|
|
|
|
@ -0,0 +1,375 @@
|
|
|
|
|
1.对话式大模型
|
|
|
|
|
@startuml
|
|
|
|
|
title Minote Iflytek LLM Interaction
|
|
|
|
|
skinparam sequenceParticipant underline
|
|
|
|
|
skinparam stereotypePosition top
|
|
|
|
|
hide footbox
|
|
|
|
|
|
|
|
|
|
actor user
|
|
|
|
|
participant NoteEditActivity<<Boundary>>
|
|
|
|
|
participant IflytekActivity<<Control>>
|
|
|
|
|
participant SparkChain<<Entity>>
|
|
|
|
|
participant LLMFactory<<Entity>>
|
|
|
|
|
participant chat_llm<<Entity>>
|
|
|
|
|
participant mCallback<<Entity>>
|
|
|
|
|
|
|
|
|
|
note right of user: 用户启动应用
|
|
|
|
|
user -> NoteEditActivity : 启动LLM
|
|
|
|
|
activate NoteEditActivity
|
|
|
|
|
|
|
|
|
|
NoteEditActivity -> IflytekActivity : 初始化SDK
|
|
|
|
|
activate IflytekActivity
|
|
|
|
|
|
|
|
|
|
IflytekActivity -> SparkChain : init(Config)
|
|
|
|
|
activate SparkChain
|
|
|
|
|
SparkChain --> IflytekActivity : 初始化完成
|
|
|
|
|
deactivate SparkChain
|
|
|
|
|
|
|
|
|
|
IflytekActivity -> LLMFactory : textGeneration()
|
|
|
|
|
activate LLMFactory
|
|
|
|
|
LLMFactory --> chat_llm : 创建LLM实例
|
|
|
|
|
deactivate LLMFactory
|
|
|
|
|
|
|
|
|
|
IflytekActivity -> chat_llm : 注册回调\n注册LLMCallbacks
|
|
|
|
|
activate chat_llm
|
|
|
|
|
chat_llm -> mCallback : registerLLMCallbacks(mCallback)
|
|
|
|
|
deactivate chat_llm
|
|
|
|
|
|
|
|
|
|
alt 同步模式
|
|
|
|
|
IflytekActivity -> chat_llm : run(question)
|
|
|
|
|
activate chat_llm
|
|
|
|
|
chat_llm --> IflytekActivity : syncOutput
|
|
|
|
|
deactivate chat_llm
|
|
|
|
|
loop 循环获取结果
|
|
|
|
|
IflytekActivity -> mCallback : syncOutput.getContent()
|
|
|
|
|
mCallback --> IflytekActivity : result
|
|
|
|
|
end
|
|
|
|
|
else 异步模式
|
|
|
|
|
IflytekActivity -> chat_llm : arun(question)
|
|
|
|
|
activate chat_llm
|
|
|
|
|
loop 等待回调
|
|
|
|
|
mCallback -> chat_llm : 接收结果
|
|
|
|
|
chat_llm -> mCallback : getContent()
|
|
|
|
|
mCallback -> IflytekActivity : 处理结果
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
IflytekActivity -> NoteEditActivity : 展示结果
|
|
|
|
|
deactivate IflytekActivity
|
|
|
|
|
NoteEditActivity --> user : 展示结果
|
|
|
|
|
deactivate NoteEditActivity
|
|
|
|
|
|
|
|
|
|
user -> NoteEditActivity : 退出应用
|
|
|
|
|
activate NoteEditActivity
|
|
|
|
|
NoteEditActivity -> IflytekActivity : 停止LLM
|
|
|
|
|
activate IflytekActivity
|
|
|
|
|
|
|
|
|
|
IflytekActivity -> SparkChain : uninit()
|
|
|
|
|
activate SparkChain
|
|
|
|
|
SparkChain --> IflytekActivity : 逆初始化完成
|
|
|
|
|
deactivate SparkChain
|
|
|
|
|
|
|
|
|
|
IflytekActivity --> NoteEditActivity : 通知退出完成
|
|
|
|
|
deactivate IflytekActivity
|
|
|
|
|
NoteEditActivity --> user : 通知退出完成
|
|
|
|
|
deactivate NoteEditActivity
|
|
|
|
|
|
|
|
|
|
@enduml
|
|
|
|
|
|
|
|
|
|
2.语音听写
|
|
|
|
|
@startuml
|
|
|
|
|
title Minote Iflytek Iat
|
|
|
|
|
skinparam sequenceParticipant underline
|
|
|
|
|
skinparam stereotypePosition top
|
|
|
|
|
hide footbox
|
|
|
|
|
|
|
|
|
|
actor User as user
|
|
|
|
|
participant "NoteEditActivity" as note
|
|
|
|
|
participant "IatDemo" as demo
|
|
|
|
|
participant "SpeechRecognizer" as recognizer
|
|
|
|
|
participant "RecognizerDialog" as dialog
|
|
|
|
|
participant "JsonParser" as parser
|
|
|
|
|
participant "SpeechUtility" as utility
|
|
|
|
|
|
|
|
|
|
activate user
|
|
|
|
|
|
|
|
|
|
== 初始化阶段 ==
|
|
|
|
|
user -> note : 点击语音按钮
|
|
|
|
|
activate note
|
|
|
|
|
note -> demo : startActivity()
|
|
|
|
|
deactivate note
|
|
|
|
|
|
|
|
|
|
activate demo
|
|
|
|
|
demo -> utility : createUtility(appid)
|
|
|
|
|
activate utility
|
|
|
|
|
demo -> recognizer : createRecognizer(InitListener)
|
|
|
|
|
activate recognizer
|
|
|
|
|
demo -> dialog : new RecognizerDialog(InitListener)
|
|
|
|
|
activate dialog
|
|
|
|
|
|
|
|
|
|
== 参数配置阶段 ==
|
|
|
|
|
user -> demo : 点击开始听写
|
|
|
|
|
demo -> recognizer : setParameter(LANGUAGE, "zh_cn")
|
|
|
|
|
demo -> recognizer : setParameter(VAD_BOS, "4000")
|
|
|
|
|
demo -> recognizer : setParameter(VAD_EOS, "1000")
|
|
|
|
|
|
|
|
|
|
== 语音识别阶段 ==
|
|
|
|
|
demo -> dialog : show()
|
|
|
|
|
deactivate demo
|
|
|
|
|
|
|
|
|
|
dialog -->> user : 显示录音对话框
|
|
|
|
|
user -> dialog : 开始说话
|
|
|
|
|
dialog -->> user : onBeginOfSpeech()显示"开始说话"
|
|
|
|
|
|
|
|
|
|
loop 语音输入过程
|
|
|
|
|
dialog -->> user : onVolumeChanged()显示音量大小
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
user -> dialog : 停止说话
|
|
|
|
|
dialog -->> user : onEndOfSpeech()显示"结束说话"
|
|
|
|
|
|
|
|
|
|
== 结果处理阶段 ==
|
|
|
|
|
dialog -> recognizer : 语音数据处理
|
|
|
|
|
activate recognizer
|
|
|
|
|
recognizer -->> dialog : onResult(RecognizerResult)
|
|
|
|
|
deactivate recognizer
|
|
|
|
|
|
|
|
|
|
dialog -> parser : parseIatResult(json)
|
|
|
|
|
activate parser
|
|
|
|
|
parser -->> dialog : 返回解析后的文本
|
|
|
|
|
deactivate parser
|
|
|
|
|
deactivate dialog
|
|
|
|
|
|
|
|
|
|
activate demo
|
|
|
|
|
demo -->> user : 显示识别结果
|
|
|
|
|
|
|
|
|
|
== 错误处理 ==
|
|
|
|
|
alt 识别出错
|
|
|
|
|
demo -->> user : 显示错误信息
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
== 完成阶段 ==
|
|
|
|
|
user -> demo : 确认文本
|
|
|
|
|
demo -->> note : 返回识别文本
|
|
|
|
|
deactivate demo
|
|
|
|
|
|
|
|
|
|
activate note
|
|
|
|
|
note -->> user : 更新便签内容
|
|
|
|
|
deactivate note
|
|
|
|
|
|
|
|
|
|
deactivate user
|
|
|
|
|
|
|
|
|
|
@enduml
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3.语音合成
|
|
|
|
|
@startuml
|
|
|
|
|
title Minote Iflytek Tts
|
|
|
|
|
skinparam sequenceParticipant underline
|
|
|
|
|
skinparam stereotypePosition top
|
|
|
|
|
hide footbox
|
|
|
|
|
|
|
|
|
|
actor User as user
|
|
|
|
|
participant "NoteEditActivity" as note
|
|
|
|
|
participant "TtsDemo" as demo
|
|
|
|
|
participant "SpeechSynthesizer" as synthesizer
|
|
|
|
|
participant "TtsSettings" as settings
|
|
|
|
|
participant "SpeechUtility" as utility
|
|
|
|
|
|
|
|
|
|
activate user
|
|
|
|
|
|
|
|
|
|
== 初始化阶段 ==
|
|
|
|
|
user -> note : 点击语音合成按钮
|
|
|
|
|
activate note
|
|
|
|
|
note -> demo : startActivity()
|
|
|
|
|
deactivate note
|
|
|
|
|
|
|
|
|
|
activate demo
|
|
|
|
|
demo -> utility : createUtility(appid)
|
|
|
|
|
demo -> synthesizer : createSynthesizer(InitListener)
|
|
|
|
|
synthesizer -->> demo : onInit()回调初始化状态
|
|
|
|
|
|
|
|
|
|
== 参数配置阶段 ==
|
|
|
|
|
user -> demo : 点击设置按钮
|
|
|
|
|
demo -> settings : startActivity()
|
|
|
|
|
activate settings
|
|
|
|
|
settings -->> user : 显示设置界面(语速/音调/音量)
|
|
|
|
|
user -> settings : 调整参数
|
|
|
|
|
settings -->> demo : 保存设置参数
|
|
|
|
|
deactivate settings
|
|
|
|
|
|
|
|
|
|
user -> demo : 选择发音人
|
|
|
|
|
demo -->> user : 显示发音人列表对话框
|
|
|
|
|
user -> demo : 确认发音人选择
|
|
|
|
|
|
|
|
|
|
== 合成播放阶段 ==
|
|
|
|
|
user -> demo : 点击开始播放
|
|
|
|
|
demo -> demo : setParam()
|
|
|
|
|
demo -> synthesizer : startSpeaking(text, listener)
|
|
|
|
|
activate synthesizer
|
|
|
|
|
|
|
|
|
|
synthesizer -->> demo : onSpeakBegin()
|
|
|
|
|
synthesizer -->> user : 开始播放提示
|
|
|
|
|
|
|
|
|
|
loop 合成播放过程
|
|
|
|
|
synthesizer -->> demo : onBufferProgress()
|
|
|
|
|
synthesizer -->> demo : onSpeakProgress()
|
|
|
|
|
demo -->> user : 显示进度和高亮文本
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
alt 用户控制
|
|
|
|
|
user -> demo : 点击暂停
|
|
|
|
|
demo -> synthesizer : pauseSpeaking()
|
|
|
|
|
synthesizer -->> user : 暂停播放提示
|
|
|
|
|
|
|
|
|
|
user -> demo : 点击继续
|
|
|
|
|
demo -> synthesizer : resumeSpeaking()
|
|
|
|
|
synthesizer -->> user : 继续播放提示
|
|
|
|
|
|
|
|
|
|
user -> demo : 点击停止
|
|
|
|
|
demo -> synthesizer : stopSpeaking()
|
|
|
|
|
synthesizer -->> user : 停止播放提示
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
synthesizer -->> demo : onCompleted()
|
|
|
|
|
deactivate synthesizer
|
|
|
|
|
demo -->> user : 播放完成提示
|
|
|
|
|
|
|
|
|
|
== 结束阶段 ==
|
|
|
|
|
user -> demo : 退出界面
|
|
|
|
|
demo -> synthesizer : destroy()
|
|
|
|
|
deactivate demo
|
|
|
|
|
|
|
|
|
|
deactivate user
|
|
|
|
|
|
|
|
|
|
@enduml
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
分界线以下是弃用的UML顺序图
|
|
|
|
|
|
|
|
|
|
///////////////////////////////////////////////////////////////////////////////////////////////
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Speech版本语音听写：
|
|
|
|
|
@startuml
|
|
|
|
|
title Speech Recognition Process with NoteEditActivity and IATActivity
|
|
|
|
|
skinparam sequenceParticipant underline
|
|
|
|
|
skinparam stereotypePosition top
|
|
|
|
|
hide footbox
|
|
|
|
|
|
|
|
|
|
actor user
|
|
|
|
|
participant "NoteEditActivity" as NEA
|
|
|
|
|
participant "IATActivity" as IATA
|
|
|
|
|
participant "SpeechRecognizer" as SR
|
|
|
|
|
participant "RecognizerListener" as RL
|
|
|
|
|
participant "InitListener" as IL
|
|
|
|
|
|
|
|
|
|
note right of user: 用户启动应用并选择语音识别功能
|
|
|
|
|
user -> NEA : 选择语音识别功能
|
|
|
|
|
activate NEA
|
|
|
|
|
|
|
|
|
|
NEA -> IATA : startIATActivity()
|
|
|
|
|
activate IATA
|
|
|
|
|
|
|
|
|
|
note right of IATA: 初始化SDK和SpeechRecognizer
|
|
|
|
|
IATA -> SR : createRecognizer(Context, IL)
|
|
|
|
|
activate SR
|
|
|
|
|
SR --> IATA : onInit(int)
|
|
|
|
|
deactivate SR
|
|
|
|
|
|
|
|
|
|
note right of IATA: 设置识别参数
|
|
|
|
|
IATA -> SR : setParameter(String, String)
|
|
|
|
|
|
|
|
|
|
note right of IATA: 构建语法（如果需要）
|
|
|
|
|
IATA -> SR : buildGrammar(String, String, GL)
|
|
|
|
|
activate SR
|
|
|
|
|
SR --> IATA : onBuildFinish(String, SpeechError)
|
|
|
|
|
deactivate SR
|
|
|
|
|
|
|
|
|
|
note right of IATA: 更新词典（如果需要）
|
|
|
|
|
IATA -> SR : updateLexicon(String, String, LXL)
|
|
|
|
|
activate SR
|
|
|
|
|
SR --> IATA : onLexiconUpdated(String, SpeechError)
|
|
|
|
|
deactivate SR
|
|
|
|
|
|
|
|
|
|
note right of IATA: 开始录音进行识别
|
|
|
|
|
IATA -> SR : startListening(RL)
|
|
|
|
|
activate SR
|
|
|
|
|
SR --> IATA : onBeginOfSpeech()
|
|
|
|
|
loop Recording
|
|
|
|
|
SR --> IATA : onVolumeChanged(int, byte[])
|
|
|
|
|
end
|
|
|
|
|
SR --> IATA : onEndOfSpeech()
|
|
|
|
|
SR --> IATA : onResult(RecognizerResult, boolean)
|
|
|
|
|
SR --> IATA : onError(SpeechError)
|
|
|
|
|
deactivate SR
|
|
|
|
|
|
|
|
|
|
note right of IATA: 处理识别结果
|
|
|
|
|
IATA -> user : 展示识别结果
|
|
|
|
|
|
|
|
|
|
note right of user: 用户完成识别并退出
|
|
|
|
|
user -> IATA : 退出识别功能
|
|
|
|
|
IATA -> SR : stopListening()
|
|
|
|
|
IATA -> SR : destroy()
|
|
|
|
|
@enduml
|
|
|
|
|
|
|
|
|
|
Linux版本：
|
|
|
|
|
@startuml
|
|
|
|
|
title Speech Recognition Process with NoteEditActivity and IATActivity
|
|
|
|
|
skinparam sequenceParticipant underline
|
|
|
|
|
skinparam stereotypePosition top
|
|
|
|
|
hide footbox
|
|
|
|
|
|
|
|
|
|
actor user
|
|
|
|
|
participant "NoteEditActivity" as NoteEdit
|
|
|
|
|
participant "IATActivity" as IAT
|
|
|
|
|
participant "SpeechRecognizer" as SR
|
|
|
|
|
participant "RecognizerListener" as RL
|
|
|
|
|
participant "InitListener" as IL
|
|
|
|
|
|
|
|
|
|
note right of user: 用户启动应用并选择语音识别功能
|
|
|
|
|
user -> NoteEdit : 选择语音识别功能
|
|
|
|
|
activate NoteEdit
|
|
|
|
|
|
|
|
|
|
NoteEdit -> IAT : startIATActivity()
|
|
|
|
|
activate IAT
|
|
|
|
|
|
|
|
|
|
note right of IAT: 初始化SDK和SpeechRecognizer
|
|
|
|
|
IAT -> SR : createRecognizer(Context, IL)
|
|
|
|
|
activate SR
|
|
|
|
|
SR --> IAT : onInit(int)
|
|
|
|
|
deactivate SR
|
|
|
|
|
|
|
|
|
|
note right of IAT: 注册监听回调和配置识别参数
|
|
|
|
|
IAT -> SR : setParameter(String, String)
|
|
|
|
|
|
|
|
|
|
note right of IAT: 启动会话
|
|
|
|
|
IAT -> SR : startListening(RL)
|
|
|
|
|
activate SR
|
|
|
|
|
|
|
|
|
|
loop Recording
|
|
|
|
|
SR --> IAT : onVolumeChanged(int, byte[])
|
|
|
|
|
SR --> IAT : onBeginOfSpeech()
|
|
|
|
|
IAT -> SR : writeAudio(byte[], int, int)
|
|
|
|
|
SR --> IAT : onResult(RecognizerResult, boolean)
|
|
|
|
|
SR --> IAT : onEndOfSpeech()
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
note right of IAT: 处理识别结果
|
|
|
|
|
IAT -> user : 展示识别结果
|
|
|
|
|
|
|
|
|
|
note right of user: 用户完成识别并退出
|
|
|
|
|
user -> IAT : 退出识别功能
|
|
|
|
|
IAT -> SR : stopListening()
|
|
|
|
|
IAT -> SR : destroy()
|
|
|
|
|
deactivate SR
|
|
|
|
|
deactivate IAT
|
|
|
|
|
@enduml
|
