pu6zrsfoy
/
mcslms
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

gui,web端语音不正常修复

Browse Source
main
SLMS Development Team 5 months ago
parent dec620ccf3
commit e9d234e104
4 changed files with 292 additions and 50 deletions
Show Stats Download Patch File Download Diff File

30
backend/build.gradle
Unescape View File

@ -8,7 +8,11 @@ plugins {
}
group = 'com.smartlibrary'
version = '1.0-SNAPSHOT'
version = '1.0.0'
// 支持Jenkins构建号
def buildNumber = project.findProperty('buildNumber') ?: '0'
def fullVersion = "v${version}.${buildNumber}"
java {
sourceCompatibility = '21'
@ -70,6 +74,30 @@ springBoot {
mainClass = 'com.smartlibrary.web.WebApplication'
}
// 配置 bootJar 输出名称
bootJar {
archiveBaseName = 'mcslms-backend'
archiveVersion = fullVersion
manifest {
attributes(
'Implementation-Title': 'Smart Library Backend',
'Implementation-Version': fullVersion
)
}
}
// 配置 bootWar 输出名称
bootWar {
archiveBaseName = 'mcslms-web'
archiveVersion = fullVersion
manifest {
attributes(
'Implementation-Title': 'Smart Library Web',
'Implementation-Version': fullVersion
)
}
}
tasks.named('test') {
useJUnitPlatform()
}

151
backend/src/main/java/com/smartlibrary/web/controller/VoiceController.java
Unescape View File

@ -135,34 +135,45 @@ public class VoiceController {
private String recognizeWithXunFei(String audioBase64) throws Exception {
String wsUrl = buildAuthUrl(XunFeiConfig.STT_API_URL);
StringBuilder resultText = new StringBuilder();
// 解码音频数据
byte[] audioData = Base64.getDecoder().decode(audioBase64);
if (audioData.length < 1000) {
throw new Exception("音频数据太短，请说话时间长一些");
}
CompletableFuture<String> future = new CompletableFuture<>();
List<String> sentenceResults = new java.util.concurrent.CopyOnWriteArrayList<>();
HttpClient client = HttpClient.newHttpClient();
WebSocket webSocket = client.newWebSocketBuilder()
client.newWebSocketBuilder()
.buildAsync(URI.create(wsUrl), new WebSocket.Listener() {
private StringBuilder fullResult = new StringBuilder();
@Override
public void onOpen(WebSocket webSocket) {
// 发送第一帧
String firstFrame = buildFirstFrame(audioBase64);
webSocket.sendText(firstFrame, true);
WebSocket.Listener.super.onOpen(webSocket);
// 在新线程中分帧发送音频
new Thread(() -> {
try {
sendAudioFrames(webSocket, audioData);
} catch (Exception e) {
future.completeExceptionally(e);
}
}).start();
}
@Override
public CompletionStage<?> onText(WebSocket webSocket, CharSequence data, boolean last) {
String response = data.toString();
// 解析响应提取文本
String text = parseXunFeiResponse(response);
if (text != null) {
fullResult.append(text);
}
String json = data.toString();
parseAndUpdateResult(json, sentenceResults);
// 检查是否结束
if (response.contains("\"status\":2")) {
future.complete(fullResult.toString());
if (json.contains("\"status\":2") || json.contains("\"status\": 2")) {
StringBuilder finalBuilder = new StringBuilder();
for (String s : sentenceResults) {
if (s != null) finalBuilder.append(s);
}
String finalResult = finalBuilder.toString().trim();
future.complete(finalResult.isEmpty() ? "未识别到语音内容" : finalResult);
}
return WebSocket.Listener.super.onText(webSocket, data, last);
@ -172,18 +183,118 @@ public class VoiceController {
public void onError(WebSocket webSocket, Throwable error) {
future.completeExceptionally(error);
}
}).join();
@Override
public CompletionStage<?> onClose(WebSocket webSocket, int statusCode, String reason) {
if (!future.isDone()) {
StringBuilder finalBuilder = new StringBuilder();
for (String s : sentenceResults) {
if (s != null) finalBuilder.append(s);
}
String finalResult = finalBuilder.toString().trim();
future.complete(finalResult.isEmpty() ? "未识别到语音内容" : finalResult);
}
return WebSocket.Listener.super.onClose(webSocket, statusCode, reason);
}
});
return future.get(30, TimeUnit.SECONDS);
}
/**
* 构建讯飞语音识别第一帧数据
* 分帧发送音频数据
*/
private void sendAudioFrames(WebSocket webSocket, byte[] audioData) throws Exception {
int frameSize = 1280; // 每帧40ms音频
int status = 0; // 0-首帧, 1-中间帧, 2-尾帧
for (int i = 0; i < audioData.length; i += frameSize) {
int end = Math.min(i + frameSize, audioData.length);
byte[] frame = Arrays.copyOfRange(audioData, i, end);
if (i + frameSize >= audioData.length) {
status = 2; // 尾帧
}
String frameJson = buildFrameJson(frame, status);
webSocket.sendText(frameJson, true);
if (status == 0) {
status = 1; // 后续都是中间帧
}
// 模拟实时发送，每帧间隔40ms
Thread.sleep(40);
}
}
/**
* 构建帧数据JSON
*/
private String buildFrameJson(byte[] audioFrame, int status) {
String audioBase64 = Base64.getEncoder().encodeToString(audioFrame);
if (status == 0) {
// 首帧，包含common和business
return "{\"common\":{\"app_id\":\"" + config.getAppId() + "\"}," +
"\"business\":{\"language\":\"zh_cn\",\"domain\":\"iat\",\"accent\":\"mandarin\",\"vad_eos\":3000}," +
"\"data\":{\"status\":0,\"format\":\"audio/L16;rate=16000\",\"encoding\":\"raw\",\"audio\":\"" + audioBase64 + "\"}}";
} else {
// 中间帧或尾帧
return "{\"data\":{\"status\":" + status + ",\"format\":\"audio/L16;rate=16000\",\"encoding\":\"raw\",\"audio\":\"" + audioBase64 + "\"}}";
}
}
/**
* 解析响应并更新结果
*/
private String buildFirstFrame(String audioBase64) {
return "{\"common\":{\"app_id\":\"" + config.getAppId() + "\"}," +
"\"business\":{\"language\":\"zh_cn\",\"domain\":\"iat\",\"accent\":\"mandarin\",\"vad_eos\":3000}," +
"\"data\":{\"status\":2,\"format\":\"audio/L16;rate=16000\",\"encoding\":\"raw\",\"audio\":\"" + audioBase64 + "\"}}";
private void parseAndUpdateResult(String json, List<String> sentenceResults) {
try {
if (!json.contains("\"code\":0") && !json.contains("\"code\": 0")) {
return;
}
// 提取sn（句子序号）
int sn = 0;
int snStart = json.indexOf("\"sn\":");
if (snStart > 0) {
int snEnd = json.indexOf(",", snStart);
if (snEnd < 0) snEnd = json.indexOf("}", snStart);
if (snEnd > snStart) {
try {
sn = Integer.parseInt(json.substring(snStart + 5, snEnd).trim());
} catch (NumberFormatException ignored) {}
}
}
// 提取所有文字
StringBuilder text = new StringBuilder();
int pos = 0;
while (true) {
int wStart = json.indexOf("\"w\":", pos);
if (wStart < 0) break;
int valueStart = json.indexOf("\"", wStart + 4) + 1;
int valueEnd = json.indexOf("\"", valueStart);
if (valueEnd > valueStart) {
text.append(json.substring(valueStart, valueEnd));
pos = valueEnd;
} else {
break;
}
}
// 确保List足够大
while (sentenceResults.size() <= sn) {
sentenceResults.add("");
}
// 替换该句子的结果
sentenceResults.set(sn, text.toString());
} catch (Exception ignored) {
// 静默处理解析异常
}
}
/**

24
gui/build.gradle
Unescape View File

@ -7,6 +7,10 @@ plugins {
group = 'com.smartlibrary'
version = '1.0.0'
// 支持Jenkins构建号
def buildNumber = project.findProperty('buildNumber') ?: '0'
def fullVersion = "v${version}.${buildNumber}"
java {
sourceCompatibility = JavaVersion.VERSION_21
targetCompatibility = JavaVersion.VERSION_21
@ -36,10 +40,16 @@ application {
// 打包可执行 JAR - 使用Launcher类避免JavaFX模块问题
tasks.register('fatJar', Jar) {
archiveBaseName = 'mcslms-gui'
archiveVersion = fullVersion
archiveClassifier = 'all'
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
manifest {
attributes 'Main-Class': 'com.smartlibrary.gui.Launcher'
attributes(
'Main-Class': 'com.smartlibrary.gui.Launcher',
'Implementation-Title': 'Smart Library GUI',
'Implementation-Version': fullVersion
)
}
from sourceSets.main.output
from {
@ -47,3 +57,15 @@ tasks.register('fatJar', Jar) {
}
exclude 'META-INF/*.SF', 'META-INF/*.DSA', 'META-INF/*.RSA'
}
jar {
archiveBaseName = 'mcslms-gui'
archiveVersion = fullVersion
manifest {
attributes(
'Main-Class': 'com.smartlibrary.gui.Launcher',
'Implementation-Title': 'Smart Library GUI',
'Implementation-Version': fullVersion
)
}
}

137
gui/src/main/java/com/smartlibrary/gui/GUIApplication.java
Unescape View File

@ -502,6 +502,10 @@ public class GUIApplication extends Application {
});
}
// 语音服务实例
private com.smartlibrary.voice.XunFeiSpeechService voiceService;
private byte[] recordedAudioData;
private void showVoiceAssistantDialog() {
Dialog<String> dialog = new Dialog<>();
dialog.setTitle("语音助手");
@ -533,46 +537,123 @@ public class GUIApplication extends Application {
stopBtn.setDisable(true);
speakBtn.setDisable(true);
// 初始化语音服务
if (voiceService == null) {
voiceService = new com.smartlibrary.voice.XunFeiSpeechService();
}
recordBtn.setOnAction(e -> {
statusLabel.setText("正在录音...");
recordBtn.setDisable(true);
stopBtn.setDisable(false);
resultArea.setText("语音识别功能需要配置讯飞API密钥\n\n" +
"配置方法：\n" +
"1. 在application.properties中配置：\n" +
" xunfei.appid=your_appid\n" +
" xunfei.apikey=your_apikey\n" +
" xunfei.apisecret=your_apisecret\n\n" +
"2. 或设置环境变量：\n" +
" XUNFEI_APPID=your_appid\n" +
" XUNFEI_APIKEY=your_apikey\n" +
" XUNFEI_APISECRET=your_apisecret\n\n" +
"演示模式：模拟识别结果");
try {
voiceService.startRecording();
statusLabel.setText("正在录音... 请说话");
recordBtn.setDisable(true);
stopBtn.setDisable(false);
resultArea.setText("正在录音中，请说话...\n\n" +
"您可以说：\n" +
"- 搜索 [图书名称]\n" +
"- 借阅 [图书名称]\n" +
"- 归还 [图书名称]\n" +
"- 推荐图书");
} catch (Exception ex) {
statusLabel.setText("录音失败");
resultArea.setText("录音启动失败: " + ex.getMessage());
recordBtn.setDisable(false);
}
});
stopBtn.setOnAction(e -> {
statusLabel.setText("识别完成");
recordBtn.setDisable(false);
statusLabel.setText("正在识别...");
stopBtn.setDisable(true);
speakBtn.setDisable(false);
resultArea.setText("正在进行语音识别，请稍候...");
// 模拟识别结果
String mockResult = "搜索深入理解计算机系统";
resultArea.setText("识别结果：" + mockResult + "\n\n" +
"您可以说：\n" +
"- 搜索 [图书名称]\n" +
"- 借阅 [图书名称]\n" +
"- 归还 [图书名称]\n" +
"- 推荐图书");
// 在后台线程进行语音识别
new Thread(() -> {
try {
// 停止录音并获取音频数据
voiceService.setRecordingCallback(new com.smartlibrary.voice.XunFeiSpeechService.RecordingCallback() {
@Override
public void onStart() {}
@Override
public void onStop(byte[] audioData, String text) {
recordedAudioData = audioData;
}
@Override
public void onError(String error) {
javafx.application.Platform.runLater(() -> {
statusLabel.setText("录音错误");
resultArea.setText("录音错误: " + error);
});
}
});
voiceService.stopRecording();
// 等待录音数据
Thread.sleep(200);
// 调用讯飞语音识别
if (recordedAudioData != null && recordedAudioData.length > 1000) {
String recognizedText = com.smartlibrary.voice.XunFeiIATService.recognize(recordedAudioData);
javafx.application.Platform.runLater(() -> {
statusLabel.setText("识别完成");
recordBtn.setDisable(false);
speakBtn.setDisable(false);
resultArea.setText("识别结果：" + recognizedText + "\n\n" +
"您可以说：\n" +
"- 搜索 [图书名称]\n" +
"- 借阅 [图书名称]\n" +
"- 归还 [图书名称]\n" +
"- 推荐图书");
});
} else {
javafx.application.Platform.runLater(() -> {
statusLabel.setText("录音太短");
recordBtn.setDisable(false);
resultArea.setText("录音时间太短，请说话时间长一些（至少1秒）");
});
}
} catch (Exception ex) {
javafx.application.Platform.runLater(() -> {
statusLabel.setText("识别失败");
recordBtn.setDisable(false);
resultArea.setText("语音识别失败: " + ex.getMessage());
});
}
}).start();
});
speakBtn.setOnAction(e -> {
String text = resultArea.getText();
if (text != null && !text.isEmpty()) {
statusLabel.setText("正在朗读...");
showAlert(Alert.AlertType.INFORMATION, "语音朗读",
"语音朗读功能需要配置讯飞TTS API\n\n演示模式：模拟朗读文本");
statusLabel.setText("朗读完成");
// 提取识别结果文本进行朗读
String toSpeak = text;
if (text.startsWith("识别结果：")) {
int endIdx = text.indexOf("\n");
if (endIdx > 0) {
toSpeak = text.substring(5, endIdx);
}
}
final String speakText = toSpeak;
voiceService.setTTSCallback(new com.smartlibrary.voice.XunFeiSpeechService.TTSCallback() {
@Override
public void onStart() {
javafx.application.Platform.runLater(() -> statusLabel.setText("正在朗读..."));
}
@Override
public void onComplete() {
javafx.application.Platform.runLater(() -> statusLabel.setText("朗读完成"));
}
@Override
public void onError(String error) {
javafx.application.Platform.runLater(() -> statusLabel.setText("朗读失败: " + error));
}
});
voiceService.speak(speakText);
}
});

Write Preview
Loading…
Cancel
Save