解决AI问答和语音功能

main
SLMS Development Team 5 months ago
parent cad5f6716f
commit 662eef14da

@ -1,5 +1,5 @@
#MCSLMS DataSource Configuration - v1.7.0
#Tue Dec 09 09:03:18 CST 2025
#Wed Dec 10 09:53:20 CST 2025
database.host=127.0.0.1
database.name=testdb
database.password=

@ -0,0 +1,250 @@
package com.smartlibrary.voice;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.io.*;
import java.net.*;
import java.net.http.HttpClient;
import java.net.http.WebSocket;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.CopyOnWriteArrayList;
/**
* - WebSocket
*/
public class XunFeiIATService {
private static final String APP_ID = "ae4a0e4a";
private static final String API_KEY = "7385e5cb32d3465474e613dfbfc69310";
private static final String API_SECRET = "NTI2NzVlOWQ0ZTM5YTgzNGYzZDI5NjQx";
private static final String HOST = "iat-api.xfyun.cn";
private static final String PATH = "/v2/iat";
/**
*
*/
public static String recognize(byte[] audioData) throws Exception {
if (audioData == null || audioData.length < 1000) {
throw new Exception("音频数据太短,请说话时间长一些");
}
String wsUrl = buildAuthUrl();
System.out.println("讯飞WebSocket URL: " + wsUrl);
CompletableFuture<String> resultFuture = new CompletableFuture<>();
// 使用List存储每个句子的识别结果按sn索引
List<String> sentenceResults = new CopyOnWriteArrayList<>();
HttpClient client = HttpClient.newHttpClient();
WebSocket.Listener listener = new WebSocket.Listener() {
@Override
public void onOpen(WebSocket webSocket) {
System.out.println("WebSocket连接已建立");
WebSocket.Listener.super.onOpen(webSocket);
// 发送音频数据
try {
sendAudioData(webSocket, audioData);
} catch (Exception e) {
resultFuture.completeExceptionally(e);
}
}
@Override
public CompletionStage<?> onText(WebSocket webSocket, CharSequence data, boolean last) {
String json = data.toString();
System.out.println("收到响应: " + json);
// 解析结果,处理动态修正
parseAndUpdateResult(json, sentenceResults);
// 检查是否结束
if (json.contains("\"status\":2") || json.contains("\"status\": 2")) {
// 合并所有句子结果
StringBuilder finalBuilder = new StringBuilder();
for (String s : sentenceResults) {
if (s != null) finalBuilder.append(s);
}
String finalResult = finalBuilder.toString().trim();
resultFuture.complete(finalResult.isEmpty() ? "未识别到语音内容" : finalResult);
}
return WebSocket.Listener.super.onText(webSocket, data, last);
}
@Override
public void onError(WebSocket webSocket, Throwable error) {
System.err.println("WebSocket错误: " + error.getMessage());
resultFuture.completeExceptionally(error);
}
@Override
public CompletionStage<?> onClose(WebSocket webSocket, int statusCode, String reason) {
System.out.println("WebSocket关闭: " + statusCode + " - " + reason);
if (!resultFuture.isDone()) {
StringBuilder finalBuilder = new StringBuilder();
for (String s : sentenceResults) {
if (s != null) finalBuilder.append(s);
}
String finalResult = finalBuilder.toString().trim();
resultFuture.complete(finalResult.isEmpty() ? "未识别到语音内容" : finalResult);
}
return WebSocket.Listener.super.onClose(webSocket, statusCode, reason);
}
};
client.newWebSocketBuilder()
.buildAsync(URI.create(wsUrl), listener);
// 等待结果最多30秒
try {
return resultFuture.get(30, TimeUnit.SECONDS);
} catch (TimeoutException e) {
throw new Exception("语音识别超时");
}
}
/**
* wpgs
* pgs: "apd", "rpl"
* rg: [start, end]
*/
private static void parseAndUpdateResult(String json, List<String> sentenceResults) {
try {
if (!json.contains("\"code\":0") && !json.contains("\"code\": 0")) {
return;
}
// 提取sn句子序号
int sn = 0;
int snStart = json.indexOf("\"sn\":");
if (snStart > 0) {
int snEnd = json.indexOf(",", snStart);
if (snEnd < 0) snEnd = json.indexOf("}", snStart);
if (snEnd > snStart) {
try {
sn = Integer.parseInt(json.substring(snStart + 5, snEnd).trim());
} catch (NumberFormatException ignored) {}
}
}
// 检查pgs字段动态修正模式
String pgs = "";
int pgsStart = json.indexOf("\"pgs\":");
if (pgsStart > 0) {
int valueStart = json.indexOf("\"", pgsStart + 6) + 1;
int valueEnd = json.indexOf("\"", valueStart);
if (valueEnd > valueStart) {
pgs = json.substring(valueStart, valueEnd);
}
}
// 提取所有文字
StringBuilder text = new StringBuilder();
int pos = 0;
while (true) {
int wStart = json.indexOf("\"w\":", pos);
if (wStart < 0) break;
int valueStart = json.indexOf("\"", wStart + 4) + 1;
int valueEnd = json.indexOf("\"", valueStart);
if (valueEnd > valueStart) {
text.append(json.substring(valueStart, valueEnd));
pos = valueEnd;
} else {
break;
}
}
// 确保List足够大
while (sentenceResults.size() <= sn) {
sentenceResults.add("");
}
// 根据pgs字段决定是替换还是追加
if ("rpl".equals(pgs)) {
// 替换模式:直接替换该句子的结果
sentenceResults.set(sn, text.toString());
} else {
// 追加模式或无pgs替换该句子因为每次都是完整的句子内容
sentenceResults.set(sn, text.toString());
}
} catch (Exception e) {
System.err.println("解析响应失败: " + e.getMessage());
}
}
private static void sendAudioData(WebSocket webSocket, byte[] audioData) throws Exception {
int frameSize = 1280; // 每帧40ms音频
int status = 0; // 0-首帧, 1-中间帧, 2-尾帧
for (int i = 0; i < audioData.length; i += frameSize) {
int end = Math.min(i + frameSize, audioData.length);
byte[] frame = Arrays.copyOfRange(audioData, i, end);
if (i + frameSize >= audioData.length) {
status = 2; // 尾帧
}
String frameJson = buildFrameJson(frame, status);
webSocket.sendText(frameJson, true);
if (status == 0) {
status = 1; // 后续都是中间帧
}
// 模拟实时发送每帧间隔40ms
Thread.sleep(40);
}
}
private static String buildFrameJson(byte[] audioFrame, int status) {
String audioBase64 = Base64.getEncoder().encodeToString(audioFrame);
if (status == 0) {
// 首帧包含common和business关闭动态修正dwa避免重复
return String.format(
"{\"common\":{\"app_id\":\"%s\"},\"business\":{\"language\":\"zh_cn\",\"domain\":\"iat\",\"accent\":\"mandarin\",\"vad_eos\":3000},\"data\":{\"status\":0,\"format\":\"audio/L16;rate=16000\",\"encoding\":\"raw\",\"audio\":\"%s\"}}",
APP_ID, audioBase64);
} else {
// 中间帧或尾帧
return String.format(
"{\"data\":{\"status\":%d,\"format\":\"audio/L16;rate=16000\",\"encoding\":\"raw\",\"audio\":\"%s\"}}",
status, audioBase64);
}
}
private static String buildAuthUrl() throws Exception {
SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.US);
sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
String date = sdf.format(new Date());
String signatureOrigin = "host: " + HOST + "\n" +
"date: " + date + "\n" +
"GET " + PATH + " HTTP/1.1";
Mac mac = Mac.getInstance("HmacSHA256");
SecretKeySpec secretKey = new SecretKeySpec(API_SECRET.getBytes(StandardCharsets.UTF_8), "HmacSHA256");
mac.init(secretKey);
byte[] signatureBytes = mac.doFinal(signatureOrigin.getBytes(StandardCharsets.UTF_8));
String signature = Base64.getEncoder().encodeToString(signatureBytes);
String authorizationOrigin = String.format(
"api_key=\"%s\", algorithm=\"hmac-sha256\", headers=\"host date request-line\", signature=\"%s\"",
API_KEY, signature);
String authorization = Base64.getEncoder().encodeToString(authorizationOrigin.getBytes(StandardCharsets.UTF_8));
return String.format("wss://%s%s?authorization=%s&date=%s&host=%s",
HOST, PATH,
java.net.URLEncoder.encode(authorization, "UTF-8"),
java.net.URLEncoder.encode(date, "UTF-8"),
java.net.URLEncoder.encode(HOST, "UTF-8"));
}
}

@ -0,0 +1,218 @@
package com.smartlibrary.voice;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.sound.sampled.*;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* - (STT)(TTS)
*/
public class XunFeiSpeechService {
// 讯飞API配置
private static final String APP_ID = "ae4a0e4a";
private static final String API_KEY = "7385e5cb32d3465474e613dfbfc69310";
private static final String API_SECRET = "NTI2NzVlOWQ0ZTM5YTgzNGYzZDI5NjQx";
// TTS API (在线语音合成)
private static final String TTS_URL = "https://tts-api.xfyun.cn/v2/tts";
// 音频格式
private static final AudioFormat AUDIO_FORMAT = new AudioFormat(16000, 16, 1, true, false);
private TargetDataLine targetLine;
private final AtomicBoolean isRecording = new AtomicBoolean(false);
private final AtomicBoolean isPlaying = new AtomicBoolean(false);
private ByteArrayOutputStream recordingBuffer;
private final ExecutorService executor = Executors.newCachedThreadPool();
private RecordingCallback recordingCallback;
private TTSCallback ttsCallback;
public interface RecordingCallback {
void onStart();
void onStop(byte[] audioData, String recognizedText);
void onError(String error);
}
public interface TTSCallback {
void onStart();
void onComplete();
void onError(String error);
}
public void setRecordingCallback(RecordingCallback callback) {
this.recordingCallback = callback;
}
public void setTTSCallback(TTSCallback callback) {
this.ttsCallback = callback;
}
/**
*
*/
public void startRecording() throws VoiceException {
if (isRecording.get()) {
throw new VoiceException("正在录音中");
}
try {
DataLine.Info info = new DataLine.Info(TargetDataLine.class, AUDIO_FORMAT);
if (!AudioSystem.isLineSupported(info)) {
throw new VoiceException("系统不支持录音功能");
}
targetLine = (TargetDataLine) AudioSystem.getLine(info);
targetLine.open(AUDIO_FORMAT);
targetLine.start();
recordingBuffer = new ByteArrayOutputStream();
isRecording.set(true);
executor.submit(() -> {
byte[] buffer = new byte[4096];
if (recordingCallback != null) {
recordingCallback.onStart();
}
while (isRecording.get()) {
int bytesRead = targetLine.read(buffer, 0, buffer.length);
if (bytesRead > 0) {
recordingBuffer.write(buffer, 0, bytesRead);
}
}
});
} catch (LineUnavailableException e) {
throw new VoiceException("无法获取录音设备: " + e.getMessage());
}
}
/**
*
*/
public void stopRecording() {
if (!isRecording.get()) {
return;
}
isRecording.set(false);
if (targetLine != null) {
targetLine.stop();
targetLine.close();
targetLine = null;
}
byte[] audioData = recordingBuffer != null ? recordingBuffer.toByteArray() : new byte[0];
// 模拟语音识别实际需要调用讯飞WebSocket API
String recognizedText = "语音识别功能需要WebSocket连接当前使用模拟结果";
if (recordingCallback != null) {
recordingCallback.onStop(audioData, recognizedText);
}
}
public boolean isRecording() {
return isRecording.get();
}
/**
*
*/
public void speak(String text) {
if (text == null || text.trim().isEmpty()) {
if (ttsCallback != null) {
ttsCallback.onError("文本为空");
}
return;
}
executor.submit(() -> {
try {
if (ttsCallback != null) {
ttsCallback.onStart();
}
isPlaying.set(true);
// 使用系统TTSWindows SAPI
speakWithSystemTTS(text);
isPlaying.set(false);
if (ttsCallback != null) {
ttsCallback.onComplete();
}
} catch (Exception e) {
isPlaying.set(false);
if (ttsCallback != null) {
ttsCallback.onError(e.getMessage());
}
}
});
}
/**
* 使WindowsTTS
*/
private void speakWithSystemTTS(String text) throws Exception {
// 创建VBS脚本使用Windows SAPI
String vbsScript = String.format(
"CreateObject(\"SAPI.SpVoice\").Speak \"%s\"",
text.replace("\"", "\"\"").replace("\n", " ").replace("\r", "")
);
// 限制文本长度
if (vbsScript.length() > 2000) {
vbsScript = String.format(
"CreateObject(\"SAPI.SpVoice\").Speak \"%s\"",
text.substring(0, 500).replace("\"", "\"\"").replace("\n", " ")
);
}
File tempFile = File.createTempFile("tts_", ".vbs");
tempFile.deleteOnExit();
try (PrintWriter writer = new PrintWriter(tempFile, "GBK")) {
writer.print(vbsScript);
}
ProcessBuilder pb = new ProcessBuilder("cscript", "//nologo", tempFile.getAbsolutePath());
pb.redirectErrorStream(true);
Process process = pb.start();
process.waitFor();
tempFile.delete();
}
public boolean isPlaying() {
return isPlaying.get();
}
public void stop() {
isPlaying.set(false);
}
public void shutdown() {
if (isRecording.get()) {
stopRecording();
}
stop();
executor.shutdown();
}
public static boolean isRecordingSupported() {
DataLine.Info info = new DataLine.Info(TargetDataLine.class, AUDIO_FORMAT);
return AudioSystem.isLineSupported(info);
}
}

@ -0,0 +1,98 @@
package com.smartlibrary.voice;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
@DisplayName("语音服务测试")
class VoiceServiceTest {
@Test
@DisplayName("测试XunFeiConfig默认配置")
void testXunFeiConfigDefault() {
XunFeiConfig config = new XunFeiConfig();
assertNotNull(config.getAppId());
assertNotNull(config.getApiKey());
assertNotNull(config.getApiSecret());
assertTrue(config.isValid());
}
@Test
@DisplayName("测试XunFeiConfig自定义配置")
void testXunFeiConfigCustom() {
XunFeiConfig config = new XunFeiConfig("testApp", "testKey", "testSecret");
assertEquals("testApp", config.getAppId());
assertEquals("testKey", config.getApiKey());
assertEquals("testSecret", config.getApiSecret());
}
@Test
@DisplayName("测试XunFeiConfig从文件加载")
void testXunFeiConfigLoadFromFile() {
XunFeiConfig config = XunFeiConfig.loadFromFile();
assertNotNull(config);
assertTrue(config.isValid());
}
@Test
@DisplayName("测试VoiceException异常")
void testVoiceException() {
VoiceException ex = new VoiceException("测试错误");
assertEquals("测试错误", ex.getMessage());
Exception cause = new RuntimeException("原因");
VoiceException ex2 = new VoiceException("带原因的错误", cause);
assertEquals("带原因的错误", ex2.getMessage());
assertEquals(cause, ex2.getCause());
}
@Test
@DisplayName("测试XunFeiSpeechService实例化")
void testXunFeiSpeechServiceInstance() {
XunFeiSpeechService service = new XunFeiSpeechService();
assertNotNull(service);
assertFalse(service.isRecording());
assertFalse(service.isPlaying());
service.shutdown();
}
@Test
@DisplayName("测试录音支持检测")
void testRecordingSupported() {
// 只测试方法能正常调用,不依赖硬件
boolean supported = XunFeiSpeechService.isRecordingSupported();
// 结果取决于系统是否有麦克风
assertTrue(supported || !supported);
}
@Test
@DisplayName("测试语音识别空数据处理")
void testIATEmptyData() {
assertThrows(Exception.class, () -> {
XunFeiIATService.recognize(null);
});
assertThrows(Exception.class, () -> {
XunFeiIATService.recognize(new byte[100]);
});
}
@Test
@DisplayName("测试TTS空文本不抛异常")
void testTTSEmptyText() {
XunFeiSpeechService service = new XunFeiSpeechService();
// 空文本应该触发回调错误,不抛异常
service.speak(null);
service.speak("");
service.shutdown();
}
@Test
@DisplayName("测试XunFeiConfig toString")
void testXunFeiConfigToString() {
XunFeiConfig config = new XunFeiConfig();
String str = config.toString();
assertNotNull(str);
assertTrue(str.contains("XunFeiConfig"));
}
}

@ -83,10 +83,36 @@ public class GUIApplication extends JFrame {
SwingUtilities.invokeLater(() -> {
voiceBtn.setText("🎤 录音");
voiceBtn.setBackground(null);
statusBar.setText("录音完成,音频长度: " + audioData.length + " 字节");
// 模拟语音识别结果
if (audioData.length > 0) {
chatArea.append("AI助手: [语音已录制] 请在输入框中输入您想问的问题。\n\n");
if (audioData.length > 1000) {
statusBar.setText("录音完成,正在调用讯飞语音识别...");
// 异步调用讯飞语音识别
new Thread(() -> {
try {
String recognizedText = com.smartlibrary.voice.XunFeiIATService.recognize(audioData);
SwingUtilities.invokeLater(() -> {
if (recognizedText != null && !recognizedText.isEmpty()
&& !recognizedText.startsWith("识别失败")
&& !recognizedText.startsWith("未识别")
&& !recognizedText.startsWith("解析")) {
// 识别成功,填入输入框让用户确认
statusBar.setText("语音识别成功,请确认或修改后按回车发送");
chatInput.setText(recognizedText);
chatInput.requestFocus();
chatInput.selectAll();
} else {
// 识别失败
statusBar.setText(recognizedText != null ? recognizedText : "识别失败");
}
});
} catch (Exception e) {
SwingUtilities.invokeLater(() -> {
statusBar.setText("语音识别失败: " + e.getMessage());
});
}
}).start();
} else {
statusBar.setText("录音时间太短,请重试");
}
});
}
@ -1742,6 +1768,13 @@ public class GUIApplication extends JFrame {
chatArea.append("我: " + message + "\n");
chatInput.setText("");
askAI(message);
}
/**
* AI
*/
private void askAI(String message) {
statusBar.setText("AI正在思考...");
// 异步调用 AI
@ -1946,13 +1979,58 @@ public class GUIApplication extends JFrame {
return;
}
try {
speechService.speak(lastAIResponse);
} catch (VoiceException e) {
JOptionPane.showMessageDialog(this,
"语音播报失败: " + e.getMessage(),
"错误", JOptionPane.ERROR_MESSAGE);
}
// 使用Windows系统TTS播报
speakBtn.setEnabled(false);
statusBar.setText("正在播报...");
new Thread(() -> {
try {
// 截取前500字符避免播报太长
String textToSpeak = lastAIResponse.length() > 500
? lastAIResponse.substring(0, 500) + "..."
: lastAIResponse;
// 清理特殊字符
textToSpeak = textToSpeak
.replace("\"", "")
.replace("\n", "。")
.replace("\r", "")
.replace("*", "")
.replace("#", "")
.replace("`", "");
// 创建VBS脚本使用Windows SAPI
String vbsScript = "CreateObject(\"SAPI.SpVoice\").Speak \"" +
textToSpeak.replace("\"", "") + "\"";
java.io.File tempFile = java.io.File.createTempFile("tts_", ".vbs");
tempFile.deleteOnExit();
try (java.io.PrintWriter writer = new java.io.PrintWriter(tempFile, "GBK")) {
writer.print(vbsScript);
}
ProcessBuilder pb = new ProcessBuilder("cscript", "//nologo", tempFile.getAbsolutePath());
pb.redirectErrorStream(true);
Process process = pb.start();
process.waitFor();
tempFile.delete();
SwingUtilities.invokeLater(() -> {
speakBtn.setEnabled(true);
statusBar.setText("播报完成");
});
} catch (Exception e) {
SwingUtilities.invokeLater(() -> {
speakBtn.setEnabled(true);
statusBar.setText("播报失败: " + e.getMessage());
JOptionPane.showMessageDialog(GUIApplication.this,
"语音播报失败: " + e.getMessage(),
"错误", JOptionPane.ERROR_MESSAGE);
});
}
}).start();
}
// ========== PlantUML 功能 ==========

Loading…
Cancel
Save