import { useState, useRef, useCallback } from "react"; // 新增音频源类型,playback表示音频播放,mic表示麦克风 type AudioSourceType = "mic" | "playback"; interface ProcessState { recording: boolean; transcribing: boolean; generating: boolean; synthesizing: boolean; error?: string; } interface VoiceAssistantProps { greeting: string; } const ANALYSER_FFT_SIZE = 128; const VOLUME_SENSITIVITY = 1.5; const SMOOTHING_FACTOR = 0.7; const BAR_COUNT = 12; const VoiceAssistant = ({ greeting }: VoiceAssistantProps) => { const [isListening, setIsListening] = useState(false); const [processState, setProcessState] = useState({ recording: false, transcribing: false, generating: false, synthesizing: false, }); const [asrText, setAsrText] = useState(""); const [answerText, setAnswerText] = useState(""); const mediaRecorder = useRef(null); const audioChunks = useRef([]); const audioElement = useRef(null); const barsRef = useRef(null); const mediaStreamRef = useRef(null); const audioContextRef = useRef(null); const analyserRef = useRef(null); const animationFrameRef = useRef(null); const dataArrayRef = useRef(null); const lastValuesRef = useRef(new Array(BAR_COUNT).fill(10)); const [audioSourceType, setAudioSourceType] = useState("mic"); const updateState = (newState: Partial) => { setProcessState((prev) => ({ ...prev, ...newState })); }; const cleanupAudio = useCallback(async () => { mediaStreamRef.current?.getTracks().forEach((track) => track.stop()); if (audioContextRef.current?.state !== "closed") { await audioContextRef.current?.close(); } if (animationFrameRef.current) { cancelAnimationFrame(animationFrameRef.current); animationFrameRef.current = null; } }, []); const initializeAudioContext = useCallback(() => { const AudioContextClass = window.AudioContext || (window as any).webkitAudioContext; audioContextRef.current = new AudioContextClass(); analyserRef.current = audioContextRef.current.createAnalyser(); analyserRef.current.fftSize = ANALYSER_FFT_SIZE; analyserRef.current.smoothingTimeConstant = SMOOTHING_FACTOR; dataArrayRef.current = new Uint8Array( analyserRef.current.frequencyBinCount ); }, []); const startRecording = async () => { try { const stream = await navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000, channelCount: 1, sampleSize: 16 }, }); mediaRecorder.current = new MediaRecorder(stream); audioChunks.current = []; mediaRecorder.current.ondataavailable = (e) => { audioChunks.current.push(e.data); }; mediaRecorder.current.start(500); updateState({ recording: true, error: undefined }); } catch (err) { updateState({ error: "麦克风访问失败,请检查权限设置" }); } }; // 新增切换音频源的函数 const stopRecording = async () => { // 如果当前没有录音器,则返回 if (!mediaRecorder.current) return; // 停止录音器 mediaRecorder.current.stop(); // 更新状态为未录音 updateState({ recording: false }); // 等待录音器停止录音 mediaRecorder.current.onstop = async () => { try { // 停止录音器 const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" }); await processAudio(audioBlob); } finally { audioChunks.current = []; } }; }; const processAudio = async (audioBlob: Blob) => { const formData = new FormData(); formData.append("audio", audioBlob, "recording.wav"); try { updateState({ transcribing: true }); // 发送请求到后端 const asrResponse = await fetch("http://localhost:5000/asr", { method: "POST", body: formData, }); // 如果请求失败,则抛出错误 if (!asrResponse.ok) throw new Error("语音识别失败"); // 获取后端返回的文本 const asrData = await asrResponse.json(); setAsrText(asrData.asr_text); updateState({ transcribing: false, generating: true }); // 发送请求到后端,生成回答 const generateResponse = await fetch("http://localhost:5000/generate", { method: "POST", headers: { "Content-Type": "application/json", }, body: JSON.stringify({ asr_text: asrData.asr_text }), }); if (!generateResponse.ok) throw new Error("生成回答失败"); const generateData = await generateResponse.json(); setAnswerText(generateData.answer_text); updateState({ generating: false, synthesizing: true }); // 播放合成的音频,增加可视化效果 if (audioElement.current) { startVisualization(); // 播放合成的音频 audioElement.current.src = `http://localhost:5000${generateData.audio_url}`; // 播放音频 audioElement.current.play() .catch((err) => { console.error("播放失败:", err); updateState({ error: "音频播放失败" }); }); } } catch (err) { updateState({ error: err instanceof Error ? err.message : "未知错误" }); } finally { updateState({ transcribing: false, generating: false, synthesizing: false, }); } }; const getStatusText = () => { if (processState.error) return processState.error; if (processState.recording) return "录音中... 🎤"; if (processState.transcribing) return "语音识别中... 🔍"; if (processState.generating) return "生成回答中... 💡"; if (processState.synthesizing) return "语音合成中... 🎵"; return "点击开始对话"; }; const startVisualization = useCallback(() => { if (!analyserRef.current || !dataArrayRef.current || !barsRef.current) { console.warn("可视化组件未就绪"); return; } if (animationFrameRef.current) { cancelAnimationFrame(animationFrameRef.current); animationFrameRef.current = null; } const bufferLength = analyserRef.current.frequencyBinCount; const updateBars = () => { try { analyserRef.current!.getByteFrequencyData(dataArrayRef.current!); const bars = barsRef.current!.children; for (let i = 0; i < bars.length; i++) { const bar = bars[i] as HTMLElement; const dataIndex = Math.floor((i / BAR_COUNT) * (bufferLength / 2)); const rawValue = (dataArrayRef.current![dataIndex] / 255) * 100 * VOLUME_SENSITIVITY; const smoothValue = Math.min( 100, Math.max(10, rawValue * 0.6 + lastValuesRef.current[i] * 0.4) ); lastValuesRef.current[i] = smoothValue; bar.style.cssText = ` height: ${smoothValue}%; transform: scaleY(${0.8 + (smoothValue / 100) * 0.6}); transition: ${i === 0 ? "none" : "height 50ms linear"}; `; } animationFrameRef.current = requestAnimationFrame(updateBars); } catch (err) { console.error("可视化更新失败:", err); } }; animationFrameRef.current = requestAnimationFrame(updateBars); }, [analyserRef, dataArrayRef, barsRef]); // 切换监听状态 const toggleListening = useCallback(async () => { if (isListening) { // 如果正在监听 await cleanupAudio(); // 清理现有音频 } else { // 否则 try { // 尝试 await cleanupAudio(); // 清理现有音频 initializeAudioContext(); // 初始化音频上下文 if (audioSourceType === "mic") { // 如果音频源类型是麦克风 const stream = await navigator.mediaDevices.getUserMedia({ // 获取用户媒体 audio: { noiseSuppression: true, echoCancellation: true }, // 音频配置 }); // 等待获取用户媒体 mediaStreamRef.current = stream; // 设置媒体流 const source = // 创建音频源 audioContextRef.current!.createMediaStreamSource(stream); // 创建音频源 source.connect(analyserRef.current!); // 连接到分析器 } else { const audio = new Audio("/test-audio.mp3"); // 创建音频元素 const source = // 创建音频源 audioContextRef.current!.createMediaElementSource(audio); // 创建音频源 source.connect(analyserRef.current!); // 连接到分析器 audio.play(); // 播放音频 } analyserRef.current!.connect(audioContextRef.current!.destination); // 连接到目标 startVisualization(); // 开始可视化 } catch (err) { console.error("初始化失败:", err); updateState({ error: "音频初始化失败" }); } } setIsListening((prev) => !prev); }, [ isListening, audioSourceType, cleanupAudio, initializeAudioContext, startVisualization, ]); // 示例音频播放 const handlePlaySample = async () => { try { await cleanupAudio(); // 清理现有音频 initializeAudioContext(); // 初始化音频上下文 const audio = new Audio("/test-audio.mp3"); // 创建音频元素 const source = audioContextRef.current!.createMediaElementSource(audio); // 创建音频源 source.connect(analyserRef.current!); // 连接到分析器 analyserRef.current!.connect(audioContextRef.current!.destination); // 连接到目标 await audio.play(); // 播放音频 startVisualization(); // 开始可视化 audio.onended = () => { setIsListening(false); if (animationFrameRef.current) { cancelAnimationFrame(animationFrameRef.current); animationFrameRef.current = null; } }; } catch (err) { console.error("播放示例失败:", err); updateState({ error: "示例播放失败" }); } }; return (
{/* 问候语 */}

{greeting}

{/* 较小较细的字体显示{asrText || "等待语音输入..."}*/}

{asrText || "等待中..."}

{/*较小较细的字体显示{answerText || "等待生成回答..."}*/}

{answerText || "AI助手待命中"}

{/* 音频源切换 */}
{/* 示例播放按钮 */} {audioSourceType === "playback" && ( )} {/* 优化后的音频波形可视化 */}
{/* 底部状态信息 */}

支持唤醒词:"魔镜魔镜"

{/* 呼吸圆点指示器 */}
{/* 扩散波纹效果 */} {isListening && (
)}
{getStatusText()}
{/* 音频播放 */}
); }; export default VoiceAssistant;