diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx index 6cbcfbf8d5a42c58138520b26f9ada6635b5bccb..ee88735b085b119af3d237db4fc3b89a271290fd 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx @@ -1,10 +1,11 @@ -import { useEffect } from "react"; +import { useEffect, useCallback } from "react"; import { Microphone } from "@phosphor-icons/react"; import { Tooltip } from "react-tooltip"; import _regeneratorRuntime from "regenerator-runtime"; import SpeechRecognition, { useSpeechRecognition, } from "react-speech-recognition"; +import { PROMPT_INPUT_EVENT } from "../../PromptInput"; let timeout; const SILENCE_INTERVAL = 3_200; // wait in seconds of silence before closing. @@ -45,15 +46,49 @@ export default function SpeechToText({ sendCommand }) { clearTimeout(timeout); } + const handleKeyPress = useCallback( + (event) => { + if (event.ctrlKey && event.keyCode === 77) { + if (listening) { + endTTSSession(); + } else { + startSTTSession(); + } + } + }, + [listening, endTTSSession, startSTTSession] + ); + + function handlePromptUpdate(e) { + if (!e?.detail && timeout) { + endTTSSession(); + clearTimeout(timeout); + } + } + + useEffect(() => { + document.addEventListener("keydown", handleKeyPress); + return () => { + document.removeEventListener("keydown", handleKeyPress); + }; + }, [handleKeyPress]); + + useEffect(() => { + if (!!window) + window.addEventListener(PROMPT_INPUT_EVENT, handlePromptUpdate); + return () => + window?.removeEventListener(PROMPT_INPUT_EVENT, handlePromptUpdate); + }, []); + useEffect(() => { - if (transcript?.length > 0) { + if (transcript?.length > 0 && listening) { sendCommand(transcript, false); clearTimeout(timeout); timeout = setTimeout(() => { endTTSSession(); }, SILENCE_INTERVAL); } - }, [transcript]); + }, [transcript, listening]); if (!browserSupportsSpeechRecognition) return null; return ( @@ -69,7 +104,9 @@ export default function SpeechToText({ sendCommand }) { > <Microphone weight="fill" - className="w-6 h-6 pointer-events-none text-white" + className={`w-6 h-6 pointer-events-none text-white overflow-hidden rounded-full ${ + listening ? "animate-pulse" : "" + }`} /> <Tooltip id="tooltip-text-size-btn" diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx index d42ace39a3b69ce28d8de18a3905c81e4e075051..4f09c0b44d318c719fac958c2f002835836efd04 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx @@ -14,6 +14,9 @@ import handleSocketResponse, { AGENT_SESSION_START, } from "@/utils/chat/agent"; import DnDFileUploaderWrapper from "./DnDWrapper"; +import SpeechRecognition, { + useSpeechRecognition, +} from "react-speech-recognition"; export default function ChatContainer({ workspace, knownHistory = [] }) { const { threadSlug = null } = useParams(); @@ -29,6 +32,10 @@ export default function ChatContainer({ workspace, knownHistory = [] }) { setMessage(event.target.value); }; + const { listening, resetTranscript } = useSpeechRecognition({ + clearTranscriptOnListen: true, + }); + // Emit an update to the state of the prompt input without directly // passing a prop in so that it does not re-render constantly. function setMessageEmit(messageContent = "") { @@ -57,11 +64,20 @@ export default function ChatContainer({ workspace, knownHistory = [] }) { }, ]; + if (listening) { + // Stop the mic if the send button is clicked + endTTSSession(); + } setChatHistory(prevChatHistory); setMessageEmit(""); setLoadingResponse(true); }; + function endTTSSession() { + SpeechRecognition.stopListening(); + resetTranscript(); + } + const regenerateAssistantMessage = (chatId) => { const updatedHistory = chatHistory.slice(0, -1); const lastUserMessage = updatedHistory.slice(-1)[0]; diff --git a/frontend/tailwind.config.js b/frontend/tailwind.config.js index acf68d493d95e9706e586750eae5224ff0eaf926..2092c50ae4e48df7a62f32d92a7bafa6697e6d42 100644 --- a/frontend/tailwind.config.js +++ b/frontend/tailwind.config.js @@ -86,7 +86,8 @@ export default { ] }, animation: { - sweep: "sweep 0.5s ease-in-out" + sweep: "sweep 0.5s ease-in-out", + pulse: "pulse 1.5s infinite" }, keyframes: { sweep: { @@ -100,6 +101,26 @@ export default { fadeOut: { "0%": { opacity: 1 }, "100%": { opacity: 0 } + }, + pulse: { + "0%": { + opacity: 1, + transform: "scale(1)", + boxShadow: "0 0 0 rgba(255, 255, 255, 0.0)", + backgroundColor: "rgba(255, 255, 255, 0.0)" + }, + "50%": { + opacity: 1, + transform: "scale(1.1)", + boxShadow: "0 0 15px rgba(255, 255, 255, 0.2)", + backgroundColor: "rgba(255, 255, 255, 0.1)" + }, + "100%": { + opacity: 1, + transform: "scale(1)", + boxShadow: "0 0 0 rgba(255, 255, 255, 0.0)", + backgroundColor: "rgba(255, 255, 255, 0.0)" + } } } }