From 6a0f068becf64fb0069d76953302912e7aeaccb1 Mon Sep 17 00:00:00 2001
From: Roman Rojas <32991189+Rrojaski@users.noreply.github.com>
Date: Wed, 7 Aug 2024 15:00:01 -0400
Subject: [PATCH] 1603 speech to text hotkey (#1771)
* Added ctrl + enter hotkeys to init speach to text
* Ran linter
* Fixed speech transcript from being submitted twice when the user clicks the send button. Updated speech hotkeys.
* Added pulse animation to mic
* Fixed prompt double-send when clicking the send button or ending the TTS session.
* Fixed comment grammar
* Update mic hotkeys
---------
Co-authored-by: Timothy Carambat <rambat1010@gmail.com>
---
.../PromptInput/SpeechToText/index.jsx | 45 +++++++++++++++++--
.../WorkspaceChat/ChatContainer/index.jsx | 16 +++++++
frontend/tailwind.config.js | 23 +++++++++-
3 files changed, 79 insertions(+), 5 deletions(-)
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx
index 6cbcfbf8d..ee88735b0 100644
--- a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx
+++ b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx
@@ -1,10 +1,11 @@
-import { useEffect } from "react";
+import { useEffect, useCallback } from "react";
import { Microphone } from "@phosphor-icons/react";
import { Tooltip } from "react-tooltip";
import _regeneratorRuntime from "regenerator-runtime";
import SpeechRecognition, {
useSpeechRecognition,
} from "react-speech-recognition";
+import { PROMPT_INPUT_EVENT } from "../../PromptInput";
let timeout;
const SILENCE_INTERVAL = 3_200; // wait in seconds of silence before closing.
@@ -45,15 +46,49 @@ export default function SpeechToText({ sendCommand }) {
clearTimeout(timeout);
}
+ const handleKeyPress = useCallback(
+ (event) => {
+ if (event.ctrlKey && event.keyCode === 77) {
+ if (listening) {
+ endTTSSession();
+ } else {
+ startSTTSession();
+ }
+ }
+ },
+ [listening, endTTSSession, startSTTSession]
+ );
+
+ function handlePromptUpdate(e) {
+ if (!e?.detail && timeout) {
+ endTTSSession();
+ clearTimeout(timeout);
+ }
+ }
+
+ useEffect(() => {
+ document.addEventListener("keydown", handleKeyPress);
+ return () => {
+ document.removeEventListener("keydown", handleKeyPress);
+ };
+ }, [handleKeyPress]);
+
+ useEffect(() => {
+ if (!!window)
+ window.addEventListener(PROMPT_INPUT_EVENT, handlePromptUpdate);
+ return () =>
+ window?.removeEventListener(PROMPT_INPUT_EVENT, handlePromptUpdate);
+ }, []);
+
useEffect(() => {
- if (transcript?.length > 0) {
+ if (transcript?.length > 0 && listening) {
sendCommand(transcript, false);
clearTimeout(timeout);
timeout = setTimeout(() => {
endTTSSession();
}, SILENCE_INTERVAL);
}
- }, [transcript]);
+ }, [transcript, listening]);
if (!browserSupportsSpeechRecognition) return null;
return (
@@ -69,7 +104,9 @@ export default function SpeechToText({ sendCommand }) {
>
<Microphone
weight="fill"
- className="w-6 h-6 pointer-events-none text-white"
+ className={`w-6 h-6 pointer-events-none text-white overflow-hidden rounded-full ${
+ listening ? "animate-pulse" : ""
+ }`}
/>
<Tooltip
id="tooltip-text-size-btn"
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx
index d42ace39a..4f09c0b44 100644
--- a/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx
+++ b/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx
@@ -14,6 +14,9 @@ import handleSocketResponse, {
AGENT_SESSION_START,
} from "@/utils/chat/agent";
import DnDFileUploaderWrapper from "./DnDWrapper";
+import SpeechRecognition, {
+ useSpeechRecognition,
+} from "react-speech-recognition";
export default function ChatContainer({ workspace, knownHistory = [] }) {
const { threadSlug = null } = useParams();
@@ -29,6 +32,10 @@ export default function ChatContainer({ workspace, knownHistory = [] }) {
setMessage(event.target.value);
};
+ const { listening, resetTranscript } = useSpeechRecognition({
+ clearTranscriptOnListen: true,
+ });
+
// Emit an update to the state of the prompt input without directly
// passing a prop in so that it does not re-render constantly.
function setMessageEmit(messageContent = "") {
@@ -57,11 +64,20 @@ export default function ChatContainer({ workspace, knownHistory = [] }) {
},
];
+ if (listening) {
+ // Stop the mic if the send button is clicked
+ endTTSSession();
+ }
setChatHistory(prevChatHistory);
setMessageEmit("");
setLoadingResponse(true);
};
+ function endTTSSession() {
+ SpeechRecognition.stopListening();
+ resetTranscript();
+ }
+
const regenerateAssistantMessage = (chatId) => {
const updatedHistory = chatHistory.slice(0, -1);
const lastUserMessage = updatedHistory.slice(-1)[0];
diff --git a/frontend/tailwind.config.js b/frontend/tailwind.config.js
index acf68d493..2092c50ae 100644
--- a/frontend/tailwind.config.js
+++ b/frontend/tailwind.config.js
@@ -86,7 +86,8 @@ export default {
]
},
animation: {
- sweep: "sweep 0.5s ease-in-out"
+ sweep: "sweep 0.5s ease-in-out",
+ pulse: "pulse 1.5s infinite"
},
keyframes: {
sweep: {
@@ -100,6 +101,26 @@ export default {
fadeOut: {
"0%": { opacity: 1 },
"100%": { opacity: 0 }
+ },
+ pulse: {
+ "0%": {
+ opacity: 1,
+ transform: "scale(1)",
+ boxShadow: "0 0 0 rgba(255, 255, 255, 0.0)",
+ backgroundColor: "rgba(255, 255, 255, 0.0)"
+ },
+ "50%": {
+ opacity: 1,
+ transform: "scale(1.1)",
+ boxShadow: "0 0 15px rgba(255, 255, 255, 0.2)",
+ backgroundColor: "rgba(255, 255, 255, 0.1)"
+ },
+ "100%": {
+ opacity: 1,
+ transform: "scale(1)",
+ boxShadow: "0 0 0 rgba(255, 255, 255, 0.0)",
+ backgroundColor: "rgba(255, 255, 255, 0.0)"
+ }
}
}
}
--
GitLab