diff --git a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx index 470340d2ff3d9b6c293d5871dfaf8c160764fa0a..2b3e72b3fc1257dbc305d2b749a38316f6ecd62a 100644 --- a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx +++ b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx @@ -1,27 +1,72 @@ import System from "@/models/system"; +import { CaretDown, CaretUp } from "@phosphor-icons/react"; import { useState, useEffect } from "react"; export default function OpenRouterOptions({ settings }) { + const [showAdvancedControls, setShowAdvancedControls] = useState(false); + return ( - <div className="flex gap-[36px] mt-1.5"> - <div className="flex flex-col w-60"> - <label className="text-white text-sm font-semibold block mb-3"> - OpenRouter API Key - </label> - <input - type="password" - name="OpenRouterApiKey" - className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5" - placeholder="OpenRouter API Key" - defaultValue={settings?.OpenRouterApiKey ? "*".repeat(20) : ""} - required={true} - autoComplete="off" - spellCheck={false} - /> + <div className="flex flex-col gap-y-4 mt-1.5"> + <div className="flex gap-[36px]"> + <div className="flex flex-col w-60"> + <label className="text-white text-sm font-semibold block mb-3"> + OpenRouter API Key + </label> + <input + type="password" + name="OpenRouterApiKey" + className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5" + placeholder="OpenRouter API Key" + defaultValue={settings?.OpenRouterApiKey ? "*".repeat(20) : ""} + required={true} + autoComplete="off" + spellCheck={false} + /> + </div> + {!settings?.credentialsOnly && ( + <OpenRouterModelSelection settings={settings} /> + )} + </div> + <AdvancedControls settings={settings} /> + </div> + ); +} + +function AdvancedControls({ settings }) { + const [showAdvancedControls, setShowAdvancedControls] = useState(false); + + return ( + <div className="flex flex-col gap-y-4"> + <button + type="button" + onClick={() => setShowAdvancedControls(!showAdvancedControls)} + className="text-white hover:text-white/70 flex items-center text-sm" + > + {showAdvancedControls ? "Hide" : "Show"} advanced controls + {showAdvancedControls ? ( + <CaretUp size={14} className="ml-1" /> + ) : ( + <CaretDown size={14} className="ml-1" /> + )} + </button> + <div hidden={!showAdvancedControls}> + <div className="flex flex-col w-60"> + <label className="text-white text-sm font-semibold block mb-3"> + Stream Timeout (ms) + </label> + <input + type="number" + name="OpenRouterTimeout" + className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5" + placeholder="Timeout value between token responses to auto-timeout the stream" + defaultValue={settings?.OpenRouterTimeout ?? 500} + autoComplete="off" + onScroll={(e) => e.target.blur()} + min={500} + step={1} + /> + </div> </div> - {!settings?.credentialsOnly && ( - <OpenRouterModelSelection settings={settings} /> - )} </div> ); } diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 6f5b4238df6d31918a9144303d81d0b6b8ee17a7..485837506ab12c1d5d813cc3570d4cd4da442429 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -424,6 +424,7 @@ const SystemSettings = { // OpenRouter Keys OpenRouterApiKey: !!process.env.OPENROUTER_API_KEY, OpenRouterModelPref: process.env.OPENROUTER_MODEL_PREF, + OpenRouterTimeout: process.env.OPENROUTER_TIMEOUT_MS, // Mistral AI (API) Keys MistralApiKey: !!process.env.MISTRAL_API_KEY, diff --git a/server/utils/AiProviders/openRouter/index.js b/server/utils/AiProviders/openRouter/index.js index 7d0ff3e3b5c7b6f7782d12fc9a5eafaf6ee5254b..c7d4dfb0b6c60ede62edec8c7788d25ae4175dd3 100644 --- a/server/utils/AiProviders/openRouter/index.js +++ b/server/utils/AiProviders/openRouter/index.js @@ -38,6 +38,7 @@ class OpenRouterLLM { this.embedder = embedder ?? new NativeEmbedder(); this.defaultTemp = 0.7; + this.timeout = this.#parseTimeout(); if (!fs.existsSync(cacheFolder)) fs.mkdirSync(cacheFolder, { recursive: true }); @@ -49,6 +50,22 @@ class OpenRouterLLM { console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args); } + /** + * OpenRouter has various models that never return `finish_reasons` and thus leave the stream open + * which causes issues in subsequent messages. This timeout value forces us to close the stream after + * x milliseconds. This is a configurable value via the OPENROUTER_TIMEOUT_MS value + * @returns {number} The timeout value in milliseconds (default: 500) + */ + #parseTimeout() { + this.log( + `OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? 500}ms` + ); + if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS))) return 500; + const setValue = Number(process.env.OPENROUTER_TIMEOUT_MS); + if (setValue < 500) return 500; + return setValue; + } + // This checks if the .cached_at file has a timestamp that is more than 1Week (in millis) // from the current date. If it is, then we will refetch the API so that all the models are up // to date. @@ -161,7 +178,7 @@ class OpenRouterLLM { } handleStream(response, stream, responseProps) { - const timeoutThresholdMs = 500; + const timeoutThresholdMs = this.timeout; const { uuid = uuidv4(), sources = [] } = responseProps; return new Promise(async (resolve) => { diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index d39941ec40fbb5115266ebf0f4474a84bd51b9f8..afacb7279fb86c41a70a418293c73bfc0200408c 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -365,6 +365,10 @@ const KEY_MAPPING = { envKey: "OPENROUTER_MODEL_PREF", checks: [isNotEmpty], }, + OpenRouterTimeout: { + envKey: "OPENROUTER_TIMEOUT_MS", + checks: [], + }, // Groq Options GroqApiKey: {