From 5e73dce506a3e26ba3ad12802742a0e5003c03b0 Mon Sep 17 00:00:00 2001
From: Timothy Carambat <rambat1010@gmail.com>
Date: Mon, 29 Jul 2024 11:49:14 -0700
Subject: [PATCH] Enable editing of OpenRouter stream timeout for slower
connections (#1994)
---
.../LLMSelection/OpenRouterOptions/index.jsx | 81 ++++++++++++++-----
server/models/systemSettings.js | 1 +
server/utils/AiProviders/openRouter/index.js | 19 ++++-
server/utils/helpers/updateENV.js | 4 +
4 files changed, 86 insertions(+), 19 deletions(-)
diff --git a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx
index 470340d2f..2b3e72b3f 100644
--- a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx
+++ b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx
@@ -1,27 +1,72 @@
import System from "@/models/system";
+import { CaretDown, CaretUp } from "@phosphor-icons/react";
import { useState, useEffect } from "react";
export default function OpenRouterOptions({ settings }) {
+ const [showAdvancedControls, setShowAdvancedControls] = useState(false);
+
return (
- <div className="flex gap-[36px] mt-1.5">
- <div className="flex flex-col w-60">
- <label className="text-white text-sm font-semibold block mb-3">
- OpenRouter API Key
- </label>
- <input
- type="password"
- name="OpenRouterApiKey"
- className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
- placeholder="OpenRouter API Key"
- defaultValue={settings?.OpenRouterApiKey ? "*".repeat(20) : ""}
- required={true}
- autoComplete="off"
- spellCheck={false}
- />
+ <div className="flex flex-col gap-y-4 mt-1.5">
+ <div className="flex gap-[36px]">
+ <div className="flex flex-col w-60">
+ <label className="text-white text-sm font-semibold block mb-3">
+ OpenRouter API Key
+ </label>
+ <input
+ type="password"
+ name="OpenRouterApiKey"
+ className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+ placeholder="OpenRouter API Key"
+ defaultValue={settings?.OpenRouterApiKey ? "*".repeat(20) : ""}
+ required={true}
+ autoComplete="off"
+ spellCheck={false}
+ />
+ </div>
+ {!settings?.credentialsOnly && (
+ <OpenRouterModelSelection settings={settings} />
+ )}
+ </div>
+ <AdvancedControls settings={settings} />
+ </div>
+ );
+}
+
+function AdvancedControls({ settings }) {
+ const [showAdvancedControls, setShowAdvancedControls] = useState(false);
+
+ return (
+ <div className="flex flex-col gap-y-4">
+ <button
+ type="button"
+ onClick={() => setShowAdvancedControls(!showAdvancedControls)}
+ className="text-white hover:text-white/70 flex items-center text-sm"
+ >
+ {showAdvancedControls ? "Hide" : "Show"} advanced controls
+ {showAdvancedControls ? (
+ <CaretUp size={14} className="ml-1" />
+ ) : (
+ <CaretDown size={14} className="ml-1" />
+ )}
+ </button>
+ <div hidden={!showAdvancedControls}>
+ <div className="flex flex-col w-60">
+ <label className="text-white text-sm font-semibold block mb-3">
+ Stream Timeout (ms)
+ </label>
+ <input
+ type="number"
+ name="OpenRouterTimeout"
+ className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+ placeholder="Timeout value between token responses to auto-timeout the stream"
+ defaultValue={settings?.OpenRouterTimeout ?? 500}
+ autoComplete="off"
+ onScroll={(e) => e.target.blur()}
+ min={500}
+ step={1}
+ />
+ </div>
</div>
- {!settings?.credentialsOnly && (
- <OpenRouterModelSelection settings={settings} />
- )}
</div>
);
}
diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
index 6f5b4238d..485837506 100644
--- a/server/models/systemSettings.js
+++ b/server/models/systemSettings.js
@@ -424,6 +424,7 @@ const SystemSettings = {
// OpenRouter Keys
OpenRouterApiKey: !!process.env.OPENROUTER_API_KEY,
OpenRouterModelPref: process.env.OPENROUTER_MODEL_PREF,
+ OpenRouterTimeout: process.env.OPENROUTER_TIMEOUT_MS,
// Mistral AI (API) Keys
MistralApiKey: !!process.env.MISTRAL_API_KEY,
diff --git a/server/utils/AiProviders/openRouter/index.js b/server/utils/AiProviders/openRouter/index.js
index 7d0ff3e3b..c7d4dfb0b 100644
--- a/server/utils/AiProviders/openRouter/index.js
+++ b/server/utils/AiProviders/openRouter/index.js
@@ -38,6 +38,7 @@ class OpenRouterLLM {
this.embedder = embedder ?? new NativeEmbedder();
this.defaultTemp = 0.7;
+ this.timeout = this.#parseTimeout();
if (!fs.existsSync(cacheFolder))
fs.mkdirSync(cacheFolder, { recursive: true });
@@ -49,6 +50,22 @@ class OpenRouterLLM {
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
}
+ /**
+ * OpenRouter has various models that never return `finish_reasons` and thus leave the stream open
+ * which causes issues in subsequent messages. This timeout value forces us to close the stream after
+ * x milliseconds. This is a configurable value via the OPENROUTER_TIMEOUT_MS value
+ * @returns {number} The timeout value in milliseconds (default: 500)
+ */
+ #parseTimeout() {
+ this.log(
+ `OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? 500}ms`
+ );
+ if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS))) return 500;
+ const setValue = Number(process.env.OPENROUTER_TIMEOUT_MS);
+ if (setValue < 500) return 500;
+ return setValue;
+ }
+
// This checks if the .cached_at file has a timestamp that is more than 1Week (in millis)
// from the current date. If it is, then we will refetch the API so that all the models are up
// to date.
@@ -161,7 +178,7 @@ class OpenRouterLLM {
}
handleStream(response, stream, responseProps) {
- const timeoutThresholdMs = 500;
+ const timeoutThresholdMs = this.timeout;
const { uuid = uuidv4(), sources = [] } = responseProps;
return new Promise(async (resolve) => {
diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js
index d39941ec4..afacb7279 100644
--- a/server/utils/helpers/updateENV.js
+++ b/server/utils/helpers/updateENV.js
@@ -365,6 +365,10 @@ const KEY_MAPPING = {
envKey: "OPENROUTER_MODEL_PREF",
checks: [isNotEmpty],
},
+ OpenRouterTimeout: {
+ envKey: "OPENROUTER_TIMEOUT_MS",
+ checks: [],
+ },
// Groq Options
GroqApiKey: {
--
GitLab