1173 dynamic cache openrouter (#1176)

* patch agent invocation rule

* Add dynamic model cache from OpenRouter API for context length and available models

frontend/src/hooks/useGetProvidersModels.js

@@ -40,7 +40,7 @@ function groupModels(models) {
   }, {});
 }
 
-const groupedProviders = ["togetherai", "openai"];
+const groupedProviders = ["togetherai", "openai", "openrouter"];
 export default function useGetProviderModels(provider = null) {
   const [defaultModels, setDefaultModels] = useState([]);
   const [customModels, setCustomModels] = useState([]);

server/storage/models/.gitignore

 Xenova
 downloaded/*
-!downloaded/.placeholder
\ No newline at end of file
+!downloaded/.placeholder
+openrouter
\ No newline at end of file

server/utils/AiProviders/openRouter/index.js

@@ -5,11 +5,9 @@ const {
   writeResponseChunk,
   clientAbortedHandler,
 } = require("../../helpers/chat/responses");
-
-function openRouterModels() {
-  const { MODELS } = require("./models.js");
-  return MODELS || {};
-}
+const fs = require("fs");
+const path = require("path");
+const { safeJsonParse } = require("../../http");
 
 class OpenRouterLLM {
   constructor(embedder = null, modelPreference = null) {
@@ -17,8 +15,9 @@ class OpenRouterLLM {
     if (!process.env.OPENROUTER_API_KEY)
       throw new Error("No OpenRouter API key was set.");
 
+    this.basePath = "https://openrouter.ai/api/v1";
     const config = new Configuration({
-      basePath: "https://openrouter.ai/api/v1",
+      basePath: this.basePath,
       apiKey: process.env.OPENROUTER_API_KEY,
       baseOptions: {
         headers: {
@@ -38,6 +37,81 @@ class OpenRouterLLM {
 
     this.embedder = !embedder ? new NativeEmbedder() : embedder;
     this.defaultTemp = 0.7;
+
+    const cacheFolder = path.resolve(
+      process.env.STORAGE_DIR
+        ? path.resolve(process.env.STORAGE_DIR, "models", "openrouter")
+        : path.resolve(__dirname, `../../../storage/models/openrouter`)
+    );
+    fs.mkdirSync(cacheFolder, { recursive: true });
+    this.cacheModelPath = path.resolve(cacheFolder, "models.json");
+    this.cacheAtPath = path.resolve(cacheFolder, ".cached_at");
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
+  }
+
+  async init() {
+    await this.#syncModels();
+    return this;
+  }
+
+  // This checks if the .cached_at file has a timestamp that is more than 1Week (in millis)
+  // from the current date. If it is, then we will refetch the API so that all the models are up
+  // to date.
+  #cacheIsStale() {
+    const MAX_STALE = 6.048e8; // 1 Week in MS
+    if (!fs.existsSync(this.cacheAtPath)) return true;
+    const now = Number(new Date());
+    const timestampMs = Number(fs.readFileSync(this.cacheAtPath));
+    return now - timestampMs > MAX_STALE;
+  }
+
+  // The OpenRouter model API has a lot of models, so we cache this locally in the directory
+  // as if the cache directory JSON file is stale or does not exist we will fetch from API and store it.
+  // This might slow down the first request, but we need the proper token context window
+  // for each model and this is a constructor property - so we can really only get it if this cache exists.
+  // We used to have this as a chore, but given there is an API to get the info - this makes little sense.
+  async #syncModels() {
+    if (fs.existsSync(this.cacheModelPath) && !this.#cacheIsStale())
+      return false;
+
+    this.log(
+      "Model cache is not present or stale. Fetching from OpenRouter API."
+    );
+    await fetch(`${this.basePath}/models`, {
+      method: "GET",
+      headers: {
+        "Content-Type": "application/json",
+      },
+    })
+      .then((res) => res.json())
+      .then(({ data = [] }) => {
+        const models = {};
+        data.forEach((model) => {
+          models[model.id] = {
+            id: model.id,
+            name: model.name,
+            organization:
+              model.id.split("/")[0].charAt(0).toUpperCase() +
+              model.id.split("/")[0].slice(1),
+            maxLength: model.context_length,
+          };
+        });
+        fs.writeFileSync(this.cacheModelPath, JSON.stringify(models), {
+          encoding: "utf-8",
+        });
+        fs.writeFileSync(this.cacheAtPath, String(Number(new Date())), {
+          encoding: "utf-8",
+        });
+        return models;
+      })
+      .catch((e) => {
+        console.error(e);
+        return {};
+      });
+    return;
   }
 
   #appendContext(contextTexts = []) {
@@ -52,8 +126,12 @@ class OpenRouterLLM {
     );
   }
 
-  allModelInformation() {
-    return openRouterModels();
+  models() {
+    if (!fs.existsSync(this.cacheModelPath)) return {};
+    return safeJsonParse(
+      fs.readFileSync(this.cacheModelPath, { encoding: "utf-8" }),
+      {}
+    );
   }
 
   streamingEnabled() {
@@ -61,12 +139,13 @@ class OpenRouterLLM {
   }
 
   promptWindowLimit() {
-    const availableModels = this.allModelInformation();
+    const availableModels = this.models();
     return availableModels[this.model]?.maxLength || 4096;
   }
 
   async isValidChatCompletionModel(model = "") {
-    const availableModels = this.allModelInformation();
+    await this.#syncModels();
+    const availableModels = this.models();
     return availableModels.hasOwnProperty(model);
   }
 
@@ -343,5 +422,4 @@ class OpenRouterLLM {
 
 module.exports = {
   OpenRouterLLM,
-  openRouterModels,
 };

server/utils/AiProviders/openRouter/scripts/.gitignore

-*.json
\ No newline at end of file

server/utils/AiProviders/openRouter/scripts/parse.mjs

-// OpenRouter has lots of models we can use so we use this script
-// to cache all the models. We can see the list of all the models
-// here: https://openrouter.ai/docs#models
-
-// To run, cd into this directory and run `node parse.mjs`
-// copy outputs into the export in ../models.js
-
-// Update the date below if you run this again because OpenRouter added new models.
-// Last Collected: Apr 14, 2024
-
-import fs from "fs";
-
-async function parseChatModels() {
-  const models = {};
-  const response = await fetch("https://openrouter.ai/api/v1/models");
-  const data = await response.json();
-  data.data.forEach((model) => {
-    models[model.id] = {
-      id: model.id,
-      name: model.name,
-      // capitalize first letter
-      organization:
-        model.id.split("/")[0].charAt(0).toUpperCase() +
-        model.id.split("/")[0].slice(1),
-      maxLength: model.context_length,
-    };
-  });
-
-  fs.writeFileSync(
-    "chat_models.json",
-    JSON.stringify(models, null, 2),
-    "utf-8"
-  );
-  return models;
-}
-
-parseChatModels();

server/utils/helpers/customModels.js

-const { openRouterModels } = require("../AiProviders/openRouter");
+const { OpenRouterLLM } = require("../AiProviders/openRouter");
 const { perplexityModels } = require("../AiProviders/perplexity");
 const { togetherAiModels } = require("../AiProviders/togetherAi");
 const SUPPORT_CUSTOM_MODELS = [
@@ -232,7 +232,8 @@ async function getPerplexityModels() {
 }
 
 async function getOpenRouterModels() {
-  const knownModels = await openRouterModels();
+  const openrouter = await new OpenRouterLLM().init();
+  const knownModels = openrouter.models();
   if (!Object.keys(knownModels).length === 0)
     return { models: [], error: null };