diff --git a/.vscode/settings.json b/.vscode/settings.json
index 14efd3fae50a760ffa453965451689caa2700b43..307bbe6c71969cabfa69d747f80a4bdeb2a32570 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -33,6 +33,7 @@
"Mintplex",
"mixtral",
"moderations",
+ "novita",
"numpages",
"Ollama",
"Oobabooga",
diff --git a/README.md b/README.md
index 4edf494820ebc08bf277133297a874460fceafed..861e4fa59e49d324d2b795805b05f248a4585baa 100644
--- a/README.md
+++ b/README.md
@@ -96,6 +96,7 @@ AnythingLLM divides your documents into objects called `workspaces`. A Workspace
- [Text Generation Web UI](https://github.com/oobabooga/text-generation-webui)
- [Apipie](https://apipie.ai/)
- [xAI](https://x.ai/)
+- [Novita AI (chat models)](https://novita.ai/model-api/product/llm-api?utm_source=github_anything-llm&utm_medium=github_readme&utm_campaign=link)
**Embedder models:**
diff --git a/docker/.env.example b/docker/.env.example
index 2f6f896b0fca0c1519aa8d2ddb49b8343b2e064f..0580465965abc5c56cf6c38f437b21533769bb90 100644
--- a/docker/.env.example
+++ b/docker/.env.example
@@ -90,6 +90,10 @@ GID='1000'
# LITE_LLM_BASE_PATH='http://127.0.0.1:4000'
# LITE_LLM_API_KEY='sk-123abc'
+# LLM_PROVIDER='novita'
+# NOVITA_LLM_API_KEY='your-novita-api-key-here' check on https://novita.ai/settings#key-management
+# NOVITA_LLM_MODEL_PREF='gryphe/mythomax-l2-13b'
+
# LLM_PROVIDER='cohere'
# COHERE_API_KEY=
# COHERE_MODEL_PREF='command-r'
diff --git a/frontend/src/components/LLMSelection/NovitaLLMOptions/index.jsx b/frontend/src/components/LLMSelection/NovitaLLMOptions/index.jsx
new file mode 100644
index 0000000000000000000000000000000000000000..26e1fe04b7fb310c5ed4b04800a5500cc1b0607e
--- /dev/null
+++ b/frontend/src/components/LLMSelection/NovitaLLMOptions/index.jsx
@@ -0,0 +1,142 @@
+import System from "@/models/system";
+import { CaretDown, CaretUp } from "@phosphor-icons/react";
+import { useState, useEffect } from "react";
+
+export default function NovitaLLMOptions({ settings }) {
+ return (
+ <div className="flex flex-col gap-y-4 mt-1.5">
+ <div className="flex gap-[36px]">
+ <div className="flex flex-col w-60">
+ <label className="text-white text-sm font-semibold block mb-3">
+ Novita API Key
+ </label>
+ <input
+ type="password"
+ name="NovitaLLMApiKey"
+ className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+ placeholder="Novita API Key"
+ defaultValue={settings?.NovitaLLMApiKey ? "*".repeat(20) : ""}
+ required={true}
+ autoComplete="off"
+ spellCheck={false}
+ />
+ </div>
+ {!settings?.credentialsOnly && (
+ <NovitaModelSelection settings={settings} />
+ )}
+ </div>
+ <AdvancedControls settings={settings} />
+ </div>
+ );
+}
+
+function AdvancedControls({ settings }) {
+ const [showAdvancedControls, setShowAdvancedControls] = useState(false);
+
+ return (
+ <div className="flex flex-col gap-y-4">
+ <button
+ type="button"
+ onClick={() => setShowAdvancedControls(!showAdvancedControls)}
+ className="text-white hover:text-white/70 flex items-center text-sm"
+ >
+ {showAdvancedControls ? "Hide" : "Show"} advanced controls
+ {showAdvancedControls ? (
+ <CaretUp size={14} className="ml-1" />
+ ) : (
+ <CaretDown size={14} className="ml-1" />
+ )}
+ </button>
+ <div hidden={!showAdvancedControls}>
+ <div className="flex flex-col w-60">
+ <label className="text-white text-sm font-semibold block mb-3">
+ Stream Timeout (ms)
+ </label>
+ <input
+ type="number"
+ name="NovitaLLMTimeout"
+ className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+ placeholder="Timeout value between token responses to auto-timeout the stream"
+ defaultValue={settings?.NovitaLLMTimeout ?? 500}
+ autoComplete="off"
+ onScroll={(e) => e.target.blur()}
+ min={500}
+ step={1}
+ />
+ </div>
+ </div>
+ </div>
+ );
+}
+
+function NovitaModelSelection({ settings }) {
+ const [groupedModels, setGroupedModels] = useState({});
+ const [loading, setLoading] = useState(true);
+
+ useEffect(() => {
+ async function findCustomModels() {
+ setLoading(true);
+ const { models } = await System.customModels("novita");
+ if (models?.length > 0) {
+ const modelsByOrganization = models.reduce((acc, model) => {
+ acc[model.organization] = acc[model.organization] || [];
+ acc[model.organization].push(model);
+ return acc;
+ }, {});
+
+ setGroupedModels(modelsByOrganization);
+ }
+
+ setLoading(false);
+ }
+ findCustomModels();
+ }, []);
+
+ if (loading || Object.keys(groupedModels).length === 0) {
+ return (
+ <div className="flex flex-col w-60">
+ <label className="text-white text-sm font-semibold block mb-3">
+ Chat Model Selection
+ </label>
+ <select
+ name="NovitaLLMModelPref"
+ disabled={true}
+ className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+ >
+ <option disabled={true} selected={true}>
+ -- loading available models --
+ </option>
+ </select>
+ </div>
+ );
+ }
+
+ return (
+ <div className="flex flex-col w-60">
+ <label className="text-white text-sm font-semibold block mb-3">
+ Chat Model Selection
+ </label>
+ <select
+ name="NovitaLLMModelPref"
+ required={true}
+ className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+ >
+ {Object.keys(groupedModels)
+ .sort()
+ .map((organization) => (
+ <optgroup key={organization} label={organization}>
+ {groupedModels[organization].map((model) => (
+ <option
+ key={model.id}
+ value={model.id}
+ selected={settings?.NovitaLLMModelPref === model.id}
+ >
+ {model.name}
+ </option>
+ ))}
+ </optgroup>
+ ))}
+ </select>
+ </div>
+ );
+}
diff --git a/frontend/src/hooks/useGetProvidersModels.js b/frontend/src/hooks/useGetProvidersModels.js
index a493438c74aa601e762d5cb199d9e957a281e533..8245872455b303b57c833f07be3e9ea20d5657f3 100644
--- a/frontend/src/hooks/useGetProvidersModels.js
+++ b/frontend/src/hooks/useGetProvidersModels.js
@@ -63,7 +63,13 @@ function groupModels(models) {
}, {});
}
-const groupedProviders = ["togetherai", "fireworksai", "openai", "openrouter"];
+const groupedProviders = [
+ "togetherai",
+ "fireworksai",
+ "openai",
+ "novita",
+ "openrouter",
+];
export default function useGetProviderModels(provider = null) {
const [defaultModels, setDefaultModels] = useState([]);
const [customModels, setCustomModels] = useState([]);
diff --git a/frontend/src/media/llmprovider/novita.png b/frontend/src/media/llmprovider/novita.png
new file mode 100644
index 0000000000000000000000000000000000000000..85f18acb240516dde86aef9c99ceb55cd29523c1
Binary files /dev/null and b/frontend/src/media/llmprovider/novita.png differ
diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
index e7b06e172c1a22eade8f2d78064bb071447a30c7..9ca39d52b0c56bb574bff7e77bdd39d6bfe16cb7 100644
--- a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
+++ b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
@@ -11,6 +11,7 @@ import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
import GeminiLogo from "@/media/llmprovider/gemini.png";
import OllamaLogo from "@/media/llmprovider/ollama.png";
+import NovitaLogo from "@/media/llmprovider/novita.png";
import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
import LocalAiLogo from "@/media/llmprovider/localai.png";
import TogetherAILogo from "@/media/llmprovider/togetherai.png";
@@ -39,6 +40,7 @@ import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions";
import NativeLLMOptions from "@/components/LLMSelection/NativeLLMOptions";
import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions";
import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions";
+import NovitaLLMOptions from "@/components/LLMSelection/NovitaLLMOptions";
import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions";
import FireworksAiOptions from "@/components/LLMSelection/FireworksAiOptions";
import MistralOptions from "@/components/LLMSelection/MistralOptions";
@@ -113,6 +115,15 @@ export const AVAILABLE_LLM_PROVIDERS = [
description: "Run LLMs locally on your own machine.",
requiredConfig: ["OllamaLLMBasePath"],
},
+ {
+ name: "Novita AI",
+ value: "novita",
+ logo: NovitaLogo,
+ options: (settings) => <NovitaLLMOptions settings={settings} />,
+ description:
+ "Reliable, Scalable, and Cost-Effective for LLMs from Novita AI",
+ requiredConfig: ["NovitaLLMApiKey"],
+ },
{
name: "LM Studio",
value: "lmstudio",
diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
index 33750cba2e9079601e646c93af8f4f282ebd4894..d200f60b1f70d592d007d2c11f594baea0640833 100644
--- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
@@ -15,6 +15,7 @@ import MistralLogo from "@/media/llmprovider/mistral.jpeg";
import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
import PerplexityLogo from "@/media/llmprovider/perplexity.png";
import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
+import NovitaLogo from "@/media/llmprovider/novita.png";
import GroqLogo from "@/media/llmprovider/groq.png";
import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
@@ -149,6 +150,14 @@ export const LLM_SELECTION_PRIVACY = {
],
logo: OpenRouterLogo,
},
+ novita: {
+ name: "Novita AI",
+ description: [
+ "Your chats will not be used for training",
+ "Your prompts and document text used in response creation are visible to Novita AI",
+ ],
+ logo: NovitaLogo,
+ },
groq: {
name: "Groq",
description: [
diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
index cc17acfd31b70ed620709a0c471b7324f6f603a2..5f58cba1aad1ec4cc05a2eea14b2903fb4f7db57 100644
--- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
@@ -22,6 +22,7 @@ import LiteLLMLogo from "@/media/llmprovider/litellm.png";
import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
import DeepSeekLogo from "@/media/llmprovider/deepseek.png";
import APIPieLogo from "@/media/llmprovider/apipie.png";
+import NovitaLogo from "@/media/llmprovider/novita.png";
import XAILogo from "@/media/llmprovider/xai.png";
import CohereLogo from "@/media/llmprovider/cohere.png";
@@ -48,6 +49,7 @@ import LiteLLMOptions from "@/components/LLMSelection/LiteLLMOptions";
import AWSBedrockLLMOptions from "@/components/LLMSelection/AwsBedrockLLMOptions";
import DeepSeekOptions from "@/components/LLMSelection/DeepSeekOptions";
import ApiPieLLMOptions from "@/components/LLMSelection/ApiPieOptions";
+import NovitaLLMOptions from "@/components/LLMSelection/NovitaLLMOptions";
import XAILLMOptions from "@/components/LLMSelection/XAiLLMOptions";
import LLMItem from "@/components/LLMSelection/LLMItem";
@@ -104,6 +106,14 @@ const LLMS = [
options: (settings) => <OllamaLLMOptions settings={settings} />,
description: "Run LLMs locally on your own machine.",
},
+ {
+ name: "Novita AI",
+ value: "novita",
+ logo: NovitaLogo,
+ options: (settings) => <NovitaLLMOptions settings={settings} />,
+ description:
+ "Reliable, Scalable, and Cost-Effective for LLMs from Novita AI",
+ },
{
name: "LM Studio",
value: "lmstudio",
diff --git a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx
index c59a77e715c7e822e53112c1bab5009c4bffec56..1e21e50b35b961ece160b09db6040ffa1eca3338 100644
--- a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx
+++ b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx
@@ -17,6 +17,7 @@ const ENABLED_PROVIDERS = [
"koboldcpp",
"togetherai",
"openrouter",
+ "novita",
"mistral",
"perplexity",
"textgenwebui",
@@ -40,6 +41,7 @@ const WARN_PERFORMANCE = [
"ollama",
"localai",
"openrouter",
+ "novita",
"generic-openai",
"textgenwebui",
];
diff --git a/locales/README.ja-JP.md b/locales/README.ja-JP.md
index e273576af05ef2817fb241c60a3fa024e1e9b26d..9ba566eb911736075cb0dc0df0eab1ef05f66fb8 100644
--- a/locales/README.ja-JP.md
+++ b/locales/README.ja-JP.md
@@ -85,6 +85,7 @@ AnythingLLMのいくつかのクールな機能
- [Fireworks AI (チャットモデル)](https://fireworks.ai/)
- [Perplexity (チャットモデル)](https://www.perplexity.ai/)
- [OpenRouter (チャットモデル)](https://openrouter.ai/)
+- [Novita AI (チャットモデル)](https://novita.ai/model-api/product/llm-api?utm_source=github_anything-llm&utm_medium=github_readme&utm_campaign=link)
- [Mistral](https://mistral.ai/)
- [Groq](https://groq.com/)
- [Cohere](https://cohere.com/)
diff --git a/locales/README.zh-CN.md b/locales/README.zh-CN.md
index 03e9ece135d71ee6b5e19084b92286fb99a5a2d4..df14a8b622fcb2be3a618f15969225b5070dac59 100644
--- a/locales/README.zh-CN.md
+++ b/locales/README.zh-CN.md
@@ -81,6 +81,7 @@ AnythingLLM的一些酷炫特性
- [Fireworks AI (聊天模型)](https://fireworks.ai/)
- [Perplexity (聊天模型)](https://www.perplexity.ai/)
- [OpenRouter (聊天模型)](https://openrouter.ai/)
+- [Novita AI (聊天模型)](https://novita.ai/model-api/product/llm-api?utm_source=github_anything-llm&utm_medium=github_readme&utm_campaign=link)
- [Mistral](https://mistral.ai/)
- [Groq](https://groq.com/)
- [Cohere](https://cohere.com/)
diff --git a/server/.env.example b/server/.env.example
index 1995892780666664e5e56868b22b3c693f97bb8e..723b3a644c231b2c00d63be375d368f9fcce08e9 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -91,6 +91,10 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long.
# LITE_LLM_BASE_PATH='http://127.0.0.1:4000'
# LITE_LLM_API_KEY='sk-123abc'
+# LLM_PROVIDER='novita'
+# NOVITA_LLM_API_KEY='your-novita-api-key-here' check on https://novita.ai/settings#key-management
+# NOVITA_LLM_MODEL_PREF='gryphe/mythomax-l2-13b'
+
# LLM_PROVIDER='cohere'
# COHERE_API_KEY=
# COHERE_MODEL_PREF='command-r'
diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
index 41dfc92931687001c38d42201114b38f09d0baea..f43118c62626b85a95da9d529df06693d86fc8f8 100644
--- a/server/models/systemSettings.js
+++ b/server/models/systemSettings.js
@@ -448,6 +448,11 @@ const SystemSettings = {
OllamaLLMKeepAliveSeconds: process.env.OLLAMA_KEEP_ALIVE_TIMEOUT ?? 300,
OllamaLLMPerformanceMode: process.env.OLLAMA_PERFORMANCE_MODE ?? "base",
+ // Novita LLM Keys
+ NovitaLLMApiKey: !!process.env.NOVITA_LLM_API_KEY,
+ NovitaLLMModelPref: process.env.NOVITA_LLM_MODEL_PREF,
+ NovitaLLMTimeout: process.env.NOVITA_LLM_TIMEOUT_MS,
+
// TogetherAI Keys
TogetherAiApiKey: !!process.env.TOGETHER_AI_API_KEY,
TogetherAiModelPref: process.env.TOGETHER_AI_MODEL_PREF,
diff --git a/server/storage/models/.gitignore b/server/storage/models/.gitignore
index b78160e7973478e7bfbe0f7f1cf64720032dfb56..e669b51b277a7fcfbef1dab2ba71832c4b72097a 100644
--- a/server/storage/models/.gitignore
+++ b/server/storage/models/.gitignore
@@ -2,4 +2,5 @@ Xenova
downloaded/*
!downloaded/.placeholder
openrouter
-apipie
\ No newline at end of file
+apipie
+novita
\ No newline at end of file
diff --git a/server/utils/AiProviders/novita/index.js b/server/utils/AiProviders/novita/index.js
new file mode 100644
index 0000000000000000000000000000000000000000..f15d20d41f52e1b1acfacd892049947146f292d2
--- /dev/null
+++ b/server/utils/AiProviders/novita/index.js
@@ -0,0 +1,376 @@
+const { NativeEmbedder } = require("../../EmbeddingEngines/native");
+const { v4: uuidv4 } = require("uuid");
+const {
+ writeResponseChunk,
+ clientAbortedHandler,
+} = require("../../helpers/chat/responses");
+const fs = require("fs");
+const path = require("path");
+const { safeJsonParse } = require("../../http");
+const cacheFolder = path.resolve(
+ process.env.STORAGE_DIR
+ ? path.resolve(process.env.STORAGE_DIR, "models", "novita")
+ : path.resolve(__dirname, `../../../storage/models/novita`)
+);
+
+class NovitaLLM {
+ constructor(embedder = null, modelPreference = null) {
+ if (!process.env.NOVITA_LLM_API_KEY)
+ throw new Error("No Novita API key was set.");
+
+ const { OpenAI: OpenAIApi } = require("openai");
+ this.basePath = "https://api.novita.ai/v3/openai";
+ this.openai = new OpenAIApi({
+ baseURL: this.basePath,
+ apiKey: process.env.NOVITA_LLM_API_KEY ?? null,
+ defaultHeaders: {
+ "HTTP-Referer": "https://anythingllm.com",
+ "X-Novita-Source": "anythingllm",
+ },
+ });
+ this.model =
+ modelPreference ||
+ process.env.NOVITA_LLM_MODEL_PREF ||
+ "gryphe/mythomax-l2-13b";
+ this.limits = {
+ history: this.promptWindowLimit() * 0.15,
+ system: this.promptWindowLimit() * 0.15,
+ user: this.promptWindowLimit() * 0.7,
+ };
+
+ this.embedder = embedder ?? new NativeEmbedder();
+ this.defaultTemp = 0.7;
+ this.timeout = this.#parseTimeout();
+
+ if (!fs.existsSync(cacheFolder))
+ fs.mkdirSync(cacheFolder, { recursive: true });
+ this.cacheModelPath = path.resolve(cacheFolder, "models.json");
+ this.cacheAtPath = path.resolve(cacheFolder, ".cached_at");
+
+ this.log(`Loaded with model: ${this.model}`);
+ }
+
+ log(text, ...args) {
+ console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
+ }
+
+ /**
+ * Novita has various models that never return `finish_reasons` and thus leave the stream open
+ * which causes issues in subsequent messages. This timeout value forces us to close the stream after
+ * x milliseconds. This is a configurable value via the NOVITA_LLM_TIMEOUT_MS value
+ * @returns {number} The timeout value in milliseconds (default: 500)
+ */
+ #parseTimeout() {
+ if (isNaN(Number(process.env.NOVITA_LLM_TIMEOUT_MS))) return 500;
+ const setValue = Number(process.env.NOVITA_LLM_TIMEOUT_MS);
+ if (setValue < 500) return 500;
+ return setValue;
+ }
+
+ // This checks if the .cached_at file has a timestamp that is more than 1Week (in millis)
+ // from the current date. If it is, then we will refetch the API so that all the models are up
+ // to date.
+ #cacheIsStale() {
+ const MAX_STALE = 6.048e8; // 1 Week in MS
+ if (!fs.existsSync(this.cacheAtPath)) return true;
+ const now = Number(new Date());
+ const timestampMs = Number(fs.readFileSync(this.cacheAtPath));
+ return now - timestampMs > MAX_STALE;
+ }
+
+ // The Novita model API has a lot of models, so we cache this locally in the directory
+ // as if the cache directory JSON file is stale or does not exist we will fetch from API and store it.
+ // This might slow down the first request, but we need the proper token context window
+ // for each model and this is a constructor property - so we can really only get it if this cache exists.
+ // We used to have this as a chore, but given there is an API to get the info - this makes little sense.
+ async #syncModels() {
+ if (fs.existsSync(this.cacheModelPath) && !this.#cacheIsStale())
+ return false;
+
+ this.log("Model cache is not present or stale. Fetching from Novita API.");
+ await fetchNovitaModels();
+ return;
+ }
+
+ #appendContext(contextTexts = []) {
+ if (!contextTexts || !contextTexts.length) return "";
+ return (
+ "\nContext:\n" +
+ contextTexts
+ .map((text, i) => {
+ return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+ })
+ .join("")
+ );
+ }
+
+ models() {
+ if (!fs.existsSync(this.cacheModelPath)) return {};
+ return safeJsonParse(
+ fs.readFileSync(this.cacheModelPath, { encoding: "utf-8" }),
+ {}
+ );
+ }
+
+ streamingEnabled() {
+ return "streamGetChatCompletion" in this;
+ }
+
+ static promptWindowLimit(modelName) {
+ const cacheModelPath = path.resolve(cacheFolder, "models.json");
+ const availableModels = fs.existsSync(cacheModelPath)
+ ? safeJsonParse(
+ fs.readFileSync(cacheModelPath, { encoding: "utf-8" }),
+ {}
+ )
+ : {};
+ return availableModels[modelName]?.maxLength || 4096;
+ }
+
+ promptWindowLimit() {
+ const availableModels = this.models();
+ return availableModels[this.model]?.maxLength || 4096;
+ }
+
+ async isValidChatCompletionModel(model = "") {
+ await this.#syncModels();
+ const availableModels = this.models();
+ return availableModels.hasOwnProperty(model);
+ }
+
+ /**
+ * Generates appropriate content array for a message + attachments.
+ * @param {{userPrompt:string, attachments: import("../../helpers").Attachment[]}}
+ * @returns {string|object[]}
+ */
+ #generateContent({ userPrompt, attachments = [] }) {
+ if (!attachments.length) {
+ return userPrompt;
+ }
+
+ const content = [{ type: "text", text: userPrompt }];
+ for (let attachment of attachments) {
+ content.push({
+ type: "image_url",
+ image_url: {
+ url: attachment.contentString,
+ detail: "auto",
+ },
+ });
+ }
+ return content.flat();
+ }
+
+ constructPrompt({
+ systemPrompt = "",
+ contextTexts = [],
+ chatHistory = [],
+ userPrompt = "",
+ attachments = [],
+ }) {
+ const prompt = {
+ role: "system",
+ content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
+ };
+ return [
+ prompt,
+ ...chatHistory,
+ {
+ role: "user",
+ content: this.#generateContent({ userPrompt, attachments }),
+ },
+ ];
+ }
+
+ async getChatCompletion(messages = null, { temperature = 0.7 }) {
+ if (!(await this.isValidChatCompletionModel(this.model)))
+ throw new Error(
+ `Novita chat: ${this.model} is not valid for chat completion!`
+ );
+
+ const result = await this.openai.chat.completions
+ .create({
+ model: this.model,
+ messages,
+ temperature,
+ })
+ .catch((e) => {
+ throw new Error(e.message);
+ });
+
+ if (!result.hasOwnProperty("choices") || result.choices.length === 0)
+ return null;
+ return result.choices[0].message.content;
+ }
+
+ async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
+ if (!(await this.isValidChatCompletionModel(this.model)))
+ throw new Error(
+ `Novita chat: ${this.model} is not valid for chat completion!`
+ );
+
+ const streamRequest = await this.openai.chat.completions.create({
+ model: this.model,
+ stream: true,
+ messages,
+ temperature,
+ });
+ return streamRequest;
+ }
+
+ handleStream(response, stream, responseProps) {
+ const timeoutThresholdMs = this.timeout;
+ const { uuid = uuidv4(), sources = [] } = responseProps;
+
+ return new Promise(async (resolve) => {
+ let fullText = "";
+ let lastChunkTime = null; // null when first token is still not received.
+
+ // Establish listener to early-abort a streaming response
+ // in case things go sideways or the user does not like the response.
+ // We preserve the generated text but continue as if chat was completed
+ // to preserve previously generated content.
+ const handleAbort = () => clientAbortedHandler(resolve, fullText);
+ response.on("close", handleAbort);
+
+ // NOTICE: Not all Novita models will return a stop reason
+ // which keeps the connection open and so the model never finalizes the stream
+ // like the traditional OpenAI response schema does. So in the case the response stream
+ // never reaches a formal close state we maintain an interval timer that if we go >=timeoutThresholdMs with
+ // no new chunks then we kill the stream and assume it to be complete. Novita is quite fast
+ // so this threshold should permit most responses, but we can adjust `timeoutThresholdMs` if
+ // we find it is too aggressive.
+ const timeoutCheck = setInterval(() => {
+ if (lastChunkTime === null) return;
+
+ const now = Number(new Date());
+ const diffMs = now - lastChunkTime;
+ if (diffMs >= timeoutThresholdMs) {
+ this.log(
+ `Novita stream did not self-close and has been stale for >${timeoutThresholdMs}ms. Closing response stream.`
+ );
+ writeResponseChunk(response, {
+ uuid,
+ sources,
+ type: "textResponseChunk",
+ textResponse: "",
+ close: true,
+ error: false,
+ });
+ clearInterval(timeoutCheck);
+ response.removeListener("close", handleAbort);
+ resolve(fullText);
+ }
+ }, 500);
+
+ try {
+ for await (const chunk of stream) {
+ const message = chunk?.choices?.[0];
+ const token = message?.delta?.content;
+ lastChunkTime = Number(new Date());
+
+ if (token) {
+ fullText += token;
+ writeResponseChunk(response, {
+ uuid,
+ sources: [],
+ type: "textResponseChunk",
+ textResponse: token,
+ close: false,
+ error: false,
+ });
+ }
+
+ if (message.finish_reason !== null) {
+ writeResponseChunk(response, {
+ uuid,
+ sources,
+ type: "textResponseChunk",
+ textResponse: "",
+ close: true,
+ error: false,
+ });
+ response.removeListener("close", handleAbort);
+ resolve(fullText);
+ }
+ }
+ } catch (e) {
+ writeResponseChunk(response, {
+ uuid,
+ sources,
+ type: "abort",
+ textResponse: null,
+ close: true,
+ error: e.message,
+ });
+ response.removeListener("close", handleAbort);
+ resolve(fullText);
+ }
+ });
+ }
+
+ // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
+ async embedTextInput(textInput) {
+ return await this.embedder.embedTextInput(textInput);
+ }
+ async embedChunks(textChunks = []) {
+ return await this.embedder.embedChunks(textChunks);
+ }
+
+ async compressMessages(promptArgs = {}, rawHistory = []) {
+ const { messageArrayCompressor } = require("../../helpers/chat");
+ const messageArray = this.constructPrompt(promptArgs);
+ return await messageArrayCompressor(this, messageArray, rawHistory);
+ }
+}
+
+async function fetchNovitaModels() {
+ return await fetch(`https://api.novita.ai/v3/openai/models`, {
+ method: "GET",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ })
+ .then((res) => res.json())
+ .then(({ data = [] }) => {
+ const models = {};
+ data.forEach((model) => {
+ models[model.id] = {
+ id: model.id,
+ name: model.title,
+ organization:
+ model.id.split("/")[0].charAt(0).toUpperCase() +
+ model.id.split("/")[0].slice(1),
+ maxLength: model.context_size,
+ };
+ });
+
+ // Cache all response information
+ if (!fs.existsSync(cacheFolder))
+ fs.mkdirSync(cacheFolder, { recursive: true });
+ fs.writeFileSync(
+ path.resolve(cacheFolder, "models.json"),
+ JSON.stringify(models),
+ {
+ encoding: "utf-8",
+ }
+ );
+ fs.writeFileSync(
+ path.resolve(cacheFolder, ".cached_at"),
+ String(Number(new Date())),
+ {
+ encoding: "utf-8",
+ }
+ );
+ return models;
+ })
+ .catch((e) => {
+ console.error(e);
+ return {};
+ });
+}
+
+module.exports = {
+ NovitaLLM,
+ fetchNovitaModels,
+};
diff --git a/server/utils/agents/aibitat/index.js b/server/utils/agents/aibitat/index.js
index 24f027cff6c0bcecf659d8817deebd12f2ec2059..d61867f4d4520725da77be39cc1396e423ef5f54 100644
--- a/server/utils/agents/aibitat/index.js
+++ b/server/utils/agents/aibitat/index.js
@@ -791,6 +791,8 @@ ${this.getHistory({ to: route.to })
return new Providers.ApiPieProvider({ model: config.model });
case "xai":
return new Providers.XAIProvider({ model: config.model });
+ case "novita":
+ return new Providers.NovitaProvider({ model: config.model });
default:
throw new Error(
diff --git a/server/utils/agents/aibitat/providers/ai-provider.js b/server/utils/agents/aibitat/providers/ai-provider.js
index c9925d1cd084c4166dfcc425ce7591480d7604fb..1bbf4a0a4ed7985247dac0c520df0bbd65af98fc 100644
--- a/server/utils/agents/aibitat/providers/ai-provider.js
+++ b/server/utils/agents/aibitat/providers/ai-provider.js
@@ -206,6 +206,14 @@ class Provider {
apiKey: process.env.LITE_LLM_API_KEY ?? null,
...config,
});
+ case "novita":
+ return new ChatOpenAI({
+ configuration: {
+ baseURL: "https://api.novita.ai/v3/openai",
+ },
+ apiKey: process.env.NOVITA_LLM_API_KEY ?? null,
+ ...config,
+ });
default:
throw new Error(`Unsupported provider ${provider} for this task.`);
diff --git a/server/utils/agents/aibitat/providers/index.js b/server/utils/agents/aibitat/providers/index.js
index 47e2d871687ec1ff8228b89b89868c19234bf222..c454c39387573b66a85cf00a852b30b886ab197f 100644
--- a/server/utils/agents/aibitat/providers/index.js
+++ b/server/utils/agents/aibitat/providers/index.js
@@ -18,6 +18,7 @@ const DeepSeekProvider = require("./deepseek.js");
const LiteLLMProvider = require("./litellm.js");
const ApiPieProvider = require("./apipie.js");
const XAIProvider = require("./xai.js");
+const NovitaProvider = require("./novita.js");
module.exports = {
OpenAIProvider,
@@ -40,4 +41,5 @@ module.exports = {
LiteLLMProvider,
ApiPieProvider,
XAIProvider,
+ NovitaProvider,
};
diff --git a/server/utils/agents/aibitat/providers/novita.js b/server/utils/agents/aibitat/providers/novita.js
new file mode 100644
index 0000000000000000000000000000000000000000..16251aa25e2e3dc6d5ef6dec45cd94a3daa1302c
--- /dev/null
+++ b/server/utils/agents/aibitat/providers/novita.js
@@ -0,0 +1,115 @@
+const OpenAI = require("openai");
+const Provider = require("./ai-provider.js");
+const InheritMultiple = require("./helpers/classes.js");
+const UnTooled = require("./helpers/untooled.js");
+
+/**
+ * The agent provider for the Novita AI provider.
+ */
+class NovitaProvider extends InheritMultiple([Provider, UnTooled]) {
+ model;
+
+ constructor(config = {}) {
+ const { model = "gryphe/mythomax-l2-13b" } = config;
+ super();
+ const client = new OpenAI({
+ baseURL: "https://api.novita.ai/v3/openai",
+ apiKey: process.env.NOVITA_LLM_API_KEY,
+ maxRetries: 3,
+ defaultHeaders: {
+ "HTTP-Referer": "https://anythingllm.com",
+ "X-Novita-Source": "anythingllm",
+ },
+ });
+
+ this._client = client;
+ this.model = model;
+ this.verbose = true;
+ }
+
+ get client() {
+ return this._client;
+ }
+
+ async #handleFunctionCallChat({ messages = [] }) {
+ return await this.client.chat.completions
+ .create({
+ model: this.model,
+ temperature: 0,
+ messages,
+ })
+ .then((result) => {
+ if (!result.hasOwnProperty("choices"))
+ throw new Error("Novita chat: No results!");
+ if (result.choices.length === 0)
+ throw new Error("Novita chat: No results length!");
+ return result.choices[0].message.content;
+ })
+ .catch((_) => {
+ return null;
+ });
+ }
+
+ /**
+ * Create a completion based on the received messages.
+ *
+ * @param messages A list of messages to send to the API.
+ * @param functions
+ * @returns The completion.
+ */
+ async complete(messages, functions = null) {
+ let completion;
+ if (functions.length > 0) {
+ const { toolCall, text } = await this.functionCall(
+ messages,
+ functions,
+ this.#handleFunctionCallChat.bind(this)
+ );
+
+ if (toolCall !== null) {
+ this.providerLog(`Valid tool call found - running ${toolCall.name}.`);
+ this.deduplicator.trackRun(toolCall.name, toolCall.arguments);
+ return {
+ result: null,
+ functionCall: {
+ name: toolCall.name,
+ arguments: toolCall.arguments,
+ },
+ cost: 0,
+ };
+ }
+ completion = { content: text };
+ }
+
+ if (!completion?.content) {
+ this.providerLog("Will assume chat completion without tool call inputs.");
+ const response = await this.client.chat.completions.create({
+ model: this.model,
+ messages: this.cleanMsgs(messages),
+ });
+ completion = response.choices[0].message;
+ }
+
+ // The UnTooled class inherited Deduplicator is mostly useful to prevent the agent
+ // from calling the exact same function over and over in a loop within a single chat exchange
+ // _but_ we should enable it to call previously used tools in a new chat interaction.
+ this.deduplicator.reset("runs");
+ return {
+ result: completion.content,
+ cost: 0,
+ };
+ }
+
+ /**
+ * Get the cost of the completion.
+ *
+ * @param _usage The completion to get the cost for.
+ * @returns The cost of the completion.
+ * Stubbed since Novita AI has no cost basis.
+ */
+ getCost() {
+ return 0;
+ }
+}
+
+module.exports = NovitaProvider;
diff --git a/server/utils/agents/index.js b/server/utils/agents/index.js
index fd7d06e8bd1d32078c6ce11ca8842e8b888df1fc..6b1d42af298a2a6c4469fe1e784849ba71297c6d 100644
--- a/server/utils/agents/index.js
+++ b/server/utils/agents/index.js
@@ -173,6 +173,10 @@ class AgentHandler {
if (!process.env.XAI_LLM_API_KEY)
throw new Error("xAI API Key must be provided to use agents.");
break;
+ case "novita":
+ if (!process.env.NOVITA_LLM_API_KEY)
+ throw new Error("Novita API Key must be provided to use agents.");
+ break;
default:
throw new Error(
@@ -234,6 +238,8 @@ class AgentHandler {
return process.env.APIPIE_LLM_MODEL_PREF ?? null;
case "xai":
return process.env.XAI_LLM_MODEL_PREF ?? "grok-beta";
+ case "novita":
+ return process.env.NOVITA_LLM_MODEL_PREF ?? "gryphe/mythomax-l2-13b";
default:
return null;
}
diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js
index 7ccbf13c73874571a5b2ba66a172da1bb7f22044..1639337695bcccd4e49c8d80744c95737d0cc2a8 100644
--- a/server/utils/helpers/customModels.js
+++ b/server/utils/helpers/customModels.js
@@ -4,6 +4,7 @@ const { perplexityModels } = require("../AiProviders/perplexity");
const { togetherAiModels } = require("../AiProviders/togetherAi");
const { fireworksAiModels } = require("../AiProviders/fireworksAi");
const { ElevenLabsTTS } = require("../TextToSpeech/elevenLabs");
+const { fetchNovitaModels } = require("../AiProviders/novita");
const SUPPORT_CUSTOM_MODELS = [
"openai",
"localai",
@@ -21,6 +22,7 @@ const SUPPORT_CUSTOM_MODELS = [
"groq",
"deepseek",
"apipie",
+ "novita",
"xai",
];
@@ -61,6 +63,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) {
return await getDeepSeekModels(apiKey);
case "apipie":
return await getAPIPieModels(apiKey);
+ case "novita":
+ return await getNovitaModels();
case "xai":
return await getXAIModels(apiKey);
default:
@@ -362,6 +366,20 @@ async function getOpenRouterModels() {
return { models, error: null };
}
+async function getNovitaModels() {
+ const knownModels = await fetchNovitaModels();
+ if (!Object.keys(knownModels).length === 0)
+ return { models: [], error: null };
+ const models = Object.values(knownModels).map((model) => {
+ return {
+ id: model.id,
+ organization: model.organization,
+ name: model.name,
+ };
+ });
+ return { models, error: null };
+}
+
async function getAPIPieModels(apiKey = null) {
const knownModels = await fetchApiPieModels(apiKey);
if (!Object.keys(knownModels).length === 0)
diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js
index 84f971cc63644d99447eebeba16a9fdcc207004e..57ec191e77eeced0c4700be6320443569814b6d6 100644
--- a/server/utils/helpers/index.js
+++ b/server/utils/helpers/index.js
@@ -165,6 +165,9 @@ function getLLMProvider({ provider = null, model = null } = {}) {
case "apipie":
const { ApiPieLLM } = require("../AiProviders/apipie");
return new ApiPieLLM(embedder, model);
+ case "novita":
+ const { NovitaLLM } = require("../AiProviders/novita");
+ return new NovitaLLM(embedder, model);
case "xai":
const { XAiLLM } = require("../AiProviders/xai");
return new XAiLLM(embedder, model);
@@ -297,6 +300,9 @@ function getLLMProviderClass({ provider = null } = {}) {
case "apipie":
const { ApiPieLLM } = require("../AiProviders/apipie");
return ApiPieLLM;
+ case "novita":
+ const { NovitaLLM } = require("../AiProviders/novita");
+ return NovitaLLM;
case "xai":
const { XAiLLM } = require("../AiProviders/xai");
return XAiLLM;
diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js
index 6081159a5242f0b72bc2befaf1532a60f6d88882..676eb812f5ec495ab91f402002860898f4d25591 100644
--- a/server/utils/helpers/updateENV.js
+++ b/server/utils/helpers/updateENV.js
@@ -395,6 +395,20 @@ const KEY_MAPPING = {
checks: [],
},
+ // Novita Options
+ NovitaLLMApiKey: {
+ envKey: "NOVITA_LLM_API_KEY",
+ checks: [isNotEmpty],
+ },
+ NovitaLLMModelPref: {
+ envKey: "NOVITA_LLM_MODEL_PREF",
+ checks: [isNotEmpty],
+ },
+ NovitaLLMTimeout: {
+ envKey: "NOVITA_LLM_TIMEOUT_MS",
+ checks: [],
+ },
+
// Groq Options
GroqApiKey: {
envKey: "GROQ_API_KEY",
@@ -655,6 +669,7 @@ function supportedLLM(input = "") {
"huggingface",
"perplexity",
"openrouter",
+ "novita",
"groq",
"koboldcpp",
"textgenwebui",