diff --git a/README.md b/README.md index 4249c42bcf8bfcd171cdd3a5b38abe97ae9bea5d..6e3df0df4ab27100067ddb16c4c1863dd6ef3717 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Some cool features of AnythingLLM - [LM Studio (all models)](https://lmstudio.ai) - [LocalAi (all models)](https://localai.io/) - [Together AI (chat models)](https://www.together.ai/) +- [Mistral](https://mistral.ai/) **Supported Embedding models:** diff --git a/docker/.env.example b/docker/.env.example index 5bd909af66b46a90fdda3683cf406f6f5f01e0a7..8d33a809d45c9d606a9b8823c8ea97838513b64a 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -44,6 +44,10 @@ GID='1000' # TOGETHER_AI_API_KEY='my-together-ai-key' # TOGETHER_AI_MODEL_PREF='mistralai/Mixtral-8x7B-Instruct-v0.1' +# LLM_PROVIDER='mistral' +# MISTRAL_API_KEY='example-mistral-ai-api-key' +# MISTRAL_MODEL_PREF='mistral-tiny' + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/frontend/src/components/LLMSelection/MistralOptions/index.jsx b/frontend/src/components/LLMSelection/MistralOptions/index.jsx new file mode 100644 index 0000000000000000000000000000000000000000..d5c666415952fa71d316d28088a114aeb53a9aeb --- /dev/null +++ b/frontend/src/components/LLMSelection/MistralOptions/index.jsx @@ -0,0 +1,103 @@ +import { useState, useEffect } from "react"; +import System from "@/models/system"; + +export default function MistralOptions({ settings }) { + const [inputValue, setInputValue] = useState(settings?.MistralApiKey); + const [mistralKey, setMistralKey] = useState(settings?.MistralApiKey); + + return ( + <div className="flex gap-x-4"> + <div className="flex flex-col w-60"> + <label className="text-white text-sm font-semibold block mb-4"> + Mistral API Key + </label> + <input + type="password" + name="MistralApiKey" + className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5" + placeholder="Mistral API Key" + defaultValue={settings?.MistralApiKey ? "*".repeat(20) : ""} + required={true} + autoComplete="off" + spellCheck={false} + onChange={(e) => setInputValue(e.target.value)} + onBlur={() => setMistralKey(inputValue)} + /> + </div> + <MistralModelSelection settings={settings} apiKey={mistralKey} /> + </div> + ); +} + +function MistralModelSelection({ apiKey, settings }) { + const [customModels, setCustomModels] = useState([]); + const [loading, setLoading] = useState(true); + + useEffect(() => { + async function findCustomModels() { + if (!apiKey) { + setCustomModels([]); + setLoading(false); + return; + } + setLoading(true); + const { models } = await System.customModels( + "mistral", + typeof apiKey === "boolean" ? null : apiKey + ); + setCustomModels(models || []); + setLoading(false); + } + findCustomModels(); + }, [apiKey]); + + if (loading || customModels.length == 0) { + return ( + <div className="flex flex-col w-60"> + <label className="text-white text-sm font-semibold block mb-4"> + Chat Model Selection + </label> + <select + name="MistralModelPref" + disabled={true} + className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" + > + <option disabled={true} selected={true}> + {!!apiKey + ? "-- loading available models --" + : "-- waiting for API key --"} + </option> + </select> + </div> + ); + } + + return ( + <div className="flex flex-col w-60"> + <label className="text-white text-sm font-semibold block mb-4"> + Chat Model Selection + </label> + <select + name="MistralModelPref" + required={true} + className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" + > + {customModels.length > 0 && ( + <optgroup label="Available Mistral Models"> + {customModels.map((model) => { + return ( + <option + key={model.id} + value={model.id} + selected={settings?.MistralModelPref === model.id} + > + {model.id} + </option> + ); + })} + </optgroup> + )} + </select> + </div> + ); +} diff --git a/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx index a3089d68843552c31cccfe27d3c83f05e2e9fcd8..da0e7b9f02a815659209a356a27b0d5c5e5bbe57 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx @@ -27,11 +27,21 @@ function castToType(key, value) { return definitions[key].cast(value); } +function recommendedSettings(provider = null) { + switch (provider) { + case "mistral": + return { temp: 0 }; + default: + return { temp: 0.7 }; + } +} + export default function WorkspaceSettings({ active, workspace, settings }) { const { slug } = useParams(); const formEl = useRef(null); const [saving, setSaving] = useState(false); const [hasChanges, setHasChanges] = useState(false); + const defaults = recommendedSettings(settings?.LLMProvider); const handleUpdate = async (e) => { setSaving(true); @@ -143,20 +153,20 @@ export default function WorkspaceSettings({ active, workspace, settings }) { This setting controls how "random" or dynamic your chat responses will be. <br /> - The higher the number (2.0 maximum) the more random and + The higher the number (1.0 maximum) the more random and incoherent. <br /> - <i>Recommended: 0.7</i> + <i>Recommended: {defaults.temp}</i> </p> </div> <input name="openAiTemp" type="number" min={0.0} - max={2.0} + max={1.0} step={0.1} onWheel={(e) => e.target.blur()} - defaultValue={workspace?.openAiTemp ?? 0.7} + defaultValue={workspace?.openAiTemp ?? defaults.temp} className="bg-zinc-900 text-white text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5" placeholder="0.7" required={true} diff --git a/frontend/src/media/llmprovider/mistral.jpeg b/frontend/src/media/llmprovider/mistral.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..1019f495d4d690dd639aa9f4e5751c403b2eff27 Binary files /dev/null and b/frontend/src/media/llmprovider/mistral.jpeg differ diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx index bd6ae511dc7d069f5a1c97e23250339a64f797d5..1efa818d3e900e60dd75cdf2dfebcd03b7067c15 100644 --- a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx @@ -12,6 +12,7 @@ import OllamaLogo from "@/media/llmprovider/ollama.png"; import LMStudioLogo from "@/media/llmprovider/lmstudio.png"; import LocalAiLogo from "@/media/llmprovider/localai.png"; import TogetherAILogo from "@/media/llmprovider/togetherai.png"; +import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import PreLoader from "@/components/Preloader"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions"; @@ -21,9 +22,10 @@ import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions"; import NativeLLMOptions from "@/components/LLMSelection/NativeLLMOptions"; import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions"; import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions"; +import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions"; +import MistralOptions from "@/components/LLMSelection/MistralOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import { MagnifyingGlass } from "@phosphor-icons/react"; -import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions"; export default function GeneralLLMPreference() { const [saving, setSaving] = useState(false); @@ -134,6 +136,13 @@ export default function GeneralLLMPreference() { options: <TogetherAiOptions settings={settings} />, description: "Run open source models from Together AI.", }, + { + name: "Mistral", + value: "mistral", + logo: MistralLogo, + options: <MistralOptions settings={settings} />, + description: "Run open source models from Mistral AI.", + }, { name: "Native", value: "native", diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx index 281f1e8cdd97e8c5f2f5f6a366e6e7856026ba19..3b0046382a9fedc45ba4635a8833d8d3e575a29c 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx @@ -9,6 +9,7 @@ import OllamaLogo from "@/media/llmprovider/ollama.png"; import TogetherAILogo from "@/media/llmprovider/togetherai.png"; import LMStudioLogo from "@/media/llmprovider/lmstudio.png"; import LocalAiLogo from "@/media/llmprovider/localai.png"; +import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import ChromaLogo from "@/media/vectordbs/chroma.png"; import PineconeLogo from "@/media/vectordbs/pinecone.png"; import LanceDbLogo from "@/media/vectordbs/lancedb.png"; @@ -91,6 +92,13 @@ const LLM_SELECTION_PRIVACY = { ], logo: TogetherAILogo, }, + mistral: { + name: "Mistral", + description: [ + "Your prompts and document text used in response creation are visible to Mistral", + ], + logo: MistralLogo, + }, }; const VECTOR_DB_PRIVACY = { diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx index dc060594edf41910f94f565016e06e537ef21a6e..9e8ab84a908947e834b9bcf6067b5d42fbe8d1f6 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx @@ -9,6 +9,7 @@ import LMStudioLogo from "@/media/llmprovider/lmstudio.png"; import LocalAiLogo from "@/media/llmprovider/localai.png"; import TogetherAILogo from "@/media/llmprovider/togetherai.png"; import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png"; +import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions"; import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions"; @@ -17,6 +18,7 @@ import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions"; import NativeLLMOptions from "@/components/LLMSelection/NativeLLMOptions"; import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions"; import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions"; +import MistralOptions from "@/components/LLMSelection/MistralOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import System from "@/models/system"; import paths from "@/utils/paths"; @@ -109,6 +111,13 @@ export default function LLMPreference({ options: <TogetherAiOptions settings={settings} />, description: "Run open source models from Together AI.", }, + { + name: "Mistral", + value: "mistral", + logo: MistralLogo, + options: <MistralOptions settings={settings} />, + description: "Run open source models from Mistral AI.", + }, { name: "Native", value: "native", diff --git a/server/.env.example b/server/.env.example index d060e0ab50133b2dea8bfa1814a6be8e8e54606d..26c51927cfeac0a9397b43021f17c013cb129b7c 100644 --- a/server/.env.example +++ b/server/.env.example @@ -41,6 +41,10 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea # TOGETHER_AI_API_KEY='my-together-ai-key' # TOGETHER_AI_MODEL_PREF='mistralai/Mixtral-8x7B-Instruct-v0.1' +# LLM_PROVIDER='mistral' +# MISTRAL_API_KEY='example-mistral-ai-api-key' +# MISTRAL_MODEL_PREF='mistral-tiny' + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index cd008d420f02342d8bf5e12e59f504bfb00e0363..53d42f2e2ed61174f7d6cae4552be1711b704d02 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -159,6 +159,18 @@ const SystemSettings = { AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, } : {}), + ...(llmProvider === "mistral" + ? { + MistralApiKey: !!process.env.MISTRAL_API_KEY, + MistralModelPref: process.env.MISTRAL_MODEL_PREF, + + // For embedding credentials when mistral is selected. + OpenAiKey: !!process.env.OPEN_AI_KEY, + AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT, + AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY, + AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, + } + : {}), ...(llmProvider === "native" ? { NativeLLMModelPref: process.env.NATIVE_LLM_MODEL_PREF, diff --git a/server/utils/AiProviders/anthropic/index.js b/server/utils/AiProviders/anthropic/index.js index 17f2abc4acde3ed4b3fcd0d74303df2fbd6df01b..56d3a80f0a4232bbfa107c1f2290a56a7fdacb06 100644 --- a/server/utils/AiProviders/anthropic/index.js +++ b/server/utils/AiProviders/anthropic/index.js @@ -26,6 +26,7 @@ class AnthropicLLM { ); this.embedder = embedder; this.answerKey = v4().split("-")[0]; + this.defaultTemp = 0.7; } streamingEnabled() { diff --git a/server/utils/AiProviders/azureOpenAi/index.js b/server/utils/AiProviders/azureOpenAi/index.js index f59fc51fa11b73010662dd0d0870d8ade102a908..639ac102ed14e6966f56c76417ac01028b507764 100644 --- a/server/utils/AiProviders/azureOpenAi/index.js +++ b/server/utils/AiProviders/azureOpenAi/index.js @@ -25,6 +25,7 @@ class AzureOpenAiLLM { "No embedding provider defined for AzureOpenAiLLM - falling back to AzureOpenAiEmbedder for embedding!" ); this.embedder = !embedder ? new AzureOpenAiEmbedder() : embedder; + this.defaultTemp = 0.7; } #appendContext(contextTexts = []) { @@ -93,7 +94,7 @@ class AzureOpenAiLLM { ); const textResponse = await this.openai .getChatCompletions(this.model, messages, { - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, }) .then((res) => { @@ -130,7 +131,7 @@ class AzureOpenAiLLM { this.model, messages, { - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, } ); diff --git a/server/utils/AiProviders/gemini/index.js b/server/utils/AiProviders/gemini/index.js index 348c8f5ed4c8e2de3cec084f0823100fad4ae823..63549fb8dd86043aae7a6db53ae5e765b9caa867 100644 --- a/server/utils/AiProviders/gemini/index.js +++ b/server/utils/AiProviders/gemini/index.js @@ -22,6 +22,7 @@ class GeminiLLM { "INVALID GEMINI LLM SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use Gemini as your LLM." ); this.embedder = embedder; + this.defaultTemp = 0.7; // not used for Gemini } #appendContext(contextTexts = []) { diff --git a/server/utils/AiProviders/lmStudio/index.js b/server/utils/AiProviders/lmStudio/index.js index 614808034c501eb41d24842ba582c3853acb32b4..08950a7b964f42e1f1235ff4444e103998716468 100644 --- a/server/utils/AiProviders/lmStudio/index.js +++ b/server/utils/AiProviders/lmStudio/index.js @@ -25,6 +25,7 @@ class LMStudioLLM { "INVALID LM STUDIO SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use LMStudio as your LLM." ); this.embedder = embedder; + this.defaultTemp = 0.7; } #appendContext(contextTexts = []) { @@ -85,7 +86,7 @@ class LMStudioLLM { const textResponse = await this.lmstudio .createChatCompletion({ model: this.model, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { @@ -122,7 +123,7 @@ class LMStudioLLM { const streamRequest = await this.lmstudio.createChatCompletion( { model: this.model, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, stream: true, messages: await this.compressMessages( diff --git a/server/utils/AiProviders/localAi/index.js b/server/utils/AiProviders/localAi/index.js index 6623ac88ee357708956a4b35bb5ae2c508eba84e..6d265cf82886ff9a9b41ae0360f0cce6c0cb2f2c 100644 --- a/server/utils/AiProviders/localAi/index.js +++ b/server/utils/AiProviders/localAi/index.js @@ -27,6 +27,7 @@ class LocalAiLLM { "INVALID LOCAL AI SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use LocalAI as your LLM." ); this.embedder = embedder; + this.defaultTemp = 0.7; } #appendContext(contextTexts = []) { @@ -85,7 +86,7 @@ class LocalAiLLM { const textResponse = await this.openai .createChatCompletion({ model: this.model, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { @@ -123,7 +124,7 @@ class LocalAiLLM { { model: this.model, stream: true, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { diff --git a/server/utils/AiProviders/mistral/index.js b/server/utils/AiProviders/mistral/index.js new file mode 100644 index 0000000000000000000000000000000000000000..a25185c763126c6a35bb477be78ec895db987183 --- /dev/null +++ b/server/utils/AiProviders/mistral/index.js @@ -0,0 +1,184 @@ +const { chatPrompt } = require("../../chats"); + +class MistralLLM { + constructor(embedder = null, modelPreference = null) { + const { Configuration, OpenAIApi } = require("openai"); + if (!process.env.MISTRAL_API_KEY) + throw new Error("No Mistral API key was set."); + + const config = new Configuration({ + basePath: "https://api.mistral.ai/v1", + apiKey: process.env.MISTRAL_API_KEY, + }); + this.openai = new OpenAIApi(config); + this.model = + modelPreference || process.env.MISTRAL_MODEL_PREF || "mistral-tiny"; + this.limits = { + history: this.promptWindowLimit() * 0.15, + system: this.promptWindowLimit() * 0.15, + user: this.promptWindowLimit() * 0.7, + }; + + if (!embedder) + console.warn( + "No embedding provider defined for MistralLLM - falling back to OpenAiEmbedder for embedding!" + ); + this.embedder = embedder; + this.defaultTemp = 0.0; + } + + #appendContext(contextTexts = []) { + if (!contextTexts || !contextTexts.length) return ""; + return ( + "\nContext:\n" + + contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("") + ); + } + + streamingEnabled() { + return "streamChat" in this && "streamGetChatCompletion" in this; + } + + promptWindowLimit() { + return 32000; + } + + async isValidChatCompletionModel(modelName = "") { + return true; + } + + constructPrompt({ + systemPrompt = "", + contextTexts = [], + chatHistory = [], + userPrompt = "", + }) { + const prompt = { + role: "system", + content: `${systemPrompt}${this.#appendContext(contextTexts)}`, + }; + return [prompt, ...chatHistory, { role: "user", content: userPrompt }]; + } + + async isSafe(_ = "") { + return { safe: true, reasons: [] }; + } + + async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Mistral chat: ${this.model} is not valid for chat completion!` + ); + + const textResponse = await this.openai + .createChatCompletion({ + model: this.model, + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), + messages: await this.compressMessages( + { + systemPrompt: chatPrompt(workspace), + userPrompt: prompt, + chatHistory, + }, + rawHistory + ), + }) + .then((json) => { + const res = json.data; + if (!res.hasOwnProperty("choices")) + throw new Error("Mistral chat: No results!"); + if (res.choices.length === 0) + throw new Error("Mistral chat: No results length!"); + return res.choices[0].message.content; + }) + .catch((error) => { + throw new Error( + `Mistral::createChatCompletion failed with: ${error.message}` + ); + }); + + return textResponse; + } + + async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Mistral chat: ${this.model} is not valid for chat completion!` + ); + + const streamRequest = await this.openai.createChatCompletion( + { + model: this.model, + stream: true, + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), + messages: await this.compressMessages( + { + systemPrompt: chatPrompt(workspace), + userPrompt: prompt, + chatHistory, + }, + rawHistory + ), + }, + { responseType: "stream" } + ); + + return streamRequest; + } + + async getChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Mistral chat: ${this.model} is not valid for chat completion!` + ); + + const { data } = await this.openai.createChatCompletion({ + model: this.model, + messages, + temperature, + }); + + if (!data.hasOwnProperty("choices")) return null; + return data.choices[0].message.content; + } + + async streamGetChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Mistral chat: ${this.model} is not valid for chat completion!` + ); + + const streamRequest = await this.openai.createChatCompletion( + { + model: this.model, + stream: true, + messages, + temperature, + }, + { responseType: "stream" } + ); + return streamRequest; + } + + // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations + async embedTextInput(textInput) { + return await this.embedder.embedTextInput(textInput); + } + async embedChunks(textChunks = []) { + return await this.embedder.embedChunks(textChunks); + } + + async compressMessages(promptArgs = {}, rawHistory = []) { + const { messageArrayCompressor } = require("../../helpers/chat"); + const messageArray = this.constructPrompt(promptArgs); + return await messageArrayCompressor(this, messageArray, rawHistory); + } +} + +module.exports = { + MistralLLM, +}; diff --git a/server/utils/AiProviders/native/index.js b/server/utils/AiProviders/native/index.js index 66cc84d0ca3e69bb3cabaff125937ef305185eff..fff904c462548b09ffbb7ca622780b5ae1209ef1 100644 --- a/server/utils/AiProviders/native/index.js +++ b/server/utils/AiProviders/native/index.js @@ -29,6 +29,7 @@ class NativeLLM { // Make directory when it does not exist in existing installations if (!fs.existsSync(this.cacheDir)) fs.mkdirSync(this.cacheDir); + this.defaultTemp = 0.7; } async #initializeLlamaModel(temperature = 0.7) { @@ -132,7 +133,7 @@ class NativeLLM { ); const model = await this.#llamaClient({ - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), }); const response = await model.call(messages); return response.content; @@ -145,7 +146,7 @@ class NativeLLM { async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { const model = await this.#llamaClient({ - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), }); const messages = await this.compressMessages( { diff --git a/server/utils/AiProviders/ollama/index.js b/server/utils/AiProviders/ollama/index.js index fce96f369844bbd0269a59e05186c3f8b3fee27c..af7fe8210f29bba55518f0e8314a75566b2a6b3f 100644 --- a/server/utils/AiProviders/ollama/index.js +++ b/server/utils/AiProviders/ollama/index.js @@ -20,6 +20,7 @@ class OllamaAILLM { "INVALID OLLAMA SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use Ollama as your LLM." ); this.embedder = embedder; + this.defaultTemp = 0.7; } #ollamaClient({ temperature = 0.07 }) { @@ -113,7 +114,7 @@ class OllamaAILLM { ); const model = this.#ollamaClient({ - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), }); const textResponse = await model .pipe(new StringOutputParser()) @@ -136,7 +137,7 @@ class OllamaAILLM { ); const model = this.#ollamaClient({ - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), }); const stream = await model .pipe(new StringOutputParser()) diff --git a/server/utils/AiProviders/openAi/index.js b/server/utils/AiProviders/openAi/index.js index 038d201d1b395bdfad9188b69c180e030c3eb115..582bc054d2ae8e682644e971f202e94d83aca924 100644 --- a/server/utils/AiProviders/openAi/index.js +++ b/server/utils/AiProviders/openAi/index.js @@ -23,6 +23,7 @@ class OpenAiLLM { "No embedding provider defined for OpenAiLLM - falling back to OpenAiEmbedder for embedding!" ); this.embedder = !embedder ? new OpenAiEmbedder() : embedder; + this.defaultTemp = 0.7; } #appendContext(contextTexts = []) { @@ -127,7 +128,7 @@ class OpenAiLLM { const textResponse = await this.openai .createChatCompletion({ model: this.model, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { @@ -165,7 +166,7 @@ class OpenAiLLM { { model: this.model, stream: true, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { diff --git a/server/utils/AiProviders/togetherAi/index.js b/server/utils/AiProviders/togetherAi/index.js index 44061dd0a4f101f29e3973e1352d3401db366a1a..341661f8dba9c5377b6c9720f02e8560d4e35fdb 100644 --- a/server/utils/AiProviders/togetherAi/index.js +++ b/server/utils/AiProviders/togetherAi/index.js @@ -28,6 +28,7 @@ class TogetherAiLLM { "INVALID TOGETHER AI SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use Together AI as your LLM." ); this.embedder = embedder; + this.defaultTemp = 0.7; } #appendContext(contextTexts = []) { @@ -89,7 +90,7 @@ class TogetherAiLLM { const textResponse = await this.openai .createChatCompletion({ model: this.model, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { @@ -127,7 +128,7 @@ class TogetherAiLLM { { model: this.model, stream: true, - temperature: Number(workspace?.openAiTemp ?? 0.7), + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), n: 1, messages: await this.compressMessages( { diff --git a/server/utils/chats/index.js b/server/utils/chats/index.js index d63de47d5ef353609072fe7bcfe84f25b185850f..764c7795a64664f072c99e14179cf6acc844bd84 100644 --- a/server/utils/chats/index.js +++ b/server/utils/chats/index.js @@ -171,7 +171,7 @@ async function chatWithWorkspace( // Send the text completion. const textResponse = await LLMConnector.getChatCompletion(messages, { - temperature: workspace?.openAiTemp ?? 0.7, + temperature: workspace?.openAiTemp ?? LLMConnector.defaultTemp, }); if (!textResponse) { diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index ceea8d7d2fdbd88c2961f9caf8c7b1ed03bc85d2..cff565ed6e2bf7ad647adc3cb4a3d4d2a7c6ca1c 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -141,7 +141,7 @@ async function streamChatWithWorkspace( `\x1b[31m[STREAMING DISABLED]\x1b[0m Streaming is not available for ${LLMConnector.constructor.name}. Will use regular chat method.` ); completeText = await LLMConnector.getChatCompletion(messages, { - temperature: workspace?.openAiTemp ?? 0.7, + temperature: workspace?.openAiTemp ?? LLMConnector.defaultTemp, }); writeResponseChunk(response, { uuid, @@ -153,7 +153,7 @@ async function streamChatWithWorkspace( }); } else { const stream = await LLMConnector.streamGetChatCompletion(messages, { - temperature: workspace?.openAiTemp ?? 0.7, + temperature: workspace?.openAiTemp ?? LLMConnector.defaultTemp, }); completeText = await handleStreamResponses(response, stream, { uuid, diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js index 87fe976ec7df06c7e991669f143efed759bce1e9..53c641e75ec1c4fe2729951ee6e6fd5e6ee6dc0f 100644 --- a/server/utils/helpers/customModels.js +++ b/server/utils/helpers/customModels.js @@ -5,6 +5,7 @@ const SUPPORT_CUSTOM_MODELS = [ "ollama", "native-llm", "togetherai", + "mistral", ]; async function getCustomModels(provider = "", apiKey = null, basePath = null) { @@ -20,6 +21,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) { return await ollamaAIModels(basePath); case "togetherai": return await getTogetherAiModels(); + case "mistral": + return await getMistralModels(apiKey); case "native-llm": return nativeLLMModels(); default: @@ -117,6 +120,26 @@ async function getTogetherAiModels() { return { models, error: null }; } +async function getMistralModels(apiKey = null) { + const { Configuration, OpenAIApi } = require("openai"); + const config = new Configuration({ + apiKey: apiKey || process.env.MISTRAL_API_KEY, + basePath: "https://api.mistral.ai/v1", + }); + const openai = new OpenAIApi(config); + const models = await openai + .listModels() + .then((res) => res.data.data.filter((model) => !model.id.includes("embed"))) + .catch((e) => { + console.error(`Mistral:listModels`, e.message); + return []; + }); + + // Api Key was successful so lets save it for future uses + if (models.length > 0 && !!apiKey) process.env.MISTRAL_API_KEY = apiKey; + return { models, error: null }; +} + function nativeLLMModels() { const fs = require("fs"); const path = require("path"); diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 2b1f3dacf489d2f140f1cd3b996435acb089c266..2eed9057cac2dc90750e821cd5c31af07e2be339 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -52,6 +52,9 @@ function getLLMProvider(modelPreference = null) { case "togetherai": const { TogetherAiLLM } = require("../AiProviders/togetherAi"); return new TogetherAiLLM(embedder, modelPreference); + case "mistral": + const { MistralLLM } = require("../AiProviders/mistral"); + return new MistralLLM(embedder, modelPreference); case "native": const { NativeLLM } = require("../AiProviders/native"); return new NativeLLM(embedder, modelPreference); @@ -76,6 +79,7 @@ function getEmbeddingEngineSelection() { return new LocalAiEmbedder(); case "native": const { NativeEmbedder } = require("../EmbeddingEngines/native"); + console.log("\x1b[34m[INFO]\x1b[0m Using Native Embedder"); return new NativeEmbedder(); default: return null; diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 5c43da519489c1986bb05fedb9a757709019d988..54e684029122fe0989175ba48f1cc1c4afdbbfd8 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -95,6 +95,15 @@ const KEY_MAPPING = { checks: [nonZero], }, + MistralApiKey: { + envKey: "MISTRAL_API_KEY", + checks: [isNotEmpty], + }, + MistralModelPref: { + envKey: "MISTRAL_MODEL_PREF", + checks: [isNotEmpty], + }, + // Native LLM Settings NativeLLMModelPref: { envKey: "NATIVE_LLM_MODEL_PREF", @@ -268,6 +277,7 @@ function supportedLLM(input = "") { "ollama", "native", "togetherai", + "mistral", ].includes(input); return validSelection ? null : `${input} is not a valid LLM provider.`; }