diff --git a/aws/cloudformation/cf_template.template b/aws/cloudformation/cf_template.template
index 405be8cdb9bf3a85020bd7cbf7abeadfc5bda09d..b261a2d89cb24d593a51bc5cd2d609978cf6f610 100644
--- a/aws/cloudformation/cf_template.template
+++ b/aws/cloudformation/cf_template.template
@@ -96,6 +96,7 @@
"!SUB::USER::CONTENT!",
"UID=\"1000\"\n",
"GID=\"1000\"\n",
+ "NO_DEBUG=\"true\"\n",
"END\n",
"cd ../frontend\n",
"rm -rf .env.production\n",
diff --git a/docker/.env.example b/docker/.env.example
index b43c4c263b9c249a9149b43d2ce7a4aa0d5ccb15..e44acd022560c7ef0e67d1962429dc11c273989f 100644
--- a/docker/.env.example
+++ b/docker/.env.example
@@ -19,6 +19,7 @@ PINECONE_INDEX=
# CLOUD DEPLOYMENT VARIRABLES ONLY
# AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.
# JWT_SECRET="my-random-string-for-seeding" # Only needed if AUTH_TOKEN is set. Please generate random string at least 12 chars long.
+# NO_DEBUG="true"
STORAGE_DIR="./server/storage"
GOOGLE_APIS_KEY=
UID='1000'
diff --git a/frontend/src/components/Modals/Keys.jsx b/frontend/src/components/Modals/Keys.jsx
index 7f472d86f6b2b856b690630aee5879d5f5c5b5e5..272f52241532fbc459ecacb11aab591f3c3c8499 100644
--- a/frontend/src/components/Modals/Keys.jsx
+++ b/frontend/src/components/Modals/Keys.jsx
@@ -1,5 +1,5 @@
import React, { useState, useEffect } from "react";
-import { AlertCircle, X } from "react-feather";
+import { AlertCircle, Loader, X } from "react-feather";
import System from "../../models/system";
const noop = () => false;
@@ -55,36 +55,48 @@ export default function KeysModal({ hideModal = noop }) {
</div>
<ShowKey
name="OpenAI API Key"
+ env="OpenAiKey"
value={settings?.OpenAiKey ? "*".repeat(20) : ""}
valid={settings?.OpenAiKey}
+ allowDebug={settings?.CanDebug}
/>
<ShowKey
name="OpenAI Model for chats"
+ env="OpenAiModelPref"
value={settings?.OpenAiModelPref}
valid={!!settings?.OpenAiModelPref}
+ allowDebug={settings?.CanDebug}
/>
<div className="h-[2px] w-full bg-gray-200 dark:bg-stone-600" />
<ShowKey
name="Vector DB Choice"
+ env="VectorDB"
value={settings?.VectorDB}
valid={!!settings?.VectorDB}
+ allowDebug={settings?.CanDebug}
/>
{settings?.VectorDB === "pinecone" && (
<>
<ShowKey
name="Pinecone DB API Key"
+ env="PineConeKey"
value={settings?.PineConeKey ? "*".repeat(20) : ""}
valid={!!settings?.PineConeKey}
+ allowDebug={settings?.CanDebug}
/>
<ShowKey
name="Pinecone DB Environment"
+ env="PineConeEnvironment"
value={settings?.PineConeEnvironment}
valid={!!settings?.PineConeEnvironment}
+ allowDebug={settings?.CanDebug}
/>
<ShowKey
name="Pinecone DB Index"
+ env="PineConeIndex"
value={settings?.PineConeIndex}
valid={!!settings?.PineConeIndex}
+ allowDebug={settings?.CanDebug}
/>
</>
)}
@@ -92,8 +104,10 @@ export default function KeysModal({ hideModal = noop }) {
<>
<ShowKey
name="Chroma Endpoint"
+ env="ChromaEndpoint"
value={settings?.ChromaEndpoint}
valid={!!settings?.ChromaEndpoint}
+ allowDebug={settings?.CanDebug}
/>
</>
)}
@@ -115,47 +129,150 @@ export default function KeysModal({ hideModal = noop }) {
);
}
-function ShowKey({ name, value, valid }) {
- if (!valid) {
+function ShowKey({ name, env, value, valid, allowDebug = true }) {
+ const [isValid, setIsValid] = useState(valid);
+ const [debug, setDebug] = useState(false);
+ const [saving, setSaving] = useState(false);
+ const handleSubmit = async (e) => {
+ e.preventDefault();
+ setSaving(true);
+ const data = {};
+ const form = new FormData(e.target);
+ for (var [key, value] of form.entries()) data[key] = value;
+ const { newValues, error } = await System.updateSystem(data);
+ if (!!error) {
+ alert(error);
+ setSaving(false);
+ setIsValid(false);
+ return;
+ }
+
+ setSaving(false);
+ setDebug(false);
+ setIsValid(true);
+ };
+
+ if (!isValid) {
return (
- <div>
+ <form onSubmit={handleSubmit}>
+ <div>
+ <label
+ htmlFor="error"
+ className="block mb-2 text-sm font-medium text-red-700 dark:text-red-500"
+ >
+ {name}
+ </label>
+ <input
+ type="text"
+ id="error"
+ name={env}
+ disabled={!debug}
+ className="bg-red-50 border border-red-500 text-red-900 placeholder-red-700 text-sm rounded-lg focus:ring-red-500 dark:bg-gray-700 focus:border-red-500 block w-full p-2.5 dark:text-red-500 dark:placeholder-red-500 dark:border-red-500"
+ placeholder={name}
+ defaultValue={value}
+ required={true}
+ autoComplete="off"
+ />
+ <div className="flex items-center justify-between">
+ <p className="mt-2 text-sm text-red-600 dark:text-red-500">
+ Need setup in .env file.
+ </p>
+ {allowDebug && (
+ <>
+ {debug ? (
+ <div className="flex items-center gap-x-2 mt-2">
+ {saving ? (
+ <>
+ <Loader className="animate-spin h-4 w-4 text-slate-300 dark:text-slate-500" />
+ </>
+ ) : (
+ <>
+ <button
+ type="button"
+ onClick={() => setDebug(false)}
+ className="text-xs text-slate-300 dark:text-slate-500"
+ >
+ Cancel
+ </button>
+ <button
+ type="submit"
+ className="text-xs text-blue-300 dark:text-blue-500"
+ >
+ Save
+ </button>
+ </>
+ )}
+ </div>
+ ) : (
+ <button
+ type="button"
+ onClick={() => setDebug(true)}
+ className="mt-2 text-xs text-slate-300 dark:text-slate-500"
+ >
+ Debug
+ </button>
+ )}
+ </>
+ )}
+ </div>
+ </div>
+ </form>
+ );
+ }
+
+ return (
+ <form onSubmit={handleSubmit}>
+ <div className="mb-6">
<label
- htmlFor="error"
- className="block mb-2 text-sm font-medium text-red-700 dark:text-red-500"
+ htmlFor="success"
+ className="block mb-2 text-sm font-medium text-gray-800 dark:text-slate-200"
>
{name}
</label>
<input
type="text"
- id="error"
- disabled={true}
- className="bg-red-50 border border-red-500 text-red-900 placeholder-red-700 text-sm rounded-lg focus:ring-red-500 dark:bg-gray-700 focus:border-red-500 block w-full p-2.5 dark:text-red-500 dark:placeholder-red-500 dark:border-red-500"
- placeholder={name}
+ id="success"
+ name={env}
+ disabled={!debug}
+ className="border border-white text-green-900 dark:text-green-400 placeholder-green-700 dark:placeholder-green-500 text-sm rounded-lg focus:ring-green-500 focus:border-green-500 block w-full p-2.5 dark:bg-gray-700 dark:border-green-500"
defaultValue={value}
+ required={true}
+ autoComplete="off"
/>
- <p className="mt-2 text-sm text-red-600 dark:text-red-500">
- Need setup in .env file.
- </p>
+ {allowDebug && (
+ <div className="flex items-center justify-end">
+ {debug ? (
+ <div className="flex items-center gap-x-2 mt-2">
+ {saving ? (
+ <>
+ <Loader className="animate-spin h-4 w-4 text-slate-300 dark:text-slate-500" />
+ </>
+ ) : (
+ <>
+ <button
+ onClick={() => setDebug(false)}
+ className="text-xs text-slate-300 dark:text-slate-500"
+ >
+ Cancel
+ </button>
+ <button className="text-xs text-blue-300 dark:text-blue-500">
+ Save
+ </button>
+ </>
+ )}
+ </div>
+ ) : (
+ <button
+ onClick={() => setDebug(true)}
+ className="mt-2 text-xs text-slate-300 dark:text-slate-500"
+ >
+ Debug
+ </button>
+ )}
+ </div>
+ )}
</div>
- );
- }
-
- return (
- <div className="mb-6">
- <label
- htmlFor="success"
- className="block mb-2 text-sm font-medium text-gray-800 dark:text-slate-200"
- >
- {name}
- </label>
- <input
- type="text"
- id="success"
- disabled={true}
- className="border border-white text-green-900 dark:text-green-400 placeholder-green-700 dark:placeholder-green-500 text-sm rounded-lg focus:ring-green-500 focus:border-green-500 block w-full p-2.5 dark:bg-gray-700 dark:border-green-500"
- defaultValue={value}
- />
- </div>
+ </form>
);
}
diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js
index 41d6e3556b311d9e5ed3c9c58db500636b5025c5..1ce003d51d80416b6c5338d9beec48c417266770 100644
--- a/frontend/src/models/system.js
+++ b/frontend/src/models/system.js
@@ -74,6 +74,18 @@ const System = {
.then((res) => res?.types)
.catch(() => null);
},
+ updateSystem: async (data) => {
+ return await fetch(`${API_BASE}/system/update-env`, {
+ method: "POST",
+ headers: baseHeaders(),
+ body: JSON.stringify(data),
+ })
+ .then((res) => res.json())
+ .catch((e) => {
+ console.error(e);
+ return { newValues: null, error: e.message };
+ });
+ },
};
export default System;
diff --git a/server/.env.example b/server/.env.example
index 3a7bf9c0b2f65eabb708c302662229a5f1e4ee32..a74cec57059ddea45cc6f6341d942f68ca7d5251 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -19,4 +19,5 @@ PINECONE_INDEX=
# CLOUD DEPLOYMENT VARIRABLES ONLY
# AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.
# JWT_SECRET="my-random-string-for-seeding" # Only needed if AUTH_TOKEN is set. Please generate random string at least 12 chars long.
-# STORAGE_DIR= # absolute filesystem path with no trailing slash
\ No newline at end of file
+# STORAGE_DIR= # absolute filesystem path with no trailing slash
+# NO_DEBUG="true"
\ No newline at end of file
diff --git a/server/endpoints/system.js b/server/endpoints/system.js
index ab47203791e64ddfa33de50c13b74ffc0193b728..ba16b2e5b7b97628012f29a79b41d5d0d0b3e54e 100644
--- a/server/endpoints/system.js
+++ b/server/endpoints/system.js
@@ -8,6 +8,7 @@ const {
acceptedFileTypes,
} = require("../utils/files/documentProcessor");
const { getVectorDbClass } = require("../utils/helpers");
+const { updateENV } = require("../utils/helpers/updateENV");
const { reqBody, makeJWT } = require("../utils/http");
function systemEndpoints(app) {
@@ -26,10 +27,14 @@ function systemEndpoints(app) {
try {
const vectorDB = process.env.VECTOR_DB || "pinecone";
const results = {
+ CanDebug: !!!process.env.NO_DEBUG,
RequiresAuth: !!process.env.AUTH_TOKEN,
VectorDB: vectorDB,
OpenAiKey: !!process.env.OPEN_AI_KEY,
OpenAiModelPref: process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo",
+ AuthToken: !!process.env.AUTH_TOKEN,
+ JWTSecret: !!process.env.JWT_SECRET,
+ StorageDir: process.env.STORAGE_DIR,
...(vectorDB === "pinecone"
? {
PineConeEnvironment: process.env.PINECONE_ENVIRONMENT,
@@ -123,6 +128,17 @@ function systemEndpoints(app) {
response.sendStatus(500).end();
}
});
+
+ app.post("/system/update-env", async (request, response) => {
+ try {
+ const body = reqBody(request);
+ const { newValues, error } = updateENV(body);
+ response.status(200).json({ newValues, error });
+ } catch (e) {
+ console.log(e.message, e);
+ response.sendStatus(500).end();
+ }
+ });
}
module.exports = { systemEndpoints };
diff --git a/server/storage/README.md b/server/storage/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..1282f4b37241a00af4282f2a2ee90f03fb187c14
--- /dev/null
+++ b/server/storage/README.md
@@ -0,0 +1,24 @@
+# AnythingLLM Storage
+
+This folder is for the local or disk storage of ready-to-embed documents, vector-cached embeddings, and the disk-storage of LanceDB and the local SQLite database.
+
+This folder should contain the following folders.
+`documents`
+`lancedb` (if using lancedb)
+`vector-cache`
+and a file named exactly `anythingllm.db`
+
+
+### Common issues
+**SQLITE_FILE_CANNOT_BE_OPENED** in the server log = The DB file does not exist probably because the node instance does not have the correct permissions to write a file to the disk. To solve this..
+
+- Local dev
+ - Create a `anythingllm.db` empty file in this directory. Thats all. No need to reboot the server or anything. If your permissions are correct this should not ever occur since the server will create the file if it does not exist automatically.
+
+- Docker Instance
+ - Get your AnythingLLM docker container id with `docker ps -a`. The container must be running to execute the next commands.
+ - Run `docker container exec -u 0 -t <ANYTHINGLLM DOCKER CONTAINER ID> mkdir -p /app/server/storage /app/server/storage/documents /app/server/storage/vector-cache /app/server/storage/lancedb`
+ - Run `docker container exec -u 0 -t <ANYTHINGLLM DOCKER CONTAINER ID> touch /app/server/storage/anythingllm.db`
+ - Run `docker container exec -u 0 -t <ANYTHINGLLM DOCKER CONTAINER ID> chown -R anythingllm:anythingllm /app/collector /app/server`
+
+ - The above commands will create the appropriate folders inside of the docker container and will persist as long as you do not destroy the container and volume. This will also fix any ownership issues of folder files in the collector and the server.
\ No newline at end of file
diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js
new file mode 100644
index 0000000000000000000000000000000000000000..4161aec13af3ca941effe6d7f8a1b32c31dae60d
--- /dev/null
+++ b/server/utils/helpers/updateENV.js
@@ -0,0 +1,115 @@
+const KEY_MAPPING = {
+ OpenAiKey: {
+ envKey: "OPEN_AI_KEY",
+ checks: [isNotEmpty, validOpenAIKey],
+ },
+ OpenAiModelPref: {
+ envKey: "OPEN_MODEL_PREF",
+ checks: [isNotEmpty, validOpenAIModel],
+ },
+ VectorDB: {
+ envKey: "VECTOR_DB",
+ checks: [isNotEmpty, supportedVectorDB],
+ },
+ ChromaEndpoint: {
+ envKey: "CHROMA_ENDPOINT",
+ checks: [isValidURL, validChromaURL],
+ },
+ PineConeEnvironment: {
+ envKey: "PINECONE_ENVIRONMENT",
+ checks: [],
+ },
+ PineConeKey: {
+ envKey: "PINECONE_API_KEY",
+ checks: [],
+ },
+ PineConeIndex: {
+ envKey: "PINECONE_INDEX",
+ checks: [],
+ },
+ // Not supported yet.
+ // 'AuthToken': 'AUTH_TOKEN',
+ // 'JWTSecret': 'JWT_SECRET',
+ // 'StorageDir': 'STORAGE_DIR',
+};
+
+function isNotEmpty(input = "") {
+ return !input || input.length === 0 ? "Value cannot be empty" : null;
+}
+
+function isValidURL(input = "") {
+ try {
+ new URL(input);
+ return null;
+ } catch (e) {
+ return "URL is not a valid URL.";
+ }
+}
+
+function validOpenAIKey(input = "") {
+ return input.startsWith("sk-") ? null : "OpenAI Key must start with sk-";
+}
+
+function validOpenAIModel(input = "") {
+ const validModels = [
+ "gpt-4",
+ "gpt-4-0613",
+ "gpt-4-32k",
+ "gpt-4-32k-0613",
+ "gpt-3.5-turbo",
+ "gpt-3.5-turbo-0613",
+ "gpt-3.5-turbo-16k",
+ "gpt-3.5-turbo-16k-0613",
+ ];
+ return validModels.includes(input)
+ ? null
+ : `Invalid Model type. Must be one of ${validModels.join(", ")}.`;
+}
+
+function supportedVectorDB(input = "") {
+ const supported = ["chroma", "pinecone", "lancedb"];
+ return supported.includes(input)
+ ? null
+ : `Invalid VectorDB type. Must be one of ${supported.join(", ")}.`;
+}
+
+function validChromaURL(input = "") {
+ return input.slice(-1) === "/"
+ ? `Chroma Instance URL should not end in a trailing slash.`
+ : null;
+}
+
+// This will force update .env variables which for any which reason were not able to be parsed or
+// read from an ENV file as this seems to be a complicating step for many so allowing people to write
+// to the process will at least alleviate that issue. It does not perform comprehensive validity checks or sanity checks
+// and is simply for debugging when the .env not found issue many come across.
+function updateENV(newENVs = {}) {
+ let error = "";
+ const validKeys = Object.keys(KEY_MAPPING);
+ const ENV_KEYS = Object.keys(newENVs).filter((key) =>
+ validKeys.includes(key)
+ );
+ const newValues = {};
+
+ ENV_KEYS.forEach((key) => {
+ const { envKey, checks } = KEY_MAPPING[key];
+ const value = newENVs[key];
+ const errors = checks
+ .map((validityCheck) => validityCheck(value))
+ .filter((err) => typeof err === "string");
+
+ if (errors.length > 0) {
+ error += errors.join("\n");
+ return;
+ }
+
+ newValues[key] = value;
+ process.env[envKey] = value;
+ });
+
+ return { newValues, error: error?.length > 0 ? error : false };
+}
+
+module.exports = {
+ updateENV,
+};