From d5951fd5a2552d30120525479114233234aa71e9 Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Thu, 8 Jun 2023 14:00:43 -0700
Subject: [PATCH] change path for file storage for cloud-base persistance
---
server/models/documents.js | 4 +++-
server/models/vectors.js | 4 +++-
server/models/workspace.js | 4 +++-
server/models/workspaceChats.js | 4 +++-
server/utils/files/index.js | 36 ++++++++++++++++++++++++++-------
5 files changed, 41 insertions(+), 11 deletions(-)
diff --git a/server/models/documents.js b/server/models/documents.js
index 148ae8c4e..9e98ab60b 100644
--- a/server/models/documents.js
+++ b/server/models/documents.js
@@ -19,7 +19,9 @@ const Document = {
const { open } = require("sqlite");
const db = await open({
- filename: "anythingllm.db",
+ filename: `${
+ !!process.env.STORAGE_DIR ? `${process.env.STORAGE_DIR}/` : ""
+ }anythingllm.db`,
driver: sqlite3.Database,
});
diff --git a/server/models/vectors.js b/server/models/vectors.js
index d3ad9cc87..6531ac4f7 100644
--- a/server/models/vectors.js
+++ b/server/models/vectors.js
@@ -17,7 +17,9 @@ const DocumentVectors = {
const { open } = require("sqlite");
const db = await open({
- filename: "anythingllm.db",
+ filename: `${
+ !!process.env.STORAGE_DIR ? `${process.env.STORAGE_DIR}/` : ""
+ }anythingllm.db`,
driver: sqlite3.Database,
});
diff --git a/server/models/workspace.js b/server/models/workspace.js
index ddb8faa93..8c6c21255 100644
--- a/server/models/workspace.js
+++ b/server/models/workspace.js
@@ -16,7 +16,9 @@ const Workspace = {
const { open } = require("sqlite");
const db = await open({
- filename: "anythingllm.db",
+ filename: `${
+ !!process.env.STORAGE_DIR ? `${process.env.STORAGE_DIR}/` : ""
+ }anythingllm.db`,
driver: sqlite3.Database,
});
diff --git a/server/models/workspaceChats.js b/server/models/workspaceChats.js
index 8a545b15c..22763527c 100644
--- a/server/models/workspaceChats.js
+++ b/server/models/workspaceChats.js
@@ -14,7 +14,9 @@ const WorkspaceChats = {
const { open } = require("sqlite");
const db = await open({
- filename: "anythingllm.db",
+ filename: `${
+ !!process.env.STORAGE_DIR ? `${process.env.STORAGE_DIR}/` : ""
+ }anythingllm.db`,
driver: sqlite3.Database,
});
diff --git a/server/utils/files/index.js b/server/utils/files/index.js
index 21021f004..978f785fd 100644
--- a/server/utils/files/index.js
+++ b/server/utils/files/index.js
@@ -4,7 +4,11 @@ const { v5: uuidv5 } = require("uuid");
async function collectDocumentData(folderName = null) {
if (!folderName) throw new Error("No docPath provided in request");
- const folder = path.resolve(__dirname, `../../documents/${folderName}`);
+ const folder =
+ process.env.NODE_ENV === "development"
+ ? path.resolve(__dirname, `../../documents/${folderName}`)
+ : path.resolve(process.env.STORAGE_DIR, `documents/${folderName}`);
+
const dirExists = fs.existsSync(folder);
if (!dirExists)
throw new Error(
@@ -28,7 +32,11 @@ async function collectDocumentData(folderName = null) {
// eg: youtube-subject/video-123.json
async function fileData(filePath = null) {
if (!filePath) throw new Error("No docPath provided in request");
- const fullPath = path.resolve(__dirname, `../../documents/${filePath}`);
+
+ const fullPath =
+ process.env.NODE_ENV === "development"
+ ? path.resolve(__dirname, `../../documents/${filePath}`)
+ : path.resolve(process.env.STORAGE_DIR, `documents/${filePath}`);
const fileExists = fs.existsSync(fullPath);
if (!fileExists) return null;
@@ -37,9 +45,12 @@ async function fileData(filePath = null) {
}
async function viewLocalFiles() {
- const folder = path.resolve(__dirname, `../../documents`);
+ const folder =
+ process.env.NODE_ENV === "development"
+ ? path.resolve(__dirname, `../../documents`)
+ : path.resolve(process.env.STORAGE_DIR, `documents`);
const dirExists = fs.existsSync(folder);
- if (!dirExists) return {};
+ if (!dirExists) fs.mkdirSync(folder);
const directory = {
name: "documents",
@@ -49,7 +60,12 @@ async function viewLocalFiles() {
for (const file of fs.readdirSync(folder)) {
if (path.extname(file) === ".md") continue;
- const folderPath = path.resolve(__dirname, `../../documents/${file}`);
+
+ const folderPath =
+ process.env.NODE_ENV === "development"
+ ? path.resolve(__dirname, `../../documents/${file}`)
+ : path.resolve(process.env.STORAGE_DIR, `documents/${file}`);
+
const isFolder = fs.lstatSync(folderPath).isDirectory();
if (isFolder) {
const subdocs = {
@@ -88,7 +104,10 @@ async function cachedVectorInformation(filename = null, checkOnly = false) {
if (!filename) return checkOnly ? false : { exists: false, chunks: [] };
const digest = uuidv5(filename, uuidv5.URL);
- const file = path.resolve(__dirname, `../../vector-cache/${digest}.json`);
+ const file =
+ process.env.NODE_ENV === "development"
+ ? path.resolve(__dirname, `../../vector-cache/${digest}.json`)
+ : path.resolve(process.env.STORAGE_DIR, `vector-cache/${digest}.json`);
const exists = fs.existsSync(file);
if (checkOnly) return exists;
@@ -109,7 +128,10 @@ async function storeVectorResult(vectorData = [], filename = null) {
console.log(
`Caching vectorized results of ${filename} to prevent duplicated embedding.`
);
- const folder = path.resolve(__dirname, `../../vector-cache`);
+ const folder =
+ process.env.NODE_ENV === "development"
+ ? path.resolve(__dirname, `../../vector-cache`)
+ : path.resolve(process.env.STORAGE_DIR, `vector-cache`);
if (!fs.existsSync(folder)) fs.mkdirSync(folder);
--
GitLab