patch storage dirs for extensions

collector/utils/extensions/Confluence/index.js

@@ -66,11 +66,17 @@ async function loadConfluence({ pageUrl, username, accessToken }) {
   const outFolder = slugify(
     `${subdomain}-confluence-${v4().slice(0, 4)}`
   ).toLowerCase();
-  const outFolderPath = path.resolve(
-    __dirname,
-    `../../../../server/storage/documents/${outFolder}`
-  );
-  fs.mkdirSync(outFolderPath);
+
+  const outFolderPath =
+    process.env.NODE_ENV === "development"
+      ? path.resolve(
+          __dirname,
+          `../../../../server/storage/documents/${outFolder}`
+        )
+      : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`);
+
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
 
   docs.forEach((doc) => {
     const data = {

collector/utils/extensions/GithubRepo/index.js

@@ -31,11 +31,17 @@ async function loadGithubRepo(args) {
   const outFolder = slugify(
     `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}`
   ).toLowerCase();
-  const outFolderPath = path.resolve(
-    __dirname,
-    `../../../../server/storage/documents/${outFolder}`
-  );
-  fs.mkdirSync(outFolderPath);
+
+  const outFolderPath =
+    process.env.NODE_ENV === "development"
+      ? path.resolve(
+          __dirname,
+          `../../../../server/storage/documents/${outFolder}`
+        )
+      : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`);
+
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
 
   for (const doc of docs) {
     if (!doc.pageContent) continue;

collector/utils/extensions/YoutubeTranscript/index.js

@@ -67,11 +67,17 @@ async function loadYouTubeTranscript({ url }) {
   const outFolder = slugify(
     `${metadata.author} YouTube transcripts`
   ).toLowerCase();
-  const outFolderPath = path.resolve(
-    __dirname,
-    `../../../../server/storage/documents/${outFolder}`
-  );
-  if (!fs.existsSync(outFolderPath)) fs.mkdirSync(outFolderPath);
+
+  const outFolderPath =
+    process.env.NODE_ENV === "development"
+      ? path.resolve(
+          __dirname,
+          `../../../../server/storage/documents/${outFolder}`
+        )
+      : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`);
+
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
 
   const data = {
     id: v4(),