add ability to purge document from custom documents as well as cleanup its...

add ability to purge document from custom documents as well as cleanup its associated cache file (#113)

* add ability to purge document from custom documents as well as cleanup its assoicated cache file

* update alert text

frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx

@@ -8,6 +8,7 @@ import {
   Zap,
 } from "react-feather";
 import { nFormatter } from "../../../../../utils/numbers";
+import System from "../../../../../models/system";
 
 export default function Directory({
   files,
@@ -19,6 +20,16 @@ export default function Directory({
   const [isExpanded, toggleExpanded] = useState(false);
   const [showDetails, toggleDetails] = useState(false);
   const [showZap, setShowZap] = useState(false);
+  const handleDelete = async (name, meta) => {
+    if (
+      !window.confirm(
+        "Are you sure you want to delete this document?\nThis will require you to re-upload and re-embed it.\nThis document will be removed from any workspace that is currently referencing it.\nThis action is not reversible."
+      )
+    )
+      return false;
+    document?.getElementById(meta?.id)?.remove();
+    await System.deleteDocument(name, meta);
+  };
 
   if (files.type === "folder") {
     return (
@@ -73,7 +84,7 @@ export default function Directory({
 
   const { name, type: _type, ...meta } = files;
   return (
-    <div className="ml-[20px] my-2">
+    <div className="ml-[20px] my-2" id={meta.id}>
       <div className="flex items-center">
         {meta?.cached && (
           <button
@@ -134,15 +145,25 @@ export default function Directory({
         </div>
       </div>
       {showDetails && (
-        <div className="ml-[20px] flex flex-col gap-y-1 my-1 p-2 rounded-md bg-slate-200 font-mono text-sm overflow-x-scroll">
-          {Object.entries(meta).map(([key, value], i) => {
-            if (key === "cached") return null;
-            return (
-              <p key={i} className="whitespace-pre">
-                {key}: {value}
-              </p>
-            );
-          })}
+        <div className="w-full flex flex-col">
+          <div className="ml-[20px] flex flex-col gap-y-1 my-1 p-2 rounded-md bg-slate-200 font-mono text-sm overflow-x-scroll">
+            {Object.entries(meta).map(([key, value], i) => {
+              if (key === "cached") return null;
+              return (
+                <p key={i} className="whitespace-pre">
+                  {key}: {value}
+                </p>
+              );
+            })}
+          </div>
+          <div
+            onClick={() => handleDelete(`${parent}/${name}`, meta)}
+            className="flex items-center justify-end w-full"
+          >
+            <button className="text-sm text-slate-400 dark:text-stone-500 hover:text-red-500">
+              Purge Document
+            </button>
+          </div>
         </div>
       )}
     </div>

frontend/src/models/system.js

@@ -86,6 +86,18 @@ const System = {
         return { newValues: null, error: e.message };
       });
   },
+  deleteDocument: async (name, meta) => {
+    return await fetch(`${API_BASE}/system/remove-document`, {
+      method: "DELETE",
+      headers: baseHeaders(),
+      body: JSON.stringify({ name, meta }),
+    })
+      .then((res) => res.ok)
+      .catch((e) => {
+        console.error(e);
+        return false;
+      });
+  },
 };
 
 export default System;

server/endpoints/system.js

@@ -7,6 +7,7 @@ const {
   checkPythonAppAlive,
   acceptedFileTypes,
 } = require("../utils/files/documentProcessor");
+const { purgeDocument } = require("../utils/files/purgeDocument");
 const { getVectorDbClass } = require("../utils/helpers");
 const { updateENV } = require("../utils/helpers/updateENV");
 const { reqBody, makeJWT } = require("../utils/http");
@@ -94,6 +95,17 @@ function systemEndpoints(app) {
     }
   });
 
+  app.delete("/system/remove-document", async (request, response) => {
+    try {
+      const { name, meta } = reqBody(request);
+      await purgeDocument(name, meta);
+      response.sendStatus(200).end();
+    } catch (e) {
+      console.log(e.message, e);
+      response.sendStatus(500).end();
+    }
+  });
+
   app.get("/system/local-files", async (_, response) => {
     try {
       const localFiles = await viewLocalFiles();

server/utils/files/index.js

@@ -141,10 +141,51 @@ async function storeVectorResult(vectorData = [], filename = null) {
   return;
 }
 
+// Purges a file from the documents/ folder.
+async function purgeSourceDocument(filename = null) {
+  if (!filename) return;
+  console.log(`Purging document of ${filename}.`);
+
+  const filePath =
+    process.env.NODE_ENV === "development"
+      ? path.resolve(__dirname, `../../storage/documents`, filename)
+      : path.resolve(process.env.STORAGE_DIR, `documents`, filename);
+
+  if (!fs.existsSync(filePath)) {
+    console.log(`Could not located cachefile for ${filename}`, filePath);
+    return;
+  }
+
+  fs.rmSync(filePath);
+  return;
+}
+
+// Purges a vector-cache file from the vector-cache/ folder.
+async function purgeVectorCache(filename = null) {
+  if (!filename) return;
+  console.log(`Purging cached vectorized results of ${filename}.`);
+
+  const digest = uuidv5(filename, uuidv5.URL);
+  const filePath =
+    process.env.NODE_ENV === "development"
+      ? path.resolve(__dirname, `../../storage/vector-cache`, `${digest}.json`)
+      : path.resolve(process.env.STORAGE_DIR, `vector-cache`, `${digest}.json`);
+
+  if (!fs.existsSync(filePath)) {
+    console.log(`Could not located cache file for ${filename}`, filePath);
+    return;
+  }
+
+  fs.rmSync(filePath);
+  return;
+}
+
 module.exports = {
   cachedVectorInformation,
   collectDocumentData,
   viewLocalFiles,
+  purgeSourceDocument,
+  purgeVectorCache,
   storeVectorResult,
   fileData,
 };

server/utils/files/purgeDocument.js

+const { purgeVectorCache, purgeSourceDocument } = require(".");
+const { Document } = require("../../models/documents");
+const { Workspace } = require("../../models/workspace");
+
+async function purgeDocument(filename, meta) {
+  const workspaces = await Workspace.where();
+  for (const workspace of workspaces) {
+    await Document.removeDocuments(workspace, [filename]);
+  }
+  await purgeVectorCache(filename);
+  await purgeSourceDocument(filename);
+  return;
+}
+
+module.exports = {
+  purgeDocument,
+};

server/utils/vectorDbProviders/chroma/index.js

@@ -350,10 +350,10 @@ const Chroma = {
       content: `Given the following conversation, relevant context, and a follow up question, reply with an answer to the current question the user is asking. Return only your response to the question given the above information following the users instructions as needed.
     Context:
     ${contextTexts
-          .map((text, i) => {
-            return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
-          })
-          .join("")}`,
+      .map((text, i) => {
+        return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+      })
+      .join("")}`,
     };
     const memory = [prompt, ...chatHistory, { role: "user", content: input }];
     const responseText = await this.getChatCompletion(this.openai(), memory, {

server/utils/vectorDbProviders/pinecone/index.js

@@ -321,10 +321,10 @@ const Pinecone = {
       content: `Given the following conversation, relevant context, and a follow up question, reply with an answer to the current question the user is asking. Return only your response to the question given the above information following the users instructions as needed.
 Context:
 ${contextTexts
-          .map((text, i) => {
-            return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
-          })
-          .join("")}`,
+  .map((text, i) => {
+    return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+  })
+  .join("")}`,
     };
     const memory = [prompt, ...chatHistory, { role: "user", content: input }];