diff --git a/frontend/src/components/Modals/ManageWorkspace.jsx b/frontend/src/components/Modals/ManageWorkspace.jsx
deleted file mode 100644
index 4d64bb41726ec2e406664a2af2fa41e0fc476eb6..0000000000000000000000000000000000000000
--- a/frontend/src/components/Modals/ManageWorkspace.jsx
+++ /dev/null
@@ -1,537 +0,0 @@
-import React, { useState, useEffect } from "react";
-import {
- FileMinus,
- FilePlus,
- Folder,
- FolderMinus,
- FolderPlus,
- X,
- Zap,
-} from "react-feather";
-import System from "../../models/system";
-import Workspace from "../../models/workspace";
-import { nFormatter } from "../../utils/numbers";
-import { dollarFormat } from "../../utils/numbers";
-import paths from "../../utils/paths";
-import { useParams } from "react-router-dom";
-import { titleCase } from "text-case";
-
-const noop = () => false;
-export default function ManageWorkspace({ hideModal = noop, workspace }) {
- const { slug } = useParams();
- const [loading, setLoading] = useState(true);
- const [saving, setSaving] = useState(false);
- const [showConfirmation, setShowConfirmation] = useState(false);
- const [directories, setDirectories] = useState(null);
- const [originalDocuments, setOriginalDocuments] = useState([]);
- const [selectedFiles, setSelectFiles] = useState([]);
- const [vectordb, setVectorDB] = useState(null);
- const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false);
-
- useEffect(() => {
- async function fetchKeys() {
- const _workspace = await Workspace.bySlug(workspace.slug);
- const localFiles = await System.localFiles();
- const settings = await System.keys();
- const originalDocs = _workspace.documents.map((doc) => doc.docpath) || [];
- setDirectories(localFiles);
- setOriginalDocuments([...originalDocs]);
- setSelectFiles([...originalDocs]);
- setVectorDB(settings?.VectorDB);
- setLoading(false);
- }
- fetchKeys();
- }, []);
-
- const deleteWorkspace = async () => {
- if (
- !window.confirm(
- `You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.`
- )
- )
- return false;
- await Workspace.delete(workspace.slug);
- workspace.slug === slug
- ? (window.location = paths.home())
- : window.location.reload();
- };
-
- const docChanges = () => {
- const changes = {
- adds: [],
- deletes: [],
- };
-
- selectedFiles.map((doc) => {
- const inOriginal = !!originalDocuments.find((oDoc) => oDoc === doc);
- if (!inOriginal) {
- changes.adds.push(doc);
- }
- });
-
- originalDocuments.map((doc) => {
- const selected = !!selectedFiles.find((oDoc) => oDoc === doc);
- if (!selected) {
- changes.deletes.push(doc);
- }
- });
-
- return changes;
- };
-
- const confirmChanges = (e) => {
- e.preventDefault();
- const changes = docChanges();
- changes.adds.length > 0 ? setShowConfirmation(true) : updateWorkspace(e);
- };
-
- const updateWorkspace = async (e) => {
- e.preventDefault();
- setSaving(true);
- setShowConfirmation(false);
- const changes = docChanges();
- await Workspace.modifyEmbeddings(workspace.slug, changes);
- setSaving(false);
- window.location.reload();
- };
-
- const isSelected = (filepath) => {
- const isFolder = !filepath.includes("/");
- return isFolder
- ? selectedFiles.some((doc) => doc.includes(filepath.split("/")[0]))
- : selectedFiles.some((doc) => doc.includes(filepath));
- };
-
- const isOriginalDoc = (filepath) => {
- const isFolder = !filepath.includes("/");
- return isFolder
- ? originalDocuments.some((doc) => doc.includes(filepath.split("/")[0]))
- : originalDocuments.some((doc) => doc.includes(filepath));
- };
-
- const toggleSelection = (filepath) => {
- const isFolder = !filepath.includes("/");
- const parent = isFolder ? filepath : filepath.split("/")[0];
-
- if (isSelected(filepath)) {
- // Certain vector DBs do not contain the ability to delete vectors
- // so we cannot remove from these. The user will have to clear the entire workspace.
- if (["lancedb"].includes(vectordb) && isOriginalDoc(filepath)) {
- setShowingNoRemovalModal(true);
- return false;
- }
-
- const updatedDocs = isFolder
- ? selectedFiles.filter((doc) => !doc.includes(parent))
- : selectedFiles.filter((doc) => !doc.includes(filepath));
- setSelectFiles([...new Set(updatedDocs)]);
- } else {
- var newDocs = [];
- if (isFolder) {
- const folderItems = directories.items.find(
- (item) => item.name === parent
- ).items;
- newDocs = folderItems.map((item) => parent + "/" + item.name);
- } else {
- newDocs = [filepath];
- }
-
- const combined = [...selectedFiles, ...newDocs];
- setSelectFiles([...new Set(combined)]);
- }
- };
-
- if (loading) {
- return (
- <div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
- <div
- className="flex fixed top-0 left-0 right-0 w-full h-full"
- onClick={hideModal}
- />
- <div className="relative w-full max-w-2xl max-h-full">
- <div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
- <div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
- <h3 className="text-xl font-semibold text-gray-900 dark:text-white">
- {workspace.name} Settings
- </h3>
- <button
- onClick={hideModal}
- type="button"
- className="text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center dark:hover:bg-gray-600 dark:hover:text-white"
- data-modal-hide="staticModal"
- >
- <X className="text-gray-300 text-lg" />
- </button>
- </div>
- <div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
- <div className="flex flex-col gap-y-1 w-full">
- <p className="text-slate-200 dark:text-stone-300 text-center">
- loading workspace files
- </p>
- </div>
- </div>
- <div className="flex items-center p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600"></div>
- </div>
- </div>
- </div>
- );
- }
-
- return (
- <>
- {showConfirmation && (
- <ConfirmationModal
- directories={directories}
- hideConfirm={() => setShowConfirmation(false)}
- additions={docChanges().adds}
- updateWorkspace={updateWorkspace}
- />
- )}
- {showingNoRemovalModal && (
- <CannotRemoveModal
- hideModal={() => setShowingNoRemovalModal(false)}
- vectordb={vectordb}
- />
- )}
- <div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
- <div
- className="flex fixed top-0 left-0 right-0 w-full h-full"
- onClick={hideModal}
- />
- <div className="relative w-full max-w-2xl max-h-full">
- <div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
- <div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
- <h3 className="text-xl font-semibold text-gray-900 dark:text-white">
- "{workspace.name}" workspace settings
- </h3>
- <button
- onClick={hideModal}
- type="button"
- className="text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center dark:hover:bg-gray-600 dark:hover:text-white"
- data-modal-hide="staticModal"
- >
- <X className="text-gray-300 text-lg" />
- </button>
- </div>
- <div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
- <div className="flex flex-col gap-y-1 w-full">
- <div className="flex flex-col mb-2">
- <p className="text-gray-800 dark:text-stone-200 text-base ">
- Select folders to add or remove from workspace.
- </p>
- <p className="text-gray-800 dark:text-stone-400 text-xs italic">
- {selectedFiles.length} documents in workspace selected.
- </p>
- </div>
- <div className="w-full h-auto border border-slate-200 dark:border-stone-600 rounded-lg px-4 py-2">
- {!!directories && (
- <Directory
- files={directories}
- toggleSelection={toggleSelection}
- isSelected={isSelected}
- />
- )}
- </div>
- </div>
- </div>
-
- <div className="flex items-center justify-between p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
- <button
- onClick={deleteWorkspace}
- type="button"
- className="border border-transparent text-gray-500 bg-white hover:bg-red-100 rounded-lg text-sm font-medium px-5 py-2.5 hover:text-red-900 focus:z-10 dark:bg-transparent dark:text-gray-300 dark:hover:text-white dark:hover:bg-red-600"
- >
- Delete Workspace
- </button>
- <div className="flex items-center">
- <button
- disabled={saving}
- onClick={confirmChanges}
- type="submit"
- className="text-slate-200 bg-black-900 px-4 py-2 rounded-lg hover:bg-gray-900"
- >
- {saving ? "Saving..." : "Confirm Changes"}
- </button>
- </div>
- </div>
- </div>
- </div>
- </div>
- </>
- );
-}
-
-function Directory({
- files,
- parent = null,
- nested = 0,
- toggleSelection,
- isSelected,
-}) {
- const [isExpanded, toggleExpanded] = useState(false);
- const [showDetails, toggleDetails] = useState(false);
- const [showZap, setShowZap] = useState(false);
-
- if (files.type === "folder") {
- return (
- <div style={{ marginLeft: nested }} className="mb-2">
- <div
- className={`flex items-center hover:bg-gray-100 gap-x-2 text-gray-800 dark:text-stone-200 dark:hover:bg-stone-800 px-2 rounded-lg`}
- >
- {files.items.some((files) => files.type === "folder") ? (
- <Folder className="w-6 h-6" />
- ) : (
- <button onClick={() => toggleSelection(files.name)}>
- {isSelected(files.name) ? (
- <FolderMinus className="w-6 h-6 stroke-red-800 hover:fill-red-500" />
- ) : (
- <FolderPlus className="w-6 h-6 hover:stroke-green-800 hover:fill-green-500" />
- )}
- </button>
- )}
-
- <div
- className="flex gap-x-2 items-center cursor-pointer w-full"
- onClick={() => toggleExpanded(!isExpanded)}
- >
- <h2 className="text-2xl">{files.name}</h2>
- {files.items.some((files) => files.type === "folder") ? (
- <p className="text-xs italic">{files.items.length} folders</p>
- ) : (
- <p className="text-xs italic">
- {files.items.length} documents |{" "}
- {nFormatter(
- files.items.reduce((a, b) => a + b.token_count_estimate, 0)
- )}{" "}
- tokens
- </p>
- )}
- </div>
- </div>
- {isExpanded &&
- files.items.map((item) => (
- <Directory
- key={item.name}
- parent={files.name}
- files={item}
- nested={nested + 20}
- toggleSelection={toggleSelection}
- isSelected={isSelected}
- />
- ))}
- </div>
- );
- }
-
- const { name, type: _type, ...meta } = files;
- return (
- <div className="ml-[20px] my-2">
- <div className="flex items-center">
- {meta?.cached && (
- <button
- type="button"
- onClick={() => setShowZap(true)}
- className="rounded-full p-1 hover:bg-stone-500 hover:bg-opacity-75"
- >
- <Zap className="h-4 w-4 stroke-yellow-500 fill-yellow-400" />
- </button>
- )}
- {showZap && (
- <dialog
- open={true}
- style={{ zIndex: 100 }}
- className="fixed top-0 flex bg-black bg-opacity-50 w-[100vw] h-full items-center justify-center "
- >
- <div className="w-fit px-10 py-4 w-[25%] rounded-lg bg-white shadow dark:bg-stone-700 text-black dark:text-slate-200">
- <div className="flex flex-col w-full">
- <p className="font-semibold text-xl flex items-center gap-x-1 justify-left">
- What does{" "}
- <Zap className="h-4 w-4 stroke-yellow-500 fill-yellow-400" />{" "}
- mean?
- </p>
- <p className="text-base mt-4">
- This symbol indicates that you have embed this document before
- and will not have to pay to re-embed this document.
- </p>
- <div className="flex w-full justify-center items-center mt-4">
- <button
- onClick={() => setShowZap(false)}
- className="border border-gray-800 text-gray-800 hover:bg-gray-100 px-4 py-1 rounded-lg dark:text-slate-200 dark:border-slate-200 dark:hover:bg-stone-900"
- >
- Close
- </button>
- </div>
- </div>
- </div>
- </dialog>
- )}
-
- <div
- className={`flex items-center gap-x-2 text-gray-800 dark:text-stone-200 hover:bg-gray-100 dark:hover:bg-stone-800 px-2 rounded-lg`}
- >
- <button onClick={() => toggleSelection(`${parent}/${name}`)}>
- {isSelected(`${parent}/${name}`) ? (
- <FileMinus className="w-6 h-6 stroke-red-800 hover:fill-red-500" />
- ) : (
- <FilePlus className="w-6 h-6 hover:stroke-green-800 hover:fill-green-500" />
- )}
- </button>
- <div
- className="w-full items-center flex cursor-pointer"
- onClick={() => toggleDetails(!showDetails)}
- >
- <h3 className="text-sm">{name}</h3>
- <br />
- </div>
- </div>
- </div>
- {showDetails && (
- <div className="ml-[20px] flex flex-col gap-y-1 my-1 p-2 rounded-md bg-slate-200 font-mono text-sm overflow-x-scroll">
- {Object.entries(meta).map(([key, value]) => {
- if (key === "cached") return null;
- return (
- <p className="whitespace-pre">
- {key}: {value}
- </p>
- );
- })}
- </div>
- )}
- </div>
- );
-}
-
-function ConfirmationModal({
- directories,
- hideConfirm,
- additions,
- updateWorkspace,
-}) {
- function estimateCosts() {
- const cachedTokens = additions.map((filepath) => {
- const [parent, filename] = filepath.split("/");
- const details = directories.items
- .find((folder) => folder.name === parent)
- .items.find((file) => file.name === filename);
-
- const { token_count_estimate = 0, cached = false } = details;
- return cached ? token_count_estimate : 0;
- });
- const tokenEstimates = additions.map((filepath) => {
- const [parent, filename] = filepath.split("/");
- const details = directories.items
- .find((folder) => folder.name === parent)
- .items.find((file) => file.name === filename);
-
- const { token_count_estimate = 0 } = details;
- return token_count_estimate;
- });
-
- const totalTokens = tokenEstimates.reduce((a, b) => a + b, 0);
- const cachedTotal = cachedTokens.reduce((a, b) => a + b, 0);
- const dollarValue = 0.0004 * ((totalTokens - cachedTotal) / 1_000);
-
- return {
- dollarValue,
- dollarText:
- dollarValue < 0.01 ? "< $0.01" : `about ${dollarFormat(dollarValue)}`,
- };
- }
-
- const { dollarValue, dollarText } = estimateCosts();
- return (
- <dialog
- open={true}
- style={{ zIndex: 100 }}
- className="fixed top-0 flex bg-black bg-opacity-50 w-[100vw] h-full items-center justify-center "
- >
- <div className="w-fit px-10 p-4 min-w-1/2 rounded-lg bg-white shadow dark:bg-stone-700 text-black dark:text-slate-200">
- <div className="flex flex-col w-full">
- <p className="font-semibold">
- Are you sure you want to embed these documents?
- </p>
-
- <div className="flex flex-col gap-y-1">
- {dollarValue <= 0 ? (
- <p className="text-base mt-4">
- You will be embedding {additions.length} new documents into this
- workspace.
- <br />
- This will not incur any costs for OpenAI credits.
- </p>
- ) : (
- <p className="text-base mt-4">
- You will be embedding {additions.length} new documents into this
- workspace. <br />
- This will cost {dollarText} in OpenAI credits.
- </p>
- )}
- </div>
-
- <div className="flex w-full justify-between items-center mt-4">
- <button
- onClick={hideConfirm}
- className="text-gray-800 hover:bg-gray-100 px-4 py-1 rounded-lg dark:text-slate-200 dark:hover:bg-stone-900"
- >
- Cancel
- </button>
- <button
- onClick={updateWorkspace}
- className="border border-gray-800 text-gray-800 hover:bg-gray-100 px-4 py-1 rounded-lg dark:text-slate-200 dark:border-slate-200 dark:hover:bg-stone-900"
- >
- Continue
- </button>
- </div>
- </div>
- </div>
- </dialog>
- );
-}
-
-function CannotRemoveModal({ hideModal, vectordb }) {
- return (
- <dialog
- open={true}
- style={{ zIndex: 100 }}
- className="fixed top-0 flex bg-black bg-opacity-50 w-[100vw] h-full items-center justify-center "
- >
- <div className="px-10 p-4 w-1/2 rounded-lg bg-white shadow dark:bg-stone-700 text-black dark:text-slate-200">
- <div className="flex flex-col w-full">
- <p className="text-lg font-semibold text-red-500">
- You cannot remove this document!
- </p>
-
- <div className="flex flex-col gap-y-1">
- <p className="text-base mt-4">
- {titleCase(vectordb)} does not support atomic removal of
- documents.
- <br />
- Unfortunately, you will have to delete the entire workspace to
- remove this document from being referenced.
- </p>
- </div>
- <div className="flex w-full justify-center items-center mt-4">
- <button
- onClick={hideModal}
- className="text-gray-800 hover:bg-gray-100 px-4 py-1 rounded-lg dark:text-slate-200 dark:hover:bg-stone-900"
- >
- I Understand
- </button>
- </div>
- </div>
- </div>
- </dialog>
- );
-}
-
-export function useManageWorkspaceModal() {
- const [showing, setShowing] = useState(false);
- const showModal = () => {
- setShowing(true);
- };
- const hideModal = () => {
- setShowing(false);
- };
-
- return { showing, showModal, hideModal };
-}
diff --git a/frontend/src/components/Modals/MangeWorkspace/CannotRemoveModal/index.jsx b/frontend/src/components/Modals/MangeWorkspace/CannotRemoveModal/index.jsx
new file mode 100644
index 0000000000000000000000000000000000000000..a66442fd2181408013a4bf945c6f3ab30eb2c99c
--- /dev/null
+++ b/frontend/src/components/Modals/MangeWorkspace/CannotRemoveModal/index.jsx
@@ -0,0 +1,37 @@
+import React from "react";
+import { titleCase } from "text-case";
+
+export default function CannotRemoveModal({
+ hideModal,
+ vectordb,
+}) {
+ return (
+ <dialog
+ open={true}
+ style={{ zIndex: 100 }}
+ className="fixed top-0 flex bg-black bg-opacity-50 w-[100vw] h-full items-center justify-center "
+ >
+ <div className="px-10 p-4 w-1/2 rounded-lg bg-white shadow dark:bg-stone-700 text-black dark:text-slate-200">
+ <div className="flex flex-col w-full">
+ <p className="text-lg font-semibold text-red-500">
+ You cannot remove this document!
+ </p>
+
+ <div className="flex flex-col gap-y-1">
+ <p className="text-base mt-4">
+ {titleCase(vectordb)} does not support atomic removal of documents.<br />Unfortunately, you will have to delete the entire workspace to remove this document from being referenced.
+ </p>
+ </div>
+ <div className="flex w-full justify-center items-center mt-4">
+ <button
+ onClick={hideModal}
+ className="text-gray-800 hover:bg-gray-100 px-4 py-1 rounded-lg dark:text-slate-200 dark:hover:bg-stone-900"
+ >
+ I Understand
+ </button>
+ </div>
+ </div>
+ </div>
+ </dialog>
+ );
+}
diff --git a/frontend/src/components/Modals/MangeWorkspace/ConfirmationModal/index.jsx b/frontend/src/components/Modals/MangeWorkspace/ConfirmationModal/index.jsx
new file mode 100644
index 0000000000000000000000000000000000000000..6dce423549df07185227625b0af97e172fb5530d
--- /dev/null
+++ b/frontend/src/components/Modals/MangeWorkspace/ConfirmationModal/index.jsx
@@ -0,0 +1,89 @@
+import React from "react";
+import { dollarFormat } from "../../../../utils/numbers";
+
+export default function ConfirmationModal({
+ directories,
+ hideConfirm,
+ additions,
+ updateWorkspace,
+}) {
+ function estimateCosts() {
+ const cachedTokens = additions.map((filepath) => {
+ const [parent, filename] = filepath.split("/");
+ const details = directories.items
+ .find((folder) => folder.name === parent)
+ .items.find((file) => file.name === filename);
+
+ const { token_count_estimate = 0, cached = false } = details;
+ return cached ? token_count_estimate : 0;
+ });
+ const tokenEstimates = additions.map((filepath) => {
+ const [parent, filename] = filepath.split("/");
+ const details = directories.items
+ .find((folder) => folder.name === parent)
+ .items.find((file) => file.name === filename);
+
+ const { token_count_estimate = 0 } = details;
+ return token_count_estimate;
+ });
+
+ const totalTokens = tokenEstimates.reduce((a, b) => a + b, 0);
+ const cachedTotal = cachedTokens.reduce((a, b) => a + b, 0);
+ const dollarValue = 0.0004 * ((totalTokens - cachedTotal) / 1_000);
+
+ return {
+ dollarValue,
+ dollarText:
+ dollarValue < 0.01 ? "< $0.01" : `about ${dollarFormat(dollarValue)}`,
+ };
+ }
+
+ const { dollarValue, dollarText } = estimateCosts();
+ return (
+ <dialog
+ open={true}
+ style={{ zIndex: 100 }}
+ className="fixed top-0 flex bg-black bg-opacity-50 w-[100vw] h-full items-center justify-center "
+ >
+ <div className="w-fit px-10 p-4 min-w-1/2 rounded-lg bg-white shadow dark:bg-stone-700 text-black dark:text-slate-200">
+ <div className="flex flex-col w-full">
+ <p className="font-semibold">
+ Are you sure you want to embed these documents?
+ </p>
+
+ <div className="flex flex-col gap-y-1">
+ {dollarValue <= 0 ? (
+ <p className="text-base mt-4">
+ You will be embedding {additions.length} new documents into this
+ workspace.
+ <br />
+ This will not incur any costs for OpenAI credits.
+ </p>
+ ) : (
+ <p className="text-base mt-4">
+ You will be embedding {additions.length} new documents into this
+ workspace. <br />
+ This will cost {dollarText} in OpenAI credits.
+ </p>
+ )}
+ </div>
+
+ <div className="flex w-full justify-between items-center mt-4">
+ <button
+ onClick={hideConfirm}
+ className="text-gray-800 hover:bg-gray-100 px-4 py-1 rounded-lg dark:text-slate-200 dark:hover:bg-stone-900"
+ >
+ Cancel
+ </button>
+ <button
+ onClick={updateWorkspace}
+ className="border border-gray-800 text-gray-800 hover:bg-gray-100 px-4 py-1 rounded-lg dark:text-slate-200 dark:border-slate-200 dark:hover:bg-stone-900"
+ >
+ Continue
+ </button>
+ </div>
+ </div>
+ </div>
+ </dialog>
+ );
+}
\ No newline at end of file
diff --git a/frontend/src/components/Modals/MangeWorkspace/Directory/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Directory/index.jsx
new file mode 100644
index 0000000000000000000000000000000000000000..b2eb412508f1308ca2b264e3507091eae380a3f2
--- /dev/null
+++ b/frontend/src/components/Modals/MangeWorkspace/Directory/index.jsx
@@ -0,0 +1,150 @@
+import React, { useState } from "react";
+import {
+ FileMinus,
+ FilePlus,
+ Folder,
+ FolderMinus,
+ FolderPlus,
+ Zap,
+} from "react-feather";
+import { nFormatter } from "../../../../utils/numbers";
+
+export default function Directory({
+ files,
+ parent = null,
+ nested = 0,
+ toggleSelection,
+ isSelected,
+}) {
+ const [isExpanded, toggleExpanded] = useState(false);
+ const [showDetails, toggleDetails] = useState(false);
+ const [showZap, setShowZap] = useState(false);
+
+ if (files.type === "folder") {
+ return (
+ <div style={{ marginLeft: nested }} className="mb-2">
+ <div
+ className={`flex items-center hover:bg-gray-100 gap-x-2 text-gray-800 dark:text-stone-200 dark:hover:bg-stone-800 px-2 rounded-lg`}
+ >
+ {files.items.some((files) => files.type === "folder") ? (
+ <Folder className="w-6 h-6" />
+ ) : (
+ <button onClick={() => toggleSelection(files.name)}>
+ {isSelected(files.name) ? (
+ <FolderMinus className="w-6 h-6 stroke-red-800 hover:fill-red-500" />
+ ) : (
+ <FolderPlus className="w-6 h-6 hover:stroke-green-800 hover:fill-green-500" />
+ )}
+ </button>
+ )}
+
+ <div
+ className="flex gap-x-2 items-center cursor-pointer w-full"
+ onClick={() => toggleExpanded(!isExpanded)}
+ >
+ <h2 className="text-2xl">{files.name}</h2>
+ {files.items.some((files) => files.type === "folder") ? (
+ <p className="text-xs italic">{files.items.length} folders</p>
+ ) : (
+ <p className="text-xs italic">
+ {files.items.length} documents |{" "}
+ {nFormatter(
+ files.items.reduce((a, b) => a + b.token_count_estimate, 0)
+ )}{" "}
+ tokens
+ </p>
+ )}
+ </div>
+ </div>
+ {isExpanded &&
+ files.items.map((item) => (
+ <Directory
+ key={item.name}
+ parent={files.name}
+ files={item}
+ nested={nested + 20}
+ toggleSelection={toggleSelection}
+ isSelected={isSelected}
+ />
+ ))}
+ </div>
+ );
+ }
+
+ const { name, type: _type, ...meta } = files;
+ return (
+ <div className="ml-[20px] my-2">
+ <div className="flex items-center">
+ {meta?.cached && (
+ <button
+ type="button"
+ onClick={() => setShowZap(true)}
+ className="rounded-full p-1 hover:bg-stone-500 hover:bg-opacity-75"
+ >
+ <Zap className="h-4 w-4 stroke-yellow-500 fill-yellow-400" />
+ </button>
+ )}
+ {showZap && (
+ <dialog
+ open={true}
+ style={{ zIndex: 100 }}
+ className="fixed top-0 flex bg-black bg-opacity-50 w-[100vw] h-full items-center justify-center "
+ >
+ <div className="w-fit px-10 py-4 w-[25%] rounded-lg bg-white shadow dark:bg-stone-700 text-black dark:text-slate-200">
+ <div className="flex flex-col w-full">
+ <p className="font-semibold text-xl flex items-center gap-x-1 justify-left">
+ What does{" "}
+ <Zap className="h-4 w-4 stroke-yellow-500 fill-yellow-400" />{" "}
+ mean?
+ </p>
+ <p className="text-base mt-4">
+ This symbol indicates that you have embed this document before
+ and will not have to pay to re-embed this document.
+ </p>
+ <div className="flex w-full justify-center items-center mt-4">
+ <button
+ onClick={() => setShowZap(false)}
+ className="border border-gray-800 text-gray-800 hover:bg-gray-100 px-4 py-1 rounded-lg dark:text-slate-200 dark:border-slate-200 dark:hover:bg-stone-900"
+ >
+ Close
+ </button>
+ </div>
+ </div>
+ </div>
+ </dialog>
+ )}
+
+ <div
+ className={`flex items-center gap-x-2 text-gray-800 dark:text-stone-200 hover:bg-gray-100 dark:hover:bg-stone-800 px-2 rounded-lg`}
+ >
+ <button onClick={() => toggleSelection(`${parent}/${name}`)}>
+ {isSelected(`${parent}/${name}`) ? (
+ <FileMinus className="w-6 h-6 stroke-red-800 hover:fill-red-500" />
+ ) : (
+ <FilePlus className="w-6 h-6 hover:stroke-green-800 hover:fill-green-500" />
+ )}
+ </button>
+ <div
+ className="w-full items-center flex cursor-pointer"
+ onClick={() => toggleDetails(!showDetails)}
+ >
+ <h3 className="text-sm">{name}</h3>
+ <br />
+ </div>
+ </div>
+ </div>
+ {showDetails && (
+ <div className="ml-[20px] flex flex-col gap-y-1 my-1 p-2 rounded-md bg-slate-200 font-mono text-sm overflow-x-scroll">
+ {Object.entries(meta).map(([key, value]) => {
+ if (key === "cached") return null;
+ return (
+ <p className="whitespace-pre">
+ {key}: {value}
+ </p>
+ );
+ })}
+ </div>
+ )}
+ </div>
+ );
+}
\ No newline at end of file
diff --git a/frontend/src/components/Modals/MangeWorkspace/index.jsx b/frontend/src/components/Modals/MangeWorkspace/index.jsx
new file mode 100644
index 0000000000000000000000000000000000000000..4c3cde12a39bd6df2f24ec6b593cc74715673fc9
--- /dev/null
+++ b/frontend/src/components/Modals/MangeWorkspace/index.jsx
@@ -0,0 +1,266 @@
+import React, { useState, useEffect } from "react";
+import { X } from 'react-feather';
+import System from "../../../models/system";
+import Workspace from "../../../models/workspace";
+import paths from "../../../utils/paths";
+import { useParams } from "react-router-dom";
+import Directory from "./Directory";
+import ConfirmationModal from "./ConfirmationModal";
+import CannotRemoveModal from "./CannotRemoveModal";
+
+const noop = () => false;
+export default function ManageWorkspace({ hideModal = noop, workspace }) {
+ const { slug } = useParams();
+ const [loading, setLoading] = useState(true);
+ const [saving, setSaving] = useState(false);
+ const [showConfirmation, setShowConfirmation] = useState(false);
+ const [directories, setDirectories] = useState(null);
+ const [originalDocuments, setOriginalDocuments] = useState([]);
+ const [selectedFiles, setSelectFiles] = useState([]);
+ const [vectordb, setVectorDB] = useState(null);
+ const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false)
+
+ useEffect(() => {
+ async function fetchKeys() {
+ const _workspace = await Workspace.bySlug(workspace.slug);
+ const localFiles = await System.localFiles();
+ const settings = await System.keys();
+ const originalDocs = _workspace.documents.map((doc) => doc.docpath) || [];
+ setDirectories(localFiles);
+ setOriginalDocuments([...originalDocs]);
+ setSelectFiles([...originalDocs]);
+ setVectorDB(settings?.VectorDB)
+ setLoading(false);
+ }
+ fetchKeys();
+ }, []);
+
+ const deleteWorkspace = async () => {
+ if (
+ !window.confirm(
+ `You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.`
+ )
+ )
+ return false;
+ await Workspace.delete(workspace.slug);
+ workspace.slug === slug
+ ? (window.location = paths.home())
+ : window.location.reload();
+ };
+
+ const docChanges = () => {
+ const changes = {
+ adds: [],
+ deletes: [],
+ };
+
+ selectedFiles.map((doc) => {
+ const inOriginal = !!originalDocuments.find((oDoc) => oDoc === doc);
+ if (!inOriginal) {
+ changes.adds.push(doc);
+ }
+ });
+
+ originalDocuments.map((doc) => {
+ const selected = !!selectedFiles.find((oDoc) => oDoc === doc);
+ if (!selected) {
+ changes.deletes.push(doc);
+ }
+ });
+
+ return changes;
+ };
+
+ const confirmChanges = (e) => {
+ e.preventDefault();
+ const changes = docChanges();
+ changes.adds.length > 0 ? setShowConfirmation(true) : updateWorkspace(e);
+ };
+
+ const updateWorkspace = async (e) => {
+ e.preventDefault();
+ setSaving(true);
+ setShowConfirmation(false);
+ const changes = docChanges();
+ await Workspace.modifyEmbeddings(workspace.slug, changes);
+ setSaving(false);
+ window.location.reload();
+ };
+
+ const isSelected = (filepath) => {
+ const isFolder = !filepath.includes("/");
+ return isFolder
+ ? selectedFiles.some((doc) => doc.includes(filepath.split("/")[0]))
+ : selectedFiles.some((doc) => doc.includes(filepath));
+ };
+
+ const isOriginalDoc = (filepath) => {
+ const isFolder = !filepath.includes("/");
+ return isFolder
+ ? originalDocuments.some((doc) => doc.includes(filepath.split("/")[0]))
+ : originalDocuments.some((doc) => doc.includes(filepath));
+ }
+
+ const toggleSelection = (filepath) => {
+ const isFolder = !filepath.includes("/");
+ const parent = isFolder ? filepath : filepath.split("/")[0];
+
+ if (isSelected(filepath)) {
+ // Certain vector DBs do not contain the ability to delete vectors
+ // so we cannot remove from these. The user will have to clear the entire workspace.
+ if (['lancedb'].includes(vectordb) && isOriginalDoc(filepath)) {
+ setShowingNoRemovalModal(true);
+ return false;
+ }
+
+ const updatedDocs = isFolder
+ ? selectedFiles.filter((doc) => !doc.includes(parent))
+ : selectedFiles.filter((doc) => !doc.includes(filepath));
+ setSelectFiles([...new Set(updatedDocs)]);
+ } else {
+ var newDocs = [];
+ if (isFolder) {
+ const folderItems = directories.items.find(
+ (item) => item.name === parent
+ ).items;
+ newDocs = folderItems.map((item) => parent + "/" + item.name);
+ } else {
+ newDocs = [filepath];
+ }
+
+ const combined = [...selectedFiles, ...newDocs];
+ setSelectFiles([...new Set(combined)]);
+ }
+ };
+
+ if (loading) {
+ return (
+ <div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
+ <div
+ className="flex fixed top-0 left-0 right-0 w-full h-full"
+ onClick={hideModal}
+ />
+ <div className="relative w-full max-w-2xl max-h-full">
+ <div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
+ <div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
+ <h3 className="text-xl font-semibold text-gray-900 dark:text-white">
+ {workspace.name} Settings
+ </h3>
+ <button
+ onClick={hideModal}
+ type="button"
+ className="text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center dark:hover:bg-gray-600 dark:hover:text-white"
+ data-modal-hide="staticModal"
+ >
+ <X className="text-gray-300 text-lg" />
+ </button>
+ </div>
+ <div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
+ <div className="flex flex-col gap-y-1 w-full">
+ <p className="text-slate-200 dark:text-stone-300 text-center">
+ loading workspace files
+ </p>
+ </div>
+ </div>
+ <div className="flex items-center p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600"></div>
+ </div>
+ </div>
+ </div>
+ );
+ }
+
+ return (
+ <>
+ {showConfirmation && (
+ <ConfirmationModal
+ directories={directories}
+ hideConfirm={() => setShowConfirmation(false)}
+ additions={docChanges().adds}
+ updateWorkspace={updateWorkspace}
+ />
+ )}
+ {showingNoRemovalModal && (
+ <CannotRemoveModal
+ hideModal={() => setShowingNoRemovalModal(false)}
+ vectordb={vectordb}
+ />
+ )}
+ <div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
+ <div
+ className="flex fixed top-0 left-0 right-0 w-full h-full"
+ onClick={hideModal}
+ />
+ <div className="relative w-full max-w-2xl max-h-full">
+ <div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
+ <div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
+ <h3 className="text-xl font-semibold text-gray-900 dark:text-white">
+ "{workspace.name}" workspace settings
+ </h3>
+ <button
+ onClick={hideModal}
+ type="button"
+ className="text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center dark:hover:bg-gray-600 dark:hover:text-white"
+ data-modal-hide="staticModal"
+ >
+ <X className="text-gray-300 text-lg" />
+ </button>
+ </div>
+ <div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
+ <div className="flex flex-col gap-y-1 w-full">
+ <div className="flex flex-col mb-2">
+ <p className="text-gray-800 dark:text-stone-200 text-base ">
+ Select folders to add or remove from workspace.
+ </p>
+ <p className="text-gray-800 dark:text-stone-400 text-xs italic">
+ {selectedFiles.length} documents in workspace selected.
+ </p>
+ </div>
+ <div className="w-full h-auto border border-slate-200 dark:border-stone-600 rounded-lg px-4 py-2">
+ {!!directories && (
+ <Directory
+ files={directories}
+ toggleSelection={toggleSelection}
+ isSelected={isSelected}
+ />
+ )}
+ </div>
+ </div>
+ </div>
+
+ <div className="flex items-center justify-between p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
+ <button
+ onClick={deleteWorkspace}
+ type="button"
+ className="border border-transparent text-gray-500 bg-white hover:bg-red-100 rounded-lg text-sm font-medium px-5 py-2.5 hover:text-red-900 focus:z-10 dark:bg-transparent dark:text-gray-300 dark:hover:text-white dark:hover:bg-red-600"
+ >
+ Delete Workspace
+ </button>
+ <div className="flex items-center">
+ <button
+ disabled={saving}
+ onClick={confirmChanges}
+ type="submit"
+ className="text-slate-200 bg-black-900 px-4 py-2 rounded-lg hover:bg-gray-900"
+ >
+ {saving ? "Saving..." : "Confirm Changes"}
+ </button>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </>
+ );
+}
+
+export function useManageWorkspaceModal() {
+ const [showing, setShowing] = useState(false);
+ const showModal = () => {
+ setShowing(true);
+ };
+ const hideModal = () => {
+ setShowing(false);
+ };
+
+ return { showing, showModal, hideModal };
+}
diff --git a/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx b/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx
index af9ddefb8c39d3911c8ec7eeee9d4ee0d3d4596b..67da8086b67f417035dc8645a8ffaae60b6bad60 100644
--- a/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx
+++ b/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx
@@ -5,7 +5,7 @@ import "react-loading-skeleton/dist/skeleton.css";
import Workspace from "../../../models/workspace";
import ManageWorkspace, {
useManageWorkspaceModal,
-} from "../../Modals/ManageWorkspace";
+} from "../../Modals/MangeWorkspace";
import paths from "../../../utils/paths";
import { useParams } from "react-router-dom";
@@ -51,11 +51,10 @@ export default function ActiveWorkspaces() {
>
<a
href={isActive ? null : paths.workspace.chat(workspace.slug)}
- className={`flex flex-grow w-[75%] h-[36px] gap-x-2 py-[5px] px-4 border border-slate-400 rounded-lg text-slate-800 dark:text-slate-200 justify-start items-center ${
- isActive
+ className={`flex flex-grow w-[75%] h-[36px] gap-x-2 py-[5px] px-4 border border-slate-400 rounded-lg text-slate-800 dark:text-slate-200 justify-start items-center ${isActive
? "bg-gray-100 dark:bg-stone-600"
: "hover:bg-slate-100 dark:hover:bg-stone-900 "
- }`}
+ }`}
>
<Book className="h-4 w-4" />
<p className="text-slate-800 dark:text-slate-200 text-xs leading-loose font-semibold">
diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js
index 08a84cc881c676c1c4361608938843bcaaaa9057..6c9ea2cbf18e092a640627049777384367877be2 100644
--- a/server/utils/helpers/index.js
+++ b/server/utils/helpers/index.js
@@ -1,7 +1,7 @@
function getVectorDbClass() {
- const { Pinecone } = require("../pinecone");
- const { Chroma } = require("../chroma");
- const { LanceDb } = require("../lancedb");
+ const { Pinecone } = require("../vectorDbProviders/pinecone");
+ const { Chroma } = require("../vectorDbProviders/chroma");
+ const { LanceDb } = require("../vectorDbProviders/lance");
const vectorSelection = process.env.VECTOR_DB || "pinecone";
switch (vectorSelection) {
diff --git a/server/utils/chroma/CHROMA_SETUP.md b/server/utils/vectorDbProviders/chroma/CHROMA_SETUP.md
similarity index 100%
rename from server/utils/chroma/CHROMA_SETUP.md
rename to server/utils/vectorDbProviders/chroma/CHROMA_SETUP.md
diff --git a/server/utils/chroma/index.js b/server/utils/vectorDbProviders/chroma/index.js
similarity index 98%
rename from server/utils/chroma/index.js
rename to server/utils/vectorDbProviders/chroma/index.js
index f6fee744a163c478125e0b9bd07d011d7b400def..9584df47f9705b4a014a2b1f1fe5589a27219fba 100644
--- a/server/utils/chroma/index.js
+++ b/server/utils/vectorDbProviders/chroma/index.js
@@ -5,10 +5,10 @@ const { ChatOpenAI } = require("langchain/chat_models/openai");
const { VectorDBQAChain } = require("langchain/chains");
const { OpenAIEmbeddings } = require("langchain/embeddings/openai");
const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
-const { storeVectorResult, cachedVectorInformation } = require("../files");
+const { storeVectorResult, cachedVectorInformation } = require("../../files");
const { Configuration, OpenAIApi } = require("openai");
const { v4: uuidv4 } = require("uuid");
-const { toChunks, curateSources } = require("../helpers");
+const { toChunks, curateSources } = require("../../helpers");
const Chroma = {
name: "Chroma",
@@ -112,7 +112,7 @@ const Chroma = {
documentData = {},
fullFilePath = null
) {
- const { DocumentVectors } = require("../../models/vectors");
+ const { DocumentVectors } = require("../../../models/vectors");
try {
const { pageContent, docId, ...metadata } = documentData;
if (!pageContent || pageContent.length == 0) return false;
@@ -235,7 +235,7 @@ const Chroma = {
}
},
deleteDocumentFromNamespace: async function (namespace, docId) {
- const { DocumentVectors } = require("../../models/vectors");
+ const { DocumentVectors } = require("../../../models/vectors");
const { client } = await this.connect();
if (!(await this.namespaceExists(client, namespace))) return;
const collection = await client.getCollection({
diff --git a/server/utils/lancedb/index.js b/server/utils/vectorDbProviders/lance/index.js
similarity index 98%
rename from server/utils/lancedb/index.js
rename to server/utils/vectorDbProviders/lance/index.js
index f2851d3292af2106c3701500e856c34e6db189d1..8617a423f090027bdfb9534b231351a24300de4f 100644
--- a/server/utils/lancedb/index.js
+++ b/server/utils/vectorDbProviders/lance/index.js
@@ -1,8 +1,8 @@
const lancedb = require("vectordb");
-const { toChunks } = require("../helpers");
+const { toChunks } = require("../../helpers");
const { OpenAIEmbeddings } = require("langchain/embeddings/openai");
const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
-const { storeVectorResult, cachedVectorInformation } = require("../files");
+const { storeVectorResult, cachedVectorInformation } = require("../../files");
const { Configuration, OpenAIApi } = require("openai");
const { v4: uuidv4 } = require("uuid");
@@ -126,7 +126,7 @@ const LanceDb = {
documentData = {},
fullFilePath = null
) {
- const { DocumentVectors } = require("../../models/vectors");
+ const { DocumentVectors } = require("../../../models/vectors");
try {
const { pageContent, docId, ...metadata } = documentData;
if (!pageContent || pageContent.length == 0) return false;
diff --git a/server/utils/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js
similarity index 98%
rename from server/utils/pinecone/index.js
rename to server/utils/vectorDbProviders/pinecone/index.js
index e57c34a8a3db30699cfbfd432e60b142b70ab26f..9167b790ecef6933e8ce440b5d64bc33d421de8e 100644
--- a/server/utils/pinecone/index.js
+++ b/server/utils/vectorDbProviders/pinecone/index.js
@@ -7,10 +7,10 @@ const { OpenAIEmbeddings } = require("langchain/embeddings/openai");
const { VectorStoreRetrieverMemory } = require("langchain/memory");
const { PromptTemplate } = require("langchain/prompts");
const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
-const { storeVectorResult, cachedVectorInformation } = require("../files");
+const { storeVectorResult, cachedVectorInformation } = require("../../files");
const { Configuration, OpenAIApi } = require("openai");
const { v4: uuidv4 } = require("uuid");
-const { toChunks, curateSources } = require("../helpers");
+const { toChunks, curateSources } = require("../../helpers");
const Pinecone = {
name: "Pinecone",
@@ -98,7 +98,7 @@ const Pinecone = {
documentData = {},
fullFilePath = null
) {
- const { DocumentVectors } = require("../../models/vectors");
+ const { DocumentVectors } = require("../../../models/vectors");
try {
const { pageContent, docId, ...metadata } = documentData;
if (!pageContent || pageContent.length == 0) return false;
@@ -192,7 +192,7 @@ const Pinecone = {
}
},
deleteDocumentFromNamespace: async function (namespace, docId) {
- const { DocumentVectors } = require("../../models/vectors");
+ const { DocumentVectors } = require("../../../models/vectors");
const { pineconeIndex } = await this.connect();
if (!(await this.namespaceExists(pineconeIndex, namespace))) return;