From 2a556c275c0171c9e6852622c8b1d3f72583a89d Mon Sep 17 00:00:00 2001
From: Timothy Carambat <rambat1010@gmail.com>
Date: Wed, 14 Jun 2023 23:12:59 -0700
Subject: [PATCH] Implement Workspace-specific setting configs + other
technical features (#58)
* 1. Define LLM Temperature as a workspace setting
2. Implement rudimentry table migration code for both new and existing repos to bring tables up to date
3. Trigger for workspace on update to update timestamp
4. Always fallback temp to 0.7
5. Extract WorkspaceModal into Tabbed content
6. Remove workspace name UNIQUE constraint (cannot be migrated :()
7. Add slug +seed when existing slug is already take
8. Seperate name from slug so display names can be changed
* remove blocking test return
---
.../CannotRemoveModal/index.jsx | 0
.../ConfirmationModal/index.jsx | 2 +-
.../{ => Documents}/Directory/index.jsx | 2 +-
.../Modals/MangeWorkspace/Documents/index.jsx | 205 ++++++++++++
.../Modals/MangeWorkspace/Settings/index.jsx | 176 ++++++++++
.../Modals/MangeWorkspace/index.jsx | 305 +++++-------------
.../src/components/Modals/NewWorkspace.jsx | 22 +-
frontend/src/components/Modals/Password.jsx | 20 +-
.../Sidebar/ActiveWorkspaces/index.jsx | 2 +-
frontend/src/models/workspace.js | 16 +
frontend/src/utils/chat/markdown.js | 2 +-
server/endpoints/workspaces.js | 22 ++
server/index.js | 7 +-
server/models/documents.js | 14 +-
server/models/vectors.js | 17 +-
server/models/workspace.js | 93 +++++-
server/models/workspaceChats.js | 32 +-
server/utils/chats/index.js | 8 +-
server/utils/database/index.js | 54 ++++
server/utils/openAi/index.js | 4 +-
.../utils/vectorDbProviders/chroma/index.js | 11 +-
server/utils/vectorDbProviders/lance/index.js | 13 +-
.../utils/vectorDbProviders/pinecone/index.js | 25 +-
23 files changed, 764 insertions(+), 288 deletions(-)
rename frontend/src/components/Modals/MangeWorkspace/{ => Documents}/CannotRemoveModal/index.jsx (100%)
rename frontend/src/components/Modals/MangeWorkspace/{ => Documents}/ConfirmationModal/index.jsx (98%)
rename frontend/src/components/Modals/MangeWorkspace/{ => Documents}/Directory/index.jsx (98%)
create mode 100644 frontend/src/components/Modals/MangeWorkspace/Documents/index.jsx
create mode 100644 frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx
create mode 100644 server/utils/database/index.js
diff --git a/frontend/src/components/Modals/MangeWorkspace/CannotRemoveModal/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/CannotRemoveModal/index.jsx
similarity index 100%
rename from frontend/src/components/Modals/MangeWorkspace/CannotRemoveModal/index.jsx
rename to frontend/src/components/Modals/MangeWorkspace/Documents/CannotRemoveModal/index.jsx
diff --git a/frontend/src/components/Modals/MangeWorkspace/ConfirmationModal/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/ConfirmationModal/index.jsx
similarity index 98%
rename from frontend/src/components/Modals/MangeWorkspace/ConfirmationModal/index.jsx
rename to frontend/src/components/Modals/MangeWorkspace/Documents/ConfirmationModal/index.jsx
index bec3beec8..b5d00a219 100644
--- a/frontend/src/components/Modals/MangeWorkspace/ConfirmationModal/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/Documents/ConfirmationModal/index.jsx
@@ -1,5 +1,5 @@
import React from "react";
-import { dollarFormat } from "../../../../utils/numbers";
+import { dollarFormat } from "../../../../../utils/numbers";
export default function ConfirmationModal({
directories,
diff --git a/frontend/src/components/Modals/MangeWorkspace/Directory/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx
similarity index 98%
rename from frontend/src/components/Modals/MangeWorkspace/Directory/index.jsx
rename to frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx
index b444f9793..b332011de 100644
--- a/frontend/src/components/Modals/MangeWorkspace/Directory/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx
@@ -7,7 +7,7 @@ import {
FolderPlus,
Zap,
} from "react-feather";
-import { nFormatter } from "../../../../utils/numbers";
+import { nFormatter } from "../../../../../utils/numbers";
export default function Directory({
files,
diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/index.jsx
new file mode 100644
index 000000000..b6ce6924a
--- /dev/null
+++ b/frontend/src/components/Modals/MangeWorkspace/Documents/index.jsx
@@ -0,0 +1,205 @@
+import React, { useState, useEffect } from "react";
+import System from "../../../../models/system";
+import Workspace from "../../../../models/workspace";
+import paths from "../../../../utils/paths";
+import { useParams } from "react-router-dom";
+import Directory from "./Directory";
+import ConfirmationModal from "./ConfirmationModal";
+import CannotRemoveModal from "./CannotRemoveModal";
+
+export default function DocumentSettings({ workspace }) {
+ const { slug } = useParams();
+ const [loading, setLoading] = useState(true);
+ const [saving, setSaving] = useState(false);
+ const [showConfirmation, setShowConfirmation] = useState(false);
+ const [directories, setDirectories] = useState(null);
+ const [originalDocuments, setOriginalDocuments] = useState([]);
+ const [selectedFiles, setSelectFiles] = useState([]);
+ const [vectordb, setVectorDB] = useState(null);
+ const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false);
+
+ useEffect(() => {
+ async function fetchKeys() {
+ const localFiles = await System.localFiles();
+ const settings = await System.keys();
+ const originalDocs = workspace.documents.map((doc) => doc.docpath) || [];
+ setDirectories(localFiles);
+ setOriginalDocuments([...originalDocs]);
+ setSelectFiles([...originalDocs]);
+ setVectorDB(settings?.VectorDB);
+ setLoading(false);
+ }
+ fetchKeys();
+ }, []);
+
+ const deleteWorkspace = async () => {
+ if (
+ !window.confirm(
+ `You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.`
+ )
+ )
+ return false;
+ await Workspace.delete(workspace.slug);
+ workspace.slug === slug
+ ? (window.location = paths.home())
+ : window.location.reload();
+ };
+
+ const docChanges = () => {
+ const changes = {
+ adds: [],
+ deletes: [],
+ };
+
+ selectedFiles.map((doc) => {
+ const inOriginal = !!originalDocuments.find((oDoc) => oDoc === doc);
+ if (!inOriginal) {
+ changes.adds.push(doc);
+ }
+ });
+
+ originalDocuments.map((doc) => {
+ const selected = !!selectedFiles.find((oDoc) => oDoc === doc);
+ if (!selected) {
+ changes.deletes.push(doc);
+ }
+ });
+
+ return changes;
+ };
+
+ const confirmChanges = (e) => {
+ e.preventDefault();
+ const changes = docChanges();
+ changes.adds.length > 0 ? setShowConfirmation(true) : updateWorkspace(e);
+ };
+
+ const updateWorkspace = async (e) => {
+ e.preventDefault();
+ setSaving(true);
+ setShowConfirmation(false);
+ const changes = docChanges();
+ await Workspace.modifyEmbeddings(workspace.slug, changes);
+ setSaving(false);
+ window.location.reload();
+ };
+
+ const isSelected = (filepath) => {
+ const isFolder = !filepath.includes("/");
+ return isFolder
+ ? selectedFiles.some((doc) => doc.includes(filepath.split("/")[0]))
+ : selectedFiles.some((doc) => doc.includes(filepath));
+ };
+
+ const isOriginalDoc = (filepath) => {
+ const isFolder = !filepath.includes("/");
+ return isFolder
+ ? originalDocuments.some((doc) => doc.includes(filepath.split("/")[0]))
+ : originalDocuments.some((doc) => doc.includes(filepath));
+ };
+
+ const toggleSelection = (filepath) => {
+ const isFolder = !filepath.includes("/");
+ const parent = isFolder ? filepath : filepath.split("/")[0];
+
+ if (isSelected(filepath)) {
+ // Certain vector DBs do not contain the ability to delete vectors
+ // so we cannot remove from these. The user will have to clear the entire workspace.
+ if (["lancedb"].includes(vectordb) && isOriginalDoc(filepath)) {
+ setShowingNoRemovalModal(true);
+ return false;
+ }
+
+ const updatedDocs = isFolder
+ ? selectedFiles.filter((doc) => !doc.includes(parent))
+ : selectedFiles.filter((doc) => !doc.includes(filepath));
+ setSelectFiles([...new Set(updatedDocs)]);
+ } else {
+ var newDocs = [];
+ var parentDirs = directories.items.find((item) => item.name === parent);
+ if (isFolder && parentDirs) {
+ const folderItems = parentDirs.items;
+ newDocs = folderItems.map((item) => parent + "/" + item.name);
+ } else {
+ newDocs = [filepath];
+ }
+
+ const combined = [...selectedFiles, ...newDocs];
+ setSelectFiles([...new Set(combined)]);
+ }
+ };
+
+ if (loading) {
+ return (
+ <>
+ <div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
+ <div className="flex flex-col gap-y-1 w-full">
+ <p className="text-slate-200 dark:text-stone-300 text-center">
+ loading workspace files
+ </p>
+ </div>
+ </div>
+ <div className="flex items-center p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600"></div>
+ </>
+ );
+ }
+
+ return (
+ <>
+ {showConfirmation && (
+ <ConfirmationModal
+ directories={directories}
+ hideConfirm={() => setShowConfirmation(false)}
+ additions={docChanges().adds}
+ updateWorkspace={updateWorkspace}
+ />
+ )}
+ {showingNoRemovalModal && (
+ <CannotRemoveModal
+ hideModal={() => setShowingNoRemovalModal(false)}
+ vectordb={vectordb}
+ />
+ )}
+ <div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
+ <div className="flex flex-col gap-y-1 w-full">
+ <div className="flex flex-col mb-2">
+ <p className="text-gray-800 dark:text-stone-200 text-base ">
+ Select folders to add or remove from workspace.
+ </p>
+ <p className="text-gray-800 dark:text-stone-400 text-xs italic">
+ {selectedFiles.length} documents in workspace selected.
+ </p>
+ </div>
+ <div className="w-full h-auto border border-slate-200 dark:border-stone-600 rounded-lg px-4 py-2">
+ {!!directories && (
+ <Directory
+ files={directories}
+ toggleSelection={toggleSelection}
+ isSelected={isSelected}
+ />
+ )}
+ </div>
+ </div>
+ </div>
+ <div className="flex items-center justify-between p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
+ <button
+ onClick={deleteWorkspace}
+ type="button"
+ className="border border-transparent text-gray-500 bg-white hover:bg-red-100 rounded-lg text-sm font-medium px-5 py-2.5 hover:text-red-900 focus:z-10 dark:bg-transparent dark:text-gray-300 dark:hover:text-white dark:hover:bg-red-600"
+ >
+ Delete Workspace
+ </button>
+ <div className="flex items-center">
+ <button
+ disabled={saving}
+ onClick={confirmChanges}
+ type="submit"
+ className="text-slate-200 bg-black-900 px-4 py-2 rounded-lg hover:bg-gray-900"
+ >
+ {saving ? "Saving..." : "Confirm Changes"}
+ </button>
+ </div>
+ </div>
+ </>
+ );
+}
diff --git a/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx
new file mode 100644
index 000000000..8e6529869
--- /dev/null
+++ b/frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx
@@ -0,0 +1,176 @@
+import React, { useState, useRef, useEffect } from "react";
+import Workspace from "../../../../models/workspace";
+import paths from "../../../../utils/paths";
+
+export default function WorkspaceSettings({ workspace }) {
+ const formEl = useRef(null);
+ const [saving, setSaving] = useState(false);
+ const [hasChanges, setHasChanges] = useState(false);
+ const [error, setError] = useState(null);
+ const [success, setSuccess] = useState(null);
+
+ useEffect(() => {
+ function setTimer() {
+ if (success !== null) {
+ setTimeout(() => {
+ setSuccess(null);
+ }, 3_000);
+ }
+
+ if (error !== null) {
+ setTimeout(() => {
+ setError(null);
+ }, 3_000);
+ }
+ }
+ setTimer();
+ }, [success, error]);
+
+ const handleUpdate = async (e) => {
+ setError(null);
+ setSuccess(null);
+ setSaving(true);
+ e.preventDefault();
+ const data = {};
+ const form = new FormData(formEl.current);
+ for (var [key, value] of form.entries()) data[key] = value;
+ const { workspace: updatedWorkspace, message } = await Workspace.update(
+ workspace.slug,
+ data
+ );
+ if (!!updatedWorkspace) {
+ setSuccess("Workspace updated!");
+ } else {
+ setError(message);
+ }
+ setSaving(false);
+ };
+
+ const deleteWorkspace = async () => {
+ if (
+ !window.confirm(
+ `You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.`
+ )
+ )
+ return false;
+ await Workspace.delete(workspace.slug);
+ workspace.slug === slug
+ ? (window.location = paths.home())
+ : window.location.reload();
+ };
+
+ return (
+ <form ref={formEl} onSubmit={handleUpdate}>
+ <div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
+ <div className="flex flex-col gap-y-1 w-full">
+ <div className="flex flex-col mb-2">
+ <p className="text-gray-800 dark:text-stone-200 text-base ">
+ Edit your workspace's settings
+ </p>
+ </div>
+
+ <div className="w-full flex flex-col gap-y-4">
+ <div>
+ <input
+ type="text"
+ disabled={true}
+ defaultValue={workspace?.slug}
+ className="bg-gray-50 border disabled:bg-gray-400 disabled:text-gray-700 disabled:border-gray-400 disabled:dark:bg-stone-800 disabled:dark:border-stone-900 disabled:dark:text-stone-600 disabled:cursor-not-allowed border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
+ required={true}
+ autoComplete="off"
+ />
+ </div>
+
+ <div>
+ <div className="flex flex-col gap-y-1 mb-4">
+ <label
+ htmlFor="name"
+ className="block text-sm font-medium text-gray-900 dark:text-white"
+ >
+ Workspace Name
+ </label>
+ <p className="text-xs text-gray-600 dark:text-stone-400">
+ This will only change the display name of your workspace.
+ </p>
+ </div>
+ <input
+ name="name"
+ type="text"
+ minLength={2}
+ maxLength={80}
+ defaultValue={workspace?.name}
+ className="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
+ placeholder="My Workspace"
+ required={true}
+ autoComplete="off"
+ onChange={() => setHasChanges(true)}
+ />
+ </div>
+
+ <div>
+ <div className="flex flex-col gap-y-1 mb-4">
+ <label
+ htmlFor="name"
+ className="block text-sm font-medium text-gray-900 dark:text-white"
+ >
+ LLM Temperature
+ </label>
+ <p className="text-xs text-gray-600 dark:text-stone-400">
+ This setting controls how "random" or dynamic your chat
+ responses will be.
+ <br />
+ The higher the number (2.0 maximum) the more random and
+ incoherent.
+ <br />
+ Recommended: 0.7
+ </p>
+ </div>
+ <input
+ name="openAiTemp"
+ type="number"
+ min={0.0}
+ max={2.0}
+ step={0.1}
+ onWheel={(e) => e.target.blur()}
+ defaultValue={workspace?.openAiTemp ?? 0.7}
+ className="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
+ placeholder="0.7"
+ required={true}
+ autoComplete="off"
+ onChange={() => setHasChanges(true)}
+ />
+ </div>
+ </div>
+
+ {error && (
+ <p className="text-red-600 dark:text-red-400 text-sm">
+ Error: {error}
+ </p>
+ )}
+ {success && (
+ <p className="text-green-600 dark:text-green-400 text-sm">
+ Success: {success}
+ </p>
+ )}
+ </div>
+ </div>
+ <div className="flex items-center justify-between p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
+ <button
+ onClick={deleteWorkspace}
+ type="button"
+ className="border border-transparent text-gray-500 bg-white hover:bg-red-100 rounded-lg text-sm font-medium px-5 py-2.5 hover:text-red-900 focus:z-10 dark:bg-transparent dark:text-gray-300 dark:hover:text-white dark:hover:bg-red-600"
+ >
+ Delete Workspace
+ </button>
+ {hasChanges && (
+ <button
+ type="submit"
+ className="text-gray-500 bg-white hover:bg-gray-100 focus:ring-4 focus:outline-none focus:ring-blue-300 rounded-lg border border-gray-200 text-sm font-medium px-5 py-2.5 hover:text-gray-900 focus:z-10 dark:bg-black dark:text-slate-200 dark:border-transparent dark:hover:text-slate-200 dark:hover:bg-gray-900 dark:focus:ring-gray-800"
+ >
+ {saving ? "Updating..." : "Update workspace"}
+ </button>
+ )}
+ </div>
+ </form>
+ );
+}
diff --git a/frontend/src/components/Modals/MangeWorkspace/index.jsx b/frontend/src/components/Modals/MangeWorkspace/index.jsx
index a97a614d1..0fe25f0b4 100644
--- a/frontend/src/components/Modals/MangeWorkspace/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/index.jsx
@@ -1,149 +1,47 @@
import React, { useState, useEffect } from "react";
-import { X } from "react-feather";
-import System from "../../../models/system";
-import Workspace from "../../../models/workspace";
-import paths from "../../../utils/paths";
+import { Archive, Sliders, X } from "react-feather";
+import DocumentSettings from "./Documents";
+import WorkspaceSettings from "./Settings";
import { useParams } from "react-router-dom";
-import Directory from "./Directory";
-import ConfirmationModal from "./ConfirmationModal";
-import CannotRemoveModal from "./CannotRemoveModal";
+import Workspace from "../../../models/workspace";
+
+const TABS = {
+ documents: DocumentSettings,
+ settings: WorkspaceSettings,
+};
const noop = () => false;
-export default function ManageWorkspace({ hideModal = noop, workspace }) {
+export default function ManageWorkspace({
+ hideModal = noop,
+ providedSlug = null,
+}) {
const { slug } = useParams();
- const [loading, setLoading] = useState(true);
- const [saving, setSaving] = useState(false);
- const [showConfirmation, setShowConfirmation] = useState(false);
- const [directories, setDirectories] = useState(null);
- const [originalDocuments, setOriginalDocuments] = useState([]);
- const [selectedFiles, setSelectFiles] = useState([]);
- const [vectordb, setVectorDB] = useState(null);
- const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false);
+ const [selectedTab, setSelectedTab] = useState("documents");
+ const [workspace, setWorkspace] = useState(null);
useEffect(() => {
- async function fetchKeys() {
- const _workspace = await Workspace.bySlug(workspace.slug);
- const localFiles = await System.localFiles();
- const settings = await System.keys();
- const originalDocs = _workspace.documents.map((doc) => doc.docpath) || [];
- setDirectories(localFiles);
- setOriginalDocuments([...originalDocs]);
- setSelectFiles([...originalDocs]);
- setVectorDB(settings?.VectorDB);
- setLoading(false);
+ async function fetchWorkspace() {
+ const workspace = await Workspace.bySlug(providedSlug ?? slug);
+ setWorkspace(workspace);
}
- fetchKeys();
- }, []);
-
- const deleteWorkspace = async () => {
- if (
- !window.confirm(
- `You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.`
- )
- )
- return false;
- await Workspace.delete(workspace.slug);
- workspace.slug === slug
- ? (window.location = paths.home())
- : window.location.reload();
- };
-
- const docChanges = () => {
- const changes = {
- adds: [],
- deletes: [],
- };
-
- selectedFiles.map((doc) => {
- const inOriginal = !!originalDocuments.find((oDoc) => oDoc === doc);
- if (!inOriginal) {
- changes.adds.push(doc);
- }
- });
-
- originalDocuments.map((doc) => {
- const selected = !!selectedFiles.find((oDoc) => oDoc === doc);
- if (!selected) {
- changes.deletes.push(doc);
- }
- });
-
- return changes;
- };
-
- const confirmChanges = (e) => {
- e.preventDefault();
- const changes = docChanges();
- changes.adds.length > 0 ? setShowConfirmation(true) : updateWorkspace(e);
- };
-
- const updateWorkspace = async (e) => {
- e.preventDefault();
- setSaving(true);
- setShowConfirmation(false);
- const changes = docChanges();
- await Workspace.modifyEmbeddings(workspace.slug, changes);
- setSaving(false);
- window.location.reload();
- };
-
- const isSelected = (filepath) => {
- const isFolder = !filepath.includes("/");
- return isFolder
- ? selectedFiles.some((doc) => doc.includes(filepath.split("/")[0]))
- : selectedFiles.some((doc) => doc.includes(filepath));
- };
-
- const isOriginalDoc = (filepath) => {
- const isFolder = !filepath.includes("/");
- return isFolder
- ? originalDocuments.some((doc) => doc.includes(filepath.split("/")[0]))
- : originalDocuments.some((doc) => doc.includes(filepath));
- };
-
- const toggleSelection = (filepath) => {
- const isFolder = !filepath.includes("/");
- const parent = isFolder ? filepath : filepath.split("/")[0];
-
- if (isSelected(filepath)) {
- // Certain vector DBs do not contain the ability to delete vectors
- // so we cannot remove from these. The user will have to clear the entire workspace.
- if (["lancedb"].includes(vectordb) && isOriginalDoc(filepath)) {
- setShowingNoRemovalModal(true);
- return false;
- }
+ fetchWorkspace();
+ }, [selectedTab, slug]);
- const updatedDocs = isFolder
- ? selectedFiles.filter((doc) => !doc.includes(parent))
- : selectedFiles.filter((doc) => !doc.includes(filepath));
- setSelectFiles([...new Set(updatedDocs)]);
- } else {
- var newDocs = [];
- var parentDirs = directories.items.find((item) => item.name === parent);
- if (isFolder && parentDirs) {
- const folderItems = parentDirs.items;
- newDocs = folderItems.map((item) => parent + "/" + item.name);
- } else {
- newDocs = [filepath];
- }
+ if (!workspace) return null;
- const combined = [...selectedFiles, ...newDocs];
- setSelectFiles([...new Set(combined)]);
- }
- };
-
- if (loading) {
- return (
- <div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
- <div
- className="flex fixed top-0 left-0 right-0 w-full h-full"
- onClick={hideModal}
- />
- <div className="relative w-full max-w-2xl max-h-full">
- <div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
- <div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
+ const Component = TABS[selectedTab || "documents"];
+ return (
+ <div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
+ <div
+ className="flex fixed top-0 left-0 right-0 w-full h-full"
+ onClick={hideModal}
+ />
+ <div className="relative w-full max-w-2xl max-h-full">
+ <div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
+ <div className="flex flex-col gap-y-1 border-b dark:border-gray-600 px-4 pt-4 ">
+ <div className="flex items-start justify-between rounded-t ">
<h3 className="text-xl font-semibold text-gray-900 dark:text-white">
- {workspace.name} Settings
+ Update "{workspace.name}"
</h3>
<button
onClick={hideModal}
@@ -154,101 +52,64 @@ export default function ManageWorkspace({ hideModal = noop, workspace }) {
<X className="text-gray-300 text-lg" />
</button>
</div>
- <div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
- <div className="flex flex-col gap-y-1 w-full">
- <p className="text-slate-200 dark:text-stone-300 text-center">
- loading workspace files
- </p>
- </div>
- </div>
- <div className="flex items-center p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600"></div>
+ <WorkspaceSettingTabs
+ selectedTab={selectedTab}
+ changeTab={setSelectedTab}
+ />
</div>
+ <Component hideModal={hideModal} workspace={workspace} />
</div>
</div>
- );
- }
+ </div>
+ );
+}
+function WorkspaceSettingTabs({ selectedTab, changeTab }) {
return (
- <>
- {showConfirmation && (
- <ConfirmationModal
- directories={directories}
- hideConfirm={() => setShowConfirmation(false)}
- additions={docChanges().adds}
- updateWorkspace={updateWorkspace}
+ <div>
+ <ul className="flex flex-wrap -mb-px text-sm gap-x-2 font-medium text-center text-gray-500 dark:text-gray-400">
+ <WorkspaceTab
+ active={selectedTab === "documents"}
+ displayName="Documents"
+ tabName="documents"
+ icon={<Archive className="h-4 w-4" />}
+ onClick={changeTab}
/>
- )}
- {showingNoRemovalModal && (
- <CannotRemoveModal
- hideModal={() => setShowingNoRemovalModal(false)}
- vectordb={vectordb}
+ <WorkspaceTab
+ active={selectedTab === "settings"}
+ displayName="Settings"
+ tabName="settings"
+ icon={<Sliders className="h-4 w-4" />}
+ onClick={changeTab}
/>
- )}
- <div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
- <div
- className="flex fixed top-0 left-0 right-0 w-full h-full"
- onClick={hideModal}
- />
- <div className="relative w-full max-w-2xl max-h-full">
- <div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
- <div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
- <h3 className="text-xl font-semibold text-gray-900 dark:text-white">
- "{workspace.name}" workspace settings
- </h3>
- <button
- onClick={hideModal}
- type="button"
- className="text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center dark:hover:bg-gray-600 dark:hover:text-white"
- data-modal-hide="staticModal"
- >
- <X className="text-gray-300 text-lg" />
- </button>
- </div>
- <div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
- <div className="flex flex-col gap-y-1 w-full">
- <div className="flex flex-col mb-2">
- <p className="text-gray-800 dark:text-stone-200 text-base ">
- Select folders to add or remove from workspace.
- </p>
- <p className="text-gray-800 dark:text-stone-400 text-xs italic">
- {selectedFiles.length} documents in workspace selected.
- </p>
- </div>
- <div className="w-full h-auto border border-slate-200 dark:border-stone-600 rounded-lg px-4 py-2">
- {!!directories && (
- <Directory
- files={directories}
- toggleSelection={toggleSelection}
- isSelected={isSelected}
- />
- )}
- </div>
- </div>
- </div>
+ </ul>
+ </div>
+ );
+}
- <div className="flex items-center justify-between p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
- <button
- onClick={deleteWorkspace}
- type="button"
- className="border border-transparent text-gray-500 bg-white hover:bg-red-100 rounded-lg text-sm font-medium px-5 py-2.5 hover:text-red-900 focus:z-10 dark:bg-transparent dark:text-gray-300 dark:hover:text-white dark:hover:bg-red-600"
- >
- Delete Workspace
- </button>
- <div className="flex items-center">
- <button
- disabled={saving}
- onClick={confirmChanges}
- type="submit"
- className="text-slate-200 bg-black-900 px-4 py-2 rounded-lg hover:bg-gray-900"
- >
- {saving ? "Saving..." : "Confirm Changes"}
- </button>
- </div>
- </div>
- </div>
- </div>
- </div>
- </>
+function WorkspaceTab({
+ active = false,
+ displayName,
+ tabName,
+ icon = "",
+ onClick,
+}) {
+ const classes = active
+ ? "text-blue-600 border-blue-600 active dark:text-blue-400 dark:border-blue-400 bg-blue-500 bg-opacity-5"
+ : "border-transparent hover:text-gray-600 hover:border-gray-300 dark:hover:text-gray-300";
+ return (
+ <li className="mr-2">
+ <button
+ disabled={active}
+ onClick={() => onClick(tabName)}
+ className={
+ "flex items-center gap-x-1 p-4 border-b-2 rounded-t-lg group " +
+ classes
+ }
+ >
+ {icon} {displayName}
+ </button>
+ </li>
);
}
diff --git a/frontend/src/components/Modals/NewWorkspace.jsx b/frontend/src/components/Modals/NewWorkspace.jsx
index 44aed7d99..81a87903b 100644
--- a/frontend/src/components/Modals/NewWorkspace.jsx
+++ b/frontend/src/components/Modals/NewWorkspace.jsx
@@ -18,33 +18,33 @@ export default function NewWorkspaceModal({ hideModal = noop }) {
};
return (
- <div class="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
+ <div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
<div
className="flex fixed top-0 left-0 right-0 w-full h-full"
onClick={hideModal}
/>
- <div class="relative w-full max-w-2xl max-h-full">
- <div class="relative bg-white rounded-lg shadow dark:bg-stone-700">
- <div class="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
- <h3 class="text-xl font-semibold text-gray-900 dark:text-white">
+ <div className="relative w-full max-w-2xl max-h-full">
+ <div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
+ <div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
+ <h3 className="text-xl font-semibold text-gray-900 dark:text-white">
Create a New Workspace
</h3>
<button
onClick={hideModal}
type="button"
- class="text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center dark:hover:bg-gray-600 dark:hover:text-white"
+ className="text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center dark:hover:bg-gray-600 dark:hover:text-white"
data-modal-hide="staticModal"
>
<X className="text-gray-300 text-lg" />
</button>
</div>
<form ref={formEl} onSubmit={handleCreate}>
- <div class="p-6 space-y-6 flex h-full w-full">
+ <div className="p-6 space-y-6 flex h-full w-full">
<div className="w-full flex flex-col gap-y-4">
<div>
<label
htmlFor="name"
- class="block mb-2 text-sm font-medium text-gray-900 dark:text-white"
+ className="block mb-2 text-sm font-medium text-gray-900 dark:text-white"
>
Workspace Name
</label>
@@ -52,7 +52,7 @@ export default function NewWorkspaceModal({ hideModal = noop }) {
name="name"
type="text"
id="name"
- class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
+ className="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
placeholder="My Workspace"
required={true}
autoComplete="off"
@@ -69,7 +69,7 @@ export default function NewWorkspaceModal({ hideModal = noop }) {
</p>
</div>
</div>
- <div class="flex w-full justify-between items-center p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
+ <div className="flex w-full justify-between items-center p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
<button
onClick={hideModal}
type="button"
@@ -79,7 +79,7 @@ export default function NewWorkspaceModal({ hideModal = noop }) {
</button>
<button
type="submit"
- class="text-gray-500 bg-white hover:bg-gray-100 focus:ring-4 focus:outline-none focus:ring-blue-300 rounded-lg border border-gray-200 text-sm font-medium px-5 py-2.5 hover:text-gray-900 focus:z-10 dark:bg-black dark:text-slate-200 dark:border-transparent dark:hover:text-slate-200 dark:hover:bg-gray-900 dark:focus:ring-gray-800"
+ className="text-gray-500 bg-white hover:bg-gray-100 focus:ring-4 focus:outline-none focus:ring-blue-300 rounded-lg border border-gray-200 text-sm font-medium px-5 py-2.5 hover:text-gray-900 focus:z-10 dark:bg-black dark:text-slate-200 dark:border-transparent dark:hover:text-slate-200 dark:hover:bg-gray-900 dark:focus:ring-gray-800"
>
Create Workspace
</button>
diff --git a/frontend/src/components/Modals/Password.jsx b/frontend/src/components/Modals/Password.jsx
index 1a0fb1966..280572feb 100644
--- a/frontend/src/components/Modals/Password.jsx
+++ b/frontend/src/components/Modals/Password.jsx
@@ -25,22 +25,22 @@ export default function PasswordModal() {
};
return (
- <div class="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
+ <div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
<div className="flex fixed top-0 left-0 right-0 w-full h-full" />
- <div class="relative w-full max-w-2xl max-h-full">
+ <div className="relative w-full max-w-2xl max-h-full">
<form ref={formEl} onSubmit={handleLogin}>
- <div class="relative bg-white rounded-lg shadow dark:bg-stone-700">
- <div class="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
- <h3 class="text-xl font-semibold text-gray-900 dark:text-white">
+ <div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
+ <div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
+ <h3 className="text-xl font-semibold text-gray-900 dark:text-white">
This workspace is password protected.
</h3>
</div>
- <div class="p-6 space-y-6 flex h-full w-full">
+ <div className="p-6 space-y-6 flex h-full w-full">
<div className="w-full flex flex-col gap-y-4">
<div>
<label
htmlFor="password"
- class="block mb-2 text-sm font-medium text-gray-900 dark:text-white"
+ className="block mb-2 text-sm font-medium text-gray-900 dark:text-white"
>
Workspace Password
</label>
@@ -48,7 +48,7 @@ export default function PasswordModal() {
name="password"
type="password"
id="password"
- class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
+ className="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
required={true}
autoComplete="off"
/>
@@ -64,11 +64,11 @@ export default function PasswordModal() {
</p>
</div>
</div>
- <div class="flex items-center justify-end p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
+ <div className="flex items-center justify-end p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
<button
disabled={loading}
type="submit"
- class="text-gray-500 bg-white hover:bg-gray-100 focus:ring-4 focus:outline-none focus:ring-blue-300 rounded-lg border border-gray-200 text-sm font-medium px-5 py-2.5 hover:text-gray-900 focus:z-10 dark:bg-gray-700 dark:text-gray-300 dark:border-gray-500 dark:hover:text-white dark:hover:bg-gray-600 dark:focus:ring-gray-600"
+ className="text-gray-500 bg-white hover:bg-gray-100 focus:ring-4 focus:outline-none focus:ring-blue-300 rounded-lg border border-gray-200 text-sm font-medium px-5 py-2.5 hover:text-gray-900 focus:z-10 dark:bg-gray-700 dark:text-gray-300 dark:border-gray-500 dark:hover:text-white dark:hover:bg-gray-600 dark:focus:ring-gray-600"
>
{loading ? "Validating..." : "Submit"}
</button>
diff --git a/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx b/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx
index 9a3cdc586..549d3f346 100644
--- a/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx
+++ b/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx
@@ -75,7 +75,7 @@ export default function ActiveWorkspaces() {
);
})}
{showing && !!selectedWs && (
- <ManageWorkspace hideModal={hideModal} workspace={selectedWs} />
+ <ManageWorkspace hideModal={hideModal} providedSlug={selectedWs.slug} />
)}
</>
);
diff --git a/frontend/src/models/workspace.js b/frontend/src/models/workspace.js
index 11f97437d..cf0a6120c 100644
--- a/frontend/src/models/workspace.js
+++ b/frontend/src/models/workspace.js
@@ -15,6 +15,22 @@ const Workspace = {
return { workspace, message };
},
+ update: async function (slug, data = {}) {
+ const { workspace, message } = await fetch(
+ `${API_BASE}/workspace/${slug}/update`,
+ {
+ method: "POST",
+ body: JSON.stringify(data),
+ headers: baseHeaders(),
+ }
+ )
+ .then((res) => res.json())
+ .catch((e) => {
+ return { workspace: null, message: e.message };
+ });
+
+ return { workspace, message };
+ },
modifyEmbeddings: async function (slug, changes = {}) {
const { workspace, message } = await fetch(
`${API_BASE}/workspace/${slug}/update-embeddings`,
diff --git a/frontend/src/utils/chat/markdown.js b/frontend/src/utils/chat/markdown.js
index ea8d794d0..7c6ccd6e1 100644
--- a/frontend/src/utils/chat/markdown.js
+++ b/frontend/src/utils/chat/markdown.js
@@ -31,6 +31,6 @@ window.copySnippet = function () {
}, 5000);
};
-export default function renderMarkdown(text) {
+export default function renderMarkdown(text = "") {
return markdown.render(text);
}
diff --git a/server/endpoints/workspaces.js b/server/endpoints/workspaces.js
index d37b1ef61..73c9e17a2 100644
--- a/server/endpoints/workspaces.js
+++ b/server/endpoints/workspaces.js
@@ -20,6 +20,28 @@ function workspaceEndpoints(app) {
}
});
+ app.post("/workspace/:slug/update", async (request, response) => {
+ try {
+ const { slug = null } = request.params;
+ const data = reqBody(request);
+ const currWorkspace = await Workspace.get(`slug = '${slug}'`);
+
+ if (!currWorkspace) {
+ response.sendStatus(400).end();
+ return;
+ }
+
+ const { workspace, message } = await Workspace.update(
+ currWorkspace.id,
+ data
+ );
+ response.status(200).json({ workspace, message });
+ } catch (e) {
+ console.log(e.message, e);
+ response.sendStatus(500).end();
+ }
+ });
+
app.post("/workspace/:slug/update-embeddings", async (request, response) => {
try {
const { slug = null } = request.params;
diff --git a/server/index.js b/server/index.js
index 5a506ae66..e96deda48 100644
--- a/server/index.js
+++ b/server/index.js
@@ -12,6 +12,7 @@ const { systemEndpoints } = require("./endpoints/system");
const { workspaceEndpoints } = require("./endpoints/workspaces");
const { chatEndpoints } = require("./endpoints/chat");
const { getVectorDbClass } = require("./utils/helpers");
+const { validateTablePragmas } = require("./utils/database");
const app = express();
const apiRouter = express.Router();
@@ -25,8 +26,9 @@ app.use(
);
apiRouter.use("/system/*", validatedRequest);
-apiRouter.use("/workspace/*", validatedRequest);
systemEndpoints(apiRouter);
+
+apiRouter.use("/workspace/*", validatedRequest);
workspaceEndpoints(apiRouter);
chatEndpoints(apiRouter);
@@ -75,7 +77,8 @@ app.all("*", function (_, response) {
});
app
- .listen(process.env.SERVER_PORT || 3001, () => {
+ .listen(process.env.SERVER_PORT || 3001, async () => {
+ await validateTablePragmas();
console.log(
`Example app listening on port ${process.env.SERVER_PORT || 3001}`
);
diff --git a/server/models/documents.js b/server/models/documents.js
index 732771195..777bd7175 100644
--- a/server/models/documents.js
+++ b/server/models/documents.js
@@ -1,6 +1,7 @@
const { fileData } = require("../utils/files");
const { v4: uuidv4 } = require("uuid");
const { getVectorDbClass } = require("../utils/helpers");
+const { checkForMigrations } = require("../utils/database");
const Document = {
tablename: "workspace_documents",
@@ -14,7 +15,15 @@ const Document = {
createdAt TEXT DEFAULT CURRENT_TIMESTAMP,
lastUpdatedAt TEXT DEFAULT CURRENT_TIMESTAMP
`,
- db: async function () {
+ migrateTable: async function () {
+ console.log(`\x1b[34m[MIGRATING]\x1b[0m Checking for Document migrations`);
+ const db = await this.db(false);
+ await checkForMigrations(this, db);
+ },
+ migrations: function () {
+ return [];
+ },
+ db: async function (tracing = true) {
const sqlite3 = require("sqlite3").verbose();
const { open } = require("sqlite");
@@ -28,7 +37,8 @@ const Document = {
await db.exec(
`CREATE TABLE IF NOT EXISTS ${this.tablename} (${this.colsInit})`
);
- db.on("trace", (sql) => console.log(sql));
+
+ if (tracing) db.on("trace", (sql) => console.log(sql));
return db;
},
forWorkspace: async function (workspaceId = null) {
diff --git a/server/models/vectors.js b/server/models/vectors.js
index 776179d02..9e1a8dd42 100644
--- a/server/models/vectors.js
+++ b/server/models/vectors.js
@@ -1,8 +1,8 @@
+const { checkForMigrations } = require("../utils/database");
const { Document } = require("./documents");
// TODO: Do we want to store entire vectorized chunks in here
// so that we can easily spin up temp-namespace clones for threading
-//
const DocumentVectors = {
tablename: "document_vectors",
colsInit: `
@@ -12,7 +12,17 @@ const DocumentVectors = {
createdAt TEXT DEFAULT CURRENT_TIMESTAMP,
lastUpdatedAt TEXT DEFAULT CURRENT_TIMESTAMP
`,
- db: async function () {
+ migrateTable: async function () {
+ console.log(
+ `\x1b[34m[MIGRATING]\x1b[0m Checking for DocumentVector migrations`
+ );
+ const db = await this.db(false);
+ await checkForMigrations(this, db);
+ },
+ migrations: function () {
+ return [];
+ },
+ db: async function (tracing = true) {
const sqlite3 = require("sqlite3").verbose();
const { open } = require("sqlite");
@@ -26,7 +36,8 @@ const DocumentVectors = {
await db.exec(
`CREATE TABLE IF NOT EXISTS ${this.tablename} (${this.colsInit})`
);
- db.on("trace", (sql) => console.log(sql));
+
+ if (tracing) db.on("trace", (sql) => console.log(sql));
return db;
},
bulkInsert: async function (vectorRecords = []) {
diff --git a/server/models/workspace.js b/server/models/workspace.js
index 6472f4779..09c3712ce 100644
--- a/server/models/workspace.js
+++ b/server/models/workspace.js
@@ -1,17 +1,50 @@
const slugify = require("slugify");
const { Document } = require("./documents");
+const { checkForMigrations } = require("../utils/database");
const Workspace = {
tablename: "workspaces",
+ writable: [
+ // Used for generic updates so we can validate keys in request body
+ "name",
+ "slug",
+ "vectorTag",
+ "openAiTemp",
+ "lastUpdatedAt",
+ ],
colsInit: `
id INTEGER PRIMARY KEY AUTOINCREMENT,
- name TEXT NOT NULL UNIQUE,
+ name TEXT NOT NULL,
slug TEXT NOT NULL UNIQUE,
vectorTag TEXT DEFAULT NULL,
createdAt TEXT DEFAULT CURRENT_TIMESTAMP,
+ openAiTemp REAL DEFAULT NULL,
lastUpdatedAt TEXT DEFAULT CURRENT_TIMESTAMP
`,
- db: async function () {
+ migrateTable: async function () {
+ console.log(`\x1b[34m[MIGRATING]\x1b[0m Checking for Workspace migrations`);
+ const db = await this.db(false);
+ await checkForMigrations(this, db);
+ },
+ migrations: function () {
+ return [
+ {
+ colName: "openAiTemp",
+ execCmd: `ALTER TABLE ${this.tablename} ADD COLUMN openAiTemp REAL DEFAULT NULL`,
+ doif: false,
+ },
+ {
+ colName: "id",
+ execCmd: `CREATE TRIGGER IF NOT EXISTS Trg_LastUpdated AFTER UPDATE ON ${this.tablename}
+ FOR EACH ROW
+ BEGIN
+ UPDATE ${this.tablename} SET lastUpdatedAt = CURRENT_TIMESTAMP WHERE id = old.id;
+ END`,
+ doif: true,
+ },
+ ];
+ },
+ db: async function (tracing = true) {
const sqlite3 = require("sqlite3").verbose();
const { open } = require("sqlite");
@@ -25,17 +58,25 @@ const Workspace = {
await db.exec(
`CREATE TABLE IF NOT EXISTS ${this.tablename} (${this.colsInit})`
);
- db.on("trace", (sql) => console.log(sql));
+
+ if (tracing) db.on("trace", (sql) => console.log(sql));
return db;
},
new: async function (name = null) {
if (!name) return { result: null, message: "name cannot be null" };
+ var slug = slugify(name, { lower: true });
+
+ const existingBySlug = await this.get(`slug = '${slug}'`);
+ if (existingBySlug !== null) {
+ const slugSeed = Math.floor(10000000 + Math.random() * 90000000);
+ slug = slugify(`${name}-${slugSeed}`, { lower: true });
+ }
const db = await this.db();
const { id, success, message } = await db
.run(`INSERT INTO ${this.tablename} (name, slug) VALUES (?, ?)`, [
name,
- slugify(name, { lower: true }),
+ slug,
])
.then((res) => {
return { id: res.lastID, success: true, message: null };
@@ -43,19 +84,57 @@ const Workspace = {
.catch((error) => {
return { id: null, success: false, message: error.message };
});
- if (!success) return { workspace: null, message };
+
+ if (!success) {
+ db.close();
+ return { workspace: null, message };
+ }
const workspace = await db.get(
`SELECT * FROM ${this.tablename} WHERE id = ${id}`
);
+ db.close();
+
return { workspace, message: null };
},
+ update: async function (id = null, data = {}) {
+ if (!id) throw new Error("No workspace id provided for update");
+
+ const validKeys = Object.keys(data).filter((key) =>
+ this.writable.includes(key)
+ );
+ const values = Object.values(data);
+ if (validKeys.length === 0 || validKeys.length !== values.length)
+ return { workspace: { id }, message: "No valid fields to update!" };
+
+ const template = `UPDATE ${this.tablename} SET ${validKeys.map((key) => {
+ return `${key}=?`;
+ })} WHERE id = ?`;
+ const db = await this.db();
+ const { success, message } = await db
+ .run(template, [...values, id])
+ .then(() => {
+ return { success: true, message: null };
+ })
+ .catch((error) => {
+ return { success: false, message: error.message };
+ });
+
+ db.close();
+ if (!success) {
+ return { workspace: null, message };
+ }
+
+ const updatedWorkspace = await this.get(`id = ${id}`);
+ return { workspace: updatedWorkspace, message: null };
+ },
get: async function (clause = "") {
const db = await this.db();
const result = await db
.get(`SELECT * FROM ${this.tablename} WHERE ${clause}`)
.then((res) => res || null);
if (!result) return null;
+ db.close();
const documents = await Document.forWorkspace(result.id);
return { ...result, documents };
@@ -63,6 +142,8 @@ const Workspace = {
delete: async function (clause = "") {
const db = await this.db();
await db.get(`DELETE FROM ${this.tablename} WHERE ${clause}`);
+ db.close();
+
return true;
},
where: async function (clause = "", limit = null) {
@@ -72,6 +153,8 @@ const Workspace = {
!!limit ? `LIMIT ${limit}` : ""
}`
);
+ db.close();
+
return results;
},
};
diff --git a/server/models/workspaceChats.js b/server/models/workspaceChats.js
index 2ded62b42..3b90cc614 100644
--- a/server/models/workspaceChats.js
+++ b/server/models/workspaceChats.js
@@ -1,3 +1,5 @@
+const { checkForMigrations } = require("../utils/database");
+
const WorkspaceChats = {
tablename: "workspace_chats",
colsInit: `
@@ -9,7 +11,17 @@ const WorkspaceChats = {
createdAt TEXT DEFAULT CURRENT_TIMESTAMP,
lastUpdatedAt TEXT DEFAULT CURRENT_TIMESTAMP
`,
- db: async function () {
+ migrateTable: async function () {
+ console.log(
+ `\x1b[34m[MIGRATING]\x1b[0m Checking for WorkspaceChats migrations`
+ );
+ const db = await this.db(false);
+ await checkForMigrations(this, db);
+ },
+ migrations: function () {
+ return [];
+ },
+ db: async function (tracing = true) {
const sqlite3 = require("sqlite3").verbose();
const { open } = require("sqlite");
@@ -23,7 +35,8 @@ const WorkspaceChats = {
await db.exec(
`CREATE TABLE IF NOT EXISTS ${this.tablename} (${this.colsInit})`
);
- db.on("trace", (sql) => console.log(sql));
+
+ if (tracing) db.on("trace", (sql) => console.log(sql));
return db;
},
new: async function ({ workspaceId, prompt, response = {} }) {
@@ -39,11 +52,16 @@ const WorkspaceChats = {
.catch((error) => {
return { id: null, success: false, message: error.message };
});
- if (!success) return { chat: null, message };
+ if (!success) {
+ db.close();
+ return { chat: null, message };
+ }
const chat = await db.get(
`SELECT * FROM ${this.tablename} WHERE id = ${id}`
);
+ db.close();
+
return { chat, message: null };
},
forWorkspace: async function (workspaceId = null) {
@@ -61,6 +79,8 @@ const WorkspaceChats = {
`UPDATE ${this.tablename} SET include = false WHERE workspaceId = ?`,
[workspaceId]
);
+ db.close();
+
return;
},
get: async function (clause = "") {
@@ -68,12 +88,16 @@ const WorkspaceChats = {
const result = await db
.get(`SELECT * FROM ${this.tablename} WHERE ${clause}`)
.then((res) => res || null);
+ db.close();
+
if (!result) return null;
return result;
},
delete: async function (clause = "") {
const db = await this.db();
await db.get(`DELETE FROM ${this.tablename} WHERE ${clause}`);
+ db.close();
+
return true;
},
where: async function (clause = "", limit = null, order = null) {
@@ -83,6 +107,8 @@ const WorkspaceChats = {
!!limit ? `LIMIT ${limit}` : ""
} ${!!order ? order : ""}`
);
+ db.close();
+
return results;
},
};
diff --git a/server/utils/chats/index.js b/server/utils/chats/index.js
index 7459e37e2..9be40b695 100644
--- a/server/utils/chats/index.js
+++ b/server/utils/chats/index.js
@@ -87,7 +87,7 @@ async function chatWithWorkspace(workspace, message, chatMode = "query") {
if (!hasVectorizedSpace) {
const rawHistory = await WorkspaceChats.forWorkspace(workspace.id);
const chatHistory = convertToPromptHistory(rawHistory);
- const response = await openai.sendChat(chatHistory, message);
+ const response = await openai.sendChat(chatHistory, message, workspace);
const data = { text: response, sources: [], type: "chat" };
await WorkspaceChats.new({
@@ -108,7 +108,11 @@ async function chatWithWorkspace(workspace, message, chatMode = "query") {
response,
sources,
message: error,
- } = await VectorDb[chatMode]({ namespace: workspace.slug, input: message });
+ } = await VectorDb[chatMode]({
+ namespace: workspace.slug,
+ input: message,
+ workspace,
+ });
if (!response) {
return {
id: uuid,
diff --git a/server/utils/database/index.js b/server/utils/database/index.js
new file mode 100644
index 000000000..f240d63b1
--- /dev/null
+++ b/server/utils/database/index.js
@@ -0,0 +1,54 @@
+function checkColumnTemplate(tablename = null, column = null) {
+ if (!tablename || !column)
+ throw new Error(`Migration Error`, { tablename, column });
+ return `SELECT COUNT(*) AS _exists FROM pragma_table_info('${tablename}') WHERE name='${column}'`;
+}
+
+// Note (tcarambat): Since there is no good way to track migrations in Node/SQLite we use this simple system
+// Each model has a `migrations` method that will return an array like...
+// { colName: 'stringColName', execCmd: `SQL Command to run when`, doif: boolean },
+// colName = name of column
+// execCmd = Command to run when doif matches the state of the DB
+// doif = condition to match that determines if execCmd will run.
+// eg: Table workspace has slug column.
+// execCmd: ALTER TABLE DROP COLUMN slug;
+// doif: true
+// => Will drop the slug column if the workspace table has a column named 'slug' otherwise nothing happens.
+// If you are adding a new table column if needs to exist in the Models `colsInit` and as a migration.
+// So both new and existing DBs will get the column when code is pulled in.
+
+async function checkForMigrations(model, db) {
+ if (model.migrations().length === 0) return;
+ const toMigrate = [];
+ for (const { colName, execCmd, doif } of model.migrations()) {
+ const { _exists } = await db.get(
+ checkColumnTemplate(model.tablename, colName)
+ );
+ const colExists = _exists !== 0;
+ if (colExists !== doif) continue;
+
+ toMigrate.push(execCmd);
+ }
+
+ if (toMigrate.length === 0) return;
+
+ console.log(`Running ${toMigrate.length} migrations`, toMigrate);
+ await db.exec(toMigrate.join(";\n"));
+ return;
+}
+
+async function validateTablePragmas() {
+ const { Workspace } = require("../../models/workspace");
+ const { Document } = require("../../models/documents");
+ const { DocumentVectors } = require("../../models/vectors");
+ const { WorkspaceChats } = require("../../models/workspaceChats");
+ await Workspace.migrateTable();
+ await Document.migrateTable();
+ await DocumentVectors.migrateTable();
+ await WorkspaceChats.migrateTable();
+}
+
+module.exports = {
+ checkForMigrations,
+ validateTablePragmas,
+};
diff --git a/server/utils/openAi/index.js b/server/utils/openAi/index.js
index 72742fb74..00ec1326b 100644
--- a/server/utils/openAi/index.js
+++ b/server/utils/openAi/index.js
@@ -40,7 +40,7 @@ class OpenAi {
return { safe: false, reasons };
}
- async sendChat(chatHistory = [], prompt) {
+ async sendChat(chatHistory = [], prompt, workspace = {}) {
const model = process.env.OPEN_MODEL_PREF;
if (!this.isValidChatModel(model))
throw new Error(
@@ -50,7 +50,7 @@ class OpenAi {
const textResponse = await this.openai
.createChatCompletion({
model,
- temperature: 0.7,
+ temperature: Number(workspace?.openAiTemp ?? 0.7),
n: 1,
messages: [
{ role: "system", content: "" },
diff --git a/server/utils/vectorDbProviders/chroma/index.js b/server/utils/vectorDbProviders/chroma/index.js
index fd08c1e35..bd1c6058e 100644
--- a/server/utils/vectorDbProviders/chroma/index.js
+++ b/server/utils/vectorDbProviders/chroma/index.js
@@ -56,12 +56,12 @@ const Chroma = {
const openai = new OpenAIApi(config);
return openai;
},
- llm: function () {
+ llm: function ({ temperature = 0.7 }) {
const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo";
return new OpenAI({
openAIApiKey: process.env.OPEN_AI_KEY,
- temperature: 0.7,
modelName: model,
+ temperature,
});
},
embedChunk: async function (openai, textChunk) {
@@ -253,7 +253,7 @@ const Chroma = {
return true;
},
query: async function (reqBody = {}) {
- const { namespace = null, input } = reqBody;
+ const { namespace = null, input, workspace = {} } = reqBody;
if (!namespace || !input) throw new Error("Invalid request body");
const { client } = await this.connect();
@@ -269,7 +269,10 @@ const Chroma = {
this.embedder(),
{ collectionName: namespace, url: process.env.CHROMA_ENDPOINT }
);
- const model = this.llm();
+ const model = this.llm({
+ temperature: workspace?.openAiTemp,
+ });
+
const chain = VectorDBQAChain.fromLLM(model, vectorStore, {
k: 5,
returnSourceDocuments: true,
diff --git a/server/utils/vectorDbProviders/lance/index.js b/server/utils/vectorDbProviders/lance/index.js
index f4cc18985..d6aced156 100644
--- a/server/utils/vectorDbProviders/lance/index.js
+++ b/server/utils/vectorDbProviders/lance/index.js
@@ -69,11 +69,16 @@ const LanceDb = {
? data[0].embedding
: null;
},
- getChatCompletion: async function (openai, messages = []) {
+ getChatCompletion: async function (
+ openai,
+ messages = [],
+ { temperature = 0.7 }
+ ) {
const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo";
const { data } = await openai.createChatCompletion({
model,
messages,
+ temperature,
});
if (!data.hasOwnProperty("choices")) return null;
@@ -213,7 +218,7 @@ const LanceDb = {
}
},
query: async function (reqBody = {}) {
- const { namespace = null, input } = reqBody;
+ const { namespace = null, input, workspace = {} } = reqBody;
if (!namespace || !input) throw new Error("Invalid request body");
const { client } = await this.connect();
@@ -242,7 +247,9 @@ const LanceDb = {
},
{ role: "user", content: input },
];
- const responseText = await this.getChatCompletion(this.openai(), messages);
+ const responseText = await this.getChatCompletion(this.openai(), messages, {
+ temperature: workspace?.openAiTemp,
+ });
return {
response: responseText,
diff --git a/server/utils/vectorDbProviders/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js
index 9167b790e..2dcf2b526 100644
--- a/server/utils/vectorDbProviders/pinecone/index.js
+++ b/server/utils/vectorDbProviders/pinecone/index.js
@@ -1,7 +1,6 @@
const { PineconeClient } = require("@pinecone-database/pinecone");
const { PineconeStore } = require("langchain/vectorstores/pinecone");
const { OpenAI } = require("langchain/llms/openai");
-const { ChatOpenAI } = require("langchain/chat_models/openai");
const { VectorDBQAChain, LLMChain } = require("langchain/chains");
const { OpenAIEmbeddings } = require("langchain/embeddings/openai");
const { VectorStoreRetrieverMemory } = require("langchain/memory");
@@ -50,20 +49,12 @@ const Pinecone = {
? data[0].embedding
: null;
},
- llm: function () {
+ llm: function ({ temperature = 0.7 }) {
const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo";
return new OpenAI({
openAIApiKey: process.env.OPEN_AI_KEY,
- temperature: 0.7,
- modelName: model,
- });
- },
- chatLLM: function () {
- const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo";
- return new ChatOpenAI({
- openAIApiKey: process.env.OPEN_AI_KEY,
- temperature: 0.7,
modelName: model,
+ temperature,
});
},
totalIndicies: async function () {
@@ -233,7 +224,7 @@ const Pinecone = {
};
},
query: async function (reqBody = {}) {
- const { namespace = null, input } = reqBody;
+ const { namespace = null, input, workspace = {} } = reqBody;
if (!namespace || !input) throw new Error("Invalid request body");
const { pineconeIndex } = await this.connect();
@@ -250,7 +241,9 @@ const Pinecone = {
namespace,
});
- const model = this.llm();
+ const model = this.llm({
+ temperature: workspace?.openAiTemp,
+ });
const chain = VectorDBQAChain.fromLLM(model, vectorStore, {
k: 5,
returnSourceDocuments: true,
@@ -265,7 +258,7 @@ const Pinecone = {
// This implementation of chat also expands the memory of the chat itself
// and adds more tokens to the PineconeDB instance namespace
chat: async function (reqBody = {}) {
- const { namespace = null, input } = reqBody;
+ const { namespace = null, input, workspace = {} } = reqBody;
if (!namespace || !input) throw new Error("Invalid request body");
const { pineconeIndex } = await this.connect();
@@ -284,7 +277,9 @@ const Pinecone = {
memoryKey: "history",
});
- const model = this.llm();
+ const model = this.llm({
+ temperature: workspace?.openAiTemp,
+ });
const prompt =
PromptTemplate.fromTemplate(`The following is a friendly conversation between a human and an AI. The AI is very casual and talkative and responds with a friendly tone. If the AI does not know the answer to a question, it truthfully says it does not know.
Relevant pieces of previous conversation:
--
GitLab