From 997482ef8f9f0ebc8c8557d7860315ecc51dd328 Mon Sep 17 00:00:00 2001
From: Sean Hatfield <seanhatfield5@gmail.com>
Date: Wed, 8 Nov 2023 17:36:54 -0800
Subject: [PATCH] added JSONL export to workspace chats (#345)
* added JSONL export to workspace chats
* change permissions for workspace chat settings
* change permissions for workspace chat settings
* Show error for correct limit on fine-tune
Change sidebar position and permission
Remove check for MUM
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
---
frontend/src/App.jsx | 11 +-
.../src/components/SettingsSidebar/index.jsx | 25 ++--
frontend/src/models/admin.js | 25 ----
frontend/src/models/system.js | 34 +++++
.../Chats/ChatRow/index.jsx | 4 +-
.../Chats/index.jsx | 38 ++++-
frontend/src/utils/paths.js | 3 +
server/endpoints/admin.js | 50 -------
server/endpoints/system.js | 130 ++++++++++++++++++
server/models/workspaceChats.js | 2 +-
10 files changed, 224 insertions(+), 98 deletions(-)
rename frontend/src/pages/{Admin => GeneralSettings}/Chats/ChatRow/index.jsx (97%)
rename frontend/src/pages/{Admin => GeneralSettings}/Chats/index.jsx (75%)
diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx
index 98ffaf039..042fde70f 100644
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@@ -12,8 +12,8 @@ const WorkspaceChat = lazy(() => import("./pages/WorkspaceChat"));
const AdminUsers = lazy(() => import("./pages/Admin/Users"));
const AdminInvites = lazy(() => import("./pages/Admin/Invitations"));
const AdminWorkspaces = lazy(() => import("./pages/Admin/Workspaces"));
-const AdminChats = lazy(() => import("./pages/Admin/Chats"));
const AdminSystem = lazy(() => import("./pages/Admin/System"));
+const GeneralChats = lazy(() => import("./pages/GeneralSettings/Chats"));
const GeneralAppearance = lazy(() =>
import("./pages/GeneralSettings/Appearance")
);
@@ -77,6 +77,10 @@ export default function App() {
path="/general/api-keys"
element={<PrivateRoute Component={GeneralApiKeys} />}
/>
+ <Route
+ path="/general/workspace-chats"
+ element={<PrivateRoute Component={GeneralChats} />}
+ />
{/* Admin Routes */}
<Route
@@ -95,11 +99,6 @@ export default function App() {
path="/admin/workspaces"
element={<AdminRoute Component={AdminWorkspaces} />}
/>
- <Route
- path="/admin/workspace-chats"
- element={<AdminRoute Component={AdminChats} />}
- />
-
{/* Onboarding Flow */}
<Route path="/onboarding" element={<OnboardingFlow />} />
</Routes>
diff --git a/frontend/src/components/SettingsSidebar/index.jsx b/frontend/src/components/SettingsSidebar/index.jsx
index c50ae878e..9b0c25531 100644
--- a/frontend/src/components/SettingsSidebar/index.jsx
+++ b/frontend/src/components/SettingsSidebar/index.jsx
@@ -91,7 +91,7 @@ export default function SettingsSidebar() {
icon={<BookOpen className="h-5 w-5 flex-shrink-0" />}
/>
<Option
- href={paths.admin.chats()}
+ href={paths.general.chats()}
btnText="Workspace Chat"
icon={
<ChatCenteredText className="h-5 w-5 flex-shrink-0" />
@@ -131,6 +131,15 @@ export default function SettingsSidebar() {
btnText="Export or Import"
icon={<DownloadSimple className="h-5 w-5 flex-shrink-0" />}
/>
+ {!user && (
+ <Option
+ href={paths.general.chats()}
+ btnText="Chat History"
+ icon={
+ <ChatCenteredText className="h-5 w-5 flex-shrink-0" />
+ }
+ />
+ )}
<Option
href={paths.general.security()}
btnText="Security"
@@ -292,17 +301,17 @@ export function SidebarMobileHeader() {
btnText="Workspaces"
icon={<BookOpen className="h-5 w-5 flex-shrink-0" />}
/>
- <Option
- href={paths.admin.chats()}
- btnText="Workspace Chat"
- icon={
- <ChatCenteredText className="h-5 w-5 flex-shrink-0" />
- }
- />
</>
)}
{/* General Settings */}
+ <Option
+ href={paths.general.chats()}
+ btnText="Workspace Chat"
+ icon={
+ <ChatCenteredText className="h-5 w-5 flex-shrink-0" />
+ }
+ />
<Option
href={paths.general.appearance()}
btnText="Appearance"
diff --git a/frontend/src/models/admin.js b/frontend/src/models/admin.js
index a2a8746fb..ade884d58 100644
--- a/frontend/src/models/admin.js
+++ b/frontend/src/models/admin.js
@@ -139,31 +139,6 @@ const Admin = {
});
},
- // Workspace Chats Mgmt
- chats: async (offset = 0) => {
- return await fetch(`${API_BASE}/admin/workspace-chats`, {
- method: "POST",
- headers: baseHeaders(),
- body: JSON.stringify({ offset }),
- })
- .then((res) => res.json())
- .catch((e) => {
- console.error(e);
- return [];
- });
- },
- deleteChat: async (chatId) => {
- return await fetch(`${API_BASE}/admin/workspace-chats/${chatId}`, {
- method: "DELETE",
- headers: baseHeaders(),
- })
- .then((res) => res.json())
- .catch((e) => {
- console.error(e);
- return { success: false, error: e.message };
- });
- },
-
// System Preferences
systemPreferences: async () => {
return await fetch(`${API_BASE}/admin/system-preferences`, {
diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js
index a90bddb23..43f7a9182 100644
--- a/frontend/src/models/system.js
+++ b/frontend/src/models/system.js
@@ -339,6 +339,40 @@ const System = {
return { models: [], error: e.message };
});
},
+ chats: async (offset = 0) => {
+ return await fetch(`${API_BASE}/system/workspace-chats`, {
+ method: "POST",
+ headers: baseHeaders(),
+ body: JSON.stringify({ offset }),
+ })
+ .then((res) => res.json())
+ .catch((e) => {
+ console.error(e);
+ return [];
+ });
+ },
+ deleteChat: async (chatId) => {
+ return await fetch(`${API_BASE}/system/workspace-chats/${chatId}`, {
+ method: "DELETE",
+ headers: baseHeaders(),
+ })
+ .then((res) => res.json())
+ .catch((e) => {
+ console.error(e);
+ return { success: false, error: e.message };
+ });
+ },
+ exportChats: async () => {
+ return await fetch(`${API_BASE}/system/export-chats`, {
+ method: "GET",
+ headers: baseHeaders(),
+ })
+ .then((res) => res.text())
+ .catch((e) => {
+ console.error(e);
+ return null;
+ });
+ },
};
export default System;
diff --git a/frontend/src/pages/Admin/Chats/ChatRow/index.jsx b/frontend/src/pages/GeneralSettings/Chats/ChatRow/index.jsx
similarity index 97%
rename from frontend/src/pages/Admin/Chats/ChatRow/index.jsx
rename to frontend/src/pages/GeneralSettings/Chats/ChatRow/index.jsx
index 16e4dfaee..96c3970d5 100644
--- a/frontend/src/pages/Admin/Chats/ChatRow/index.jsx
+++ b/frontend/src/pages/GeneralSettings/Chats/ChatRow/index.jsx
@@ -1,7 +1,7 @@
import { useRef } from "react";
-import Admin from "../../../../models/admin";
import truncate from "truncate";
import { X, Trash } from "@phosphor-icons/react";
+import System from "../../../../models/system";
export default function ChatRow({ chat }) {
const rowRef = useRef(null);
@@ -13,7 +13,7 @@ export default function ChatRow({ chat }) {
)
return false;
rowRef?.current?.remove();
- await Admin.deleteChat(chat.id);
+ await System.deleteChat(chat.id);
};
return (
diff --git a/frontend/src/pages/Admin/Chats/index.jsx b/frontend/src/pages/GeneralSettings/Chats/index.jsx
similarity index 75%
rename from frontend/src/pages/Admin/Chats/index.jsx
rename to frontend/src/pages/GeneralSettings/Chats/index.jsx
index 673e39108..3b2a1e951 100644
--- a/frontend/src/pages/Admin/Chats/index.jsx
+++ b/frontend/src/pages/GeneralSettings/Chats/index.jsx
@@ -5,12 +5,33 @@ import Sidebar, {
import { isMobile } from "react-device-detect";
import * as Skeleton from "react-loading-skeleton";
import "react-loading-skeleton/dist/skeleton.css";
-import Admin from "../../../models/admin";
import useQuery from "../../../hooks/useQuery";
import ChatRow from "./ChatRow";
+import showToast from "../../../utils/toast";
+import System from "../../../models/system";
const PAGE_SIZE = 20;
-export default function AdminChats() {
+export default function WorkspaceChats() {
+ const handleDumpChats = async () => {
+ const chats = await System.exportChats();
+ if (chats) {
+ const blob = new Blob([chats], { type: "application/jsonl" });
+ const link = document.createElement("a");
+ link.href = window.URL.createObjectURL(blob);
+ link.download = "chats.jsonl";
+ document.body.appendChild(link);
+ link.click();
+ window.URL.revokeObjectURL(link.href);
+ document.body.removeChild(link);
+ showToast(
+ "Chats exported successfully. Note: Must have at least 10 chats to be valid for OpenAI fine tuning.",
+ "success"
+ );
+ } else {
+ showToast("Failed to export chats.", "error");
+ }
+ };
+
return (
<div className="w-screen h-screen overflow-hidden bg-sidebar flex">
{!isMobile && <Sidebar />}
@@ -25,6 +46,12 @@ export default function AdminChats() {
<p className="text-2xl font-semibold text-white">
Workspace Chats
</p>
+ <button
+ onClick={handleDumpChats}
+ className="border border-slate-200 px-4 py-1 rounded-lg text-slate-200 text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800"
+ >
+ Export Chats to JSONL
+ </button>
</div>
<p className="text-sm font-base text-white text-opacity-60">
These are all the recorded chats and messages that have been sent
@@ -54,7 +81,7 @@ function ChatsContainer() {
useEffect(() => {
async function fetchChats() {
- const { chats: _chats, hasPages = false } = await Admin.chats(offset);
+ const { chats: _chats, hasPages = false } = await System.chats(offset);
setChats(_chats);
setCanNext(hasPages);
setLoading(false);
@@ -105,9 +132,8 @@ function ChatsContainer() {
</tr>
</thead>
<tbody>
- {chats.map((chat) => (
- <ChatRow key={chat.id} chat={chat} />
- ))}
+ {!!chats &&
+ chats.map((chat) => <ChatRow key={chat.id} chat={chat} />)}
</tbody>
</table>
<div className="flex w-full justify-between items-center">
diff --git a/frontend/src/utils/paths.js b/frontend/src/utils/paths.js
index 1d77abe5a..cbe19795f 100644
--- a/frontend/src/utils/paths.js
+++ b/frontend/src/utils/paths.js
@@ -61,6 +61,9 @@ export default {
apiKeys: () => {
return "/general/api-keys";
},
+ chats: () => {
+ return "/general/workspace-chats";
+ },
},
admin: {
system: () => {
diff --git a/server/endpoints/admin.js b/server/endpoints/admin.js
index a3852bd5a..23949d922 100644
--- a/server/endpoints/admin.js
+++ b/server/endpoints/admin.js
@@ -251,56 +251,6 @@ function adminEndpoints(app) {
}
);
- app.post(
- "/admin/workspace-chats",
- [validatedRequest],
- async (request, response) => {
- try {
- const user = await userFromSession(request, response);
- if (!user || user?.role !== "admin") {
- response.sendStatus(401).end();
- return;
- }
-
- const { offset = 0, limit = 20 } = reqBody(request);
- const chats = await WorkspaceChats.whereWithData(
- {},
- limit,
- offset * limit,
- { id: "desc" }
- );
- const totalChats = await WorkspaceChats.count();
- const hasPages = totalChats > (offset + 1) * limit;
-
- response.status(200).json({ chats: chats, hasPages, totalChats });
- } catch (e) {
- console.error(e);
- response.sendStatus(500).end();
- }
- }
- );
-
- app.delete(
- "/admin/workspace-chats/:id",
- [validatedRequest],
- async (request, response) => {
- try {
- const user = await userFromSession(request, response);
- if (!user || user?.role !== "admin") {
- response.sendStatus(401).end();
- return;
- }
-
- const { id } = request.params;
- await WorkspaceChats.delete({ id: Number(id) });
- response.status(200).json({ success, error });
- } catch (e) {
- console.error(e);
- response.sendStatus(500).end();
- }
- }
- );
-
app.get(
"/admin/system-preferences",
[validatedRequest],
diff --git a/server/endpoints/system.js b/server/endpoints/system.js
index 7e565874d..202ba3762 100644
--- a/server/endpoints/system.js
+++ b/server/endpoints/system.js
@@ -38,6 +38,8 @@ const { Telemetry } = require("../models/telemetry");
const { WelcomeMessages } = require("../models/welcomeMessages");
const { ApiKey } = require("../models/apiKeys");
const { getCustomModels } = require("../utils/helpers/customModels");
+const { WorkspaceChats } = require("../models/workspaceChats");
+const { Workspace } = require("../models/workspace");
function systemEndpoints(app) {
if (!app) return;
@@ -646,6 +648,134 @@ function systemEndpoints(app) {
}
}
);
+
+ app.post(
+ "/system/workspace-chats",
+ [validatedRequest],
+ async (request, response) => {
+ try {
+ if (
+ response.locals.multiUserMode &&
+ response.locals.user?.role !== "admin"
+ ) {
+ return response.sendStatus(401).end();
+ }
+
+ const { offset = 0, limit = 20 } = reqBody(request);
+ const chats = await WorkspaceChats.whereWithData(
+ {},
+ limit,
+ offset * limit,
+ { id: "desc" }
+ );
+ const totalChats = await WorkspaceChats.count();
+ const hasPages = totalChats > (offset + 1) * limit;
+
+ response.status(200).json({ chats: chats, hasPages, totalChats });
+ } catch (e) {
+ console.error(e);
+ response.sendStatus(500).end();
+ }
+ }
+ );
+
+ app.delete(
+ "/system/workspace-chats/:id",
+ [validatedRequest],
+ async (request, response) => {
+ try {
+ if (
+ response.locals.multiUserMode &&
+ response.locals.user?.role !== "admin"
+ ) {
+ return response.sendStatus(401).end();
+ }
+
+ const { id } = request.params;
+ await WorkspaceChats.delete({ id: Number(id) });
+ response.status(200).json({ success, error });
+ } catch (e) {
+ console.error(e);
+ response.sendStatus(500).end();
+ }
+ }
+ );
+
+ app.get(
+ "/system/export-chats",
+ [validatedRequest],
+ async (request, response) => {
+ try {
+ if (
+ response.locals.multiUserMode &&
+ response.locals.user?.role !== "admin"
+ ) {
+ return response.sendStatus(401).end();
+ }
+
+ const chats = await WorkspaceChats.whereWithData({}, null, null, {
+ id: "asc",
+ });
+ const workspaceIds = [
+ ...new Set(chats.map((chat) => chat.workspaceId)),
+ ];
+
+ const workspacesWithPrompts = await Promise.all(
+ workspaceIds.map((id) => Workspace.get({ id: Number(id) }))
+ );
+
+ const workspacePromptsMap = workspacesWithPrompts.reduce(
+ (acc, workspace) => {
+ acc[workspace.id] = workspace.openAiPrompt;
+ return acc;
+ },
+ {}
+ );
+
+ const workspaceChatsMap = chats.reduce((acc, chat) => {
+ const { prompt, response, workspaceId } = chat;
+ const responseJson = JSON.parse(response);
+
+ if (!acc[workspaceId]) {
+ acc[workspaceId] = {
+ messages: [
+ {
+ role: "system",
+ content:
+ workspacePromptsMap[workspaceId] ||
+ "Given the following conversation, relevant context, and a follow up question, reply with an answer to the current question the user is asking. Return only your response to the question given the above information following the users instructions as needed.",
+ },
+ ],
+ };
+ }
+
+ acc[workspaceId].messages.push(
+ {
+ role: "user",
+ content: prompt,
+ },
+ {
+ role: "assistant",
+ content: responseJson.text,
+ }
+ );
+
+ return acc;
+ }, {});
+
+ // Convert to JSONL
+ const jsonl = Object.values(workspaceChatsMap)
+ .map((workspaceChats) => JSON.stringify(workspaceChats))
+ .join("\n");
+
+ response.setHeader("Content-Type", "application/jsonl");
+ response.status(200).send(jsonl);
+ } catch (e) {
+ console.error(e);
+ response.sendStatus(500).end();
+ }
+ }
+ );
}
module.exports = { systemEndpoints };
diff --git a/server/models/workspaceChats.js b/server/models/workspaceChats.js
index 36703efa4..6dfbafeff 100644
--- a/server/models/workspaceChats.js
+++ b/server/models/workspaceChats.js
@@ -161,7 +161,7 @@ const WorkspaceChats = {
const user = await User.get({ id: res.user_id });
res.user = user
? { username: user.username }
- : { username: "deleted user" };
+ : { username: "unknown user" };
}
return results;
--
GitLab