From 7fb76cfef0ca80a314b44f866af6541f20bf867f Mon Sep 17 00:00:00 2001
From: Sean Hatfield <seanhatfield5@gmail.com>
Date: Thu, 18 Jan 2024 17:59:51 -0800
Subject: [PATCH] add support for exporting to json and csv in workspace chats
(#622)
* add support for exporting to json and csv in workspace chats
* safety encode URL options
* remove message about openai fine tuning on export success
* all defaults to jsonl
---
frontend/src/index.css | 32 +++++++
frontend/src/models/system.js | 6 +-
.../src/pages/GeneralSettings/Chats/index.jsx | 91 ++++++++++++++++---
server/endpoints/system.js | 30 ++++--
server/endpoints/utils.js | 36 +++++++-
5 files changed, 172 insertions(+), 23 deletions(-)
diff --git a/frontend/src/index.css b/frontend/src/index.css
index 729cccb5f..b9e6976da 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -407,3 +407,35 @@ dialog::backdrop {
.Toastify__toast-body {
white-space: pre-line;
}
+
+@keyframes slideDown {
+ from {
+ max-height: 0;
+ opacity: 0;
+ }
+
+ to {
+ max-height: 400px;
+ opacity: 1;
+ }
+}
+
+.slide-down {
+ animation: slideDown 0.3s ease-out forwards;
+}
+
+@keyframes slideUp {
+ from {
+ max-height: 400px;
+ opacity: 1;
+ }
+
+ to {
+ max-height: 0;
+ opacity: 0;
+ }
+}
+
+.slide-up {
+ animation: slideUp 0.3s ease-out forwards;
+}
diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js
index 6dc003123..e504fcb26 100644
--- a/frontend/src/models/system.js
+++ b/frontend/src/models/system.js
@@ -398,8 +398,10 @@ const System = {
return { success: false, error: e.message };
});
},
- exportChats: async () => {
- return await fetch(`${API_BASE}/system/export-chats`, {
+ exportChats: async (type = "csv") => {
+ const url = new URL(`${fullApiUrl()}/system/export-chats`);
+ url.searchParams.append("type", encodeURIComponent(type));
+ return await fetch(url, {
method: "GET",
headers: baseHeaders(),
})
diff --git a/frontend/src/pages/GeneralSettings/Chats/index.jsx b/frontend/src/pages/GeneralSettings/Chats/index.jsx
index d925232c3..f0ae8e973 100644
--- a/frontend/src/pages/GeneralSettings/Chats/index.jsx
+++ b/frontend/src/pages/GeneralSettings/Chats/index.jsx
@@ -1,4 +1,4 @@
-import { useEffect, useState } from "react";
+import { useEffect, useRef, useState } from "react";
import Sidebar, { SidebarMobileHeader } from "@/components/SettingsSidebar";
import { isMobile } from "react-device-detect";
import * as Skeleton from "react-loading-skeleton";
@@ -7,22 +7,32 @@ import useQuery from "@/hooks/useQuery";
import ChatRow from "./ChatRow";
import showToast from "@/utils/toast";
import System from "@/models/system";
-
-const PAGE_SIZE = 20;
+import { CaretDown } from "@phosphor-icons/react";
export default function WorkspaceChats() {
+ const [showMenu, setShowMenu] = useState(false);
+ const [exportType, setExportType] = useState("jsonl");
+ const menuRef = useRef();
+ const openMenuButton = useRef();
+
+ const exportOptions = {
+ csv: { mimeType: "text/csv", fileExtension: "csv" },
+ json: { mimeType: "application/json", fileExtension: "json" },
+ jsonl: { mimeType: "application/jsonl", fileExtension: "jsonl" },
+ };
const handleDumpChats = async () => {
- const chats = await System.exportChats();
+ const chats = await System.exportChats(exportType);
if (chats) {
- const blob = new Blob([chats], { type: "application/jsonl" });
+ const { mimeType, fileExtension } = exportOptions[exportType];
+ const blob = new Blob([chats], { type: mimeType });
const link = document.createElement("a");
link.href = window.URL.createObjectURL(blob);
- link.download = "chats.jsonl";
+ link.download = `chats.${fileExtension}`;
document.body.appendChild(link);
link.click();
window.URL.revokeObjectURL(link.href);
document.body.removeChild(link);
showToast(
- "Chats exported successfully. Note: Must have at least 10 chats to be valid for OpenAI fine tuning.",
+ `Chats exported successfully as ${fileExtension.toUpperCase()}.`,
"success"
);
} else {
@@ -30,6 +40,27 @@ export default function WorkspaceChats() {
}
};
+ const toggleMenu = () => {
+ setShowMenu(!showMenu);
+ };
+
+ useEffect(() => {
+ function handleClickOutside(event) {
+ if (
+ menuRef.current &&
+ !menuRef.current.contains(event.target) &&
+ !openMenuButton.current.contains(event.target)
+ ) {
+ setShowMenu(false);
+ }
+ }
+
+ document.addEventListener("mousedown", handleClickOutside);
+ return () => {
+ document.removeEventListener("mousedown", handleClickOutside);
+ };
+ }, []);
+
return (
<div className="w-screen h-screen overflow-hidden bg-sidebar flex">
{!isMobile && <Sidebar />}
@@ -44,12 +75,46 @@ export default function WorkspaceChats() {
<p className="text-2xl font-semibold text-white">
Workspace Chats
</p>
- <button
- onClick={handleDumpChats}
- className="border border-slate-200 px-4 py-1 rounded-lg text-slate-200 text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800"
- >
- Export Chats to JSONL
- </button>
+ <div className="flex gap-x-1 relative">
+ <button
+ onClick={handleDumpChats}
+ className="border border-slate-200 px-4 py-1 rounded-lg text-slate-200 text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800"
+ >
+ Export Chats to {exportType.toUpperCase()}
+ </button>
+ <button
+ ref={openMenuButton}
+ onClick={toggleMenu}
+ className={`transition-all duration-300 border border-slate-200 p-1 rounded-lg text-slate-200 text-sm items-center flex hover:bg-slate-200 hover:text-slate-800 ${
+ showMenu ? "bg-slate-200 text-slate-800" : ""
+ }`}
+ >
+ <CaretDown weight="bold" className="h-4 w-4" />
+ </button>
+ <div
+ ref={menuRef}
+ className={`${
+ showMenu ? "slide-down" : "slide-up hidden"
+ } z-20 w-fit rounded-lg absolute top-full right-0 bg-sidebar p-4 flex items-center justify-center mt-2`}
+ >
+ <div className="flex flex-col gap-y-2">
+ {Object.keys(exportOptions)
+ .filter((type) => type !== exportType)
+ .map((type) => (
+ <button
+ key={type}
+ onClick={() => {
+ setExportType(type);
+ setShowMenu(false);
+ }}
+ className="text-white hover:bg-slate-200/20 w-full text-left px-4 py-1.5 rounded-md"
+ >
+ {type.toUpperCase()}
+ </button>
+ ))}
+ </div>
+ </div>
+ </div>
</div>
<p className="text-sm font-base text-white text-opacity-60">
These are all the recorded chats and messages that have been sent
diff --git a/server/endpoints/system.js b/server/endpoints/system.js
index 4bed2b160..7e535c004 100644
--- a/server/endpoints/system.js
+++ b/server/endpoints/system.js
@@ -47,6 +47,7 @@ const { WorkspaceChats } = require("../models/workspaceChats");
const { Workspace } = require("../models/workspace");
const { flexUserRoleValid } = require("../utils/middleware/multiUserProtected");
const { fetchPfp, determinePfpFilepath } = require("../utils/files/pfp");
+const { convertToCSV, convertToJSON, convertToJSONL } = require("./utils");
function systemEndpoints(app) {
if (!app) return;
@@ -776,8 +777,9 @@ function systemEndpoints(app) {
app.get(
"/system/export-chats",
[validatedRequest, flexUserRoleValid],
- async (_request, response) => {
+ async (request, response) => {
try {
+ const { type = "jsonl" } = request.query;
const chats = await WorkspaceChats.whereWithData({}, null, null, {
id: "asc",
});
@@ -828,13 +830,27 @@ function systemEndpoints(app) {
return acc;
}, {});
- // Convert to JSONL
- const jsonl = Object.values(workspaceChatsMap)
- .map((workspaceChats) => JSON.stringify(workspaceChats))
- .join("\n");
+ let output;
+ switch (type.toLowerCase()) {
+ case "json": {
+ response.setHeader("Content-Type", "application/json");
+ output = await convertToJSON(workspaceChatsMap);
+ break;
+ }
+ case "csv": {
+ response.setHeader("Content-Type", "text/csv");
+ output = await convertToCSV(workspaceChatsMap);
+ break;
+ }
+ // JSONL default
+ default: {
+ response.setHeader("Content-Type", "application/jsonl");
+ output = await convertToJSONL(workspaceChatsMap);
+ break;
+ }
+ }
- response.setHeader("Content-Type", "application/jsonl");
- response.status(200).send(jsonl);
+ response.status(200).send(output);
} catch (e) {
console.error(e);
response.sendStatus(500).end();
diff --git a/server/endpoints/utils.js b/server/endpoints/utils.js
index 82be00f9d..3c639b3e5 100644
--- a/server/endpoints/utils.js
+++ b/server/endpoints/utils.js
@@ -32,6 +32,34 @@ async function getDiskStorage() {
}
}
+async function convertToCSV(workspaceChatsMap) {
+ const rows = ["role,content"];
+ for (const workspaceChats of Object.values(workspaceChatsMap)) {
+ for (const message of workspaceChats.messages) {
+ // Escape double quotes and wrap content in double quotes
+ const escapedContent = `"${message.content
+ .replace(/"/g, '""')
+ .replace(/\n/g, " ")}"`;
+ rows.push(`${message.role},${escapedContent}`);
+ }
+ }
+ return rows.join("\n");
+}
+
+async function convertToJSON(workspaceChatsMap) {
+ const allMessages = [].concat.apply(
+ [],
+ Object.values(workspaceChatsMap).map((workspace) => workspace.messages)
+ );
+ return JSON.stringify(allMessages);
+}
+
+async function convertToJSONL(workspaceChatsMap) {
+ return Object.values(workspaceChatsMap)
+ .map((workspaceChats) => JSON.stringify(workspaceChats))
+ .join("\n");
+}
+
function utilEndpoints(app) {
if (!app) return;
@@ -54,4 +82,10 @@ function utilEndpoints(app) {
});
}
-module.exports = { utilEndpoints, getGitVersion };
+module.exports = {
+ utilEndpoints,
+ getGitVersion,
+ convertToCSV,
+ convertToJSON,
+ convertToJSONL,
+};
--
GitLab