diff --git a/frontend/src/index.css b/frontend/src/index.css
index e8d7e2d8ca3a0af0df5499a261f77a7a8c2135a7..729cccb5ff8875838c1f0758f7818a026686888f 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -403,3 +403,7 @@ dialog::backdrop {
.tooltip {
@apply !bg-black !text-white !py-2 !px-3 !rounded-md;
}
+
+.Toastify__toast-body {
+ white-space: pre-line;
+}
diff --git a/server/endpoints/workspaces.js b/server/endpoints/workspaces.js
index 7873ef76a16a9490dc59383ec1eff5e4f536ef8b..7119297f6a5aa39e1a35a56ff619db2646a821f0 100644
--- a/server/endpoints/workspaces.js
+++ b/server/endpoints/workspaces.js
@@ -159,7 +159,7 @@ function workspaceEndpoints(app) {
}
await Document.removeDocuments(currWorkspace, deletes);
- const { failed = [] } = await Document.addDocuments(
+ const { failedToEmbed = [], errors = [] } = await Document.addDocuments(
currWorkspace,
adds
);
@@ -167,8 +167,10 @@ function workspaceEndpoints(app) {
response.status(200).json({
workspace: updatedWorkspace,
message:
- failed.length > 0
- ? `${failed.length} documents could not be embedded.`
+ failedToEmbed.length > 0
+ ? `${failedToEmbed.length} documents failed to add.\n\n${errors
+ .map((msg) => `${msg}`)
+ .join("\n\n")}`
: null,
});
} catch (e) {
diff --git a/server/models/documents.js b/server/models/documents.js
index 4505089d5c38ba35da70e3f521b75ad052239013..8f3b88fbd8d2e2d93342ef6d38944ab0fe452fb7 100644
--- a/server/models/documents.js
+++ b/server/models/documents.js
@@ -39,6 +39,7 @@ const Document = {
if (additions.length === 0) return { failed: [], embedded: [] };
const embedded = [];
const failedToEmbed = [];
+ const errors = new Set();
for (const path of additions) {
const data = await fileData(path);
@@ -53,14 +54,20 @@ const Document = {
workspaceId: workspace.id,
metadata: JSON.stringify(metadata),
};
- const vectorized = await VectorDb.addDocumentToNamespace(
+
+ const { vectorized, error } = await VectorDb.addDocumentToNamespace(
workspace.slug,
{ ...data, docId },
path
);
+
if (!vectorized) {
- console.error("Failed to vectorize", path);
- failedToEmbed.push(path);
+ console.error(
+ "Failed to vectorize",
+ metadata?.title || newDoc.filename
+ );
+ failedToEmbed.push(metadata?.title || newDoc.filename);
+ errors.add(error);
continue;
}
@@ -77,7 +84,7 @@ const Document = {
Embedder: process.env.EMBEDDING_ENGINE || "inherit",
VectorDbSelection: process.env.VECTOR_DB || "pinecone",
});
- return { failed: failedToEmbed, embedded };
+ return { failedToEmbed, errors: Array.from(errors), embedded };
},
removeDocuments: async function (workspace, removals = []) {
diff --git a/server/utils/EmbeddingEngines/azureOpenAi/index.js b/server/utils/EmbeddingEngines/azureOpenAi/index.js
index e80b4b734b3a852a6485d9c1e28454c453d498c3..4193e860d1e05ee601c6361c827162a2cfc4503f 100644
--- a/server/utils/EmbeddingEngines/azureOpenAi/index.js
+++ b/server/utils/EmbeddingEngines/azureOpenAi/index.js
@@ -46,7 +46,12 @@ class AzureOpenAiEmbedder {
resolve({ data: res.data, error: null });
})
.catch((e) => {
- resolve({ data: [], error: e?.error });
+ e.type =
+ e?.response?.data?.error?.code ||
+ e?.response?.status ||
+ "failed_to_embed";
+ e.message = e?.response?.data?.error?.message || e.message;
+ resolve({ data: [], error: e });
});
})
);
@@ -62,11 +67,14 @@ class AzureOpenAiEmbedder {
.map((res) => res.error)
.flat();
if (errors.length > 0) {
+ let uniqueErrors = new Set();
+ errors.map((error) =>
+ uniqueErrors.add(`[${error.type}]: ${error.message}`)
+ );
+
return {
data: [],
- error: `(${errors.length}) Embedding Errors! ${errors
- .map((error) => `[${error.type}]: ${error.message}`)
- .join(", ")}`,
+ error: Array.from(uniqueErrors).join(", "),
};
}
return {
diff --git a/server/utils/EmbeddingEngines/localAi/index.js b/server/utils/EmbeddingEngines/localAi/index.js
index 1480755d76ab0b7c385cbcf9b37859d73fa4863b..2c9db2c734dda2a57875c52ddd52a4003efde1bd 100644
--- a/server/utils/EmbeddingEngines/localAi/index.js
+++ b/server/utils/EmbeddingEngines/localAi/index.js
@@ -41,7 +41,12 @@ class LocalAiEmbedder {
resolve({ data: res.data?.data, error: null });
})
.catch((e) => {
- resolve({ data: [], error: e?.error });
+ e.type =
+ e?.response?.data?.error?.code ||
+ e?.response?.status ||
+ "failed_to_embed";
+ e.message = e?.response?.data?.error?.message || e.message;
+ resolve({ data: [], error: e });
});
})
);
@@ -57,11 +62,14 @@ class LocalAiEmbedder {
.map((res) => res.error)
.flat();
if (errors.length > 0) {
+ let uniqueErrors = new Set();
+ errors.map((error) =>
+ uniqueErrors.add(`[${error.type}]: ${error.message}`)
+ );
+
return {
data: [],
- error: `(${errors.length}) Embedding Errors! ${errors
- .map((error) => `[${error.type}]: ${error.message}`)
- .join(", ")}`,
+ error: Array.from(uniqueErrors).join(", "),
};
}
return {
diff --git a/server/utils/EmbeddingEngines/openAi/index.js b/server/utils/EmbeddingEngines/openAi/index.js
index 105be9d73ab58976ab9afb9bb4aaebce9343c35e..1f9ba432a1a40a4894dd903117578ff4d628433b 100644
--- a/server/utils/EmbeddingEngines/openAi/index.js
+++ b/server/utils/EmbeddingEngines/openAi/index.js
@@ -37,7 +37,12 @@ class OpenAiEmbedder {
resolve({ data: res.data?.data, error: null });
})
.catch((e) => {
- resolve({ data: [], error: e?.error });
+ e.type =
+ e?.response?.data?.error?.code ||
+ e?.response?.status ||
+ "failed_to_embed";
+ e.message = e?.response?.data?.error?.message || e.message;
+ resolve({ data: [], error: e });
});
})
);
@@ -53,11 +58,14 @@ class OpenAiEmbedder {
.map((res) => res.error)
.flat();
if (errors.length > 0) {
+ let uniqueErrors = new Set();
+ errors.map((error) =>
+ uniqueErrors.add(`[${error.type}]: ${error.message}`)
+ );
+
return {
data: [],
- error: `(${errors.length}) Embedding Errors! ${errors
- .map((error) => `[${error.type}]: ${error.message}`)
- .join(", ")}`,
+ error: Array.from(uniqueErrors).join(", "),
};
}
return {
diff --git a/server/utils/vectorDbProviders/chroma/index.js b/server/utils/vectorDbProviders/chroma/index.js
index 878cf05f8526cf3a5b9b0bb89810b4dcf635b287..28af39e66636907214b3ab0769cbcc72fb6a23c0 100644
--- a/server/utils/vectorDbProviders/chroma/index.js
+++ b/server/utils/vectorDbProviders/chroma/index.js
@@ -171,7 +171,7 @@ const Chroma = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
}
// If we are here then we are going to embed and store a novel document.
@@ -242,11 +242,10 @@ const Chroma = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
} catch (e) {
- console.error(e);
console.error("addDocumentToNamespace", e.message);
- return false;
+ return { vectorized: false, error: e.message };
}
},
deleteDocumentFromNamespace: async function (namespace, docId) {
diff --git a/server/utils/vectorDbProviders/lance/index.js b/server/utils/vectorDbProviders/lance/index.js
index 5e58ef1c8618b4f52190332c5c1daf98669358e0..8f243cf9b2243e60f0fec23fe4eea59cb05b6e59 100644
--- a/server/utils/vectorDbProviders/lance/index.js
+++ b/server/utils/vectorDbProviders/lance/index.js
@@ -172,7 +172,7 @@ const LanceDb = {
await this.updateOrCreateCollection(client, submissions, namespace);
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
}
// If we are here then we are going to embed and store a novel document.
@@ -229,11 +229,10 @@ const LanceDb = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
} catch (e) {
- console.error(e);
console.error("addDocumentToNamespace", e.message);
- return false;
+ return { vectorized: false, error: e.message };
}
},
performSimilaritySearch: async function ({
diff --git a/server/utils/vectorDbProviders/milvus/index.js b/server/utils/vectorDbProviders/milvus/index.js
index cc934a9a2b129436f0ba8f55375416097904e3ae..79a1324139d2bfa3d4e5ea966d34b3b611a9d51c 100644
--- a/server/utils/vectorDbProviders/milvus/index.js
+++ b/server/utils/vectorDbProviders/milvus/index.js
@@ -167,7 +167,7 @@ const Milvus = {
}
await DocumentVectors.bulkInsert(documentVectors);
await client.flushSync({ collection_names: [namespace] });
- return true;
+ return { vectorized: true, error: null };
}
const textSplitter = new RecursiveCharacterTextSplitter({
@@ -231,11 +231,10 @@ const Milvus = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
} catch (e) {
- console.error(e);
console.error("addDocumentToNamespace", e.message);
- return false;
+ return { vectorized: false, error: e.message };
}
},
deleteDocumentFromNamespace: async function (namespace, docId) {
diff --git a/server/utils/vectorDbProviders/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js
index 7a7f862c2319c335dcc9befb08930055d47a0e85..594a9aaf3dc3ff06ea11e2cbb4e4ce7de99864d8 100644
--- a/server/utils/vectorDbProviders/pinecone/index.js
+++ b/server/utils/vectorDbProviders/pinecone/index.js
@@ -125,7 +125,7 @@ const Pinecone = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
}
// If we are here then we are going to embed and store a novel document.
@@ -183,11 +183,10 @@ const Pinecone = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
} catch (e) {
- console.error(e);
console.error("addDocumentToNamespace", e.message);
- return false;
+ return { vectorized: false, error: e.message };
}
},
deleteDocumentFromNamespace: async function (namespace, docId) {
diff --git a/server/utils/vectorDbProviders/qdrant/index.js b/server/utils/vectorDbProviders/qdrant/index.js
index 2783cde932f3e66b43ce129ca74b270b7178dc42..70c069e843308311e0fb8d93aa412478e64a9a0b 100644
--- a/server/utils/vectorDbProviders/qdrant/index.js
+++ b/server/utils/vectorDbProviders/qdrant/index.js
@@ -190,7 +190,7 @@ const QDrant = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
}
// If we are here then we are going to embed and store a novel document.
@@ -272,11 +272,10 @@ const QDrant = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
} catch (e) {
- console.error(e);
console.error("addDocumentToNamespace", e.message);
- return false;
+ return { vectorized: false, error: e.message };
}
},
deleteDocumentFromNamespace: async function (namespace, docId) {
diff --git a/server/utils/vectorDbProviders/weaviate/index.js b/server/utils/vectorDbProviders/weaviate/index.js
index 91faff64ef54ce2f4a55839cb022d30b634bbea4..ac89315af7f2f8b760cf5f1c3996c2f677bee3c2 100644
--- a/server/utils/vectorDbProviders/weaviate/index.js
+++ b/server/utils/vectorDbProviders/weaviate/index.js
@@ -233,7 +233,7 @@ const Weaviate = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
}
// If we are here then we are going to embed and store a novel document.
@@ -316,11 +316,10 @@ const Weaviate = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
} catch (e) {
- console.error(e);
console.error("addDocumentToNamespace", e.message);
- return false;
+ return { vectorized: false, error: e.message };
}
},
deleteDocumentFromNamespace: async function (namespace, docId) {
diff --git a/server/utils/vectorDbProviders/zilliz/index.js b/server/utils/vectorDbProviders/zilliz/index.js
index b8493e1c286ed907e045fbe19b7611272fdd0f71..31afab35a30acba66436355a88787f00bf1d5f6b 100644
--- a/server/utils/vectorDbProviders/zilliz/index.js
+++ b/server/utils/vectorDbProviders/zilliz/index.js
@@ -168,7 +168,7 @@ const Zilliz = {
}
await DocumentVectors.bulkInsert(documentVectors);
await client.flushSync({ collection_names: [namespace] });
- return true;
+ return { vectorized: true, error: null };
}
const textSplitter = new RecursiveCharacterTextSplitter({
@@ -232,11 +232,10 @@ const Zilliz = {
}
await DocumentVectors.bulkInsert(documentVectors);
- return true;
+ return { vectorized: true, error: null };
} catch (e) {
- console.error(e);
console.error("addDocumentToNamespace", e.message);
- return false;
+ return { vectorized: false, error: e.message };
}
},
deleteDocumentFromNamespace: async function (namespace, docId) {