Skip to content
Snippets Groups Projects
Unverified Commit a58f2711 authored by Sean Hatfield's avatar Sean Hatfield Committed by GitHub
Browse files

Milvus bug fix (#2183)

* patch no text results for milvus chunks

* wrap addDocumentToNamespace in try catch for handling milvus errors

* lint

* revert milvus db changes

* add try catch to handle grpc error from milvus
parent cf6928fd
No related branches found
No related tags found
No related merge requests found
...@@ -156,30 +156,38 @@ const Milvus = { ...@@ -156,30 +156,38 @@ const Milvus = {
vectorDimension = chunks[0][0].values.length || null; vectorDimension = chunks[0][0].values.length || null;
await this.getOrCreateCollection(client, namespace, vectorDimension); await this.getOrCreateCollection(client, namespace, vectorDimension);
for (const chunk of chunks) { try {
// Before sending to Pinecone and saving the records to our db for (const chunk of chunks) {
// we need to assign the id of each chunk that is stored in the cached file. // Before sending to Milvus and saving the records to our db
const newChunks = chunk.map((chunk) => { // we need to assign the id of each chunk that is stored in the cached file.
const id = uuidv4(); const newChunks = chunk.map((chunk) => {
documentVectors.push({ docId, vectorId: id }); const id = uuidv4();
return { id, vector: chunk.values, metadata: chunk.metadata }; documentVectors.push({ docId, vectorId: id });
}); return { id, vector: chunk.values, metadata: chunk.metadata };
const insertResult = await client.insert({ });
collection_name: this.normalize(namespace), const insertResult = await client.insert({
data: newChunks, collection_name: this.normalize(namespace),
}); data: newChunks,
});
if (insertResult?.status.error_code !== "Success") { if (insertResult?.status.error_code !== "Success") {
throw new Error( throw new Error(
`Error embedding into Milvus! Reason:${insertResult?.status.reason}` `Error embedding into Milvus! Reason:${insertResult?.status.reason}`
); );
}
} }
await DocumentVectors.bulkInsert(documentVectors);
await client.flushSync({
collection_names: [this.normalize(namespace)],
});
return { vectorized: true, error: null };
} catch (insertError) {
console.error(
"Error inserting cached chunks:",
insertError.message
);
return { vectorized: false, error: insertError.message };
} }
await DocumentVectors.bulkInsert(documentVectors);
await client.flushSync({
collection_names: [this.normalize(namespace)],
});
return { vectorized: true, error: null };
} }
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment