From c019f5abfafbf3de324d996e09e84d1bf5e1cf52 Mon Sep 17 00:00:00 2001
From: Timothy Carambat <rambat1010@gmail.com>
Date: Tue, 22 Aug 2023 09:25:55 -0700
Subject: [PATCH] Enable batch deletion of Pinecone Ids by max limit of 1000
(#210)
* Enable batch deletion of Pinecone Ids by max limit of 1000
* lint
---
server/utils/vectorDbProviders/pinecone/index.js | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/server/utils/vectorDbProviders/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js
index 91d97578f..4109ee653 100644
--- a/server/utils/vectorDbProviders/pinecone/index.js
+++ b/server/utils/vectorDbProviders/pinecone/index.js
@@ -185,10 +185,12 @@ const Pinecone = {
if (knownDocuments.length === 0) return;
const vectorIds = knownDocuments.map((doc) => doc.vectorId);
- await pineconeIndex.delete1({
- ids: vectorIds,
- namespace,
- });
+ for (const batchOfVectorIds of toChunks(vectorIds, 1000)) {
+ await pineconeIndex.delete1({
+ ids: batchOfVectorIds,
+ namespace,
+ });
+ }
const indexes = knownDocuments.map((doc) => doc.id);
await DocumentVectors.deleteIds(indexes);
--
GitLab