diff --git a/.changeset/twelve-bottles-wait.md b/.changeset/twelve-bottles-wait.md new file mode 100644 index 0000000000000000000000000000000000000000..fa5b176d4e950a3d0e7fbb267da3adda08473d04 --- /dev/null +++ b/.changeset/twelve-bottles-wait.md @@ -0,0 +1,6 @@ +--- +"llamaindex": minor +"docs": minor +--- + +setDocumentHash should be async diff --git a/apps/docs/i18n/fr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md b/apps/docs/i18n/fr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md index 0743da72baded5149a8bf33820769f826f576e4b..690ae542fcefaf825235789d7c5f7e6b345afd98 100644 --- a/apps/docs/i18n/fr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md +++ b/apps/docs/i18n/fr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md @@ -271,7 +271,7 @@ custom_edit_url: null ### setDocumentHash -â–¸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void` +â–¸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\> #### Parameters diff --git a/apps/docs/i18n/hr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md b/apps/docs/i18n/hr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md index f90b88e5f341d7e2577be49ed7e07b41a9fb0278..b7ae4720bf08cc1143fda380cc580ead7ad9b76f 100644 --- a/apps/docs/i18n/hr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md +++ b/apps/docs/i18n/hr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md @@ -271,7 +271,7 @@ custom_edit_url: null ### setDocumentHash -â–¸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void` +â–¸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\> #### Parameters diff --git a/apps/docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md b/apps/docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md index 4c3f59628ed8387c7b36c871c9495129dbecacc1..03bcc954e6058670af8bf9c439258871b167489c 100644 --- a/apps/docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md +++ b/apps/docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md @@ -271,7 +271,7 @@ custom_edit_url: null ### setDocumentHash -â–¸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void` +â–¸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\> #### Parameters diff --git a/packages/core/src/indices/BaseIndex.ts b/packages/core/src/indices/BaseIndex.ts index cd199a7d73898ac58c1d1fcc9d798e6d49f4a843..80a1a9d23def0b0e23bfba17be31133ff8d9331b 100644 --- a/packages/core/src/indices/BaseIndex.ts +++ b/packages/core/src/indices/BaseIndex.ts @@ -95,7 +95,7 @@ export abstract class BaseIndex<T> { [nodeParserFromSettingsOrContext(this.serviceContext)], ); await this.insertNodes(nodes); - this.docStore.setDocumentHash(document.id_, document.hash); + await this.docStore.setDocumentHash(document.id_, document.hash); } abstract insertNodes(nodes: BaseNode[]): Promise<void>; diff --git a/packages/core/src/indices/keyword/index.ts b/packages/core/src/indices/keyword/index.ts index 7ff2c8255bc31fb8f0a6ab819826776200608da0..48dc4c2a45682db258294b178c137111496bc8ad 100644 --- a/packages/core/src/indices/keyword/index.ts +++ b/packages/core/src/indices/keyword/index.ts @@ -281,7 +281,7 @@ export class KeywordTableIndex extends BaseIndex<KeywordTable> { await docStore.addDocuments(documents, true); for (const doc of documents) { - docStore.setDocumentHash(doc.id_, doc.hash); + await docStore.setDocumentHash(doc.id_, doc.hash); } const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents); diff --git a/packages/core/src/indices/summary/index.ts b/packages/core/src/indices/summary/index.ts index 317ee00c0e81794a85c72cae4d09d967697446f6..affbf717829e62826ee94314dd45b77b6041a624 100644 --- a/packages/core/src/indices/summary/index.ts +++ b/packages/core/src/indices/summary/index.ts @@ -138,7 +138,7 @@ export class SummaryIndex extends BaseIndex<IndexList> { await docStore.addDocuments(documents, true); for (const doc of documents) { - docStore.setDocumentHash(doc.id_, doc.hash); + await docStore.setDocumentHash(doc.id_, doc.hash); } const nodes = diff --git a/packages/core/src/ingestion/strategies/DuplicatesStrategy.ts b/packages/core/src/ingestion/strategies/DuplicatesStrategy.ts index 3aee977bc7f496c6850865d5d7aaca97a34eecd9..f47f82af2f3a017517012565459de0a3ce8b4cb3 100644 --- a/packages/core/src/ingestion/strategies/DuplicatesStrategy.ts +++ b/packages/core/src/ingestion/strategies/DuplicatesStrategy.ts @@ -19,7 +19,7 @@ export class DuplicatesStrategy implements TransformComponent { for (const node of nodes) { if (!(node.hash in hashes) && !currentHashes.has(node.hash)) { - this.docStore.setDocumentHash(node.id_, node.hash); + await this.docStore.setDocumentHash(node.id_, node.hash); nodesToRun.push(node); currentHashes.add(node.hash); } diff --git a/packages/core/src/storage/docStore/types.ts b/packages/core/src/storage/docStore/types.ts index f5206b9e43ead5b03214b2cb5caaedf8a0424a50..0067219356e5f0852c6d72d169a25244a140b09c 100644 --- a/packages/core/src/storage/docStore/types.ts +++ b/packages/core/src/storage/docStore/types.ts @@ -32,7 +32,7 @@ export abstract class BaseDocumentStore { abstract documentExists(docId: string): Promise<boolean>; // Hash - abstract setDocumentHash(docId: string, docHash: string): void; + abstract setDocumentHash(docId: string, docHash: string): Promise<void>; abstract getDocumentHash(docId: string): Promise<string | undefined>;