From cc4bac899eff3debe5d1de0279ca93aeb88a3d56 Mon Sep 17 00:00:00 2001 From: Yi Ding <yi.s.ding@gmail.com> Date: Tue, 15 Aug 2023 08:59:09 -0700 Subject: [PATCH] remove node with embedding --- .../docs/api/classes/SimpleVectorStore.md | 14 +++--- .../docs/docs/api/classes/VectorStoreIndex.md | 24 ++++----- .../docs/api/enums/VectorStoreQueryMode.md | 14 +++--- .../docs/api/interfaces/ExactMatchFilter.md | 6 +-- .../docs/api/interfaces/MetadataFilters.md | 2 +- apps/docs/docs/api/interfaces/MetadataInfo.md | 6 +-- apps/docs/docs/api/interfaces/VectorStore.md | 16 +++--- .../docs/api/interfaces/VectorStoreInfo.md | 4 +- .../docs/api/interfaces/VectorStoreQuery.md | 16 +++--- .../api/interfaces/VectorStoreQueryResult.md | 6 +-- .../api/interfaces/VectorStoreQuerySpec.md | 6 +-- apps/docs/docs/api/modules.md | 1 - packages/core/src/Node.ts | 8 --- .../indices/vectorStore/VectorStoreIndex.ts | 19 +++---- .../storage/vectorStore/SimpleVectorStore.ts | 49 +++++++++---------- .../core/src/storage/vectorStore/types.ts | 3 +- 16 files changed, 88 insertions(+), 106 deletions(-) diff --git a/apps/docs/docs/api/classes/SimpleVectorStore.md b/apps/docs/docs/api/classes/SimpleVectorStore.md index 6b5fbef83..48e35a23c 100644 --- a/apps/docs/docs/api/classes/SimpleVectorStore.md +++ b/apps/docs/docs/api/classes/SimpleVectorStore.md @@ -99,7 +99,7 @@ VectorStore.client | Name | Type | | :------ | :------ | -| `embeddingResults` | [`NodeWithEmbedding`](../interfaces/NodeWithEmbedding.md)[] | +| `embeddingResults` | [`BaseNode`](BaseNode.md)[] | #### Returns @@ -135,7 +135,7 @@ ___ #### Defined in -[storage/vectorStore/SimpleVectorStore.ts:78](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L78) +[storage/vectorStore/SimpleVectorStore.ts:77](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L77) ___ @@ -180,7 +180,7 @@ ___ #### Defined in -[storage/vectorStore/SimpleVectorStore.ts:147](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L147) +[storage/vectorStore/SimpleVectorStore.ts:146](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L146) ___ @@ -204,7 +204,7 @@ ___ #### Defined in -[storage/vectorStore/SimpleVectorStore.ts:89](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L89) +[storage/vectorStore/SimpleVectorStore.ts:88](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L88) ___ @@ -218,7 +218,7 @@ ___ #### Defined in -[storage/vectorStore/SimpleVectorStore.ts:197](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L197) +[storage/vectorStore/SimpleVectorStore.ts:196](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L196) ___ @@ -238,7 +238,7 @@ ___ #### Defined in -[storage/vectorStore/SimpleVectorStore.ts:190](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L190) +[storage/vectorStore/SimpleVectorStore.ts:189](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L189) ___ @@ -280,4 +280,4 @@ ___ #### Defined in -[storage/vectorStore/SimpleVectorStore.ts:160](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L160) +[storage/vectorStore/SimpleVectorStore.ts:159](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/SimpleVectorStore.ts#L159) diff --git a/apps/docs/docs/api/classes/VectorStoreIndex.md b/apps/docs/docs/api/classes/VectorStoreIndex.md index 32f6a9eb9..aaa264826 100644 --- a/apps/docs/docs/api/classes/VectorStoreIndex.md +++ b/apps/docs/docs/api/classes/VectorStoreIndex.md @@ -32,7 +32,7 @@ The VectorStoreIndex, an index that stores the nodes only according to their vec #### Defined in -[indices/vectorStore/VectorStoreIndex.ts:36](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L36) +[indices/vectorStore/VectorStoreIndex.ts:31](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L31) ## Properties @@ -116,7 +116,7 @@ ___ #### Defined in -[indices/vectorStore/VectorStoreIndex.ts:34](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L34) +[indices/vectorStore/VectorStoreIndex.ts:29](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L29) ## Methods @@ -145,7 +145,7 @@ and response synthezier if they are not provided. #### Defined in -[indices/vectorStore/VectorStoreIndex.ts:223](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L223) +[indices/vectorStore/VectorStoreIndex.ts:216](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L216) ___ @@ -171,7 +171,7 @@ Create a new retriever from the index. #### Defined in -[indices/vectorStore/VectorStoreIndex.ts:219](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L219) +[indices/vectorStore/VectorStoreIndex.ts:212](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L212) ___ @@ -196,7 +196,7 @@ ___ #### Defined in -[indices/vectorStore/VectorStoreIndex.ts:259](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L259) +[indices/vectorStore/VectorStoreIndex.ts:252](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L252) ___ @@ -246,7 +246,7 @@ ___ #### Defined in -[indices/vectorStore/VectorStoreIndex.ts:234](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L234) +[indices/vectorStore/VectorStoreIndex.ts:227](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L227) ___ @@ -272,7 +272,7 @@ Get embeddings for nodes and place them into the index. #### Defined in -[indices/vectorStore/VectorStoreIndex.ts:147](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L147) +[indices/vectorStore/VectorStoreIndex.ts:143](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L143) ___ @@ -297,13 +297,13 @@ High level API: split documents, get embeddings, and build index. #### Defined in -[indices/vectorStore/VectorStoreIndex.ts:194](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L194) +[indices/vectorStore/VectorStoreIndex.ts:187](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L187) ___ ### getNodeEmbeddingResults -â–¸ `Static` **getNodeEmbeddingResults**(`nodes`, `serviceContext`, `logProgress?`): `Promise`<[`NodeWithEmbedding`](../interfaces/NodeWithEmbedding.md)[]\> +â–¸ `Static` **getNodeEmbeddingResults**(`nodes`, `serviceContext`, `logProgress?`): `Promise`<[`BaseNode`](BaseNode.md)[]\> Get the embeddings for nodes. @@ -317,11 +317,11 @@ Get the embeddings for nodes. #### Returns -`Promise`<[`NodeWithEmbedding`](../interfaces/NodeWithEmbedding.md)[]\> +`Promise`<[`BaseNode`](BaseNode.md)[]\> #### Defined in -[indices/vectorStore/VectorStoreIndex.ts:119](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L119) +[indices/vectorStore/VectorStoreIndex.ts:114](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L114) ___ @@ -344,4 +344,4 @@ This is needed to handle persistence. #### Defined in -[indices/vectorStore/VectorStoreIndex.ts:47](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L47) +[indices/vectorStore/VectorStoreIndex.ts:42](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/indices/vectorStore/VectorStoreIndex.ts#L42) diff --git a/apps/docs/docs/api/enums/VectorStoreQueryMode.md b/apps/docs/docs/api/enums/VectorStoreQueryMode.md index ab30e61ff..5f7db5ded 100644 --- a/apps/docs/docs/api/enums/VectorStoreQueryMode.md +++ b/apps/docs/docs/api/enums/VectorStoreQueryMode.md @@ -14,7 +14,7 @@ custom_edit_url: null #### Defined in -[storage/vectorStore/types.ts:12](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L12) +[storage/vectorStore/types.ts:11](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L11) ___ @@ -24,7 +24,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:14](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L14) +[storage/vectorStore/types.ts:13](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L13) ___ @@ -34,7 +34,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:18](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L18) +[storage/vectorStore/types.ts:17](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L17) ___ @@ -44,7 +44,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:17](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L17) +[storage/vectorStore/types.ts:16](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L16) ___ @@ -54,7 +54,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:20](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L20) +[storage/vectorStore/types.ts:19](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L19) ___ @@ -64,7 +64,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:13](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L13) +[storage/vectorStore/types.ts:12](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L12) ___ @@ -74,4 +74,4 @@ ___ #### Defined in -[storage/vectorStore/types.ts:16](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L16) +[storage/vectorStore/types.ts:15](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L15) diff --git a/apps/docs/docs/api/interfaces/ExactMatchFilter.md b/apps/docs/docs/api/interfaces/ExactMatchFilter.md index 8775dc725..9ec89fdcc 100644 --- a/apps/docs/docs/api/interfaces/ExactMatchFilter.md +++ b/apps/docs/docs/api/interfaces/ExactMatchFilter.md @@ -14,7 +14,7 @@ custom_edit_url: null #### Defined in -[storage/vectorStore/types.ts:24](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L24) +[storage/vectorStore/types.ts:23](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L23) ___ @@ -24,7 +24,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:25](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L25) +[storage/vectorStore/types.ts:24](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L24) ___ @@ -34,4 +34,4 @@ ___ #### Defined in -[storage/vectorStore/types.ts:26](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L26) +[storage/vectorStore/types.ts:25](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L25) diff --git a/apps/docs/docs/api/interfaces/MetadataFilters.md b/apps/docs/docs/api/interfaces/MetadataFilters.md index 7ba21aa70..8c0cf31d0 100644 --- a/apps/docs/docs/api/interfaces/MetadataFilters.md +++ b/apps/docs/docs/api/interfaces/MetadataFilters.md @@ -14,4 +14,4 @@ custom_edit_url: null #### Defined in -[storage/vectorStore/types.ts:30](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L30) +[storage/vectorStore/types.ts:29](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L29) diff --git a/apps/docs/docs/api/interfaces/MetadataInfo.md b/apps/docs/docs/api/interfaces/MetadataInfo.md index dd15fbf73..4bac991df 100644 --- a/apps/docs/docs/api/interfaces/MetadataInfo.md +++ b/apps/docs/docs/api/interfaces/MetadataInfo.md @@ -14,7 +14,7 @@ custom_edit_url: null #### Defined in -[storage/vectorStore/types.ts:42](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L42) +[storage/vectorStore/types.ts:41](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L41) ___ @@ -24,7 +24,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:40](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L40) +[storage/vectorStore/types.ts:39](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L39) ___ @@ -34,4 +34,4 @@ ___ #### Defined in -[storage/vectorStore/types.ts:41](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L41) +[storage/vectorStore/types.ts:40](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L40) diff --git a/apps/docs/docs/api/interfaces/VectorStore.md b/apps/docs/docs/api/interfaces/VectorStore.md index 9f47de8ae..302b05672 100644 --- a/apps/docs/docs/api/interfaces/VectorStore.md +++ b/apps/docs/docs/api/interfaces/VectorStore.md @@ -18,7 +18,7 @@ custom_edit_url: null #### Defined in -[storage/vectorStore/types.ts:63](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L63) +[storage/vectorStore/types.ts:62](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L62) ___ @@ -28,7 +28,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:62](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L62) +[storage/vectorStore/types.ts:61](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L61) ## Methods @@ -40,7 +40,7 @@ ___ | Name | Type | | :------ | :------ | -| `embeddingResults` | [`NodeWithEmbedding`](NodeWithEmbedding.md)[] | +| `embeddingResults` | [`BaseNode`](../classes/BaseNode.md)[] | #### Returns @@ -48,7 +48,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:65](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L65) +[storage/vectorStore/types.ts:64](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L64) ___ @@ -62,7 +62,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:64](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L64) +[storage/vectorStore/types.ts:63](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L63) ___ @@ -83,7 +83,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:66](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L66) +[storage/vectorStore/types.ts:65](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L65) ___ @@ -104,7 +104,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:68](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L68) +[storage/vectorStore/types.ts:67](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L67) ___ @@ -125,4 +125,4 @@ ___ #### Defined in -[storage/vectorStore/types.ts:67](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L67) +[storage/vectorStore/types.ts:66](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L66) diff --git a/apps/docs/docs/api/interfaces/VectorStoreInfo.md b/apps/docs/docs/api/interfaces/VectorStoreInfo.md index 682922493..adf63a369 100644 --- a/apps/docs/docs/api/interfaces/VectorStoreInfo.md +++ b/apps/docs/docs/api/interfaces/VectorStoreInfo.md @@ -14,7 +14,7 @@ custom_edit_url: null #### Defined in -[storage/vectorStore/types.ts:47](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L47) +[storage/vectorStore/types.ts:46](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L46) ___ @@ -24,4 +24,4 @@ ___ #### Defined in -[storage/vectorStore/types.ts:46](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L46) +[storage/vectorStore/types.ts:45](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L45) diff --git a/apps/docs/docs/api/interfaces/VectorStoreQuery.md b/apps/docs/docs/api/interfaces/VectorStoreQuery.md index edb875271..794e821e4 100644 --- a/apps/docs/docs/api/interfaces/VectorStoreQuery.md +++ b/apps/docs/docs/api/interfaces/VectorStoreQuery.md @@ -14,7 +14,7 @@ custom_edit_url: null #### Defined in -[storage/vectorStore/types.ts:56](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L56) +[storage/vectorStore/types.ts:55](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L55) ___ @@ -24,7 +24,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:53](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L53) +[storage/vectorStore/types.ts:52](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L52) ___ @@ -34,7 +34,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:57](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L57) +[storage/vectorStore/types.ts:56](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L56) ___ @@ -44,7 +44,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:58](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L58) +[storage/vectorStore/types.ts:57](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L57) ___ @@ -54,7 +54,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:55](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L55) +[storage/vectorStore/types.ts:54](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L54) ___ @@ -64,7 +64,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:51](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L51) +[storage/vectorStore/types.ts:50](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L50) ___ @@ -74,7 +74,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:54](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L54) +[storage/vectorStore/types.ts:53](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L53) ___ @@ -84,4 +84,4 @@ ___ #### Defined in -[storage/vectorStore/types.ts:52](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L52) +[storage/vectorStore/types.ts:51](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L51) diff --git a/apps/docs/docs/api/interfaces/VectorStoreQueryResult.md b/apps/docs/docs/api/interfaces/VectorStoreQueryResult.md index aa5223f33..89ac4f510 100644 --- a/apps/docs/docs/api/interfaces/VectorStoreQueryResult.md +++ b/apps/docs/docs/api/interfaces/VectorStoreQueryResult.md @@ -14,7 +14,7 @@ custom_edit_url: null #### Defined in -[storage/vectorStore/types.ts:8](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L8) +[storage/vectorStore/types.ts:7](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L7) ___ @@ -24,7 +24,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:6](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L6) +[storage/vectorStore/types.ts:5](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L5) ___ @@ -34,4 +34,4 @@ ___ #### Defined in -[storage/vectorStore/types.ts:7](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L7) +[storage/vectorStore/types.ts:6](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L6) diff --git a/apps/docs/docs/api/interfaces/VectorStoreQuerySpec.md b/apps/docs/docs/api/interfaces/VectorStoreQuerySpec.md index 979bcd862..75451bb7f 100644 --- a/apps/docs/docs/api/interfaces/VectorStoreQuerySpec.md +++ b/apps/docs/docs/api/interfaces/VectorStoreQuerySpec.md @@ -14,7 +14,7 @@ custom_edit_url: null #### Defined in -[storage/vectorStore/types.ts:35](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L35) +[storage/vectorStore/types.ts:34](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L34) ___ @@ -24,7 +24,7 @@ ___ #### Defined in -[storage/vectorStore/types.ts:34](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L34) +[storage/vectorStore/types.ts:33](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L33) ___ @@ -34,4 +34,4 @@ ___ #### Defined in -[storage/vectorStore/types.ts:36](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L36) +[storage/vectorStore/types.ts:35](https://github.com/run-llama/LlamaIndexTS/blob/main/packages/core/src/storage/vectorStore/types.ts#L35) diff --git a/apps/docs/docs/api/modules.md b/apps/docs/docs/api/modules.md index 5dd4b6dcd..1cb9fbca5 100644 --- a/apps/docs/docs/api/modules.md +++ b/apps/docs/docs/api/modules.md @@ -80,7 +80,6 @@ custom_edit_url: null - [MetadataFilters](interfaces/MetadataFilters.md) - [MetadataInfo](interfaces/MetadataInfo.md) - [NodeParser](interfaces/NodeParser.md) -- [NodeWithEmbedding](interfaces/NodeWithEmbedding.md) - [NodeWithScore](interfaces/NodeWithScore.md) - [QueryEngineTool](interfaces/QueryEngineTool.md) - [RelatedNodeInfo](interfaces/RelatedNodeInfo.md) diff --git a/packages/core/src/Node.ts b/packages/core/src/Node.ts index 510351b77..5b7adb296 100644 --- a/packages/core/src/Node.ts +++ b/packages/core/src/Node.ts @@ -296,11 +296,3 @@ export interface NodeWithScore { node: BaseNode; score: number; } - -/** - * A node with an embedding - */ -export interface NodeWithEmbedding { - node: BaseNode; - embedding: number[]; -} diff --git a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts index 04624bc89..488dd873a 100644 --- a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts +++ b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts @@ -1,9 +1,4 @@ -import { - BaseNode, - Document, - MetadataMode, - NodeWithEmbedding, -} from "../../Node"; +import { BaseNode, Document, MetadataMode } from "../../Node"; import { BaseQueryEngine, RetrieverQueryEngine } from "../../QueryEngine"; import { ResponseSynthesizer } from "../../ResponseSynthesizer"; import { BaseRetriever } from "../../Retriever"; @@ -121,7 +116,7 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> { serviceContext: ServiceContext, logProgress = false, ) { - const nodesWithEmbeddings: NodeWithEmbedding[] = []; + const nodesWithEmbeddings: BaseNode[] = []; for (let i = 0; i < nodes.length; ++i) { const node = nodes[i]; @@ -131,7 +126,8 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> { const embedding = await serviceContext.embedModel.getTextEmbedding( node.getContent(MetadataMode.EMBED), ); - nodesWithEmbeddings.push({ node, embedding }); + node.embedding = embedding; + nodesWithEmbeddings.push(node); } return nodesWithEmbeddings; @@ -171,13 +167,10 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> { await vectorStore.add(embeddingResults); if (!vectorStore.storesText) { - await docStore.addDocuments( - embeddingResults.map((result) => result.node), - true, - ); + await docStore.addDocuments(embeddingResults, true); } - for (const { node } of embeddingResults) { + for (const node of embeddingResults) { indexDict.addNode(node); } diff --git a/packages/core/src/storage/vectorStore/SimpleVectorStore.ts b/packages/core/src/storage/vectorStore/SimpleVectorStore.ts index e7e62fe39..08ffd8cd2 100644 --- a/packages/core/src/storage/vectorStore/SimpleVectorStore.ts +++ b/packages/core/src/storage/vectorStore/SimpleVectorStore.ts @@ -1,19 +1,19 @@ -import * as path from "path"; import _ from "lodash"; -import { GenericFileSystem, exists } from "../FileSystem"; +import * as path from "path"; +import { + getTopKEmbeddings, + getTopKEmbeddingsLearner, + getTopKMMREmbeddings, +} from "../../Embedding"; +import { BaseNode } from "../../Node"; +import { DEFAULT_FS, DEFAULT_PERSIST_DIR } from "../constants"; +import { exists, GenericFileSystem } from "../FileSystem"; import { VectorStore, VectorStoreQuery, VectorStoreQueryMode, VectorStoreQueryResult, } from "./types"; -import { - getTopKEmbeddings, - getTopKEmbeddingsLearner, - getTopKMMREmbeddings, -} from "../../Embedding"; -import { DEFAULT_PERSIST_DIR, DEFAULT_FS } from "../constants"; -import { NodeWithEmbedding } from "../../Node"; const LEARNER_MODES = new Set<VectorStoreQueryMode>([ VectorStoreQueryMode.SVM, @@ -41,7 +41,7 @@ export class SimpleVectorStore implements VectorStore { static async fromPersistDir( persistDir: string = DEFAULT_PERSIST_DIR, - fs: GenericFileSystem = DEFAULT_FS + fs: GenericFileSystem = DEFAULT_FS, ): Promise<SimpleVectorStore> { let persistPath = `${persistDir}/vector_store.json`; return await SimpleVectorStore.fromPersistPath(persistPath, fs); @@ -55,29 +55,28 @@ export class SimpleVectorStore implements VectorStore { return this.data.embeddingDict[textId]; } - async add(embeddingResults: NodeWithEmbedding[]): Promise<string[]> { - for (let result of embeddingResults) { - this.data.embeddingDict[result.node.id_] = result.embedding; + async add(embeddingResults: BaseNode[]): Promise<string[]> { + for (let node of embeddingResults) { + this.data.embeddingDict[node.id_] = node.getEmbedding(); - if (!result.node.sourceNode) { + if (!node.sourceNode) { console.error("Missing source node from TextNode."); continue; } - this.data.textIdToRefDocId[result.node.id_] = - result.node.sourceNode?.nodeId; + this.data.textIdToRefDocId[node.id_] = node.sourceNode?.nodeId; } if (this.persistPath) { await this.persist(this.persistPath, this.fs); } - return embeddingResults.map((result) => result.node.id_); + return embeddingResults.map((result) => result.id_); } async delete(refDocId: string): Promise<void> { let textIdsToDelete = Object.keys(this.data.textIdToRefDocId).filter( - (textId) => this.data.textIdToRefDocId[textId] === refDocId + (textId) => this.data.textIdToRefDocId[textId] === refDocId, ); for (let textId of textIdsToDelete) { delete this.data.embeddingDict[textId]; @@ -89,7 +88,7 @@ export class SimpleVectorStore implements VectorStore { async query(query: VectorStoreQuery): Promise<VectorStoreQueryResult> { if (!_.isNil(query.filters)) { throw new Error( - "Metadata filters not implemented for SimpleVectorStore yet." + "Metadata filters not implemented for SimpleVectorStore yet.", ); } @@ -115,7 +114,7 @@ export class SimpleVectorStore implements VectorStore { queryEmbedding, embeddings, query.similarityTopK, - nodeIds + nodeIds, ); } else if (query.mode === MMR_MODE) { let mmrThreshold = query.mmrThreshold; @@ -125,14 +124,14 @@ export class SimpleVectorStore implements VectorStore { null, query.similarityTopK, nodeIds, - mmrThreshold + mmrThreshold, ); } else if (query.mode === VectorStoreQueryMode.DEFAULT) { [topSimilarities, topIds] = getTopKEmbeddings( queryEmbedding, embeddings, query.similarityTopK, - nodeIds + nodeIds, ); } else { throw new Error(`Invalid query mode: ${query.mode}`); @@ -146,7 +145,7 @@ export class SimpleVectorStore implements VectorStore { async persist( persistPath: string = `${DEFAULT_PERSIST_DIR}/vector_store.json`, - fs?: GenericFileSystem + fs?: GenericFileSystem, ): Promise<void> { fs = fs || this.fs; let dirPath = path.dirname(persistPath); @@ -159,7 +158,7 @@ export class SimpleVectorStore implements VectorStore { static async fromPersistPath( persistPath: string, - fs?: GenericFileSystem + fs?: GenericFileSystem, ): Promise<SimpleVectorStore> { fs = fs || DEFAULT_FS; @@ -174,7 +173,7 @@ export class SimpleVectorStore implements VectorStore { dataDict = JSON.parse(fileData.toString()); } catch (e) { console.error( - `No valid data found at path: ${persistPath} starting new store.` + `No valid data found at path: ${persistPath} starting new store.`, ); } diff --git a/packages/core/src/storage/vectorStore/types.ts b/packages/core/src/storage/vectorStore/types.ts index 67b4afc98..24424c669 100644 --- a/packages/core/src/storage/vectorStore/types.ts +++ b/packages/core/src/storage/vectorStore/types.ts @@ -1,6 +1,5 @@ import { BaseNode } from "../../Node"; import { GenericFileSystem } from "../FileSystem"; -import { NodeWithEmbedding } from "../../Node"; export interface VectorStoreQueryResult { nodes?: BaseNode[]; @@ -62,7 +61,7 @@ export interface VectorStore { storesText: boolean; isEmbeddingQuery?: boolean; client(): any; - add(embeddingResults: NodeWithEmbedding[]): Promise<string[]>; + add(embeddingResults: BaseNode[]): Promise<string[]>; delete(refDocId: string, deleteKwargs?: any): Promise<void>; query(query: VectorStoreQuery, kwargs?: any): Promise<VectorStoreQueryResult>; persist(persistPath: string, fs?: GenericFileSystem): Promise<void>; -- GitLab