diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1fe96968a982ad5c015836a09c922ed883956dcc..e216ad507ad338c1204982e9a4a2cf14e1712793 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -126,6 +126,7 @@ These serve as the main data store and retrieval engine for our vector index.
- [Pinecone](https://github.com/jerryjliu/llama_index/blob/main/llama_index/vector_stores/pinecone.py)
- [Faiss](https://github.com/jerryjliu/llama_index/blob/main/llama_index/vector_stores/faiss.py)
- [Chroma](https://github.com/jerryjliu/llama_index/blob/main/llama_index/vector_stores/chroma.py)
+- [DashVector](https://github.com/jerryjliu/llama_index/blob/main/llama_index/vector_stores/dashvector.py)
**Ideas**:
diff --git a/docs/community/integrations/vector_stores.md b/docs/community/integrations/vector_stores.md
index b2ad80612ff259aa887b1a6ebccf670a51b35ccd..01138c75a23ae1c88d685862d4854bfdc2f74f9b 100644
--- a/docs/community/integrations/vector_stores.md
+++ b/docs/community/integrations/vector_stores.md
@@ -32,6 +32,7 @@ as the storage backend for `VectorStoreIndex`.
- Redis (`RedisVectorStore`). [Installation](https://redis.io/docs/getting-started/installation/).
- Neo4j (`Neo4jVectorIndex`). [Installation](https://neo4j.com/docs/operations-manual/current/installation/).
- TimeScale (`TimescaleVectorStore`). [Installation](https://github.com/timescale/python-vector).
+- DashVector(`DashVectorStore`).[Installation](https://help.aliyun.com/document_detail/2510230.html).
A detailed API reference is [found here](/api_reference/indices/vector_store.rst).
@@ -486,6 +487,23 @@ vector_store = CognitiveSearchVectorStore(
)
```
+**DashVector**
+
+```python
+import dashvector
+from llama_index.vector_stores import DashVectorStore
+
+# init dashvector client
+client = dashvector.Client(api_key='your-dashvector-api-key')
+
+# creating a DashVector collection
+client.create("quickstart", dimension=1536)
+collection = client.get("quickstart")
+
+# construct vector store
+vector_store = DashVectorStore(collection)
+```
+
[Example notebooks can be found here](https://github.com/jerryjliu/llama_index/tree/main/docs/examples/vector_stores).
## Loading Data from Vector Stores using Data Connector
@@ -625,4 +643,5 @@ maxdepth: 1
../../examples/vector_stores/Neo4jVectorDemo.ipynb
../../examples/vector_stores/CognitiveSearchIndexDemo.ipynb
../../examples/vector_stores/Timescalevector.ipynb
+../../examples/vector_stores/DashvectorIndexDemo.ipynb
```
diff --git a/docs/core_modules/data_modules/storage/vector_stores.md b/docs/core_modules/data_modules/storage/vector_stores.md
index 61a19815be35e3e583173755fd1ebb75e6c040de..e9f251851e09180458724cfb6a0f97c00dfc244d 100644
--- a/docs/core_modules/data_modules/storage/vector_stores.md
+++ b/docs/core_modules/data_modules/storage/vector_stores.md
@@ -32,6 +32,7 @@ We are actively adding more integrations and improving feature coverage for each
| OpenSearch | self-hosted / cloud | ✓ | | ✓ | ✓ | |
| Neo4jVector | self-hosted / cloud | | | ✓ | ✓ | |
| Azure Cognitive Search | cloud | | ✓ | ✓ | ✓ | |
+| DashVector | cloud | ✓ | | ✓ | ✓ | |
| DynamoDB | cloud | | | ✓ | | |
| LanceDB | cloud | ✓ | | ✓ | ✓ | |
| Metal | cloud | ✓ | | ✓ | ✓ | |
@@ -79,5 +80,6 @@ maxdepth: 1
/examples/vector_stores/CognitiveSearchIndexDemo.ipynb
/examples/vector_stores/EpsillaIndexDemo.ipynb
/examples/vector_stores/Timescalevector.ipynb
+/examples/vector_stores/DashvectorIndexDemo.ipynb
/examples/vector_stores/TencentVectorDBIndexDemo.ipynb
```
diff --git a/docs/examples/data_connectors/DashvectorReaderDemo.ipynb b/docs/examples/data_connectors/DashvectorReaderDemo.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..92acbbcf18db95830328b6b4e5c9df5d43e106ff
--- /dev/null
+++ b/docs/examples/data_connectors/DashvectorReaderDemo.ipynb
@@ -0,0 +1,150 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "f3ca56f0-6ef1-426f-bac5-fd7c374d0f51",
+ "metadata": {},
+ "source": [
+ "# DashVector Reader"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b2bd3c59",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import logging\n",
+ "import sys\n",
+ "import os\n",
+ "\n",
+ "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n",
+ "logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e2f49003-b952-4b9b-b935-2941f9303773",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "api_key = os.environ[\"DASHVECTOR_API_KEY\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "262f990a-79c8-413a-9f3c-cd9a3c191307",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from llama_index.readers import DashVectorReader\n",
+ "\n",
+ "reader = DashVectorReader(api_key=api_key)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "53b49187-8477-436c-9718-5d2f8cc6fad0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "\n",
+ "# the id_to_text_map specifies a mapping from the ID specified in DashVector to your text.\n",
+ "id_to_text_map = {\n",
+ " \"id1\": \"text blob 1\",\n",
+ " \"id2\": \"text blob 2\",\n",
+ "}\n",
+ "\n",
+ "# the query_vector is an embedding representation of your query_vector\n",
+ "query_vector = [n1, n2, n3, ...]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a88be1c4-603f-48b9-ac64-10a219af4951",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# NOTE: Required args are index_name, id_to_text_map, vector.\n",
+ "# In addition, we can pass through the metadata filter that meet the SQL syntax.\n",
+ "# See the Python client: https://pypi.org/project/dashvector/ for more details.\n",
+ "documents = reader.load_data(\n",
+ " collection_name=\"quickstart\",\n",
+ " id_to_text_map=id_to_text_map,\n",
+ " top_k=3,\n",
+ " vector=query_vector,\n",
+ " filter=\"key = 'value'\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a4baf59e-fc97-4a1e-947f-354a6438ffa6",
+ "metadata": {},
+ "source": [
+ "### Create index "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "109d083e-f3b4-420b-886b-087c8cf3f98b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from llama_index.indices import ListIndex\n",
+ "from IPython.display import Markdown, display\n",
+ "\n",
+ "index = ListIndex.from_documents(documents)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e15b9177-9e94-4e4e-9a2e-cd3a288a7faf",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set Logging to DEBUG for more detailed outputs\n",
+ "query_engine = index.as_query_engine()\n",
+ "response = query_engine.query(\"<query_text>\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "67b50613-a589-4acf-ba16-10571b415268",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown(f\"<b>{response}</b>\"))"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/docs/examples/vector_stores/DashvectorIndexDemo.ipynb b/docs/examples/vector_stores/DashvectorIndexDemo.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..c53512bca0dcf657a08adbf1be801349308de8ab
--- /dev/null
+++ b/docs/examples/vector_stores/DashvectorIndexDemo.ipynb
@@ -0,0 +1,214 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "307804a3-c02b-4a57-ac0d-172c30ddc851",
+ "metadata": {},
+ "source": [
+ "# DashVector Vector Store"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d48af8e1",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import logging\n",
+ "import sys\n",
+ "import os\n",
+ "\n",
+ "logging.basicConfig(stream=sys.stdout, level=logging.INFO)\n",
+ "logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "f7010b1d-d1bb-4f08-9309-a328bb4ea396",
+ "metadata": {},
+ "source": [
+ "#### Creating a DashVector Collection"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0ce3143d-198c-4dd2-8e5a-c5cdf94f017a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import dashvector"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4ad14111-0bbb-4c62-906d-6d6253e0cdee",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "api_key = os.environ[\"DASHVECTOR_API_KEY\"]\n",
+ "client = dashvector.Client(api_key=api_key)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c2c90087-bdd9-4ca4-b06b-2af883559f88",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{\"code\": 0, \"message\": \"\", \"requests_id\": \"82b969d2-2568-4e18-b0dc-aa159b503c84\"}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# dimensions are for text-embedding-ada-002\n",
+ "client.create(\"llama-demo\", dimension=1536)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "667f3cb3-ce18-48d5-b9aa-bfc1a1f0f0f6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dashvector_collection = client.get(\"quickstart\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "8ee4473a-094f-4d0a-a825-e1213db07240",
+ "metadata": {},
+ "source": [
+ "#### Load documents, build the DashVectorStore and VectorStoreIndex"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0a2bcc07",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "INFO:numexpr.utils:Note: NumExpr detected 12 cores but \"NUMEXPR_MAX_THREADS\" not set, so enforcing safe limit of 8.\n",
+ "Note: NumExpr detected 12 cores but \"NUMEXPR_MAX_THREADS\" not set, so enforcing safe limit of 8.\n",
+ "INFO:numexpr.utils:NumExpr defaulting to 8 threads.\n",
+ "NumExpr defaulting to 8 threads.\n"
+ ]
+ }
+ ],
+ "source": [
+ "from llama_index import VectorStoreIndex, SimpleDirectoryReader\n",
+ "from llama_index.vector_stores import DashVectorStore\n",
+ "from IPython.display import Markdown, display"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "68cbd239-880e-41a3-98d8-dbb3fab55431",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# load documents\n",
+ "documents = SimpleDirectoryReader(\"../data/paul_graham\").load_data()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ba1558b3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# initialize without metadata filter\n",
+ "from llama_index.storage.storage_context import StorageContext\n",
+ "\n",
+ "vector_store = DashVectorStore(dashvector_collection)\n",
+ "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+ "index = VectorStoreIndex.from_documents(\n",
+ " documents, storage_context=storage_context\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "04304299-fc3e-40a0-8600-f50c3292767e",
+ "metadata": {},
+ "source": [
+ "#### Query Index"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "35369eda",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set Logging to DEBUG for more detailed outputs\n",
+ "query_engine = index.as_query_engine()\n",
+ "response = query_engine.query(\"What did the author do growing up?\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bedbb693-725f-478f-be26-fa7180ea38b2",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/markdown": [
+ "<b>The author worked on writing and programming outside of school. They wrote short stories and tried writing programs on the IBM 1401 computer. They also built a microcomputer and started programming on it, writing simple games and a word processor.</b>"
+ ],
+ "text/plain": [
+ "<IPython.core.display.Markdown object>"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "display(Markdown(f\"<b>{response}</b>\"))"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/llama_index/data_structs/struct_type.py b/llama_index/data_structs/struct_type.py
index bdf399c9105fab294383c6496040475ba25212ab..342bd4cf86d706d0991fbac2c47d7f2065f4ebdf 100644
--- a/llama_index/data_structs/struct_type.py
+++ b/llama_index/data_structs/struct_type.py
@@ -53,6 +53,9 @@ class IndexStructType(str, Enum):
SQL ("SQL"): SQL Structured Store Index.
See :ref:`Ref-Indices-StructStore`
for more information on the SQL vector store index.
+ DASHVECTOR ("dashvector"): DashVector Vector Store Index.
+ See :ref:`Ref-Indices-VectorStore`
+ for more information on the Dashvecotor vector store index.
KG ("kg"): Knowledge Graph index.
See :ref:`Ref-Indices-Knowledge-Graph` for KG indices.
DOCUMENT_SUMMARY ("document_summary"): Document Summary Index.
@@ -80,6 +83,7 @@ class IndexStructType(str, Enum):
MYSCALE = "myscale"
VECTOR_STORE = "vector_store"
OPENSEARCH = "opensearch"
+ DASHVECTOR = "dashvector"
CHATGPT_RETRIEVAL_PLUGIN = "chatgpt_retrieval_plugin"
DEEPLAKE = "deeplake"
EPSILLA = "epsilla"
diff --git a/llama_index/readers/__init__.py b/llama_index/readers/__init__.py
index 97cf401ffa66c5f38e3f489391bb7d76dffc2a80..f2d43665f276d0b9a14185abc7dd6b1242e13e3b 100644
--- a/llama_index/readers/__init__.py
+++ b/llama_index/readers/__init__.py
@@ -13,6 +13,7 @@ definition of a Document - the bare minimum is a `text` property.
from llama_index.readers.bagel import BagelReader
from llama_index.readers.chatgpt_plugin import ChatGPTRetrievalPluginReader
from llama_index.readers.chroma import ChromaReader
+from llama_index.readers.dashvector import DashVectorReader
from llama_index.readers.deeplake import DeepLakeReader
from llama_index.readers.discord_reader import DiscordReader
from llama_index.readers.elasticsearch import ElasticsearchReader
@@ -90,4 +91,5 @@ __all__ = [
"BagelReader",
"HTMLTagReader",
"PDFReader",
+ "DashVectorReader",
]
diff --git a/llama_index/readers/dashvector.py b/llama_index/readers/dashvector.py
new file mode 100644
index 0000000000000000000000000000000000000000..e431be38f7c4db56655860af592285d8ab460281
--- /dev/null
+++ b/llama_index/readers/dashvector.py
@@ -0,0 +1,85 @@
+"""DashVector reader."""
+
+from typing import Dict, List, Optional
+
+from llama_index.readers.base import BaseReader
+from llama_index.schema import Document
+
+
+class DashVectorReader(BaseReader):
+ """DashVector reader.
+
+ Args:
+ api_key (str): DashVector API key.
+ """
+
+ def __init__(self, api_key: str):
+ """Initialize with parameters."""
+ try:
+ import dashvector
+ except ImportError:
+ raise ImportError(
+ "`dashvector` package not found, please run `pip install dashvector`"
+ )
+
+ self._api_key = api_key
+ self._client = dashvector.Client(api_key=api_key)
+
+ def load_data(
+ self,
+ collection_name: str,
+ id_to_text_map: Dict[str, str],
+ vector: Optional[List[float]],
+ top_k: int,
+ separate_documents: bool = True,
+ filter: Optional[str] = None,
+ include_vector: bool = True,
+ ) -> List[Document]:
+ """Load data from DashVector.
+
+ Args:
+ collection_name (str): Name of the collection.
+ id_to_text_map (Dict[str, str]): A map from ID's to text.
+ separate_documents (Optional[bool]): Whether to return separate
+ documents per retrieved entry. Defaults to True.
+ vector (List[float]): Query vector.
+ top_k (int): Number of results to return.
+ filter (Optional[str]): doc fields filter conditions that meet the SQL
+ where clause specification.
+ include_vector (bool): Whether to include the embedding in the response.
+ Defaults to True.
+
+ Returns:
+ List[Document]: A list of documents.
+ """
+ collection = self._client.get(collection_name)
+ if not collection:
+ raise ValueError(
+ f"Failed to get collection: {collection_name}," f"Error: {collection}"
+ )
+
+ resp = collection.query(
+ vector=vector,
+ topk=top_k,
+ filter=filter,
+ include_vector=include_vector,
+ )
+ if not resp:
+ raise Exception(f"Failed to query document," f"Error: {resp}")
+
+ documents = []
+ for doc in resp:
+ if doc.id not in id_to_text_map:
+ raise ValueError("ID not found in id_to_text_map.")
+ text = id_to_text_map[doc.id]
+ embedding = doc.vector
+ if len(embedding) == 0:
+ embedding = None
+ documents.append(Document(text=text, embedding=embedding))
+
+ if not separate_documents:
+ text_list = [doc.get_content() for doc in documents]
+ text = "\n\n".join(text_list)
+ documents = [Document(text=text)]
+
+ return documents
diff --git a/llama_index/vector_stores/__init__.py b/llama_index/vector_stores/__init__.py
index 2fb2e8faeb06e4506121142f4bc5f76f42e67a66..b57ae6f3052e38caee1c54203f8a550a4c6fe726 100644
--- a/llama_index/vector_stores/__init__.py
+++ b/llama_index/vector_stores/__init__.py
@@ -7,6 +7,7 @@ from llama_index.vector_stores.cassandra import CassandraVectorStore
from llama_index.vector_stores.chatgpt_plugin import ChatGPTRetrievalPluginClient
from llama_index.vector_stores.chroma import ChromaVectorStore
from llama_index.vector_stores.cogsearch import CognitiveSearchVectorStore
+from llama_index.vector_stores.dashvector import DashVectorStore
from llama_index.vector_stores.deeplake import DeepLakeVectorStore
from llama_index.vector_stores.docarray import (
DocArrayHnswVectorStore,
@@ -77,5 +78,6 @@ __all__ = [
"VectorStoreQuery",
"VectorStoreQueryResult",
"MetadataFilters",
+ "DashVectorStore",
"TencentVectorDB",
]
diff --git a/llama_index/vector_stores/dashvector.py b/llama_index/vector_stores/dashvector.py
new file mode 100644
index 0000000000000000000000000000000000000000..eddfccd045c47608c62a8529e36c222ee072108e
--- /dev/null
+++ b/llama_index/vector_stores/dashvector.py
@@ -0,0 +1,166 @@
+"""DashVector Vector Store."""
+import logging
+from typing import Any, List, Optional, cast
+
+from llama_index.schema import BaseNode, TextNode
+from llama_index.vector_stores.types import (
+ MetadataFilters,
+ VectorStore,
+ VectorStoreQuery,
+ VectorStoreQueryResult,
+)
+from llama_index.vector_stores.utils import (
+ DEFAULT_DOC_ID_KEY,
+ DEFAULT_TEXT_KEY,
+ legacy_metadata_dict_to_node,
+ metadata_dict_to_node,
+ node_to_metadata_dict,
+)
+
+DEFAULT_BATCH_SIZE = 100
+logger = logging.getLogger(__name__)
+
+
+def _to_dashvector_filter(
+ standard_filters: Optional[MetadataFilters] = None,
+) -> Optional[str]:
+ """Convert from standard filter to dashvector filter dict."""
+ if standard_filters is None:
+ return None
+
+ filters = []
+ for filter in standard_filters.filters:
+ if isinstance(filter.value, str):
+ value = f"'{filter.value}'"
+ else:
+ value = f"{filter.value}"
+ filters.append(f"{filter.key} = {value}")
+ return " and ".join(filters)
+
+
+class DashVectorStore(VectorStore):
+ """Dash Vector Store.
+
+ In this vector store, embeddings and docs are stored within a
+ DashVector collection.
+
+ During query time, the index uses DashVector to query for the top
+ k most similar nodes.
+
+ Args:
+ collection (Optional[dashvector.Collection]): DashVector collection instance
+ """
+
+ stores_text: bool = True
+ flat_metadata: bool = True
+
+ def __init__(
+ self,
+ collection: Optional[Any] = None,
+ ) -> None:
+ """Initialize params."""
+ import_err_msg = (
+ "`dashvector` package not found, please run `pip install dashvector`"
+ )
+ try:
+ import dashvector
+ except ImportError:
+ raise ImportError(import_err_msg)
+
+ if collection is not None:
+ self._collection = cast(dashvector.Collection, collection)
+
+ def add(
+ self,
+ nodes: List[BaseNode],
+ ) -> List[str]:
+ """Add nodes to vector store.
+
+ Args:
+ nodes (List[BaseNode]): list of nodes with embeddings
+ """
+ from dashvector import Doc
+
+ for i in range(0, len(nodes), DEFAULT_BATCH_SIZE):
+ # batch end
+ end = min(i + DEFAULT_BATCH_SIZE, len(nodes))
+ docs = [
+ Doc(
+ id=node.node_id,
+ vector=node.embedding,
+ fields=node_to_metadata_dict(
+ node, remove_text=False, flat_metadata=self.flat_metadata
+ ),
+ )
+ for node in nodes[i:end]
+ ]
+
+ resp = self._collection.upsert(docs)
+ if not resp:
+ raise Exception(f"Failed to upsert docs, error: {resp}")
+
+ return [node.node_id for node in nodes]
+
+ def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None:
+ """
+ Delete nodes using with ref_doc_id.
+
+ Args:
+ ref_doc_id (str): The doc_id of the document to delete.
+
+ """
+ filter = f"{DEFAULT_DOC_ID_KEY}='{ref_doc_id}'"
+ resp = self._collection.query(filter=filter)
+ if not resp:
+ raise Exception(f"Failed to query doc by {filter}")
+
+ self._collection.delete(ids=[doc.id for doc in resp])
+
+ def query(
+ self,
+ query: VectorStoreQuery,
+ **kwargs: Any,
+ ) -> VectorStoreQueryResult:
+ """Query vector store."""
+ query_embedding = (
+ [float(e) for e in query.query_embedding] if query.query_embedding else []
+ )
+ filter = _to_dashvector_filter(query.filters)
+ rsp = self._collection.query(
+ vector=query_embedding,
+ topk=query.similarity_top_k,
+ filter=filter,
+ include_vector=True,
+ )
+ if not rsp:
+ raise Exception(f"Failed to query docs, error: {rsp}")
+
+ top_k_ids = []
+ top_k_nodes = []
+ top_k_scores = []
+ for doc in rsp:
+ try:
+ node = metadata_dict_to_node(doc.fields)
+ except Exception:
+ # NOTE: deprecated legacy logic for backward compatibility
+ logger.debug("Failed to parse Node metadata, fallback to legacy logic.")
+ metadata, node_info, relationships = legacy_metadata_dict_to_node(
+ doc.fields
+ )
+
+ text = doc.fields[DEFAULT_TEXT_KEY]
+ node = TextNode(
+ id_=doc.id,
+ text=text,
+ metadata=metadata,
+ start_char_idx=node_info.get("start", None),
+ end_char_idx=node_info.get("end", None),
+ relationships=relationships,
+ )
+ top_k_ids.append(doc.id)
+ top_k_nodes.append(node)
+ top_k_scores.append(doc.score)
+
+ return VectorStoreQueryResult(
+ nodes=top_k_nodes, similarities=top_k_scores, ids=top_k_ids
+ )
diff --git a/pyproject.toml b/pyproject.toml
index 76590a1310c5eac277113f60dd2f78ef930c9d82..0ec041ff66a57b48140dac6fae187b12722e9afb 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -254,6 +254,7 @@ betabageldb = {optional = true, version = "^0.2.34"}
cassio = {optional = true, version = "^0.1.3"}
chromadb = {optional = true, version = ">=0.4.0"}
clickhouse-connect = {optional = true, version = "^0.6.14"}
+dashvector = {optional = true, version = "^1.0.5"}
deeplake = {optional = true, version = "^3.7.3"}
docarray = {optional = true, version = "^0.39.0"}
elasticsearch = {optional = true, version = "^8.10.0"}