diff --git a/.gitignore b/.gitignore
index 1ef6f088dad1cb8e3730347fa6f9b8bf40fb20e4..217cb9a7c39fbd17b254dcf403763e02f6e9e102 100644
--- a/.gitignore
+++ b/.gitignore
@@ -140,3 +140,7 @@ dmypy.json
.idea
modules/
*.swp
+
+# pipenv
+Pipfile
+Pipfile.lock
\ No newline at end of file
diff --git a/docs/examples/vector_stores/RedisIndexDemo.ipynb b/docs/examples/vector_stores/RedisIndexDemo.ipynb
index 991590904f6597d65d324eea8981e2456619724f..6f39ae9f3376c825748949b730bfbe0c06ba408c 100644
--- a/docs/examples/vector_stores/RedisIndexDemo.ipynb
+++ b/docs/examples/vector_stores/RedisIndexDemo.ipynb
@@ -20,7 +20,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 31,
"id": "47264e32",
"metadata": {
"ExecuteTime": {
@@ -81,7 +81,7 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": 2,
"id": "0c9f4d21-145a-401e-95ff-ccb259e8ef84",
"metadata": {
"ExecuteTime": {
@@ -110,7 +110,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 32,
"id": "68cbd239-880e-41a3-98d8-dbb3fab55431",
"metadata": {
"ExecuteTime": {
@@ -126,7 +126,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Document ID: 09206095-be73-4069-b9f6-ff76d1f03343 Document Hash: 77ae91ab542f3abb308c4d7c77c9bc4c9ad0ccd63144802b7cbe7e1bb3a4094e\n"
+ "Document ID: faa23c94-ac9e-4763-92ba-e0f87bf38195 Document Hash: 77ae91ab542f3abb308c4d7c77c9bc4c9ad0ccd63144802b7cbe7e1bb3a4094e\n"
]
}
],
@@ -151,23 +151,35 @@
"```python\n",
"class RedisVectorStore(VectorStore):\n",
" \n",
- " def __init__(\n",
+ "def __init__(\n",
" self,\n",
- " index_name: Optional[str],\n",
- " index_prefix: Optional[str] = \"gpt_index\",\n",
+ " index_name: str,\n",
+ " index_prefix: str = \"llama_index\",\n",
" index_args: Optional[Dict[str, Any]] = None,\n",
- " redis_url: Optional[str] = \"redis://localhost:6379\",\n",
+ " metadata_fields: Optional[List[str]] = None,\n",
+ " redis_url: str = \"redis://localhost:6379\",\n",
" overwrite: bool = False,\n",
" **kwargs: Any,\n",
" ) -> None:\n",
" \"\"\"Initialize RedisVectorStore.\n",
"\n",
+ " For index arguments that can be passed to RediSearch, see\n",
+ " https://redis.io/docs/stack/search/reference/vectors/\n",
+ "\n",
+ " The index arguments will depend on the index type chosen. There\n",
+ " are two available index types\n",
+ " - FLAT: a flat index that uses brute force search\n",
+ " - HNSW: a hierarchical navigable small world graph index\n",
+ "\n",
" Args:\n",
" index_name (str): Name of the index.\n",
- " index_prefix (str): Prefix for the index. Defaults to \"gpt_index\".\n",
+ " index_prefix (str): Prefix for the index. Defaults to \"llama_index\".\n",
" index_args (Dict[str, Any]): Arguments for the index. Defaults to None.\n",
- " redis_url (str): URL for the redis instance. Defaults to \"redis://localhost:6379\".\n",
- " overwrite (bool): Whether to overwrite the index if it already exists. Defaults to False.\n",
+ " metadata_fields (List[str]): List of metadata fields to store in the index (only supports TAG fields).\n",
+ " redis_url (str): URL for the redis instance.\n",
+ " Defaults to \"redis://localhost:6379\".\n",
+ " overwrite (bool): Whether to overwrite the index if it already exists.\n",
+ " Defaults to False.\n",
" kwargs (Any): Additional arguments to pass to the redis client.\n",
"\n",
" Raises:\n",
@@ -175,12 +187,13 @@
" ValueError: If RediSearch is not installed\n",
"\n",
" Examples:\n",
- " >>> from gpt_index.vector_stores.redis import RedisVectorStore\n",
+ " >>> from llama_index.vector_stores.redis import RedisVectorStore\n",
" >>> # Create a RedisVectorStore\n",
" >>> vector_store = RedisVectorStore(\n",
" >>> index_name=\"my_index\",\n",
" >>> index_prefix=\"gpt_index\",\n",
- " >>> index_args={\"algorithm\": \"HNSW\", \"m\": 16, \"efConstruction\": 200, \"distance_metric\": \"cosine\"},\n",
+ " >>> index_args={\"algorithm\": \"HNSW\", \"m\": 16, \"ef_construction\": 200,\n",
+ " \"distance_metric\": \"cosine\"},\n",
" >>> redis_url=\"redis://localhost:6379/\",\n",
" >>> overwrite=True)\n",
"\n",
@@ -190,7 +203,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 33,
"id": "ba1558b3",
"metadata": {
"ExecuteTime": {
@@ -228,7 +241,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 34,
"id": "35369eda",
"metadata": {
"ExecuteTime": {
@@ -241,10 +254,11 @@
"name": "stdout",
"output_type": "stream",
"text": [
- " The author learned that the AI programs of the time were not capable of understanding natural\n",
- "language, and that the field of AI was a hoax. He also learned that he could make art, and that he\n",
- "could pass the entrance exam for the Accademia di Belli Arti in Florence. He also learned Lisp\n",
- "hacking and wrote his dissertation on applications of continuations.\n"
+ " The author learned that it is possible to publish essays online, and that working on things that\n",
+ "are not prestigious can be a sign that one is on the right track. They also learned that impure\n",
+ "motives can lead ambitious people astray, and that it is possible to make connections with people\n",
+ "through cleverly planned events. Finally, the author learned that they could find love through a\n",
+ "chance meeting at a party.\n"
]
}
],
@@ -256,7 +270,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 35,
"id": "99212d33",
"metadata": {
"ExecuteTime": {
@@ -269,10 +283,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
- " A hard moment for the author was when he realized that the AI programs of the time were a hoax and\n",
- "that there was an unbridgeable gap between what they could do and actually understanding natural\n",
- "language. He had invested a lot of time and energy into learning about AI and was disappointed to\n",
- "find out that the field was not as promising as he had thought.\n"
+ " A hard moment for the author was when he realized that he had been working on things that weren't\n",
+ "prestigious. He had been drawn to these types of work despite their lack of prestige, and he was\n",
+ "worried that his ambition was leading him astray. He was also concerned that people would give him a\n",
+ "\"glassy eye\" when he explained what he was writing.\n"
]
}
],
@@ -339,6 +353,7 @@
]
},
{
+ "attachments": {},
"cell_type": "markdown",
"id": "52b975a7",
"metadata": {},
@@ -350,17 +365,17 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 36,
"id": "6fe322f7",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "'09206095-be73-4069-b9f6-ff76d1f03343'"
+ "'faa23c94-ac9e-4763-92ba-e0f87bf38195'"
]
},
- "execution_count": 16,
+ "execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
@@ -372,7 +387,7 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 37,
"id": "ae4fb2b0",
"metadata": {},
"outputs": [
@@ -380,7 +395,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Number of documents 24\n"
+ "Number of documents 20\n"
]
}
],
@@ -391,7 +406,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 38,
"id": "0ce45788",
"metadata": {},
"outputs": [],
@@ -401,7 +416,7 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 39,
"id": "4a1ac683",
"metadata": {},
"outputs": [
@@ -409,7 +424,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Number of documents 14\n"
+ "Number of documents 10\n"
]
}
],
@@ -419,7 +434,7 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": 40,
"id": "c380605a",
"metadata": {},
"outputs": [],
@@ -431,7 +446,7 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": 41,
"id": "474ad4ee",
"metadata": {},
"outputs": [
@@ -446,6 +461,174 @@
"source": [
"print(\"Number of documents\", len(redis_client.keys()))"
]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "61b67496",
+ "metadata": {},
+ "source": [
+ "# Working with Metadata\n",
+ "\n",
+ "RedisVectorStore supports adding metadata and then using it in your queries (for example, to limit the scope of documents retrieved). However, there are a couple of important caveats:\n",
+ "1. Currently, only [Tag fields](https://redis.io/docs/stack/search/reference/tags/) are supported, and only with exact match.\n",
+ "2. You must declare the metadata when creating the index (usually when initializing RedisVectorStore). If you do not do this, your queries will come back empty. There is no way to modify an existing index after it had already been created (this is a Redis limitation).\n",
+ "\n",
+ "Here's how to work with Metadata:\n",
+ "\n",
+ "\n",
+ "### When **creating** the index\n",
+ "\n",
+ "Make sure to declare the metadata when you **first** create the index:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "id": "9889ec79",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "vector_store = RedisVectorStore(\n",
+ " index_name=\"pg_essays_with_metadata\",\n",
+ " index_prefix=\"llama\",\n",
+ " redis_url=\"redis://localhost:6379\",\n",
+ " overwrite=True,\n",
+ " metadata_fields=[\"user_id\", \"favorite_color\"],\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "f8d6dc21",
+ "metadata": {},
+ "source": [
+ "Note: the field names `text`, `doc_id`, `id` and the name of your vector field (`vector` by default) should **not** be used as metadata field names, as they are are reserved."
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "429947d5",
+ "metadata": {},
+ "source": [
+ "### When adding a document\n",
+ "\n",
+ "Add your metadata under the `extra_info` key. You can add metadata to documents you load in just by looping over them:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "id": "89781b7d",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Document ID: 6a5aa8dd-2771-454b-befc-bcfc311d2008 Document Hash: 77ae91ab542f3abb308c4d7c77c9bc4c9ad0ccd63144802b7cbe7e1bb3a4094e Metadata: {'user_id': '12345', 'favorite_color': 'blue'}\n"
+ ]
+ }
+ ],
+ "source": [
+ "# load your documents normally, then add your metadata\n",
+ "documents = SimpleDirectoryReader('../data/paul_graham').load_data()\n",
+ "\n",
+ "for document in documents:\n",
+ " document.extra_info = {\"user_id\": \"12345\", \"favorite_color\": \"blue\"}\n",
+ "\n",
+ "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+ "index = GPTVectorStoreIndex.from_documents(documents, storage_context=storage_context)\n",
+ " \n",
+ "# load documents\n",
+ "print('Document ID:', documents[0].doc_id, 'Document Hash:', documents[0].doc_hash, 'Metadata:', documents[0].extra_info)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "42b24e76",
+ "metadata": {},
+ "source": [
+ "### When querying the index\n",
+ "\n",
+ "To filter by your metadata fields, include one or more of your metadata keys, like so:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "id": "0b01f346",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " The author learned that it was possible to publish anything online, and that working on things that\n",
+ "weren't prestigious could lead to discovering something real. They also learned that impure motives\n",
+ "were a big danger for the ambitious, and that it was possible for programs not to terminate.\n",
+ "Finally, they learned that computers were expensive in those days, and that they could write\n",
+ "programs on the IBM 1401.\n"
+ ]
+ }
+ ],
+ "source": [
+ "from llama_index.vector_stores.types import MetadataFilters, ExactMatchFilter\n",
+ "\n",
+ "query_engine = index.as_query_engine(\n",
+ " similarity_top_k=3,\n",
+ " filters=MetadataFilters(\n",
+ " filters=[\n",
+ " ExactMatchFilter(key=\"user_id\", value=\"12345\"),\n",
+ " ExactMatchFilter(key=\"favorite_color\", value=\"blue\")\n",
+ " ]\n",
+ " ),\n",
+ ")\n",
+ "\n",
+ "response = query_engine.query(\"What did the author learn?\")\n",
+ "print(textwrap.fill(str(response), 100))"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "07514f85",
+ "metadata": {},
+ "source": [
+ "## Troubleshooting\n",
+ "\n",
+ "In case you run into issues retrieving your documents from the index, you might get a message similar to this.\n",
+ "```\n",
+ "No docs found on index 'pg_essays' with prefix 'llama' and filters '(@user_id:{12345} & @favorite_color:{blue})'.\n",
+ "* Did you originally create the index with a different prefix?\n",
+ "* Did you index your metadata fields when you created the index?\n",
+ "```\n",
+ "\n",
+ "If you get this error, there a couple of gotchas to be aware of when working with Redis:\n",
+ "#### Prefix issues\n",
+ "\n",
+ "If you first create your index with a specific `prefix` but later change that prefix in your code, your query will come back empty. Redis saves the prefix your originally created your index with and expects it to be consistent.\n",
+ "\n",
+ "To see what prefix your index was created with, you can run `FT.INFO <name of your index>` in the Redis CLI and look under `index_definition` => `prefixes`.\n",
+ "\n",
+ "#### Empty queries when using metadata\n",
+ "\n",
+ "If you add metadata to the index *after* it has already been created and then try to query over that metadata, your queries will come back empty.\n",
+ "\n",
+ "Redis indexes fields upon index creation only (similar to how it indexes the prefixes, above).\n",
+ "\n",
+ "If you have an existing index and want to make sure it's dropped, you can run `FT.DROPINDEX <name of your index>` in the Redis CLI. Note that this will *not* drop your actual data."
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "c09d1199",
+ "metadata": {},
+ "source": []
}
],
"metadata": {
diff --git a/llama_index/vector_stores/redis.py b/llama_index/vector_stores/redis.py
index 0218345aad4c0a8bfca87e6e0c1bee706f297af9..741c3877be6df14109a009835d1c240e7be57048 100644
--- a/llama_index/vector_stores/redis.py
+++ b/llama_index/vector_stores/redis.py
@@ -3,9 +3,10 @@
An index that that is built on top of an existing vector store.
"""
import logging
-import fsspec
from typing import TYPE_CHECKING, Any, Dict, List, Optional
+import fsspec
+
from llama_index.data_structs.node import DocumentRelationship, Node
from llama_index.readers.redis.utils import (
TokenEscaper,
@@ -15,6 +16,7 @@ from llama_index.readers.redis.utils import (
get_redis_query,
)
from llama_index.vector_stores.types import (
+ MetadataFilters,
NodeWithEmbedding,
VectorStore,
VectorStoreQuery,
@@ -41,6 +43,7 @@ class RedisVectorStore(VectorStore):
index_name: str,
index_prefix: str = "llama_index",
index_args: Optional[Dict[str, Any]] = None,
+ metadata_fields: Optional[List[str]] = None,
redis_url: str = "redis://localhost:6379",
overwrite: bool = False,
**kwargs: Any,
@@ -59,6 +62,8 @@ class RedisVectorStore(VectorStore):
index_name (str): Name of the index.
index_prefix (str): Prefix for the index. Defaults to "llama_index".
index_args (Dict[str, Any]): Arguments for the index. Defaults to None.
+ metadata_fields (List[str]): List of metadata fields to store in the index
+ (only supports TAG fields).
redis_url (str): URL for the redis instance.
Defaults to "redis://localhost:6379".
overwrite (bool): Whether to overwrite the index if it already exists.
@@ -100,6 +105,7 @@ class RedisVectorStore(VectorStore):
self._prefix = index_prefix
self._index_name = index_name
self._index_args = index_args if index_args is not None else {}
+ self._metadata_fields = metadata_fields if metadata_fields is not None else []
self._overwrite = overwrite
self._vector_field = str(self._index_args.get("vector_field", "vector"))
self._vector_key = str(self._index_args.get("vector_key", "vector"))
@@ -201,21 +207,24 @@ class RedisVectorStore(VectorStore):
ValueError: If query.query_embedding is None.
redis.exceptions.RedisError: If there is an error querying the index.
redis.exceptions.TimeoutError: If there is a timeout querying the index.
+ ValueError: If no documents are found when querying the index.
"""
from redis.exceptions import RedisError
from redis.exceptions import TimeoutError as RedisTimeoutError
- # TODO: implement this
- if query.filters is not None:
- raise ValueError("Metadata filters not implemented for Redis yet.")
-
return_fields = ["id", "doc_id", "text", self._vector_key, "vector_score"]
+ filters = _to_redis_filters(query.filters) if query.filters is not None else "*"
+
+ _logger.info(f"Using filters: {filters}")
+
redis_query = get_redis_query(
return_fields=return_fields,
top_k=query.similarity_top_k,
vector_field=self._vector_field,
+ filters=filters,
)
+
if not query.query_embedding:
raise ValueError("Query embedding is required for querying.")
@@ -235,6 +244,14 @@ class RedisVectorStore(VectorStore):
_logger.error(f"Error querying {self._index_name}: {e}")
raise e
+ if len(results.docs) == 0:
+ raise ValueError(
+ f"No docs found on index '{self._index_name}' with "
+ f"prefix '{self._prefix}' and filters '{filters}'. "
+ "* Did you originally create the index with a different prefix? "
+ "* Did you index your metadata fields when you created the index?"
+ )
+
ids = []
nodes = []
scores = []
@@ -298,10 +315,18 @@ class RedisVectorStore(VectorStore):
]
# add vector field to list of index fields. Create lazily to allow user
# to specify index and search attributes in creation.
+
fields = default_fields + [
self._create_vector_field(self._vector_field, **self._index_args)
]
+ # add metadata fields to list of index fields or we won't be able to search them
+ for metadata_field in self._metadata_fields:
+ # TODO: allow addition of text fields as metadata
+ # TODO: make sure we're preventing overwriting other keys (e.g. text,
+ # doc_id, id, and other vector fields)
+ fields.append(TagField(metadata_field, sortable=False))
+
_logger.info(f"Creating index {self._index_name}")
self._redis_client.ft(self._index_name).create_index(
fields=fields,
@@ -387,13 +412,18 @@ class RedisVectorStore(VectorStore):
)
-def cast_metadata_types(mapping: Optional[Dict[str, Any]]) -> Dict[str, str]:
- metadata = {}
- if mapping:
- for key, value in mapping.items():
- try:
- metadata[str(key)] = str(value)
- except (TypeError, ValueError) as e:
- # warn the user and continue
- _logger.warning("Failed to cast metadata to string", e)
- return metadata
+# currently only supports exact tag match - {} denotes a tag
+# must create the index with the correct metadata field before using a field as a
+# filter, or it will return no results
+def _to_redis_filters(metadata_filters: MetadataFilters) -> str:
+ tokenizer = TokenEscaper()
+
+ filter_strings = []
+ for filter in metadata_filters.filters:
+ # adds quotes around the value to ensure that the filter is treated as an
+ # exact match
+ filter_string = "@%s:{%s}" % (filter.key, tokenizer.escape(str(filter.value)))
+ filter_strings.append(filter_string)
+
+ joined_filter_strings = " & ".join(filter_strings)
+ return f"({joined_filter_strings})"