Skip to content
Snippets Groups Projects
Unverified Commit e2c7441d authored by Doc Emmett Brown's avatar Doc Emmett Brown Committed by GitHub
Browse files

Add Reader for String Iterable (#173)

parent ab1aaf9d
Branches
Tags
No related merge requests found
.vscode/
.DS_Store
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
......@@ -129,6 +130,3 @@ dmypy.json
# Pyre type checker
.pyre/
# OS generated files
.DS_Store
......@@ -8,4 +8,7 @@ lint:
mypy .
black . --check
isort . --check
flake8 .
\ No newline at end of file
flake8 .
test:
pytest tests
\ No newline at end of file
......@@ -58,6 +58,7 @@ from gpt_index.readers import (
SimpleDirectoryReader,
SimpleMongoReader,
SlackReader,
StringIterableReader,
WeaviateReader,
WikipediaReader,
)
......@@ -92,6 +93,7 @@ __all__ = [
"NotionPageReader",
"GoogleDocsReader",
"SlackReader",
"StringIterableReader",
"WeaviateReader",
"FaissReader",
"PineconeReader",
......
......@@ -21,6 +21,7 @@ from gpt_index.readers.notion import NotionPageReader
from gpt_index.readers.pinecone import PineconeReader
from gpt_index.readers.schema.base import Document
from gpt_index.readers.slack import SlackReader
from gpt_index.readers.string_iterable import StringIterableReader
from gpt_index.readers.weaviate.reader import WeaviateReader
from gpt_index.readers.wikipedia import WikipediaReader
......@@ -36,4 +37,5 @@ __all__ = [
"PineconeReader",
"FaissReader",
"Document",
"StringIterableReader",
]
"""Simple reader that turns an iterable of strings into a list of Documents."""
from typing import Any, List
from gpt_index.readers.base import BaseReader
from gpt_index.readers.schema.base import Document
class StringIterableReader(BaseReader):
"""String Iterable Reader.
Gets a list of documents, given an iterable (e.g. list) of strings.
Example:
.. code-block:: python
from gpt_index import StringIterableReader, GPTTreeIndex
documents = StringIterableReader().load_data(
texts=["I went to the store", "I bought an apple"])
index = GPTTreeIndex(documents)
index.query("what did I buy?")
# response should be something like "You bought an apple."
"""
def load_data(self, **load_kwargs: Any) -> List[Document]:
"""Load the data."""
texts = load_kwargs.get("texts", list())
results = []
for text in texts:
results.append(Document(text))
return results
"""Test String Iterable Reader."""
from gpt_index.readers.string_iterable import StringIterableReader
def test_load() -> None:
"""Test loading data into StringIterableReader."""
reader = StringIterableReader()
documents = reader.load_data(texts=["I went to the store", "I bought an apple"])
assert len(documents) == 2
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment