Skip to content
Snippets Groups Projects
Commit 928a9f3b authored by Logan Markewich's avatar Logan Markewich
Browse files

add video 3 resources

parent 777ec2e4
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
3_eval_baseline/3_eval_basline.ipynb 0 → 100644
+ 933
− 0
View file @ 928a9f3b
%% Cell type:markdown id: tags:
# LlamaIndex Bottoms-Up Development - Evaluation Baseline
LlamaIndex provides some basic evaluation of query engines! We can setup an evaluator that will measure both hallucinations, as well as if the query was actually answered!
This is provided by two main evaluations:
- `ResponseSourceEvaluator` - uses an LLM to decide if the response is similar enough to the sources -- a good measure for hallunication detection!
- `QueryResponseEvaluator` - uses an LLM to decide if a response is similar enough to the original query -- a good measure for checking if the query was answered!
You may have noticed that we are using an LLM for this task. That means we will want to pick a powerful LLM, like GPT-4 or Claude-2.
Lastly, using these methods, we can also use the LLM to generate syntheic questions to evaluate with!
%% Cell type:markdown id: tags:
## Setup the Baseline Query Engine
%% Cell type:markdown id: tags:
### Loading our Docs
%% Cell type:code id: tags:
``` python
import openai
import os
os.environ["OPENAI_API_KEY"] = "API_KEY_HERE"
openai.api_key = os.environ["OPENAI_API_KEY"]
```
%% Cell type:code id: tags:
``` python
import os
import sys
sys.path.append(os.path.join(os.getcwd(), '..'))
```
%% Cell type:code id: tags:
``` python
from llama_docs_bot.markdown_docs_reader import MarkdownDocsReader
from llama_index import SimpleDirectoryReader
def load_markdown_docs(filepath):
"""Load markdown docs from a directory, excluding all other file types."""
loader = SimpleDirectoryReader(
input_dir=filepath,
required_exts=[".md"],
file_extractor={".md": MarkdownDocsReader()},
recursive=True
)
documents = loader.load_data()
# exclude some metadata from the LLM
for doc in documents:
doc.excluded_llm_metadata_keys = ["File Name", "Content Type", "Header Path"]
return documents
```
%% Cell type:code id: tags:
``` python
# load our documents from each folder.
# we keep them seperate for now, in order to create seperate indexes later
getting_started_docs = load_markdown_docs("../docs/getting_started")
community_docs = load_markdown_docs("../docs/community")
data_docs = load_markdown_docs("../docs/core_modules/data_modules")
agent_docs = load_markdown_docs("../docs/core_modules/agent_modules")
model_docs = load_markdown_docs("../docs/core_modules/model_modules")
query_docs = load_markdown_docs("../docs/core_modules/query_modules")
supporting_docs = load_markdown_docs("../docs/core_modules/supporting_modules")
tutorials_docs = load_markdown_docs("../docs/end_to_end_tutorials")
contributing_docs = load_markdown_docs("../docs/development")
```
%% Cell type:markdown id: tags:
### Create the indicies
%% Cell type:code id: tags:
``` python
from llama_index import ServiceContext, set_global_service_context
from llama_index.llms import OpenAI
# create a global service context
service_context = ServiceContext.from_defaults(llm=OpenAI(model="gpt-3.5-turbo", temperature=0))
set_global_service_context(service_context)
```
%% Cell type:code id: tags:
``` python
from llama_index import VectorStoreIndex, StorageContext, load_index_from_storage
# create a vector store index for each folder
try:
getting_started_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./getting_started_index"))
community_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./community_index"))
data_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./data_index"))
agent_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./agent_index"))
model_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./model_index"))
query_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./query_index"))
supporting_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./supporting_index"))
tutorials_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./tutorials_index"))
contributing_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./contributing_index"))
except:
getting_started_index = VectorStoreIndex.from_documents(getting_started_docs)
getting_started_index.storage_context.persist(persist_dir="./getting_started_index")
community_index = VectorStoreIndex.from_documents(community_docs)
community_index.storage_context.persist(persist_dir="./community_index")
data_index = VectorStoreIndex.from_documents(data_docs)
data_index.storage_context.persist(persist_dir="./data_index")
agent_index = VectorStoreIndex.from_documents(agent_docs)
agent_index.storage_context.persist(persist_dir="./agent_index")
model_index = VectorStoreIndex.from_documents(model_docs)
model_index.storage_context.persist(persist_dir="./model_index")
query_index = VectorStoreIndex.from_documents(query_docs)
query_index.storage_context.persist(persist_dir="./query_index")
supporting_index = VectorStoreIndex.from_documents(supporting_docs)
supporting_index.storage_context.persist(persist_dir="./supporting_index")
tutorials_index = VectorStoreIndex.from_documents(tutorials_docs)
tutorials_index.storage_context.persist(persist_dir="./tutorials_index")
contributing_index = VectorStoreIndex.from_documents(contributing_docs)
contributing_index.storage_context.persist(persist_dir="./contributing_index")
```
%% Cell type:markdown id: tags:
### Create Query Engine Tools
Since we have so many indicies, we can create a query engine tool for each and then use them in a single query engine!
%% Cell type:code id: tags:
``` python
from llama_index.tools import QueryEngineTool
# create a query engine tool for each folder
getting_started_tool = QueryEngineTool.from_defaults(
query_engine=getting_started_index.as_query_engine(),
name="Getting Started",
description="Useful for answering questions about installing and running llama index, as well as basic explanations of how llama index works."
)
community_tool = QueryEngineTool.from_defaults(
query_engine=community_index.as_query_engine(),
name="Community",
description="Useful for answering questions about integrations and other apps built by the community."
)
data_tool = QueryEngineTool.from_defaults(
query_engine=data_index.as_query_engine(),
name="Data Modules",
description="Useful for answering questions about data loaders, documents, nodes, and index structures."
)
agent_tool = QueryEngineTool.from_defaults(
query_engine=agent_index.as_query_engine(),
name="Agent Modules",
description="Useful for answering questions about data agents, agent configurations, and tools."
)
model_tool = QueryEngineTool.from_defaults(
query_engine=model_index.as_query_engine(),
name="Model Modules",
description="Useful for answering questions about using and configuring LLMs, embedding modles, and prompts."
)
query_tool = QueryEngineTool.from_defaults(
query_engine=query_index.as_query_engine(),
name="Query Modules",
description="Useful for answering questions about query engines, query configurations, and using various parts of the query engine pipeline."
)
supporting_tool = QueryEngineTool.from_defaults(
query_engine=supporting_index.as_query_engine(),
name="Supporting Modules",
description="Useful for answering questions about supporting modules, such as callbacks, service context, and avaluation."
)
tutorials_tool = QueryEngineTool.from_defaults(
query_engine=tutorials_index.as_query_engine(),
name="Tutorials",
description="Useful for answering questions about end-to-end tutorials and giving examples of specific use-cases."
)
contributing_tool = QueryEngineTool.from_defaults(
query_engine=contributing_index.as_query_engine(),
name="Contributing",
description="Useful for answering questions about contributing to llama index, including how to contribute to the codebase and how to build documentation."
)
```
%% Cell type:markdown id: tags:
### Create Unified Query Engine
%% Cell type:code id: tags:
``` python
# needed for notebooks
import nest_asyncio
nest_asyncio.apply()
from llama_index.query_engine import SubQuestionQueryEngine
from llama_index.response_synthesizers import get_response_synthesizer
query_engine = SubQuestionQueryEngine.from_defaults(
query_engine_tools=[
getting_started_tool,
community_tool,
data_tool,
agent_tool,
model_tool,
query_tool,
supporting_tool,
tutorials_tool,
contributing_tool
],
# enable this for streaming
# response_synthesizer=get_response_synthesizer(streaming=True),
verbose=False
)
```
%% Cell type:markdown id: tags:
### Test the Query Engine!
%% Cell type:code id: tags:
``` python
response = query_engine.query("How do I install llama index?")
print(str(response))
```
%% Output
To install LlamaIndex, you can follow these steps:
1. Install the package using pip by running the command: `pip install llama-index`.
2. Clone the repository using Git by running the command: `git clone https://github.com/jerryjliu/llama_index.git`.
3. If you want to do an editable install of just the package itself, run the command: `pip install -e .`.
4. If you want to install optional dependencies and dependencies used for development, run the command: `pip install -r requirements.txt`.
Please note that these steps assume you have pip and Git installed on your system.
%% Cell type:markdown id: tags:
## Evaluate the Basline!
Now that we have our baseline query engine created, we can create a basic evaluation pipeline!
Our pipeline will:
- Generate a small dataset of questions
- Save/cache these questions (so we can properly compare performance later!)
- Evaluate both response quality and hallucination
To do this reliably, we need to use an LLM smarter than `gpt-3.5-turbo`, so we will setup `gpt-4` for the evaluation process!
%% Cell type:markdown id: tags:
### Generate the Dataset
In order to make the question generation more effecient, we can remove small documents and combine all documents into a giant single docoument.
I also modify the question generation prompt, to generate a single question for each chunk, along with extra context for what it is reading.
%% Cell type:code id: tags:
``` python
from llama_index import Document
documents = SimpleDirectoryReader("../docs", recursive=True, required_exts=[".md"]).load_data()
all_text = ""
for doc in documents:
all_text += doc.text
giant_document = Document(text=all_text)
```
%% Cell type:code id: tags:
``` python
import os
import random
random.seed(42)
from llama_index import ServiceContext
from llama_index.prompts import Prompt
from llama_index.llms import OpenAI
from llama_index.evaluation import DatasetGenerator
gpt4_service_context = ServiceContext.from_defaults(llm=OpenAI(llm="gpt-4", temperature=0))
question_dataset = []
if os.path.exists("question_dataset.txt"):
with open("question_dataset.txt", "r") as f:
for line in f:
question_dataset.append(line.strip())
else:
# generate questions
data_generator = DatasetGenerator.from_documents(
[giant_document],
text_question_template=Prompt(
"A sample from the LlamaIndex documentation is below.\n"
"---------------------\n"
"{context_str}\n"
"---------------------\n"
"Using the documentation sample, carefully follow the instructions below:\n"
"{query_str}"
),
question_gen_query=(
"You are an evaluator for a search pipeline. Your task is to write a single question "
"using the provided documentation sample above to test the search pipeline. The question should "
"reference specific names, functions, and terms. Restrict the question to the "
"context information provided.\n"
"Question: "
),
# set this to be low, so we can generate more questions
service_context=gpt4_service_context
)
generated_questions = data_generator.generate_questions_from_nodes()
# randomly pick 40 questions from each dataset
generated_questions = random.sample(generated_questions, 40)
question_dataset.extend(generated_questions)
print(f"Generated {len(question_dataset)} questions.")
# save the questions!
with open("question_dataset.txt", "w") as f:
for question in question_dataset:
f.write(f"{question.strip()}\n")
```
%% Cell type:code id: tags:
``` python
print(random.sample(question_dataset, 5))
```
%% Output
['What is the function used to specify the metadata visible to the embedding model and how can it be customized?', 'How can I convert tools to LangChain tools using the provided documentation sample?', 'What is the purpose of the "router query engine" in the LlamaIndex framework?', 'What are the different vector stores supported by LlamaIndex for use as the storage backend for `VectorStoreIndex`?', 'What is the default number of LLM calls required for the ListIndex?']
%% Cell type:markdown id: tags:
### Evaluate with the Dataset
Now that we have our dataset, let's measure performance!
%% Cell type:markdown id: tags:
#### Evaluating Response for Hallucination
%% Cell type:code id: tags:
``` python
import time
import asyncio
import nest_asyncio
nest_asyncio.apply()
from llama_index import Response
def evaluate_query_engine(evaluator, query_engine, questions):
async def run_query(query_engine, q):
try:
return await query_engine.aquery(q)
except:
return Response(response="Error, query failed.")
total_correct = 0
all_results = []
for batch_size in range(0, len(questions), 5):
batch_qs = questions[batch_size:batch_size+5]
tasks = [run_query(query_engine, q) for q in batch_qs]
responses = asyncio.run(asyncio.gather(*tasks))
print(f"finished batch {(batch_size // 5) + 1} out of {len(questions) // 5}")
for response in responses:
eval_result = 1 if "YES" in evaluator.evaluate(response) else 0
total_correct += eval_result
all_results.append(eval_result)
# helps avoid rate limits
time.sleep(1)
return total_correct, all_results
```
%% Cell type:code id: tags:
``` python
from llama_index.evaluation import ResponseEvaluator
# gpt-4 evaluator!
evaluator = ResponseEvaluator(service_context=gpt4_service_context)
total_correct, all_results = evaluate_query_engine(evaluator, query_engine, question_dataset)
print(f"Hallucination? Scored {total_correct} out of {len(question_dataset)} questions correctly.")
```
%% Output
finished batch 1 out of 8
finished batch 2 out of 8
finished batch 3 out of 8
finished batch 4 out of 8
finished batch 5 out of 8
finished batch 6 out of 8
finished batch 7 out of 8
finished batch 8 out of 8
Hallucination? Scored 29 out of 40 questions correctly.
%% Cell type:markdown id: tags:
#### Investigating Hallucinations
%% Cell type:code id: tags:
``` python
import numpy as np
hallucinated_questions = np.array(question_dataset)[np.array(all_results) == 0]
print(hallucinated_questions)
```
%% Output
['What is the purpose of the `GuidancePydanticProgram` class in the LlamaIndex documentation?'
'What is the purpose of the SubQuestionQueryEngine class in LlamaIndex?'
'What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?'
'What are the different vector stores supported by LlamaIndex for use as the storage backend for `VectorStoreIndex`?'
'What is the purpose of the `RefinePrompt` class in the LlamaIndex documentation?'
'What is the purpose of the Algovera tool built on top of LlamaIndex?'
'What are the three primary sections within the layout of the ChatView component?'
'What is the purpose of the `insert_terms` function in the LlamaIndex app?'
'What is the purpose of the `load_collection_model` function in the LlamaIndex documentation?'
'What is the purpose of the SQLTableNodeMapping object in the LlamaIndex documentation sample?'
'What is the purpose of the `RouterQueryEngine` in LlamaIndex and how can it be used in the search pipeline?']
%% Cell type:code id: tags:
``` python
response = query_engine.query('What is the purpose of the `GuidancePydanticProgram` class in the LlamaIndex documentation?')
print(str(response))
print("-----------------")
print(response.get_formatted_sources(length=1000))
```
%% Output
Based on the given context information, there is no specific mention of the "GuidancePydanticProgram" class in the LlamaIndex documentation. Therefore, it is not possible to determine the purpose of this class without further information.
-----------------
> Source (Doc id: bbf2dedc-90fb-49e8-be57-28a2783ea7c1): Sub question: What is the purpose of the GuidancePydanticProgram class?
Response: Based on the given context information, there is no specific mention of the "GuidancePydanticProgram" class. Therefore, it is not possible to determine the purpose of this class without further information....
> Source (Doc id: b393e1a2-75d2-4017-bd12-d3b536aa3c4a): Sub question: How do I install and run LlamaIndex?
Response: To install LlamaIndex, you can use the pip package manager by running the command "pip install llama-index" in your terminal or command prompt.
After installing LlamaIndex, you can use the following starter example to run it:
1. Import the LlamaIndex module in your Python script:
```python
import llama_index
```
2. Use the LlamaIndex functions and classes as needed in your script.
Make sure you have Python and pip installed on your system before running the installation command....
> Source (Doc id: a9c75753-964e-4fb3-9798-b6b8ba6c044b): Sub question: What are the basic explanations of how LlamaIndex works?
Response: LlamaIndex is a platform that allows you to create LLM-powered applications using custom data. The platform follows the retrieval augmented generation (RAG) paradigm, which combines LLM (Language Model) with custom data.
The basic explanation of how LlamaIndex works is by using LLM to generate responses based on the given input. It does this by retrieving relevant information from the custom data and then using LLM to generate a response that is relevant to the input.
To compose your own RAG pipeline in LlamaIndex, you need to understand key concepts and modules. These include understanding how to retrieve information from the custom data, how to use LLM for generation, and how to combine these components effectively to create your desired application.
Overall, LlamaIndex provides a framework for building LLM-powered applications by combining LLM with custom data, allowing you to create applications...
> Source (Doc id: 60e1c688-4acc-428b-9282-240d0c7092df): Sub question: Are there any integrations or other apps built by the community for LlamaIndex?
Response: Yes, there are integrations and other apps built by the community for LlamaIndex....
> Source (Doc id: 8880918b-13f3-4695-b805-bb2d99df393b): Sub question: What are data loaders, documents, nodes, and index structures in LlamaIndex?
Response: In LlamaIndex, data loaders are components that are responsible for ingesting and loading external data into the system. They handle the process of converting the external data into LlamaIndex's internal representation, which consists of documents and nodes.
Documents in LlamaIndex refer to the ingested data, which can be text or any other form of content. These documents are internally parsed and chunked into smaller units called nodes. Nodes represent chunks of text from the documents and serve as the basic building blocks for indexing and querying.
Index structures in LlamaIndex are the data structures used to organize and store the index metadata. The index metadata includes information about the nodes, such as their location and properties, which enables efficient retrieval and querying of the data. LlamaIndex supports customizable index stores, allowing users to choose the st...
> Source (Doc id: adb3de98-42f7-4ea9-8c53-67c14aa30000): Sub question: How do I work with data agents, agent configurations, and tools in LlamaIndex?
Response: To work with data agents, agent configurations, and tools in LlamaIndex, you need to understand the core components and functionalities involved.
Data agents in LlamaIndex are knowledge workers powered by LLM (Llama Language Model). They can perform various tasks on your data, both in a "read" and "write" function. They are capable of automated search and retrieval of different types of data, including unstructured, semi-structured, and structured data. Additionally, they can call external service APIs in a structured manner and process the response, storing it for later use.
To build a data agent, you need two core components: a reasoning loop and tool abstractions. The reasoning loop helps the agent decide which tools to use, in what sequence, and with what parameters, based on the input task. The tool abstractions represent the APIs or tools that the agent can interact with. T...
> Source (Doc id: e274a384-c44f-429a-9f4b-044b7f80ed2f): Sub question: What is the purpose of LLMs, embedding models, and prompts in LlamaIndex?
Response: The purpose of LLMs (Large Language Models) in LlamaIndex is to provide expressive power and enhance the functionality of the framework. LLMs can be used as standalone modules or integrated into other core LlamaIndex components. They are primarily used during the response synthesis step, but depending on the type of index being used, they may also be utilized during index construction, insertion, and query traversal.
Embedding models are used in LlamaIndex to generate embeddings or representations of text data. These embeddings can be used for various tasks such as similarity matching, clustering, or classification. Embedding models help in organizing and structuring the data within LlamaIndex.
Prompts play a crucial role in LlamaIndex as they are used for various purposes. They are used to build the index, perform insertion, traverse queries, and synthesize the final answer. LlamaInd...
> Source (Doc id: 5c3e397e-95e1-4a9a-8bf6-0c6a1d7f3589): Sub question: How do I use query engines, query configurations, and the query engine pipeline in LlamaIndex?
Response: To use query engines, query configurations, and the query engine pipeline in LlamaIndex, you need to follow these steps:
1. Import the necessary modules from the LlamaIndex library. In this case, you need to import the `VectorStoreIndex`, `get_response_synthesizer`, `VectorIndexRetriever`, and `RetrieverQueryEngine` modules.
2. Create an instance of the `VectorStoreIndex` class. This class represents the index structure that you want to perform queries on.
3. Create an instance of the `VectorIndexRetriever` class, passing the `VectorStoreIndex` instance as a parameter. This class is responsible for retrieving responses from the index.
4. Create an instance of the `RetrieverQueryEngine` class, passing the `VectorIndexRetriever` instance as a parameter. This class represents the query engine that will execute the queries and retrieve the responses.
5. Configure t...
> Source (Doc id: 5c1412b2-165a-45f8-aa66-019bbae04d35): Sub question: What are the supporting modules in LlamaIndex, such as callbacks, service context, and evaluation?
Response: The supporting modules in LlamaIndex include callbacks, service context, and evaluation....
> Source (Doc id: 66e6e99d-20d9-48f5-b41c-2ab714af20a1): Sub question: Are there any end-to-end tutorials or examples of specific use-cases for LlamaIndex?
Response: Based on the given context information, it is not explicitly mentioned whether there are any end-to-end tutorials or examples of specific use-cases for LlamaIndex....
> Source (Doc id: bae0667b-f1e5-4303-906a-2ff1c823181a): Sub question: How can I contribute to LlamaIndex, including contributing to the codebase and building documentation?
Response: To contribute to LLamaIndex, you can start by contributing to its codebase. This can be done by forking the LLamaIndex repository on a platform like GitHub, making your desired changes or additions to the code, and then submitting a pull request to the main LLamaIndex repository. The LLamaIndex team will review your changes and, if approved, merge them into the codebase.
In addition to code contributions, you can also contribute to LLamaIndex by building documentation. This can involve creating or updating documentation that helps users understand how to use LLamaIndex, its features, and its configuration options. You can contribute to the documentation by submitting pull requests to the LLamaIndex documentation repository.
By contributing to the codebase and building documentation, you can help improve LLamaIndex and make it more accessible and useful for...
%% Cell type:code id: tags:
``` python
response = query_engine.query('What is the purpose of the `RouterQueryEngine` in LlamaIndex and how can it be used in the search pipeline?')
print(str(response))
print("-----------------")
print(response.get_formatted_sources(length=1000))
```
%% Output
The purpose of the `RouterQueryEngine` in LlamaIndex is to perform query transformations over index structures. It is responsible for converting a query into another query, either in a single-step or multi-step process. Additionally, the `RouterQueryEngine` supports streaming the response as it is being generated, allowing for the processing or printing of the beginning of the response before the full response is finished, thereby reducing the perceived latency of queries.
As for how the `RouterQueryEngine` can be used in the search pipeline, the given context information does not provide any specific details. Therefore, without prior knowledge, it is not possible to determine the exact usage of the `RouterQueryEngine` in the search pipeline.
-----------------
> Source (Doc id: b1fba297-c258-4d6f-9f18-205177456c59): Sub question: What is the purpose of the `RouterQueryEngine` in LlamaIndex?
Response: Based on the given context information, the purpose of the `RouterQueryEngine` in LlamaIndex is to perform query transformations over index structures. It is responsible for converting a query into another query, either in a single-step or multi-step process. The `RouterQueryEngine` also supports streaming the response as it is being generated, allowing for the processing or printing of the beginning of the response before the full response is finished, thereby reducing the perceived latency of queries....
> Source (Doc id: 8cefb2af-73d6-4edf-baab-13651453ea56): Sub question: How can the `RouterQueryEngine` be used in the search pipeline?
Response: The given context information does not provide any specific details about the `RouterQueryEngine`. Therefore, without prior knowledge, it is not possible to determine how the `RouterQueryEngine` can be used in the search pipeline....
%% Cell type:markdown id: tags:
#### Evaluating Response for Answer Quality
%% Cell type:code id: tags:
``` python
import time
import asyncio
import nest_asyncio
nest_asyncio.apply()
from llama_index import Response
def evaluate_query_engine(evaluator, query_engine, questions):
async def run_query(query_engine, q):
try:
return await query_engine.aquery(q)
except:
return Response(response="Error, query failed.")
total_correct = 0
all_results = []
for batch_size in range(0, len(questions), 5):
batch_qs = questions[batch_size:batch_size+5]
tasks = [run_query(query_engine, q) for q in batch_qs]
responses = asyncio.run(asyncio.gather(*tasks))
print(f"finished batch {(batch_size // 5) + 1} out of {len(questions) // 5}")
for question, response in zip(batch_qs, responses):
eval_result = 1 if "YES" in evaluator.evaluate(question, response) else 0
total_correct += eval_result
all_results.append(eval_result)
# helps avoid rate limits
time.sleep(1)
return total_correct, all_results
```
%% Cell type:code id: tags:
``` python
from llama_index.evaluation import QueryResponseEvaluator
evaluator = QueryResponseEvaluator(service_context=gpt4_service_context)
total_correct, all_results = evaluate_query_engine(evaluator, query_engine, question_dataset)
print(f"Response satisfies the query? Scored {total_correct} out of {len(question_dataset)} questions correctly.")
```
%% Output
finished batch 1 out of 8
finished batch 2 out of 8
finished batch 3 out of 8
finished batch 4 out of 8
finished batch 5 out of 8
finished batch 6 out of 8
finished batch 7 out of 8
finished batch 8 out of 8
Response satisfies the query? Scored 19 out of 40 questions correctly.
%% Cell type:markdown id: tags:
#### Investigating Incorrect Answers
%% Cell type:code id: tags:
``` python
import numpy as np
unanswered_queries = np.array(question_dataset)[np.array(all_results) == 0]
print(unanswered_queries)
```
%% Output
['What is the purpose of the `GuidancePydanticProgram` class in the LlamaIndex documentation?'
'What is the purpose of the SubQuestionQueryEngine class in LlamaIndex?'
'What are the available options for the storage backend of the index store in LlamaIndex?'
'What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?'
"What is the purpose of the `CollectionQueryConsumer` class in the Delphic application's WebSocket handling?"
'What is the purpose of the `ReActAgent` and how can it be initialized with other agents as tools?'
'How can I create a Django superuser using the Delphic application?'
'What is the default number of LLM calls required for the ListIndex?'
'What are the different vector stores supported by LlamaIndex for use as the storage backend for `VectorStoreIndex`?'
'What is the purpose of the "router query engine" in the LlamaIndex framework?'
'What storage backends are supported by LlamaIndex for persisting data?'
'What is the purpose of the `fetchDocuments` function in the `fetchDocuments.tsx` file in the React frontend?'
'What is the purpose of the `RefinePrompt` class in the LlamaIndex documentation?'
"What is the function used to retrieve the collections for the logged-in user in the Delphic project's frontend?"
'What is the purpose of the ResponseEvaluator class in the LlamaIndex library?'
'What is the purpose of the Algovera tool built on top of LlamaIndex?'
'What is the purpose of the HyDE query transform in the LlamaIndex?'
'What are the three primary sections within the layout of the ChatView component?'
'What is the purpose of the `insert_terms` function in the LlamaIndex app?'
'What is the purpose of the `load_collection_model` function in the LlamaIndex documentation?'
'What is the purpose of the SQLTableNodeMapping object in the LlamaIndex documentation sample?']
%% Cell type:code id: tags:
``` python
response = query_engine.query('What is the purpose of the `ReActAgent` and how can it be initialized with other agents as tools?')
print(str(response))
print("-----------------")
print(response.get_formatted_sources(length=256))
```
%% Output
The purpose of the `ReActAgent` is to instantiate an agent from a set of Tools. It can be initialized with other agents as tools by passing them as parameters to the `from_tools()` method.
-----------------
> Source (Doc id: 809486e9-ee91-42cb-a620-66c534c6890f): Sub question: What is the purpose of the ReActAgent?
Response: The purpose of the ReActAgent is to instantiate an agent from a set of Tools....
> Source (Doc id: 581d674e-6550-4a40-aa78-8936b53867e8): Sub question: How can the ReActAgent be initialized with other agents as tools?
Response: Based on the given context information, it is not possible to initialize the ReActAgent with other agents as tools. The ReActAgent can only be initialized with a s...
%% Cell type:code id: tags:
``` python
response = query_engine.query('What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?')
print(str(response))
print("-----------------")
print(response.get_formatted_sources(length=256))
```
%% Output
The purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation is to provide a high-level interface for ingesting, indexing, and querying external data. It allows users to customize storage components such as document stores, index stores, and vector stores. The LoadAndSearchToolSpec also supports persisting data to various storage backends and offers different ways of querying a list index, including embedding-based queries and keyword filters.
-----------------
> Source (Doc id: 8ad62670-cacc-4a34-bb9b-00504904b04c): Sub question: What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?
Response: The purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation is not mentioned in the given context information....
> Source (Doc id: f5974d32-081a-4366-9e8a-ea9b4bccbdaf): Sub question: What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?
Response: The purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation is to provide a high-level interface for ingesting, indexing, and querying...
> Source (Doc id: 69d8cee2-a4eb-4183-acbe-34047f3b2649): Sub question: What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?
Response: The purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation is to provide a tool specification that can be used to load and search dat...
> Source (Doc id: be322260-e24b-4dc3-9540-8b973b1afecf): Sub question: What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?
Response: Based on the given context information, it is not possible to determine the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation. T...
> Source (Doc id: f0f035ac-2c2d-4ba6-b84b-77d88bb7aaed): Sub question: What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?
Response: Based on the given context information, there is no mention of the "LoadAndSearchToolSpec" in the LlamaIndex documentation. Therefore, it is not po...
> Source (Doc id: b0384b51-bb28-48dd-98fd-63f57d1c4791): Sub question: What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?
Response: Based on the given context information, there is no mention of the "LoadAndSearchToolSpec" in the LlamaIndex documentation. Therefore, the purpose ...
> Source (Doc id: 9d0c096e-9049-4c36-9f1a-5b166c93a635): Sub question: What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?
Response: The purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation is not mentioned in the given context information....
> Source (Doc id: 65259c15-1ad2-4211-bd11-7c7b6f182cf8): Sub question: What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?
Response: The purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation is to provide a tool that allows users to load and search for embeddings a...
%% Cell type:markdown id: tags:
# Conclusion
In this notebook, we covered several key topics!
- setting up a sub-question query engine
- generating a dataset of evaluation questions
- evaluating responses for hallucination
- evaluating responses for answer quality
3_eval_baseline/question_dataset.txt 0 → 100644
+ 40
− 0
View file @ 928a9f3b
What are the possible settings for the LLM and how can the user set the prompt for term extraction?
How can I convert tools to LangChain tools using the provided documentation sample?
What is the purpose of the `GuidancePydanticProgram` class in the LlamaIndex documentation?
What is the purpose of the SubQuestionQueryEngine class in LlamaIndex?
What is the purpose of the `query_wrapper_prompt` in the `HuggingFaceLLM` class?
What embedding model does LlamaIndex use by default?
What are the available options for the storage backend of the index store in LlamaIndex?
What is the function used to specify the metadata visible to the embedding model and how can it be customized?
What are the node postprocessors available in the LlamaIndex documentation?
What is the purpose of the LoadAndSearchToolSpec in the LlamaIndex documentation?
What is the purpose of the DEFAULT_REFINE_PROMPT_SEL_LC in the LlamaIndex documentation?
What is the purpose of the `CollectionQueryConsumer` class in the Delphic application's WebSocket handling?
What is the purpose of the `ReActAgent` and how can it be initialized with other agents as tools?
How can I create a Django superuser using the Delphic application?
What is the default number of LLM calls required for the ListIndex?
What are the different vector stores supported by LlamaIndex for use as the storage backend for `VectorStoreIndex`?
What are the key building blocks in LlamaIndex for composing a Retrieval Augmented Generation (RAG) pipeline?
What is the purpose of the "router query engine" in the LlamaIndex framework?
What is the purpose of the `VectorStoreIndex` class in the LlamaIndex documentation sample?
What storage backends are supported by LlamaIndex for persisting data?
What is the purpose of the `fetchDocuments` function in the `fetchDocuments.tsx` file in the React frontend?
What is the purpose of the `DecomposeQueryTransform` module in the LlamaIndex documentation?
What is the purpose of the `RefinePrompt` class in the LlamaIndex documentation?
What is the function used to retrieve the collections for the logged-in user in the Delphic project's frontend?
What is the default value for the `include_metadata` parameter in the `SimpleNodeParser` class?
What is the purpose of the `RouterQueryEngine` in the LlamaIndex documentation?
What is the purpose of the TokenCountingHandler callback in the LlamaIndex library?
What is the purpose of the `RelatedNodeInfo` class in the LlamaIndex documentation?
What is the purpose of the ResponseEvaluator class in the LlamaIndex library?
What is the purpose of the `SimilarityPostprocessor` in the query engine configuration?
What is the purpose of the Algovera tool built on top of LlamaIndex?
How can you delete a document from the index data structures and specify whether to delete it from the document store as well?
How can we extract terms from documents using LlamaIndex and insert them into the index?
What is the purpose of the HyDE query transform in the LlamaIndex?
What are the three primary sections within the layout of the ChatView component?
What is the default value for the child_branch_factor parameter when configuring the ComposableGraphQueryEngine?
What is the purpose of the `insert_terms` function in the LlamaIndex app?
What is the purpose of the `load_collection_model` function in the LlamaIndex documentation?
What is the purpose of the SQLTableNodeMapping object in the LlamaIndex documentation sample?
What is the purpose of the `RouterQueryEngine` in LlamaIndex and how can it be used in the search pipeline?
llama_docs_bot/indexing.py 0 → 100644
+ 174
− 0
View file @ 928a9f3b
import nest_asyncio
nest_asyncio.apply()
from .markdown_docs_reader import MarkdownDocsReader
from llama_index import (
SimpleDirectoryReader,
VectorStoreIndex,
ServiceContext,
StorageContext,
load_index_from_storage,
set_global_service_context
)
from llama_index.query_engine import SubQuestionQueryEngine
from llama_index.response_synthesizers import get_response_synthesizer
from llama_index.tools import QueryEngineTool
def load_markdown_docs(filepath):
"""Load markdown docs from a directory, excluding all other file types."""
loader = SimpleDirectoryReader(
input_dir=filepath,
required_exts=[".md"],
file_extractor={".md": MarkdownDocsReader()},
recursive=True
)
documents = loader.load_data()
# exclude some metadata from the LLM
for doc in documents:
doc.excluded_llm_metadata_keys = ["File Name", "Content Type", "Header Path"]
return documents
def load_docs():
getting_started_docs = load_markdown_docs("../docs/getting_started")
community_docs = load_markdown_docs("../docs/community")
data_docs = load_markdown_docs("../docs/core_modules/data_modules")
agent_docs = load_markdown_docs("../docs/core_modules/agent_modules")
model_docs = load_markdown_docs("../docs/core_modules/model_modules")
query_docs = load_markdown_docs("../docs/core_modules/query_modules")
supporting_docs = load_markdown_docs("../docs/core_modules/supporting_modules")
tutorials_docs = load_markdown_docs("../docs/end_to_end_tutorials")
contributing_docs = load_markdown_docs("../docs/development")
return (
getting_started_docs,
community_docs,
data_docs,
agent_docs,
model_docs,
query_docs,
supporting_docs,
tutorials_docs,
contributing_docs,
)
def create_query_engine():
"""Create a query engine."""
getting_started_docs, community_docs, data_docs, agent_docs, model_docs, query_docs, supporting_docs, tutorials_docs, contributing_docs = load_docs()
try:
getting_started_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./getting_started_index"))
community_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./community_index"))
data_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./data_index"))
agent_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./agent_index"))
model_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./model_index"))
query_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./query_index"))
supporting_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./supporting_index"))
tutorials_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./tutorials_index"))
contributing_index = load_index_from_storage(StorageContext.from_defaults(persist_dir="./contributing_index"))
except Exception:
getting_started_index = VectorStoreIndex.from_documents(getting_started_docs)
getting_started_index.storage_context.persist(persist_dir="./getting_started_index")
community_index = VectorStoreIndex.from_documents(community_docs)
community_index.storage_context.persist(persist_dir="./community_index")
data_index = VectorStoreIndex.from_documents(data_docs)
data_index.storage_context.persist(persist_dir="./data_index")
agent_index = VectorStoreIndex.from_documents(agent_docs)
agent_index.storage_context.persist(persist_dir="./agent_index")
model_index = VectorStoreIndex.from_documents(model_docs)
model_index.storage_context.persist(persist_dir="./model_index")
query_index = VectorStoreIndex.from_documents(query_docs)
query_index.storage_context.persist(persist_dir="./query_index")
supporting_index = VectorStoreIndex.from_documents(supporting_docs)
supporting_index.storage_context.persist(persist_dir="./supporting_index")
tutorials_index = VectorStoreIndex.from_documents(tutorials_docs)
tutorials_index.storage_context.persist(persist_dir="./tutorials_index")
contributing_index = VectorStoreIndex.from_documents(contributing_docs)
contributing_index.storage_context.persist(persist_dir="./contributing_index")
# create a query engine tool for each folder
getting_started_tool = QueryEngineTool.from_defaults(
query_engine=getting_started_index.as_query_engine(),
name="Getting Started",
description="Useful for answering questions about installing and running llama index, as well as basic explanations of how llama index works."
)
community_tool = QueryEngineTool.from_defaults(
query_engine=community_index.as_query_engine(),
name="Community",
description="Useful for answering questions about integrations and other apps built by the community."
)
data_tool = QueryEngineTool.from_defaults(
query_engine=data_index.as_query_engine(),
name="Data Modules",
description="Useful for answering questions about data loaders, documents, nodes, and index structures."
)
agent_tool = QueryEngineTool.from_defaults(
query_engine=agent_index.as_query_engine(),
name="Agent Modules",
description="Useful for answering questions about data agents, agent configurations, and tools."
)
model_tool = QueryEngineTool.from_defaults(
query_engine=model_index.as_query_engine(),
name="Model Modules",
description="Useful for answering questions about using and configuring LLMs, embedding modles, and prompts."
)
query_tool = QueryEngineTool.from_defaults(
query_engine=query_index.as_query_engine(),
name="Query Modules",
description="Useful for answering questions about query engines, query configurations, and using various parts of the query engine pipeline."
)
supporting_tool = QueryEngineTool.from_defaults(
query_engine=supporting_index.as_query_engine(),
name="Supporting Modules",
description="Useful for answering questions about supporting modules, such as callbacks, service context, and avaluation."
)
tutorials_tool = QueryEngineTool.from_defaults(
query_engine=tutorials_index.as_query_engine(),
name="Tutorials",
description="Useful for answering questions about end-to-end tutorials and giving examples of specific use-cases."
)
contributing_tool = QueryEngineTool.from_defaults(
query_engine=contributing_index.as_query_engine(),
name="Contributing",
description="Useful for answering questions about contributing to llama index, including how to contribute to the codebase and how to build documentation."
)
query_engine = SubQuestionQueryEngine.from_defaults(
query_engine_tools=[
getting_started_tool,
community_tool,
data_tool,
agent_tool,
model_tool,
query_tool,
supporting_tool,
tutorials_tool,
contributing_tool
],
# enable this for streaming
response_synthesizer=get_response_synthesizer(streaming=True),
verbose=False
)
return query_engine
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

:feet::heart: Strive to be the person your dogs believe you are :heart::feet: