From 9ea840142b29d86bd1129e5df874506b5cd64b57 Mon Sep 17 00:00:00 2001
From: Sourabh Desai <sourabhdesai@gmail.com>
Date: Tue, 4 Jul 2023 07:40:41 +0000
Subject: [PATCH] changes to get test script running
---
packages/core/src/Node.ts | 2 +-
packages/core/src/Prompt.ts | 60 +++++++++----------
packages/core/src/index/list/ListIndex.ts | 10 +++-
.../core/src/index/list/ListIndexRetriever.ts | 1 +
.../src/storage/docStore/KVDocumentStore.ts | 2 +-
packages/core/src/storage/docStore/utils.ts | 9 ++-
6 files changed, 44 insertions(+), 40 deletions(-)
diff --git a/packages/core/src/Node.ts b/packages/core/src/Node.ts
index 970ddd2ce..95d3081c8 100644
--- a/packages/core/src/Node.ts
+++ b/packages/core/src/Node.ts
@@ -227,7 +227,7 @@ export class ImageDocument extends Document {
export interface NodeWithScore {
node: BaseNode;
- score?: number;
+ score: number;
}
export interface NodeWithEmbedding {
diff --git a/packages/core/src/Prompt.ts b/packages/core/src/Prompt.ts
index e9ce13964..74d02c4bd 100644
--- a/packages/core/src/Prompt.ts
+++ b/packages/core/src/Prompt.ts
@@ -84,35 +84,33 @@ Given the new context, refine the original answer to better answer the question.
export const defaultChoiceSelectPrompt: SimplePrompt = (input) => {
const { context = "", query = "" } = input;
- return `
- A list of documents is shown below. Each document has a number next to it along
- with a summary of the document. A question is also provided.
- Respond with the numbers of the documents
- you should consult to answer the question, in order of relevance, as well
- as the relevance score. The relevance score is a number from 1-10 based on
- how relevant you think the document is to the question.
- Do not include any documents that are not relevant to the question.
- Example format:
- Document 1:
- <summary of document 1>
-
- Document 2:
- <summary of document 2>
-
- ...
-
- Document 10:\n<summary of document 10>
-
- Question: <question>
- Answer:
- Doc: 9, Relevance: 7
- Doc: 3, Relevance: 4
- Doc: 7, Relevance: 3
-
- Let's try this now:
-
- ${context}
- Question: ${query}
- Answer:
-`.trim();
+ return `A list of documents is shown below. Each document has a number next to it along
+with a summary of the document. A question is also provided.
+Respond with the numbers of the documents
+you should consult to answer the question, in order of relevance, as well
+as the relevance score. The relevance score is a number from 1-10 based on
+how relevant you think the document is to the question.
+Do not include any documents that are not relevant to the question.
+Example format:
+Document 1:
+<summary of document 1>
+
+Document 2:
+<summary of document 2>
+
+...
+
+Document 10:\n<summary of document 10>
+
+Question: <question>
+Answer:
+Doc: 9, Relevance: 7
+Doc: 3, Relevance: 4
+Doc: 7, Relevance: 3
+
+Let's try this now:
+
+${context}
+Question: ${query}
+Answer:`;
};
diff --git a/packages/core/src/index/list/ListIndex.ts b/packages/core/src/index/list/ListIndex.ts
index 78c1971b9..56a17e0fe 100644
--- a/packages/core/src/index/list/ListIndex.ts
+++ b/packages/core/src/index/list/ListIndex.ts
@@ -11,7 +11,7 @@ import {
ServiceContext,
serviceContextFromDefaults,
} from "../../ServiceContext";
-import { RefDocInfo } from "../../storage/docStore/types";
+import { BaseDocumentStore, RefDocInfo } from "../../storage/docStore/types";
import _ from "lodash";
export enum ListRetrieverMode {
@@ -53,7 +53,10 @@ export class ListIndex extends BaseIndex<IndexList> {
"Cannot initialize VectorStoreIndex without nodes or indexStruct"
);
}
- indexStruct = ListIndex._buildIndexFromNodes(options.nodes);
+ indexStruct = ListIndex._buildIndexFromNodes(
+ options.nodes,
+ storageContext.docStore
+ );
}
return new ListIndex({
@@ -74,6 +77,7 @@ export class ListIndex extends BaseIndex<IndexList> {
serviceContext = serviceContext ?? serviceContextFromDefaults({});
const docStore = storageContext.docStore;
+ docStore.addDocuments(documents, true);
for (const doc of documents) {
docStore.setDocumentHash(doc.id_, doc.hash);
}
@@ -108,10 +112,12 @@ export class ListIndex extends BaseIndex<IndexList> {
static _buildIndexFromNodes(
nodes: BaseNode[],
+ docStore: BaseDocumentStore,
indexStruct?: IndexList
): IndexList {
indexStruct = indexStruct || new IndexList();
+ docStore.addDocuments(nodes, true);
for (const node of nodes) {
indexStruct.addNode(node);
}
diff --git a/packages/core/src/index/list/ListIndexRetriever.ts b/packages/core/src/index/list/ListIndexRetriever.ts
index 86b0a7827..33e742075 100644
--- a/packages/core/src/index/list/ListIndexRetriever.ts
+++ b/packages/core/src/index/list/ListIndexRetriever.ts
@@ -26,6 +26,7 @@ export class ListIndexRetriever implements BaseRetriever {
const nodes = await this.index.docStore.getNodes(nodeIds);
return nodes.map((node) => ({
node: node,
+ score: 1,
}));
}
}
diff --git a/packages/core/src/storage/docStore/KVDocumentStore.ts b/packages/core/src/storage/docStore/KVDocumentStore.ts
index 64b9780d7..027672e6d 100644
--- a/packages/core/src/storage/docStore/KVDocumentStore.ts
+++ b/packages/core/src/storage/docStore/KVDocumentStore.ts
@@ -77,7 +77,7 @@ export class KVDocumentStore extends BaseDocumentStore {
let json = await this.kvstore.get(docId, this.nodeCollection);
if (_.isNil(json)) {
if (raiseError) {
- throw new Error(`doc_id ${docId} not found.`);
+ throw new Error(`docId ${docId} not found.`);
} else {
return;
}
diff --git a/packages/core/src/storage/docStore/utils.ts b/packages/core/src/storage/docStore/utils.ts
index 8c80a3c87..a7329df67 100644
--- a/packages/core/src/storage/docStore/utils.ts
+++ b/packages/core/src/storage/docStore/utils.ts
@@ -23,12 +23,11 @@ export function jsonToDoc(docDict: Record<string, any>): BaseNode {
hash: dataDict.hash,
});
} else if (docType === ObjectType.TEXT) {
- const relationships = dataDict.relationships;
+ console.log({ dataDict });
doc = new TextNode({
- text: relationships.text,
- id_: relationships.id_,
- embedding: relationships.embedding,
- hash: relationships.hash,
+ text: dataDict.text,
+ id_: dataDict.id_,
+ hash: dataDict.hash,
});
} else {
throw new Error(`Unknown doc type: ${docType}`);
--
GitLab