From 4ea034082b1f6ca54470f50e00fe2c45f7b1916c Mon Sep 17 00:00:00 2001
From: Yi Ding <yi.s.ding@gmail.com>
Date: Tue, 18 Jul 2023 19:45:44 -0700
Subject: [PATCH] changed default response synthesizer to compact
---
packages/core/examples/abramov.txt | 4 +-
packages/core/src/LLMPredictor.ts | 8 +-
packages/core/src/Prompt.ts | 2 +-
packages/core/src/QueryEngine.ts | 2 +-
packages/core/src/ResponseSynthesizer.ts | 108 +++++++++++++++++------
5 files changed, 91 insertions(+), 33 deletions(-)
diff --git a/packages/core/examples/abramov.txt b/packages/core/examples/abramov.txt
index 11b2c1856..7dddad30a 100644
--- a/packages/core/examples/abramov.txt
+++ b/packages/core/examples/abramov.txt
@@ -1,4 +1,4 @@
-export default `I started this decade as a first-year college student fresh out of high school. I was 17, didn’t have a job, didn’t have any industry connections, and really didn’t know shit. And now you’re reading my blog! I would have been proud.
+I started this decade as a first-year college student fresh out of high school. I was 17, didn’t have a job, didn’t have any industry connections, and really didn’t know shit. And now you’re reading my blog! I would have been proud.
I’ve told bits and pieces of my story on different podcasts. Now feels like an appropriate time to write down the parts that were most memorable to me.
Every person’s story is unique and not directly reproducible. I’ve benefited immensely from the privilege of being born in an upper middle class family and looking like a typical coder stereotype. People took chances on me. Still, I hope that sharing my story can be helpful to compare our experiences. Even if our circumstances are too different, at least you might find some of it entertaining.
2010
@@ -117,4 +117,4 @@ I got promoted. I started this blog as a side project. I have another side proje
I always knew that I liked building UIs. I got hooked on Visual Basic. I spent this decade building UIs, and then building a way to build UIs. And then talking about it and explaining it. But I realize now that my drive to explain things is just as important to me as my drive to build. Perhaps, even more important.
I look forward to doing more of that in the next decade.
Or, should I say, this decade?
-Welcome to the twenties.`
+Welcome to the twenties.
\ No newline at end of file
diff --git a/packages/core/src/LLMPredictor.ts b/packages/core/src/LLMPredictor.ts
index 69c3c6284..5bd95ec03 100644
--- a/packages/core/src/LLMPredictor.ts
+++ b/packages/core/src/LLMPredictor.ts
@@ -49,13 +49,17 @@ export class ChatGPTLLMPredictor implements BaseLLMPredictor {
async predict(
prompt: string | SimplePrompt,
input?: Record<string, string>,
- parentEvent?: Event
+ parentEvent?: Event,
+ logProgress: boolean = false
): Promise<string> {
if (typeof prompt === "string") {
+ if (logProgress) {
+ console.log("PROMPT", prompt);
+ }
const result = await this.languageModel.complete(prompt, parentEvent);
return result.message.content;
} else {
- return this.predict(prompt(input ?? {}));
+ return this.predict(prompt(input ?? {}), undefined, parentEvent);
}
}
}
diff --git a/packages/core/src/Prompt.ts b/packages/core/src/Prompt.ts
index 6eb994281..a82c4c517 100644
--- a/packages/core/src/Prompt.ts
+++ b/packages/core/src/Prompt.ts
@@ -258,7 +258,7 @@ ${JSON.stringify(exampleOutput, null, 4)}
# Example 2
<Tools>
\`\`\`json
-${toolsStr}}
+${toolsStr}
\`\`\`
<User Question>
diff --git a/packages/core/src/QueryEngine.ts b/packages/core/src/QueryEngine.ts
index b4a3bc115..39f941779 100644
--- a/packages/core/src/QueryEngine.ts
+++ b/packages/core/src/QueryEngine.ts
@@ -141,7 +141,7 @@ export class SubQuestionQueryEngine implements BaseQueryEngine {
const response = await queryEngine.query(question, parentEvent);
const responseText = response.response;
- const nodeText = `Sub question: ${question}\nResponse: ${responseText}}`;
+ const nodeText = `Sub question: ${question}\nResponse: ${responseText}`;
const node = new TextNode({ text: nodeText });
return { node, score: 0 };
} catch (error) {
diff --git a/packages/core/src/ResponseSynthesizer.ts b/packages/core/src/ResponseSynthesizer.ts
index c0e5faaa8..1077310f9 100644
--- a/packages/core/src/ResponseSynthesizer.ts
+++ b/packages/core/src/ResponseSynthesizer.ts
@@ -7,9 +7,19 @@ import {
} from "./Prompt";
import { getBiggestPrompt } from "./PromptHelper";
import { Response } from "./Response";
-import { ServiceContext } from "./ServiceContext";
+import { ServiceContext, serviceContextFromDefaults } from "./ServiceContext";
import { Event } from "./callbacks/CallbackManager";
+/**
+ * Response modes of the response synthesizer
+ */
+enum ResponseMode {
+ REFINE = "refine",
+ COMPACT = "compact",
+ TREE_SUMMARIZE = "tree_summarize",
+ SIMPLE = "simple",
+}
+
/**
* A ResponseBuilder is used in a response synthesizer to generate a response from multiple response chunks.
*/
@@ -19,11 +29,13 @@ interface BaseResponseBuilder {
* @param query
* @param textChunks
* @param parentEvent
+ * @param prevResponse
*/
getResponse(
query: string,
textChunks: string[],
- parentEvent?: Event
+ parentEvent?: Event,
+ prevResponse?: string
): Promise<string>;
}
@@ -34,9 +46,8 @@ export class SimpleResponseBuilder implements BaseResponseBuilder {
llmPredictor: BaseLLMPredictor;
textQATemplate: SimplePrompt;
- constructor(serviceContext?: ServiceContext) {
- this.llmPredictor =
- serviceContext?.llmPredictor ?? new ChatGPTLLMPredictor();
+ constructor(serviceContext: ServiceContext) {
+ this.llmPredictor = serviceContext.llmPredictor;
this.textQATemplate = defaultTextQaPrompt;
}
@@ -76,15 +87,21 @@ export class Refine implements BaseResponseBuilder {
async getResponse(
query: string,
textChunks: string[],
- prevResponse?: any
+ parentEvent?: Event,
+ prevResponse?: string
): Promise<string> {
let response: string | undefined = undefined;
for (const chunk of textChunks) {
if (!prevResponse) {
- response = await this.giveResponseSingle(query, chunk);
+ response = await this.giveResponseSingle(query, chunk, parentEvent);
} else {
- response = await this.refineResponseSingle(prevResponse, query, chunk);
+ response = await this.refineResponseSingle(
+ prevResponse,
+ query,
+ chunk,
+ parentEvent
+ );
}
prevResponse = response;
}
@@ -94,7 +111,8 @@ export class Refine implements BaseResponseBuilder {
private async giveResponseSingle(
queryStr: string,
- textChunk: string
+ textChunk: string,
+ parentEvent?: Event
): Promise<string> {
const textQATemplate: SimplePrompt = (input) =>
this.textQATemplate({ ...input, query: queryStr });
@@ -110,10 +128,16 @@ export class Refine implements BaseResponseBuilder {
textQATemplate,
{
context: chunk,
- }
+ },
+ parentEvent
);
} else {
- response = await this.refineResponseSingle(response, queryStr, chunk);
+ response = await this.refineResponseSingle(
+ response,
+ queryStr,
+ chunk,
+ parentEvent
+ );
}
}
@@ -123,7 +147,8 @@ export class Refine implements BaseResponseBuilder {
private async refineResponseSingle(
response: string,
queryStr: string,
- textChunk: string
+ textChunk: string,
+ parentEvent?: Event
) {
const refineTemplate: SimplePrompt = (input) =>
this.refineTemplate({ ...input, query: queryStr });
@@ -138,7 +163,8 @@ export class Refine implements BaseResponseBuilder {
{
context: chunk,
existingAnswer: response,
- }
+ },
+ parentEvent
);
}
return response;
@@ -152,7 +178,8 @@ export class CompactAndRefine extends Refine {
async getResponse(
query: string,
textChunks: string[],
- prevResponse?: any
+ parentEvent?: Event,
+ prevResponse?: string
): Promise<string> {
const textQATemplate: SimplePrompt = (input) =>
this.textQATemplate({ ...input, query: query });
@@ -164,7 +191,12 @@ export class CompactAndRefine extends Refine {
maxPrompt,
textChunks
);
- const response = super.getResponse(query, newTexts, prevResponse);
+ const response = super.getResponse(
+ query,
+ newTexts,
+ parentEvent,
+ prevResponse
+ );
return response;
}
}
@@ -178,7 +210,11 @@ export class TreeSummarize implements BaseResponseBuilder {
this.serviceContext = serviceContext;
}
- async getResponse(query: string, textChunks: string[]): Promise<string> {
+ async getResponse(
+ query: string,
+ textChunks: string[],
+ parentEvent?: Event
+ ): Promise<string> {
const summaryTemplate: SimplePrompt = (input) =>
defaultTextQaPrompt({ ...input, query: query });
@@ -192,15 +228,23 @@ export class TreeSummarize implements BaseResponseBuilder {
);
if (packedTextChunks.length === 1) {
- return this.serviceContext.llmPredictor.predict(summaryTemplate, {
- context: packedTextChunks[0],
- });
+ return this.serviceContext.llmPredictor.predict(
+ summaryTemplate,
+ {
+ context: packedTextChunks[0],
+ },
+ parentEvent
+ );
} else {
const summaries = await Promise.all(
packedTextChunks.map((chunk) =>
- this.serviceContext.llmPredictor.predict(summaryTemplate, {
- context: chunk,
- })
+ this.serviceContext.llmPredictor.predict(
+ summaryTemplate,
+ {
+ context: chunk,
+ },
+ parentEvent
+ )
)
);
@@ -210,9 +254,19 @@ export class TreeSummarize implements BaseResponseBuilder {
}
export function getResponseBuilder(
- serviceContext?: ServiceContext
-): SimpleResponseBuilder {
- return new SimpleResponseBuilder(serviceContext);
+ serviceContext: ServiceContext,
+ responseMode?: ResponseMode
+): BaseResponseBuilder {
+ switch (responseMode) {
+ case ResponseMode.SIMPLE:
+ return new SimpleResponseBuilder(serviceContext);
+ case ResponseMode.REFINE:
+ return new Refine(serviceContext);
+ case ResponseMode.TREE_SUMMARIZE:
+ return new TreeSummarize(serviceContext);
+ default:
+ return new CompactAndRefine(serviceContext);
+ }
}
/**
@@ -220,7 +274,7 @@ export function getResponseBuilder(
*/
export class ResponseSynthesizer {
responseBuilder: BaseResponseBuilder;
- serviceContext?: ServiceContext;
+ serviceContext: ServiceContext;
constructor({
responseBuilder,
@@ -229,7 +283,7 @@ export class ResponseSynthesizer {
responseBuilder?: BaseResponseBuilder;
serviceContext?: ServiceContext;
} = {}) {
- this.serviceContext = serviceContext;
+ this.serviceContext = serviceContext ?? serviceContextFromDefaults();
this.responseBuilder =
responseBuilder ?? getResponseBuilder(this.serviceContext);
}
--
GitLab