diff --git a/packages/core/e2e/node/snapshot/llm.snap b/packages/core/e2e/node/snapshot/llm.snap index a27bcffc100c89a73306d7dde343a7a92f20fbae..6074f1524fe5a188f5cb705dbf1af1d21daa5cc2 100644 --- a/packages/core/e2e/node/snapshot/llm.snap +++ b/packages/core/e2e/node/snapshot/llm.snap @@ -56,10 +56,11 @@ { "id": "de46b84e-7345-430f-b8fa-423354b630c9", "response": { - "raw": {}, + "raw": null, "message": { "content": "Hello! How can I assist you today?", - "role": "assistant" + "role": "assistant", + "options": {} } } } diff --git a/packages/core/src/llm/base.ts b/packages/core/src/llm/base.ts index 8c245f4373296c96c95b17b55227842fd50c3361..d0f434909c55b8746b238146a3c82dd7f6eec9fd 100644 --- a/packages/core/src/llm/base.ts +++ b/packages/core/src/llm/base.ts @@ -41,6 +41,7 @@ export abstract class BaseLLM< }); return streamConverter(stream, (chunk) => { return { + raw: null, text: chunk.delta, }; }); diff --git a/packages/core/src/llm/ollama.ts b/packages/core/src/llm/ollama.ts index aad435628dbcdb3a890803332f67f56d2cf4798c..85e475cf27f90112f93108fa9303cbffea5d3582 100644 --- a/packages/core/src/llm/ollama.ts +++ b/packages/core/src/llm/ollama.ts @@ -17,8 +17,9 @@ const messageAccessor = (data: any): ChatResponseChunk => { delta: data.message.content, }; }; + const completionAccessor = (data: any): CompletionResponse => { - return { text: data.response }; + return { text: data.response, raw: null }; }; // https://github.com/jmorganca/ollama diff --git a/packages/core/src/llm/types.ts b/packages/core/src/llm/types.ts index 1231885b1f350b41077e3396a09931b71822dea1..1b27b130921e0778a96184a4b83ca7a4ea4616d6 100644 --- a/packages/core/src/llm/types.ts +++ b/packages/core/src/llm/types.ts @@ -126,8 +126,10 @@ export interface ChatResponse< message: ChatMessage<AdditionalMessageOptions>; /** * Raw response from the LLM + * + * It's possible that this is `null` if the LLM response an iterable of chunks */ - raw: object; + raw: object | null; } export type ChatResponseChunk< @@ -148,7 +150,12 @@ export type ChatResponseChunk< export interface CompletionResponse { text: string; - raw?: Record<string, any>; + /** + * Raw response from the LLM + * + * It's possible that this is `null` if the LLM response an iterable of chunks + */ + raw: object | null; } export type LLMMetadata = { diff --git a/packages/core/src/llm/utils.ts b/packages/core/src/llm/utils.ts index fbbc0c6c796516b377e94f2c89efdbc91da0ac6d..b37cc05daebde88b20b4f60994b0f5afedf85b6a 100644 --- a/packages/core/src/llm/utils.ts +++ b/packages/core/src/llm/utils.ts @@ -84,10 +84,11 @@ export function wrapLLMEvent( }; response[Symbol.asyncIterator] = async function* () { const finalResponse: ChatResponse = { - raw: response, + raw: null, message: { content: "", role: "assistant", + options: {}, }, }; let firstOne = false; @@ -98,6 +99,12 @@ export function wrapLLMEvent( } else { finalResponse.message.content += chunk.delta; } + if (chunk.options) { + finalResponse.message.options = { + ...finalResponse.message.options, + ...chunk.options, + }; + } yield chunk; } snapshot(() => {