diff --git a/packages/core/src/NodeParser.ts b/packages/core/src/NodeParser.ts index 99f9daba6b577159829d4a10ff102f0e56959b81..4860e66258ceafbb17dc263a2ef5faaa61bd78c1 100644 --- a/packages/core/src/NodeParser.ts +++ b/packages/core/src/NodeParser.ts @@ -2,6 +2,12 @@ import { Document, NodeRelationship, TextNode } from "./Node"; import { SentenceSplitter } from "./TextSplitter"; import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants"; +/** + * Splits the text of a document into smaller parts. + * @param document - The document to split. + * @param textSplitter - The text splitter to use. + * @returns An array of text splits. + */ export function getTextSplitsFromDocument( document: Document, textSplitter: SentenceSplitter @@ -12,6 +18,14 @@ export function getTextSplitsFromDocument( return splits; } +/** + * Generates an array of nodes from a document. + * @param document - The document to generate nodes from. + * @param textSplitter - The text splitter to use. + * @param includeMetadata - Whether to include metadata in the nodes. + * @param includePrevNextRel - Whether to include previous and next relationships in the nodes. + * @returns An array of nodes. + */ export function getNodesFromDocument( document: Document, textSplitter: SentenceSplitter, @@ -46,10 +60,16 @@ export function getNodesFromDocument( return nodes; } + /** - * A node parser generates TextNodes from Documents + * A NodeParser generates TextNodes from Documents */ export interface NodeParser { + /** + * Generates an array of nodes from an array of documents. + * @param documents - The documents to generate nodes from. + * @returns An array of nodes. + */ getNodesFromDocuments(documents: Document[]): TextNode[]; } @@ -57,8 +77,17 @@ export interface NodeParser { * SimpleNodeParser is the default NodeParser. It splits documents into TextNodes using a splitter, by default SentenceSplitter */ export class SimpleNodeParser implements NodeParser { + /** + * The text splitter to use. + */ textSplitter: SentenceSplitter; + /** + * Whether to include metadata in the nodes. + */ includeMetadata: boolean; + /** + * Whether to include previous and next relationships in the nodes. + */ includePrevNextRel: boolean; constructor(init?: { diff --git a/packages/core/src/callbacks/utility/handleOpenAIStream.ts b/packages/core/src/callbacks/utility/handleOpenAIStream.ts index 0ecaa7650c0ba6e155837b547c768ecfa457fb0f..d86a6e67a0f3b47a18e6b478505cc367dbee0de9 100644 --- a/packages/core/src/callbacks/utility/handleOpenAIStream.ts +++ b/packages/core/src/callbacks/utility/handleOpenAIStream.ts @@ -5,6 +5,13 @@ import { APIResponse } from "openai/core"; import { Stream } from "openai/streaming"; import { MessageType } from "../../llm/LLM"; +/** + * Handles the OpenAI streaming interface and pipes it to the callback function + * @param response - The response from the OpenAI API. + * @param onLLMStream - A callback function to handle the LLM stream. + * @param parentEvent - An optional parent event. + * @returns A promise that resolves to an object with a message and a role. + */ export async function handleOpenAIStream({ response, onLLMStream,