diff --git a/packages/core/src/ServiceContext.ts b/packages/core/src/ServiceContext.ts index e1490a1995f8dd699f4dcb09cb37a5d968b86ecb..7bdb2c66484d6e3ce55b5632346af4cf199cb1b9 100644 --- a/packages/core/src/ServiceContext.ts +++ b/packages/core/src/ServiceContext.ts @@ -1,7 +1,7 @@ import { CallbackManager } from "./callbacks/CallbackManager"; import { BaseEmbedding, OpenAIEmbedding } from "./embeddings"; -import { LLM, OpenAI } from "./llm/LLM"; -import { NodeParser, SimpleNodeParser } from "./NodeParser"; +import { LLM, OpenAI } from "./llm"; +import { NodeParser, SimpleNodeParser } from "./nodeParsers"; import { PromptHelper } from "./PromptHelper"; /** diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e170e99d8dc8a0df82add70a49624efed8d63225..e995ca0156dca67847c0011b13f59ee673a2bfc6 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -2,7 +2,6 @@ export * from "./ChatEngine"; export * from "./ChatHistory"; export * from "./GlobalsHelper"; export * from "./Node"; -export * from "./NodeParser"; export * from "./OutputParser"; export * from "./Prompt"; export * from "./PromptHelper"; @@ -18,6 +17,7 @@ export * from "./constants"; export * from "./embeddings"; export * from "./indices"; export * from "./llm"; +export * from "./nodeParsers"; export * from "./readers/AssemblyAI"; export * from "./readers/CSVReader"; export * from "./readers/HTMLReader"; diff --git a/packages/core/src/NodeParser.ts b/packages/core/src/nodeParsers/SimpleNodeParser.ts similarity index 82% rename from packages/core/src/NodeParser.ts rename to packages/core/src/nodeParsers/SimpleNodeParser.ts index d39aae5ae98fd5ef25103850f990439e55ed94b2..8a1738f571ceae6ce652cd23d31e262407b3ea63 100644 --- a/packages/core/src/NodeParser.ts +++ b/packages/core/src/nodeParsers/SimpleNodeParser.ts @@ -4,9 +4,10 @@ import { ImageDocument, NodeRelationship, TextNode, -} from "./Node"; -import { SentenceSplitter } from "./TextSplitter"; -import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants"; +} from "../Node"; +import { SentenceSplitter } from "../TextSplitter"; +import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "../constants"; +import { NodeParser } from "./types"; /** * Splits the text of a document into smaller parts. @@ -14,25 +15,23 @@ import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants"; * @param textSplitter - The text splitter to use. * @returns An array of text splits. */ -export function getTextSplitsFromDocument( +function getTextSplitsFromDocument( document: Document, textSplitter: SentenceSplitter, ) { const text = document.getText(); - const splits = textSplitter.splitText(text); - - return splits; + return textSplitter.splitText(text); } /** * Generates an array of nodes from a document. - * @param document - The document to generate nodes from. + * @param doc * @param textSplitter - The text splitter to use. * @param includeMetadata - Whether to include metadata in the nodes. * @param includePrevNextRel - Whether to include previous and next relationships in the nodes. * @returns An array of nodes. */ -export function getNodesFromDocument( +function getNodesFromDocument( doc: BaseNode, textSplitter: SentenceSplitter, includeMetadata: boolean = true, @@ -74,18 +73,6 @@ export function getNodesFromDocument( return nodes; } -/** - * A NodeParser generates Nodes from Documents - */ -export interface NodeParser { - /** - * Generates an array of nodes from an array of documents. - * @param documents - The documents to generate nodes from. - * @returns An array of nodes. - */ - getNodesFromDocuments(documents: BaseNode[]): BaseNode[]; -} - /** * SimpleNodeParser is the default NodeParser. It splits documents into TextNodes using a splitter, by default SentenceSplitter */ diff --git a/packages/core/src/nodeParsers/index.ts b/packages/core/src/nodeParsers/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..4094a976711a71da70cbd7cac74ae897c3b14893 --- /dev/null +++ b/packages/core/src/nodeParsers/index.ts @@ -0,0 +1,2 @@ +export * from "./SimpleNodeParser"; +export * from "./types"; diff --git a/packages/core/src/nodeParsers/types.ts b/packages/core/src/nodeParsers/types.ts new file mode 100644 index 0000000000000000000000000000000000000000..23a4a6f4887469d799a2df039f86de8e485ec2e7 --- /dev/null +++ b/packages/core/src/nodeParsers/types.ts @@ -0,0 +1,13 @@ +import { BaseNode } from "../Node"; + +/** + * A NodeParser generates Nodes from Documents + */ +export interface NodeParser { + /** + * Generates an array of nodes from an array of documents. + * @param documents - The documents to generate nodes from. + * @returns An array of nodes. + */ + getNodesFromDocuments(documents: BaseNode[]): BaseNode[]; +}