diff --git a/.changeset/late-apricots-yell.md b/.changeset/late-apricots-yell.md new file mode 100644 index 0000000000000000000000000000000000000000..96c8694587b54cb29fe59e5b953deb46c0151b84 --- /dev/null +++ b/.changeset/late-apricots-yell.md @@ -0,0 +1,5 @@ +--- +"@llamaindex/core": patch +--- + +Remove chunk size limit for prompt helper (use LLM default) diff --git a/packages/core/src/indices/prompt-helper.ts b/packages/core/src/indices/prompt-helper.ts index 960ffe31f7ed156f8e65e26619421e45aabdd1cb..477c5037cb1b06841fa422adfef0fe9392ba31e8 100644 --- a/packages/core/src/indices/prompt-helper.ts +++ b/packages/core/src/indices/prompt-helper.ts @@ -1,7 +1,6 @@ import { type Tokenizer, tokenizers } from "@llamaindex/env"; import { DEFAULT_CHUNK_OVERLAP_RATIO, - DEFAULT_CHUNK_SIZE, DEFAULT_CONTEXT_WINDOW, DEFAULT_NUM_OUTPUTS, DEFAULT_PADDING, @@ -171,7 +170,7 @@ export class PromptHelper { ) { const { chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO, - chunkSizeLimit = DEFAULT_CHUNK_SIZE, + chunkSizeLimit = undefined, tokenizer = Settings.tokenizer, separator = " ", } = options ?? {};