-
- Downloads
feat: node parser refactor (#1065)
Showing
- .changeset/young-schools-kneel.md 11 additions, 0 deletions.changeset/young-schools-kneel.md
- examples/longText.ts 2 additions, 3 deletionsexamples/longText.ts
- examples/split.ts 1 addition, 1 deletionexamples/split.ts
- packages/core/package.json 16 additions, 1 deletionpackages/core/package.json
- packages/core/src/embeddings/base.ts 1 addition, 3 deletionspackages/core/src/embeddings/base.ts
- packages/core/src/global/settings.ts 15 additions, 0 deletionspackages/core/src/global/settings.ts
- packages/core/src/global/settings/callback-manager.ts 23 additions, 6 deletionspackages/core/src/global/settings/callback-manager.ts
- packages/core/src/global/settings/chunk-size.ts 2 additions, 2 deletionspackages/core/src/global/settings/chunk-size.ts
- packages/core/src/global/settings/tokenizer.ts 21 additions, 0 deletionspackages/core/src/global/settings/tokenizer.ts
- packages/core/src/node-parser/base.ts 147 additions, 0 deletionspackages/core/src/node-parser/base.ts
- packages/core/src/node-parser/index.ts 24 additions, 0 deletionspackages/core/src/node-parser/index.ts
- packages/core/src/node-parser/markdown.ts 87 additions, 0 deletionspackages/core/src/node-parser/markdown.ts
- packages/core/src/node-parser/sentence-splitter.ts 226 additions, 0 deletionspackages/core/src/node-parser/sentence-splitter.ts
- packages/core/src/node-parser/sentence-tokenizer-parser.d.ts 5 additions, 0 deletionspackages/core/src/node-parser/sentence-tokenizer-parser.d.ts
- packages/core/src/node-parser/sentence-tokenizer-parser.js 1571 additions, 0 deletionspackages/core/src/node-parser/sentence-tokenizer-parser.js
- packages/core/src/node-parser/sentence-window.ts 85 additions, 0 deletionspackages/core/src/node-parser/sentence-window.ts
- packages/core/src/node-parser/type.ts 5 additions, 0 deletionspackages/core/src/node-parser/type.ts
- packages/core/src/node-parser/utils.ts 53 additions, 0 deletionspackages/core/src/node-parser/utils.ts
- packages/core/src/schema/node.ts 45 additions, 0 deletionspackages/core/src/schema/node.ts
- packages/core/src/schema/type.ts 5 additions, 2 deletionspackages/core/src/schema/type.ts
Loading
Please register or sign in to comment