From 03ed2b4129590e137021d3bbe5e45cb0f5422fad Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Fri, 5 Jan 2024 16:54:12 +0700
Subject: [PATCH] feat: add sentenceWindow.ts example
---
examples/package.json | 2 +-
examples/sentenceWindow.ts | 36 ++++++++++++++++++++++++++++++++++++
packages/core/src/index.ts | 1 +
3 files changed, 38 insertions(+), 1 deletion(-)
create mode 100644 examples/sentenceWindow.ts
diff --git a/examples/package.json b/examples/package.json
index d7c4023a1..07e116249 100644
--- a/examples/package.json
+++ b/examples/package.json
@@ -1,7 +1,7 @@
{
"version": "0.0.3",
"private": true,
- "name": "simple",
+ "name": "examples",
"dependencies": {
"@notionhq/client": "^2.2.13",
"@datastax/astra-db-ts": "^0.1.2",
diff --git a/examples/sentenceWindow.ts b/examples/sentenceWindow.ts
new file mode 100644
index 000000000..66b4fd9ad
--- /dev/null
+++ b/examples/sentenceWindow.ts
@@ -0,0 +1,36 @@
+import {
+ Document,
+ MetadataReplacementPostProcessor,
+ SentenceWindowNodeParser,
+ VectorStoreIndex,
+ serviceContextFromDefaults,
+} from "llamaindex";
+import essay from "./essay";
+
+async function main() {
+ const document = new Document({ text: essay, id_: "essay" });
+
+ // create service context with sentence window parser
+ const nodeParser = new SentenceWindowNodeParser({
+ windowSize: 3,
+ windowMetadataKey: "window",
+ originalTextMetadataKey: "original_text",
+ });
+ const serviceContext = serviceContextFromDefaults({ nodeParser });
+
+ // Split text and create embeddings. Store them in a VectorStoreIndex
+ const index = await VectorStoreIndex.fromDocuments([document], {
+ serviceContext,
+ });
+
+ // Query the index
+ const queryEngine = index.asQueryEngine({
+ nodePostprocessors: [new MetadataReplacementPostProcessor("window")],
+ });
+ const response = await queryEngine.query("What mistakes did they make?");
+
+ // Output response
+ console.log(response.toString());
+}
+
+main().catch(console.error);
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index e995ca015..8676bd393 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -18,6 +18,7 @@ export * from "./embeddings";
export * from "./indices";
export * from "./llm";
export * from "./nodeParsers";
+export * from "./postprocessors";
export * from "./readers/AssemblyAI";
export * from "./readers/CSVReader";
export * from "./readers/HTMLReader";
--
GitLab