fixed tokenizer decoder

.changeset/silver-ties-walk.md

+---
+"llamaindex": patch
+---
+
+Fixed tokenizer decoder

packages/core/src/GlobalsHelper.ts

@@ -5,19 +5,27 @@ import { v4 as uuidv4 } from "uuid";
  * Helper class singleton
  */
 class GlobalsHelper {
-  defaultTokenizer: ((text: string) => string[]) | null = null;
+  defaultTokenizer: {
+    encode: (text: string) => number[];
+    decode: (tokens: number[]) => string;
+  } | null = null;
 
   tokenizer() {
-    if (this.defaultTokenizer) {
-      return this.defaultTokenizer;
+    if (!this.defaultTokenizer) {
+      const tiktoken = require("tiktoken-node");
+      this.defaultTokenizer = tiktoken.getEncoding("gpt2");
     }
 
-    const tiktoken = require("tiktoken-node");
-    let enc = new tiktoken.getEncoding("gpt2");
-    this.defaultTokenizer = (text: string) => {
-      return enc.encode(text);
-    };
-    return this.defaultTokenizer;
+    return this.defaultTokenizer!.encode.bind(this.defaultTokenizer);
+  }
+
+  tokenizerDecoder() {
+    if (!this.defaultTokenizer) {
+      const tiktoken = require("tiktoken-node");
+      this.defaultTokenizer = tiktoken.getEncoding("gpt2");
+    }
+
+    return this.defaultTokenizer!.decode.bind(this.defaultTokenizer);
   }
 
   createEvent({

packages/core/src/PromptHelper.ts

@@ -34,7 +34,7 @@ export class PromptHelper {
   numOutput = DEFAULT_NUM_OUTPUTS;
   chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO;
   chunkSizeLimit?: number;
-  tokenizer: (text: string) => string[];
+  tokenizer: (text: string) => number[];
   separator = " ";
 
   constructor(
@@ -42,7 +42,7 @@ export class PromptHelper {
     numOutput = DEFAULT_NUM_OUTPUTS,
     chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO,
     chunkSizeLimit?: number,
-    tokenizer?: (text: string) => string[],
+    tokenizer?: (text: string) => number[],
     separator = " "
   ) {
     this.contextWindow = contextWindow;

packages/core/src/TextSplitter.ts

@@ -60,7 +60,7 @@ export class SentenceSplitter {
 
     if (tokenizer == undefined || tokenizerDecoder == undefined) {
       tokenizer = globalsHelper.tokenizer();
-      tokenizerDecoder = globalsHelper.tokenizer;
+      tokenizerDecoder = globalsHelper.tokenizerDecoder();
     }
     this.tokenizer = tokenizer;
     this.tokenizerDecoder = tokenizerDecoder;