migrated to tiktoken lite

Hopefully fixes the Windows issue

packages/core/package.json

@@ -14,7 +14,7 @@
     "pdf-parse": "^1.1.1",
     "rake-modified": "^1.0.8",
     "replicate": "^0.18.1",
-    "tiktoken-node": "^0.0.6",
+    "tiktoken": "^1.0.10",
     "uuid": "^9.0.1",
     "wink-nlp": "^1.14.3"
   },

packages/core/src/GlobalsHelper.ts

+import cl100k_base from "tiktoken/encoders/cl100k_base.json";
+import { Tiktoken } from "tiktoken/lite";
+
 import { v4 as uuidv4 } from "uuid";
 import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
 
@@ -6,14 +9,30 @@ import { Event, EventTag, EventType } from "./callbacks/CallbackManager";
  */
 class GlobalsHelper {
   defaultTokenizer: {
-    encode: (text: string) => number[];
-    decode: (tokens: number[]) => string;
+    encode: (text: string) => Uint32Array;
+    decode: (tokens: Uint32Array) => string;
   } | null = null;
 
+  private initDefaultTokenizer() {
+    const encoding = new Tiktoken(
+      cl100k_base.bpe_ranks,
+      cl100k_base.special_tokens,
+      cl100k_base.pat_str,
+    );
+
+    this.defaultTokenizer = {
+      encode: (text: string) => {
+        return encoding.encode(text);
+      },
+      decode: (tokens: Uint32Array) => {
+        return new TextDecoder().decode(encoding.decode(tokens));
+      },
+    };
+  }
+
   tokenizer() {
     if (!this.defaultTokenizer) {
-      const tiktoken = require("tiktoken-node");
-      this.defaultTokenizer = tiktoken.getEncoding("gpt2");
+      this.initDefaultTokenizer();
     }
 
     return this.defaultTokenizer!.encode.bind(this.defaultTokenizer);
@@ -21,8 +40,7 @@ class GlobalsHelper {
 
   tokenizerDecoder() {
     if (!this.defaultTokenizer) {
-      const tiktoken = require("tiktoken-node");
-      this.defaultTokenizer = tiktoken.getEncoding("gpt2");
+      this.initDefaultTokenizer();
     }
 
     return this.defaultTokenizer!.decode.bind(this.defaultTokenizer);

packages/core/src/PromptHelper.ts

@@ -34,7 +34,7 @@ export class PromptHelper {
   numOutput = DEFAULT_NUM_OUTPUTS;
   chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO;
   chunkSizeLimit?: number;
-  tokenizer: (text: string) => number[];
+  tokenizer: (text: string) => Uint32Array;
   separator = " ";
 
   constructor(
@@ -42,7 +42,7 @@ export class PromptHelper {
     numOutput = DEFAULT_NUM_OUTPUTS,
     chunkOverlapRatio = DEFAULT_CHUNK_OVERLAP_RATIO,
     chunkSizeLimit?: number,
-    tokenizer?: (text: string) => number[],
+    tokenizer?: (text: string) => Uint32Array,
     separator = " ",
   ) {
     this.contextWindow = contextWindow;

packages/core/src/storage/vectorStore/types.ts

@@ -63,6 +63,9 @@ export interface VectorStore {
   client(): any;
   add(embeddingResults: BaseNode[]): Promise<string[]>;
   delete(refDocId: string, deleteKwargs?: any): Promise<void>;
-  query(query: VectorStoreQuery, kwargs?: any): Promise<VectorStoreQueryResult>;
+  query(
+    query: VectorStoreQuery,
+    options?: any,
+  ): Promise<VectorStoreQueryResult>;
   persist(persistPath: string, fs?: GenericFileSystem): Promise<void>;
 }

packages/core/tsconfig.json

@@ -9,7 +9,8 @@
     "noEmit": true,
     "strict": true,
     "lib": ["es2015", "dom"],
-    "target": "ES2015"
+    "target": "ES2015",
+    "resolveJsonModule": true
   },
   "exclude": ["node_modules"]
 }