diff --git a/package.json b/package.json
index ff79321eb97d42fed707446280bec596be9c379a..14637f2ade3cf975acd9e55cff25b04ca18fa7e5 100644
--- a/package.json
+++ b/package.json
@@ -18,7 +18,7 @@
"prettier": "^2.8.8",
"prettier-plugin-tailwindcss": "^0.3.0",
"ts-jest": "^29.1.0",
- "turbo": "^1.10.6"
+ "turbo": "^1.10.7"
},
"packageManager": "pnpm@7.15.0",
"name": "llamascript"
diff --git a/packages/core/src/BaseIndex.ts b/packages/core/src/BaseIndex.ts
index 0a0d00dfae1982a6273ce072a80366a76132b2ce..dd6e7461b20ea86a6133aad94dd0adf71a014748 100644
--- a/packages/core/src/BaseIndex.ts
+++ b/packages/core/src/BaseIndex.ts
@@ -141,7 +141,12 @@ export class VectorStoreIndex extends BaseIndex<IndexDict> {
vectorStore.add(embeddingResults);
- throw new Error("not implemented");
+ const indexDict = new IndexDict();
+ for (const { node } of embeddingResults) {
+ indexDict.addNode(node);
+ }
+
+ return indexDict;
}
static async fromDocuments(
diff --git a/packages/core/src/GlobalsHelper.ts b/packages/core/src/GlobalsHelper.ts
index 631608cf4c861fdbabda3e1a9b548b945a1e845e..afd4549bae03386299630feed449188dd45cb6d5 100644
--- a/packages/core/src/GlobalsHelper.ts
+++ b/packages/core/src/GlobalsHelper.ts
@@ -1,11 +1,17 @@
class GlobalsHelper {
+ defaultTokenizer: ((text: string) => string[]) | null = null;
+
tokenizer() {
+ if (this.defaultTokenizer) {
+ return this.defaultTokenizer;
+ }
+
const tiktoken = require("tiktoken-node");
let enc = new tiktoken.getEncoding("gpt2");
- const defaultTokenizer = (text: string) => {
+ this.defaultTokenizer = (text: string) => {
return enc.encode(text);
};
- return defaultTokenizer;
+ return this.defaultTokenizer;
}
}
diff --git a/packages/core/src/embeddingUtils.ts b/packages/core/src/embeddingUtils.ts
deleted file mode 100644
index d9d9b7522b5ea617d7853e252bbbb439035858d9..0000000000000000000000000000000000000000
--- a/packages/core/src/embeddingUtils.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-import _ from "lodash";
-import { VectorStoreQueryMode } from "./storage/vectorStore/types";
-
-export function getTopKEmbeddings(
- queryEmbedding: number[],
- embeddings: number[][],
- similarityFn?: (queryEmbedding: number[], emb: number[]) => number,
- similarityTopK?: number,
- embeddingIds?: number[],
- similarityCutoff?: number
-): [number[], number[]] {
- throw new Error("Not implemented");
-}
-
-export function getTopKEmbeddingsLearner(
- queryEmbedding: number[],
- embeddings: number[][],
- similarityTopK?: number,
- embeddingIds?: number[],
- queryMode: VectorStoreQueryMode = VectorStoreQueryMode.SVM
-): [number[], number[]] {
- throw new Error("Not implemented");
-}
-
-export function getTopKMMREmbeddings(
- queryEmbedding: number[],
- embeddings: number[][],
- similarityFn?: (queryEmbedding: number[], emb: number[]) => number,
- similarityTopK?: number,
- embeddingIds?: number[],
- similarityCutoff?: number,
- mmrThreshold?: number
-): [number[], number[]] {
- throw new Error("Not implemented");
-}
diff --git a/packages/core/src/storage/StorageContext.ts b/packages/core/src/storage/StorageContext.ts
index d7c5e03bcd3254586aab19c2427b9f3efed528cc..d5c51c8d2abdd30e09baea086bde63d6bbaa3ea0 100644
--- a/packages/core/src/storage/StorageContext.ts
+++ b/packages/core/src/storage/StorageContext.ts
@@ -32,21 +32,24 @@ export async function storageContextFromDefaults({
persistDir,
fs,
}: BuilderParams): Promise<StorageContext> {
- persistDir = persistDir || DEFAULT_PERSIST_DIR;
-
- fs = fs || DEFAULT_FS;
-
- docStore =
- docStore ||
- (await SimpleDocumentStore.fromPersistDir(
- persistDir,
- DEFAULT_NAMESPACE,
- fs
- ));
- indexStore =
- indexStore || (await SimpleIndexStore.fromPersistDir(persistDir, fs));
- vectorStore =
- vectorStore || (await SimpleVectorStore.fromPersistDir(persistDir, fs));
+ if (!persistDir) {
+ docStore = docStore || new SimpleDocumentStore();
+ indexStore = indexStore || new SimpleIndexStore();
+ vectorStore = vectorStore || new SimpleVectorStore();
+ } else {
+ fs = fs || DEFAULT_FS;
+ docStore =
+ docStore ||
+ (await SimpleDocumentStore.fromPersistDir(
+ persistDir,
+ DEFAULT_NAMESPACE,
+ fs
+ ));
+ indexStore =
+ indexStore || (await SimpleIndexStore.fromPersistDir(persistDir, fs));
+ vectorStore =
+ vectorStore || (await SimpleVectorStore.fromPersistDir(persistDir, fs));
+ }
return {
docStore,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 0fc66c118bc9d0f9dfe1fcce910ebc49b0abf2ee..0bd001b095993916642c99b576755f9e6b699b28 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -36,8 +36,8 @@ importers:
specifier: ^29.1.0
version: 29.1.0(@babel/core@7.22.5)(jest@29.5.0)(typescript@4.9.5)
turbo:
- specifier: ^1.10.6
- version: 1.10.6
+ specifier: ^1.10.7
+ version: 1.10.7
apps/docs:
dependencies:
@@ -5444,65 +5444,65 @@ packages:
resolution: {integrity: sha512-C3TaO7K81YvjCgQH9Q1S3R3P3BtN3RIM8n+OvX4il1K1zgE8ZhI0op7kClgkxtutIE8hQrcrHBXvIheqKUUCxw==}
dev: true
- /turbo-darwin-64@1.10.6:
- resolution: {integrity: sha512-s2Gc7i9Ud+H9GDcrGJjPIyscJfzDGQ6il4Sl2snfvwngJs4/TaqKuBoX3HNt/7F4NiFRs7ZhlLV1/Yu9zGBRhw==}
+ /turbo-darwin-64@1.10.7:
+ resolution: {integrity: sha512-N2MNuhwrl6g7vGuz4y3fFG2aR1oCs0UZ5HKl8KSTn/VC2y2YIuLGedQ3OVbo0TfEvygAlF3QGAAKKtOCmGPNKA==}
cpu: [x64]
os: [darwin]
requiresBuild: true
dev: true
optional: true
- /turbo-darwin-arm64@1.10.6:
- resolution: {integrity: sha512-tgl70t5PBLyRcNTdP9N6NjvdvQ5LUk8Z60JGUhBhnc+oCOdA4pltrDJNPyel3tQAXXt1dDpl8pp9vUrbwoVyGg==}
+ /turbo-darwin-arm64@1.10.7:
+ resolution: {integrity: sha512-WbJkvjU+6qkngp7K4EsswOriO3xrNQag7YEGRtfLoDdMTk4O4QTeU6sfg2dKfDsBpTidTvEDwgIYJhYVGzrz9Q==}
cpu: [arm64]
os: [darwin]
requiresBuild: true
dev: true
optional: true
- /turbo-linux-64@1.10.6:
- resolution: {integrity: sha512-h7eyAA3xtAVpamcYJYUwe0xm0LWdbv7/I7QiM09AZ67TTNpyUgqW8giFN3h793BHEQ2Rcnk9FNkpIbjWBbyamg==}
+ /turbo-linux-64@1.10.7:
+ resolution: {integrity: sha512-x1CF2CDP1pDz/J8/B2T0hnmmOQI2+y11JGIzNP0KtwxDM7rmeg3DDTtDM/9PwGqfPotN9iVGgMiMvBuMFbsLhg==}
cpu: [x64]
os: [linux]
requiresBuild: true
dev: true
optional: true
- /turbo-linux-arm64@1.10.6:
- resolution: {integrity: sha512-8cZhOeLqu3QZ27yLd6bw4FNaB8y5pLdWeRLJeiWHkIb/cptKnRKJFP+keBJzJi8ovaMqdBpabrxiBRN2lhau5Q==}
+ /turbo-linux-arm64@1.10.7:
+ resolution: {integrity: sha512-JtnBmaBSYbs7peJPkXzXxsRGSGBmBEIb6/kC8RRmyvPAMyqF8wIex0pttsI+9plghREiGPtRWv/lfQEPRlXnNQ==}
cpu: [arm64]
os: [linux]
requiresBuild: true
dev: true
optional: true
- /turbo-windows-64@1.10.6:
- resolution: {integrity: sha512-qx5jcfCJodN1Mh0KtSVQau7pK8CxDvtif7+joPHI2HbQPAADgdUl0LHfA5tFHh6aWgfvhxbvIXqJd6v7Mqkj9g==}
+ /turbo-windows-64@1.10.7:
+ resolution: {integrity: sha512-7A/4CByoHdolWS8dg3DPm99owfu1aY/W0V0+KxFd0o2JQMTQtoBgIMSvZesXaWM57z3OLsietFivDLQPuzE75w==}
cpu: [x64]
os: [win32]
requiresBuild: true
dev: true
optional: true
- /turbo-windows-arm64@1.10.6:
- resolution: {integrity: sha512-vTQaRG3/s2XTreOBr6J9HKFtjzusvwGQg0GtuW2+9Z7fizdzP8MuhaDbN6FhKHcWC81PQPD61TBIKTVTsYOEZg==}
+ /turbo-windows-arm64@1.10.7:
+ resolution: {integrity: sha512-D36K/3b6+hqm9IBAymnuVgyePktwQ+F0lSXr2B9JfAdFPBktSqGmp50JNC7pahxhnuCLj0Vdpe9RqfnJw5zATA==}
cpu: [arm64]
os: [win32]
requiresBuild: true
dev: true
optional: true
- /turbo@1.10.6:
- resolution: {integrity: sha512-0/wbjw4HvmPP1abVWHTdeFRfCA9cn5oxCPP5bDixagLzvDgGWE3xfdlsyGmq779Ekr9vjtDPgC2Y4JlXEhyryw==}
+ /turbo@1.10.7:
+ resolution: {integrity: sha512-xm0MPM28TWx1e6TNC3wokfE5eaDqlfi0G24kmeHupDUZt5Wd0OzHFENEHMPqEaNKJ0I+AMObL6nbSZonZBV2HA==}
hasBin: true
requiresBuild: true
optionalDependencies:
- turbo-darwin-64: 1.10.6
- turbo-darwin-arm64: 1.10.6
- turbo-linux-64: 1.10.6
- turbo-linux-arm64: 1.10.6
- turbo-windows-64: 1.10.6
- turbo-windows-arm64: 1.10.6
+ turbo-darwin-64: 1.10.7
+ turbo-darwin-arm64: 1.10.7
+ turbo-linux-64: 1.10.7
+ turbo-linux-arm64: 1.10.7
+ turbo-windows-64: 1.10.7
+ turbo-windows-arm64: 1.10.7
dev: true
/type-check@0.4.0: