From d09614205fec9d57c78e98fbea38fa1c1f0a4c41 Mon Sep 17 00:00:00 2001
From: Sourabh Desai <sourabhdesai@gmail.com>
Date: Wed, 21 Jun 2023 22:39:26 +0000
Subject: [PATCH] small bug/syntax fixes
---
packages/core/src/storage/StorageContext.ts | 10 +++++-----
packages/core/src/storage/docStore/KVDocumentStore.ts | 8 ++++----
.../core/src/storage/docStore/SimpleDocumentStore.ts | 8 ++++----
packages/core/src/storage/docStore/types.ts | 2 +-
packages/core/src/storage/docStore/utils.ts | 4 ++--
packages/core/src/storage/indexStore/KVIndexStore.ts | 2 +-
.../core/src/storage/vectorStore/SimpleVectorStore.ts | 1 +
7 files changed, 18 insertions(+), 17 deletions(-)
diff --git a/packages/core/src/storage/StorageContext.ts b/packages/core/src/storage/StorageContext.ts
index c3e88ccbd..c39893e65 100644
--- a/packages/core/src/storage/StorageContext.ts
+++ b/packages/core/src/storage/StorageContext.ts
@@ -5,7 +5,7 @@ import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore";
import { SimpleIndexStore } from "./indexStore/SimpleIndexStore";
import { SimpleVectorStore } from "./vectorStore/SimpleVectorStore";
import { GenericFileSystem } from "./FileSystem";
-import { DEFAULT_PERSIST_DIR, DEFAULT_FS } from "./constants";
+import { DEFAULT_PERSIST_DIR, DEFAULT_FS, DEFAULT_NAMESPACE } from "./constants";
export interface StorageContext {
docStore?: BaseDocumentStore;
@@ -21,16 +21,16 @@ type BuilderParams = {
fs?: GenericFileSystem,
};
-export function storageContextFromDefaults({
+export async function storageContextFromDefaults({
docStore, indexStore, vectorStore, persistDir, fs
}: BuilderParams): StorageContext {
persistDir = persistDir || DEFAULT_PERSIST_DIR;
fs = fs || DEFAULT_FS;
- docStore = docStore || SimpleDocumentStore.fromPersistDir(persistDir, fs=fs);
- indexStore = indexStore || SimpleIndexStore.fromPersistDir(persistDir, fs=fs);
- vectorStore = vectorStore || SimpleVectorStore.fromPersistDir(persistDir, fs=fs);
+ docStore = docStore || await SimpleDocumentStore.fromPersistDir(persistDir, DEFAULT_NAMESPACE, fs);
+ indexStore = indexStore || await SimpleIndexStore.fromPersistDir(persistDir, fs);
+ vectorStore = vectorStore || await SimpleVectorStore.fromPersistDir(persistDir, fs);
return {
docStore,
diff --git a/packages/core/src/storage/docStore/KVDocumentStore.ts b/packages/core/src/storage/docStore/KVDocumentStore.ts
index 772221276..7f72bc266 100644
--- a/packages/core/src/storage/docStore/KVDocumentStore.ts
+++ b/packages/core/src/storage/docStore/KVDocumentStore.ts
@@ -3,7 +3,7 @@ import { BaseDocument } from '../../Document';
import { BaseDocumentStore, RefDocInfo } from './types';
import { BaseKVStore } from '../kvStore/types';
import _, * as lodash from 'lodash';
-import { docToJson, jsonToDoc } from './docstore-utils';
+import { docToJson, jsonToDoc } from './utils';
import { DEFAULT_NAMESPACE } from '../constants';
type DocMetaData = { docHash: string, refDocId?: string };
@@ -22,11 +22,11 @@ export class KVDocumentStore extends BaseDocumentStore {
this.metadataCollection = `${namespace}/metadata`;
}
- get docs(): Record<string, BaseDocument> {
- let jsonDict = this.kvstore.getAll(this.nodeCollection);
+ async docs(): Promise<Record<string, BaseDocument>> {
+ let jsonDict = await this.kvstore.getAll(this.nodeCollection);
let docs: Record<string, BaseDocument> = {};
for (let key in jsonDict) {
- docs[key] = jsonToDoc(jsonDict[key]);
+ docs[key] = jsonToDoc(jsonDict[key] as Record<string, any>);
}
return docs;
}
diff --git a/packages/core/src/storage/docStore/SimpleDocumentStore.ts b/packages/core/src/storage/docStore/SimpleDocumentStore.ts
index 3d664b4e8..968d6247f 100644
--- a/packages/core/src/storage/docStore/SimpleDocumentStore.ts
+++ b/packages/core/src/storage/docStore/SimpleDocumentStore.ts
@@ -11,7 +11,7 @@ import {
DEFAULT_FS
} from '../constants';
-type SaveDict = {[key: string]: any}; // Replace `any` with the appropriate type if possible.
+type SaveDict = {[key: string]: any};
export class SimpleDocumentStore extends KVDocumentStore {
private kvStore: SimpleKVStore;
@@ -23,13 +23,13 @@ export class SimpleDocumentStore extends KVDocumentStore {
this.kvStore = kvStore;
}
- static fromPersistDir(
+ static async fromPersistDir(
persistDir: string = DEFAULT_PERSIST_DIR,
namespace?: string,
fsModule?: GenericFileSystem
- ): SimpleDocumentStore {
+ ): Promise<SimpleDocumentStore> {
const persistPath = path.join(persistDir, DEFAULT_DOC_STORE_PERSIST_FILENAME);
- return SimpleDocumentStore.fromPersistPath(persistPath, namespace, fsModule);
+ return await SimpleDocumentStore.fromPersistPath(persistPath, namespace, fsModule);
}
static async fromPersistPath(
diff --git a/packages/core/src/storage/docStore/types.ts b/packages/core/src/storage/docStore/types.ts
index bde842a46..9084e2d04 100644
--- a/packages/core/src/storage/docStore/types.ts
+++ b/packages/core/src/storage/docStore/types.ts
@@ -18,7 +18,7 @@ export abstract class BaseDocumentStore {
}
// Main interface
- abstract get docs(): {[key: string]: BaseDocument};
+ abstract docs(): Promise<Record<string, BaseDocument>>;
abstract addDocuments(docs: BaseDocument[], allowUpdate: boolean): void;
diff --git a/packages/core/src/storage/docStore/utils.ts b/packages/core/src/storage/docStore/utils.ts
index 519b5682a..f6f1d23bb 100644
--- a/packages/core/src/storage/docStore/utils.ts
+++ b/packages/core/src/storage/docStore/utils.ts
@@ -3,14 +3,14 @@ import { BaseDocument, NodeType } from '../../Document';
import { DATA_KEY, TYPE_KEY } from '../constants';
-function docToJson(doc: BaseDocument): Record<string, any> {
+export function docToJson(doc: BaseDocument): Record<string, any> {
return {
[DATA_KEY]: JSON.stringify(doc),
[TYPE_KEY]: doc.getType(),
};
}
-function jsonToDoc(docDict: Record<string, any>): BaseDocument {
+export function jsonToDoc(docDict: Record<string, any>): BaseDocument {
let docType = docDict[TYPE_KEY];
let dataDict = docDict[DATA_KEY];
let doc: BaseDocument;
diff --git a/packages/core/src/storage/indexStore/KVIndexStore.ts b/packages/core/src/storage/indexStore/KVIndexStore.ts
index 8e31f7760..7e2f5eed0 100644
--- a/packages/core/src/storage/indexStore/KVIndexStore.ts
+++ b/packages/core/src/storage/indexStore/KVIndexStore.ts
@@ -1,7 +1,7 @@
import { BaseKVStore } from '../kvStore/types';
import { IndexStruct, indexStructToJson, jsonToIndexStruct } from '../../dataStructs';
import _ from 'lodash';
-import { DEFAULT_NAMESPACE } from './constants';
+import { DEFAULT_NAMESPACE } from '../constants';
import { BaseIndexStore } from './types';
export class KVIndexStore extends BaseIndexStore {
diff --git a/packages/core/src/storage/vectorStore/SimpleVectorStore.ts b/packages/core/src/storage/vectorStore/SimpleVectorStore.ts
index 20d4a15fa..339e707e6 100644
--- a/packages/core/src/storage/vectorStore/SimpleVectorStore.ts
+++ b/packages/core/src/storage/vectorStore/SimpleVectorStore.ts
@@ -18,6 +18,7 @@ class SimpleVectorStoreData {
}
export class SimpleVectorStore implements VectorStore {
+ storesText: boolean = false;
private data: SimpleVectorStoreData = new SimpleVectorStoreData();
private fs: GenericFileSystem = DEFAULT_FS;
--
GitLab