diff --git a/packages/core/src/global/constants.ts b/packages/core/src/global/constants.ts new file mode 100644 index 0000000000000000000000000000000000000000..4c742d4b92675536d5134437a530bd644ff042af --- /dev/null +++ b/packages/core/src/global/constants.ts @@ -0,0 +1,24 @@ +import { path } from "@llamaindex/env"; + +//#region llm +export const DEFAULT_CONTEXT_WINDOW = 3900; +export const DEFAULT_NUM_OUTPUTS = 256; +export const DEFAULT_CHUNK_SIZE = 1024; +export const DEFAULT_CHUNK_OVERLAP = 20; +export const DEFAULT_CHUNK_OVERLAP_RATIO = 0.1; +export const DEFAULT_PADDING = 5; +//#endregion +//#region storage +export const DEFAULT_COLLECTION = "data"; +export const DEFAULT_PERSIST_DIR = path.join("./storage"); +export const DEFAULT_INDEX_STORE_PERSIST_FILENAME = "index_store.json"; +export const DEFAULT_DOC_STORE_PERSIST_FILENAME = "doc_store.json"; +export const DEFAULT_VECTOR_STORE_PERSIST_FILENAME = "vector_store.json"; +export const DEFAULT_GRAPH_STORE_PERSIST_FILENAME = "graph_store.json"; +export const DEFAULT_NAMESPACE = "docstore"; +export const DEFAULT_IMAGE_VECTOR_NAMESPACE = "images"; +//#endregion +//#region llama cloud +export const DEFAULT_PROJECT_NAME = "Default"; +export const DEFAULT_BASE_URL = "https://api.cloud.llamaindex.ai"; +//#endregion diff --git a/packages/core/src/global/index.ts b/packages/core/src/global/index.ts index d3afd40b98b70ea1dfc8325dedf35a554ef0f6c6..78167fb407f0e42ae0141b418a49bf0e1d88821b 100644 --- a/packages/core/src/global/index.ts +++ b/packages/core/src/global/index.ts @@ -1,3 +1,4 @@ +export * from "./constants"; export { Settings } from "./settings"; export { CallbackManager } from "./settings/callback-manager"; export type { diff --git a/packages/llamaindex/src/PromptHelper.ts b/packages/llamaindex/src/PromptHelper.ts index 40ff503d57742e8859218ab26f4e998706f454fd..c4618ecde56f3fe0baee0cdcea1b48be28a42c2d 100644 --- a/packages/llamaindex/src/PromptHelper.ts +++ b/packages/llamaindex/src/PromptHelper.ts @@ -1,12 +1,12 @@ -import { SentenceSplitter } from "@llamaindex/core/node-parser"; -import type { PromptTemplate } from "@llamaindex/core/prompts"; -import { type Tokenizer, tokenizers } from "@llamaindex/env"; import { DEFAULT_CHUNK_OVERLAP_RATIO, DEFAULT_CONTEXT_WINDOW, DEFAULT_NUM_OUTPUTS, DEFAULT_PADDING, -} from "./constants.js"; +} from "@llamaindex/core/global"; +import { SentenceSplitter } from "@llamaindex/core/node-parser"; +import type { PromptTemplate } from "@llamaindex/core/prompts"; +import { type Tokenizer, tokenizers } from "@llamaindex/env"; /** * Get the empty prompt text given a prompt. diff --git a/packages/llamaindex/src/cloud/LlamaCloudIndex.ts b/packages/llamaindex/src/cloud/LlamaCloudIndex.ts index ebd25a94334a7e77470c3690adbdacf168502a52..d29b52e31ba60a339a907f6cd0666775b4c76f42 100644 --- a/packages/llamaindex/src/cloud/LlamaCloudIndex.ts +++ b/packages/llamaindex/src/cloud/LlamaCloudIndex.ts @@ -7,7 +7,7 @@ import type { QueryEngine } from "../types.js"; import type { CloudRetrieveParams } from "./LlamaCloudRetriever.js"; import { LlamaCloudRetriever } from "./LlamaCloudRetriever.js"; import { getPipelineCreate } from "./config.js"; -import type { CloudConstructorParams } from "./constants.js"; +import type { CloudConstructorParams } from "./type.js"; import { getAppBaseUrl, getProjectId, initService } from "./utils.js"; import { PipelinesService, ProjectsService } from "@llamaindex/cloud/api"; diff --git a/packages/llamaindex/src/cloud/LlamaCloudRetriever.ts b/packages/llamaindex/src/cloud/LlamaCloudRetriever.ts index b75f48f90d2908deb03bd98b1c9c803f013e399e..a947dfb6022eec99d0379e85109db8963f90b60f 100644 --- a/packages/llamaindex/src/cloud/LlamaCloudRetriever.ts +++ b/packages/llamaindex/src/cloud/LlamaCloudRetriever.ts @@ -4,13 +4,12 @@ import { type RetrievalParams, type TextNodeWithScore, } from "@llamaindex/cloud/api"; -import { Settings } from "@llamaindex/core/global"; +import { DEFAULT_PROJECT_NAME, Settings } from "@llamaindex/core/global"; import type { NodeWithScore } from "@llamaindex/core/schema"; import { jsonToNode, ObjectType } from "@llamaindex/core/schema"; import { extractText, wrapEventCaller } from "@llamaindex/core/utils"; import type { BaseRetriever, RetrieveParams } from "../Retriever.js"; -import type { ClientParams, CloudConstructorParams } from "./constants.js"; -import { DEFAULT_PROJECT_NAME } from "./constants.js"; +import type { ClientParams, CloudConstructorParams } from "./type.js"; import { getProjectId, initService } from "./utils.js"; export type CloudRetrieveParams = Omit< diff --git a/packages/llamaindex/src/cloud/index.ts b/packages/llamaindex/src/cloud/index.ts index d1c16b0f01e1a256298a2b718f6f07736e8230ec..99768801432d89f5d742bca7dc8f4d7b79855a44 100644 --- a/packages/llamaindex/src/cloud/index.ts +++ b/packages/llamaindex/src/cloud/index.ts @@ -1,7 +1,7 @@ -export type { CloudConstructorParams } from "./constants.js"; export { LLamaCloudFileService } from "./LLamaCloudFileService.js"; export { LlamaCloudIndex } from "./LlamaCloudIndex.js"; export { LlamaCloudRetriever, type CloudRetrieveParams, } from "./LlamaCloudRetriever.js"; +export type { CloudConstructorParams } from "./type.js"; diff --git a/packages/llamaindex/src/cloud/constants.ts b/packages/llamaindex/src/cloud/type.ts similarity index 71% rename from packages/llamaindex/src/cloud/constants.ts rename to packages/llamaindex/src/cloud/type.ts index 7d75e13613cc5d2b33f755af9d506412f3bab675..cd77474c3fdc8fbd9cdbab6e6a31a6c541507aa5 100644 --- a/packages/llamaindex/src/cloud/constants.ts +++ b/packages/llamaindex/src/cloud/type.ts @@ -1,8 +1,5 @@ import type { ServiceContext } from "../ServiceContext.js"; -export const DEFAULT_PROJECT_NAME = "Default"; -export const DEFAULT_BASE_URL = "https://api.cloud.llamaindex.ai"; - export type ClientParams = { apiKey?: string; baseUrl?: string }; export type CloudConstructorParams = { diff --git a/packages/llamaindex/src/cloud/utils.ts b/packages/llamaindex/src/cloud/utils.ts index b31ae9cf1c8d16a262a212da754d4e29d98536bd..033a37c7fe89590388febc917b2a65648f316bbe 100644 --- a/packages/llamaindex/src/cloud/utils.ts +++ b/packages/llamaindex/src/cloud/utils.ts @@ -1,7 +1,7 @@ import { OpenAPI, ProjectsService } from "@llamaindex/cloud/api"; +import { DEFAULT_BASE_URL } from "@llamaindex/core/global"; import { getEnv } from "@llamaindex/env"; -import type { ClientParams } from "./constants.js"; -import { DEFAULT_BASE_URL } from "./constants.js"; +import type { ClientParams } from "./type.js"; function getBaseUrl(baseUrl?: string): string { return baseUrl ?? getEnv("LLAMA_CLOUD_BASE_URL") ?? DEFAULT_BASE_URL; diff --git a/packages/llamaindex/src/constants.ts b/packages/llamaindex/src/constants.ts deleted file mode 100644 index 004d5d3e8b028b92ef28ecc9d799438ee439e81d..0000000000000000000000000000000000000000 --- a/packages/llamaindex/src/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -export const DEFAULT_CONTEXT_WINDOW = 3900; -export const DEFAULT_NUM_OUTPUTS = 256; - -export const DEFAULT_CHUNK_SIZE = 1024; -export const DEFAULT_CHUNK_OVERLAP = 20; -export const DEFAULT_CHUNK_OVERLAP_RATIO = 0.1; - -export const DEFAULT_PADDING = 5; diff --git a/packages/llamaindex/src/index.edge.ts b/packages/llamaindex/src/index.edge.ts index 4332921b458b0d45b7cfe8eaa096fb86445be889..86908724560970e815f1c96866ba27bdc614bcaf 100644 --- a/packages/llamaindex/src/index.edge.ts +++ b/packages/llamaindex/src/index.edge.ts @@ -1,6 +1,36 @@ import type { AgentEndEvent, AgentStartEvent } from "./agent/types.js"; import type { RetrievalEndEvent, RetrievalStartEvent } from "./llm/types.js"; +export { + CallbackManager, + DEFAULT_BASE_URL, + DEFAULT_CHUNK_OVERLAP, + DEFAULT_CHUNK_OVERLAP_RATIO, + DEFAULT_CHUNK_SIZE, + DEFAULT_COLLECTION, + DEFAULT_CONTEXT_WINDOW, + DEFAULT_DOC_STORE_PERSIST_FILENAME, + DEFAULT_GRAPH_STORE_PERSIST_FILENAME, + DEFAULT_IMAGE_VECTOR_NAMESPACE, + DEFAULT_INDEX_STORE_PERSIST_FILENAME, + DEFAULT_NAMESPACE, + DEFAULT_NUM_OUTPUTS, + DEFAULT_PADDING, + DEFAULT_PERSIST_DIR, + DEFAULT_PROJECT_NAME, + DEFAULT_VECTOR_STORE_PERSIST_FILENAME, +} from "@llamaindex/core/global"; +export type { + JSONArray, + JSONObject, + JSONValue, + LLMEndEvent, + LLMStartEvent, + LLMStreamEvent, + LLMToolCallEvent, + LLMToolResultEvent, +} from "@llamaindex/core/global"; +export * from "@llamaindex/core/llms"; export * from "@llamaindex/core/prompts"; export * from "@llamaindex/core/schema"; @@ -14,23 +44,11 @@ declare module "@llamaindex/core/global" { } } -export { CallbackManager } from "@llamaindex/core/global"; -export type { - JSONArray, - JSONObject, - JSONValue, - LLMEndEvent, - LLMStartEvent, - LLMStreamEvent, - LLMToolCallEvent, - LLMToolResultEvent, -} from "@llamaindex/core/global"; export * from "@llamaindex/core/llms"; export * from "@llamaindex/core/schema"; export * from "./agent/index.js"; export * from "./ChatHistory.js"; export * from "./cloud/index.js"; -export * from "./constants.js"; export * from "./embeddings/index.js"; export * from "./engines/chat/index.js"; export * from "./engines/query/index.js"; diff --git a/packages/llamaindex/src/storage/StorageContext.ts b/packages/llamaindex/src/storage/StorageContext.ts index 15e13ec2dfb7a2a752892f62e5fafc56e04f7337..b1b4a392772146725ba5913ed1c0a738e574a48c 100644 --- a/packages/llamaindex/src/storage/StorageContext.ts +++ b/packages/llamaindex/src/storage/StorageContext.ts @@ -1,10 +1,10 @@ -import { ModalityType, ObjectType } from "@llamaindex/core/schema"; -import { path } from "@llamaindex/env"; -import { getImageEmbedModel } from "../internal/settings/image-embed-model.js"; import { DEFAULT_IMAGE_VECTOR_NAMESPACE, DEFAULT_NAMESPACE, -} from "./constants.js"; +} from "@llamaindex/core/global"; +import { ModalityType, ObjectType } from "@llamaindex/core/schema"; +import { path } from "@llamaindex/env"; +import { getImageEmbedModel } from "../internal/settings/image-embed-model.js"; import { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js"; import type { BaseDocumentStore } from "./docStore/types.js"; import { SimpleIndexStore } from "./indexStore/SimpleIndexStore.js"; diff --git a/packages/llamaindex/src/storage/constants.ts b/packages/llamaindex/src/storage/constants.ts deleted file mode 100644 index 8d58a29a7af9a81f3f3dfa6610eb6c53bb7204fa..0000000000000000000000000000000000000000 --- a/packages/llamaindex/src/storage/constants.ts +++ /dev/null @@ -1,8 +0,0 @@ -export const DEFAULT_COLLECTION = "data"; -export const DEFAULT_PERSIST_DIR = "./storage"; -export const DEFAULT_INDEX_STORE_PERSIST_FILENAME = "index_store.json"; -export const DEFAULT_DOC_STORE_PERSIST_FILENAME = "doc_store.json"; -export const DEFAULT_VECTOR_STORE_PERSIST_FILENAME = "vector_store.json"; -export const DEFAULT_GRAPH_STORE_PERSIST_FILENAME = "graph_store.json"; -export const DEFAULT_NAMESPACE = "docstore"; -export const DEFAULT_IMAGE_VECTOR_NAMESPACE = "images"; diff --git a/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts b/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts index b239c4c69b464d54ceada064a1136c59a8fe4cf0..0e65af62ae90c5f33c088487562e9bba17901692 100644 --- a/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts +++ b/packages/llamaindex/src/storage/docStore/KVDocumentStore.ts @@ -1,7 +1,7 @@ +import { DEFAULT_NAMESPACE } from "@llamaindex/core/global"; import type { BaseNode } from "@llamaindex/core/schema"; import { ObjectType } from "@llamaindex/core/schema"; import _ from "lodash"; -import { DEFAULT_NAMESPACE } from "../constants.js"; import type { BaseKVStore } from "../kvStore/types.js"; import type { RefDocInfo } from "./types.js"; import { BaseDocumentStore } from "./types.js"; diff --git a/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts b/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts index 710b72d297d18068a75e563025cb19eac8762c15..41ee129001a2b9dab3c05112e8427c6fa86ec611 100644 --- a/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts +++ b/packages/llamaindex/src/storage/docStore/SimpleDocumentStore.ts @@ -1,10 +1,10 @@ -import { path } from "@llamaindex/env"; -import _ from "lodash"; import { DEFAULT_DOC_STORE_PERSIST_FILENAME, DEFAULT_NAMESPACE, DEFAULT_PERSIST_DIR, -} from "../constants.js"; +} from "@llamaindex/core/global"; +import { path } from "@llamaindex/env"; +import _ from "lodash"; import { SimpleKVStore } from "../kvStore/SimpleKVStore.js"; import { BaseInMemoryKVStore } from "../kvStore/types.js"; import { KVDocumentStore } from "./KVDocumentStore.js"; diff --git a/packages/llamaindex/src/storage/docStore/types.ts b/packages/llamaindex/src/storage/docStore/types.ts index 8c2fd5a49762d97cdce4ec235dde6bcb08c69e23..eed71e34351029d3a8c00f5e6fbc8af25b14341d 100644 --- a/packages/llamaindex/src/storage/docStore/types.ts +++ b/packages/llamaindex/src/storage/docStore/types.ts @@ -1,8 +1,8 @@ -import { BaseNode } from "@llamaindex/core/schema"; import { DEFAULT_DOC_STORE_PERSIST_FILENAME, DEFAULT_PERSIST_DIR, -} from "../constants.js"; +} from "@llamaindex/core/global"; +import { BaseNode } from "@llamaindex/core/schema"; const defaultPersistPath = `${DEFAULT_PERSIST_DIR}/${DEFAULT_DOC_STORE_PERSIST_FILENAME}`; diff --git a/packages/llamaindex/src/storage/index.ts b/packages/llamaindex/src/storage/index.ts index d839410e98c312009362b7ffcf39ab681c36deda..27af70716d7566673d22cf0277bd7b7fd0c178bc 100644 --- a/packages/llamaindex/src/storage/index.ts +++ b/packages/llamaindex/src/storage/index.ts @@ -1,6 +1,5 @@ export { SimpleChatStore } from "./chatStore/SimpleChatStore.js"; export * from "./chatStore/types.js"; -export * from "./constants.js"; export { SimpleDocumentStore } from "./docStore/SimpleDocumentStore.js"; export * from "./docStore/types.js"; export * from "./FileSystem.js"; diff --git a/packages/llamaindex/src/storage/indexStore/KVIndexStore.ts b/packages/llamaindex/src/storage/indexStore/KVIndexStore.ts index 87ce07fa267975354910e9006f98e69c01ba9a84..38736333a3e0234844aa9b4b7a385f37b19cc407 100644 --- a/packages/llamaindex/src/storage/indexStore/KVIndexStore.ts +++ b/packages/llamaindex/src/storage/indexStore/KVIndexStore.ts @@ -1,7 +1,7 @@ +import { DEFAULT_NAMESPACE } from "@llamaindex/core/global"; import _ from "lodash"; import type { IndexStruct } from "../../indices/IndexStruct.js"; import { jsonToIndexStruct } from "../../indices/json-to-index-struct.js"; -import { DEFAULT_NAMESPACE } from "../constants.js"; import type { BaseKVStore } from "../kvStore/types.js"; import { BaseIndexStore } from "./types.js"; diff --git a/packages/llamaindex/src/storage/indexStore/SimpleIndexStore.ts b/packages/llamaindex/src/storage/indexStore/SimpleIndexStore.ts index 5447b5638f61dd21ff3e5ae749006114badc8f1c..516024be54ef341b264482ba6437ad054451b73f 100644 --- a/packages/llamaindex/src/storage/indexStore/SimpleIndexStore.ts +++ b/packages/llamaindex/src/storage/indexStore/SimpleIndexStore.ts @@ -1,8 +1,8 @@ -import { path } from "@llamaindex/env"; import { DEFAULT_INDEX_STORE_PERSIST_FILENAME, DEFAULT_PERSIST_DIR, -} from "../constants.js"; +} from "@llamaindex/core/global"; +import { path } from "@llamaindex/env"; import type { DataType } from "../kvStore/SimpleKVStore.js"; import { SimpleKVStore } from "../kvStore/SimpleKVStore.js"; import type { BaseInMemoryKVStore } from "../kvStore/types.js"; diff --git a/packages/llamaindex/src/storage/indexStore/types.ts b/packages/llamaindex/src/storage/indexStore/types.ts index e503af83c2b3c4e6528a52e24ba82dce48a8ca7b..2cab6aa77ff284c8951c55919fee7d9173e6782e 100644 --- a/packages/llamaindex/src/storage/indexStore/types.ts +++ b/packages/llamaindex/src/storage/indexStore/types.ts @@ -1,10 +1,14 @@ -import type { IndexStruct } from "../../indices/IndexStruct.js"; import { DEFAULT_INDEX_STORE_PERSIST_FILENAME, DEFAULT_PERSIST_DIR, -} from "../constants.js"; +} from "@llamaindex/core/global"; +import { path } from "@llamaindex/env"; +import type { IndexStruct } from "../../indices/IndexStruct.js"; -const defaultPersistPath = `${DEFAULT_PERSIST_DIR}/${DEFAULT_INDEX_STORE_PERSIST_FILENAME}`; +const defaultPersistPath = path.join( + DEFAULT_PERSIST_DIR, + DEFAULT_INDEX_STORE_PERSIST_FILENAME, +); export abstract class BaseIndexStore { abstract getIndexStructs(): Promise<IndexStruct[]>; diff --git a/packages/llamaindex/src/storage/kvStore/SimpleKVStore.ts b/packages/llamaindex/src/storage/kvStore/SimpleKVStore.ts index 07c3a696e3a5d1301a0d1b0512937e207d0d25c6..38b3335285c7fe9bf46064fb9a17550f833f92c1 100644 --- a/packages/llamaindex/src/storage/kvStore/SimpleKVStore.ts +++ b/packages/llamaindex/src/storage/kvStore/SimpleKVStore.ts @@ -1,6 +1,6 @@ +import { DEFAULT_COLLECTION } from "@llamaindex/core/global"; import { fs, path } from "@llamaindex/env"; import { exists } from "../FileSystem.js"; -import { DEFAULT_COLLECTION } from "../constants.js"; import { BaseKVStore } from "./types.js"; export type DataType = Record<string, Record<string, any>>; diff --git a/packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts b/packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts index 342a52966b465afff43334ad95139a8cf81db7b0..7c9c380dc13e28961f88c910b02d6ae28e4c36e3 100644 --- a/packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts +++ b/packages/llamaindex/src/storage/vectorStore/SimpleVectorStore.ts @@ -1,4 +1,5 @@ import type { BaseEmbedding } from "@llamaindex/core/embeddings"; +import { DEFAULT_PERSIST_DIR } from "@llamaindex/core/global"; import type { BaseNode } from "@llamaindex/core/schema"; import { fs, path } from "@llamaindex/env"; import { @@ -6,7 +7,6 @@ import { getTopKMMREmbeddings, } from "../../internal/utils.js"; import { exists } from "../FileSystem.js"; -import { DEFAULT_PERSIST_DIR } from "../constants.js"; import { FilterOperator, VectorStoreBase,