diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000000000000000000000000000000000..de93269a3abf52fa44eaa7c0ed009a4af0576546 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "editor.tabSize": 2, + "editor.formatOnSave": true, + "editor.defaultFormatter": "esbenp.prettier-vscode" +} diff --git a/packages/core/Document.ts b/packages/core/Document.ts new file mode 100644 index 0000000000000000000000000000000000000000..88b4cefa5e350cc1f5c3f9b8d5f9296340355dd1 --- /dev/null +++ b/packages/core/Document.ts @@ -0,0 +1,38 @@ +export interface BaseDocument { + getText(): string; + getDocId(): string; + getDocHash(): string; + getEmbedding(): number[]; +} + +export class Document implements BaseDocument { + docId: string; + text: string; + embedding: number[]; + docHash: string; + + constructor(docId: string, text: string) { + this.docId = docId; + this.text = text; + } + + getText() { + console.log("getText"); + return ""; + } + + getDocId() { + console.log("getDocId"); + return ""; + } + + getDocHash() { + console.log("getDocHash"); + return ""; + } + + getEmbedding() { + console.log("getEmbedding"); + return []; + } +} diff --git a/packages/core/Embedding.ts b/packages/core/Embedding.ts new file mode 100644 index 0000000000000000000000000000000000000000..7ff4e81580911d66f56429d7eb3d5968c7c434e4 --- /dev/null +++ b/packages/core/Embedding.ts @@ -0,0 +1,23 @@ +export enum SimilarityType { + DEFAULT = "cosine", + DOT_PRODUCT = "dot_product", + EUCLIDEAN = "euclidean", +} + +export class BaseEmbedding { + getQueryEmbedding(query: string): number[] { + return []; + } + + getTextEmbedding(text: string): number[] { + return []; + } + + similarity( + embedding1: number[], + embedding2: number[], + mode: SimilarityType + ): number { + return 0; + } +} diff --git a/packages/core/Index.ts b/packages/core/Index.ts new file mode 100644 index 0000000000000000000000000000000000000000..3525e1690b96b41f5333a4d2eca4667185609657 --- /dev/null +++ b/packages/core/Index.ts @@ -0,0 +1,16 @@ +import { Document } from "./Document"; +import { Node } from "./Node"; + +export class BaseIndex { + constructor(nodes?: Node[]) {} + + fromDocuments(documents: Document[]) { + console.log("fromDocuments"); + } + + asQueryEngine() { + console.log("asQueryEngine"); + } +} + +export class VectorStoreIndex extends BaseIndex {} diff --git a/packages/core/LLMPredictor.ts b/packages/core/LLMPredictor.ts new file mode 100644 index 0000000000000000000000000000000000000000..522965a709322a989fbc58d64251fb770de2715a --- /dev/null +++ b/packages/core/LLMPredictor.ts @@ -0,0 +1,27 @@ +export interface BaseLLMPredictor { + getLlmMetadata(): Promise<any>; + predict(prompt: string, options: any): Promise<any>; + stream(prompt: string, options: any): Promise<any>; +} + +export class LLMPredictor implements BaseLLMPredictor { + llm: string; + retryOnThrottling: boolean; + + constructor(llm: string, retryOnThrottling: boolean = true) { + this.llm = llm; + this.retryOnThrottling = retryOnThrottling; + } + + async getLlmMetadata() { + console.log("getLlmMetadata"); + } + + async predict(prompt: string, options: any) { + console.log("predict"); + } + + async stream(prompt: string, options: any) { + console.log("stream"); + } +} diff --git a/packages/core/Node.ts b/packages/core/Node.ts new file mode 100644 index 0000000000000000000000000000000000000000..9be3427ae498ecaf757827b670bd747fd31ee5bd --- /dev/null +++ b/packages/core/Node.ts @@ -0,0 +1,56 @@ +import { BaseDocument } from "./Document"; + +export enum DocumentRelationship { + SOURCE = "source", + PREVIOUS = "previous", + NEXT = "next", + PARENT = "parent", + CHILD = "child", +} + +export enum NodeType { + TEXT, + IMAGE, + INDEX, +} + +export class Node implements BaseDocument { + relationships: { [key in DocumentRelationship]: string | string[] }; + + getText(): string { + throw new Error("Method not implemented."); + } + getDocId(): string { + throw new Error("Method not implemented."); + } + getDocHash(): string { + throw new Error("Method not implemented."); + } + getEmbedding(): number[] { + throw new Error("Method not implemented."); + } + + getNodeInfo(): { [key: string]: any } { + return {}; + } + + refDocId(): string | null { + return ""; + } + + prevNodeId(): string { + throw new Error("Node does not have previous node"); + } + + nextNodeId(): string { + throw new Error("Node does not have next node"); + } + + parentNodeId(): string { + throw new Error("Node does not have parent node"); + } + + childNodeIds(): string[] { + return []; + } +} diff --git a/packages/core/Prompt.ts b/packages/core/Prompt.ts new file mode 100644 index 0000000000000000000000000000000000000000..741e08f0027d889915fad8d0371a82d4c7d59412 --- /dev/null +++ b/packages/core/Prompt.ts @@ -0,0 +1,58 @@ +/** + * A prompt is a function that takes a dictionary of inputs and returns a string. + * NOTE this is a different interface compared to LlamaIndex Python + */ +export type Prompt = (input: { [key: string]: string }) => string; + +/* +DEFAULT_TEXT_QA_PROMPT_TMPL = ( + "Context information is below. \n" + "---------------------\n" + "{context_str}" + "\n---------------------\n" + "Given the context information and not prior knowledge, " + "answer the question: {query_str}\n" +) +*/ + +export const defaultTextQaPrompt: Prompt = (input: { + [key: string]: string; +}) => { + const { context, query } = input; + + return `Context information is below. +--------------------- +${context} +--------------------- +Given the context information and not prior knowledge, answer the question: ${query} +`; +}; + +/* +DEFAULT_SUMMARY_PROMPT_TMPL = ( + "Write a summary of the following. Try to use only the " + "information provided. " + "Try to include as many key details as possible.\n" + "\n" + "\n" + "{context_str}\n" + "\n" + "\n" + 'SUMMARY:"""\n' +) +*/ + +export const defaultSummaryPrompt: Prompt = (input: { + [key: string]: string; +}) => { + const { context } = input; + + return `Write a summary of the following. Try to use only the information provided. Try to include as many key details as possible. + + +${input.context} + + +SUMMARY:""" +`; +}; diff --git a/packages/core/QueryEngine.ts b/packages/core/QueryEngine.ts new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/packages/core/Reader.ts b/packages/core/Reader.ts new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/packages/core/ResponseSynthesizer.ts b/packages/core/ResponseSynthesizer.ts new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/packages/core/Retriever.ts b/packages/core/Retriever.ts new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/packages/core/ServiceContext.ts b/packages/core/ServiceContext.ts new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/packages/core/StorageContext.ts b/packages/core/StorageContext.ts new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391