diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fd866123c29b707b279d8be7ac2c24fb6dee928c..f87865ebf51a688a2ad3154f8c9ac471b81a9943 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -36,7 +36,7 @@ jobs: working-directory: ./packages/create-llama - name: Run Playwright tests run: pnpm exec playwright test - env: + env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} working-directory: ./packages/create-llama - uses: actions/upload-artifact@v3 diff --git a/.github/workflows/lint_on_push_or_pull.yml b/.github/workflows/lint_on_push_or_pull.yml index 576725646009a403f4dfccdd651a2378f17f7667..96b8eae5268e7ee1e45c8574ddb5e10356c6f3de 100644 --- a/.github/workflows/lint_on_push_or_pull.yml +++ b/.github/workflows/lint_on_push_or_pull.yml @@ -17,9 +17,11 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version-file: '.nvmrc' + node-version-file: ".nvmrc" cache: "pnpm" - name: Install dependencies run: pnpm install - name: Run lint run: pnpm run lint + - name: Run Prettier + run: pnpm run format diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4436b635736d0e77f05882ae0189d0a56e3567bd..ce0fc78fe58505f2a89fe4fa79d98d245ee7f3b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version-file: '.nvmrc' + node-version-file: ".nvmrc" cache: "pnpm" - name: Install dependencies run: pnpm install diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000000000000000000000000000000000..f44098dd48e74a870fc6b5baf97691d812ab234a --- /dev/null +++ b/.prettierignore @@ -0,0 +1,3 @@ +apps/docs/i18n +pnpm-lock.yaml + diff --git a/apps/docs/docusaurus.config.js b/apps/docs/docusaurus.config.js index 0b7b88b4d7e825091e388a5b5f4f87d44ef7151b..1af9d27c40c6004d1389124a68f101f94cecb271 100644 --- a/apps/docs/docusaurus.config.js +++ b/apps/docs/docusaurus.config.js @@ -30,21 +30,21 @@ const config = { i18n: { defaultLocale: "en", locales: [ - "en", - "zh-Hans", - "es", - "fr", - "de", - "ja", - "ko", - "pt", - "ar", - "it", - "tr", - "pl", - "nl", - "vi", - "th", + "en", + "zh-Hans", + "es", + "fr", + "de", + "ja", + "ko", + "pt", + "ar", + "it", + "tr", + "pl", + "nl", + "vi", + "th", ], // "fa", "ru", "ro", "sv", "hu", "cs", "el", "da", "fi", "he", "no", "hi", "in", "sl", "se", "sk", "uk", "bg", "hr", "lt", "lv", "et", "cat" }, @@ -66,10 +66,10 @@ const config = { }, blog: false, gtag: { - trackingID: 'G-NB9B8LW9W5', + trackingID: "G-NB9B8LW9W5", anonymizeIP: true, }, - }), + }), ], ], diff --git a/examples/assemblyai.ts b/examples/assemblyai.ts index b1cc37e95369438f9b88ecb5a29503b12820700d..986788196ab50b946827fc61bb40d03dbf80607b 100644 --- a/examples/assemblyai.ts +++ b/examples/assemblyai.ts @@ -1,15 +1,16 @@ import { program } from "commander"; -import { AudioTranscriptReader, TranscribeParams, VectorStoreIndex } from "llamaindex"; +import { + AudioTranscriptReader, + TranscribeParams, + VectorStoreIndex, +} from "llamaindex"; import { stdin as input, stdout as output } from "node:process"; // readline/promises is still experimental so not in @types/node yet // @ts-ignore import readline from "node:readline/promises"; program - .option( - "-a, --audio [string]", - "URL or path of the audio file to transcribe", - ) + .option("-a, --audio [string]", "URL or path of the audio file to transcribe") .option("-i, --transcript-id [string]", "ID of the AssemblyAI transcript") .action(async (options) => { if (!process.env.ASSEMBLYAI_API_KEY) { @@ -26,9 +27,7 @@ program } else if (options.transcriptId) { params = options.transcriptId; } else { - console.log( - "You must provide either an --audio or a --transcript-id", - ); + console.log("You must provide either an --audio or a --transcript-id"); return; } diff --git a/examples/data/tinytweets.json b/examples/data/tinytweets.json index 2e3a8e66d405749eb47f76eb8303bba37612cd38..dabac683fcea7a9dacc680552c6af0719263d139 100644 Binary files a/examples/data/tinytweets.json and b/examples/data/tinytweets.json differ diff --git a/examples/pinecone-vector-store/load-docs.ts b/examples/pinecone-vector-store/load-docs.ts index 518de54f386cb000f0f7d3e3df16f097bb820599..12a50f54649408e75b7a9963abfd9f158aa0495f 100755 --- a/examples/pinecone-vector-store/load-docs.ts +++ b/examples/pinecone-vector-store/load-docs.ts @@ -1,11 +1,11 @@ // load-docs.ts import fs from "fs/promises"; import { + PineconeVectorStore, SimpleDirectoryReader, storageContextFromDefaults, VectorStoreIndex, } from "llamaindex"; -import { PineconeVectorStore } from "llamaindex"; async function getSourceFilenames(sourceDir: string) { return await fs diff --git a/examples/pinecone-vector-store/query.ts b/examples/pinecone-vector-store/query.ts index 51b7465510cde06cd44f6a2e537e90c1e0843e64..41ca987b05d9a2bafc053c57c014b87438fd28d1 100755 --- a/examples/pinecone-vector-store/query.ts +++ b/examples/pinecone-vector-store/query.ts @@ -1,6 +1,8 @@ -import { VectorStoreIndex } from "llamaindex"; -import { serviceContextFromDefaults } from "llamaindex"; -import { PineconeVectorStore } from "llamaindex"; +import { + PineconeVectorStore, + VectorStoreIndex, + serviceContextFromDefaults, +} from "llamaindex"; async function main() { const readline = require("readline").createInterface({ diff --git a/package.json b/package.json index aabdde73c46704aee4a8e5ca032427d9ef803a95..3939443bd6157ddf8ba8babb61484614459adf90 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,8 @@ "scripts": { "build": "turbo run build", "dev": "turbo run dev", - "format": "prettier --write \"**/*.{js,jsx,ts,tsx,md}\"", + "format": "prettier --ignore-unknown --cache --check .", + "format:write": "prettier --ignore-unknown --write .", "lint": "turbo run lint", "prepare": "husky install", "test": "turbo run test", diff --git a/packages/core/src/TextSplitter.ts b/packages/core/src/TextSplitter.ts index 36708004c59ea5931dfdc17acc99336dee2bcc20..f6d2a831fe513166997c41e6622335790f0b6116 100644 --- a/packages/core/src/TextSplitter.ts +++ b/packages/core/src/TextSplitter.ts @@ -1,5 +1,5 @@ -import nlp from 'compromise' -import { EOL } from 'node:os' +import nlp from "compromise"; +import { EOL } from "node:os"; // GitHub translated import { globalsHelper } from "./GlobalsHelper"; import { DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE } from "./constants"; @@ -20,11 +20,15 @@ class TextSplit { type SplitRep = { text: string; numTokens: number }; export const defaultSentenceTokenizer = (text: string): string[] => { - return nlp(text).sentences().json().map((sentence: any) => sentence.text); + return nlp(text) + .sentences() + .json() + .map((sentence: any) => sentence.text); }; // Refs: https://github.com/fxsjy/jieba/issues/575#issuecomment-359637511 -const resentencesp = /([﹒﹔﹖﹗.;。!?]["’”」』]{0,2}|:(?=["‘“「『]{1,2}|$))/; +const resentencesp = + /([﹒﹔﹖﹗.;。!?]["’”」』]{0,2}|:(?=["‘“「『]{1,2}|$))/; /** * Tokenizes sentences. Suitable for Chinese, Japanese, and Korean. Use instead of `defaultSentenceTokenizer`. * @param text @@ -46,7 +50,7 @@ export function cjkSentenceTokenizer(sentence: string): string[] { return slist.filter((s) => s.length > 0); } -export const defaultParagraphSeparator = EOL + EOL + EOL +export const defaultParagraphSeparator = EOL + EOL + EOL; // In theory there's also Mac style \r only, but it's pre-OSX and I don't think // many documents will use it. @@ -227,9 +231,10 @@ export class SentenceSplitter { // push curent doc list to docs docs.push( new TextSplit( - curChunkSentences.map((sentence) => sentence.text). - join(" "). - trim(), + curChunkSentences + .map((sentence) => sentence.text) + .join(" ") + .trim(), ), ); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d71560dfd10e1490f73be37289d090cc996a2105..e170e99d8dc8a0df82add70a49624efed8d63225 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -9,7 +9,6 @@ export * from "./PromptHelper"; export * from "./QueryEngine"; export * from "./QuestionGenerator"; export * from "./Response"; -export * from "./synthesizers"; export * from "./Retriever"; export * from "./ServiceContext"; export * from "./TextSplitter"; @@ -29,3 +28,4 @@ export * from "./readers/SimpleDirectoryReader"; export * from "./readers/SimpleMongoReader"; export * from "./readers/base"; export * from "./storage"; +export * from "./synthesizers"; diff --git a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts index 2b818f006395157f66e7b6951a2c28eba54e1366..b588b34eea6bfc2783774530e61aeda4d1d3eab1 100644 --- a/packages/core/src/indices/vectorStore/VectorStoreIndex.ts +++ b/packages/core/src/indices/vectorStore/VectorStoreIndex.ts @@ -24,7 +24,7 @@ import { } from "../../storage/StorageContext"; import { BaseIndexStore } from "../../storage/indexStore/types"; import { VectorStore } from "../../storage/vectorStore/types"; -import { ResponseSynthesizer, BaseSynthesizer } from "../../synthesizers"; +import { BaseSynthesizer } from "../../synthesizers"; import { BaseIndex, BaseIndexInit, diff --git a/packages/core/src/llm/LLM.ts b/packages/core/src/llm/LLM.ts index 7964f5bd55794e27bc25206bebc518c4d48a3717..06a0857a6e23d242154d22ff74c6751a72f1e93f 100644 --- a/packages/core/src/llm/LLM.ts +++ b/packages/core/src/llm/LLM.ts @@ -10,6 +10,7 @@ import { import { ChatCompletionMessageParam } from "openai/resources"; import { LLMOptions } from "portkey-ai"; +import { MessageContent } from "../ChatEngine"; import { globalsHelper, Tokenizers } from "../GlobalsHelper"; import { ANTHROPIC_AI_PROMPT, @@ -27,7 +28,6 @@ import { import { getOpenAISession, OpenAISession } from "./openai"; import { getPortkeySession, PortkeySession } from "./portkey"; import { ReplicateSession } from "./replicate"; -import { MessageContent } from "../ChatEngine"; export type MessageType = | "user" diff --git a/packages/core/src/readers/DocxReader.ts b/packages/core/src/readers/DocxReader.ts index fae5b696d28a671c4c3ea6d56d79d9e4ac5a96b3..4b72bc6728d9c7ca4b8760fb5e3f135ef26a8d85 100644 --- a/packages/core/src/readers/DocxReader.ts +++ b/packages/core/src/readers/DocxReader.ts @@ -1,7 +1,7 @@ import mammoth from "mammoth"; import { Document } from "../Node"; -import { DEFAULT_FS } from "../storage/constants"; import { GenericFileSystem } from "../storage/FileSystem"; +import { DEFAULT_FS } from "../storage/constants"; import { BaseReader } from "./base"; export class DocxReader implements BaseReader { diff --git a/packages/core/src/storage/index.ts b/packages/core/src/storage/index.ts index cbd82e8ce8bc9a39f61c6bf245e20048a2a9e1c4..83cedd3894d69fc9648c693ef17afe772befa7ed 100644 --- a/packages/core/src/storage/index.ts +++ b/packages/core/src/storage/index.ts @@ -9,7 +9,7 @@ export { SimpleKVStore } from "./kvStore/SimpleKVStore"; export * from "./kvStore/types"; export { AstraDBVectorStore } from "./vectorStore/AstraDBVectorStore"; export { MongoDBAtlasVectorSearch } from "./vectorStore/MongoDBAtlasVectorStore"; +export { PGVectorStore } from "./vectorStore/PGVectorStore"; export { PineconeVectorStore } from "./vectorStore/PineconeVectorStore"; export { SimpleVectorStore } from "./vectorStore/SimpleVectorStore"; -export { PGVectorStore } from "./vectorStore/PGVectorStore"; export * from "./vectorStore/types"; diff --git a/packages/core/src/storage/vectorStore/PineconeVectorStore.ts b/packages/core/src/storage/vectorStore/PineconeVectorStore.ts index ae424c050b1298ba77766ba62797e72bf2534b8c..68a43973b64388f4eb8a28683eb956db7611985f 100644 --- a/packages/core/src/storage/vectorStore/PineconeVectorStore.ts +++ b/packages/core/src/storage/vectorStore/PineconeVectorStore.ts @@ -1,8 +1,10 @@ -import { VectorStore, - VectorStoreQuery, - VectorStoreQueryResult, - ExactMatchFilter, - MetadataFilters } from "./types"; +import { + ExactMatchFilter, + MetadataFilters, + VectorStore, + VectorStoreQuery, + VectorStoreQueryResult, +} from "./types"; import { BaseNode, Document, Metadata, MetadataMode } from "../../Node"; import { GenericFileSystem } from "../FileSystem"; @@ -142,7 +144,7 @@ export class PineconeVectorStore implements VectorStore { topK: query.similarityTopK, include_values: true, include_metadara: true, - filter: filter + filter: filter, }; const idx = await this.index(); diff --git a/packages/core/src/tests/CallbackManager.test.ts b/packages/core/src/tests/CallbackManager.test.ts index f9ef370b469709f17463097173b73a2440ace017..38a95028b9242943d9d30167ad0c52e116630c42 100644 --- a/packages/core/src/tests/CallbackManager.test.ts +++ b/packages/core/src/tests/CallbackManager.test.ts @@ -1,3 +1,5 @@ +import { Document } from "../Node"; +import { ServiceContext, serviceContextFromDefaults } from "../ServiceContext"; import { CallbackManager, RetrievalCallbackResponse, @@ -7,12 +9,7 @@ import { OpenAIEmbedding } from "../embeddings"; import { SummaryIndex } from "../indices/summary"; import { VectorStoreIndex } from "../indices/vectorStore/VectorStoreIndex"; import { OpenAI } from "../llm/LLM"; -import { Document } from "../Node"; -import { - ResponseSynthesizer, - SimpleResponseBuilder, -} from "../synthesizers"; -import { ServiceContext, serviceContextFromDefaults } from "../ServiceContext"; +import { ResponseSynthesizer, SimpleResponseBuilder } from "../synthesizers"; import { mockEmbeddingModel, mockLlmGeneration } from "./utility/mockOpenAI"; // Mock the OpenAI getOpenAISession function during testing diff --git a/packages/core/src/tests/TextSplitter.test.ts b/packages/core/src/tests/TextSplitter.test.ts index 591fd493c1c96127458443cb4d41f1d02a878872..4678e2725b36f9b00c8b79be1970297278646b1b 100644 --- a/packages/core/src/tests/TextSplitter.test.ts +++ b/packages/core/src/tests/TextSplitter.test.ts @@ -1,4 +1,4 @@ -import { cjkSentenceTokenizer, SentenceSplitter } from '../TextSplitter' +import { cjkSentenceTokenizer, SentenceSplitter } from "../TextSplitter"; describe("SentenceSplitter", () => { test("initializes", () => { @@ -88,7 +88,9 @@ describe("SentenceSplitter", () => { chunkingTokenizerFn: cjkSentenceTokenizer, }); - const splits = sentenceSplitter.splitText("此后如竟没有炬火:我便是唯一的光。倘若有了炬火,出了太阳,我们自然心悦诚服的消失。不但毫无不平,而且还要随喜赞美这炬火或太阳;因为他照了人类,连我都在内。"); + const splits = sentenceSplitter.splitText( + "此后如竟没有炬火:我便是唯一的光。倘若有了炬火,出了太阳,我们自然心悦诚服的消失。不但毫无不平,而且还要随喜赞美这炬火或太阳;因为他照了人类,连我都在内。", + ); expect(splits).toEqual([ "此后如竟没有炬火:我便是唯一的光。", "倘若有了炬火,出了太阳,我们自然心悦诚服的消失。", diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index 37fda5e0572aae0a24cd1a08585fa1dcd0aed180..0c3397b798962a8dd2a89d35775cb990a0e1c1dd 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -8,18 +8,10 @@ "preserveWatchOutput": true, "skipLibCheck": true, "strict": true, - "lib": [ - "es2015", - "dom" - ], + "lib": ["es2015", "dom"], "target": "ES2015", "resolveJsonModule": true, - "typeRoots": [ - "./types", - "./node_modules/@types" - ] + "typeRoots": ["./types", "./node_modules/@types"] }, - "exclude": [ - "node_modules" - ], -} \ No newline at end of file + "exclude": ["node_modules"] +} diff --git a/packages/create-llama/package.json b/packages/create-llama/package.json index 72e5d3362ffa2044a36270c29e5cd29285c0e620..e893e9e3cb4a0765e97c88bfbc101d866f978b97 100644 --- a/packages/create-llama/package.json +++ b/packages/create-llama/package.json @@ -58,4 +58,4 @@ "engines": { "node": ">=16.14.0" } -} \ No newline at end of file +} diff --git a/packages/create-llama/templates/types/simple/express/eslintrc.json b/packages/create-llama/templates/types/simple/express/eslintrc.json index c19581799d886ce5be17def8ad268c2f4b175109..cf20cdc7a788ed949375c3e013b27e692d6df234 100644 --- a/packages/create-llama/templates/types/simple/express/eslintrc.json +++ b/packages/create-llama/templates/types/simple/express/eslintrc.json @@ -1,3 +1,3 @@ { "extends": "eslint:recommended" -} \ No newline at end of file +} diff --git a/packages/create-llama/templates/types/simple/express/tsconfig.json b/packages/create-llama/templates/types/simple/express/tsconfig.json index e886da1ef324ab02d14102f3a4a36c8f90f70eef..bc819cab43220b31fac6abb1d4a36ac5880ef011 100644 --- a/packages/create-llama/templates/types/simple/express/tsconfig.json +++ b/packages/create-llama/templates/types/simple/express/tsconfig.json @@ -7,4 +7,4 @@ "skipLibCheck": true, "moduleResolution": "node" } -} \ No newline at end of file +} diff --git a/packages/create-llama/templates/types/streaming/express/eslintrc.json b/packages/create-llama/templates/types/streaming/express/eslintrc.json index c19581799d886ce5be17def8ad268c2f4b175109..cf20cdc7a788ed949375c3e013b27e692d6df234 100644 --- a/packages/create-llama/templates/types/streaming/express/eslintrc.json +++ b/packages/create-llama/templates/types/streaming/express/eslintrc.json @@ -1,3 +1,3 @@ { "extends": "eslint:recommended" -} \ No newline at end of file +} diff --git a/packages/create-llama/templates/types/streaming/express/tsconfig.json b/packages/create-llama/templates/types/streaming/express/tsconfig.json index e886da1ef324ab02d14102f3a4a36c8f90f70eef..bc819cab43220b31fac6abb1d4a36ac5880ef011 100644 --- a/packages/create-llama/templates/types/streaming/express/tsconfig.json +++ b/packages/create-llama/templates/types/streaming/express/tsconfig.json @@ -7,4 +7,4 @@ "skipLibCheck": true, "moduleResolution": "node" } -} \ No newline at end of file +} diff --git a/packages/create-llama/templates/types/streaming/nextjs/tsconfig.json b/packages/create-llama/templates/types/streaming/nextjs/tsconfig.json index e779aa667e9e83943147b44d06fa930e9175a2b7..40c136b8255d33d2f7335e82f141d9d16005d878 100644 --- a/packages/create-llama/templates/types/streaming/nextjs/tsconfig.json +++ b/packages/create-llama/templates/types/streaming/nextjs/tsconfig.json @@ -1,11 +1,7 @@ { "compilerOptions": { "target": "es5", - "lib": [ - "dom", - "dom.iterable", - "esnext" - ], + "lib": ["dom", "dom.iterable", "esnext"], "allowJs": true, "skipLibCheck": true, "strict": true, @@ -23,19 +19,10 @@ } ], "paths": { - "@/*": [ - "./*" - ] + "@/*": ["./*"] }, - "forceConsistentCasingInFileNames": true, + "forceConsistentCasingInFileNames": true }, - "include": [ - "next-env.d.ts", - "**/*.ts", - "**/*.tsx", - ".next/types/**/*.ts" - ], - "exclude": [ - "node_modules" - ] -} \ No newline at end of file + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/packages/create-llama/tsconfig.json b/packages/create-llama/tsconfig.json index a653e907d965449ec6e9f82778b7a2e722f9bc02..e4edad9e127c7de102ca7bdd6b243168a83bbc20 100644 --- a/packages/create-llama/tsconfig.json +++ b/packages/create-llama/tsconfig.json @@ -7,8 +7,5 @@ "esModuleInterop": true, "skipLibCheck": false }, - "exclude": [ - "templates", - "dist" - ] -} \ No newline at end of file + "exclude": ["templates", "dist"] +} diff --git a/packages/eslint-config-custom/index.js b/packages/eslint-config-custom/index.js index 3fc7c9ef1b9650762f599a9c261acb6346bce91a..37100f452993b962ad82ca89d51058874fb25159 100644 --- a/packages/eslint-config-custom/index.js +++ b/packages/eslint-config-custom/index.js @@ -28,7 +28,7 @@ module.exports = { "PINECONE_ENVIRONMENT", "PINECONE_PROJECT_ID", "PINECONE_INDEX_NAME", - "PINECONE_CHUNK_SIZE", + "PINECONE_CHUNK_SIZE", "PINECONE_INDEX_NAME", "AZURE_OPENAI_API_KEY", diff --git a/sweep.yaml b/sweep.yaml index ccb434b595de09eaeb16712ad5f3cfb224e4d31a..9d453b943b4c9e6f550b74bdd5a9120bb0f78681 100644 --- a/sweep.yaml +++ b/sweep.yaml @@ -19,5 +19,4 @@ sandbox: - pnpm eslint --fix {file_path} - pnpx ts-node --type-check {file_path} - pnpm test - # Default Values: https://github.com/sweepai/sweep/blob/main/sweep.yaml