diff --git a/.changeset/weak-ants-kick.md b/.changeset/weak-ants-kick.md new file mode 100644 index 0000000000000000000000000000000000000000..62bc3c2a11ecfcae83a689e11c4668620fcf8b3d --- /dev/null +++ b/.changeset/weak-ants-kick.md @@ -0,0 +1,8 @@ +--- +"llamaindex": minor +"@llamaindex/doc": minor +"@llamaindex/examples": minor +"@llamaindex/unit-test": minor +--- + +Remove readers package from llamaindex diff --git a/.vscode/ltex.dictionary.en-US.txt b/.vscode/ltex.dictionary.en-US.txt new file mode 100644 index 0000000000000000000000000000000000000000..120ae4c4b23542006620a499814b41550998751a --- /dev/null +++ b/.vscode/ltex.dictionary.en-US.txt @@ -0,0 +1 @@ +LlamaIndexTS diff --git a/README.md b/README.md index 433e92c73bc43cc410166caf3dc80dc6ab18e38c..563556922eba33ecce662d0d545fd157d5608e0b 100644 --- a/README.md +++ b/README.md @@ -65,44 +65,18 @@ yarn add llamaindex See our official document: <https://ts.llamaindex.ai/docs/llamaindex/getting_started/> -### Tips when using in non-Node.js environments +### Adding provider packages -When you are importing `llamaindex` in a non-Node.js environment(such as Vercel Edge, Cloudflare Workers, etc.) -Some classes are not exported from top-level entry file. +In most cases, you'll also need to install provider packages to use LlamaIndexTS. These are for adding AI models, file readers for ingestion or storing documents, e.g. in vector databases. -The reason is that some classes are only compatible with Node.js runtime,(e.g. `PDFReader`) which uses Node.js specific APIs(like `fs`, `child_process`, `crypto`). +For example, to use the OpenAI LLM, you would install the following package: -If you need any of those classes, you have to import them instead directly though their file path in the package. -Here's an example for importing the `PineconeVectorStore` class: - -```typescript -import { PineconeVectorStore } from "llamaindex/vector-store/PineconeVectorStore"; -``` - -As the `PDFReader` is not working with the Edge runtime, here's how to use the `SimpleDirectoryReader` with the `LlamaParseReader` to load PDFs: - -```typescript -import { SimpleDirectoryReader } from "llamaindex/readers/SimpleDirectoryReader"; -import { LlamaParseReader } from "llamaindex/readers/LlamaParseReader"; - -export const DATA_DIR = "./data"; - -export async function getDocuments() { - const reader = new SimpleDirectoryReader(); - // Load PDFs using LlamaParseReader - return await reader.loadData({ - directoryPath: DATA_DIR, - fileExtToReader: { - pdf: new LlamaParseReader({ resultType: "markdown" }), - }, - }); -} +```shell +npm install @llamaindex/openai +pnpm install @llamaindex/openai +yarn add @llamaindex/openai ``` -> _Note_: Reader classes have to be added explictly to the `fileExtToReader` map in the Edge version of the `SimpleDirectoryReader`. - -You'll find a complete example with LlamaIndexTS here: https://github.com/run-llama/create_llama_projects/tree/main/nextjs-edge-llamaparse - ## Playground Check out our NextJS playground at https://llama-playground.vercel.app/. The source is available at https://github.com/run-llama/ts-playground diff --git a/apps/next/src/content/docs/llamaindex/guide/agents/1_setup.mdx b/apps/next/src/content/docs/llamaindex/guide/agents/1_setup.mdx index 022308116dbb44904b412c7fb3e2d5c4de80b6c4..5a35bd32255f9a074aea36d6b87867cca3d0ffcd 100644 --- a/apps/next/src/content/docs/llamaindex/guide/agents/1_setup.mdx +++ b/apps/next/src/content/docs/llamaindex/guide/agents/1_setup.mdx @@ -15,7 +15,7 @@ In LlamaIndex, an agent is a semi-autonomous piece of software powered by an LLM You'll need to have a recent version of [Node.js](https://nodejs.org/en) installed. Then you can install LlamaIndex.TS by running ```bash -npm install llamaindex +npm install llamaindex @llamaindex/openai @llamaindex/readers @llamaindex/huggingface ``` ## Choose your model diff --git a/apps/next/src/content/docs/llamaindex/guide/agents/4_agentic_rag.mdx b/apps/next/src/content/docs/llamaindex/guide/agents/4_agentic_rag.mdx index b690120fbcbc1b20e34d4372e23e8ad5fef52433..ff991b346bd3ed89f1b81f1cb804a396a43ac5eb 100644 --- a/apps/next/src/content/docs/llamaindex/guide/agents/4_agentic_rag.mdx +++ b/apps/next/src/content/docs/llamaindex/guide/agents/4_agentic_rag.mdx @@ -40,7 +40,7 @@ We'll be bringing in `SimpleDirectoryReader`, `HuggingFaceEmbedding`, `VectorSto import { FunctionTool, QueryEngineTool, Settings, VectorStoreIndex } from "llamaindex"; import { OpenAI, OpenAIAgent } from "@llamaindex/openai"; import { HuggingFaceEmbedding } from "@llamaindex/huggingface"; -import { SimpleDirectoryReader } from "llamaindex"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; ``` ### Add an embedding model diff --git a/apps/next/src/content/docs/llamaindex/guide/loading/index.mdx b/apps/next/src/content/docs/llamaindex/guide/loading/index.mdx index 91d91868f7d32f2aa62b33aa6863da4067c998c6..42611dd03feb46df6df48cbb80889cd513d4c049 100644 --- a/apps/next/src/content/docs/llamaindex/guide/loading/index.mdx +++ b/apps/next/src/content/docs/llamaindex/guide/loading/index.mdx @@ -10,7 +10,7 @@ import { Accordion, Accordions } from 'fumadocs-ui/components/accordion'; <Accordions> <Accordion title="Install @llamaindex/readers"> - If you want to only use reader modules, you can install `@llamaindex/readers` + If you want to use the reader module, you need to install `@llamaindex/readers` <Tabs groupId="install-llamaindex" items={["npm", "yarn", "pnpm"]} persist> ```shell tab="npm" @@ -31,72 +31,73 @@ import { Accordion, Accordions } from 'fumadocs-ui/components/accordion'; We offer readers for different file formats. -<Tabs groupId="llamaindex-or-readers" items={["llamaindex", "@llamaindex/readers"]} persist> - ```ts twoslash tab="llamaindex" - import { CSVReader } from '@llamaindex/readers/csv' - import { PDFReader } from '@llamaindex/readers/pdf' - import { JSONReader } from '@llamaindex/readers/json' - import { MarkdownReader } from '@llamaindex/readers/markdown' - import { HTMLReader } from '@llamaindex/readers/html' - // you can find more readers in the documentation - ``` - - ```ts twoslash tab="@llamaindex/readers" - import { CSVReader } from '@llamaindex/readers/csv' - import { PDFReader } from '@llamaindex/readers/pdf' - import { JSONReader } from '@llamaindex/readers/json' - import { MarkdownReader } from '@llamaindex/readers/markdown' - import { HTMLReader } from '@llamaindex/readers/html' - // you can find more readers in the documentation - ``` - -</Tabs> +```ts twoslash +import { CSVReader } from '@llamaindex/readers/csv' +import { PDFReader } from '@llamaindex/readers/pdf' +import { JSONReader } from '@llamaindex/readers/json' +import { MarkdownReader } from '@llamaindex/readers/markdown' +import { HTMLReader } from '@llamaindex/readers/html' +// you can find more readers in the documentation +``` ## SimpleDirectoryReader `SimpleDirectoryReader` is the simplest way to load data from local files into LlamaIndex. -<Tabs groupId="llamaindex-or-readers" items={["llamaindex", "@llamaindex/readers"]} persist> +```ts twoslash +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; - ```ts twoslash tab="llamaindex" - import { SimpleDirectoryReader } from "llamaindex"; +const reader = new SimpleDirectoryReader() +const documents = await reader.loadData("./data") +// ^? - const reader = new SimpleDirectoryReader() - const documents = await reader.loadData("./data") - // ^? +const texts = documents.map(doc => doc.getText()) +// ^? +``` - const texts = documents.map(doc => doc.getText()) - // ^? - ``` - ```ts twoslash tab="@llamaindex/readers" - import { SimpleDirectoryReader } from "llamaindex"; +## Tips when using in non-Node.js environments - const reader = new SimpleDirectoryReader() - const documents = await reader.loadData("./data") - // ^? +When using `@llamaindex/readers` in a non-Node.js environment (such as Vercel Edge, Cloudflare Workers, etc.) +Some classes are not exported from top-level entry file. +The reason is that some classes are only compatible with Node.js runtime, (e.g. `PDFReader`) which uses Node.js specific APIs (like `fs`, `child_process`, `crypto`). - const texts = documents.map(doc => doc.getText()) - // ^? - ``` +If you need any of those classes, you have to import them instead directly through their file path in the package. + +As the `PDFReader` is not working with the Edge runtime, here's how to use the `SimpleDirectoryReader` with the `LlamaParseReader` to load PDFs: + +```typescript +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; +import { LlamaParseReader } from "@llamaindex/cloud"; + +export const DATA_DIR = "./data"; + +export async function getDocuments() { + const reader = new SimpleDirectoryReader(); + // Load PDFs using LlamaParseReader + return await reader.loadData({ + directoryPath: DATA_DIR, + fileExtToReader: { + pdf: new LlamaParseReader({ resultType: "markdown" }), + }, + }); +} +``` + +> _Note_: Reader classes have to be added explicitly to the `fileExtToReader` map in the Edge version of the `SimpleDirectoryReader`. + +You'll find a complete example with LlamaIndexTS here: https://github.com/run-llama/create_llama_projects/tree/main/nextjs-edge-llamaparse -</Tabs> ## Load file natively using Node.js Customization Hooks We have a helper utility to allow you to import a file in Node.js script. -<Tabs groupId="llamaindex-or-readers" items={["llamaindex", "@llamaindex/readers"]} persist> - ```shell tab="llamaindex" - node --import llamaindex/register ./script.js - ``` - - ```shell tab="@llamaindex/readers" - node --import @llamaindex/readers/node ./script.js - ``` -</Tabs> +```shell +node --import @llamaindex/readers/node ./script.js +``` ```ts import csv from './path/to/data.csv'; diff --git a/apps/next/src/content/docs/llamaindex/modules/query_engines/router_query_engine.mdx b/apps/next/src/content/docs/llamaindex/modules/query_engines/router_query_engine.mdx index 5322ecbf2b2d7c37d12f9135a5398bed4e720cad..ba07d0909999605ff3a9fc6fd3a2a7d4dc21f520 100644 --- a/apps/next/src/content/docs/llamaindex/modules/query_engines/router_query_engine.mdx +++ b/apps/next/src/content/docs/llamaindex/modules/query_engines/router_query_engine.mdx @@ -34,7 +34,7 @@ import { Settings, } from "llamaindex"; import { OpenAI } from "@llamaindex/openai"; -import { SimpleDirectoryReader } from "llamaindex"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; ``` ## Loading Data @@ -124,7 +124,7 @@ import { Settings, } from "llamaindex"; import { OpenAI } from "@llamaindex/openai"; -import { SimpleDirectoryReader } from "llamaindex"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; Settings.llm = new OpenAI(); Settings.nodeParser = new SentenceSplitter({ diff --git a/e2e/examples/nextjs-node-runtime/package.json b/e2e/examples/nextjs-node-runtime/package.json index 6f2350627f1fffc3cc1034f5a2b364a4cd5d493e..d9b91b45d1c3c0d475547510aea10a07b5478bf5 100644 --- a/e2e/examples/nextjs-node-runtime/package.json +++ b/e2e/examples/nextjs-node-runtime/package.json @@ -10,6 +10,7 @@ "dependencies": { "llamaindex": "workspace:*", "@llamaindex/huggingface": "workspace:*", + "@llamaindex/readers": "workspace:*", "next": "15.0.3", "react": "18.3.1", "react-dom": "18.3.1" diff --git a/e2e/examples/nextjs-node-runtime/src/actions/openai.ts b/e2e/examples/nextjs-node-runtime/src/actions/openai.ts index a69877a93340d84ddaf2043cfe5b692d1be3bba3..65464435ffa61e333cac3840cf37567d3553ec03 100644 --- a/e2e/examples/nextjs-node-runtime/src/actions/openai.ts +++ b/e2e/examples/nextjs-node-runtime/src/actions/openai.ts @@ -1,11 +1,11 @@ "use server"; import { HuggingFaceEmbedding } from "@llamaindex/huggingface"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; import { OpenAI, OpenAIAgent, QueryEngineTool, Settings, - SimpleDirectoryReader, VectorStoreIndex, } from "llamaindex"; diff --git a/e2e/examples/waku-query-engine/package.json b/e2e/examples/waku-query-engine/package.json index c3f2953cd548f7d50be841247b55aab423ef0a08..c6bf2ff15737adc908c7d9d17a5ece717e9a54cf 100644 --- a/e2e/examples/waku-query-engine/package.json +++ b/e2e/examples/waku-query-engine/package.json @@ -9,6 +9,7 @@ "start": "waku start" }, "dependencies": { + "@llamaindex/env": "workspace:*", "llamaindex": "workspace:*", "react": "19.0.0-rc-5c56b873-20241107", "react-dom": "19.0.0-rc-5c56b873-20241107", diff --git a/e2e/examples/waku-query-engine/src/actions.ts b/e2e/examples/waku-query-engine/src/actions.ts index 8978b12c881ac0708d7fe519f762792191883b94..8e683c1d8cdfb0b8ceb1c2349c5a31acb6038bae 100644 --- a/e2e/examples/waku-query-engine/src/actions.ts +++ b/e2e/examples/waku-query-engine/src/actions.ts @@ -1,13 +1,14 @@ "use server"; +import { fs } from "@llamaindex/env"; import { BaseQueryEngine, Document, VectorStoreIndex } from "llamaindex"; -import { readFile } from "node:fs/promises"; + let _queryEngine: BaseQueryEngine; async function lazyLoadQueryEngine() { if (!_queryEngine) { const path = "node_modules/llamaindex/examples/abramov.txt"; - const essay = await readFile(path, "utf-8"); + const essay = await fs.readFile(path, "utf-8"); // Create Document object with essay const document = new Document({ text: essay, id_: path }); diff --git a/examples/agent/query_openai_agent.ts b/examples/agent/query_openai_agent.ts index 7ad7d0760a0ed3e8b928dafd97f23388c9f5ff00..58f126c0155c0e55b0a009a240deb3f846b9c430 100644 --- a/examples/agent/query_openai_agent.ts +++ b/examples/agent/query_openai_agent.ts @@ -1,9 +1,6 @@ import { OpenAIAgent } from "@llamaindex/openai"; -import { - QueryEngineTool, - SimpleDirectoryReader, - VectorStoreIndex, -} from "llamaindex"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; +import { QueryEngineTool, VectorStoreIndex } from "llamaindex"; async function main() { // Load the documents diff --git a/examples/agent/retriever_openai_agent.ts b/examples/agent/retriever_openai_agent.ts index a3b19628517cbb7ff44bfbcb1c98a18d670a82d6..ffa67310c1b0c6045d907fa3974c8ad59df68670 100644 --- a/examples/agent/retriever_openai_agent.ts +++ b/examples/agent/retriever_openai_agent.ts @@ -1,9 +1,9 @@ import { OpenAIAgent } from "@llamaindex/openai"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; import { FunctionTool, MetadataMode, NodeWithScore, - SimpleDirectoryReader, VectorStoreIndex, } from "llamaindex"; diff --git a/examples/agent/step_wise_query_tool.ts b/examples/agent/step_wise_query_tool.ts index c3d0f575d3c35c810b15c9e5351e8c1a55a72eb9..fbaf7e486def2f5707a87c88df8812348e9b5809 100644 --- a/examples/agent/step_wise_query_tool.ts +++ b/examples/agent/step_wise_query_tool.ts @@ -1,9 +1,6 @@ import { OpenAIAgent } from "@llamaindex/openai"; -import { - QueryEngineTool, - SimpleDirectoryReader, - VectorStoreIndex, -} from "llamaindex"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; +import { QueryEngineTool, VectorStoreIndex } from "llamaindex"; async function main() { // Load the documents diff --git a/examples/directory.ts b/examples/directory.ts index b92ad82da3261e40355a25536399acb350262bbe..800ac4ceb86ad77f69229ca14abdd23e7fd9b4ea 100644 --- a/examples/directory.ts +++ b/examples/directory.ts @@ -1,4 +1,4 @@ -import { SimpleDirectoryReader } from "llamaindex"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; function callback( category: string, diff --git a/examples/multimodal/jina.ts b/examples/multimodal/jina.ts index 7ce2f3cf86d5d171814dc637a804824bda268fd2..1c3081d0fcba4f4b51556198f503660913d5c051 100644 --- a/examples/multimodal/jina.ts +++ b/examples/multimodal/jina.ts @@ -1,9 +1,9 @@ +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; import { ImageDocument, JinaAIEmbedding, similarity, SimilarityType, - SimpleDirectoryReader, } from "llamaindex"; import path from "path"; diff --git a/examples/multimodal/load.ts b/examples/multimodal/load.ts index c1a9f68c43999a210ae8019cf3fb10ce2f921b9a..a824f132b0955d21e2fa18e8f5f318864f32f73a 100644 --- a/examples/multimodal/load.ts +++ b/examples/multimodal/load.ts @@ -1,4 +1,5 @@ -import { Settings, SimpleDirectoryReader, VectorStoreIndex } from "llamaindex"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; +import { Settings, VectorStoreIndex } from "llamaindex"; import path from "path"; import { getStorageContext } from "./storage"; diff --git a/examples/pinecone-vector-store/load-docs.ts b/examples/pinecone-vector-store/load-docs.ts index 1c6a1575876f05e4d520197072183d168569c2d4..736a6cc8df910bb23a00d1a3c92bd85ec5741f5d 100755 --- a/examples/pinecone-vector-store/load-docs.ts +++ b/examples/pinecone-vector-store/load-docs.ts @@ -1,11 +1,8 @@ // load-docs.ts import { PineconeVectorStore } from "@llamaindex/pinecone"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; import fs from "fs/promises"; -import { - SimpleDirectoryReader, - storageContextFromDefaults, - VectorStoreIndex, -} from "llamaindex"; +import { storageContextFromDefaults, VectorStoreIndex } from "llamaindex"; async function getSourceFilenames(sourceDir: string) { return await fs diff --git a/examples/readers/src/custom-simple-directory-reader.ts b/examples/readers/src/custom-simple-directory-reader.ts index 32fedd137e716b32b126619274d25fac2223aa97..e5ec35c519bcd45f6a015e07f7e29a8a0e4a816f 100644 --- a/examples/readers/src/custom-simple-directory-reader.ts +++ b/examples/readers/src/custom-simple-directory-reader.ts @@ -1,10 +1,10 @@ -import { TextFileReader } from "@llamaindex/readers/text"; -import type { Document, Metadata } from "llamaindex"; import { FILE_EXT_TO_READER, - FileReader, SimpleDirectoryReader, -} from "llamaindex"; +} from "@llamaindex/readers/directory"; +import { TextFileReader } from "@llamaindex/readers/text"; +import type { Document, Metadata } from "llamaindex"; +import { FileReader } from "llamaindex"; class ZipReader extends FileReader { loadDataAsContent(fileContent: Uint8Array): Promise<Document<Metadata>[]> { diff --git a/examples/readers/src/simple-directory-reader-with-llamaparse.ts b/examples/readers/src/simple-directory-reader-with-llamaparse.ts index e71683657799d5fb80ceb41f035ef15663b0c1ce..7ef35e6e462f190b44908855987e101cdb200df9 100644 --- a/examples/readers/src/simple-directory-reader-with-llamaparse.ts +++ b/examples/readers/src/simple-directory-reader-with-llamaparse.ts @@ -1,5 +1,6 @@ import { LlamaParseReader } from "@llamaindex/cloud"; -import { SimpleDirectoryReader, VectorStoreIndex } from "llamaindex"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; +import { VectorStoreIndex } from "llamaindex"; async function main() { const reader = new SimpleDirectoryReader(); diff --git a/examples/readers/src/simple-directory-reader.ts b/examples/readers/src/simple-directory-reader.ts index 784b5b5efc8aad5d8ace96d1434d26a0b0e24881..b815a88415371aba802abe320544c14253518a55 100644 --- a/examples/readers/src/simple-directory-reader.ts +++ b/examples/readers/src/simple-directory-reader.ts @@ -1,6 +1,4 @@ -import { SimpleDirectoryReader } from "llamaindex"; -// or -// import { SimpleDirectoryReader } from 'llamaindex' +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; const reader = new SimpleDirectoryReader(); const documents = await reader.loadData("../data"); diff --git a/examples/routerQueryEngine.ts b/examples/routerQueryEngine.ts index 51f5025bd15f14533e48e5c6fc68f709d9e1a147..1bbbe0bcf2e361b196de0129bb7c84cf08671bfa 100644 --- a/examples/routerQueryEngine.ts +++ b/examples/routerQueryEngine.ts @@ -1,9 +1,9 @@ import { OpenAI } from "@llamaindex/openai"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; import { RouterQueryEngine, SentenceSplitter, Settings, - SimpleDirectoryReader, SummaryIndex, VectorStoreIndex, } from "llamaindex"; diff --git a/examples/vector-store/azure/index.ts b/examples/vector-store/azure/index.ts index 2a3f1e57b9f8631a81166af701213bd6a16315ca..2f6322d88055b38a34ef481229c2f8d6f85c09ca 100644 --- a/examples/vector-store/azure/index.ts +++ b/examples/vector-store/azure/index.ts @@ -14,6 +14,7 @@ import { MetadataIndexFieldType, } from "@llamaindex/azure"; import { OpenAI, OpenAIEmbedding } from "@llamaindex/openai"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; import dotenv from "dotenv"; import { Document, @@ -22,7 +23,6 @@ import { Metadata, NodeWithScore, Settings, - SimpleDirectoryReader, storageContextFromDefaults, TextNode, VectorStoreIndex, diff --git a/examples/vector-store/pg/load-docs.ts b/examples/vector-store/pg/load-docs.ts index ac37877159a331fa197db3d39849d79ea9d6fcc1..af85641f2857e0a356327750d5d1fe28af3b15e6 100755 --- a/examples/vector-store/pg/load-docs.ts +++ b/examples/vector-store/pg/load-docs.ts @@ -1,10 +1,7 @@ // load-docs.ts import { PGVectorStore } from "@llamaindex/postgres"; -import { - SimpleDirectoryReader, - storageContextFromDefaults, - VectorStoreIndex, -} from "llamaindex"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; +import { storageContextFromDefaults, VectorStoreIndex } from "llamaindex"; import fs from "node:fs/promises"; async function getSourceFilenames(sourceDir: string) { diff --git a/examples/vector-store/pg/supabase.ts b/examples/vector-store/pg/supabase.ts index db157b8625b9c948655ce658b649ce876dbed522..97e1e171da773d7fe55fd5bf7d9f9aba592ff36c 100644 --- a/examples/vector-store/pg/supabase.ts +++ b/examples/vector-store/pg/supabase.ts @@ -1,10 +1,7 @@ import { PGVectorStore } from "@llamaindex/postgres"; +import { SimpleDirectoryReader } from "@llamaindex/readers/directory"; import dotenv from "dotenv"; -import { - SimpleDirectoryReader, - storageContextFromDefaults, - VectorStoreIndex, -} from "llamaindex"; +import { storageContextFromDefaults, VectorStoreIndex } from "llamaindex"; dotenv.config(); diff --git a/packages/llamaindex/package.json b/packages/llamaindex/package.json index 12d1eb120430ee54d7c5844f8012df67565ca3b8..73add5213c2855ad68280b152cf4c2b8e79aea05 100644 --- a/packages/llamaindex/package.json +++ b/packages/llamaindex/package.json @@ -25,7 +25,6 @@ "@llamaindex/env": "workspace:*", "@llamaindex/node-parser": "workspace:*", "@llamaindex/openai": "workspace:*", - "@llamaindex/readers": "workspace:*", "@types/lodash": "^4.17.7", "@types/node": "^22.9.0", "ajv": "^8.17.1", diff --git a/packages/llamaindex/src/index.ts b/packages/llamaindex/src/index.ts index f3fd6ab5bf9e54e7ed857d3273bd5f3bfd47732f..ccdd367ae7286a779e1f08450de06081dfb06d9d 100644 --- a/packages/llamaindex/src/index.ts +++ b/packages/llamaindex/src/index.ts @@ -1,9 +1,9 @@ export * from "./index.edge.js"; -export * from "./readers/index.js"; -export * from "./storage/index.js"; +// TODO: clean up, move to jinaai package export { JinaAIEmbedding } from "./embeddings/JinaAIEmbedding.js"; -// Don't export SimpleVectorStore for non-node.js runtime on top level, +// Don't export file-system stores for non-node.js runtime on top level, // as we cannot guarantee that they will work in other environments +export * from "./storage/index.js"; export * from "./vector-store.js"; diff --git a/packages/llamaindex/src/readers/index.ts b/packages/llamaindex/src/readers/index.ts deleted file mode 100644 index 03045ab761e6afb13cb1d4ca844015438176dd63..0000000000000000000000000000000000000000 --- a/packages/llamaindex/src/readers/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -export { - LlamaParseReader, - type Language, - type ResultType, -} from "@llamaindex/cloud/reader"; -export * from "@llamaindex/readers/assembly-ai"; -export * from "@llamaindex/readers/cosmosdb"; -export * from "@llamaindex/readers/csv"; -export * from "@llamaindex/readers/directory"; -export * from "@llamaindex/readers/discord"; -export * from "@llamaindex/readers/docx"; -export * from "@llamaindex/readers/html"; -export * from "@llamaindex/readers/image"; -export * from "@llamaindex/readers/json"; -export * from "@llamaindex/readers/markdown"; -export * from "@llamaindex/readers/mongo"; -export * from "@llamaindex/readers/notion"; -export * from "@llamaindex/readers/obsidian"; -export * from "@llamaindex/readers/pdf"; -export * from "@llamaindex/readers/text"; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 50e3726e4de988a7568e53a0f9c5e216ccfefe8d..ae3cc07d4ccb454bcfba2e0eca6e21c65607dddf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -476,6 +476,9 @@ importers: '@llamaindex/huggingface': specifier: workspace:* version: link:../../../packages/providers/huggingface + '@llamaindex/readers': + specifier: workspace:* + version: link:../../../packages/readers llamaindex: specifier: workspace:* version: link:../../../packages/llamaindex @@ -516,6 +519,9 @@ importers: e2e/examples/waku-query-engine: dependencies: + '@llamaindex/env': + specifier: workspace:* + version: link:../../../packages/env llamaindex: specifier: workspace:* version: link:../../../packages/llamaindex @@ -1034,9 +1040,6 @@ importers: '@llamaindex/openai': specifier: workspace:* version: link:../providers/openai - '@llamaindex/readers': - specifier: workspace:* - version: link:../readers '@types/lodash': specifier: ^4.17.7 version: 4.17.15 diff --git a/unit/readers/SimpleCosmosDBReader.test.ts b/unit/readers/SimpleCosmosDBReader.test.ts index c7370a223ff0def8b315a18560485d8430290d8a..e87569977ba6c355ff0bfbe35e99db437510ced9 100644 --- a/unit/readers/SimpleCosmosDBReader.test.ts +++ b/unit/readers/SimpleCosmosDBReader.test.ts @@ -3,7 +3,7 @@ import { Document } from "@llamaindex/core/schema"; import { SimpleCosmosDBReader, type SimpleCosmosDBReaderLoaderConfig, -} from "llamaindex"; +} from "@llamaindex/readers/cosmosdb"; import { describe, expect, it, vi } from "vitest"; const createMockClient = (mockData?: unknown[]) => { diff --git a/unit/readers/register.test.ts b/unit/readers/register.test.ts index 14b1491358aebbd1ab8ec4beae43bea69492e985..e3701c005cb4fbf4455f854d42cc0b54954cfc57 100644 --- a/unit/readers/register.test.ts +++ b/unit/readers/register.test.ts @@ -5,7 +5,7 @@ test("node register", async () => { const code = `import csvDocument from '../examples/data/movie_reviews.csv';console.log(csvDocument.getText())`; const cp = spawnSync( process.execPath, - ["--input-type=module", "--import=llamaindex/register"], + ["--input-type=module", "--import=@llamaindex/readers/node"], { input: code, stdio: "pipe",