diff --git a/packages/core/src/readers/PDFReader.ts b/packages/core/src/readers/PDFReader.ts index caa9580b93cfb41232ffae331f19bf971fb784ff..bda8ab753ebd897c50349ab346d06f0273e20228 100644 --- a/packages/core/src/readers/PDFReader.ts +++ b/packages/core/src/readers/PDFReader.ts @@ -1,11 +1,11 @@ import { fs } from "@llamaindex/env"; import { Document } from "../Node.js"; -import type { BaseReader } from "./type.js"; +import type { FileReader } from "./type.js"; /** * Read the text of a PDF */ -export class PDFReader implements BaseReader { +export class PDFReader implements FileReader { async loadData(file: string): Promise<Document[]> { const content = await fs.readFile(file); const pages = await readPDF(content); diff --git a/packages/core/src/readers/SimpleDirectoryReader.edge.ts b/packages/core/src/readers/SimpleDirectoryReader.edge.ts index 5362391e418ce3a7bdd1f0182e097d9ea22446a6..589f455853c59d54ac1aede7a9fe6a69ed650b62 100644 --- a/packages/core/src/readers/SimpleDirectoryReader.edge.ts +++ b/packages/core/src/readers/SimpleDirectoryReader.edge.ts @@ -1,8 +1,8 @@ -import { fs, path } from "@llamaindex/env"; +import { path } from "@llamaindex/env"; import { Document, type Metadata } from "../Node.js"; import { walk } from "../storage/FileSystem.js"; import { TextFileReader } from "./TextFileReader.js"; -import type { BaseReader } from "./type.js"; +import type { BaseReader, FileReader } from "./type.js"; import pLimit from "./utils.js"; type ReaderCallback = ( @@ -20,13 +20,13 @@ enum ReaderStatus { export type SimpleDirectoryReaderLoadDataParams = { directoryPath: string; // Fallback Reader, defaults to TextFileReader - defaultReader?: BaseReader | null; + defaultReader?: FileReader | null; // Map file extensions individually to readers - fileExtToReader?: Record<string, BaseReader>; + fileExtToReader?: Record<string, FileReader>; // Number of workers, defaults to 1. Must be between 1 and 9. numWorkers?: number; // Overrides reader for all file extensions - overrideReader?: BaseReader; + overrideReader?: FileReader; }; type ProcessFileParams = Omit< @@ -115,7 +115,7 @@ export class SimpleDirectoryReader implements BaseReader { return []; } - let reader: BaseReader; + let reader: FileReader; if (params.overrideReader) { reader = params.overrideReader; @@ -135,7 +135,7 @@ export class SimpleDirectoryReader implements BaseReader { return []; } - const fileDocs = await reader.loadData(filePath, fs); + const fileDocs = await reader.loadData(filePath); fileDocs.forEach(addMetaData(filePath)); // Observer can still cancel addition of the resulting docs from this file diff --git a/packages/core/src/readers/SimpleDirectoryReader.ts b/packages/core/src/readers/SimpleDirectoryReader.ts index 70d49ef3be414d935d308a077e826597ac89d5ca..10b3b9bdbcb071681167eba78a01ad6ea4572ded 100644 --- a/packages/core/src/readers/SimpleDirectoryReader.ts +++ b/packages/core/src/readers/SimpleDirectoryReader.ts @@ -10,9 +10,9 @@ import { type SimpleDirectoryReaderLoadDataParams, } from "./SimpleDirectoryReader.edge.js"; import { TextFileReader } from "./TextFileReader.js"; -import type { BaseReader } from "./type.js"; +import type { FileReader } from "./type.js"; -export const FILE_EXT_TO_READER: Record<string, BaseReader> = { +export const FILE_EXT_TO_READER: Record<string, FileReader> = { txt: new TextFileReader(), pdf: new PDFReader(), csv: new PapaCSVReader(), diff --git a/packages/core/src/readers/TextFileReader.ts b/packages/core/src/readers/TextFileReader.ts index 7d51ac53b8fd6390e81f34b8bcd63325860267f4..0b91e69bfde1520dbc853679a6300bae88e567cd 100644 --- a/packages/core/src/readers/TextFileReader.ts +++ b/packages/core/src/readers/TextFileReader.ts @@ -1,12 +1,12 @@ import { fs } from "@llamaindex/env"; import { Document } from "../Node.js"; -import type { BaseReader } from "./type.js"; +import type { FileReader } from "./type.js"; /** * Read a .txt file */ -export class TextFileReader implements BaseReader { +export class TextFileReader implements FileReader { async loadData(file: string): Promise<Document[]> { const dataBuffer = await fs.readFile(file, "utf-8"); return [new Document({ text: dataBuffer, id_: file })];