feat: data connector for obsidian note taking app (#1529)

examples/readers/package.json

@@ -15,7 +15,8 @@
     "start:llamaparse-dir": "node --import tsx ./src/simple-directory-reader-with-llamaparse.ts",
     "start:llamaparse-json": "node --import tsx ./src/llamaparse-json.ts",
     "start:discord": "node --import tsx ./src/discord.ts",
-    "start:json": "node --import tsx ./src/json.ts"
+    "start:json": "node --import tsx ./src/json.ts",
+    "start:obsidian": "node --import tsx ./src/obsidian.ts"
   },
   "dependencies": {
     "@llamaindex/readers": "*",

examples/readers/src/obsidian.ts

+import { ObsidianReader } from "@llamaindex/readers/obsidian";
+
+const obsidianReader = new ObsidianReader(
+  "/Users/jingyi/Documents/jingyi-vault",
+);
+
+obsidianReader.loadData().then((documents) => {
+  console.log("documents:", documents.length);
+  documents.forEach((doc) => {
+    console.log(`document (${doc.id_}):`, doc.getText());
+  });
+});

packages/readers/package.json

@@ -142,6 +142,24 @@
         "default": "./notion/dist/index.js"
       }
     },
+    "./obsidian": {
+      "edge-light": {
+        "types": "./obsidian/dist/index.edge-light.d.ts",
+        "default": "./obsidian/dist/index.edge-light.js"
+      },
+      "workerd": {
+        "types": "./obsidian/dist/index.workerd.d.ts",
+        "default": "./obsidian/dist/index.workerd.js"
+      },
+      "require": {
+        "types": "./obsidian/dist/index.d.cts",
+        "default": "./obsidian/dist/index.cjs"
+      },
+      "import": {
+        "types": "./obsidian/dist/index.d.ts",
+        "default": "./obsidian/dist/index.js"
+      }
+    },
     "./pdf": {
       "require": {
         "types": "./pdf/dist/index.d.cts",
@@ -176,6 +194,7 @@
     "markdown",
     "mongo",
     "notion",
+    "obsidian",
     "pdf",
     "text",
     "node"

packages/readers/src/obsidian.ts

+import { type BaseReader, Document } from "@llamaindex/core/schema";
+import * as fs from "node:fs";
+import path from "node:path";
+import { MarkdownReader } from "./markdown";
+
+export class ObsidianReader implements BaseReader<Document> {
+  protected inputDir: string;
+  protected docs: Document[] = [];
+
+  constructor(inputDir: string) {
+    this.inputDir = inputDir;
+  }
+
+  private async processPath(file: fs.Dirent, filepath: string) {
+    if (file.isDirectory() && !file.name.startsWith(".")) {
+      await this.readFromPath(filepath);
+    } else if (file.isFile() && file.name.endsWith(".md")) {
+      await this.convertToDocuments(filepath);
+    } else {
+      console.log(`Skipping ${filepath}`);
+    }
+  }
+
+  private async readFromPath(dir: string) {
+    const files = await fs.promises.readdir(dir, { withFileTypes: true });
+    for (const file of files) {
+      const filepath = path.join(dir, file.name);
+      await this.processPath(file, filepath);
+    }
+  }
+
+  private async convertToDocuments(filepath: string) {
+    const content = await new MarkdownReader().loadData(filepath);
+    this.docs.push(...content);
+  }
+
+  async loadData(): Promise<Document[]> {
+    await this.readFromPath(this.inputDir);
+    return this.docs;
+  }
+}