fix: files in sub folders of 'data' are not displayed (#241)

.changeset/good-oranges-cover.md

+---
+"create-llama": patch
+---
+
+Fix error that files in sub folders of 'data' are not displayed

templates/components/llamaindex/typescript/streaming/events.ts

@@ -7,6 +7,8 @@ import {
   ToolCall,
   ToolOutput,
 } from "llamaindex";
+import path from "node:path";
+import { DATA_DIR } from "../../engine/loader";
 import { LLamaCloudFileService } from "./service";
 
 export function appendSourceData(
@@ -122,8 +124,16 @@ function getNodeUrl(metadata: Metadata) {
       return `${process.env.FILESERVER_URL_PREFIX}/output/llamacloud/${name}`;
     }
     const isPrivate = metadata["private"] === "true";
-    const folder = isPrivate ? "output/uploaded" : "data";
-    return `${process.env.FILESERVER_URL_PREFIX}/${folder}/${fileName}`;
+    if (isPrivate) {
+      return `${process.env.FILESERVER_URL_PREFIX}/output/uploaded/${fileName}`;
+    }
+    const filePath = metadata["file_path"];
+    const dataDir = path.resolve(DATA_DIR);
+
+    if (filePath && dataDir) {
+      const relativePath = path.relative(dataDir, filePath);
+      return `${process.env.FILESERVER_URL_PREFIX}/data/${relativePath}`;
+    }
   }
   // fallback to URL in metadata (e.g. for websites)
   return metadata["URL"];

templates/components/loaders/python/file.py

@@ -2,21 +2,16 @@ import os
 import logging
 from typing import Dict
 from llama_parse import LlamaParse
-from pydantic import BaseModel, validator
+from pydantic import BaseModel
+
+from app.config import DATA_DIR
 
 logger = logging.getLogger(__name__)
 
 
 class FileLoaderConfig(BaseModel):
-    data_dir: str = "data"
     use_llama_parse: bool = False
 
-    @validator("data_dir")
-    def data_dir_must_exist(cls, v):
-        if not os.path.isdir(v):
-            raise ValueError(f"Directory '{v}' does not exist")
-        return v
-
 
 def llama_parse_parser():
     if os.getenv("LLAMA_CLOUD_API_KEY") is None:
@@ -54,7 +49,7 @@ def get_file_documents(config: FileLoaderConfig):
 
             file_extractor = llama_parse_extractor()
         reader = SimpleDirectoryReader(
-            config.data_dir,
+            DATA_DIR,
             recursive=True,
             filename_as_id=True,
             raise_on_error=True,

templates/types/streaming/fastapi/app/api/routers/models.py

@@ -7,6 +7,8 @@ from llama_index.core.schema import NodeWithScore
 from pydantic import BaseModel, Field, validator
 from pydantic.alias_generators import to_camel
 
+from app.config import DATA_DIR
+
 logger = logging.getLogger("uvicorn")
 
 
@@ -175,6 +177,7 @@ class SourceNodes(BaseModel):
                 "Warning: FILESERVER_URL_PREFIX not set in environment variables. Can't use file server"
             )
         file_name = metadata.get("file_name")
+
         if file_name and url_prefix:
             # file_name exists and file server is configured
             pipeline_id = metadata.get("pipeline_id")
@@ -184,11 +187,17 @@ class SourceNodes(BaseModel):
                 return f"{url_prefix}/output/llamacloud/{file_name}"
             is_private = metadata.get("private", "false") == "true"
             if is_private:
+                # file is a private upload
                 return f"{url_prefix}/output/uploaded/{file_name}"
-            return f"{url_prefix}/data/{file_name}"
-        else:
-            # fallback to URL in metadata (e.g. for websites)
-            return metadata.get("URL")
+            # file is from calling the 'generate' script
+            # Get the relative path of file_path to data_dir
+            file_path = metadata.get("file_path")
+            data_dir = os.path.abspath(DATA_DIR)
+            if file_path and data_dir:
+                relative_path = os.path.relpath(file_path, data_dir)
+                return f"{url_prefix}/data/{relative_path}"
+        # fallback to URL in metadata (e.g. for websites)
+        return metadata.get("URL")
 
     @classmethod
     def from_source_nodes(cls, source_nodes: List[NodeWithScore]):

templates/types/streaming/fastapi/app/config.py

+DATA_DIR = "data"

templates/types/streaming/fastapi/main.py

 # flake8: noqa: E402
 from dotenv import load_dotenv
 
+from app.config import DATA_DIR
+
 load_dotenv()
 
 import logging
@@ -43,15 +45,16 @@ if environment == "dev":
 
 def mount_static_files(directory, path):
     if os.path.exists(directory):
-        for dir, _, _ in os.walk(directory):
-            relative_path = os.path.relpath(dir, directory)
-            mount_path = path if relative_path == "." else f"{path}/{relative_path}"
-            logger.info(f"Mounting static files '{dir}' at {mount_path}")
-            app.mount(mount_path, StaticFiles(directory=dir), name=f"{dir}-static")
+        logger.info(f"Mounting static files '{directory}' at '{path}'")
+        app.mount(
+            path,
+            StaticFiles(directory=directory, check_dir=False),
+            name=f"{directory}-static",
+        )
 
 
 # Mount the data files to serve the file viewer
-mount_static_files("data", "/api/files/data")
+mount_static_files(DATA_DIR, "/api/files/data")
 # Mount the output files from tools
 mount_static_files("output", "/api/files/output")
 

templates/types/streaming/nextjs/app/api/files/[...slug]/route.ts

 import { readFile } from "fs/promises";
 import { NextRequest, NextResponse } from "next/server";
 import path from "path";
+import { DATA_DIR } from "../../chat/engine/loader";
 
 /**
  * This API is to get file data from allowed folders
@@ -28,7 +29,11 @@ export async function GET(
   }
 
   try {
-    const filePath = path.join(process.cwd(), folder, path.join(...pathTofile));
+    const filePath = path.join(
+      process.cwd(),
+      folder === "data" ? DATA_DIR : folder,
+      path.join(...pathTofile),
+    );
     const blob = await readFile(filePath);
 
     return new NextResponse(blob, {