From 3e8057a83a7a7b5ac1df15d2ece6929bbc77ee26 Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Tue, 1 Oct 2024 16:22:22 +0700
Subject: [PATCH] improve saveDocument
---
.../llamaindex/typescript/documents/helper.ts | 44 ++++++++++++-------
1 file changed, 29 insertions(+), 15 deletions(-)
diff --git a/templates/components/llamaindex/typescript/documents/helper.ts b/templates/components/llamaindex/typescript/documents/helper.ts
index 3e878f08..bfe74522 100644
--- a/templates/components/llamaindex/typescript/documents/helper.ts
+++ b/templates/components/llamaindex/typescript/documents/helper.ts
@@ -1,4 +1,5 @@
-import fs from "fs";
+import fs from "node:fs";
+import path from "node:path";
import { getExtractors } from "../../engine/loader";
const MIME_TYPE_TO_EXT: Record<string, string> = {
@@ -19,7 +20,8 @@ export async function storeAndParseFile(
if (!fileExt) throw new Error(`Unsupported document type: ${mimeType}`);
const documents = await loadDocuments(fileBuffer, mimeType);
- await saveDocument(filename, fileBuffer);
+ const filepath = path.join(UPLOADED_FOLDER, filename);
+ await saveDocument(filepath, fileBuffer);
for (const document of documents) {
document.metadata = {
...document.metadata,
@@ -41,19 +43,31 @@ async function loadDocuments(fileBuffer: Buffer, mimeType: string) {
return await reader.loadDataAsContent(fileBuffer);
}
-export async function saveDocument(filename: string, fileBuffer: Buffer) {
- const filepath = `${UPLOADED_FOLDER}/${filename}`;
- const fileurl = `${process.env.FILESERVER_URL_PREFIX}/${filepath}`;
+// Save document to file server and return the file url
+export async function saveDocument(filepath: string, content: string | Buffer) {
+ if (path.isAbsolute(filepath)) {
+ throw new Error("Absolute file paths are not allowed.");
+ }
+ const fileName = path.basename(filepath);
+ if (!/^[a-zA-Z0-9_.-]+$/.test(fileName)) {
+ throw new Error(
+ "File name is not allowed to contain any special characters.",
+ );
+ }
+ if (!process.env.FILESERVER_URL_PREFIX) {
+ throw new Error("FILESERVER_URL_PREFIX environment variable is not set.");
+ }
- if (!fs.existsSync(UPLOADED_FOLDER)) {
- fs.mkdirSync(UPLOADED_FOLDER, { recursive: true });
+ const dirPath = path.dirname(filepath);
+ await fs.promises.mkdir(dirPath, { recursive: true });
+
+ if (typeof content === "string") {
+ await fs.promises.writeFile(filepath, content, "utf-8");
+ } else {
+ await fs.promises.writeFile(filepath, content);
}
- await fs.promises.writeFile(filepath, fileBuffer);
-
- console.log(`Saved document file to ${filepath}.\nURL: ${fileurl}`);
- return {
- filename,
- filepath,
- fileurl,
- };
+
+ const fileurl = `${process.env.FILESERVER_URL_PREFIX}/${filepath}`;
+ console.log(`Saved document to ${filepath}. Reachable at URL: ${fileurl}`);
+ return fileurl;
}
--
GitLab