From b42493c6de58dbb10a94ba9949813bebe2e92e3e Mon Sep 17 00:00:00 2001
From: Timothy Carambat <rambat1010@gmail.com>
Date: Thu, 3 Aug 2023 18:57:50 -0700
Subject: [PATCH] Split large PDFS into subfolder in documents (#176)
append time value to folder name to prevent duplicate uploads
---
collector/scripts/watch/convert/as_pdf.py | 5 +++--
collector/scripts/watch/utils.py | 4 ++--
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/collector/scripts/watch/convert/as_pdf.py b/collector/scripts/watch/convert/as_pdf.py
index 605c29c19..12163cf26 100644
--- a/collector/scripts/watch/convert/as_pdf.py
+++ b/collector/scripts/watch/convert/as_pdf.py
@@ -1,4 +1,4 @@
-import os
+import os, time
from langchain.document_loaders import PyPDFLoader
from slugify import slugify
from ..utils import guid, file_creation_time, write_to_server_documents, move_source
@@ -11,6 +11,7 @@ def as_pdf(**kwargs):
ext = kwargs.get('ext', '.txt')
remove = kwargs.get('remove_on_complete', False)
fullpath = f"{parent_dir}/{filename}{ext}"
+ destination = f"../server/storage/documents/{slugify(filename)}-{int(time.time())}"
loader = PyPDFLoader(fullpath)
pages = loader.load_and_split()
@@ -31,7 +32,7 @@ def as_pdf(**kwargs):
'pageContent': content,
'token_count_estimate': len(tokenize(content))
}
- write_to_server_documents(data, f"{slugify(filename)}-pg{pg_num}-{data.get('id')}")
+ write_to_server_documents(data, f"{slugify(filename)}-pg{pg_num}-{data.get('id')}", destination)
move_source(parent_dir, f"{filename}{ext}", remove=remove)
print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n")
\ No newline at end of file
diff --git a/collector/scripts/watch/utils.py b/collector/scripts/watch/utils.py
index b15b0b3b0..f61e39437 100644
--- a/collector/scripts/watch/utils.py
+++ b/collector/scripts/watch/utils.py
@@ -28,8 +28,8 @@ def move_source(working_dir='hotdir', new_destination_filename='', failed=False,
os.replace(f"{working_dir}/{new_destination_filename}", f"{destination}/{new_destination_filename}")
return
-def write_to_server_documents(data, filename):
- destination = f"../server/storage/documents/custom-documents"
+def write_to_server_documents(data, filename, override_destination = None):
+ destination = f"../server/storage/documents/custom-documents" if override_destination == None else override_destination
if os.path.exists(destination) == False: os.makedirs(destination)
with open(f"{destination}/{filename}.json", 'w', encoding='utf-8') as file:
json.dump(data, file, ensure_ascii=True, indent=4)
--
GitLab