Split large PDFS into subfolder in documents (#176)

append time value to folder name to prevent duplicate uploads

collector/scripts/watch/convert/as_pdf.py

-import os
+import os, time
 from langchain.document_loaders import PyPDFLoader
 from slugify import slugify
 from ..utils import guid, file_creation_time, write_to_server_documents, move_source
@@ -11,6 +11,7 @@ def as_pdf(**kwargs):
   ext = kwargs.get('ext', '.txt')
   remove = kwargs.get('remove_on_complete', False)
   fullpath = f"{parent_dir}/{filename}{ext}"
+  destination = f"../server/storage/documents/{slugify(filename)}-{int(time.time())}"
 
   loader = PyPDFLoader(fullpath)
   pages = loader.load_and_split()
@@ -31,7 +32,7 @@ def as_pdf(**kwargs):
       'pageContent': content,
       'token_count_estimate': len(tokenize(content))
     }
-    write_to_server_documents(data, f"{slugify(filename)}-pg{pg_num}-{data.get('id')}")
+    write_to_server_documents(data, f"{slugify(filename)}-pg{pg_num}-{data.get('id')}", destination)
 
   move_source(parent_dir, f"{filename}{ext}", remove=remove)
   print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n")
\ No newline at end of file

collector/scripts/watch/utils.py

@@ -28,8 +28,8 @@ def move_source(working_dir='hotdir', new_destination_filename='', failed=False,
   os.replace(f"{working_dir}/{new_destination_filename}", f"{destination}/{new_destination_filename}")
   return
 
-def write_to_server_documents(data, filename):
-  destination = f"../server/storage/documents/custom-documents"
+def write_to_server_documents(data, filename, override_destination = None):
+  destination = f"../server/storage/documents/custom-documents" if override_destination == None else override_destination
   if os.path.exists(destination) == False: os.makedirs(destination)
   with open(f"{destination}/{filename}.json", 'w', encoding='utf-8') as file:
     json.dump(data, file, ensure_ascii=True, indent=4)