From 6851073201ffae11fb3dd105850477dc769b03ac Mon Sep 17 00:00:00 2001
From: Javier Torres <javierandrestorresreyes@gmail.com>
Date: Thu, 21 Mar 2024 16:01:37 -0500
Subject: [PATCH] Add s3 as data source (#12151)
* use s3fs
* when key=none
* use s3fs
* changelog + version bump
* tests + serialization
* changelog
* pants update
* module mappings
* add files reader to BUILD
* more BUILD
* add s3 to enum
* fix schema
* give BaseReader a __modify_schema__
* give BaseReader a __modify_schema__
* dont use ModelField
* remove validators
* Support schema in BaseReader for PydanticV2
---------
Co-authored-by: Logan Markewich <logan.markewich@live.com>
---
.../llama_index/core/ingestion/data_sources.py | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/llama-index-core/llama_index/core/ingestion/data_sources.py b/llama-index-core/llama_index/core/ingestion/data_sources.py
index 39509b729b..ab00deecea 100644
--- a/llama-index-core/llama_index/core/ingestion/data_sources.py
+++ b/llama-index-core/llama_index/core/ingestion/data_sources.py
@@ -294,6 +294,21 @@ def build_configurable_data_source_enum():
except ImportError:
pass
+ try:
+ from llama_index.readers.s3 import S3Reader # pants: no-infer-dep
+
+ enum_members.append(
+ (
+ "S3",
+ DataSource(
+ name="S3",
+ component_type=S3Reader,
+ ),
+ )
+ )
+ except ImportError:
+ pass
+
enum_members.append(
(
"READER",
--
GitLab