diff --git a/llama_index/readers/download.py b/llama_index/readers/download.py
index 73f666d50a1ab90d4061cbd0678461aa9071c98f..48ca16446ab4ad85138c7b4f7ba752be411b8f3c 100644
--- a/llama_index/readers/download.py
+++ b/llama_index/readers/download.py
@@ -35,11 +35,17 @@ def download_loader(
     Returns:
         A Loader.
     """
+    # Only one of the `custom_dir` or `custom_path` is supported.
+    if custom_path is not None:
+        custom_dir = None
+    else:
+        custom_dir = "llamahub_modules"
+
     reader_cls = download_llama_module(
         loader_class,
         llama_hub_url=loader_hub_url,
         refresh_cache=refresh_cache,
-        custom_dir="llamahub_modules",
+        custom_dir=custom_dir,
         custom_path=custom_path,
         use_gpt_index_import=use_gpt_index_import,
     )