diff --git a/poetry.lock b/poetry.lock
index eb4ec63cfc5ccdc24224397082ed0e3e10848cec..1b4125e3829874e2a60595031fadb25dac46b148 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -4220,7 +4220,7 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more
 bedrock = ["boto3"]
 google = ["google-cloud-aiplatform"]
 hybrid = ["pinecone-text"]
-local = ["huggingface-hub", "llama-cpp-python", "tokenizers", "torch", "transformers"]
+local = ["llama-cpp-python", "tokenizers", "torch", "transformers"]
 mistralai = ["mistralai"]
 pinecone = ["pinecone-client"]
 processing = ["matplotlib"]
@@ -4230,4 +4230,4 @@ vision = ["pillow", "torch", "torchvision", "transformers"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.13"
-content-hash = "0233c0677c37714b0de5f6e319e055fceac9718e8088c6ed56cf969d443e29b6"
+content-hash = "17f2d76c59c4cb39899f69fe1e2242933a5747cfceb24a77a024d0cae0da1b3f"
diff --git a/pyproject.toml b/pyproject.toml
index 817f985966a295d5472e3d2ba2d5f4c7e6601485..88e515e82f09dc0edc0cc2f99767248c7d43e2b4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,7 +28,6 @@ pinecone-text = {version = ">=0.7.1,<0.8.0", optional = true}
 torch = {version = ">=2.1.0,<2.6.0", optional = true}
 transformers = {version = ">=4.36.2", optional = true}
 tokenizers = {version = ">=0.19", optional = true}
-huggingface-hub = {version = "!=0.23.*", optional = true}
 llama-cpp-python = {version = "^0.2.28", optional = true}
 colorama = "^0.4.6"
 pinecone-client = {version=">=3.0.0,<4.0.0", optional = true}
diff --git a/tests/unit/encoders/test_clip.py b/tests/unit/encoders/test_clip.py
index 4247b279292e919d95bd7c4adf45d1b1d3d83615..de997e36deb3c22349fb2f8eeba58878b112fe26 100644
--- a/tests/unit/encoders/test_clip.py
+++ b/tests/unit/encoders/test_clip.py
@@ -4,14 +4,8 @@ import torch
 from PIL import Image
 
 from semantic_router.encoders import CLIPEncoder
-from transformers import AutoTokenizer, AutoModel
 
 test_model_name = "aurelio-ai/sr-test-clip"
-
-# force the model download
-tokenizer = AutoTokenizer.from_pretrained(test_model_name, force_download=True)
-model = AutoModel.from_pretrained(test_model_name, force_download=True)
-
 clip_encoder = CLIPEncoder(name=test_model_name)
 embed_dim = 64
 
diff --git a/tests/unit/encoders/test_huggingface.py b/tests/unit/encoders/test_huggingface.py
index 3d21b1df684c96d0a8fc51145f2b62f01b8d5310..f14c7a685a497c4fbb7a6afca4c2f8f85c355007 100644
--- a/tests/unit/encoders/test_huggingface.py
+++ b/tests/unit/encoders/test_huggingface.py
@@ -4,14 +4,8 @@ import numpy as np
 import pytest
 
 from semantic_router.encoders.huggingface import HuggingFaceEncoder
-from transformers import AutoTokenizer, AutoModel
 
 test_model_name = "aurelio-ai/sr-test-huggingface"
-
-# force the model download
-tokenizer = AutoTokenizer.from_pretrained(test_model_name, force_download=True)
-model = AutoModel.from_pretrained(test_model_name, force_download=True)
-
 encoder = HuggingFaceEncoder(name=test_model_name)
 
 
diff --git a/tests/unit/encoders/test_vit.py b/tests/unit/encoders/test_vit.py
index d7e3d3bfd06a522fefb0e164f5ad519121aaf4a2..64f605e4720df8c3e9942eaec06ae192a4a37ce8 100644
--- a/tests/unit/encoders/test_vit.py
+++ b/tests/unit/encoders/test_vit.py
@@ -4,14 +4,8 @@ import torch
 from PIL import Image
 
 from semantic_router.encoders import VitEncoder
-from transformers import AutoTokenizer, AutoModel
 
 test_model_name = "aurelio-ai/sr-test-vit"
-
-# force the model download
-tokenizer = AutoTokenizer.from_pretrained(test_model_name, force_download=True)
-model = AutoModel.from_pretrained(test_model_name, force_download=True)
-
 vit_encoder = VitEncoder(name=test_model_name)
 embed_dim = 32