From f500963ddcda5a9d17f8d67594506dd5f286b2f3 Mon Sep 17 00:00:00 2001
From: Kirushikesh DB <49152921+Kirushikesh@users.noreply.github.com>
Date: Fri, 15 Mar 2024 05:48:08 +0530
Subject: [PATCH] Incorrect warning message on HuggingFaceLLM tokenizer
(#11883)
---
.../llama_index/llms/huggingface/base.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llama-index-integrations/llms/llama-index-llms-huggingface/llama_index/llms/huggingface/base.py b/llama-index-integrations/llms/llama-index-llms-huggingface/llama_index/llms/huggingface/base.py
index bf09798201..e8724c6f13 100644
--- a/llama-index-integrations/llms/llama-index-llms-huggingface/llama_index/llms/huggingface/base.py
+++ b/llama-index-integrations/llms/llama-index-llms-huggingface/llama_index/llms/huggingface/base.py
@@ -183,9 +183,9 @@ class HuggingFaceLLM(CustomLLM):
tokenizer_name, **tokenizer_kwargs
)
- if tokenizer_name != model_name:
+ if self._tokenizer.name_or_path != model_name:
logger.warning(
- f"The model `{model_name}` and tokenizer `{tokenizer_name}` "
+ f"The model `{model_name}` and tokenizer `{self._tokenizer.name_or_path}` "
f"are different, please ensure that they are compatible."
)
--
GitLab