Skip to content
Snippets Groups Projects
Commit 7abb86d5 authored by Ismail Ashraq's avatar Ismail Ashraq
Browse files

make default bm25 params optional

parent a558eabb
No related branches found
No related tags found
No related merge requests found
...@@ -9,7 +9,12 @@ class BM25Encoder(BaseEncoder): ...@@ -9,7 +9,12 @@ class BM25Encoder(BaseEncoder):
idx_mapping: dict[int, int] | None = None idx_mapping: dict[int, int] | None = None
type: str = "sparse" type: str = "sparse"
def __init__(self, name: str = "bm25", score_threshold: float = 0.82): def __init__(
self,
name: str = "bm25",
score_threshold: float = 0.82,
use_default_params: bool = True,
):
super().__init__(name=name, score_threshold=score_threshold) super().__init__(name=name, score_threshold=score_threshold)
try: try:
from pinecone_text.sparse import BM25Encoder as encoder from pinecone_text.sparse import BM25Encoder as encoder
...@@ -18,9 +23,15 @@ class BM25Encoder(BaseEncoder): ...@@ -18,9 +23,15 @@ class BM25Encoder(BaseEncoder):
"Please install pinecone-text to use BM25Encoder. " "Please install pinecone-text to use BM25Encoder. "
"You can install it with: `pip install semantic-router[hybrid]`" "You can install it with: `pip install semantic-router[hybrid]`"
) )
logger.info("Downloading and initializing BM25 model parameters.")
self.model = encoder.default()
self.model = encoder()
if use_default_params:
logger.info("Downloading and initializing default sBM25 model parameters.")
self.model = encoder.default()
self._set_idx_mapping()
def _set_idx_mapping(self):
params = self.model.get_params() params = self.model.get_params()
doc_freq = params["doc_freq"] doc_freq = params["doc_freq"]
if isinstance(doc_freq, dict): if isinstance(doc_freq, dict):
...@@ -53,3 +64,4 @@ class BM25Encoder(BaseEncoder): ...@@ -53,3 +64,4 @@ class BM25Encoder(BaseEncoder):
if self.model is None: if self.model is None:
raise ValueError("Model is not initialized.") raise ValueError("Model is not initialized.")
self.model.fit(docs) self.model.fit(docs)
self._set_idx_mapping()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment