Skip to content
Snippets Groups Projects
Commit ff1161f2 authored by Simonas's avatar Simonas
Browse files

Revert "fix: Added fix to encode documents within rolling window"

This reverts commit 7bb47b3efa90c8ea21b7ac756158a7f70f4bc04d.
parent ee0d792b
No related branches found
No related tags found
No related merge requests found
......@@ -100,19 +100,12 @@ class RollingWindowSplitter(BaseSplitter):
return splits
def _encode_documents(self, docs: List[str]) -> np.ndarray:
max_docs_per_batch = 2000 # OpenAI limit is 2048
embeddings = []
for i in range(0, len(docs), max_docs_per_batch):
batch_docs = docs[i : i + max_docs_per_batch]
try:
batch_embeddings = self.encoder(batch_docs)
embeddings.extend(batch_embeddings)
except Exception as e:
logger.error(f"Error encoding documents {batch_docs}: {e}")
raise
return np.array(embeddings)
try:
embeddings = self.encoder(docs)
return np.array(embeddings)
except Exception as e:
logger.error(f"Error encoding documents {docs}: {e}")
raise
def _calculate_similarity_scores(self, encoded_docs: np.ndarray) -> List[float]:
raw_similarities = []
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment