Merge pull request #263 from aurelio-labs/james/encoder-disallowed-special

fix: James/encoder disallowed special

pyproject.toml

 [tool.poetry]
 name = "semantic-router"
-version = "0.0.36"
+version = "0.0.37"
 description = "Super fast semantic router for AI decision making"
 authors = [
     "James Briggs <james@aurelio.ai>",

semantic_router/__init__.py

@@ -4,4 +4,4 @@ from semantic_router.route import Route
 
 __all__ = ["RouteLayer", "HybridRouteLayer", "Route", "LayerConfig"]
 
-__version__ = "0.0.36"
+__version__ = "0.0.37"

semantic_router/encoders/openai.py

@@ -114,7 +114,8 @@ class OpenAIEncoder(BaseEncoder):
         return embeddings
 
     def _truncate(self, text: str) -> str:
-        tokens = self._token_encoder.encode(text)
+        # we use encode_ordinary as faster equivalent to encode(text, disallowed_special=())
+        tokens = self._token_encoder.encode_ordinary(text)
         if len(tokens) > self.token_limit:
             logger.warning(
                 f"Document exceeds token limit: {len(tokens)} > {self.token_limit}"