fix: created embedding helper functions

semantic_router/hybrid_layer.py

@@ -56,45 +56,33 @@ class HybridRouteLayer:
             return None
 
     def add(self, route: Route):
+        self._add_route(route=route)
+
+    def _add_route(self, route: Route):
+        self.routes += [route]
+
+        self.update_dense_embeddings_index(route.utterances)
+
         if isinstance(self.sparse_encoder, TfidfEncoder) and hasattr(
             self.sparse_encoder, "fit"
         ):
-            self.sparse_encoder.fit(self.routes + [route])
+            self.sparse_encoder.fit(self.routes)
+            # re-build index
             self.sparse_index = None
-            for r in self.routes:
-                self.compute_and_store_sparse_embeddings(r)
-        self.routes.append(route)
-        self._add_route(route=route)
+            all_utterances = [
+                utterance for route in self.routes for utterance in route.utterances
+            ]
+            self.update_sparse_embeddings_index(all_utterances)
+        else:
+            self.update_sparse_embeddings_index(route.utterances)
 
-    def _add_route(self, route: Route):
-        # create embeddings
-        dense_embeds = np.array(self.dense_encoder(route.utterances))  # * self.alpha
-        self.compute_and_store_sparse_embeddings(route)
         # create route array
         if self.categories is None:
             self.categories = np.array([route.name] * len(route.utterances))
-            self.utterances = np.array(route.utterances)
         else:
             str_arr = np.array([route.name] * len(route.utterances))
             self.categories = np.concatenate([self.categories, str_arr])
-            self.utterances = np.concatenate(
-                [self.utterances, np.array(route.utterances)]
-            )
-        # create utterance array (the dense index)
-        if self.index is None:
-            self.index = dense_embeds
-        else:
-            self.index = np.concatenate([self.index, dense_embeds])
-
-    def compute_and_store_sparse_embeddings(self, route: Route):
-        sparse_embeds = np.array(
-            self.sparse_encoder(route.utterances)
-        )  # * (1 - self.alpha)
-        # create sparse utterance array
-        if self.sparse_index is None:
-            self.sparse_index = sparse_embeds
-        else:
-            self.sparse_index = np.concatenate([self.sparse_index, sparse_embeds])
+        self.routes.append(route)
 
     def _add_routes(self, routes: list[Route]):
         # create embeddings for all routes
@@ -102,8 +90,8 @@ class HybridRouteLayer:
         all_utterances = [
             utterance for route in routes for utterance in route.utterances
         ]
-        dense_embeds = np.array(self.dense_encoder(all_utterances))
-        sparse_embeds = np.array(self.sparse_encoder(all_utterances))
+        self.update_dense_embeddings_index(all_utterances)
+        self.update_sparse_embeddings_index(all_utterances)
 
         # create route array
         route_names = [route.name for route in routes for _ in route.utterances]
@@ -114,6 +102,8 @@ class HybridRouteLayer:
             else route_array
         )
 
+    def update_dense_embeddings_index(self, utterances: list):
+        dense_embeds = np.array(self.dense_encoder(utterances))
         # create utterance array (the dense index)
         self.index = (
             np.concatenate([self.index, dense_embeds])
@@ -121,6 +111,8 @@ class HybridRouteLayer:
             else dense_embeds
         )
 
+    def update_sparse_embeddings_index(self, utterances: list):
+        sparse_embeds = np.array(self.sparse_encoder(utterances))
         # create sparse utterance array
         self.sparse_index = (
             np.concatenate([self.sparse_index, sparse_embeds])