diff --git a/docs/00-introduction.ipynb b/docs/00-introduction.ipynb
index 60c2a07f2ce5810a7e52a388676609b40e9b739a..9c68cd0c90b9790098f3510fee64222a8e52252a 100644
--- a/docs/00-introduction.ipynb
+++ b/docs/00-introduction.ipynb
@@ -37,11 +37,21 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "[notice] A new release of pip is available: 23.1.2 -> 24.0\n",
+ "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
+ ]
+ }
+ ],
"source": [
- "!pip install -qU semantic-router==0.0.34"
+ "!pip install -qU semantic-router==0.0.35"
]
},
{
@@ -53,7 +63,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -81,7 +91,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@@ -108,7 +118,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@@ -136,14 +146,14 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "\u001b[32m2024-01-07 18:08:29 INFO semantic_router.utils.logger Initializing RouteLayer\u001b[0m\n"
+ "\u001b[32m2024-04-19 18:34:06 INFO semantic_router.utils.logger local\u001b[0m\n"
]
}
],
@@ -162,16 +172,16 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "RouteChoice(name='politics', function_call=None)"
+ "RouteChoice(name='politics', function_call=None, similarity_score=None)"
]
},
- "execution_count": 5,
+ "execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
@@ -182,16 +192,16 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "RouteChoice(name='chitchat', function_call=None)"
+ "RouteChoice(name='chitchat', function_call=None, similarity_score=None)"
]
},
- "execution_count": 6,
+ "execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
@@ -209,16 +219,16 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "RouteChoice(name=None, function_call=None)"
+ "RouteChoice(name=None, function_call=None, similarity_score=None)"
]
},
- "execution_count": 7,
+ "execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
@@ -231,8 +241,56 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "In this case, we return `None` because no matches were identified."
+ "We can also retrieve multiple routes with its associated score:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[RouteChoice(name='politics', function_call=None, similarity_score=0.8596186767854487),\n",
+ " RouteChoice(name='chitchat', function_call=None, similarity_score=0.8356239688161808)]"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "rl.retrieve_multiple_routes(\"Hi! How are you doing in politics??\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[]"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "rl.retrieve_multiple_routes(\"I'm interested in learning about llama 2\")"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -251,7 +309,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.5"
+ "version": "3.12.2"
}
},
"nbformat": 4,
diff --git a/semantic_router/layer.py b/semantic_router/layer.py
index 21c0da2766cd265a632d0e92e370c2a46a509bbd..a138893a88427fc7d7791f085d3508ee67bd6879 100644
--- a/semantic_router/layer.py
+++ b/semantic_router/layer.py
@@ -271,6 +271,32 @@ class RouteLayer:
# if no route passes threshold, return empty route choice
return RouteChoice()
+ def retrieve_multiple_routes(
+ self,
+ text: Optional[str] = None,
+ vector: Optional[List[float]] = None,
+ ) -> List[RouteChoice]:
+ if vector is None:
+ if text is None:
+ raise ValueError("Either text or vector must be provided")
+ vector_arr = self._encode(text=text)
+ else:
+ vector_arr = np.array(vector)
+ # get relevant utterances
+ results = self._retrieve(xq=vector_arr)
+
+ # decide most relevant routes
+ categories_with_scores = self._semantic_classify_multiple_routes(results)
+
+ route_choices = []
+ for category, score in categories_with_scores:
+ route = self.check_for_matching_routes(category)
+ if route:
+ route_choice = RouteChoice(name=route.name, similarity_score=score)
+ route_choices.append(route_choice)
+
+ return route_choices
+
def _retrieve_top_route(
self, vector: List[float], route_filter: Optional[List[str]] = None
) -> Tuple[Optional[Route], List[float]]:
@@ -423,14 +449,7 @@ class RouteLayer:
)
def _semantic_classify(self, query_results: List[dict]) -> Tuple[str, List[float]]:
- scores_by_class: Dict[str, List[float]] = {}
- for result in query_results:
- score = result["score"]
- route = result["route"]
- if route in scores_by_class:
- scores_by_class[route].append(score)
- else:
- scores_by_class[route] = [score]
+ scores_by_class = self.group_scores_by_class(query_results)
# Calculate total score for each class
total_scores = {
@@ -446,6 +465,49 @@ class RouteLayer:
logger.warning("No classification found for semantic classifier.")
return "", []
+ def get(self, name: str) -> Optional[Route]:
+ for route in self.routes:
+ if route.name == name:
+ return route
+ logger.error(f"Route `{name}` not found")
+ return None
+
+ def _semantic_classify_multiple_routes(
+ self, query_results: List[dict]
+ ) -> List[Tuple[str, float]]:
+ scores_by_class = self.group_scores_by_class(query_results)
+
+ # Filter classes based on threshold and find max score for each
+ classes_above_threshold = []
+ for route_name, scores in scores_by_class.items():
+ # Use the get method to find the Route object by its name
+ route_obj = self.get(route_name)
+ if route_obj is not None:
+ # Use the Route object's threshold if it exists, otherwise use the provided threshold
+ _threshold = (
+ route_obj.score_threshold
+ if route_obj.score_threshold is not None
+ else self.score_threshold
+ )
+ if self._pass_threshold(scores, _threshold):
+ max_score = max(scores)
+ classes_above_threshold.append((route_name, max_score))
+
+ return classes_above_threshold
+
+ def group_scores_by_class(
+ self, query_results: List[dict]
+ ) -> Dict[str, List[float]]:
+ scores_by_class: Dict[str, List[float]] = {}
+ for result in query_results:
+ score = result["score"]
+ route = result["route"]
+ if route in scores_by_class:
+ scores_by_class[route].append(score)
+ else:
+ scores_by_class[route] = [score]
+ return scores_by_class
+
def _pass_threshold(self, scores: List[float], threshold: float) -> bool:
if scores:
return max(scores) > threshold
diff --git a/tests/unit/test_layer.py b/tests/unit/test_layer.py
index 59830da1ac3e06060f944b5ad4e071503ebac90a..8f4833f0011225aac4f3337607c0f301083b46f9 100644
--- a/tests/unit/test_layer.py
+++ b/tests/unit/test_layer.py
@@ -97,6 +97,22 @@ def routes():
]
+@pytest.fixture
+def routes_2():
+ return [
+ Route(name="Route 1", utterances=["Hello"]),
+ Route(name="Route 2", utterances=["Hello"]),
+ ]
+
+
+@pytest.fixture
+def routes_3():
+ return [
+ Route(name="Route 1", utterances=["Hello"]),
+ Route(name="Route 2", utterances=["Asparagus"]),
+ ]
+
+
@pytest.fixture
def dynamic_routes():
return [
@@ -503,6 +519,149 @@ class TestRouteLayer:
)
assert route_layer.get_thresholds() == {"Route 1": 0.82, "Route 2": 0.82}
+ def test_with_multiple_routes_passing_threshold(
+ self, openai_encoder, routes, index_cls
+ ):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes, index=index_cls()
+ )
+ route_layer.score_threshold = 0.5 # Set the score_threshold if needed
+ # Assuming route_layer is already set up with routes "Route 1" and "Route 2"
+ query_results = [
+ {"route": "Route 1", "score": 0.6},
+ {"route": "Route 2", "score": 0.7},
+ {"route": "Route 1", "score": 0.8},
+ ]
+ # Override _pass_threshold to always return True for this test
+ route_layer._pass_threshold = lambda scores, threshold: True
+ expected = [("Route 1", 0.8), ("Route 2", 0.7)]
+ results = route_layer._semantic_classify_multiple_routes(query_results)
+ assert sorted(results) == sorted(
+ expected
+ ), "Should classify and return routes above their thresholds"
+
+ def test_with_no_routes_passing_threshold(self, openai_encoder, routes, index_cls):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes, index=index_cls()
+ )
+ route_layer.score_threshold = 0.5
+ # Override _pass_threshold to always return False for this test
+ route_layer._pass_threshold = lambda scores, threshold: False
+ query_results = [
+ {"route": "Route 1", "score": 0.3},
+ {"route": "Route 2", "score": 0.2},
+ ]
+ expected = []
+ results = route_layer._semantic_classify_multiple_routes(query_results)
+ assert (
+ results == expected
+ ), "Should return an empty list when no routes pass their thresholds"
+
+ def test_with_no_query_results(self, openai_encoder, routes, index_cls):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes, index=index_cls()
+ )
+ route_layer.score_threshold = 0.5
+ query_results = []
+ expected = []
+ results = route_layer._semantic_classify_multiple_routes(query_results)
+ assert (
+ results == expected
+ ), "Should return an empty list when there are no query results"
+
+ def test_with_unrecognized_route(self, openai_encoder, routes, index_cls):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes, index=index_cls()
+ )
+ route_layer.score_threshold = 0.5
+ # Test with a route name that does not exist in the route_layer's routes
+ query_results = [{"route": "UnrecognizedRoute", "score": 0.9}]
+ expected = []
+ results = route_layer._semantic_classify_multiple_routes(query_results)
+ assert results == expected, "Should ignore and not return unrecognized routes"
+
+ def test_retrieve_with_text(self, openai_encoder, routes, index_cls):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes, index=index_cls()
+ )
+ text = "Hello"
+ results = route_layer.retrieve_multiple_routes(text=text)
+ assert len(results) >= 1, "Expected at least one result"
+ assert any(
+ result.name in ["Route 1", "Route 2"] for result in results
+ ), "Expected the result to be either 'Route 1' or 'Route 2'"
+
+ def test_retrieve_with_vector(self, openai_encoder, routes, index_cls):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes, index=index_cls()
+ )
+ vector = [0.1, 0.2, 0.3]
+ results = route_layer.retrieve_multiple_routes(vector=vector)
+ assert len(results) >= 1, "Expected at least one result"
+ assert any(
+ result.name in ["Route 1", "Route 2"] for result in results
+ ), "Expected the result to be either 'Route 1' or 'Route 2'"
+
+ def test_retrieve_without_text_or_vector(self, openai_encoder, routes, index_cls):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes, index=index_cls()
+ )
+ with pytest.raises(ValueError, match="Either text or vector must be provided"):
+ route_layer.retrieve_multiple_routes()
+
+ def test_retrieve_no_matches(self, openai_encoder, routes, index_cls):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes, index=index_cls()
+ )
+ text = "Asparagus"
+ results = route_layer.retrieve_multiple_routes(text=text)
+ assert len(results) == 0, f"Expected no results, but got {len(results)}"
+
+ def test_retrieve_one_match(self, openai_encoder, routes_3, index_cls):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes_3, index=index_cls()
+ )
+ text = "Hello"
+ results = route_layer.retrieve_multiple_routes(text=text)
+ assert len(results) == 1, f"Expected one result, and got {len(results)}"
+ matched_routes = [result.name for result in results]
+ assert "Route 1" in matched_routes, "Expected 'Route 1' to be a match"
+
+ def test_retrieve_with_text_for_multiple_matches(
+ self, openai_encoder, routes_2, index_cls
+ ):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes_2, index=index_cls()
+ )
+ text = "Hello"
+ results = route_layer.retrieve_multiple_routes(text=text)
+ assert len(results) == 2, "Expected two results"
+ matched_routes = [result.name for result in results]
+ assert "Route 1" in matched_routes, "Expected 'Route 1' to be a match"
+ assert "Route 2" in matched_routes, "Expected 'Route 2' to be a match"
+
+ def test_set_aggregation_method_with_unsupported_value(
+ self, openai_encoder, routes, index_cls
+ ):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes, index=index_cls()
+ )
+ unsupported_aggregation = "unsupported_aggregation_method"
+ with pytest.raises(
+ ValueError,
+ match=f"Unsupported aggregation method chosen: {unsupported_aggregation}. Choose either 'SUM', 'MEAN', or 'MAX'.",
+ ):
+ route_layer._set_aggregation_method(unsupported_aggregation)
+
+ def test_refresh_routes_not_implemented(self, openai_encoder, routes, index_cls):
+ route_layer = RouteLayer(
+ encoder=openai_encoder, routes=routes, index=index_cls()
+ )
+ with pytest.raises(
+ NotImplementedError, match="This method has not yet been implemented."
+ ):
+ route_layer._refresh_routes()
+
class TestLayerFit:
def test_eval(self, openai_encoder, routes, test_data):