Merge pull request #156 from CP500/ollama

docs/07-ollama-local-execution.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/aurelio-labs/semantic-router/blob/main/docs/07-ollama-local-execution.ipynb) [![Open nbviewer](https://raw.githubusercontent.com/pinecone-io/examples/master/assets/nbviewer-shield.svg)](https://nbviewer.org/github/aurelio-labs/semantic-router/blob/main/docs/07-ollama-local-execution.ipynb)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Local Dynamic Routes - With Ollama"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Fully local Semantic Router with Ollama and HuggingFace Encoder\n",
+    "\n",
+    "There are many reasons users might choose to roll their own LLMs rather than use a third-party service. Whether it's due to cost, privacy or compliance, Semantic Router supports the use of \"local\" LLMs through `llama.cpp`.\n",
+    "\n",
+    "Below is an example of using semantic router which leverages Ollama in order to utilize the **OpenHermes** LLM. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We need `pillow`, `torch` and `transformers` for the HuggingFace encoders."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Installing the Library and Dependencies"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: semantic_router[local]==0.0.23 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (0.0.23)\n",
+      "Requirement already satisfied: pillow in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (10.2.0)\n",
+      "Requirement already satisfied: torch in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (2.2.0)\n",
+      "Requirement already satisfied: transformers in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (4.38.0)\n",
+      "Requirement already satisfied: black<24.0.0,>=23.12.1 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from semantic_router[local]==0.0.23) (23.12.1)\n",
+      "Requirement already satisfied: cohere<5.0,>=4.32 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from semantic_router[local]==0.0.23) (4.47)\n",
+      "Requirement already satisfied: colorama<0.5.0,>=0.4.6 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from semantic_router[local]==0.0.23) (0.4.6)\n",
+      "Requirement already satisfied: colorlog<7.0.0,>=6.8.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from semantic_router[local]==0.0.23) (6.8.2)\n",
+      "Requirement already satisfied: llama-cpp-python<0.3.0,>=0.2.28 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from semantic_router[local]==0.0.23) (0.2.45)\n",
+      "Requirement already satisfied: mistralai<0.0.13,>=0.0.12 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from semantic_router[local]==0.0.23) (0.0.12)\n",
+      "Requirement already satisfied: numpy<2.0.0,>=1.25.2 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from semantic_router[local]==0.0.23) (1.26.4)\n",
+      "Requirement already satisfied: openai<2.0.0,>=1.10.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from semantic_router[local]==0.0.23) (1.12.0)\n",
+      "Requirement already satisfied: pydantic<3.0.0,>=2.5.3 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from semantic_router[local]==0.0.23) (2.6.1)\n",
+      "Requirement already satisfied: pyyaml<7.0.0,>=6.0.1 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from semantic_router[local]==0.0.23) (6.0.1)\n",
+      "Requirement already satisfied: filelock in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from torch) (3.13.1)\n",
+      "Requirement already satisfied: typing-extensions>=4.8.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from torch) (4.9.0)\n",
+      "Requirement already satisfied: sympy in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from torch) (1.12)\n",
+      "Requirement already satisfied: networkx in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from torch) (3.2.1)\n",
+      "Requirement already satisfied: jinja2 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from torch) (3.1.3)\n",
+      "Requirement already satisfied: fsspec in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from torch) (2024.2.0)\n",
+      "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from transformers) (0.20.3)\n",
+      "Requirement already satisfied: packaging>=20.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from transformers) (23.2)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from transformers) (2023.12.25)\n",
+      "Requirement already satisfied: requests in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from transformers) (2.31.0)\n",
+      "Requirement already satisfied: tokenizers<0.19,>=0.14 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from transformers) (0.15.2)\n",
+      "Requirement already satisfied: safetensors>=0.4.1 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from transformers) (0.4.2)\n",
+      "Requirement already satisfied: tqdm>=4.27 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from transformers) (4.66.2)\n",
+      "Requirement already satisfied: click>=8.0.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from black<24.0.0,>=23.12.1->semantic_router[local]==0.0.23) (8.1.7)\n",
+      "Requirement already satisfied: mypy-extensions>=0.4.3 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from black<24.0.0,>=23.12.1->semantic_router[local]==0.0.23) (1.0.0)\n",
+      "Requirement already satisfied: pathspec>=0.9.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from black<24.0.0,>=23.12.1->semantic_router[local]==0.0.23) (0.12.1)\n",
+      "Requirement already satisfied: platformdirs>=2 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from black<24.0.0,>=23.12.1->semantic_router[local]==0.0.23) (4.2.0)\n",
+      "Requirement already satisfied: aiohttp<4.0,>=3.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (3.9.3)\n",
+      "Requirement already satisfied: backoff<3.0,>=2.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (2.2.1)\n",
+      "Requirement already satisfied: fastavro<2.0,>=1.8 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (1.9.4)\n",
+      "Requirement already satisfied: importlib_metadata<7.0,>=6.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (6.11.0)\n",
+      "Requirement already satisfied: urllib3<3,>=1.26 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (2.2.1)\n",
+      "Requirement already satisfied: diskcache>=5.6.1 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from llama-cpp-python<0.3.0,>=0.2.28->semantic_router[local]==0.0.23) (5.6.3)\n",
+      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from jinja2->torch) (2.1.5)\n",
+      "Requirement already satisfied: httpx<0.26.0,>=0.25.2 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from mistralai<0.0.13,>=0.0.12->semantic_router[local]==0.0.23) (0.25.2)\n",
+      "Requirement already satisfied: orjson<4.0.0,>=3.9.10 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from mistralai<0.0.13,>=0.0.12->semantic_router[local]==0.0.23) (3.9.14)\n",
+      "Requirement already satisfied: anyio<5,>=3.5.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from openai<2.0.0,>=1.10.0->semantic_router[local]==0.0.23) (4.2.0)\n",
+      "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from openai<2.0.0,>=1.10.0->semantic_router[local]==0.0.23) (1.9.0)\n",
+      "Requirement already satisfied: sniffio in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from openai<2.0.0,>=1.10.0->semantic_router[local]==0.0.23) (1.3.0)\n",
+      "Requirement already satisfied: annotated-types>=0.4.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from pydantic<3.0.0,>=2.5.3->semantic_router[local]==0.0.23) (0.6.0)\n",
+      "Requirement already satisfied: pydantic-core==2.16.2 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from pydantic<3.0.0,>=2.5.3->semantic_router[local]==0.0.23) (2.16.2)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from requests->transformers) (3.3.2)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from requests->transformers) (3.6)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from requests->transformers) (2024.2.2)\n",
+      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from sympy->torch) (1.3.0)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (1.3.1)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (23.2.0)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (1.4.1)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (6.0.5)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from aiohttp<4.0,>=3.0->cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (1.9.4)\n",
+      "Requirement already satisfied: httpcore==1.* in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from httpx<0.26.0,>=0.25.2->mistralai<0.0.13,>=0.0.12->semantic_router[local]==0.0.23) (1.0.3)\n",
+      "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from httpcore==1.*->httpx<0.26.0,>=0.25.2->mistralai<0.0.13,>=0.0.12->semantic_router[local]==0.0.23) (0.14.0)\n",
+      "Requirement already satisfied: zipp>=0.5 in c:\\users\\siraj\\documents\\personal\\work\\aurelio\\20240123 semantic router\\venvs\\semantic_router\\lib\\site-packages (from importlib_metadata<7.0,>=6.0->cohere<5.0,>=4.32->semantic_router[local]==0.0.23) (3.17.0)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "[notice] A new release of pip is available: 23.1.2 -> 24.0\n",
+      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install semantic_router[local]==0.0.23 \\\n",
+    "pillow torch transformers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "c:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20240123 Semantic Router\\venvs\\semantic_router\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from semantic_router.encoders import HuggingFaceEncoder\n",
+    "\n",
+    "encoder = HuggingFaceEncoder()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Define Static Routes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from semantic_router import Route\n",
+    "\n",
+    "# we could use this as a guide for our chatbot to avoid political conversations\n",
+    "politics = Route(\n",
+    "    name=\"politics\",\n",
+    "    utterances=[\n",
+    "        \"isn't politics the best thing ever\",\n",
+    "        \"why don't you tell me about your political opinions\",\n",
+    "        \"don't you just love the president\" \"don't you just hate the president\",\n",
+    "        \"they're going to destroy this country!\",\n",
+    "        \"they will save the country!\",\n",
+    "    ],\n",
+    ")\n",
+    "\n",
+    "# this could be used as an indicator to our chatbot to switch to a more\n",
+    "# conversational prompt\n",
+    "chitchat = Route(\n",
+    "    name=\"chitchat\",\n",
+    "    utterances=[\n",
+    "        \"how's the weather today?\",\n",
+    "        \"how are things going?\",\n",
+    "        \"lovely weather today\",\n",
+    "        \"the weather is horrendous\",\n",
+    "        \"let's go to the chippy\",\n",
+    "    ],\n",
+    ")\n",
+    "\n",
+    "# we place both of our decisions together into single list\n",
+    "routes = [politics, chitchat]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Define Route Layer with Ollama"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32m2024-02-22 10:59:54 INFO semantic_router.utils.logger local\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "from semantic_router.layer import RouteLayer\n",
+    "from semantic_router.llms.ollama import OllamaLLM\n",
+    "\n",
+    "\n",
+    "llm = OllamaLLM(llm_name=\"openhermes\") # Change llm_name if you want to use a different LLM with dynamic routes.\n",
+    "rl = RouteLayer(encoder = encoder, routes=routes, llm=llm)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test Static Routes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'politics'"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "rl(\"don't you love politics?\").name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'chitchat'"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "rl(\"how's the weather today?\").name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rl(\"I'm interested in learning about llama 2\").name"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Test Dynamic Routes\n",
+    "\n",
+    "Dynamic routes work by associating a function with a route. If the input utterance is similar enough to the utterances of the route, such that route is chosen by the semantic router, then this triggers a secondary process: \n",
+    "\n",
+    "The LLM we specified in the `RouteLayer` (we specified Ollama, which isn't strictly an LLM, but which defaults to using the `OpenHermes` LLM), is then usde to take a `function_schema`, and the input utterance, and extract values from the input utterance which can be used as arguments for `function` described by the the `funcion_schema`. The returned values can then be used in the `function` to obtain an output.\n",
+    "\n",
+    "So, in short, it's a way of generating `function` inputs from an utterance, if that utterance matches the route utterances closely enough.\n",
+    "\n",
+    "In the below example the utterance **\"what is the time in new york city?\"** is used to trigger the \"get_time\" route, which has the `function_schema` of a likewise named `get_time()` function associated with it. Then Ollama is used to run `OpenHermes` locally, which extracts the correctly formatted IANA timezone (`\"America/New York\"`), based on this utterance and information we provide it about the `function` in the `function_schema`. The returned stirng \"America/New York\" can then be used directly in the `get_time()` function to return the actual time in New York city."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datetime import datetime\n",
+    "from zoneinfo import ZoneInfo\n",
+    "\n",
+    "\n",
+    "def get_time(timezone: str) -> str:\n",
+    "    \"\"\"\n",
+    "Finds the current time in a specific timezone.\n",
+    "\n",
+    ":param timezone: The timezone to find the current time in, should\n",
+    "    be a valid timezone from the IANA Time Zone Database like\n",
+    "    \"America/New_York\" or \"Europe/London\". Do NOT put the place\n",
+    "    name itself like \"rome\", or \"new york\", you must provide\n",
+    "    the IANA format.\n",
+    ":type timezone: str\n",
+    ":return: The current time in the specified timezone.\n",
+    "    \"\"\"\n",
+    "    now = datetime.now(ZoneInfo(timezone))\n",
+    "    return now.strftime(\"%H:%M\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'01:59'"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "get_time(\"America/New_York\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'name': 'get_time',\n",
+       " 'description': 'Finds the current time in a specific timezone.\\n\\n:param timezone: The timezone to find the current time in, should\\n    be a valid timezone from the IANA Time Zone Database like\\n    \"America/New_York\" or \"Europe/London\". Do NOT put the place\\n    name itself like \"rome\", or \"new york\", you must provide\\n    the IANA format.\\n:type timezone: str\\n:return: The current time in the specified timezone.\\n    ',\n",
+       " 'signature': '(timezone: str) -> str',\n",
+       " 'output': \"<class 'str'>\"}"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from semantic_router.utils.function_call import get_schema\n",
+    "\n",
+    "schema = get_schema(get_time)\n",
+    "schema"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "time_route = Route(\n",
+    "    name=\"get_time\",\n",
+    "    utterances=[\n",
+    "        \"what is the time in new york city?\",\n",
+    "        \"what is the time in london?\",\n",
+    "        \"I live in Rome, what time is it?\",\n",
+    "    ],\n",
+    "    function_schema=schema,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32m2024-02-22 10:59:55 INFO semantic_router.utils.logger Adding `get_time` route\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "rl.add(time_route)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32m2024-02-22 11:01:29 INFO semantic_router.utils.logger Extracting function input...\u001b[0m\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32m2024-02-22 11:01:32 INFO semantic_router.utils.logger LLM output: {\n",
+      "    \"timezone\": \"America/New_York\"\n",
+      "}\u001b[0m\n",
+      "\u001b[32m2024-02-22 11:01:32 INFO semantic_router.utils.logger Function inputs: {'timezone': 'America/New_York'}\u001b[0m\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "name='get_time' function_call={'timezone': 'America/New_York'} similarity_score=None trigger=None\n"
+     ]
+    }
+   ],
+   "source": [
+    "out = rl(\"what is the time in new york city?\")\n",
+    "print(out)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'02:01'"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "\n",
+    "get_time(**out.function_call)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "semantic_splitter_1",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

semantic_router/llms/base.py

@@ -46,33 +46,55 @@ class BaseLLM(BaseModel):
         logger.info("Extracting function input...")
 
         prompt = f"""
-        You are a helpful assistant designed to output JSON.
-        Given the following function schema
-        << {function_schema} >>
-        and query
-        << {query} >>
-        extract the parameters values from the query, in a valid JSON format.
-        Example:
-        Input:
-        query: "How is the weather in Hawaii right now in International units?"
-        schema:
-        {{
-            "name": "get_weather",
-            "description": "Useful to get the weather in a specific location",
-            "signature": "(location: str, degree: str) -> str",
-            "output": "<class 'str'>",
-        }}
-
-        Result: {{
-            "location": "London",
-            "degree": "Celsius",
-        }}
-
-        Input:
-        query: {query}
-        schema: {function_schema}
-        Result:
-        """
+You are an accurate and reliable computer program that only outputs valid JSON. 
+Your task is to output JSON representing the input arguments of a Python function.
+
+This is the Python function's schema:
+
+### FUNCTION_SCHEMA Start ###
+	{function_schema}
+### FUNCTION_SCHEMA End ###
+
+This is the input query.
+
+### QUERY Start ###
+	{query}
+### QUERY End ###
+
+The arguments that you need to provide values for, together with their datatypes, are stated in "signature" in the FUNCTION_SCHEMA.
+The values these arguments must take are made clear by the QUERY.
+Use the FUNCTION_SCHEMA "description" too, as this might provide helpful clues about the arguments and their values.
+Return only JSON, stating the argument names and their corresponding values.
+
+### FORMATTING_INSTRUCTIONS Start ###
+	Return a respones in valid JSON format. Do not return any other explanation or text, just the JSON.
+	The JSON-Keys are the names of the arguments, and JSON-values are the values those arguments should take.
+### FORMATTING_INSTRUCTIONS End ###
+
+### EXAMPLE Start ###
+	=== EXAMPLE_INPUT_QUERY Start ===
+		"How is the weather in Hawaii right now in International units?"
+	=== EXAMPLE_INPUT_QUERY End ===
+	=== EXAMPLE_INPUT_SCHEMA Start ===
+		{{
+			"name": "get_weather",
+			"description": "Useful to get the weather in a specific location",
+			"signature": "(location: str, degree: str) -> str",
+			"output": "<class 'str'>",
+		}}
+	=== EXAMPLE_INPUT_QUERY End ===
+	=== EXAMPLE_OUTPUT Start ===
+		{{
+			"location": "Hawaii",
+			"degree": "Celsius",
+		}}
+	=== EXAMPLE_OUTPUT End ===
+### EXAMPLE End ###
+
+Note: I will tip $500 for and accurate JSON output. You will be penalized for an inaccurate JSON output.
+
+Provide JSON output now:
+"""
         llm_input = [Message(role="user", content=prompt)]
         output = self(llm_input)
 

semantic_router/llms/ollama.py

+from typing import List, Optional
+import requests
+
+from semantic_router.llms import BaseLLM
+from semantic_router.schema import Message
+from semantic_router.utils.logger import logger
+
+
+class OllamaLLM(BaseLLM):
+    temperature: Optional[float]
+    llm_name: Optional[str]
+    max_tokens: Optional[int]
+    stream: Optional[bool]
+
+    def __init__(
+        self,
+        name: str = "ollama",
+        temperature: float = 0.2,
+        llm_name: str = "openhermes",
+        max_tokens: Optional[int] = 200,
+        stream: bool = False,
+    ):
+        super().__init__(name=name)
+        self.temperature = temperature
+        self.llm_name = llm_name
+        self.max_tokens = max_tokens
+        self.stream = stream
+
+    def __call__(
+        self,
+        messages: List[Message],
+        temperature: Optional[float] = None,
+        llm_name: Optional[str] = None,
+        max_tokens: Optional[int] = None,
+        stream: Optional[bool] = None,
+    ) -> str:
+        # Use instance defaults if not overridden
+        temperature = temperature if temperature is not None else self.temperature
+        llm_name = llm_name if llm_name is not None else self.llm_name
+        max_tokens = max_tokens if max_tokens is not None else self.max_tokens
+        stream = stream if stream is not None else self.stream
+
+        try:
+            payload = {
+                "model": llm_name,
+                "messages": [m.to_openai() for m in messages],
+                "options": {"temperature": temperature, "num_predict": max_tokens},
+                "format": "json",
+                "stream": stream,
+            }
+            response = requests.post("http://localhost:11434/api/chat", json=payload)
+            output = response.json()["message"]["content"]
+
+            return output
+        except Exception as e:
+            logger.error(f"LLM error: {e}")
+            raise Exception(f"LLM error: {e}") from e

tests/unit/llms/test_llm_ollama.py

+import pytest
+from semantic_router.llms.ollama import OllamaLLM
+from semantic_router.schema import Message
+
+
+@pytest.fixture
+def ollama_llm():
+    return OllamaLLM()
+
+
+class TestOllamaLLM:
+    def test_ollama_llm_init_success(self, ollama_llm):
+        assert ollama_llm.name == "ollama"
+        assert ollama_llm.temperature == 0.2
+        assert ollama_llm.llm_name == "openhermes"
+        assert ollama_llm.max_tokens == 200
+        assert ollama_llm.stream is False
+
+    def test_ollama_llm_call_success(self, ollama_llm, mocker):
+        mock_response = mocker.MagicMock()
+        mock_response.json.return_value = {"message": {"content": "test response"}}
+        mocker.patch("requests.post", return_value=mock_response)
+
+        output = ollama_llm([Message(role="user", content="test")])
+        assert output == "test response"
+
+    def test_ollama_llm_error_handling(self, ollama_llm, mocker):
+        mocker.patch("requests.post", side_effect=Exception("LLM error"))
+        with pytest.raises(Exception) as exc_info:
+            ollama_llm([Message(role="user", content="test")])
+        assert "LLM error" in str(exc_info.value)