{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from decision_layer.schema import Decision\n",
"\n",
"politics = Decision(\n",
" name=\"politics\",\n",
" utterances=[\n",
" \"Who is the current Prime Minister of the UK?\",\n",
" \"What are the main political parties in Germany?\",\n",
" \"What is the role of the United Nations?\",\n",
" \"Tell me about the political system in China.\",\n",
" \"What is the political history of South Africa?\",\n",
" \"Who is the President of Russia and what is his political ideology?\",\n",
" \"What is the impact of politics on climate change?\",\n",
" \"How does the political system work in India?\",\n",
" \"What are the major political events happening in the Middle East?\",\n",
" \"What is the political structure of the European Union?\",\n",
" \"Who are the key political leaders in Australia?\",\n",
" \"What are the political implications of the recent protests in Hong Kong?\",\n",
" \"Can you explain the political crisis in Venezuela?\",\n",
" \"What is the political significance of the G7 summit?\",\n",
" \"Who are the current political leaders in the African Union?\"\n",
" ]\n",
")\n",
"\n",
"other_brands = Decision(\n",
" name=\"other_brands\",\n",
" utterances=[\n",
" \"How can I create a Google account?\",\n",
" \"What are the features of the new iPhone?\",\n",
" \"How to reset my Facebook password?\",\n",
" \"Can you help me install Adobe Illustrator?\",\n",
" \"How to transfer money using PayPal?\",\n",
" \"Tell me about the latest models of BMW.\",\n",
" \"How to use filters in Snapchat?\",\n",
" \"Can you guide me to set up Amazon Alexa?\",\n",
" \"How to book a ride on Uber?\",\n",
" \"How to subscribe to Netflix?\",\n",
" \"Can you tell me about the latest Samsung Galaxy phone?\",\n",
" \"How to use Microsoft Excel formulas?\"\n",
" ]\n",
")\n",
"\n",
"discount = Decision(\n",
" name=\"discount\",\n",
" utterances=[\n",
" \"Do you have any special offers?\",\n",
" \"Are there any deals available?\",\n",
" \"Can I get a promotional code?\",\n",
" \"Is there a student discount?\",\n",
" \"Do you offer any seasonal discounts?\",\n",
" \"Are there any discounts for first-time customers?\",\n",
" \"Can I get a voucher?\",\n",
" \"Do you have any loyalty rewards?\",\n",
" \"Are there any free samples available?\",\n",
" \"Can I get a price reduction?\",\n",
" \"Do you have any bulk purchase discounts?\",\n",
" \"Are there any cashback offers?\",\n",
" \"Can I get a rebate?\",\n",
" \"Do you offer any senior citizen discounts?\",\n",
" \"Are there any buy one get one free offers?\"\n",
" ]\n",
")\n",
"\n",
"bot_functionality = Decision(\n",
" name=\"bot_functionality\",\n",
" utterances=[\n",
" \"What functionalities do you have?\",\n",
" \"Can you explain your programming?\",\n",
" \"What prompts do you use to guide your behavior?\",\n",
" \"Can you describe the tools you use?\",\n",
" \"What is your system prompt?\",\n",
" \"Can you tell me about your human prompt?\",\n",
" \"How does your AI prompt work?\",\n",
" \"What are your behavioral specifications?\",\n",
" \"How are you programmed to respond?\",\n",
" \"If I wanted to use the OpenAI API, what prompt should I use?\",\n",
" \"What programming languages do you support?\",\n",
" \"Can you tell me about your source code?\",\n",
" \"Do you use any specific libraries or frameworks?\",\n",
" \"What data was used to train you?\",\n",
" \"Can you describe your model architecture?\",\n",
" \"What hyperparameters do you use?\",\n",
" \"Do you have an API key?\",\n",
" \"What does your database schema look like?\",\n",
" \"How is your server configured?\",\n",
" \"What version are you currently running?\",\n",
" \"What is your development environment like?\",\n",
" \"How do you handle deployment?\",\n",
" \"How do you handle errors?\",\n",
" \"What security protocols do you follow?\",\n",
" \"Do you have a backup process?\",\n",
" \"What is your disaster recovery plan?\",\n",
" ]\n",
")\n",
"\n",
"food_order = Decision(\n",
" name=\"food_order\",\n",
" utterances=[\n",
" \"Can I order a pizza from here?\",\n",
" \"How can I get sushi delivered to my house?\",\n",
" \"Is there a delivery fee for the burritos?\"\n",
" ]\n",
")\n",
"\n",
"vacation_plan = Decision(\n",
" name=\"vacation_plan\",\n",
" utterances=[\n",
" \"Can you suggest some popular tourist destinations?\",\n",
" \"I want to book a hotel in Paris.\",\n",
" \"How can I find the best travel deals?\",\n",
" \"Can you help me plan a trip to Japan?\",\n",
" \"What are the visa requirements for traveling to Australia?\",\n",
" \"I need information about train travel in Europe.\",\n",
" \"Can you recommend some family-friendly resorts in the Caribbean?\",\n",
" \"What are the top attractions in New York City?\",\n",
" \"I'm looking for a budget trip to Thailand.\",\n",
" \"Can you suggest a travel itinerary for a week in Italy?\"\n",
" ]\n",
")\n",
"\n",
"mathematics = Decision(\n",
" name=\"mathematics\",\n",
" utterances=[\n",
" \"What is the Pythagorean theorem?\",\n",
" \"Can you explain the concept of derivatives?\",\n",
" \"What is the difference between mean, median, and mode?\",\n",
" \"How do I solve quadratic equations?\",\n",
" \"What is the concept of limits in calculus?\"\n",
" ]\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"from decision_layer.encoders import OpenAIEncoder\n",
"import os\n",
"\n",
"encoder = OpenAIEncoder(name=\"text-embedding-ada-002\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"ename": "APIError",
"evalue": "Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:48:30 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e45f93ceeb482-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mAPIError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20231106 Semantic Layer\\Repo\\semantic-layer\\00_performance_tests.ipynb Cell 3\u001b[0m line \u001b[0;36m1\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mdecision_layer\u001b[39;00m \u001b[39mimport\u001b[39;00m DecisionLayer\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m decisions \u001b[39m=\u001b[39m [\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m politics,\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m other_brands,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m mathematics,\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m ]\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=12'>13</a>\u001b[0m dl \u001b[39m=\u001b[39m DecisionLayer(encoder\u001b[39m=\u001b[39;49mencoder, decisions\u001b[39m=\u001b[39;49mdecisions)\n",
"File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20231106 Semantic Layer\\Repo\\semantic-layer\\decision_layer\\decision_layer.py:16\u001b[0m, in \u001b[0;36mDecisionLayer.__init__\u001b[1;34m(self, encoder, decisions)\u001b[0m\n\u001b[0;32m 13\u001b[0m \u001b[39mif\u001b[39;00m decisions:\n\u001b[0;32m 14\u001b[0m \u001b[39m# initialize index now\u001b[39;00m\n\u001b[0;32m 15\u001b[0m \u001b[39mfor\u001b[39;00m decision \u001b[39min\u001b[39;00m decisions:\n\u001b[1;32m---> 16\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_add_decision(decision\u001b[39m=\u001b[39;49mdecision)\n",
"File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20231106 Semantic Layer\\Repo\\semantic-layer\\decision_layer\\decision_layer.py:29\u001b[0m, in \u001b[0;36mDecisionLayer._add_decision\u001b[1;34m(self, decision)\u001b[0m\n\u001b[0;32m 27\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_add_decision\u001b[39m(\u001b[39mself\u001b[39m, decision: Decision):\n\u001b[0;32m 28\u001b[0m \u001b[39m# create embeddings\u001b[39;00m\n\u001b[1;32m---> 29\u001b[0m embeds \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mencoder(decision\u001b[39m.\u001b[39;49mutterances)\n\u001b[0;32m 31\u001b[0m \u001b[39m# create decision array\u001b[39;00m\n\u001b[0;32m 32\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcategories \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
"File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20231106 Semantic Layer\\Repo\\semantic-layer\\decision_layer\\encoders\\openai.py:24\u001b[0m, in \u001b[0;36mOpenAIEncoder.__call__\u001b[1;34m(self, texts)\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[39mfor\u001b[39;00m j \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m5\u001b[39m):\n\u001b[0;32m 22\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 23\u001b[0m \u001b[39m# create embeddings\u001b[39;00m\n\u001b[1;32m---> 24\u001b[0m res \u001b[39m=\u001b[39m openai\u001b[39m.\u001b[39;49mEmbedding\u001b[39m.\u001b[39;49mcreate(\n\u001b[0;32m 25\u001b[0m \u001b[39minput\u001b[39;49m\u001b[39m=\u001b[39;49mtexts, engine\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mname\n\u001b[0;32m 26\u001b[0m )\n\u001b[0;32m 27\u001b[0m passed \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m 28\u001b[0m \u001b[39mexcept\u001b[39;00m openai\u001b[39m.\u001b[39merror\u001b[39m.\u001b[39mRateLimitError:\n",
"File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\openai\\api_resources\\embedding.py:33\u001b[0m, in \u001b[0;36mEmbedding.create\u001b[1;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[0;32m 31\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m 32\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> 33\u001b[0m response \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49mcreate(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[0;32m 35\u001b[0m \u001b[39m# If a user specifies base64, we'll just return the encoded string.\u001b[39;00m\n\u001b[0;32m 36\u001b[0m \u001b[39m# This is only for the default case.\u001b[39;00m\n\u001b[0;32m 37\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m user_provided_encoding_format:\n",
"File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\openai\\api_resources\\abstract\\engine_api_resource.py:155\u001b[0m, in \u001b[0;36mEngineAPIResource.create\u001b[1;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001b[0m\n\u001b[0;32m 129\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[0;32m 130\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcreate\u001b[39m(\n\u001b[0;32m 131\u001b[0m \u001b[39mcls\u001b[39m,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 138\u001b[0m \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mparams,\n\u001b[0;32m 139\u001b[0m ):\n\u001b[0;32m 140\u001b[0m (\n\u001b[0;32m 141\u001b[0m deployment_id,\n\u001b[0;32m 142\u001b[0m engine,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 152\u001b[0m api_key, api_base, api_type, api_version, organization, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mparams\n\u001b[0;32m 153\u001b[0m )\n\u001b[1;32m--> 155\u001b[0m response, _, api_key \u001b[39m=\u001b[39m requestor\u001b[39m.\u001b[39;49mrequest(\n\u001b[0;32m 156\u001b[0m \u001b[39m\"\u001b[39;49m\u001b[39mpost\u001b[39;49m\u001b[39m\"\u001b[39;49m,\n\u001b[0;32m 157\u001b[0m url,\n\u001b[0;32m 158\u001b[0m params\u001b[39m=\u001b[39;49mparams,\n\u001b[0;32m 159\u001b[0m headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[0;32m 160\u001b[0m stream\u001b[39m=\u001b[39;49mstream,\n\u001b[0;32m 161\u001b[0m request_id\u001b[39m=\u001b[39;49mrequest_id,\n\u001b[0;32m 162\u001b[0m request_timeout\u001b[39m=\u001b[39;49mrequest_timeout,\n\u001b[0;32m 163\u001b[0m )\n\u001b[0;32m 165\u001b[0m \u001b[39mif\u001b[39;00m stream:\n\u001b[0;32m 166\u001b[0m \u001b[39m# must be an iterator\u001b[39;00m\n\u001b[0;32m 167\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(response, OpenAIResponse)\n",
"File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\openai\\api_requestor.py:299\u001b[0m, in \u001b[0;36mAPIRequestor.request\u001b[1;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001b[0m\n\u001b[0;32m 278\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrequest\u001b[39m(\n\u001b[0;32m 279\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[0;32m 280\u001b[0m method,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 287\u001b[0m request_timeout: Optional[Union[\u001b[39mfloat\u001b[39m, Tuple[\u001b[39mfloat\u001b[39m, \u001b[39mfloat\u001b[39m]]] \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m,\n\u001b[0;32m 288\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001b[39mbool\u001b[39m, \u001b[39mstr\u001b[39m]:\n\u001b[0;32m 289\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrequest_raw(\n\u001b[0;32m 290\u001b[0m method\u001b[39m.\u001b[39mlower(),\n\u001b[0;32m 291\u001b[0m url,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 297\u001b[0m request_timeout\u001b[39m=\u001b[39mrequest_timeout,\n\u001b[0;32m 298\u001b[0m )\n\u001b[1;32m--> 299\u001b[0m resp, got_stream \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_interpret_response(result, stream)\n\u001b[0;32m 300\u001b[0m \u001b[39mreturn\u001b[39;00m resp, got_stream, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mapi_key\n",
"File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\openai\\api_requestor.py:710\u001b[0m, in \u001b[0;36mAPIRequestor._interpret_response\u001b[1;34m(self, result, stream)\u001b[0m\n\u001b[0;32m 702\u001b[0m \u001b[39mreturn\u001b[39;00m (\n\u001b[0;32m 703\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_interpret_response_line(\n\u001b[0;32m 704\u001b[0m line, result\u001b[39m.\u001b[39mstatus_code, result\u001b[39m.\u001b[39mheaders, stream\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m\n\u001b[0;32m 705\u001b[0m )\n\u001b[0;32m 706\u001b[0m \u001b[39mfor\u001b[39;00m line \u001b[39min\u001b[39;00m parse_stream(result\u001b[39m.\u001b[39miter_lines())\n\u001b[0;32m 707\u001b[0m ), \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m 708\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 709\u001b[0m \u001b[39mreturn\u001b[39;00m (\n\u001b[1;32m--> 710\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_interpret_response_line(\n\u001b[0;32m 711\u001b[0m result\u001b[39m.\u001b[39;49mcontent\u001b[39m.\u001b[39;49mdecode(\u001b[39m\"\u001b[39;49m\u001b[39mutf-8\u001b[39;49m\u001b[39m\"\u001b[39;49m),\n\u001b[0;32m 712\u001b[0m result\u001b[39m.\u001b[39;49mstatus_code,\n\u001b[0;32m 713\u001b[0m result\u001b[39m.\u001b[39;49mheaders,\n\u001b[0;32m 714\u001b[0m stream\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m 715\u001b[0m ),\n\u001b[0;32m 716\u001b[0m \u001b[39mFalse\u001b[39;00m,\n\u001b[0;32m 717\u001b[0m )\n",
"File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\openai\\api_requestor.py:775\u001b[0m, in \u001b[0;36mAPIRequestor._interpret_response_line\u001b[1;34m(self, rbody, rcode, rheaders, stream)\u001b[0m\n\u001b[0;32m 773\u001b[0m stream_error \u001b[39m=\u001b[39m stream \u001b[39mand\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39merror\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m resp\u001b[39m.\u001b[39mdata\n\u001b[0;32m 774\u001b[0m \u001b[39mif\u001b[39;00m stream_error \u001b[39mor\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39m200\u001b[39m \u001b[39m<\u001b[39m\u001b[39m=\u001b[39m rcode \u001b[39m<\u001b[39m \u001b[39m300\u001b[39m:\n\u001b[1;32m--> 775\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhandle_error_response(\n\u001b[0;32m 776\u001b[0m rbody, rcode, resp\u001b[39m.\u001b[39mdata, rheaders, stream_error\u001b[39m=\u001b[39mstream_error\n\u001b[0;32m 777\u001b[0m )\n\u001b[0;32m 778\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n",
"\u001b[1;31mAPIError\u001b[0m: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:48:30 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e45f93ceeb482-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}"
]
}
],
"source": [
"from decision_layer import DecisionLayer\n",
"\n",
"decisions = [\n",
" politics,\n",
" other_brands,\n",
" discount,\n",
" bot_functionality,\n",
" food_order,\n",
" vacation_plan,\n",
" mathematics,\n",
"]\n",
"\n",
"dl = DecisionLayer(encoder=encoder, decisions=decisions)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"queries = [\n",
" (\"What is the political system in the UK?\", \"politics\"),\n",
" (\"i'm bored today\", \"NULL\"),\n",
" (\"how do I do 2+2\", \"mathematics\"),\n",
" (\"I want to order a pizza\", \"food_order\"),\n",
" (\"Identify the current Chancellor of Germany.\", \"politics\"),\n",
" (\"List the predominant political factions in France.\", \"politics\"),\n",
" (\"Describe the functions of the World Trade Organization in global politics.\", \"politics\"),\n",
" (\"Discuss the governance framework of the United States.\", \"politics\"),\n",
" (\"Outline the foreign policy evolution of India since its independence.\", \"politics\"),\n",
" (\"Who heads the government in Canada, and what are their political principles?\", \"politics\"),\n",
" (\"Analyze how political leadership influences environmental policy.\", \"politics\"),\n",
" (\"Detail the legislative process in the Brazilian government.\", \"politics\"),\n",
" (\"Summarize recent significant political developments in Northern Africa.\", \"politics\"),\n",
" (\"Explain the governance model of the Commonwealth of Independent States.\", \"politics\"),\n",
" (\"Highlight the pivotal government figures in Italy.\", \"politics\"),\n",
" (\"Assess the political aftermath of the economic reforms in Argentina.\", \"politics\"),\n",
" (\"Elucidate the ongoing political turmoil in Syria.\", \"politics\"),\n",
" (\"What is the geopolitical importance of NATO meetings?\", \"politics\"),\n",
" (\"Identify the political powerhouses within the Southeast Asian region.\", \"politics\"),\n",
" (\"Characterize the political arena in Mexico.\", \"politics\"),\n",
" (\"Discuss the political changes occurring in Egypt.\", \"politics\"),\n",
" (\"Guide me through the process of retrieving a lost Google account.\", \"other_brands\"),\n",
" (\"Can you compare the camera specifications between the new iPhone and its predecessor?\", \"other_brands\"),\n",
" (\"What's the latest method for securing my Facebook account with two-factor authentication?\", \"other_brands\"),\n",
" (\"Is there a way to get a free trial of Adobe Illustrator?\", \"other_brands\"),\n",
" (\"What are PayPal's fees for international currency transfer?\", \"other_brands\"),\n",
" (\"Discuss the fuel efficiency of the latest BMW series.\", \"other_brands\"),\n",
" (\"Explain how to create a custom geofilter for events on Snapchat.\", \"other_brands\"),\n",
" (\"Steps to troubleshoot Amazon Alexa when it's not responding?\", \"other_brands\"),\n",
" (\"What are the safety features provided by Uber during a ride?\", \"other_brands\"),\n",
" (\"Detail the differences between Netflix's basic and premium plans.\", \"other_brands\"),\n",
" (\"How does the battery life of the newest Samsung Galaxy compare to its competitors?\", \"other_brands\"),\n",
" (\"What are the new features in the latest update of Microsoft Excel?\", \"other_brands\"),\n",
" (\"Give me a rundown on using Gmail's confidential mode for sending sensitive information.\", \"other_brands\"),\n",
" (\"What's the best way to optimize my LinkedIn profile for job searches?\", \"other_brands\"),\n",
" (\"Does McDonald's offer any special discounts when ordering online?\", \"other_brands\"),\n",
" (\"What are the benefits of pre-ordering my drink through the Starbucks app?\", \"other_brands\"),\n",
" (\"Show me how to set virtual backgrounds in Zoom.\", \"other_brands\"),\n",
" (\"Describe the autopilot advancements in the new Tesla software update.\", \"other_brands\"),\n",
" (\"What are the video capabilities of Canon's newest DSLR camera?\", \"other_brands\"),\n",
" (\"How can I discover new music tailored to my tastes on Spotify?\", \"other_brands\"),\n",
" (\"What specials are currently on offer?\", \"discount\"),\n",
" (\"Any available deals I should know about?\", \"discount\"),\n",
" (\"How can I access a promo code?\", \"discount\"),\n",
" (\"Do you provide a discount for students?\", \"discount\"),\n",
" (\"Are seasonal price reductions available at the moment?\", \"discount\"),\n",
" (\"What are the benefits for a new customer?\", \"discount\"),\n",
" (\"Is it possible to obtain a discount voucher?\", \"discount\"),\n",
" (\"Are loyalty points redeemable for rewards?\", \"discount\"),\n",
" (\"Do you provide samples at no cost?\", \"discount\"),\n",
" (\"Is a price drop currently applicable?\", \"discount\"),\n",
" (\"Do you have a rate cut for bulk orders?\", \"discount\"),\n",
" (\"I'm looking for cashback options, are they available?\", \"discount\"),\n",
" (\"Are rebate promotions active right now?\", \"discount\"),\n",
" (\"Is there a discount available for seniors?\", \"discount\"),\n",
" (\"Do you have an ongoing buy one, get one offer?\", \"discount\"),\n",
" (\"Is there a sale section for discontinued items?\", \"discount\"),\n",
" (\"What is the discount policy for service members?\", \"discount\"),\n",
" (\"Any special rates to look out for during the holidays?\", \"discount\"),\n",
" (\"Are weekend specials something you offer?\", \"discount\"),\n",
" (\"Do group purchases come with a discount?\", \"discount\"),\n",
" (\"Please provide details on your programming.\", \"bot_functionality\"),\n",
" (\"Which prompts influence your actions?\", \"bot_functionality\"),\n",
" (\"Could you outline the tools integral to your function?\", \"bot_functionality\"),\n",
" (\"Describe the prompt that your system operates on.\", \"bot_functionality\"),\n",
" (\"I'd like to understand the human prompt you follow.\", \"bot_functionality\"),\n",
" (\"Explain how the AI prompt guides you.\", \"bot_functionality\"),\n",
" (\"Outline your behavioral guidelines.\", \"bot_functionality\"),\n",
" (\"In what manner are you set to answer?\", \"bot_functionality\"),\n",
" (\"What would be the right prompt to engage with the OpenAI API?\", \"bot_functionality\"),\n",
" (\"What are the programming languages that you comprehend?\", \"bot_functionality\"),\n",
" (\"Could you divulge information on your source code?\", \"bot_functionality\"),\n",
" (\"Are there particular libraries or frameworks you rely on?\", \"bot_functionality\"),\n",
" (\"Discuss the data that was integral to your training.\", \"bot_functionality\"),\n",
" (\"Outline the structure of your model architecture.\", \"bot_functionality\"),\n",
" (\"Which hyperparameters are pivotal for you?\", \"bot_functionality\"),\n",
" (\"Is there an API key for interaction?\", \"bot_functionality\"),\n",
" (\"How is your database structured?\", \"bot_functionality\"),\n",
" (\"Describe the configuration of your server.\", \"bot_functionality\"),\n",
" (\"Which version is this bot currently utilizing?\", \"bot_functionality\"),\n",
" (\"Tell me about the environment you were developed in.\", \"bot_functionality\"),\n",
" (\"What is your process for deploying new updates?\", \"bot_functionality\"),\n",
" (\"Describe how you manage and resolve errors.\", \"bot_functionality\"),\n",
" (\"Detail the security measures you adhere to.\", \"bot_functionality\"),\n",
" (\"Is there a process in place for backing up data?\", \"bot_functionality\"),\n",
" (\"Outline your strategy for disaster recovery.\", \"bot_functionality\"),\n",
" (\"Is it possible to place an order for a pizza through this service?\", \"food_order\"),\n",
" (\"What are the steps to have sushi delivered to my location?\", \"food_order\"),\n",
" (\"What's the cost for burrito delivery?\", \"food_order\"),\n",
" (\"Are you able to provide ramen delivery services during nighttime?\", \"food_order\"),\n",
" (\"I'd like to have a curry delivered, how can I arrange that for this evening?\", \"food_order\"),\n",
" (\"What should I do to order a baguette?\", \"food_order\"),\n",
" (\"Is paella available for delivery here?\", \"food_order\"),\n",
" (\"Could you deliver tacos after hours?\", \"food_order\"),\n",
" (\"What are the charges for delivering pasta?\", \"food_order\"),\n",
" (\"I'm looking to order a bento box, can I do that for my midday meal?\", \"food_order\"),\n",
" (\"Is there a service to have dim sum delivered?\", \"food_order\"),\n",
" (\"How can a kebab be delivered to my place?\", \"food_order\"),\n",
" (\"What's the process for ordering pho from this platform?\", \"food_order\"),\n",
" (\"At these hours, do you provide delivery for gyros?\", \"food_order\"),\n",
" (\"I'm interested in getting poutine delivered, how does that work?\", \"food_order\"),\n",
" (\"Could you inform me about the delivery charge for falafel?\", \"food_order\"),\n",
" (\"Does your delivery service operate after dark for items like bibimbap?\", \"food_order\"),\n",
" (\"How can I order a schnitzel to have for my midday meal?\", \"food_order\"),\n",
" (\"Is there an option for pad thai to be delivered through your service?\", \"food_order\"),\n",
" (\"How do I go about getting jerk chicken delivered here?\", \"food_order\"),\n",
" (\"Could you list some must-visit places for tourists?\", \"vacation_plan\"),\n",
" (\"I'm interested in securing accommodation in Paris.\", \"vacation_plan\"),\n",
" (\"Where do I look for the most advantageous travel deals?\", \"vacation_plan\"),\n",
" (\"Assist me with outlining a journey to Japan.\", \"vacation_plan\"),\n",
" (\"Detail the entry permit prerequisites for Australia.\", \"vacation_plan\"),\n",
" (\"Provide details on rail journeys within Europe.\", \"vacation_plan\"),\n",
" (\"Advise on some resorts in the Caribbean suitable for families.\", \"vacation_plan\"),\n",
" (\"Highlight the premier points of interest in New York City.\", \"vacation_plan\"),\n",
" (\"Guide me towards a cost-effective voyage to Thailand.\", \"vacation_plan\"),\n",
" (\"Draft a one-week travel plan for Italy, please.\", \"vacation_plan\"),\n",
" (\"Enlighten me on the ideal season for a Hawaiian vacation.\", \"vacation_plan\"),\n",
" (\"I'm in need of vehicle hire services in Los Angeles.\", \"vacation_plan\"),\n",
" (\"I'm searching for options for a sea voyage to the Bahamas.\", \"vacation_plan\"),\n",
" (\"Enumerate the landmarks one should not miss in London.\", \"vacation_plan\"),\n",
" (\"I am mapping out a continental hike through South America.\", \"vacation_plan\"),\n",
" (\"Point out some coastal retreats in Mexico.\", \"vacation_plan\"),\n",
" (\"I require booking a flight destined for Berlin.\", \"vacation_plan\"),\n",
" (\"Assistance required in locating a holiday home in Spain.\", \"vacation_plan\"),\n",
" (\"Searching for comprehensive package resorts in Turkey.\", \"vacation_plan\"),\n",
" (\"I'm interested in learning about India's cultural sights.\", \"vacation_plan\"),\n",
" (\"How are you today?\", \"NULL\"),\n",
" (\"What's your favorite color?\", \"NULL\"),\n",
" (\"Do you like music?\", \"NULL\"),\n",
" (\"Can you tell me a joke?\", \"NULL\"),\n",
" (\"What's your favorite movie?\", \"NULL\"),\n",
" (\"Do you have any pets?\", \"NULL\"),\n",
" (\"What's your favorite food?\", \"NULL\"),\n",
" (\"Do you like to read books?\", \"NULL\"),\n",
" (\"What's your favorite sport?\", \"NULL\"),\n",
" (\"Do you have any siblings?\", \"NULL\"),\n",
" (\"What's your favorite season?\", \"NULL\"),\n",
" (\"Do you like to travel?\", \"NULL\"),\n",
" (\"What's your favorite hobby?\", \"NULL\"),\n",
" (\"Do you like to cook?\", \"NULL\"),\n",
" (\"What's your favorite type of music?\", \"NULL\"),\n",
" (\"Do you like to dance?\", \"NULL\"),\n",
" (\"What's your favorite animal?\", \"NULL\"),\n",
" (\"Do you like to watch TV?\", \"NULL\"),\n",
" (\"What's your favorite type of cuisine?\", \"NULL\"),\n",
" (\"Do you like to play video games?\", \"NULL\"),\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"def max_threshold_test(threshold: float, scores: list):\n",
" return max(scores) > threshold\n",
"\n",
"\n",
"def mean_threshold_test(threshold: float, scores: list):\n",
" return mean(scores) > threshold"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
" from .autonotebook import tqdm as notebook_tqdm\n"
]
},
{
"ename": "NameError",
"evalue": "name 'queries' is not defined",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20231106 Semantic Layer\\Repo\\semantic-layer\\00_performance_tests.ipynb Cell 6\u001b[0m line \u001b[0;36m8\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m thresholds \u001b[39m=\u001b[39m [\u001b[39m0.5\u001b[39m, \u001b[39m0.55\u001b[39m, \u001b[39m0.6\u001b[39m, \u001b[39m0.65\u001b[39m, \u001b[39m0.7\u001b[39m, \u001b[39m0.75\u001b[39m, \u001b[39m0.8\u001b[39m, \u001b[39m0.85\u001b[39m, \u001b[39m0.9\u001b[39m, \u001b[39m0.95\u001b[39m]\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m threshold_method \u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mmax\u001b[39m\u001b[39m'\u001b[39m \u001b[39m# 'mean', 'max'\u001b[39;00m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m \u001b[39mfor\u001b[39;00m q, expected \u001b[39min\u001b[39;00m tqdm(queries):\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m \n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39m# Attempt Query 3 Times.\u001b[39;00m\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m out \u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mUNDEFINED_CLASS\u001b[39m\u001b[39m'\u001b[39m \u001b[39m# Initialize actual_decision here\u001b[39;00m\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m all_attempts_failed \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m \u001b[39m# Initialize flag here\u001b[39;00m\n",
"\u001b[1;31mNameError\u001b[0m: name 'queries' is not defined"
]
}
],
"source": [
"from tqdm.auto import tqdm\n",
"import time\n",
"\n",
"results = {}\n",
"thresholds = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]\n",
"threshold_method = 'max' # 'mean', 'max'\n",
"\n",
"for q, expected in tqdm(queries):\n",
"\n",
" # Attempt Query 3 Times.\n",
" out = 'UNDEFINED_CLASS' # Initialize actual_decision here\n",
" all_attempts_failed = True # Initialize flag here\n",
" for i in range(3):\n",
" try:\n",
" start_time = time.time() # Start timer\n",
" out = dl._query(q, top_k=5)\n",
" end_time = time.time() # End timer\n",
" all_attempts_failed = False # If we reach this line, the attempt was successful\n",
" break\n",
" except Exception as e:\n",
" print(f\"\\t\\t\\tAttempt {i+1} failed with error: {str(e)}\")\n",
" if i < 2: # Don't sleep after the last attempt\n",
" time.sleep(5)\n",
" if all_attempts_failed:\n",
" print(\"\\t\\t\\tAll attempts failed. Skipping this utterance.\")\n",
" continue # Skip to the next utterance\n",
" \n",
" # Determine Top Class and the Cosine-Similarity Scores of Vectors that Contributed to Top Class score.\n",
" top_class, top_class_scores = dl._semantic_classify(query_results=out)\n",
"\n",
" # test if the top score is above the threshold for range of thresholds\n",
" for threshold in thresholds:\n",
" if threshold not in results:\n",
" results[threshold] = []\n",
" if threshold_method == 'mean':\n",
" class_pass = mean_threshold_test(threshold, top_class_scores)\n",
" elif threshold_method == 'max':\n",
" class_pass = max_threshold_test(threshold, top_class_scores)\n",
" if class_pass:\n",
" pass\n",
" else:\n",
" top_class = \"NULL\"\n",
" correct = top_class == expected\n",
" results[threshold].append(correct)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Threshold: 0.5, Accuracy: 0.8287671232876712\n",
"Threshold: 0.55, Accuracy: 0.8287671232876712\n",
"Threshold: 0.6, Accuracy: 0.8287671232876712\n",
"Threshold: 0.65, Accuracy: 0.8287671232876712\n",
"Threshold: 0.7, Accuracy: 0.8287671232876712\n",
"Threshold: 0.75, Accuracy: 0.8287671232876712\n",
"Threshold: 0.8, Accuracy: 0.8287671232876712\n",
"Threshold: 0.85, Accuracy: 0.8287671232876712\n",
"Threshold: 0.9, Accuracy: 0.8287671232876712\n",
"Threshold: 0.95, Accuracy: 0.8287671232876712\n"
]
}
],
"source": [
"for k, v in results.items():\n",
" print(f\"Threshold: {k}, Accuracy: {sum(v) / len(v)}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "decision-layer",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}