diff --git a/00_performance_tests.ipynb b/00_performance_tests.ipynb index 2ab1d1d53070d409d639306860dd36ec84f4301a..e59e88237543b8400499d476ba4917869b13f0fe 100644 --- a/00_performance_tests.ipynb +++ b/00_performance_tests.ipynb @@ -146,16 +146,34 @@ "from decision_layer.encoders import OpenAIEncoder\n", "import os\n", "\n", - "os.environ[\"OPENAI_API_KEY\"] = \"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\"\n", - "\n", "encoder = OpenAIEncoder(name=\"text-embedding-ada-002\")" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "APIError", + "evalue": "Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:48:30 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e45f93ceeb482-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mAPIError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20231106 Semantic Layer\\Repo\\semantic-layer\\00_performance_tests.ipynb Cell 3\u001b[0m line \u001b[0;36m1\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mdecision_layer\u001b[39;00m \u001b[39mimport\u001b[39;00m DecisionLayer\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m decisions \u001b[39m=\u001b[39m [\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=3'>4</a>\u001b[0m politics,\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m other_brands,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m mathematics,\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m ]\n\u001b[1;32m---> <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y135sZmlsZQ%3D%3D?line=12'>13</a>\u001b[0m dl \u001b[39m=\u001b[39m DecisionLayer(encoder\u001b[39m=\u001b[39;49mencoder, decisions\u001b[39m=\u001b[39;49mdecisions)\n", + "File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20231106 Semantic Layer\\Repo\\semantic-layer\\decision_layer\\decision_layer.py:16\u001b[0m, in \u001b[0;36mDecisionLayer.__init__\u001b[1;34m(self, encoder, decisions)\u001b[0m\n\u001b[0;32m 13\u001b[0m \u001b[39mif\u001b[39;00m decisions:\n\u001b[0;32m 14\u001b[0m \u001b[39m# initialize index now\u001b[39;00m\n\u001b[0;32m 15\u001b[0m \u001b[39mfor\u001b[39;00m decision \u001b[39min\u001b[39;00m decisions:\n\u001b[1;32m---> 16\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_add_decision(decision\u001b[39m=\u001b[39;49mdecision)\n", + "File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20231106 Semantic Layer\\Repo\\semantic-layer\\decision_layer\\decision_layer.py:29\u001b[0m, in \u001b[0;36mDecisionLayer._add_decision\u001b[1;34m(self, decision)\u001b[0m\n\u001b[0;32m 27\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_add_decision\u001b[39m(\u001b[39mself\u001b[39m, decision: Decision):\n\u001b[0;32m 28\u001b[0m \u001b[39m# create embeddings\u001b[39;00m\n\u001b[1;32m---> 29\u001b[0m embeds \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mencoder(decision\u001b[39m.\u001b[39;49mutterances)\n\u001b[0;32m 31\u001b[0m \u001b[39m# create decision array\u001b[39;00m\n\u001b[0;32m 32\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcategories \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n", + "File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20231106 Semantic Layer\\Repo\\semantic-layer\\decision_layer\\encoders\\openai.py:24\u001b[0m, in \u001b[0;36mOpenAIEncoder.__call__\u001b[1;34m(self, texts)\u001b[0m\n\u001b[0;32m 21\u001b[0m \u001b[39mfor\u001b[39;00m j \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m5\u001b[39m):\n\u001b[0;32m 22\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m 23\u001b[0m \u001b[39m# create embeddings\u001b[39;00m\n\u001b[1;32m---> 24\u001b[0m res \u001b[39m=\u001b[39m openai\u001b[39m.\u001b[39;49mEmbedding\u001b[39m.\u001b[39;49mcreate(\n\u001b[0;32m 25\u001b[0m \u001b[39minput\u001b[39;49m\u001b[39m=\u001b[39;49mtexts, engine\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mname\n\u001b[0;32m 26\u001b[0m )\n\u001b[0;32m 27\u001b[0m passed \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m 28\u001b[0m \u001b[39mexcept\u001b[39;00m openai\u001b[39m.\u001b[39merror\u001b[39m.\u001b[39mRateLimitError:\n", + "File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\openai\\api_resources\\embedding.py:33\u001b[0m, in \u001b[0;36mEmbedding.create\u001b[1;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[0;32m 31\u001b[0m \u001b[39mwhile\u001b[39;00m \u001b[39mTrue\u001b[39;00m:\n\u001b[0;32m 32\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m---> 33\u001b[0m response \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49mcreate(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[0;32m 35\u001b[0m \u001b[39m# If a user specifies base64, we'll just return the encoded string.\u001b[39;00m\n\u001b[0;32m 36\u001b[0m \u001b[39m# This is only for the default case.\u001b[39;00m\n\u001b[0;32m 37\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m user_provided_encoding_format:\n", + "File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\openai\\api_resources\\abstract\\engine_api_resource.py:155\u001b[0m, in \u001b[0;36mEngineAPIResource.create\u001b[1;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001b[0m\n\u001b[0;32m 129\u001b[0m \u001b[39m@classmethod\u001b[39m\n\u001b[0;32m 130\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcreate\u001b[39m(\n\u001b[0;32m 131\u001b[0m \u001b[39mcls\u001b[39m,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 138\u001b[0m \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mparams,\n\u001b[0;32m 139\u001b[0m ):\n\u001b[0;32m 140\u001b[0m (\n\u001b[0;32m 141\u001b[0m deployment_id,\n\u001b[0;32m 142\u001b[0m engine,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 152\u001b[0m api_key, api_base, api_type, api_version, organization, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mparams\n\u001b[0;32m 153\u001b[0m )\n\u001b[1;32m--> 155\u001b[0m response, _, api_key \u001b[39m=\u001b[39m requestor\u001b[39m.\u001b[39;49mrequest(\n\u001b[0;32m 156\u001b[0m \u001b[39m\"\u001b[39;49m\u001b[39mpost\u001b[39;49m\u001b[39m\"\u001b[39;49m,\n\u001b[0;32m 157\u001b[0m url,\n\u001b[0;32m 158\u001b[0m params\u001b[39m=\u001b[39;49mparams,\n\u001b[0;32m 159\u001b[0m headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[0;32m 160\u001b[0m stream\u001b[39m=\u001b[39;49mstream,\n\u001b[0;32m 161\u001b[0m request_id\u001b[39m=\u001b[39;49mrequest_id,\n\u001b[0;32m 162\u001b[0m request_timeout\u001b[39m=\u001b[39;49mrequest_timeout,\n\u001b[0;32m 163\u001b[0m )\n\u001b[0;32m 165\u001b[0m \u001b[39mif\u001b[39;00m stream:\n\u001b[0;32m 166\u001b[0m \u001b[39m# must be an iterator\u001b[39;00m\n\u001b[0;32m 167\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(response, OpenAIResponse)\n", + "File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\openai\\api_requestor.py:299\u001b[0m, in \u001b[0;36mAPIRequestor.request\u001b[1;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001b[0m\n\u001b[0;32m 278\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrequest\u001b[39m(\n\u001b[0;32m 279\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[0;32m 280\u001b[0m method,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 287\u001b[0m request_timeout: Optional[Union[\u001b[39mfloat\u001b[39m, Tuple[\u001b[39mfloat\u001b[39m, \u001b[39mfloat\u001b[39m]]] \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m,\n\u001b[0;32m 288\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001b[39mbool\u001b[39m, \u001b[39mstr\u001b[39m]:\n\u001b[0;32m 289\u001b[0m result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrequest_raw(\n\u001b[0;32m 290\u001b[0m method\u001b[39m.\u001b[39mlower(),\n\u001b[0;32m 291\u001b[0m url,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 297\u001b[0m request_timeout\u001b[39m=\u001b[39mrequest_timeout,\n\u001b[0;32m 298\u001b[0m )\n\u001b[1;32m--> 299\u001b[0m resp, got_stream \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_interpret_response(result, stream)\n\u001b[0;32m 300\u001b[0m \u001b[39mreturn\u001b[39;00m resp, got_stream, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mapi_key\n", + "File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\openai\\api_requestor.py:710\u001b[0m, in \u001b[0;36mAPIRequestor._interpret_response\u001b[1;34m(self, result, stream)\u001b[0m\n\u001b[0;32m 702\u001b[0m \u001b[39mreturn\u001b[39;00m (\n\u001b[0;32m 703\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_interpret_response_line(\n\u001b[0;32m 704\u001b[0m line, result\u001b[39m.\u001b[39mstatus_code, result\u001b[39m.\u001b[39mheaders, stream\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m\n\u001b[0;32m 705\u001b[0m )\n\u001b[0;32m 706\u001b[0m \u001b[39mfor\u001b[39;00m line \u001b[39min\u001b[39;00m parse_stream(result\u001b[39m.\u001b[39miter_lines())\n\u001b[0;32m 707\u001b[0m ), \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m 708\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m 709\u001b[0m \u001b[39mreturn\u001b[39;00m (\n\u001b[1;32m--> 710\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_interpret_response_line(\n\u001b[0;32m 711\u001b[0m result\u001b[39m.\u001b[39;49mcontent\u001b[39m.\u001b[39;49mdecode(\u001b[39m\"\u001b[39;49m\u001b[39mutf-8\u001b[39;49m\u001b[39m\"\u001b[39;49m),\n\u001b[0;32m 712\u001b[0m result\u001b[39m.\u001b[39;49mstatus_code,\n\u001b[0;32m 713\u001b[0m result\u001b[39m.\u001b[39;49mheaders,\n\u001b[0;32m 714\u001b[0m stream\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[0;32m 715\u001b[0m ),\n\u001b[0;32m 716\u001b[0m \u001b[39mFalse\u001b[39;00m,\n\u001b[0;32m 717\u001b[0m )\n", + "File \u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\openai\\api_requestor.py:775\u001b[0m, in \u001b[0;36mAPIRequestor._interpret_response_line\u001b[1;34m(self, rbody, rcode, rheaders, stream)\u001b[0m\n\u001b[0;32m 773\u001b[0m stream_error \u001b[39m=\u001b[39m stream \u001b[39mand\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39merror\u001b[39m\u001b[39m\"\u001b[39m \u001b[39min\u001b[39;00m resp\u001b[39m.\u001b[39mdata\n\u001b[0;32m 774\u001b[0m \u001b[39mif\u001b[39;00m stream_error \u001b[39mor\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39m200\u001b[39m \u001b[39m<\u001b[39m\u001b[39m=\u001b[39m rcode \u001b[39m<\u001b[39m \u001b[39m300\u001b[39m:\n\u001b[1;32m--> 775\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhandle_error_response(\n\u001b[0;32m 776\u001b[0m rbody, rcode, resp\u001b[39m.\u001b[39mdata, rheaders, stream_error\u001b[39m=\u001b[39mstream_error\n\u001b[0;32m 777\u001b[0m )\n\u001b[0;32m 778\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n", + "\u001b[1;31mAPIError\u001b[0m: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:48:30 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e45f93ceeb482-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}" + ] + } + ], "source": [ "from decision_layer import DecisionLayer\n", "\n", @@ -174,7 +192,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -344,159 +362,26 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - " 7%|▋ | 10/146 [00:17<03:51, 1.70s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\tAttempt 1 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:40:02 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e39a9ddbd124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n", - "\t\t\tAttempt 2 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:40:12 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e39e0ed87124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 16%|█▋ | 24/146 [01:02<03:40, 1.81s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\tAttempt 1 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:40:49 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3ac0bbcc124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 21%|██ | 31/146 [01:24<03:55, 2.05s/it]" + "c:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\Virtual Environments\\semantic_layer\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\tAttempt 1 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:41:07 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3b4b5c17124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 22%|██■| 32/146 [01:33<07:36, 4.00s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\tAttempt 1 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:41:17 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3b854b93124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n", - "\t\t\tAttempt 2 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:41:25 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3bb8d9ec124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 23%|██▎ | 33/146 [01:51<15:29, 8.22s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\tAttempt 3 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:41:32 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3be47f18124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n", - "\t\t\tAll attempts failed. Skipping this utterance.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 23%|██▎ | 34/146 [01:52<11:41, 6.26s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\tAttempt 1 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:41:36 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3bf9ffe7124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 25%|██■| 36/146 [02:04<10:19, 5.63s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\tAttempt 1 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:41:51 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3c412820124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 25%|██▌ | 37/146 [02:16<14:05, 7.76s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\tAttempt 1 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:41:59 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3c940824124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n", - "\t\t\tAttempt 2 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:42:11 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3cbf0ce2124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 26%|██▌ | 38/146 [02:37<20:56, 11.63s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\tAttempt 3 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:42:18 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3d0c6db5124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n", - "\t\t\tAll attempts failed. Skipping this utterance.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 27%|██▋ | 40/146 [02:42<12:07, 6.86s/it]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\t\t\tAttempt 1 failed with error: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Wed, 08 Nov 2023 13:42:31 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '822e3d326bcc124f-DXB', 'alt-svc': 'h3=\":443\"; ma=86400'}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 32%|███■| 46/146 [03:07<05:21, 3.21s/it]" + "ename": "NameError", + "evalue": "name 'queries' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32mc:\\Users\\Siraj\\Documents\\Personal\\Work\\Aurelio\\20231106 Semantic Layer\\Repo\\semantic-layer\\00_performance_tests.ipynb Cell 6\u001b[0m line \u001b[0;36m8\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=4'>5</a>\u001b[0m thresholds \u001b[39m=\u001b[39m [\u001b[39m0.5\u001b[39m, \u001b[39m0.55\u001b[39m, \u001b[39m0.6\u001b[39m, \u001b[39m0.65\u001b[39m, \u001b[39m0.7\u001b[39m, \u001b[39m0.75\u001b[39m, \u001b[39m0.8\u001b[39m, \u001b[39m0.85\u001b[39m, \u001b[39m0.9\u001b[39m, \u001b[39m0.95\u001b[39m]\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m threshold_method \u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mmax\u001b[39m\u001b[39m'\u001b[39m \u001b[39m# 'mean', 'max'\u001b[39;00m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m \u001b[39mfor\u001b[39;00m q, expected \u001b[39min\u001b[39;00m tqdm(queries):\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m \n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39m# Attempt Query 3 Times.\u001b[39;00m\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m out \u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mUNDEFINED_CLASS\u001b[39m\u001b[39m'\u001b[39m \u001b[39m# Initialize actual_decision here\u001b[39;00m\n\u001b[0;32m <a href='vscode-notebook-cell:/c%3A/Users/Siraj/Documents/Personal/Work/Aurelio/20231106%20Semantic%20Layer/Repo/semantic-layer/00_performance_tests.ipynb#Y141sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m all_attempts_failed \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m \u001b[39m# Initialize flag here\u001b[39;00m\n", + "\u001b[1;31mNameError\u001b[0m: name 'queries' is not defined" ] } ],