From a8270082a006f78738b05ab88d7383a7131e331d Mon Sep 17 00:00:00 2001
From: Marcus Schiesser <mail@marcusschiesser.de>
Date: Wed, 10 Jan 2024 10:55:13 +0700
Subject: [PATCH] fix: improve async handling in fastapi (#322)
---
.../templates/types/simple/fastapi/app/api/routers/chat.py | 2 +-
.../templates/types/streaming/fastapi/app/api/routers/chat.py | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/packages/create-llama/templates/types/simple/fastapi/app/api/routers/chat.py b/packages/create-llama/templates/types/simple/fastapi/app/api/routers/chat.py
index f6a246386..d4b000fd8 100644
--- a/packages/create-llama/templates/types/simple/fastapi/app/api/routers/chat.py
+++ b/packages/create-llama/templates/types/simple/fastapi/app/api/routers/chat.py
@@ -50,7 +50,7 @@ async def chat(
]
# query chat engine
- response = chat_engine.chat(lastMessage.content, messages)
+ response = await chat_engine.achat(lastMessage.content, messages)
return _Result(
result=_Message(role=MessageRole.ASSISTANT, content=response.response)
)
diff --git a/packages/create-llama/templates/types/streaming/fastapi/app/api/routers/chat.py b/packages/create-llama/templates/types/streaming/fastapi/app/api/routers/chat.py
index 9dd9eb6ed..26fd480de 100644
--- a/packages/create-llama/templates/types/streaming/fastapi/app/api/routers/chat.py
+++ b/packages/create-llama/templates/types/streaming/fastapi/app/api/routers/chat.py
@@ -49,11 +49,11 @@ async def chat(
]
# query chat engine
- response = chat_engine.stream_chat(lastMessage.content, messages)
+ response = await chat_engine.astream_chat(lastMessage.content, messages)
# stream response
async def event_generator():
- for token in response.response_gen:
+ async for token in response.async_response_gen():
# If client closes connection, stop sending events
if await request.is_disconnected():
break
--
GitLab