From 0becb754ca05a87bf613870c9c093e5ec3fe38e5 Mon Sep 17 00:00:00 2001
From: Ming <tslmy@users.noreply.github.com>
Date: Sun, 24 Mar 2024 12:09:31 -0700
Subject: [PATCH] ReAct Agent should be informed when it tried to use a
non-existent tool (#12207)
* ReAct Agents should know when it tried to use a non-existent tool
* ignore the storage folder
* async
---
llama-index-core/.gitignore | 1 +
.../llama_index/core/agent/react/step.py | 98 ++++++++++++-------
2 files changed, 63 insertions(+), 36 deletions(-)
diff --git a/llama-index-core/.gitignore b/llama-index-core/.gitignore
index d1a1c96f8..09a904e84 100644
--- a/llama-index-core/.gitignore
+++ b/llama-index-core/.gitignore
@@ -1,4 +1,5 @@
llama_index/core/_static
+storage/
.DS_Store
# Byte-compiled / optimized / DLL files
__pycache__/
diff --git a/llama-index-core/llama_index/core/agent/react/step.py b/llama-index-core/llama_index/core/agent/react/step.py
index 272ea226d..d15fb4195 100644
--- a/llama-index-core/llama_index/core/agent/react/step.py
+++ b/llama-index-core/llama_index/core/agent/react/step.py
@@ -232,24 +232,27 @@ class ReActAgentWorker(BaseAgentWorker):
# call tool with input
reasoning_step = cast(ActionReasoningStep, current_reasoning[-1])
- tool = tools_dict[reasoning_step.action]
- with self.callback_manager.event(
- CBEventType.FUNCTION_CALL,
- payload={
- EventPayload.FUNCTION_CALL: reasoning_step.action_input,
- EventPayload.TOOL: tool.metadata,
- },
- ) as event:
- try:
- tool_output = tool.call(**reasoning_step.action_input)
- except Exception as e:
- tool_output = ToolOutput(
- content=f"Error: {e!s}",
- tool_name=tool.metadata.name,
- raw_input={"kwargs": reasoning_step.action_input},
- raw_output=e,
- )
- event.on_end(payload={EventPayload.FUNCTION_OUTPUT: str(tool_output)})
+ if reasoning_step.action in tools_dict:
+ tool = tools_dict[reasoning_step.action]
+ with self.callback_manager.event(
+ CBEventType.FUNCTION_CALL,
+ payload={
+ EventPayload.FUNCTION_CALL: reasoning_step.action_input,
+ EventPayload.TOOL: tool.metadata,
+ },
+ ) as event:
+ try:
+ tool_output = tool.call(**reasoning_step.action_input)
+ except Exception as e:
+ tool_output = ToolOutput(
+ content=f"Error: {e!s}",
+ tool_name=tool.metadata.name,
+ raw_input={"kwargs": reasoning_step.action_input},
+ raw_output=e,
+ )
+ event.on_end(payload={EventPayload.FUNCTION_OUTPUT: str(tool_output)})
+ else:
+ tool_output = self._handle_nonexistent_tool_name(reasoning_step)
task.extra_state["sources"].append(tool_output)
@@ -276,24 +279,27 @@ class ReActAgentWorker(BaseAgentWorker):
# call tool with input
reasoning_step = cast(ActionReasoningStep, current_reasoning[-1])
- tool = tools_dict[reasoning_step.action]
- with self.callback_manager.event(
- CBEventType.FUNCTION_CALL,
- payload={
- EventPayload.FUNCTION_CALL: reasoning_step.action_input,
- EventPayload.TOOL: tool.metadata,
- },
- ) as event:
- try:
- tool_output = await tool.acall(**reasoning_step.action_input)
- except Exception as e:
- tool_output = ToolOutput(
- content=f"Error: {e!s}",
- tool_name=tool.metadata.name,
- raw_input={"kwargs": reasoning_step.action_input},
- raw_output=e,
- )
- event.on_end(payload={EventPayload.FUNCTION_OUTPUT: str(tool_output)})
+ if reasoning_step.action in tools_dict:
+ tool = tools_dict[reasoning_step.action]
+ with self.callback_manager.event(
+ CBEventType.FUNCTION_CALL,
+ payload={
+ EventPayload.FUNCTION_CALL: reasoning_step.action_input,
+ EventPayload.TOOL: tool.metadata,
+ },
+ ) as event:
+ try:
+ tool_output = await tool.acall(**reasoning_step.action_input)
+ except Exception as e:
+ tool_output = ToolOutput(
+ content=f"Error: {e!s}",
+ tool_name=tool.metadata.name,
+ raw_input={"kwargs": reasoning_step.action_input},
+ raw_output=e,
+ )
+ event.on_end(payload={EventPayload.FUNCTION_OUTPUT: str(tool_output)})
+ else:
+ tool_output = self._handle_nonexistent_tool_name(reasoning_step)
task.extra_state["sources"].append(tool_output)
@@ -303,6 +309,26 @@ class ReActAgentWorker(BaseAgentWorker):
print_text(f"{observation_step.get_content()}\n", color="blue")
return current_reasoning, False
+ def _handle_nonexistent_tool_name(self, reasoning_step):
+ # We still emit a `tool_output` object to the task, so that the LLM can know
+ # it has hallucinated in the next reasoning step.
+ with self.callback_manager.event(
+ CBEventType.FUNCTION_CALL,
+ payload={
+ EventPayload.FUNCTION_CALL: reasoning_step.action_input,
+ },
+ ) as event:
+ # TODO(L10N): This should be localized.
+ content = f"Error: No such tool named `{reasoning_step.action}`."
+ tool_output = ToolOutput(
+ content=content,
+ tool_name=reasoning_step.action,
+ raw_input={"kwargs": reasoning_step.action_input},
+ raw_output=content,
+ )
+ event.on_end(payload={EventPayload.FUNCTION_OUTPUT: str(tool_output)})
+ return tool_output
+
def _get_response(
self,
current_reasoning: List[BaseReasoningStep],
--
GitLab