From 56746c240fe6d854c4e468ffb2bc51427fd0c05a Mon Sep 17 00:00:00 2001
From: Parham Saidi <parham@parha.me>
Date: Fri, 12 Jul 2024 18:47:49 +0200
Subject: [PATCH] fix: bedrock handle empty content and added max tokens export
(#1034)
---
.changeset/eleven-colts-sniff.md | 5 +++++
packages/community/src/index.ts | 6 +++++-
packages/community/src/llm/bedrock/base.ts | 12 ++++++++++++
packages/community/src/llm/bedrock/utils.ts | 4 ++--
4 files changed, 24 insertions(+), 3 deletions(-)
create mode 100644 .changeset/eleven-colts-sniff.md
diff --git a/.changeset/eleven-colts-sniff.md b/.changeset/eleven-colts-sniff.md
new file mode 100644
index 000000000..b747a28c9
--- /dev/null
+++ b/.changeset/eleven-colts-sniff.md
@@ -0,0 +1,5 @@
+---
+"@llamaindex/community": patch
+---
+
+fix: llama3 patched to handle empty content (can happen with system) and added max tokens export
diff --git a/packages/community/src/index.ts b/packages/community/src/index.ts
index 457f7602b..57a5cac79 100644
--- a/packages/community/src/index.ts
+++ b/packages/community/src/index.ts
@@ -1 +1,5 @@
-export { BEDROCK_MODELS, Bedrock } from "./llm/bedrock/base.js";
+export {
+ BEDROCK_MODELS,
+ BEDROCK_MODEL_MAX_TOKENS,
+ Bedrock,
+} from "./llm/bedrock/base.js";
diff --git a/packages/community/src/llm/bedrock/base.ts b/packages/community/src/llm/bedrock/base.ts
index 1c04a88ab..6dd056bc6 100644
--- a/packages/community/src/llm/bedrock/base.ts
+++ b/packages/community/src/llm/bedrock/base.ts
@@ -150,6 +150,18 @@ export type BedrockModelParams = {
maxTokens?: number;
};
+export const BEDROCK_MODEL_MAX_TOKENS: Partial<Record<BEDROCK_MODELS, number>> =
+ {
+ [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET]: 4096,
+ [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU]: 4096,
+ [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS]: 4096,
+ [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET]: 4096,
+ [BEDROCK_MODELS.META_LLAMA2_13B_CHAT]: 2048,
+ [BEDROCK_MODELS.META_LLAMA2_70B_CHAT]: 2048,
+ [BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT]: 2048,
+ [BEDROCK_MODELS.META_LLAMA3_70B_INSTRUCT]: 2048,
+ };
+
const DEFAULT_BEDROCK_PARAMS = {
temperature: 0.1,
topP: 1,
diff --git a/packages/community/src/llm/bedrock/utils.ts b/packages/community/src/llm/bedrock/utils.ts
index b26fc385f..217c5a4d5 100644
--- a/packages/community/src/llm/bedrock/utils.ts
+++ b/packages/community/src/llm/bedrock/utils.ts
@@ -154,10 +154,10 @@ export const mapChatMessagesToMetaMessages = <T extends ChatMessage>(
messages: T[],
): MetaMessage[] => {
return messages.map((msg) => {
- let content: string;
+ let content: string = "";
if (typeof msg.content === "string") {
content = msg.content;
- } else {
+ } else if (msg.content.length) {
content = (msg.content[0] as MessageContentTextDetail).text;
}
return {
--
GitLab