From a5a75f618d58564e09805f7c742deae22525ee69 Mon Sep 17 00:00:00 2001
From: Parham Saidi <parham@parha.me>
Date: Wed, 23 Oct 2024 19:22:06 +0200
Subject: [PATCH] feat: added sonnet 3.5 v2 support via bedrock (#1371)
---
.changeset/old-gifts-talk.md | 5 ++++
.../modules/llms/available_llms/bedrock.md | 29 ++++++++++++++++++-
packages/community/README.md | 2 +-
packages/community/src/llm/bedrock/index.ts | 9 ++++++
4 files changed, 43 insertions(+), 2 deletions(-)
create mode 100644 .changeset/old-gifts-talk.md
diff --git a/.changeset/old-gifts-talk.md b/.changeset/old-gifts-talk.md
new file mode 100644
index 000000000..ed8a86176
--- /dev/null
+++ b/.changeset/old-gifts-talk.md
@@ -0,0 +1,5 @@
+---
+"@llamaindex/community": patch
+---
+
+feat: added sonnet 3.5 v2
diff --git a/apps/docs/docs/modules/llms/available_llms/bedrock.md b/apps/docs/docs/modules/llms/available_llms/bedrock.md
index 24f5a159e..2416fb9bb 100644
--- a/apps/docs/docs/modules/llms/available_llms/bedrock.md
+++ b/apps/docs/docs/modules/llms/available_llms/bedrock.md
@@ -31,7 +31,34 @@ META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0";
META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0";
META_LLAMA3_1_8B_INSTRUCT = "meta.llama3-1-8b-instruct-v1:0"; // available on us-west-2
META_LLAMA3_1_70B_INSTRUCT = "meta.llama3-1-70b-instruct-v1:0"; // available on us-west-2
-META_LLAMA3_1_405B_INSTRUCT = "meta.llama3-1-405b-instruct-v1:0"; // preview only, available on us-west-2, tool calling supported
+META_LLAMA3_1_405B_INSTRUCT = "meta.llama3-1-405b-instruct-v1:0"; // available on us-west-2, tool calling supported
+META_LLAMA3_2_1B_INSTRUCT = "meta.llama3-2-1b-instruct-v1:0"; // only available via inference endpoints (see below)
+META_LLAMA3_2_3B_INSTRUCT = "meta.llama3-2-3b-instruct-v1:0"; // only available via inference endpoints (see below)
+META_LLAMA3_2_11B_INSTRUCT = "meta.llama3-2-11b-instruct-v1:0"; // only available via inference endpoints (see below), multimodal and function call supported
+META_LLAMA3_2_90B_INSTRUCT = "meta.llama3-2-90b-instruct-v1:0"; // only available via inference endpoints (see below), multimodal and function call supported
+```
+
+You can also use Bedrock's Inference endpoints by using the model names:
+
+```ts
+// US
+US_ANTHROPIC_CLAUDE_3_HAIKU = "us.anthropic.claude-3-haiku-20240307-v1:0";
+US_ANTHROPIC_CLAUDE_3_OPUS = "us.anthropic.claude-3-opus-20240229-v1:0";
+US_ANTHROPIC_CLAUDE_3_SONNET = "us.anthropic.claude-3-sonnet-20240229-v1:0";
+US_ANTHROPIC_CLAUDE_3_5_SONNET = "us.anthropic.claude-3-5-sonnet-20240620-v1:0";
+US_ANTHROPIC_CLAUDE_3_5_SONNET_V2 =
+ "us.anthropic.claude-3-5-sonnet-20241022-v2:0";
+US_META_LLAMA_3_2_1B_INSTRUCT = "us.meta.llama3-2-1b-instruct-v1:0";
+US_META_LLAMA_3_2_3B_INSTRUCT = "us.meta.llama3-2-3b-instruct-v1:0";
+US_META_LLAMA_3_2_11B_INSTRUCT = "us.meta.llama3-2-11b-instruct-v1:0";
+US_META_LLAMA_3_2_90B_INSTRUCT = "us.meta.llama3-2-90b-instruct-v1:0";
+
+// EU
+EU_ANTHROPIC_CLAUDE_3_HAIKU = "eu.anthropic.claude-3-haiku-20240307-v1:0";
+EU_ANTHROPIC_CLAUDE_3_SONNET = "eu.anthropic.claude-3-sonnet-20240229-v1:0";
+EU_ANTHROPIC_CLAUDE_3_5_SONNET = "eu.anthropic.claude-3-5-sonnet-20240620-v1:0";
+EU_META_LLAMA_3_2_1B_INSTRUCT = "eu.meta.llama3-2-1b-instruct-v1:0";
+EU_META_LLAMA_3_2_3B_INSTRUCT = "eu.meta.llama3-2-3b-instruct-v1:0";
```
Sonnet, Haiku and Opus are multimodal, image_url only supports base64 data url format, e.g. `data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==`
diff --git a/packages/community/README.md b/packages/community/README.md
index 2457143dc..4f2d55d3f 100644
--- a/packages/community/README.md
+++ b/packages/community/README.md
@@ -4,7 +4,7 @@
## Current Features:
-- Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
+- Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock) including the latest Sonnet 3.5 v2
- Bedrock support for the Meta LLama 2, 3, 3.1 and 3.2 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
- Meta LLama3.1 405b and Llama3.2 tool call support
- Meta 3.2 11B and 90B vision support
diff --git a/packages/community/src/llm/bedrock/index.ts b/packages/community/src/llm/bedrock/index.ts
index 9e4bc9141..f85c05b04 100644
--- a/packages/community/src/llm/bedrock/index.ts
+++ b/packages/community/src/llm/bedrock/index.ts
@@ -65,6 +65,7 @@ export const BEDROCK_MODELS = {
ANTHROPIC_CLAUDE_3_HAIKU: "anthropic.claude-3-haiku-20240307-v1:0",
ANTHROPIC_CLAUDE_3_OPUS: "anthropic.claude-3-opus-20240229-v1:0",
ANTHROPIC_CLAUDE_3_5_SONNET: "anthropic.claude-3-5-sonnet-20240620-v1:0",
+ ANTHROPIC_CLAUDE_3_5_SONNET_V2: "anthropic.claude-3-5-sonnet-20241022-v2:0",
META_LLAMA2_13B_CHAT: "meta.llama2-13b-chat-v1",
META_LLAMA2_70B_CHAT: "meta.llama2-70b-chat-v1",
META_LLAMA3_8B_INSTRUCT: "meta.llama3-8b-instruct-v1:0",
@@ -89,6 +90,8 @@ export const INFERENCE_BEDROCK_MODELS = {
US_ANTHROPIC_CLAUDE_3_SONNET: "us.anthropic.claude-3-sonnet-20240229-v1:0",
US_ANTHROPIC_CLAUDE_3_5_SONNET:
"us.anthropic.claude-3-5-sonnet-20240620-v1:0",
+ US_ANTHROPIC_CLAUDE_3_5_SONNET_V2:
+ "us.anthropic.claude-3-5-sonnet-20241022-v2:0",
US_META_LLAMA_3_2_1B_INSTRUCT: "us.meta.llama3-2-1b-instruct-v1:0",
US_META_LLAMA_3_2_3B_INSTRUCT: "us.meta.llama3-2-3b-instruct-v1:0",
US_META_LLAMA_3_2_11B_INSTRUCT: "us.meta.llama3-2-11b-instruct-v1:0",
@@ -117,6 +120,8 @@ export const INFERENCE_TO_BEDROCK_MAP: Record<
BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_SONNET,
[INFERENCE_BEDROCK_MODELS.US_ANTHROPIC_CLAUDE_3_5_SONNET]:
BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
+ [INFERENCE_BEDROCK_MODELS.US_ANTHROPIC_CLAUDE_3_5_SONNET_V2]:
+ BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET_V2,
[INFERENCE_BEDROCK_MODELS.US_META_LLAMA_3_2_1B_INSTRUCT]:
BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT,
[INFERENCE_BEDROCK_MODELS.US_META_LLAMA_3_2_3B_INSTRUCT]:
@@ -163,6 +168,7 @@ const CHAT_ONLY_MODELS = {
[BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU]: 200000,
[BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS]: 200000,
[BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET]: 200000,
+ [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET_V2]: 200000,
[BEDROCK_MODELS.META_LLAMA2_13B_CHAT]: 2048,
[BEDROCK_MODELS.META_LLAMA2_70B_CHAT]: 4096,
[BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT]: 8192,
@@ -197,6 +203,7 @@ export const STREAMING_MODELS = new Set([
BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
+ BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET_V2,
BEDROCK_MODELS.META_LLAMA2_13B_CHAT,
BEDROCK_MODELS.META_LLAMA2_70B_CHAT,
BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT,
@@ -218,6 +225,7 @@ export const TOOL_CALL_MODELS: BEDROCK_MODELS[] = [
BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU,
BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS,
BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
+ BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET_V2,
BEDROCK_MODELS.META_LLAMA3_1_405B_INSTRUCT,
BEDROCK_MODELS.META_LLAMA3_2_1B_INSTRUCT,
BEDROCK_MODELS.META_LLAMA3_2_3B_INSTRUCT,
@@ -251,6 +259,7 @@ export const BEDROCK_MODEL_MAX_TOKENS: Partial<Record<BEDROCK_MODELS, number>> =
[BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_HAIKU]: 4096,
[BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_OPUS]: 4096,
[BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET]: 4096,
+ [BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET_V2]: 4096,
[BEDROCK_MODELS.META_LLAMA2_13B_CHAT]: 2048,
[BEDROCK_MODELS.META_LLAMA2_70B_CHAT]: 2048,
[BEDROCK_MODELS.META_LLAMA3_8B_INSTRUCT]: 2048,
--
GitLab