From ca8d9709e01609c8b89cd76ad8a28305bffc7664 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A1ssio=20de=20Freitas=20e=20Silva?=
<ksfreitas@gmail.com>
Date: Wed, 3 Jul 2024 05:27:58 -0300
Subject: [PATCH] feat: add support for Meta LLMs in AWS Bedrock (#960)
---
.../modules/llms/available_llms/bedrock.md | 6 +-
packages/community/README.md | 1 +
packages/community/src/llm/bedrock/base.ts | 218 ++----------------
.../community/src/llm/bedrock/provider.ts | 59 +++++
.../src/llm/bedrock/providers/anthropic.ts | 154 +++++++++++++
.../src/llm/bedrock/providers/index.ts | 9 +
.../src/llm/bedrock/providers/meta.ts | 69 ++++++
packages/community/src/llm/bedrock/types.ts | 22 +-
packages/community/src/llm/bedrock/utils.ts | 81 +++++++
9 files changed, 418 insertions(+), 201 deletions(-)
create mode 100644 packages/community/src/llm/bedrock/provider.ts
create mode 100644 packages/community/src/llm/bedrock/providers/anthropic.ts
create mode 100644 packages/community/src/llm/bedrock/providers/index.ts
create mode 100644 packages/community/src/llm/bedrock/providers/meta.ts
diff --git a/apps/docs/docs/modules/llms/available_llms/bedrock.md b/apps/docs/docs/modules/llms/available_llms/bedrock.md
index 07b934dae..56c65c8fd 100644
--- a/apps/docs/docs/modules/llms/available_llms/bedrock.md
+++ b/apps/docs/docs/modules/llms/available_llms/bedrock.md
@@ -15,7 +15,7 @@ Settings.llm = new Bedrock({
});
```
-Currently only supports Anthropic models:
+Currently only supports Anthropic and Meta models:
```ts
ANTHROPIC_CLAUDE_INSTANT_1 = "anthropic.claude-instant-v1";
@@ -25,6 +25,10 @@ ANTHROPIC_CLAUDE_3_SONNET = "anthropic.claude-3-sonnet-20240229-v1:0";
ANTHROPIC_CLAUDE_3_HAIKU = "anthropic.claude-3-haiku-20240307-v1:0";
ANTHROPIC_CLAUDE_3_OPUS = "anthropic.claude-3-opus-20240229-v1:0"; // available on us-west-2
ANTHROPIC_CLAUDE_3_5_SONNET = "anthropic.claude-3-5-sonnet-20240620-v1:0";
+META_LLAMA2_13B_CHAT = "meta.llama2-13b-chat-v1";
+META_LLAMA2_70B_CHAT = "meta.llama2-70b-chat-v1";
+META_LLAMA3_8B_INSTRUCT = "meta.llama3-8b-instruct-v1:0";
+META_LLAMA3_70B_INSTRUCT = "meta.llama3-70b-instruct-v1:0";
```
Sonnet, Haiku and Opus are multimodal, image_url only supports base64 data url format, e.g. `data:image/jpeg;base64,SGVsbG8sIFdvcmxkIQ==`
diff --git a/packages/community/README.md b/packages/community/README.md
index b759df607..5a2f0fa7a 100644
--- a/packages/community/README.md
+++ b/packages/community/README.md
@@ -5,6 +5,7 @@
## Current Features:
- Bedrock support for the Anthropic Claude Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
+- Bedrock support for the Meta LLama 2 and 3 Models [usage](https://ts.llamaindex.ai/modules/llms/available_llms/bedrock)
## LICENSE
diff --git a/packages/community/src/llm/bedrock/base.ts b/packages/community/src/llm/bedrock/base.ts
index a3d5b3405..b50f6f1c2 100644
--- a/packages/community/src/llm/bedrock/base.ts
+++ b/packages/community/src/llm/bedrock/base.ts
@@ -1,53 +1,34 @@
import {
BedrockRuntimeClient,
+ type BedrockRuntimeClientConfig,
InvokeModelCommand,
InvokeModelWithResponseStreamCommand,
- ResponseStream,
- type BedrockRuntimeClientConfig,
- type InvokeModelCommandInput,
- type InvokeModelWithResponseStreamCommandInput,
} from "@aws-sdk/client-bedrock-runtime";
import type {
- BaseTool,
ChatMessage,
ChatResponse,
- ChatResponseChunk,
CompletionResponse,
LLMChatParamsNonStreaming,
LLMChatParamsStreaming,
LLMCompletionParamsNonStreaming,
LLMCompletionParamsStreaming,
LLMMetadata,
- PartialToolCall,
- ToolCall,
ToolCallLLMMessageOptions,
} from "llamaindex";
-import { ToolCallLLM, streamConverter, wrapLLMEvent } from "llamaindex";
-import type {
- AnthropicNoneStreamingResponse,
- AnthropicTextContent,
- StreamEvent,
- ToolBlock,
- ToolChoice,
-} from "./types.js";
+import { streamConverter, ToolCallLLM, wrapLLMEvent } from "llamaindex";
import {
- mapBaseToolsToAnthropicTools,
- mapChatMessagesToAnthropicMessages,
- mapMessageContentToMessageContentDetails,
- toUtf8,
-} from "./utils.js";
-
-export type BedrockAdditionalChatOptions = { toolChoice: ToolChoice };
+ type BedrockAdditionalChatOptions,
+ type BedrockChatStreamResponse,
+ Provider,
+} from "./provider";
+import { PROVIDERS } from "./providers";
+import { mapMessageContentToMessageContentDetails } from "./utils.js";
export type BedrockChatParamsStreaming = LLMChatParamsStreaming<
BedrockAdditionalChatOptions,
ToolCallLLMMessageOptions
>;
-export type BedrockChatStreamResponse = AsyncIterable<
- ChatResponseChunk<ToolCallLLMMessageOptions>
->;
-
export type BedrockChatParamsNonStreaming = LLMChatParamsNonStreaming<
BedrockAdditionalChatOptions,
ToolCallLLMMessageOptions
@@ -151,174 +132,6 @@ export const TOOL_CALL_MODELS = [
BEDROCK_MODELS.ANTHROPIC_CLAUDE_3_5_SONNET,
];
-abstract class Provider<ProviderStreamEvent extends {} = {}> {
- abstract getTextFromResponse(response: Record<string, any>): string;
-
- abstract getToolsFromResponse<T extends {} = {}>(
- response: Record<string, any>,
- ): T[];
-
- getStreamingEventResponse(
- response: Record<string, any>,
- ): ProviderStreamEvent | undefined {
- return response.chunk?.bytes
- ? (JSON.parse(toUtf8(response.chunk?.bytes)) as ProviderStreamEvent)
- : undefined;
- }
-
- async *reduceStream(
- stream: AsyncIterable<ResponseStream>,
- ): BedrockChatStreamResponse {
- yield* streamConverter(stream, (response) => {
- return {
- delta: this.getTextFromStreamResponse(response),
- raw: response,
- };
- });
- }
-
- getTextFromStreamResponse(response: Record<string, any>): string {
- return this.getTextFromResponse(response);
- }
-
- abstract getRequestBody<T extends ChatMessage>(
- metadata: LLMMetadata,
- messages: T[],
- tools?: BaseTool[],
- options?: BedrockAdditionalChatOptions,
- ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput;
-}
-
-class AnthropicProvider extends Provider<StreamEvent> {
- getResultFromResponse(
- response: Record<string, any>,
- ): AnthropicNoneStreamingResponse {
- return JSON.parse(toUtf8(response.body));
- }
-
- getToolsFromResponse<AnthropicToolContent>(
- response: Record<string, any>,
- ): AnthropicToolContent[] {
- const result = this.getResultFromResponse(response);
- return result.content
- .filter((item) => item.type === "tool_use")
- .map((item) => item as AnthropicToolContent);
- }
-
- getTextFromResponse(response: Record<string, any>): string {
- const result = this.getResultFromResponse(response);
- return result.content
- .filter((item) => item.type === "text")
- .map((item) => (item as AnthropicTextContent).text)
- .join(" ");
- }
-
- getTextFromStreamResponse(response: Record<string, any>): string {
- const event = this.getStreamingEventResponse(response);
- if (event?.type === "content_block_delta") {
- if (event.delta.type === "text_delta") return event.delta.text;
- if (event.delta.type === "input_json_delta")
- return event.delta.partial_json;
- }
- return "";
- }
-
- async *reduceStream(
- stream: AsyncIterable<ResponseStream>,
- ): BedrockChatStreamResponse {
- let collecting = [];
- let tool: ToolBlock | undefined = undefined;
- // #TODO this should be broken down into a separate consumer
- for await (const response of stream) {
- const event = this.getStreamingEventResponse(response);
- if (
- event?.type === "content_block_start" &&
- event.content_block.type === "tool_use"
- ) {
- tool = event.content_block;
- continue;
- }
-
- if (
- event?.type === "content_block_delta" &&
- event.delta.type === "input_json_delta"
- ) {
- collecting.push(event.delta.partial_json);
- }
-
- let options: undefined | ToolCallLLMMessageOptions = undefined;
- if (tool && collecting.length) {
- const input = collecting.filter((item) => item).join("");
- // We have all we need to parse the tool_use json
- if (event?.type === "content_block_stop") {
- options = {
- toolCall: [
- {
- id: tool.id,
- name: tool.name,
- input: JSON.parse(input),
- } as ToolCall,
- ],
- };
- // reset the collection/tool
- collecting = [];
- tool = undefined;
- } else {
- options = {
- toolCall: [
- {
- id: tool.id,
- name: tool.name,
- input,
- } as PartialToolCall,
- ],
- };
- }
- }
- const delta = this.getTextFromStreamResponse(response);
- if (!delta && !options) continue;
-
- yield {
- delta,
- options,
- raw: response,
- };
- }
- }
-
- getRequestBody<T extends ChatMessage<ToolCallLLMMessageOptions>>(
- metadata: LLMMetadata,
- messages: T[],
- tools?: BaseTool[],
- options?: BedrockAdditionalChatOptions,
- ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
- const extra: Record<string, unknown> = {};
- if (options?.toolChoice) {
- extra["tool_choice"] = options?.toolChoice;
- }
- const mapped = mapChatMessagesToAnthropicMessages(messages);
- return {
- modelId: metadata.model,
- contentType: "application/json",
- accept: "application/json",
- body: JSON.stringify({
- anthropic_version: "bedrock-2023-05-31",
- messages: mapped,
- tools: mapBaseToolsToAnthropicTools(tools),
- max_tokens: metadata.maxTokens,
- temperature: metadata.temperature,
- top_p: metadata.topP,
- ...extra,
- }),
- };
- }
-}
-
-// Other providers could go here
-const PROVIDERS: { [key: string]: Provider } = {
- anthropic: new AnthropicProvider(),
-};
-
const getProvider = (model: string): Provider => {
const providerName = model.split(".")[0];
if (!(providerName in PROVIDERS)) {
@@ -373,6 +186,10 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
this.temperature = temperature ?? DEFAULT_BEDROCK_PARAMS.temperature;
this.topP = topP ?? DEFAULT_BEDROCK_PARAMS.topP;
this.client = new BedrockRuntimeClient(params);
+
+ if (!this.supportToolCall) {
+ console.warn(`The model "${this.model}" doesn't support ToolCall`);
+ }
}
get supportToolCall(): boolean {
@@ -402,10 +219,13 @@ export class Bedrock extends ToolCallLLM<BedrockAdditionalChatOptions> {
);
const command = new InvokeModelCommand(input);
const response = await this.client.send(command);
- const tools = this.provider.getToolsFromResponse(response);
- const options: ToolCallLLMMessageOptions = tools.length
- ? { toolCall: tools }
- : {};
+ let options: ToolCallLLMMessageOptions = {};
+ if (this.supportToolCall) {
+ const tools = this.provider.getToolsFromResponse(response);
+ if (tools.length) {
+ options = { toolCall: tools };
+ }
+ }
return {
raw: response,
message: {
diff --git a/packages/community/src/llm/bedrock/provider.ts b/packages/community/src/llm/bedrock/provider.ts
new file mode 100644
index 000000000..99b615354
--- /dev/null
+++ b/packages/community/src/llm/bedrock/provider.ts
@@ -0,0 +1,59 @@
+import {
+ type InvokeModelCommandInput,
+ type InvokeModelWithResponseStreamCommandInput,
+ ResponseStream,
+} from "@aws-sdk/client-bedrock-runtime";
+import {
+ type BaseTool,
+ type ChatMessage,
+ type ChatResponseChunk,
+ type LLMMetadata,
+ streamConverter,
+ type ToolCallLLMMessageOptions,
+} from "llamaindex";
+import type { ToolChoice } from "./types";
+import { toUtf8 } from "./utils";
+
+export type BedrockAdditionalChatOptions = { toolChoice: ToolChoice };
+
+export type BedrockChatStreamResponse = AsyncIterable<
+ ChatResponseChunk<ToolCallLLMMessageOptions>
+>;
+
+export abstract class Provider<ProviderStreamEvent extends {} = {}> {
+ abstract getTextFromResponse(response: Record<string, any>): string;
+
+ abstract getToolsFromResponse<T extends {} = {}>(
+ response: Record<string, any>,
+ ): T[];
+
+ getStreamingEventResponse(
+ response: Record<string, any>,
+ ): ProviderStreamEvent | undefined {
+ return response.chunk?.bytes
+ ? (JSON.parse(toUtf8(response.chunk?.bytes)) as ProviderStreamEvent)
+ : undefined;
+ }
+
+ async *reduceStream(
+ stream: AsyncIterable<ResponseStream>,
+ ): BedrockChatStreamResponse {
+ yield* streamConverter(stream, (response) => {
+ return {
+ delta: this.getTextFromStreamResponse(response),
+ raw: response,
+ };
+ });
+ }
+
+ getTextFromStreamResponse(response: Record<string, any>): string {
+ return this.getTextFromResponse(response);
+ }
+
+ abstract getRequestBody<T extends ChatMessage>(
+ metadata: LLMMetadata,
+ messages: T[],
+ tools?: BaseTool[],
+ options?: BedrockAdditionalChatOptions,
+ ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput;
+}
diff --git a/packages/community/src/llm/bedrock/providers/anthropic.ts b/packages/community/src/llm/bedrock/providers/anthropic.ts
new file mode 100644
index 000000000..d68c1b721
--- /dev/null
+++ b/packages/community/src/llm/bedrock/providers/anthropic.ts
@@ -0,0 +1,154 @@
+import {
+ type InvokeModelCommandInput,
+ type InvokeModelWithResponseStreamCommandInput,
+ ResponseStream,
+} from "@aws-sdk/client-bedrock-runtime";
+import type {
+ BaseTool,
+ ChatMessage,
+ LLMMetadata,
+ PartialToolCall,
+ ToolCall,
+ ToolCallLLMMessageOptions,
+} from "llamaindex";
+import {
+ type BedrockAdditionalChatOptions,
+ type BedrockChatStreamResponse,
+ Provider,
+} from "../provider";
+import type {
+ AnthropicNoneStreamingResponse,
+ AnthropicStreamEvent,
+ AnthropicTextContent,
+ ToolBlock,
+} from "../types";
+import {
+ mapBaseToolsToAnthropicTools,
+ mapChatMessagesToAnthropicMessages,
+ toUtf8,
+} from "../utils";
+
+export class AnthropicProvider extends Provider<AnthropicStreamEvent> {
+ getResultFromResponse(
+ response: Record<string, any>,
+ ): AnthropicNoneStreamingResponse {
+ return JSON.parse(toUtf8(response.body));
+ }
+
+ getToolsFromResponse<AnthropicToolContent>(
+ response: Record<string, any>,
+ ): AnthropicToolContent[] {
+ const result = this.getResultFromResponse(response);
+ return result.content
+ .filter((item) => item.type === "tool_use")
+ .map((item) => item as AnthropicToolContent);
+ }
+
+ getTextFromResponse(response: Record<string, any>): string {
+ const result = this.getResultFromResponse(response);
+ return result.content
+ .filter((item) => item.type === "text")
+ .map((item) => (item as AnthropicTextContent).text)
+ .join(" ");
+ }
+
+ getTextFromStreamResponse(response: Record<string, any>): string {
+ const event = this.getStreamingEventResponse(response);
+ if (event?.type === "content_block_delta") {
+ if (event.delta.type === "text_delta") return event.delta.text;
+ if (event.delta.type === "input_json_delta")
+ return event.delta.partial_json;
+ }
+ return "";
+ }
+
+ async *reduceStream(
+ stream: AsyncIterable<ResponseStream>,
+ ): BedrockChatStreamResponse {
+ let collecting = [];
+ let tool: ToolBlock | undefined = undefined;
+ // #TODO this should be broken down into a separate consumer
+ for await (const response of stream) {
+ const event = this.getStreamingEventResponse(response);
+ if (
+ event?.type === "content_block_start" &&
+ event.content_block.type === "tool_use"
+ ) {
+ tool = event.content_block;
+ continue;
+ }
+
+ if (
+ event?.type === "content_block_delta" &&
+ event.delta.type === "input_json_delta"
+ ) {
+ collecting.push(event.delta.partial_json);
+ }
+
+ let options: undefined | ToolCallLLMMessageOptions = undefined;
+ if (tool && collecting.length) {
+ const input = collecting.filter((item) => item).join("");
+ // We have all we need to parse the tool_use json
+ if (event?.type === "content_block_stop") {
+ options = {
+ toolCall: [
+ {
+ id: tool.id,
+ name: tool.name,
+ input: JSON.parse(input),
+ } as ToolCall,
+ ],
+ };
+ // reset the collection/tool
+ collecting = [];
+ tool = undefined;
+ } else {
+ options = {
+ toolCall: [
+ {
+ id: tool.id,
+ name: tool.name,
+ input,
+ } as PartialToolCall,
+ ],
+ };
+ }
+ }
+ const delta = this.getTextFromStreamResponse(response);
+ if (!delta && !options) continue;
+
+ yield {
+ delta,
+ options,
+ raw: response,
+ };
+ }
+ }
+
+ getRequestBody<T extends ChatMessage<ToolCallLLMMessageOptions>>(
+ metadata: LLMMetadata,
+ messages: T[],
+ tools?: BaseTool[],
+ options?: BedrockAdditionalChatOptions,
+ ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
+ const extra: Record<string, unknown> = {};
+ if (options?.toolChoice) {
+ extra["tool_choice"] = options?.toolChoice;
+ }
+ const mapped = mapChatMessagesToAnthropicMessages(messages);
+ return {
+ modelId: metadata.model,
+ contentType: "application/json",
+ accept: "application/json",
+ body: JSON.stringify({
+ anthropic_version: "bedrock-2023-05-31",
+ messages: mapped,
+ tools: mapBaseToolsToAnthropicTools(tools),
+ max_tokens: metadata.maxTokens,
+ temperature: metadata.temperature,
+ top_p: metadata.topP,
+ ...extra,
+ }),
+ };
+ }
+}
diff --git a/packages/community/src/llm/bedrock/providers/index.ts b/packages/community/src/llm/bedrock/providers/index.ts
new file mode 100644
index 000000000..01ba640d5
--- /dev/null
+++ b/packages/community/src/llm/bedrock/providers/index.ts
@@ -0,0 +1,9 @@
+import { Provider } from "../provider";
+import { AnthropicProvider } from "./anthropic";
+import { MetaProvider } from "./meta";
+
+// Other providers should go here
+export const PROVIDERS: { [key: string]: Provider } = {
+ anthropic: new AnthropicProvider(),
+ meta: new MetaProvider(),
+};
diff --git a/packages/community/src/llm/bedrock/providers/meta.ts b/packages/community/src/llm/bedrock/providers/meta.ts
new file mode 100644
index 000000000..26eacf086
--- /dev/null
+++ b/packages/community/src/llm/bedrock/providers/meta.ts
@@ -0,0 +1,69 @@
+import type {
+ InvokeModelCommandInput,
+ InvokeModelWithResponseStreamCommandInput,
+} from "@aws-sdk/client-bedrock-runtime";
+import type { ChatMessage, LLMMetadata } from "llamaindex";
+import type { MetaNoneStreamingResponse, MetaStreamEvent } from "../types";
+import {
+ mapChatMessagesToMetaLlama2Messages,
+ mapChatMessagesToMetaLlama3Messages,
+ toUtf8,
+} from "../utils";
+
+import { Provider } from "../provider";
+
+export class MetaProvider extends Provider<MetaStreamEvent> {
+ constructor() {
+ super();
+ }
+
+ getResultFromResponse(
+ response: Record<string, any>,
+ ): MetaNoneStreamingResponse {
+ return JSON.parse(toUtf8(response.body));
+ }
+
+ getToolsFromResponse(_response: Record<string, any>): never {
+ throw new Error("Not supported by this provider.");
+ }
+
+ getTextFromResponse(response: Record<string, any>): string {
+ const result = this.getResultFromResponse(response);
+ return result.generation;
+ }
+
+ getTextFromStreamResponse(response: Record<string, any>): string {
+ const event = this.getStreamingEventResponse(response);
+ if (event?.generation) {
+ return event.generation;
+ }
+ return "";
+ }
+
+ getRequestBody<T extends ChatMessage>(
+ metadata: LLMMetadata,
+ messages: T[],
+ ): InvokeModelCommandInput | InvokeModelWithResponseStreamCommandInput {
+ let promptFunction: (messages: ChatMessage[]) => string;
+
+ if (metadata.model.startsWith("meta.llama3")) {
+ promptFunction = mapChatMessagesToMetaLlama3Messages;
+ } else if (metadata.model.startsWith("meta.llama2")) {
+ promptFunction = mapChatMessagesToMetaLlama2Messages;
+ } else {
+ throw new Error(`Meta model ${metadata.model} is not supported`);
+ }
+
+ return {
+ modelId: metadata.model,
+ contentType: "application/json",
+ accept: "application/json",
+ body: JSON.stringify({
+ prompt: promptFunction(messages),
+ max_gen_len: metadata.maxTokens,
+ temperature: metadata.temperature,
+ top_p: metadata.topP,
+ }),
+ };
+ }
+}
diff --git a/packages/community/src/llm/bedrock/types.ts b/packages/community/src/llm/bedrock/types.ts
index 8a02d5db4..a72554c73 100644
--- a/packages/community/src/llm/bedrock/types.ts
+++ b/packages/community/src/llm/bedrock/types.ts
@@ -79,7 +79,7 @@ export type ToolChoice =
| { type: "auto" }
| { type: "tool"; name: string };
-export type StreamEvent =
+export type AnthropicStreamEvent =
| { type: "message_start"; message: Message }
| ContentBlockStart
| ContentBlockDelta
@@ -93,6 +93,8 @@ export type AnthropicContent =
| AnthropicToolContent
| AnthropicToolResultContent;
+export type MetaTextContent = string;
+
export type AnthropicTextContent = {
type: "text";
text: string;
@@ -133,6 +135,11 @@ export type AnthropicMessage = {
content: AnthropicContent[];
};
+export type MetaMessage = {
+ role: "user" | "assistant" | "system";
+ content: MetaTextContent;
+};
+
export type AnthropicNoneStreamingResponse = {
id: string;
type: "message";
@@ -143,3 +150,16 @@ export type AnthropicNoneStreamingResponse = {
stop_sequence?: string;
usage: { input_tokens: number; output_tokens: number };
};
+
+type MetaResponse = {
+ generation: string;
+ prompt_token_count: number;
+ generation_token_count: number;
+ stop_reason: "stop" | "length";
+};
+
+export type MetaStreamEvent = MetaResponse & {
+ "amazon-bedrock-invocationMetrics": InvocationMetrics;
+};
+
+export type MetaNoneStreamingResponse = MetaResponse;
diff --git a/packages/community/src/llm/bedrock/utils.ts b/packages/community/src/llm/bedrock/utils.ts
index 64bbdda1e..c301d16b2 100644
--- a/packages/community/src/llm/bedrock/utils.ts
+++ b/packages/community/src/llm/bedrock/utils.ts
@@ -4,6 +4,7 @@ import type {
JSONObject,
MessageContent,
MessageContentDetail,
+ MessageContentTextDetail,
ToolCallLLMMessageOptions,
ToolMetadata,
} from "llamaindex";
@@ -13,6 +14,7 @@ import type {
AnthropicMediaTypes,
AnthropicMessage,
AnthropicTextContent,
+ MetaMessage,
} from "./types.js";
const ACCEPTED_IMAGE_MIME_TYPES = [
@@ -148,6 +150,85 @@ export const mapChatMessagesToAnthropicMessages = <
return mergeNeighboringSameRoleMessages(mapped);
};
+export const mapChatMessagesToMetaMessages = <T extends ChatMessage>(
+ messages: T[],
+): MetaMessage[] => {
+ return messages.map((msg) => {
+ let content: string;
+ if (typeof msg.content === "string") {
+ content = msg.content;
+ } else {
+ content = (msg.content[0] as MessageContentTextDetail).text;
+ }
+ return {
+ role:
+ msg.role === "assistant"
+ ? "assistant"
+ : msg.role === "user"
+ ? "user"
+ : "system",
+ content,
+ };
+ });
+};
+
+/**
+ * Documentation at https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-3
+ */
+export const mapChatMessagesToMetaLlama3Messages = <T extends ChatMessage>(
+ messages: T[],
+): string => {
+ const mapped = mapChatMessagesToMetaMessages(messages).map((message) => {
+ const text = message.content;
+ return `<|start_header_id|>${message.role}<|end_header_id|>\n${text}\n<|eot_id|>\n`;
+ });
+ return (
+ "<|begin_of_text|>" +
+ mapped.join("\n") +
+ "\n<|start_header_id|>assistant<|end_header_id|>\n"
+ );
+};
+
+/**
+ * Documentation at https://llama.meta.com/docs/model-cards-and-prompt-formats/meta-llama-2
+ */
+export const mapChatMessagesToMetaLlama2Messages = <T extends ChatMessage>(
+ messages: T[],
+): string => {
+ const mapped = mapChatMessagesToMetaMessages(messages);
+ let output = "<s>";
+ let insideInst = false;
+ let needsStartAgain = false;
+ for (const message of mapped) {
+ if (needsStartAgain) {
+ output += "<s>";
+ needsStartAgain = false;
+ }
+ const text = message.content;
+ if (message.role === "system") {
+ if (!insideInst) {
+ output += "[INST] ";
+ insideInst = true;
+ }
+ output += `<<SYS>>\n${text}\n<</SYS>>\n`;
+ } else if (message.role === "user") {
+ output += text;
+ if (insideInst) {
+ output += " [/INST]";
+ insideInst = false;
+ }
+ } else if (message.role === "assistant") {
+ if (insideInst) {
+ output += " [/INST]";
+ insideInst = false;
+ }
+ output += ` ${text} </s>\n`;
+ needsStartAgain = true;
+ }
+ }
+ return output;
+};
+
export const mapTextContent = (text: string): AnthropicTextContent => {
return { type: "text", text };
};
--
GitLab