feat(api): unify Bedrock provider using Runtime API

Problem:
The current Bedrock implementation uses the Bedrock SDK, which requires separate handling for different model types and doesn't provide a unified streaming interface.

Solution:
Integrate the Bedrock Runtime API to provide a single, unified interface for all Bedrock models (Claude and Nova) using the ConverseStream API. This eliminates the need for separate handlers while maintaining all existing functionality.

Key Changes:
- Refactored AwsBedrockHandler to use @aws-sdk/client-bedrock-runtime
- Enhanced bedrock-converse-format.ts to handle all content types and properly transform between Anthropic and Bedrock formats
- Maintained cross-region inference support with proper region prefixing
- Added support for prompt caching configuration
- Improved AWS credentials handling to better support default providers
- Added proper error handling and token tracking for all response types

Dependencies:
- Added @aws-sdk/client-bedrock-runtime for unified API access
- Removed @anthropic-ai/bedrock-sdk dependency

Testing:
- Verified message format conversion for all content types
- Tested cross-region inference functionality
- Validated streaming responses for both Claude and Nova models

This change simplifies the codebase by providing a single, consistent interface for all Bedrock models while maintaining full compatibility with existing features.
This commit is contained in:
Cline
2024-12-10 18:33:50 +02:00
parent dffc040e7c
commit 140318cecd
5 changed files with 403 additions and 101 deletions

View File

@@ -16,11 +16,14 @@ export interface ApiHandlerOptions {
openRouterApiKey?: string
openRouterModelId?: string
openRouterModelInfo?: ModelInfo
openRouterUseMiddleOutTransform?: boolean
awsAccessKey?: string
awsSecretKey?: string
awsSessionToken?: string
awsRegion?: string
awsUseCrossRegionInference?: boolean
awsusePromptCache?: boolean
awspromptCacheId?: string
vertexProjectId?: string
vertexRegion?: string
openAiBaseUrl?: string
@@ -33,7 +36,7 @@ export interface ApiHandlerOptions {
geminiApiKey?: string
openAiNativeApiKey?: string
azureApiVersion?: string
openRouterUseMiddleOutTransform?: boolean
useBedrockRuntime?: boolean // Force use of Bedrock Runtime API instead of SDK
}
export type ApiConfiguration = ApiHandlerOptions & {
@@ -105,9 +108,63 @@ export const anthropicModels = {
// AWS Bedrock
// https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference.html
export interface MessageContent {
type: 'text' | 'image' | 'video' | 'tool_use' | 'tool_result';
text?: string;
source?: {
type: 'base64';
data: string | Uint8Array; // string for Anthropic, Uint8Array for Bedrock
media_type: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp';
};
// Video specific fields
format?: string;
s3Location?: {
uri: string;
bucketOwner?: string;
};
// Tool use and result fields
toolUseId?: string;
name?: string;
input?: any;
output?: any; // Used for tool_result type
}
export type BedrockModelId = keyof typeof bedrockModels
export const bedrockDefaultModelId: BedrockModelId = "anthropic.claude-3-5-sonnet-20241022-v2:0"
export const bedrockModels = {
"amazon.nova-pro-v1:0": {
maxTokens: 5000,
contextWindow: 300_000,
supportsImages: true,
supportsComputerUse: false,
supportsPromptCache: false,
inputPrice: 0.8,
outputPrice: 3.2,
cacheWritesPrice: 0.8, // per million tokens
cacheReadsPrice: 0.2, // per million tokens
},
"amazon.nova-lite-v1:0": {
maxTokens: 5000,
contextWindow: 300_000,
supportsImages: true,
supportsComputerUse: false,
supportsPromptCache: false,
inputPrice: 0.06,
outputPrice: 0.024,
cacheWritesPrice: 0.06, // per million tokens
cacheReadsPrice: 0.015, // per million tokens
},
"amazon.nova-micro-v1:0": {
maxTokens: 5000,
contextWindow: 128_000,
supportsImages: false,
supportsComputerUse: false,
supportsPromptCache: false,
inputPrice: 0.035,
outputPrice: 0.14,
cacheWritesPrice: 0.035, // per million tokens
cacheReadsPrice: 0.00875, // per million tokens
},
"anthropic.claude-3-5-sonnet-20241022-v2:0": {
maxTokens: 8192,
contextWindow: 200_000,
@@ -116,6 +173,9 @@ export const bedrockModels = {
supportsPromptCache: false,
inputPrice: 3.0,
outputPrice: 15.0,
cacheWritesPrice: 3.75, // per million tokens
cacheReadsPrice: 0.3, // per million tokens
},
"anthropic.claude-3-5-haiku-20241022-v1:0": {
maxTokens: 8192,
@@ -124,6 +184,9 @@ export const bedrockModels = {
supportsPromptCache: false,
inputPrice: 1.0,
outputPrice: 5.0,
cacheWritesPrice: 1.0,
cacheReadsPrice: 0.08,
},
"anthropic.claude-3-5-sonnet-20240620-v1:0": {
maxTokens: 8192,