wirejs-deploy-amplify-basic 0.0.162 → 0.0.164-llm

package/amplify-backend-assets/backend.ts

@@ -213,6 +213,18 @@ const apiUrl = backend.api.resources.lambda.addFunctionUrl({
 });
 apiUrl.grantInvokeUrl(new AnyPrincipal());
 
+backend.api.resources.lambda.addToRolePolicy(new PolicyStatement({
+  actions: [
+    'bedrock:InvokeModel',
+    'bedrock:InvokeModelWithResponseStream'
+  ],
+  resources: [
+    'arn:aws:bedrock:*::foundation-model/*',
+    'arn:aws:bedrock:*:*:custom-model/*'
+  ]
+}));
+
+
 backend.addOutput({
   custom: {
     api: apiUrl.url

package/amplify-backend-assets/constructs/realtime-service/index.ts

@@ -60,7 +60,7 @@ export class RealtimeService extends Construct {
 		const iamProvider: AppSyncAuthProvider = { authorizationType: IAM };
 
 		realtimeService = new EventApi(this, 'realtime', {
-			apiName: `${props.appId}-${props.branchId}-realtime-api`,
+			apiName: `${props.appId.slice(0, 18)}-${props.branchId.slice(0, 18)}-realtime-api`,
 			authorizationConfig: {
 				authProviders: [iamProvider, lambdaProvider],
 				connectionAuthModeTypes: [LAMBDA],

package/amplify-hosting-assets/compute/default/package.json

@@ -3,6 +3,6 @@
 	"dependencies": {
 		"jsdom": "^25.0.1",
 		"wirejs-dom": "^1.0.42",
-		"wirejs-resources": "^0.1.130"
+		"wirejs-resources": "^0.1.132-llm"
 	}
 }

package/dist/index.d.ts

@@ -1,6 +1,7 @@
 export * from 'wirejs-resources';
 export { FileService } from './services/file.js';
 export { AuthenticationService } from './services/authentication.js';
+export { LLM } from './services/llm.js';
 export { DistributedTable } from './resources/distributed-table.js';
 export { RealtimeService } from './services/realtime.js';
 export { BackgroundJob } from './resources/background-job.js';

package/dist/index.js

@@ -6,6 +6,8 @@ import { FileService } from './services/file.js';
 export { FileService } from './services/file.js';
 import { AuthenticationService } from './services/authentication.js';
 export { AuthenticationService } from './services/authentication.js';
+import { LLM } from './services/llm.js';
+export { LLM } from './services/llm.js';
 import { DistributedTable } from './resources/distributed-table.js';
 export { DistributedTable } from './resources/distributed-table.js';
 import { RealtimeService } from './services/realtime.js';
@@ -21,3 +23,4 @@ overrides.FileService = FileService;
 overrides.RealtimeService = RealtimeService;
 overrides.BackgroundJob = BackgroundJob;
 overrides.Endpoint = Endpoint;
+overrides.LLM = LLM;

package/dist/services/llm.d.ts

@@ -0,0 +1,14 @@
+import { LLM as BaseLLM, LLMMessage, LLMChunk, Resource } from 'wirejs-resources';
+export declare class LLM extends BaseLLM {
+    private bedrockClient;
+    constructor(scope: Resource | string, id: string, options: {
+        models: string[];
+    });
+    private createBedrockInstructionMessage;
+    private convertToBedrockFormat;
+    private getModelId;
+    private invokeBedrock;
+    private streamBedrock;
+    private invokeModel;
+    continueConversation(history: LLMMessage[], onChunk?: (chunk: LLMChunk) => void | Promise<void>): Promise<LLMMessage>;
+}

package/dist/services/llm.js

@@ -0,0 +1,152 @@
+import { LLM as BaseLLM } from 'wirejs-resources';
+import { BedrockRuntimeClient, InvokeModelCommand, InvokeModelWithResponseStreamCommand } from '@aws-sdk/client-bedrock-runtime';
+import { defaultProvider } from '@aws-sdk/credential-provider-node';
+export class LLM extends BaseLLM {
+    bedrockClient;
+    constructor(scope, id, options) {
+        super(scope, id, options);
+        this.models = options.models;
+        this.bedrockClient = new BedrockRuntimeClient({
+            credentials: defaultProvider(),
+        });
+    }
+    createBedrockInstructionMessage(message) {
+        return {
+            role: 'assistant',
+            content: message
+        };
+    }
+    convertToBedrockFormat(messages) {
+        return messages.map(msg => ({
+            role: msg.role === 'user' ? 'user' : 'assistant',
+            content: msg.content
+        }));
+    }
+    getModelId(model) {
+        // small set of convenience identifier aliases. maybe worth breaking out into
+        // a larger, shared, canonical JSON later?
+        const modelMap = {
+            'claude': 'anthropic.claude-3-haiku-20240307-v1:0',
+            'claude-haiku': 'anthropic.claude-3-haiku-20240307-v1:0',
+            'claude-sonnet': 'anthropic.claude-3-sonnet-20240229-v1:0',
+            'claude-opus': 'anthropic.claude-3-opus-20240229-v1:0',
+            'llama2': 'meta.llama2-70b-chat-v1',
+            'llama3': 'meta.llama3-70b-instruct-v1:0',
+        };
+        return modelMap[model] || model;
+    }
+    async invokeBedrock(modelId, messages, stream) {
+        const body = JSON.stringify({
+            anthropic_version: "bedrock-2023-05-31",
+            max_tokens: 10000,
+            messages: this.convertToBedrockFormat(messages)
+        });
+        if (stream) {
+            const command = new InvokeModelWithResponseStreamCommand({
+                modelId,
+                body,
+                contentType: 'application/json',
+                accept: 'application/json'
+            });
+            return await this.bedrockClient.send(command);
+        }
+        else {
+            const command = new InvokeModelCommand({
+                modelId,
+                body,
+                contentType: 'application/json',
+                accept: 'application/json'
+            });
+            return await this.bedrockClient.send(command);
+        }
+    }
+    async streamBedrock(response, onChunk) {
+        let content = '';
+        if (response.body) {
+            for await (const chunk of response.body) {
+                if (chunk.chunk?.bytes) {
+                    const data = JSON.parse(new TextDecoder().decode(chunk.chunk.bytes));
+                    if (data.delta?.text) {
+                        content += data.delta.text;
+                        const llmChunk = {
+                            created_at: new Date().toISOString(),
+                            message: {
+                                role: 'assistant',
+                                content: data.delta.text
+                            },
+                            done: false
+                        };
+                        if (onChunk) {
+                            try {
+                                await onChunk(llmChunk);
+                            }
+                            catch (error) {
+                                console.error(error);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return {
+            role: 'assistant',
+            content
+        };
+    }
+    async invokeModel(model, history, onChunk, maxRetries = 10) {
+        const stream = typeof onChunk === 'function';
+        for (let i = 0; i < maxRetries; i++) {
+            try {
+                const modelId = this.getModelId(model);
+                const response = await this.invokeBedrock(modelId, history, stream);
+                if (stream) {
+                    return this.streamBedrock(response, onChunk);
+                }
+                else {
+                    const data = JSON.parse(new TextDecoder().decode(response.body));
+                    return {
+                        role: 'assistant',
+                        content: data.content?.[0]?.text || 'No response generated'
+                    };
+                }
+            }
+            catch (error) {
+                if (error.name === 'ThrottlingException') {
+                    console.warn(`ThrottlingException encountered for model ${model}. Retrying...`);
+                    await new Promise(resolve => setTimeout(resolve, i * 1000 + Math.random() * 1000));
+                    continue;
+                }
+                else {
+                    throw error;
+                }
+            }
+        }
+        throw new Error("Exceeded max retries trying to invoke " + model);
+    }
+    async continueConversation(history, onChunk) {
+        // models are expected to be given in priority order. first one that doesn't
+        // throw a fit wins.
+        for (const model of this.models) {
+            try {
+                return await this.invokeModel(model, history, onChunk);
+            }
+            catch (error) {
+                console.log(error);
+                if (error.name === 'ValidationException' && error.message?.includes('model')) {
+                    continue;
+                }
+                if (error.name === 'AccessDeniedException') {
+                    continue;
+                }
+            }
+        }
+        return this.createBedrockInstructionMessage(`None of the attempted are not available or not enabled in your AWS account. ` +
+            `Please enable it in the AWS Bedrock console:\n\n` +
+            `1. Go to AWS Bedrock Console (https://console.aws.amazon.com/bedrock/)\n` +
+            `2. Navigate to "Model access"\n` +
+            `3. Request access to the model\n` +
+            `4. Accept the End User License Agreement (EULA)\n\n` +
+            `Available models attempted: ${this.models.join(', ')}\n` +
+            `You may need to enable these models: ${this.models.map(m => this.getModelId(m)).join(', ')}`);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "wirejs-deploy-amplify-basic",
-	"version": "0.0.162",
+	"version": "0.0.164-llm",
 	"type": "module",
 	"main": "./dist/index.js",
 	"types": "./dist/index.d.ts",
@@ -28,6 +28,7 @@
 	},
 	"dependencies": {
 		"@aws-crypto/sha256-js": "^5.2.0",
+		"@aws-sdk/client-bedrock-runtime": "^3.821.0",
 		"@aws-sdk/client-cognito-identity-provider": "^3.741.0",
 		"@aws-sdk/client-dynamodb": "^3.774.0",
 		"@aws-sdk/client-s3": "^3.738.0",
@@ -42,7 +43,7 @@
 		"recursive-copy": "^2.0.14",
 		"rimraf": "^6.0.1",
 		"wirejs-dom": "^1.0.42",
-		"wirejs-resources": "^0.1.130"
+		"wirejs-resources": "^0.1.132-llm"
 	},
 	"devDependencies": {
 		"@aws-amplify/backend": "^1.14.0",