sst 2.18.3 → 2.19.0

package/cdk-assets/private/handlers/container-images.js

@@ -0,0 +1,231 @@
+import * as path from "path";
+import { EventType } from "cdk-assets/lib/progress.js";
+import { replaceAwsPlaceholders } from "cdk-assets/lib/private/placeholders.js";
+import { shell } from "cdk-assets/lib/private/shell.js";
+export class ContainerImageAssetHandler {
+    workDir;
+    asset;
+    host;
+    options;
+    init;
+    constructor(workDir, asset, host, 
+    // TODO: remove after PR is merged
+    options) {
+        this.workDir = workDir;
+        this.asset = asset;
+        this.host = host;
+        this.options = options;
+    }
+    async build() {
+        const initOnce = await this.initOnce();
+        if (initOnce.destinationAlreadyExists) {
+            return;
+        }
+        if (this.host.aborted) {
+            return;
+        }
+        const dockerForBuilding = await this.host.dockerFactory.forBuild({
+            repoUri: initOnce.repoUri,
+            logger: (m) => this.host.emitMessage(EventType.DEBUG, m),
+            ecr: initOnce.ecr,
+        });
+        const builder = new ContainerImageBuilder(dockerForBuilding, this.workDir, this.asset, this.host, 
+        // TODO: remove after PR is merged
+        {
+            quiet: this.options.quiet,
+        });
+        const localTagName = await builder.build();
+        if (localTagName === undefined || this.host.aborted) {
+            return;
+        }
+        if (this.host.aborted) {
+            return;
+        }
+        await dockerForBuilding.tag(localTagName, initOnce.imageUri);
+    }
+    async isPublished() {
+        try {
+            const initOnce = await this.initOnce({ quiet: true });
+            return initOnce.destinationAlreadyExists;
+        }
+        catch (e) {
+            this.host.emitMessage(EventType.DEBUG, `${e.message}`);
+        }
+        return false;
+    }
+    async publish() {
+        const initOnce = await this.initOnce();
+        if (initOnce.destinationAlreadyExists) {
+            return;
+        }
+        if (this.host.aborted) {
+            return;
+        }
+        const dockerForPushing = await this.host.dockerFactory.forEcrPush({
+            repoUri: initOnce.repoUri,
+            logger: (m) => this.host.emitMessage(EventType.DEBUG, m),
+            ecr: initOnce.ecr,
+        });
+        if (this.host.aborted) {
+            return;
+        }
+        this.host.emitMessage(EventType.UPLOAD, `Push ${initOnce.imageUri}`);
+        // TODO: remove after PR is merged
+        await dockerForPushing.push({
+            tag: initOnce.imageUri,
+            quiet: this.options.quiet,
+        });
+    }
+    async initOnce(options = {}) {
+        if (this.init) {
+            return this.init;
+        }
+        const destination = await replaceAwsPlaceholders(this.asset.destination, this.host.aws);
+        const ecr = await this.host.aws.ecrClient({
+            ...destination,
+            quiet: options.quiet,
+        });
+        const account = async () => (await this.host.aws.discoverCurrentAccount())?.accountId;
+        const repoUri = await repositoryUri(ecr, destination.repositoryName);
+        if (!repoUri) {
+            throw new Error(`No ECR repository named '${destination.repositoryName}' in account ${await account()}. Is this account bootstrapped?`);
+        }
+        const imageUri = `${repoUri}:${destination.imageTag}`;
+        this.init = {
+            imageUri,
+            ecr,
+            repoUri,
+            destinationAlreadyExists: await this.destinationAlreadyExists(ecr, destination, imageUri),
+        };
+        return this.init;
+    }
+    /**
+     * Check whether the image already exists in the ECR repo
+     *
+     * Use the fields from the destination to do the actual check. The imageUri
+     * should correspond to that, but is only used to print Docker image location
+     * for user benefit (the format is slightly different).
+     */
+    async destinationAlreadyExists(ecr, destination, imageUri) {
+        this.host.emitMessage(EventType.CHECK, `Check ${imageUri}`);
+        if (await imageExists(ecr, destination.repositoryName, destination.imageTag)) {
+            this.host.emitMessage(EventType.FOUND, `Found ${imageUri}`);
+            return true;
+        }
+        return false;
+    }
+}
+class ContainerImageBuilder {
+    docker;
+    workDir;
+    asset;
+    host;
+    options;
+    constructor(docker, workDir, asset, host, 
+    // TODO: remove after PR is merged
+    options) {
+        this.docker = docker;
+        this.workDir = workDir;
+        this.asset = asset;
+        this.host = host;
+        this.options = options;
+    }
+    async build() {
+        return this.asset.source.executable
+            ? this.buildExternalAsset(this.asset.source.executable)
+            : this.buildDirectoryAsset();
+    }
+    /**
+     * Build a (local) Docker asset from a directory with a Dockerfile
+     *
+     * Tags under a deterministic, unique, local identifier wich will skip
+     * the build if it already exists.
+     */
+    async buildDirectoryAsset() {
+        const localTagName = `cdkasset-${this.asset.id.assetId.toLowerCase()}`;
+        if (!(await this.isImageCached(localTagName))) {
+            if (this.host.aborted) {
+                return undefined;
+            }
+            await this.buildImage(localTagName);
+        }
+        return localTagName;
+    }
+    /**
+     * Build a (local) Docker asset by running an external command
+     *
+     * External command is responsible for deduplicating the build if possible,
+     * and is expected to return the generated image identifier on stdout.
+     */
+    async buildExternalAsset(executable, cwd) {
+        const assetPath = cwd ?? this.workDir;
+        this.host.emitMessage(EventType.BUILD, `Building Docker image using command '${executable}'`);
+        if (this.host.aborted) {
+            return undefined;
+        }
+        return (await shell(executable, { cwd: assetPath, quiet: true })).trim();
+    }
+    async buildImage(localTagName) {
+        const source = this.asset.source;
+        if (!source.directory) {
+            throw new Error(`'directory' is expected in the DockerImage asset source, got: ${JSON.stringify(source)}`);
+        }
+        const fullPath = path.resolve(this.workDir, source.directory);
+        this.host.emitMessage(EventType.BUILD, `Building Docker image at ${fullPath}`);
+        await this.docker.build({
+            directory: fullPath,
+            tag: localTagName,
+            buildArgs: source.dockerBuildArgs,
+            buildSecrets: source.dockerBuildSecrets,
+            target: source.dockerBuildTarget,
+            file: source.dockerFile,
+            networkMode: source.networkMode,
+            platform: source.platform,
+            outputs: source.dockerOutputs,
+            cacheFrom: source.cacheFrom,
+            cacheTo: source.cacheTo,
+            // TODO: remove after PR is merged
+            quiet: this.options.quiet,
+        });
+    }
+    async isImageCached(localTagName) {
+        if (await this.docker.exists(localTagName)) {
+            this.host.emitMessage(EventType.CACHED, `Cached ${localTagName}`);
+            return true;
+        }
+        return false;
+    }
+}
+async function imageExists(ecr, repositoryName, imageTag) {
+    try {
+        await ecr
+            .describeImages({ repositoryName, imageIds: [{ imageTag }] })
+            .promise();
+        return true;
+    }
+    catch (e) {
+        if (e.code !== "ImageNotFoundException") {
+            throw e;
+        }
+        return false;
+    }
+}
+/**
+ * Return the URI for the repository with the given name
+ *
+ * Returns undefined if the repository does not exist.
+ */
+async function repositoryUri(ecr, repositoryName) {
+    try {
+        const response = await ecr
+            .describeRepositories({ repositoryNames: [repositoryName] })
+            .promise();
+        return (response.repositories || [])[0]?.repositoryUri;
+    }
+    catch (e) {
+        if (e.code !== "RepositoryNotFoundException") {
+            throw e;
+        }
+        return undefined;
+    }
+}

package/cdk-assets/private/handlers/index.d.ts

@@ -0,0 +1,3 @@
+import { AssetManifest, IManifestEntry } from "cdk-assets/lib/asset-manifest.js";
+import { IAssetHandler, IHandlerHost, IHandlerOptions } from "../asset-handler.js";
+export declare function makeAssetHandler(manifest: AssetManifest, asset: IManifestEntry, host: IHandlerHost, options: IHandlerOptions): IAssetHandler;

package/cdk-assets/private/handlers/index.js

@@ -0,0 +1,18 @@
+import { ContainerImageAssetHandler } from "./container-images.js";
+import { FileAssetHandler } from "cdk-assets/lib/private/handlers/files.js";
+import { DockerImageManifestEntry, FileManifestEntry, } from "cdk-assets/lib/asset-manifest.js";
+export function makeAssetHandler(manifest, asset, host, 
+// TODO: remove after PR is merged
+options) {
+    if (asset instanceof FileManifestEntry) {
+        // TODO: remove after PR is merged
+        // @ts-ignore
+        return new FileAssetHandler(manifest.directory, asset, host);
+    }
+    if (asset instanceof DockerImageManifestEntry) {
+        return new ContainerImageAssetHandler(manifest.directory, asset, host, 
+        // TODO: remove after PR is merged
+        options);
+    }
+    throw new Error(`Unrecognized asset type: '${asset}'`);
+}

package/cdk-assets/publishing.d.ts

@@ -0,0 +1,113 @@
+import { AssetManifest, IManifestEntry } from "cdk-assets/lib/asset-manifest.js";
+import { IAws } from "cdk-assets/lib/aws.js";
+import { IPublishProgress, IPublishProgressListener } from "cdk-assets/lib/progress.js";
+export interface AssetPublishingOptions {
+    /**
+     * Entry point for AWS client
+     */
+    readonly aws: IAws;
+    /**
+     * Listener for progress events
+     *
+     * @default No listener
+     */
+    readonly progressListener?: IPublishProgressListener;
+    /**
+     * Whether to throw at the end if there were errors
+     *
+     * @default true
+     */
+    readonly throwOnError?: boolean;
+    /**
+     * Whether to publish in parallel, when 'publish()' is called
+     *
+     * @default false
+     */
+    readonly publishInParallel?: boolean;
+    /**
+     * Whether to build assets, when 'publish()' is called
+     *
+     * @default true
+     */
+    readonly buildAssets?: boolean;
+    /**
+     * Whether to publish assets, when 'publish()' is called
+     *
+     * @default true
+     */
+    readonly publishAssets?: boolean;
+    /**
+     * Whether to print publishing logs
+     *
+     * @default true
+     */
+    readonly quiet?: boolean;
+}
+/**
+ * A failure to publish an asset
+ */
+export interface FailedAsset {
+    /**
+     * The asset that failed to publish
+     */
+    readonly asset: IManifestEntry;
+    /**
+     * The failure that occurred
+     */
+    readonly error: Error;
+}
+export declare class AssetPublishing implements IPublishProgress {
+    private readonly manifest;
+    private readonly options;
+    /**
+     * The message for the IPublishProgress interface
+     */
+    message: string;
+    /**
+     * The current asset for the IPublishProgress interface
+     */
+    currentAsset?: IManifestEntry;
+    readonly failures: FailedAsset[];
+    private readonly assets;
+    private readonly totalOperations;
+    private completedOperations;
+    private aborted;
+    private readonly handlerHost;
+    private readonly publishInParallel;
+    private readonly buildAssets;
+    private readonly publishAssets;
+    private readonly handlerCache;
+    constructor(manifest: AssetManifest, options: AssetPublishingOptions);
+    /**
+     * Publish all assets from the manifest
+     */
+    publish(): Promise<void>;
+    /**
+     * Build a single asset from the manifest
+     */
+    buildEntry(asset: IManifestEntry): Promise<boolean>;
+    /**
+     * Publish a single asset from the manifest
+     */
+    publishEntry(asset: IManifestEntry): Promise<boolean>;
+    /**
+     * Return whether a single asset is published
+     */
+    isEntryPublished(asset: IManifestEntry): Promise<boolean>;
+    /**
+     * publish an asset (used by 'publish()')
+     * @param asset The asset to publish
+     * @returns false when publishing should stop
+     */
+    private publishAsset;
+    get percentComplete(): number;
+    abort(): void;
+    get hasFailures(): boolean;
+    /**
+     * Publish a progress event to the listener, if present.
+     *
+     * Returns whether an abort is requested. Helper to get rid of repetitive code in publish().
+     */
+    private progressEvent;
+    private assetHandler;
+}

package/cdk-assets/publishing.js

@@ -0,0 +1,194 @@
+import { DockerFactory } from "./private/docker.js";
+import { makeAssetHandler } from "./private/handlers/index.js";
+import { EventType, } from "cdk-assets/lib/progress.js";
+export class AssetPublishing {
+    manifest;
+    options;
+    /**
+     * The message for the IPublishProgress interface
+     */
+    message = "Starting";
+    /**
+     * The current asset for the IPublishProgress interface
+     */
+    currentAsset;
+    failures = new Array();
+    assets;
+    totalOperations;
+    completedOperations = 0;
+    aborted = false;
+    handlerHost;
+    publishInParallel;
+    buildAssets;
+    publishAssets;
+    handlerCache = new Map();
+    constructor(manifest, options) {
+        this.manifest = manifest;
+        this.options = options;
+        this.assets = manifest.entries;
+        this.totalOperations = this.assets.length;
+        this.publishInParallel = options.publishInParallel ?? false;
+        this.buildAssets = options.buildAssets ?? true;
+        this.publishAssets = options.publishAssets ?? true;
+        const self = this;
+        this.handlerHost = {
+            aws: this.options.aws,
+            get aborted() {
+                return self.aborted;
+            },
+            emitMessage(t, m) {
+                self.progressEvent(t, m);
+            },
+            dockerFactory: new DockerFactory(),
+        };
+    }
+    /**
+     * Publish all assets from the manifest
+     */
+    async publish() {
+        if (this.publishInParallel) {
+            await Promise.all(this.assets.map(async (asset) => this.publishAsset(asset)));
+        }
+        else {
+            for (const asset of this.assets) {
+                if (!(await this.publishAsset(asset))) {
+                    break;
+                }
+            }
+        }
+        if ((this.options.throwOnError ?? true) && this.failures.length > 0) {
+            throw new Error(`Error publishing: ${this.failures.map((e) => e.error.message)}`);
+        }
+    }
+    /**
+     * Build a single asset from the manifest
+     */
+    async buildEntry(asset) {
+        try {
+            if (this.progressEvent(EventType.START, `Building ${asset.id}`)) {
+                return false;
+            }
+            const handler = this.assetHandler(asset);
+            await handler.build();
+            if (this.aborted) {
+                throw new Error("Aborted");
+            }
+            this.completedOperations++;
+            if (this.progressEvent(EventType.SUCCESS, `Built ${asset.id}`)) {
+                return false;
+            }
+        }
+        catch (e) {
+            this.failures.push({ asset, error: e });
+            this.completedOperations++;
+            if (this.progressEvent(EventType.FAIL, e.message)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Publish a single asset from the manifest
+     */
+    async publishEntry(asset) {
+        try {
+            if (this.progressEvent(EventType.START, `Publishing ${asset.id}`)) {
+                return false;
+            }
+            const handler = this.assetHandler(asset);
+            await handler.publish();
+            if (this.aborted) {
+                throw new Error("Aborted");
+            }
+            this.completedOperations++;
+            if (this.progressEvent(EventType.SUCCESS, `Published ${asset.id}`)) {
+                return false;
+            }
+        }
+        catch (e) {
+            this.failures.push({ asset, error: e });
+            this.completedOperations++;
+            if (this.progressEvent(EventType.FAIL, e.message)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Return whether a single asset is published
+     */
+    isEntryPublished(asset) {
+        const handler = this.assetHandler(asset);
+        return handler.isPublished();
+    }
+    /**
+     * publish an asset (used by 'publish()')
+     * @param asset The asset to publish
+     * @returns false when publishing should stop
+     */
+    async publishAsset(asset) {
+        try {
+            if (this.progressEvent(EventType.START, `Publishing ${asset.id}`)) {
+                return false;
+            }
+            const handler = this.assetHandler(asset);
+            if (this.buildAssets) {
+                await handler.build();
+            }
+            if (this.publishAssets) {
+                await handler.publish();
+            }
+            if (this.aborted) {
+                throw new Error("Aborted");
+            }
+            this.completedOperations++;
+            if (this.progressEvent(EventType.SUCCESS, `Published ${asset.id}`)) {
+                return false;
+            }
+        }
+        catch (e) {
+            this.failures.push({ asset, error: e });
+            this.completedOperations++;
+            if (this.progressEvent(EventType.FAIL, e.message)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    get percentComplete() {
+        if (this.totalOperations === 0) {
+            return 100;
+        }
+        return Math.floor((this.completedOperations / this.totalOperations) * 100);
+    }
+    abort() {
+        this.aborted = true;
+    }
+    get hasFailures() {
+        return this.failures.length > 0;
+    }
+    /**
+     * Publish a progress event to the listener, if present.
+     *
+     * Returns whether an abort is requested. Helper to get rid of repetitive code in publish().
+     */
+    progressEvent(event, message) {
+        this.message = message;
+        if (this.options.progressListener) {
+            this.options.progressListener.onPublishEvent(event, this);
+        }
+        return this.aborted;
+    }
+    assetHandler(asset) {
+        const existing = this.handlerCache.get(asset);
+        if (existing) {
+            return existing;
+        }
+        const ret = makeAssetHandler(this.manifest, asset, this.handlerHost, {
+            // TODO: remove after PR is merged
+            quiet: this.options.quiet,
+        });
+        this.handlerCache.set(asset, ret);
+        return ret;
+    }
+}

package/cli/commands/dev.js

@@ -62,19 +62,43 @@ export const dev = (program) => program.command(["dev", "start"], "Work on your
             Colors.line(prefix(evt.properties.requestID), Colors.dim.bold("Invoked"), Colors.dim(useFunctions().fromID(evt.properties.functionID)?.handler));
         });
         bus.subscribe("worker.stdout", async (evt) => {
+            const info = useFunctions().fromID(evt.properties.functionID);
             prefix(evt.properties.requestID);
             const { started } = pending.get(evt.properties.requestID);
             for (let line of evt.properties.message.split("\n")) {
+                // Remove prefix from container logs
+                if (info?.runtime === "container") {
+                    // handle Node.js container logs
+                    // ie. 2023-07-05T00:13:42.448Z\td7330533-2429-4871-a632-ed29a1d32246\tINFO\tfoo!
+                    const parts = line.split("\t");
+                    if (parts.length >= 4 &&
+                        Date.parse(parts[0]) &&
+                        parts[1].length === 36) {
+                        line = parts.slice(3).join("\t");
+                    }
+                }
                 Colors.line(prefix(evt.properties.requestID), Colors.dim(("+" + (Date.now() - started) + "ms").padEnd(7)), Colors.dim(line));
             }
         });
+        bus.subscribe("function.build.started", async (evt) => {
+            const info = useFunctions().fromID(evt.properties.functionID);
+            if (!info)
+                return;
+            if (info.enableLiveDev === false)
+                return;
+            if (info.runtime !== "container")
+                return;
+            Colors.line(Colors.dim(Colors.prefix, "Building", info.handler, "container"));
+        });
         bus.subscribe("function.build.success", async (evt) => {
             const info = useFunctions().fromID(evt.properties.functionID);
             if (!info)
                 return;
             if (info.enableLiveDev === false)
                 return;
-            Colors.line(Colors.dim(Colors.prefix, "Built", info.handler));
+            Colors.line(info.runtime === "container"
+                ? Colors.dim(Colors.prefix, "Built", info.handler, "container")
+                : Colors.dim(Colors.prefix, "Built", info.handler));
         });
         bus.subscribe("function.build.failed", async (evt) => {
             const info = useFunctions().fromID(evt.properties.functionID);

package/constructs/App.d.ts

@@ -42,7 +42,7 @@ export type AppProps = CDKAppProps;
  */
 export declare class App extends CDKApp {
     /**
-     * Whether or not the app is running locally under `sst start`
+     * Whether or not the app is running locally under `sst dev`
      */
     readonly local: boolean;
     /**

package/constructs/App.js

@@ -26,7 +26,7 @@ function exitWithMessage(message) {
  */
 export class App extends CDKApp {
     /**
-     * Whether or not the app is running locally under `sst start`
+     * Whether or not the app is running locally under `sst dev`
      */
     local = false;
     /**

package/constructs/Function.d.ts

@@ -11,6 +11,7 @@ import { Architecture, Function as CDKFunction, FunctionOptions, ILayerVersion,
 import { RetentionDays } from "aws-cdk-lib/aws-logs";
 import { Size as CDKSize, Duration as CDKDuration } from "aws-cdk-lib/core";
 declare const supportedRuntimes: {
+    container: CDKRuntime;
     rust: CDKRuntime;
     nodejs: CDKRuntime;
     "nodejs4.3": CDKRuntime;
@@ -26,6 +27,7 @@ declare const supportedRuntimes: {
     "python3.7": CDKRuntime;
     "python3.8": CDKRuntime;
     "python3.9": CDKRuntime;
+    "python3.10": CDKRuntime;
     "dotnetcore1.0": CDKRuntime;
     "dotnetcore2.0": CDKRuntime;
     "dotnetcore2.1": CDKRuntime;
@@ -76,6 +78,9 @@ export interface FunctionProps extends Omit<FunctionOptions, "functionName" | "m
      * Used to configure python function properties
      */
     python?: PythonProps;
+    /**
+     * Used to configure image function properties
+     */
     /**
      * Hooks to run before and after function builds
      */