sfnkit 0.1.0__py3-none-any.whl

sfnkit/__init__.py

@@ -0,0 +1,67 @@
+"""sfnkit: author AWS Step Functions workflows as decorated Python functions."""
+
+from sfnkit.blocks import (
+    Err,
+    Ok,
+    S3File,
+    S3Objects,
+    States,
+    attempt,
+    each,
+    fail,
+    is_null,
+    length,
+    one_of,
+    subworkflow,
+    wait,
+)
+from sfnkit.compute import ComputeBackend, ComputeSpec, Lambda, register_backend
+from sfnkit.errors import CompileError, DeployError, LintError, SfnkitError, TraceError
+from sfnkit.iam import Allow, Deny, Policy
+from sfnkit.local import WorkflowFailed
+from sfnkit.step import Retry, Step, step
+from sfnkit.workflow import Workflow, workflow
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Allow",
+    "CompileError",
+    "ComputeBackend",
+    "ComputeSpec",
+    "Deny",
+    "DeployError",
+    "Err",
+    "Lambda",
+    "LintError",
+    "Ok",
+    "Policy",
+    "Retry",
+    "S3File",
+    "S3Objects",
+    "States",
+    "Step",
+    "SfnkitError",
+    "TraceError",
+    "Workflow",
+    "WorkflowFailed",
+    "attempt",
+    "each",
+    "fail",
+    "is_null",
+    "length",
+    "one_of",
+    "register_backend",
+    "step",
+    "subworkflow",
+    "wait",
+    "workflow",
+]
+
+
+def deploy(workflow, **kwargs):
+    """Build, push and deploy a workflow. See ``sfnkit.deploy.deployer.deploy``."""
+    from sfnkit.compile.app import build_app
+    from sfnkit.deploy.deployer import deploy as _deploy
+
+    return _deploy(build_app(workflow), **kwargs)

sfnkit/app.py

@@ -0,0 +1,118 @@
+"""The compiled form of a workflow: everything CloudFormation assembly and the deployer need.
+
+``sfnkit.compile.build_app(workflow)`` produces an ``App``. Downstream code (``compile/cfn.py``,
+``build/``, ``deploy/``) reads only these types, never the tracer or IR.
+
+Conventions shared with CloudFormation assembly:
+  * State machine definitions use ``DefinitionSubstitutions`` placeholders. ``StateMachineSpec
+    .substitutions`` already maps every placeholder it uses to a CFN intrinsic, except that the
+    intrinsics may reference the template parameter ``ArtifactsBucket`` (the assets stack's bucket name)
+    and logical ids of functions in ``App.functions``.
+  * A state machine's full name is ``${AWS::StackName}-<StateMachineSpec.name>`` so ARNs can be built
+    without a CloudFormation dependency cycle (``sm_arn``).
+  * Each ``ImageVariant`` becomes a template parameter ``ImageVariant.param_name`` holding the image URI.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal
+
+from sfnkit.compute.base import ComputeSpec
+from sfnkit.iam import Policy
+
+if TYPE_CHECKING:
+    from sfnkit.step import Step
+    from sfnkit.workflow import Workflow
+
+
+@dataclass(eq=False)
+class ImageVariant:
+    key: str  # short content hash of the dependency set
+    deps: tuple[str, ...]
+    system_packages: tuple[str, ...]
+    python: str  # e.g. "3.14"
+    architecture: str  # "x86_64" | "arm64"
+
+    @property
+    def param_name(self) -> str:
+        return f"Image{self.key}Uri"
+
+
+@dataclass(eq=False)
+class FunctionSpec:
+    """One deployed unit of compute: a step with its resolved compute, IAM policy and image."""
+
+    logical_id: str
+    step: Step
+    compute: ComputeSpec
+    policy: Policy  # the user's effective policy (base set or the step's own); baseline added at synth
+    image: ImageVariant
+    env: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass(eq=False)
+class SmNeed:
+    """Something a state machine role must be allowed to do. CFN assembly turns these into statements."""
+
+    kind: Literal["invoke", "start_child", "map_run", "bucket", "s3_read"]
+    fn: FunctionSpec | None = None  # invoke
+    child: str | None = None  # start_child: StateMachineSpec.name of the child
+    bucket: str | None = None  # s3_read: a user bucket name
+    prefix: str | None = None  # s3_read: key prefix ("" for the whole bucket)
+
+
+@dataclass(eq=False)
+class StateMachineSpec:
+    logical_id: str
+    name: str  # suffix of the full name (see module docstring)
+    definition: dict[str, Any]
+    substitutions: dict[str, Any]
+    type: Literal["STANDARD", "EXPRESS"]
+    needs: list[SmNeed]
+    root: bool = False
+
+
+@dataclass(eq=False)
+class AppSettings:
+    offload_threshold: int = 32 * 1024
+    log_retention_days: int = 14
+    run_retention_days: int = 30
+    python: str = "3.14"
+    architecture: str = "x86_64"
+
+
+@dataclass(eq=False)
+class App:
+    name: str  # sanitized root workflow name; the default stack name is f"sfnkit-{name}"
+    module: str  # importable module that defines the root workflow (and registers its steps)
+    workflow: Workflow
+    functions: list[FunctionSpec]
+    state_machines: list[StateMachineSpec]  # root first
+    images: list[ImageVariant]
+    settings: AppSettings
+    project_dir: Path
+    exclude: tuple[str, ...] = ()
+
+    @property
+    def root(self) -> StateMachineSpec:
+        return self.state_machines[0]
+
+
+def sm_full_name(name: str) -> dict:
+    return {"Fn::Sub": f"${{AWS::StackName}}-{name}"}
+
+
+def sm_arn(name: str) -> dict:
+    return {
+        "Fn::Sub": "arn:${AWS::Partition}:states:${AWS::Region}:${AWS::AccountId}:stateMachine:"
+        f"${{AWS::StackName}}-{name}"
+    }
+
+
+def sm_execution_arn(name: str) -> dict:
+    return {
+        "Fn::Sub": "arn:${AWS::Partition}:states:${AWS::Region}:${AWS::AccountId}:execution:"
+        f"${{AWS::StackName}}-{name}:*"
+    }

sfnkit/blocks.py

@@ -0,0 +1,250 @@
+"""Control-flow building blocks used inside a ``@workflow`` body.
+
+* ``match attempt(fn, ...)`` with ``case Ok(value)`` / ``case Err(error, cause)``: error handling.
+* ``with each(items) as it:`` ... ``it.result(x)`` ... ``it.value``: a Map with a multi-step body.
+* ``with subworkflow("name"):``: run the block as a child state machine.
+* ``wait(seconds=...)`` / ``wait(until=...)``, ``fail(error, cause)``.
+* ``is_null``, ``one_of``, ``length``: conditions that Python syntax can't express on a traced value.
+"""
+
+from __future__ import annotations
+
+import sys
+from dataclasses import dataclass
+from typing import Any
+
+from sfnkit.errors import TraceError
+from sfnkit.expr import Expr, Get, IsNull, IsOk, Length, Lit, Not, OneOf, OutcomeGet, _decide, lift
+from sfnkit.iam import Policy
+from sfnkit.trace.tracer import TNode, require_tracer
+
+
+class States:
+    """Step Functions' built-in error names, for ``Retry(on=[...])`` and ``Err(...)`` patterns."""
+
+    ALL = "States.ALL"
+    TIMEOUT = "States.Timeout"
+    TASK_FAILED = "States.TaskFailed"
+    PERMISSIONS = "States.Permissions"
+    HEARTBEAT_TIMEOUT = "States.HeartbeatTimeout"
+
+
+# attempt / Ok / Err
+
+
+class _OutcomeMeta(type):
+    def __instancecheck__(cls, inst):
+        if isinstance(inst, OutcomeGet):
+            cond = IsOk(Get(inst.src)) if cls is Ok else Not(IsOk(Get(inst.src)))
+            return _decide(cond)
+        if isinstance(inst, Expr):
+            raise TraceError("Ok(...)/Err(...) patterns only match the outcome of attempt(...)")
+        return type.__instancecheck__(cls, inst)
+
+
+class Ok(metaclass=_OutcomeMeta):
+    """``case Ok(value):`` the attempted work succeeded with ``value``."""
+
+    __match_args__ = ("value",)
+
+    def __init__(self, value: Any = None):
+        self.value = value
+
+    def __repr__(self):
+        return f"Ok({self.value!r})"
+
+
+class Err(metaclass=_OutcomeMeta):
+    """``case Err(error, cause):`` the attempted work failed. ``error`` is the error name (a Python
+    exception's class name), ``cause`` its details (a dict for Python exceptions)."""
+
+    __match_args__ = ("error", "cause")
+
+    def __init__(self, error: str = "", cause: Any = None):
+        self.error, self.cause = error, cause
+
+    def __repr__(self):
+        return f"Err({self.error!r}, {self.cause!r})"
+
+
+def attempt(fn, *args, **kwargs) -> OutcomeGet:
+    """Run a step, a workflow or a plain function (a multi-step block) and catch its errors.
+
+    Must be the subject of a ``match`` directly in the workflow body. Errors that no ``Err`` case
+    matches are not caught and fail the workflow as they would without ``attempt``.
+    """
+    t = require_tracer("attempt()")
+    return t.attempt(fn, args, kwargs, sys._getframe(1))
+
+
+# each / map
+
+
+class each:
+    """``with each(items, distributed=False, max_concurrency=None) as it:`` run the block per item.
+
+    Inside, ``it.item`` is the current item; call ``it.result(x)`` with the per-item value. After the
+    block, ``it.value`` is the list of results in item order.
+    """
+
+    def __init__(
+        self,
+        items,
+        *,
+        distributed: bool = False,
+        max_concurrency: int | None = None,
+        tolerated_failure_percentage: float | None = None,
+        tolerated_failure_count: int | None = None,
+        child_type: str = "STANDARD",
+    ):
+        self._frame = sys._getframe(1)
+        self._items = items
+        self._opts = dict(
+            distributed=distributed,
+            max_concurrency=max_concurrency,
+            tolerated_failure_percentage=tolerated_failure_percentage,
+            tolerated_failure_count=tolerated_failure_count,
+            child_type=child_type,
+        )
+        self._node: TNode | None = None
+        self._result: Expr | None = None
+
+    def __enter__(self) -> each:
+        t = require_tracer("each()")
+        self._node = t.each_enter(self._items, self._opts, self._frame)
+        del self._frame
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        if exc_type is None:
+            require_tracer("each()").each_exit(self._node, self._result)  # type: ignore[arg-type]
+        return False
+
+    @property
+    def item(self) -> Get:
+        if self._node is None or self._node.closed:
+            raise TraceError("it.item is only available inside the `with each(...)` block")
+        return Get(self._node.item)
+
+    def result(self, value) -> None:
+        if self._node is None or self._node.closed:
+            raise TraceError("it.result() must be called inside the `with each(...)` block")
+        self._result = lift(value)
+
+    @property
+    def value(self) -> Get:
+        if self._node is None or not self._node.closed:
+            raise TraceError("it.value is only available after the `with each(...)` block")
+        return Get(self._node)
+
+
+def _map_call(target, items, kwargs: dict, opts: dict, frame) -> Get:
+    """``step.map(items, **kwargs)`` / ``workflow.map(...)``: a Map whose body is one call."""
+    from sfnkit.workflow import Workflow
+
+    t = require_tracer(f"{target.name}.map()")
+    n = t.each_enter(items, opts, frame)
+    param = target.item_param(kwargs)
+    bound = target.bind((), {**kwargs, param: Get(n.item)})
+    r = t.call(target, bound, frame) if isinstance(target, Workflow) else t.task(target, bound, frame)
+    t.each_exit(n, r)
+    return Get(n)
+
+
+@dataclass(frozen=True)
+class S3Objects:
+    """Distributed Map items: the objects under ``s3://bucket/prefix`` (each item is a listing entry
+    with ``Key``, ``Size``, ``ETag``, ``LastModified``)."""
+
+    bucket: str
+    prefix: str = ""
+
+
+@dataclass(frozen=True)
+class S3File:
+    """Distributed Map items read from one S3 object: a JSON array, JSON Lines, or CSV with a header."""
+
+    bucket: str
+    key: str
+    format: str = "json"
+
+    def __post_init__(self):
+        if self.format not in ("json", "jsonl", "csv"):
+            raise ValueError("S3File format must be json, jsonl or csv")
+
+
+# subworkflow
+
+
+class subworkflow:
+    """``with subworkflow("name", iam=None, type="STANDARD"):`` run the block as a child state machine.
+
+    Values created inside can be used after the block. Steps inside without their own ``iam=``
+    inherit this block's ``iam`` if given, otherwise the enclosing workflow's.
+    """
+
+    def __init__(self, name: str, *, iam: Policy | None = None, type: str = "STANDARD"):
+        if type not in ("STANDARD", "EXPRESS"):
+            raise ValueError("type must be STANDARD or EXPRESS")
+        self._frame = sys._getframe(1)
+        self._opts = {"name": name, "iam": iam, "type": type}
+        self._node: TNode | None = None
+
+    def __enter__(self):
+        self._node = require_tracer("subworkflow()").sub_enter(self._opts, self._frame)
+        del self._frame
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        if exc_type is None:
+            require_tracer("subworkflow()").sub_exit(self._node)  # type: ignore[arg-type]
+        return False
+
+
+# wait / fail
+
+
+def wait(*, seconds: int | Expr | None = None, until: str | Expr | None = None) -> None:
+    """Pause the workflow. Everything traced before runs first; everything after waits."""
+    require_tracer("wait()").wait(seconds, until, sys._getframe(1))
+
+
+def fail(error: str | Expr = "Failed", cause: str | Expr = "") -> None:
+    """End the workflow (or the enclosing attempt) with an error."""
+    require_tracer("fail()").fail(error, cause, sys._getframe(1))
+
+
+# condition helpers
+
+
+def is_null(value) -> IsNull:
+    return IsNull(lift(value))
+
+
+def one_of(value, values) -> OneOf:
+    vals = tuple(values)
+    for v in vals:
+        lift(v)
+    return OneOf(lift(value), vals)
+
+
+def length(value) -> Length:
+    return Length(lift(value))
+
+
+__all__ = [
+    "Err",
+    "Lit",
+    "Ok",
+    "S3File",
+    "S3Objects",
+    "States",
+    "attempt",
+    "each",
+    "fail",
+    "is_null",
+    "length",
+    "one_of",
+    "subworkflow",
+    "wait",
+]

sfnkit/build/__init__.py

@@ -0,0 +1,5 @@
+from sfnkit.build.context import BuildContext, prepare_context
+from sfnkit.build.docker import Builder, DockerBuilder
+from sfnkit.build.dockerfile import UV_VERSION, render_dockerfile
+
+__all__ = ["UV_VERSION", "BuildContext", "Builder", "DockerBuilder", "prepare_context", "render_dockerfile"]

sfnkit/build/context.py

@@ -0,0 +1,139 @@
+"""Assemble a Docker build context for one image variant.
+
+Layout under ``<project>/.sfnkit/build/<variant key>/`` (never /tmp: contexts can be large and /tmp is
+RAM on some hosts)::
+
+    Dockerfile
+    requirements.txt
+    sfnkit/      the installed sfnkit package (works for PyPI and editable installs alike)
+    project/       the import roots of the workflow module and its steps, merged
+
+The tag is a hash of everything in the context, so an unchanged project rebuilds to the same tag and the
+push is skipped.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import hashlib
+import importlib
+import importlib.metadata
+import os
+import shutil
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+from sfnkit.app import App, ImageVariant
+from sfnkit.build.dockerfile import docker_platform, render_dockerfile, render_requirements
+
+DEFAULT_EXCLUDE_NAMES = frozenset(
+    {
+        ".git",
+        ".venv",
+        "venv",
+        "__pycache__",
+        ".sfnkit",
+        "node_modules",
+        ".pytest_cache",
+        ".mypy_cache",
+        ".ruff_cache",
+        ".tox",
+        ".idea",
+        ".vscode",
+        ".DS_Store",
+    }
+)
+DEFAULT_EXCLUDE_GLOBS = ("*.pyc", "*.pyo")
+
+
+@dataclass
+class BuildContext:
+    image: ImageVariant
+    path: Path
+    tag: str
+    platform: str
+
+
+def sfnkit_version() -> str:
+    try:
+        return importlib.metadata.version("sfnkit")
+    except importlib.metadata.PackageNotFoundError:
+        return "0+unknown"
+
+
+def sfnkit_package_dir() -> Path:
+    mod = importlib.import_module("sfnkit")
+    return Path(next(iter(mod.__path__))).resolve()
+
+
+def _ignore(root: Path, user_globs: tuple[str, ...]):
+    def ignore(directory: str, names: list[str]) -> set[str]:
+        rel_dir = Path(directory).resolve().relative_to(root)
+        out = set()
+        for name in names:
+            rel = (rel_dir / name).as_posix()
+            if (
+                name in DEFAULT_EXCLUDE_NAMES
+                or any(fnmatch.fnmatch(name, g) for g in DEFAULT_EXCLUDE_GLOBS)
+                or any(fnmatch.fnmatch(rel, g) or fnmatch.fnmatch(name, g) for g in user_globs)
+            ):
+                out.add(name)
+        return out
+
+    return ignore
+
+
+def import_roots(app: App, image: ImageVariant) -> list[Path]:
+    """Directories that must sit on sys.path for the workflow module and these steps to import."""
+    project = app.project_dir.resolve()
+    names = {app.module} | {fn.step.fn.__module__ for fn in app.functions if fn.image is image}
+    roots: list[Path] = []
+    for name in sorted(names):
+        if name == "__main__":
+            continue
+        mod = sys.modules.get(name) or importlib.import_module(name)
+        file = getattr(mod, "__file__", None)
+        if not file:
+            continue
+        p = Path(file).resolve()
+        depth = name.count(".") + (1 if p.name == "__init__.py" else 0)
+        root = p.parents[depth]
+        if root.is_relative_to(project) and root not in roots:
+            roots.append(root)
+    return roots or [project]
+
+
+def content_hash(path: Path, extra: str = "") -> str:
+    h = hashlib.sha256(extra.encode())
+    for f in sorted(p for p in path.rglob("*") if p.is_file()):
+        h.update(f.relative_to(path).as_posix().encode() + b"\0")
+        h.update(f.read_bytes())
+        h.update(b"\0")
+    return h.hexdigest()[:16]
+
+
+def prepare_context(app: App, image: ImageVariant) -> BuildContext:
+    ctx_dir = app.project_dir.resolve() / ".sfnkit" / "build" / image.key
+    if ctx_dir.exists():
+        shutil.rmtree(ctx_dir)
+    ctx_dir.mkdir(parents=True)
+    (ctx_dir / "Dockerfile").write_text(render_dockerfile(image))
+    (ctx_dir / "requirements.txt").write_text(render_requirements(image))
+
+    pkg = sfnkit_package_dir()
+    shutil.copytree(pkg, ctx_dir / "sfnkit", ignore=_ignore(pkg, ()))
+
+    project_out = ctx_dir / "project"
+    project_out.mkdir()
+    user_globs = tuple(app.exclude)
+    for root in import_roots(app, image):
+        shutil.copytree(root, project_out, dirs_exist_ok=True, ignore=_ignore(root, user_globs))
+
+    extra = os.pathsep.join([sfnkit_version(), image.python, image.architecture])
+    return BuildContext(
+        image=image,
+        path=ctx_dir,
+        tag=content_hash(ctx_dir, extra),
+        platform=docker_platform(image.architecture),
+    )

sfnkit/build/docker.py

@@ -0,0 +1,75 @@
+"""Build an image variant and push it to ECR. ``Builder`` is the seam for other builders later."""
+
+from __future__ import annotations
+
+import base64
+import subprocess
+from collections.abc import Callable
+from typing import Protocol
+
+from sfnkit.build.context import BuildContext
+from sfnkit.errors import DeployError
+
+
+class Builder(Protocol):
+    def ensure_image(self, context: BuildContext, repository_uri: str, ecr) -> str:
+        """Make ``<repository_uri>:<context.tag>`` exist in ECR and return that URI."""
+        ...
+
+
+def image_exists(ecr, repository_name: str, tag: str) -> bool:
+    try:
+        resp = ecr.describe_images(repositoryName=repository_name, imageIds=[{"imageTag": tag}])
+    except ecr.exceptions.ImageNotFoundException:
+        return False
+    return bool(resp.get("imageDetails"))
+
+
+class DockerBuilder:
+    def __init__(self, docker: str = "docker", run: Callable = subprocess.run, log: Callable = print):
+        self.docker = docker
+        self._run_fn = run
+        self.log = log
+        self._logged_in: set[str] = set()
+
+    def _run(self, args: list[str], *, input: bytes | None = None) -> None:
+        proc = self._run_fn(args, input=input, capture_output=True)
+        if proc.returncode != 0:
+            err = (proc.stderr or b"").decode(errors="replace").strip()[-4000:]
+            raise DeployError(f"`{' '.join(args[:3])} ...` failed ({proc.returncode}):\n{err}")
+
+    def login(self, ecr) -> None:
+        data = ecr.get_authorization_token()["authorizationData"][0]
+        endpoint = data["proxyEndpoint"]
+        if endpoint in self._logged_in:
+            return
+        user, _, password = base64.b64decode(data["authorizationToken"]).decode().partition(":")
+        self._run(
+            [self.docker, "login", "--username", user, "--password-stdin", endpoint], input=password.encode()
+        )
+        self._logged_in.add(endpoint)
+
+    def ensure_image(self, context: BuildContext, repository_uri: str, ecr) -> str:
+        uri = f"{repository_uri}:{context.tag}"
+        repo_name = repository_uri.split("/", 1)[1]
+        if image_exists(ecr, repo_name, context.tag):
+            self.log(f"image {context.image.key}: {context.tag} already in ECR, skipping build")
+            return uri
+        self.login(ecr)
+        self.log(f"image {context.image.key}: building {context.tag} ({context.platform})")
+        # --provenance=false: Lambda rejects the image index that build attestations create.
+        self._run(
+            [
+                self.docker,
+                "build",
+                "--platform",
+                context.platform,
+                "--provenance=false",
+                "-t",
+                uri,
+                str(context.path),
+            ]
+        )
+        self.log(f"image {context.image.key}: pushing")
+        self._run([self.docker, "push", uri])
+        return uri