sfnkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,56 @@
1
+ """Dockerfile for one image variant: the AWS Lambda Python base image plus the variant's deps."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from sfnkit.app import ImageVariant
6
+ from sfnkit.compute.lambda_ import HANDLER
7
+
8
+ # Pinned so a content hash means the same image every time.
9
+ UV_VERSION = "0.13.7"
10
+
11
+ _PLATFORMS = {
12
+ "x86_64": ("linux/amd64", "x86_64-manylinux_2_28"),
13
+ "arm64": ("linux/arm64", "aarch64-manylinux_2_28"),
14
+ }
15
+
16
+
17
+ def docker_platform(architecture: str) -> str:
18
+ try:
19
+ return _PLATFORMS[architecture][0]
20
+ except KeyError:
21
+ raise ValueError(f"unsupported architecture {architecture!r} (x86_64 or arm64)") from None
22
+
23
+
24
+ def render_dockerfile(image: ImageVariant, uv_version: str = UV_VERSION) -> str:
25
+ if image.architecture not in _PLATFORMS:
26
+ raise ValueError(f"unsupported architecture {image.architecture!r} (x86_64 or arm64)")
27
+ py_platform = _PLATFORMS[image.architecture][1]
28
+ lines = [
29
+ "# Generated by sfnkit. Do not edit; it is rewritten on every build.",
30
+ f"FROM public.ecr.aws/lambda/python:{image.python}",
31
+ ]
32
+ if image.system_packages:
33
+ pkgs = " ".join(image.system_packages)
34
+ # Lambda's AL2023 base images ship microdnf, not dnf/yum.
35
+ lines.append(f"RUN microdnf install -y {pkgs} && microdnf clean all")
36
+ lines.append(f"COPY --from=ghcr.io/astral-sh/uv:{uv_version} /uv /bin/uv")
37
+ if image.deps:
38
+ lines += [
39
+ "COPY requirements.txt /opt/sfnkit-build/requirements.txt",
40
+ (
41
+ 'RUN uv pip install --no-cache --target "${LAMBDA_TASK_ROOT}" '
42
+ f"--python-version {image.python} --python-platform {py_platform} "
43
+ "-r /opt/sfnkit-build/requirements.txt"
44
+ ),
45
+ ]
46
+ lines += [
47
+ "COPY sfnkit/ ${LAMBDA_TASK_ROOT}/sfnkit/",
48
+ "COPY project/ ${LAMBDA_TASK_ROOT}/",
49
+ f'CMD ["{HANDLER}"]',
50
+ "",
51
+ ]
52
+ return "\n".join(lines)
53
+
54
+
55
+ def render_requirements(image: ImageVariant) -> str:
56
+ return "".join(f"{d}\n" for d in image.deps)
sfnkit/cli.py ADDED
@@ -0,0 +1,170 @@
1
+ """``sfnkit synth|deploy|run|local|destroy module:workflow``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import importlib
7
+ import json
8
+ import sys
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from sfnkit.errors import SfnkitError
13
+
14
+
15
+ def load_workflow(target: str, project_dir: Path):
16
+ module, sep, attr = target.partition(":")
17
+ if not sep or not module or not attr:
18
+ raise SystemExit(f"expected module:workflow, got {target!r}")
19
+ root = str(project_dir.resolve())
20
+ if root not in sys.path:
21
+ sys.path.insert(0, root)
22
+ mod = importlib.import_module(module)
23
+ obj = mod
24
+ for part in attr.split("."):
25
+ obj = getattr(obj, part)
26
+ return obj
27
+
28
+
29
+ def parse_input(raw: str | None) -> Any:
30
+ if raw is None:
31
+ return {}
32
+ if raw.startswith("@"):
33
+ raw = Path(raw[1:]).read_text()
34
+ try:
35
+ return json.loads(raw)
36
+ except ValueError as e:
37
+ raise SystemExit(f"--input is not valid JSON: {e}") from None
38
+
39
+
40
+ def build_parser() -> argparse.ArgumentParser:
41
+ p = argparse.ArgumentParser(prog="sfnkit", description="Author AWS Step Functions workflows in Python")
42
+ sub = p.add_subparsers(dest="command", required=True)
43
+
44
+ def common(sp, aws: bool):
45
+ sp.add_argument("target", help="module:workflow, e.g. myapp.flows:pipeline")
46
+ sp.add_argument("--project-dir", type=Path, default=Path.cwd(), help="project root (default: cwd)")
47
+ if aws:
48
+ sp.add_argument("--region")
49
+ sp.add_argument("--profile")
50
+ sp.add_argument("--stack-name")
51
+
52
+ sp = sub.add_parser("synth", help="write templates, state machine definitions and Dockerfiles")
53
+ common(sp, aws=False)
54
+ sp.add_argument("-o", "--out", type=Path, default=Path("sfnkit.out"))
55
+
56
+ sp = sub.add_parser("deploy", help="build images and deploy both stacks")
57
+ common(sp, aws=True)
58
+
59
+ sp = sub.add_parser("run", help="start an execution of the deployed workflow")
60
+ common(sp, aws=True)
61
+ sp.add_argument("--input", help="JSON input, or @file.json")
62
+ sp.add_argument("--no-wait", action="store_true", help="print the execution ARN and return")
63
+ sp.add_argument("--timeout", type=float)
64
+
65
+ sp = sub.add_parser("local", help="run the workflow in-process")
66
+ common(sp, aws=False)
67
+ sp.add_argument("--input", help="JSON input, or @file.json")
68
+
69
+ sp = sub.add_parser("destroy", help="delete both stacks and the run artifacts")
70
+ common(sp, aws=True)
71
+ return p
72
+
73
+
74
+ def _session(args):
75
+ import boto3
76
+
77
+ return boto3.Session(region_name=args.region, profile_name=args.profile)
78
+
79
+
80
+ def _stack(args, app) -> str:
81
+ from sfnkit.deploy.deployer import default_stack_name
82
+
83
+ return args.stack_name or default_stack_name(app)
84
+
85
+
86
+ def cmd_synth(args) -> int:
87
+ from sfnkit.build.dockerfile import render_dockerfile
88
+ from sfnkit.compile.app import build_app
89
+ from sfnkit.compile.cfn import assets_template, main_template
90
+
91
+ app = build_app(load_workflow(args.target, args.project_dir), project_dir=args.project_dir)
92
+ out: Path = args.out
93
+ out.mkdir(parents=True, exist_ok=True)
94
+ (out / "template.json").write_text(json.dumps(main_template(app), indent=2))
95
+ (out / "assets.json").write_text(json.dumps(assets_template(app), indent=2))
96
+ for sm in app.state_machines:
97
+ (out / f"{sm.name}.asl.json").write_text(json.dumps(sm.definition, indent=2))
98
+ for image in app.images:
99
+ (out / f"Dockerfile.{image.key}").write_text(render_dockerfile(image))
100
+ print(f"wrote {out}/")
101
+ return 0
102
+
103
+
104
+ def cmd_deploy(args) -> int:
105
+ from sfnkit.compile.app import build_app
106
+ from sfnkit.deploy.deployer import deploy
107
+
108
+ app = build_app(load_workflow(args.target, args.project_dir), project_dir=args.project_dir)
109
+ res = deploy(app, stack_name=_stack(args, app), session=_session(args))
110
+ print(json.dumps({"stack": res.stack_name, "state_machine": res.state_machine_arn, "bucket": res.bucket}))
111
+ return 0
112
+
113
+
114
+ def cmd_run(args) -> int:
115
+ from sfnkit.compile.app import build_app
116
+ from sfnkit.deploy import client
117
+ from sfnkit.deploy.deployer import stack_outputs
118
+
119
+ app = build_app(load_workflow(args.target, args.project_dir), project_dir=args.project_dir)
120
+ session = _session(args)
121
+ outputs = stack_outputs(session.client("cloudformation"), _stack(args, app))
122
+ arn = outputs["StateMachineArn"]
123
+ payload = parse_input(args.input)
124
+ express = outputs.get("StateMachineType") == "EXPRESS"
125
+ if args.no_wait and not express:
126
+ print(client.start(arn, payload, session=session))
127
+ return 0
128
+ result = client.run(arn, payload, express=express, timeout=args.timeout, session=session)
129
+ print(json.dumps(result, indent=2, default=str))
130
+ return 0
131
+
132
+
133
+ def cmd_local(args) -> int:
134
+ wf = load_workflow(args.target, args.project_dir)
135
+ payload = parse_input(args.input)
136
+ if not isinstance(payload, dict):
137
+ raise SystemExit("--input for `local` must be a JSON object of workflow arguments")
138
+ print(json.dumps(wf.local(**payload), indent=2, default=str))
139
+ return 0
140
+
141
+
142
+ def cmd_destroy(args) -> int:
143
+ from sfnkit.compile.app import build_app
144
+ from sfnkit.deploy.deployer import destroy
145
+
146
+ app = build_app(load_workflow(args.target, args.project_dir), project_dir=args.project_dir)
147
+ destroy(app, stack_name=_stack(args, app), session=_session(args))
148
+ return 0
149
+
150
+
151
+ COMMANDS = {
152
+ "synth": cmd_synth,
153
+ "deploy": cmd_deploy,
154
+ "run": cmd_run,
155
+ "local": cmd_local,
156
+ "destroy": cmd_destroy,
157
+ }
158
+
159
+
160
+ def main(argv: list[str] | None = None) -> int:
161
+ args = build_parser().parse_args(argv)
162
+ try:
163
+ return COMMANDS[args.command](args)
164
+ except SfnkitError as e:
165
+ print(f"sfnkit: {e}", file=sys.stderr)
166
+ return 1
167
+
168
+
169
+ if __name__ == "__main__":
170
+ sys.exit(main())
@@ -0,0 +1 @@
1
+ """Compilers: graph -> ASL (asl.py), App assembly (app.py), CloudFormation (cfn.py)."""
sfnkit/compile/app.py ADDED
@@ -0,0 +1,164 @@
1
+ """Assemble an ``App`` from a root workflow: functions, images and every state machine it needs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import re
8
+ from pathlib import Path
9
+
10
+ from sfnkit.app import App, AppSettings, FunctionSpec, ImageVariant, SmNeed, StateMachineSpec
11
+ from sfnkit.compile.asl import Compiler, Machine, render_fixpoint
12
+ from sfnkit.compute.lambda_ import Lambda
13
+ from sfnkit.errors import CompileError
14
+ from sfnkit.ir import Graph, SubN, TaskN
15
+
16
+
17
+ def _h(*parts) -> str:
18
+ return hashlib.sha256(json.dumps(parts, sort_keys=True, default=str).encode()).hexdigest()
19
+
20
+
21
+ def _camel(s: str) -> str:
22
+ return "".join(p[:1].upper() + p[1:] for p in re.split(r"[^A-Za-z0-9]+", s) if p) or "X"
23
+
24
+
25
+ def _sm_name(s: str) -> str:
26
+ return (re.sub(r"[^A-Za-z0-9-]+", "-", s).strip("-") or "child")[:24]
27
+
28
+
29
+ def build_app(workflow, project_dir: Path | None = None) -> App:
30
+ if workflow.module == "__main__":
31
+ raise CompileError(
32
+ f"workflow {workflow.name} is defined in __main__; put it in an importable module and refer to "
33
+ "it as module:attr so the deployed runtime can import it"
34
+ )
35
+ return _Builder(workflow, Path(project_dir or Path.cwd())).build()
36
+
37
+
38
+ class _Builder:
39
+ def __init__(self, root, project_dir: Path):
40
+ self.root = root
41
+ self.project_dir = project_dir
42
+ self.settings = AppSettings(
43
+ offload_threshold=root.offload_threshold, python=root.python, architecture=root.architecture
44
+ )
45
+ self.functions: dict[tuple, FunctionSpec] = {}
46
+ self.images: dict[str, ImageVariant] = {}
47
+ self.owner: dict[int, object] = {}
48
+ self.children: dict[int, tuple[str, object]] = {}
49
+ self.pending: list = []
50
+ self.sub_names: dict[int, str] = {}
51
+ self.used_names: set[str] = {"main"}
52
+ self.compiler = Compiler(self.function_for, self.child_machine_for, self.sub_name_for)
53
+
54
+ def unique(self, name: str) -> str:
55
+ base, n = name, 2
56
+ while name in self.used_names:
57
+ suffix = f"-{n}"
58
+ name = base[: 24 - len(suffix)] + suffix
59
+ n += 1
60
+ self.used_names.add(name)
61
+ return name
62
+
63
+ def note_owner(self, g: Graph) -> None:
64
+ for t in g.tasks():
65
+ self.owner[id(t)] = g.workflow
66
+
67
+ def child_machine_for(self, wf):
68
+ if id(wf) not in self.children:
69
+ if wf is self.root:
70
+ raise CompileError(f"workflow {wf.name} calls itself")
71
+ self.children[id(wf)] = (self.unique(_sm_name(wf.name)), wf)
72
+ self.pending.append(wf)
73
+ return self.children[id(wf)]
74
+
75
+ def sub_name_for(self, s: SubN) -> str:
76
+ if id(s) not in self.sub_names:
77
+ self.sub_names[id(s)] = self.unique(_sm_name(s.name))
78
+ return self.sub_names[id(s)]
79
+
80
+ def image_for(self, deps: list[str], system: list[str]) -> ImageVariant:
81
+ key = _h(deps, system, self.settings.python, self.settings.architecture)[:12]
82
+ if key not in self.images:
83
+ self.images[key] = ImageVariant(
84
+ key=key,
85
+ deps=tuple(deps),
86
+ system_packages=tuple(system),
87
+ python=self.settings.python,
88
+ architecture=self.settings.architecture,
89
+ )
90
+ return self.images[key]
91
+
92
+ def function_for(self, t: TaskN) -> FunctionSpec:
93
+ owner = self.owner.get(id(t), self.root)
94
+ step = t.step
95
+ compute = step.compute or getattr(owner, "default_compute", None) or self.root.default_compute or Lambda()
96
+ deps = sorted(set(self.root.deps) | set(getattr(owner, "deps", ())) | set(step.deps))
97
+ system = sorted(
98
+ set(self.root.system_packages) | set(getattr(owner, "system_packages", ())) | set(step.system_packages)
99
+ )
100
+ image = self.image_for(deps, system)
101
+ key = (step.key, t.policy.digest(), repr(compute), image.key)
102
+ fn = self.functions.get(key)
103
+ if fn is None:
104
+ lid = f"Fn{_camel(step.name)[:40]}{_h(*key)[:6]}"
105
+ fn = FunctionSpec(logical_id=lid, step=step, compute=compute, policy=t.policy, image=image,
106
+ env=dict(step.env))
107
+ self.functions[key] = fn
108
+ return fn
109
+
110
+ def build(self) -> App:
111
+ c = self.compiler
112
+ g = self.root.graph()
113
+ self.note_owner(g)
114
+ root_m = Machine(name="main", kind="root", type=self.root.type)
115
+ c.machines.append(root_m)
116
+ c.compile_machine(root_m, g, "root")
117
+ compiled: set[int] = set()
118
+ while self.pending:
119
+ wf = self.pending.pop(0)
120
+ if id(wf) in compiled:
121
+ continue
122
+ compiled.add(id(wf))
123
+ cg = wf.graph()
124
+ self.note_owner(cg)
125
+ name, _ = self.children[id(wf)]
126
+ m = Machine(name=name, kind="call", type=wf.type)
127
+ c.machines.append(m)
128
+ c.compile_machine(m, cg, "call")
129
+ defs = render_fixpoint([m.definition for m in c.machines])
130
+ specs = []
131
+ for m, d in zip(c.machines, defs):
132
+ needs = [SmNeed("invoke", fn=fn) for fn in m.functions]
133
+ needs += [SmNeed("start_child", child=ch) for ch in m.children]
134
+ if m.map_run:
135
+ needs.append(SmNeed("map_run"))
136
+ if "ArtifactsBucket" in m.substitutions:
137
+ needs.append(SmNeed("bucket"))
138
+ for bucket, prefix in m.s3_reads:
139
+ needs.append(SmNeed("s3_read", bucket=bucket, prefix=prefix))
140
+ specs.append(
141
+ StateMachineSpec(
142
+ logical_id=f"Sm{_camel(m.name)}",
143
+ name=m.name,
144
+ definition=d,
145
+ substitutions=dict(m.substitutions),
146
+ type=m.type, # type: ignore[arg-type]
147
+ needs=needs,
148
+ root=m is root_m,
149
+ )
150
+ )
151
+ express_with_waits = [s.name for s, m in zip(specs, c.machines) if m.type == "EXPRESS" and m.map_run]
152
+ if express_with_waits:
153
+ raise CompileError(f"EXPRESS workflows can't run a Distributed Map: {', '.join(express_with_waits)}")
154
+ return App(
155
+ name=_sm_name(self.root.name).lower(),
156
+ module=self.root.module,
157
+ workflow=self.root,
158
+ functions=list(self.functions.values()),
159
+ state_machines=specs,
160
+ images=list(self.images.values()),
161
+ settings=self.settings,
162
+ project_dir=self.project_dir,
163
+ exclude=tuple(self.root.exclude),
164
+ )