py-serverless-kit 0.1.0__py3-none-any.whl

py_serverless_kit/workflows/__init__.py

@@ -0,0 +1,35 @@
+"""Workflow (state machine) compilers.
+
+One portable definition in ``pskit.yml`` -> AWS Step Functions (ASL), Google
+Cloud Workflows, or the embedded engine in :mod:`py_serverless_kit.runtime.workflow`.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict
+
+from ..config.schema import WorkflowDefinition
+
+_CONFIG_ONLY = {"description", "engine", "timeout", "options"}
+
+
+def definition_dict(wf: WorkflowDefinition) -> Dict[str, Any]:
+    """Plain-dict form consumed by the engine and the compilers."""
+    raw = wf.model_dump(mode="json", exclude=_CONFIG_ONLY & set(type(wf).model_fields))
+    return _clean(raw)
+
+
+def _clean(value: Any) -> Any:
+    if isinstance(value, dict):
+        out = {}
+        for k, v in value.items():
+            if k == "result_path":  # None is meaningful ("discard result")
+                out[k] = v
+            elif v is None or v == [] or (k == "end" and v is False):
+                continue
+            else:
+                out[k] = _clean(v)
+        return out
+    if isinstance(value, list):
+        return [_clean(v) for v in value]
+    return value

py_serverless_kit/workflows/asl.py

@@ -0,0 +1,170 @@
+"""Compile a portable workflow definition to Amazon States Language."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List
+
+ArnResolver = Callable[[str], str]
+
+
+def _errors(errors: List[str]) -> List[str]:
+    return ["States.ALL" if e in ("*", "States.ALL") else e for e in (errors or ["*"])]
+
+
+def _typed(op: str, value: Any) -> str:
+    if isinstance(value, bool):
+        return "Boolean" + op
+    if isinstance(value, (int, float)):
+        return "Numeric" + op
+    return "String" + op
+
+
+_CMP = {"gt": "GreaterThan", "gte": "GreaterThanEquals", "lt": "LessThan", "lte": "LessThanEquals"}
+
+
+def compile_rule(rule: Dict[str, Any]) -> Dict[str, Any]:
+    if "and" in rule:
+        return {"And": [compile_rule(r) for r in rule["and"]]}
+    if "or" in rule:
+        return {"Or": [compile_rule(r) for r in rule["or"]]}
+    if "not" in rule:
+        return {"Not": compile_rule(rule["not"])}
+    var = rule.get("variable", "$")
+    if "equals" in rule:
+        value = rule["equals"]
+        if value is None:
+            return {"Variable": var, "IsNull": True}
+        return {"Variable": var, _typed("Equals", value): value}
+    if "not_equals" in rule:
+        return {"Not": compile_rule({"variable": var, "equals": rule["not_equals"]})}
+    for op, name in _CMP.items():
+        if op in rule:
+            value = rule[op]
+            if isinstance(value, bool):
+                raise ValueError(f"operator {op} cannot compare booleans")
+            return {"Variable": var, _typed(name, value): value}
+    if "in" in rule:
+        return {"Or": [compile_rule({"variable": var, "equals": v}) for v in rule["in"]]}
+    if "contains" in rule:
+        needle = str(rule["contains"]).replace("*", "\\*")
+        return {"Variable": var, "StringMatches": f"*{needle}*"}
+    if "matches" in rule:
+        return {"Variable": var, "StringMatches": rule["matches"]}
+    if "is_null" in rule:
+        return {"Variable": var, "IsNull": bool(rule["is_null"])}
+    if "is_present" in rule:
+        return {"Variable": var, "IsPresent": bool(rule["is_present"])}
+    raise ValueError(f"unsupported choice rule: {rule}")
+
+
+def _flow(state: Dict[str, Any], out: Dict[str, Any]) -> None:
+    if state.get("next"):
+        out["Next"] = state["next"]
+    else:
+        out["End"] = True
+
+
+def _io(state: Dict[str, Any], out: Dict[str, Any]) -> None:
+    if state.get("input_path"):
+        out["InputPath"] = state["input_path"]
+    if "result_path" in state and state["result_path"] != "$":
+        out["ResultPath"] = state["result_path"]
+
+
+def compile_machine(machine: Dict[str, Any], arn_of: ArnResolver) -> Dict[str, Any]:
+    states: Dict[str, Any] = {}
+    for name, st in machine["states"].items():
+        kind = st["type"]
+        out: Dict[str, Any] = {}
+        if st.get("comment"):
+            out["Comment"] = st["comment"]
+        if kind == "task":
+            out["Type"] = "Task"
+            out["Resource"] = arn_of(st["function"])
+            out["Parameters"] = {"__pskit": "workflow", "data.$": "$"}
+            _io(st, out)
+            if st.get("timeout_seconds"):
+                out["TimeoutSeconds"] = int(st["timeout_seconds"])
+            _flow(st, out)
+        elif kind == "pass":
+            out["Type"] = "Pass"
+            if "result" in st:
+                out["Result"] = st["result"]
+            _io(st, out)
+            _flow(st, out)
+        elif kind == "wait":
+            out["Type"] = "Wait"
+            if st.get("seconds_path"):
+                out["SecondsPath"] = st["seconds_path"]
+            else:
+                out["Seconds"] = int(st.get("seconds", 0))
+            if st.get("input_path"):
+                out["InputPath"] = st["input_path"]
+            _flow(st, out)
+        elif kind == "choice":
+            out["Type"] = "Choice"
+            out["Choices"] = [dict(compile_rule(r), Next=r["next"]) for r in st["choices"]]
+            if st.get("default"):
+                out["Default"] = st["default"]
+        elif kind == "parallel":
+            out["Type"] = "Parallel"
+            out["Branches"] = [compile_machine(b, arn_of) for b in st["branches"]]
+            _io(st, out)
+            _flow(st, out)
+        elif kind == "map":
+            out["Type"] = "Map"
+            if st.get("items_path"):
+                out["ItemsPath"] = st["items_path"]
+            out["ItemProcessor"] = {
+                "ProcessorConfig": {"Mode": "INLINE"},
+                **compile_machine(st["iterator"], arn_of),
+            }
+            if st.get("max_concurrency"):
+                out["MaxConcurrency"] = int(st["max_concurrency"])
+            _io(st, out)
+            _flow(st, out)
+        elif kind == "succeed":
+            out["Type"] = "Succeed"
+        elif kind == "fail":
+            out["Type"] = "Fail"
+            if st.get("error"):
+                out["Error"] = st["error"]
+            if st.get("cause"):
+                out["Cause"] = st["cause"]
+        else:  # pragma: no cover - schema prevents this
+            raise ValueError(f"unknown state type {kind}")
+
+        if kind in ("task", "parallel", "map"):
+            if st.get("retry"):
+                out["Retry"] = []
+                for r in st["retry"]:
+                    rr: Dict[str, Any] = {
+                        "ErrorEquals": _errors(r.get("errors", ["*"])),
+                        "IntervalSeconds": max(1, int(round(r.get("interval_seconds", 1)))),
+                        "MaxAttempts": int(r.get("max_attempts", 3)),
+                        "BackoffRate": float(r.get("backoff_rate", 2.0)),
+                    }
+                    if r.get("max_delay_seconds"):
+                        rr["MaxDelaySeconds"] = int(r["max_delay_seconds"])
+                    out["Retry"].append(rr)
+            if st.get("catch"):
+                out["Catch"] = [
+                    {
+                        "ErrorEquals": _errors(c.get("errors", ["*"])),
+                        "Next": c["next"],
+                        "ResultPath": c.get("result_path", "$.error"),
+                    }
+                    for c in st["catch"]
+                ]
+        states[name] = out
+    return {"StartAt": machine["start_at"], "States": states}
+
+
+def compile_asl(definition: Dict[str, Any], arn_of: ArnResolver, comment: str = "", timeout: int = 0) -> Dict[str, Any]:
+    asl: Dict[str, Any] = {}
+    if comment:
+        asl["Comment"] = comment
+    asl.update(compile_machine(definition, arn_of))
+    if timeout:
+        asl["TimeoutSeconds"] = int(timeout)
+    return asl

py_serverless_kit/workflows/gcp.py

@@ -0,0 +1,328 @@
+"""Compile a portable workflow definition to Google Cloud Workflows (JSON syntax).
+
+Each task state becomes an authenticated ``http.post`` to the Cloud Function's
+URL with the ``{"__pskit": "workflow", "data": ...}`` envelope; the GCP adapter
+answers with the handler's JSON result or a 500 ``{"errorType", "cause"}``.
+
+Limitations compared to Step Functions (validated, reported as warnings):
+
+* only the *first* retry rule of a task is used (Cloud Workflows allows one policy);
+* retry / catch are honoured on ``task`` states only.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+UrlResolver = Callable[[str], str]
+_MAX_HTTP_TIMEOUT = 1800
+
+
+def _sid(*parts: str) -> str:
+    return re.sub(r"[^A-Za-z0-9_]", "_", "_".join(p for p in parts if p))
+
+
+def _keys(path: Optional[str]) -> List[Any]:
+    if not path or path == "$":
+        return []
+    from ..runtime.workflow import _tokens  # same JSON-path dialect as the engine
+
+    return _tokens(path)
+
+
+def read_expr(path: Optional[str], var: str = "data") -> str:
+    keys = _keys(path)
+    if not keys:
+        return var
+    if all(isinstance(k, str) for k in keys):
+        return f"map.get({var}, {json.dumps(keys)})"
+    expr = var
+    for k in keys:
+        expr += f"[{json.dumps(k)}]"
+    return expr
+
+
+def assign_target(path: Optional[str], var: str = "data") -> Optional[str]:
+    if path is None:
+        return None
+    keys = _keys(path)
+    target = var
+    for k in keys:
+        target += f"[{json.dumps(k)}]" if isinstance(k, int) else f".{k}"
+    return target
+
+
+def _lit(value: Any) -> str:
+    return json.dumps(value)
+
+
+def rule_expr(rule: Dict[str, Any]) -> str:
+    if "and" in rule:
+        return "(" + " and ".join(rule_expr(r) for r in rule["and"]) + ")"
+    if "or" in rule:
+        return "(" + " or ".join(rule_expr(r) for r in rule["or"]) + ")"
+    if "not" in rule:
+        return f"(not {rule_expr(rule['not'])})"
+    v = read_expr(rule.get("variable", "$"))
+    simple = {"equals": "==", "not_equals": "!=", "gt": ">", "gte": ">=", "lt": "<", "lte": "<="}
+    for op, sym in simple.items():
+        if op in rule:
+            return f"({v} {sym} {_lit(rule[op])})"
+    if "in" in rule:
+        return f"({v} in {_lit(rule['in'])})"
+    if "contains" in rule:
+        return f"(len(text.find_all({v}, {_lit(str(rule['contains']))})) > 0)"
+    if "matches" in rule:
+        regex = "^" + re.escape(rule["matches"]).replace("\\*", ".*").replace("\\?", ".") + "$"
+        return f"text.match_regex({v}, {_lit(regex)})"
+    if "is_null" in rule:
+        return f"({v} == null)" if rule["is_null"] else f"({v} != null)"
+    if "is_present" in rule:
+        return f"({v} != null)" if rule["is_present"] else f"({v} == null)"
+    raise ValueError(f"unsupported choice rule: {rule}")
+
+
+def _errors_condition(errors: List[str]) -> str:
+    if any(e in ("*", "States.ALL") for e in errors or ["*"]):
+        return "${true}"
+    return '${map.get(e, ["body", "errorType"]) in ' + _lit(list(errors)) + "}"
+
+
+class _Compiler:
+    def __init__(self, url_of: UrlResolver, workflow: str) -> None:
+        self.url_of = url_of
+        self.workflow = workflow
+        self.subworkflows: Dict[str, Any] = {}
+        self.warnings: List[str] = []
+
+    # ------------------------------------------------------------------
+    def machine(self, machine: Dict[str, Any], prefix: str) -> List[Dict[str, Any]]:
+        states: Dict[str, Dict[str, Any]] = machine["states"]
+        entry = {name: self._entry_step(prefix, name, st) for name, st in states.items()}
+        steps: List[Dict[str, Any]] = [
+            {
+                _sid(prefix, "init"): {
+                    "assign": [{"data": "${input}"}, {"_caught": ""}],
+                    "next": entry[machine["start_at"]],
+                }
+            }
+        ]
+        for name, st in states.items():
+            steps.extend(self.state(prefix, name, st, entry))
+        return steps
+
+    def _entry_step(self, prefix: str, name: str, st: Dict[str, Any]) -> str:
+        if st["type"] == "task" and st.get("catch"):
+            return _sid(prefix, name, "pre")
+        if st["type"] in ("parallel", "map"):
+            return _sid(prefix, name, "init")
+        return _sid(prefix, name)
+
+    def _after(self, st: Dict[str, Any], entry: Dict[str, str]) -> Dict[str, Any]:
+        if st.get("next"):
+            return {"next": entry[st["next"]]}
+        return {"return": "${data}"}
+
+    def _store(
+        self, prefix: str, name: str, st: Dict[str, Any], value_expr: str, entry: Dict[str, str]
+    ) -> Dict[str, Any]:
+        target = assign_target(st.get("result_path", "$"))
+        body: Dict[str, Any] = {}
+        if target is not None:
+            body["assign"] = [{target: value_expr}]
+        else:
+            body["assign"] = [{"_discard": value_expr}]
+        body.update(self._after(st, entry))
+        return {_sid(prefix, name, "out"): body}
+
+    def _sub(self, machine: Dict[str, Any], name: str) -> str:
+        sub_name = _sid(self.workflow, name)
+        self.subworkflows[sub_name] = {"params": ["input"], "steps": self.machine(machine, sub_name)}
+        return sub_name
+
+    # ------------------------------------------------------------------
+    def state(self, prefix: str, name: str, st: Dict[str, Any], entry: Dict[str, str]) -> List[Dict[str, Any]]:
+        kind = st["type"]
+        sid = _sid(prefix, name)
+        input_expr = "${" + read_expr(st.get("input_path")) + "}"
+        if kind == "task":
+            return self.task(prefix, name, st, entry, input_expr)
+        if kind == "pass":
+            value = _lit(st["result"]) if "result" in st else read_expr(st.get("input_path"))
+            return [
+                {sid: {"assign": [{"_pass": "${" + value + "}"}], "next": _sid(prefix, name, "out")}},
+                self._store(prefix, name, st, "${_pass}", entry),
+            ]
+        if kind == "wait":
+            seconds: Any = int(st.get("seconds", 0))
+            if st.get("seconds_path"):
+                seconds = "${" + read_expr(st["seconds_path"]) + "}"
+            return [{sid: {"call": "sys.sleep", "args": {"seconds": seconds}, **self._after(st, entry)}}]
+        if kind == "choice":
+            conds: List[Dict[str, Any]] = [
+                {"condition": "${" + rule_expr(r) + "}", "next": entry[r["next"]]} for r in st["choices"]
+            ]
+            body: Dict[str, Any] = {"switch": conds}
+            if st.get("default"):
+                body["next"] = entry[st["default"]]
+            else:
+                conds.append({"condition": True, "raise": {"error": "States.NoChoiceMatched", "cause": name}})
+            return [{sid: body}]
+        if kind == "succeed":
+            return [{sid: {"return": "${data}"}}]
+        if kind == "fail":
+            return [{sid: {"raise": {"error": st.get("error", "States.Fail"), "cause": st.get("cause", "")}}}]
+        if kind == "parallel":
+            if st.get("retry") or st.get("catch"):
+                self.warnings.append(f"{self.workflow}.{name}: retry/catch on parallel states is ignored on GCP")
+            var = _sid("par", prefix, name)
+            branches = []
+            for i, branch in enumerate(st["branches"]):
+                sub = self._sub(branch, f"{prefix}_{name}_b{i}")
+                branches.append(
+                    {
+                        f"b{i}": {
+                            "steps": [
+                                {
+                                    _sid(sid, f"b{i}", "call"): {
+                                        "call": sub,
+                                        "args": {"input": input_expr},
+                                        "result": f"r{i}",
+                                    }
+                                },
+                                {_sid(sid, f"b{i}", "save"): {"assign": [{f"{var}.b{i}": f"${{r{i}}}"}]}},
+                            ]
+                        }
+                    }
+                )
+            collect = "${[" + ", ".join(f"{var}.b{i}" for i in range(len(branches))) + "]}"
+            return [
+                {_sid(prefix, name, "init"): {"assign": [{var: {}}]}},
+                {sid: {"parallel": {"shared": [var], "branches": branches}}},
+                self._store(prefix, name, st, collect, entry),
+            ]
+        if kind == "map":
+            if st.get("retry") or st.get("catch"):
+                self.warnings.append(f"{self.workflow}.{name}: retry/catch on map states is ignored on GCP")
+            sub = self._sub(st["iterator"], f"{prefix}_{name}_it")
+            items, res, out = _sid("items", prefix, name), _sid("res", prefix, name), _sid("out", prefix, name)
+            items_src = read_expr(st.get("items_path"), read_expr(st.get("input_path")))
+            parallel: Dict[str, Any] = {
+                "shared": [res],
+                "for": {
+                    "value": "item",
+                    "index": "i",
+                    "in": "${" + items + "}",
+                    "steps": [
+                        {_sid(sid, "call"): {"call": sub, "args": {"input": "${item}"}, "result": "r"}},
+                        {_sid(sid, "save"): {"assign": [{f"{res}[string(i)]": "${r}"}]}},
+                    ],
+                },
+            }
+            if st.get("max_concurrency"):
+                parallel["concurrency_limit"] = int(st["max_concurrency"])
+            return [
+                {_sid(prefix, name, "init"): {"assign": [{items: "${" + items_src + "}"}, {res: {}}, {out: []}]}},
+                {sid: {"parallel": parallel}},
+                {
+                    _sid(prefix, name, "collect"): {
+                        "for": {
+                            "value": "j",
+                            "range": "${[0, len(" + items + ") - 1]}",
+                            "steps": [
+                                {_sid(sid, "append"): {"assign": [{out: f"${{list.concat({out}, {res}[string(j)])}}"}]}}
+                            ],
+                        }
+                    }
+                },
+                self._store(prefix, name, st, "${" + out + "}", entry),
+            ]
+        raise ValueError(f"unknown state type {kind}")  # pragma: no cover
+
+    def task(
+        self, prefix: str, name: str, st: Dict[str, Any], entry: Dict[str, str], input_expr: str
+    ) -> List[Dict[str, Any]]:
+        sid = _sid(prefix, name)
+        result_var = _sid("res", prefix, name)
+        timeout = min(int(st.get("timeout_seconds") or _MAX_HTTP_TIMEOUT), _MAX_HTTP_TIMEOUT)
+        call = {
+            "call": "http.post",
+            "args": {
+                "url": self.url_of(st["function"]),
+                "auth": {"type": "OIDC"},
+                "timeout": timeout,
+                "body": {"__pskit": "workflow", "data": input_expr},
+            },
+            "result": result_var,
+        }
+        steps: List[Dict[str, Any]] = []
+        retry = st.get("retry") or []
+        catch = st.get("catch") or []
+        if len(retry) > 1:
+            self.warnings.append(f"{self.workflow}.{name}: only the first retry rule is used on GCP")
+        if not retry and not catch:
+            steps.append({sid: call})
+        else:
+            body: Dict[str, Any] = {"try": call}
+            if retry:
+                rule = retry[0]
+                pred = _sid(self.workflow, prefix, name, "retry_pred")
+                self.subworkflows[pred] = {
+                    "params": ["e"],
+                    "steps": [
+                        {
+                            "check": {
+                                "switch": [{"condition": _errors_condition(rule.get("errors", ["*"])), "return": True}]
+                            }
+                        },
+                        {"no": {"return": False}},
+                    ],
+                }
+                body["retry"] = {
+                    "predicate": "${" + pred + "}",
+                    "max_retries": int(rule.get("max_attempts", 3)),
+                    "backoff": {
+                        "initial_delay": max(1, float(rule.get("interval_seconds", 1))),
+                        "max_delay": float(rule.get("max_delay_seconds") or 300),
+                        "multiplier": float(rule.get("backoff_rate", 2.0)),
+                    },
+                }
+            if catch:
+                info = {
+                    "error": '${default(map.get(e, ["body", "errorType"]), "Error")}',
+                    "cause": '${default(map.get(e, ["body", "cause"]), "")}',
+                }
+                conds: List[Dict[str, Any]] = []
+                for i, c in enumerate(catch):
+                    assigns: List[Dict[str, Any]] = [{"_caught": c["next"]}]
+                    target = assign_target(c.get("result_path", "$.error"))
+                    if target is not None:
+                        assigns.append({target: info})
+                    conds.append(
+                        {
+                            "condition": _errors_condition(c.get("errors", ["*"])),
+                            "steps": [{_sid(sid, f"catch{i}"): {"assign": assigns}}],
+                        }
+                    )
+                conds.append({"condition": True, "raise": "${e}"})
+                body["except"] = {"as": "e", "steps": [{_sid(sid, "except"): {"switch": conds}}]}
+                steps.append({_sid(prefix, name, "pre"): {"assign": [{"_caught": ""}]}})
+            steps.append({sid: body})
+            if catch:
+                route = [
+                    {"condition": "${_caught == " + _lit(c["next"]) + "}", "next": entry[c["next"]]} for c in catch
+                ]
+                steps.append({_sid(prefix, name, "route"): {"switch": route}})
+        steps.append(self._store(prefix, name, st, "${" + result_var + ".body}", entry))
+        return steps
+
+
+def compile_gcp(definition: Dict[str, Any], url_of: UrlResolver, workflow: str) -> Tuple[Dict[str, Any], List[str]]:
+    """Return ``(workflow_source_dict, warnings)``."""
+    comp = _Compiler(url_of, _sid(workflow))
+    main = {"params": ["input"], "steps": comp.machine(definition, "s")}
+    source: Dict[str, Any] = {"main": main}
+    source.update(comp.subworkflows)
+    return source, comp.warnings