keelflow 0.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
keelflow/__init__.py ADDED
@@ -0,0 +1,2 @@
1
+ """keelflow — reusable Postgres-backed durable workflow engine."""
2
+ __version__ = "0.1.0"
@@ -0,0 +1,9 @@
1
+ """SuperAdmin — deliberately rough server-rendered ops UI (T15).
2
+
3
+ `make_app(dsn)` returns a small WSGI/HTTP handler for `python -m
4
+ keelflow.admin`. Read-only views: queue overview, job list/detail, workflow
5
+ runs/detail, failed rows, metrics. Mutations limited to `POST /retry`.
6
+ """
7
+ from .app import make_app
8
+
9
+ __all__ = ["make_app"]
keelflow/admin/app.py ADDED
@@ -0,0 +1,190 @@
1
+ """Minimal server-rendered SuperAdmin (deliberately rough).
2
+
3
+ Serves: / (metrics+queues), /jobs, /jobs/<id>, /runs, /runs/<id>,
4
+ /jobs/<id>/rows, POST /jobs/<id>/retry. No auth — bind locally or put
5
+ behind your own gateway.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import asyncio
10
+ import html
11
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
12
+ from urllib.parse import parse_qs, urlparse
13
+
14
+ import psycopg
15
+
16
+ from ..engine.monitoring import MonitoringService
17
+
18
+
19
+ def make_app(dsn: str, host: str = "127.0.0.1", port: int = 8410):
20
+ """Start the admin HTTP server (blocking). Ctrl+C to stop."""
21
+ svc = MonitoringService()
22
+
23
+ def q(coro):
24
+ async def _run():
25
+ conn = await psycopg.AsyncConnection.connect(dsn, autocommit=True)
26
+ try:
27
+ return await coro(conn)
28
+ finally:
29
+ await conn.close()
30
+ return asyncio.run(_run())
31
+
32
+ class H(BaseHTTPRequestHandler):
33
+ def _send(self, body: str, code: int = 200):
34
+ data = body.encode()
35
+ self.send_response(code)
36
+ self.send_header("Content-Type", "text/html; charset=utf-8")
37
+ self.send_header("Content-Length", str(len(data)))
38
+ self.end_headers()
39
+ self.wfile.write(data)
40
+
41
+ def log_message(self, *args): # quieter
42
+ pass
43
+
44
+ def do_GET(self):
45
+ p = urlparse(self.path)
46
+ path, qs = p.path, parse_qs(p.query)
47
+ if path == "/":
48
+ return self._page("keelflow admin", self._home())
49
+ if path == "/jobs":
50
+ return self._page("jobs", self._jobs(qs))
51
+ if path.startswith("/jobs/") and path.endswith("/rows"):
52
+ jid = path.split("/")[2]
53
+ return self._page("failed rows", self._rows(jid))
54
+ if path.startswith("/jobs/"):
55
+ return self._page("job", self._job(path.split("/")[2]))
56
+ if path == "/runs":
57
+ return self._page("runs", self._runs())
58
+ if path.startswith("/runs/"):
59
+ return self._page("run", self._run(path.split("/")[2]))
60
+ return self._send("<h1>404</h1>", 404)
61
+
62
+ def do_POST(self):
63
+ p = urlparse(self.path)
64
+ if p.path.startswith("/jobs/") and p.path.endswith("/retry"):
65
+ jid = p.path.split("/")[2]
66
+ q(lambda c: svc.retry_job(c, jid))
67
+ return self._send("<meta http-equiv='refresh' content='0;/jobs'>")
68
+ return self._send("<h1>404</h1>", 404)
69
+
70
+ # ── views ────────────────────────────────────────────────────
71
+
72
+ def _page(self, title, body):
73
+ nav = ("<p><a href='/'>home</a> · <a href='/jobs'>jobs</a> · "
74
+ "<a href='/runs'>runs</a></p>")
75
+ return self._send(
76
+ f"<html><head><title>{title}</title>"
77
+ "<style>body{{font-family:monospace;margin:2em}}"
78
+ "table{{border-collapse:collapse}}td,th{{border:1px solid #ccc;"
79
+ "padding:4px 8px}} .failed{{color:#b00}} .pending{{color:#a60}}"
80
+ " .done{{color:#070}}</style></head><body>"
81
+ f"{nav}<h1>{title}</h1>{body}</body></html>"
82
+ )
83
+
84
+ def _esc(self, v):
85
+ return html.escape(str(v)) if v is not None else ""
86
+
87
+ def _table(self, rows, cols, links=None):
88
+ links = links or {}
89
+ out = ["<table><tr>" + "".join(f"<th>{c}</th>" for c in cols) + "</tr>"]
90
+ for r in rows:
91
+ cells = []
92
+ for c in cols:
93
+ v = r.get(c)
94
+ cls = v if c == "status" and v in ("failed", "pending", "done") else ""
95
+ cell = f"<td class='{cls}'>{self._esc(v)}</td>"
96
+ if c in links:
97
+ cell = f"<td><a href='{links[c]}{r[c]}'>{self._esc(v)}</a></td>"
98
+ cells.append(cell)
99
+ out.append("<tr>" + "".join(cells) + "</tr>")
100
+ out.append("</table>")
101
+ return "".join(out)
102
+
103
+ def _home(self):
104
+ m = q(svc.metrics_snapshot)
105
+ return (
106
+ f"<h2>metrics</h2><ul>"
107
+ f"<li>queue depth: {m['queue_depth']}</li>"
108
+ f"<li>running: {m['running']}</li>"
109
+ f"<li>failed (24h): {m['failed_24h']}</li>"
110
+ f"<li>oldest pending: {self._esc(round(m['oldest_pending_s'] or 0,1))}s</li>"
111
+ f"</ul>"
112
+ )
113
+
114
+ def _jobs(self, qs):
115
+ types = qs.get("type", [None])[0]
116
+ statuses = qs.get("status", [None])[0]
117
+ res = q(lambda c: svc.jobs_list(
118
+ c, types=[types] if types else None,
119
+ statuses=[statuses] if statuses else None, limit=100,
120
+ ))
121
+ rows = [
122
+ {
123
+ "id": str(r["id"]), "type": r["type"], "status": r["status"],
124
+ "attempts": r["attempts"], "run_at": r["run_at"],
125
+ "claimed_by": r["claimed_by"], "assignee": r["assignee"],
126
+ "created_at": r["created_at"],
127
+ }
128
+ for r in res["items"]
129
+ ]
130
+ return self._table(
131
+ rows, ["id", "type", "status", "attempts", "assignee", "created_at"],
132
+ links={"id": "/jobs/"},
133
+ )
134
+
135
+ def _job(self, jid):
136
+ r = q(lambda c: svc.job_get(c, jid))
137
+ if not r:
138
+ return "<p>not found</p>"
139
+ kv = "".join(
140
+ f"<tr><td>{k}</td><td>{self._esc(v)}</td></tr>"
141
+ for k, v in r.items() if k != "id"
142
+ )
143
+ return (
144
+ f"<table><tr><td>id</td><td>{r['id']}</td></tr>{kv}</table>"
145
+ f"<p><a href='/jobs/{r['id']}/rows'>failed rows</a> · "
146
+ f"<form method='post' action='/jobs/{r['id']}/retry'>"
147
+ "<button>retry</button></form></p>"
148
+ )
149
+
150
+ def _runs(self):
151
+ rows = q(svc.runs_list)
152
+ return self._table(
153
+ [{**r, "id": str(r["id"])} for r in rows],
154
+ ["id", "workflow", "status", "created_at"],
155
+ links={"id": "/runs/"},
156
+ )
157
+
158
+ def _run(self, rid):
159
+ r = q(lambda c: svc.runs_get(c, rid))
160
+ if not r:
161
+ return "<p>not found</p>"
162
+ steps = [
163
+ {
164
+ "step_id": s["step_id"], "type": s["type"],
165
+ "status": s["status"], "attempts": s["attempts"],
166
+ "outputs": s["outputs"], "job_id": s["job_id"],
167
+ }
168
+ for s in r["steps"]
169
+ ]
170
+ return (
171
+ f"<p>workflow: {r['workflow']} · status: {r['status']}</p>"
172
+ + self._table(steps, ["step_id", "type", "status", "attempts", "outputs"])
173
+ )
174
+
175
+ def _rows(self, jid):
176
+ rows = q(lambda c: svc.rows_list_failed(c, jid))
177
+ return self._table(
178
+ [{**r, "id": str(r["id"])} for r in rows],
179
+ ["id", "row_key", "error", "updated_at"],
180
+ )
181
+
182
+ server = ThreadingHTTPServer((host, port), H)
183
+ print(f"keelflow admin → http://{host}:{port}")
184
+ server.serve_forever()
185
+
186
+
187
+ if __name__ == "__main__": # pragma: no cover
188
+ import os
189
+
190
+ make_app(os.environ["KEELFLOW_DSN"])
@@ -0,0 +1,4 @@
1
+ """YAML definition compiler — text → internal DAG/state-machine models."""
2
+ from .loader import DefinitionError, load_definitions, load_state_machine, load_workflow
3
+
4
+ __all__ = ["DefinitionError", "load_definitions", "load_state_machine", "load_workflow"]
@@ -0,0 +1,242 @@
1
+ """YAML → WorkflowDefinition / StateMachineDefinition with validation.
2
+
3
+ Validation at load time (not at run):
4
+ - every `needs` references a declared step;
5
+ - every `type` is a registered job type in the registry;
6
+ - every `creates_job`/`triggers_workflow` references registered names;
7
+ - `needs` graph is acyclic;
8
+ - `if:` guards parse under the minimal expression grammar;
9
+ - SM transitions use declared states; actions are unique.
10
+
11
+ Definitions are versioned by their `name@vN` — the name in the file is
12
+ what callers reference.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ import re
17
+ from pathlib import Path
18
+ from typing import Any
19
+
20
+ import yaml
21
+
22
+ from ..engine.workflow import validate_guard
23
+ from ..sdk import (
24
+ StateMachineDefinition,
25
+ StepSpec,
26
+ TransitionSpec,
27
+ WorkflowDefinition,
28
+ WorkflowTriggerSpec,
29
+ )
30
+
31
+ _NAME = re.compile(r"^[\w.\-]+@v\d+$")
32
+
33
+
34
+ class DefinitionError(ValueError):
35
+ """A definition file failed validation at load."""
36
+
37
+
38
+ def load_workflow(path: str | Path, registry) -> WorkflowDefinition:
39
+ data = yaml.safe_load(Path(path).read_text())
40
+ if not isinstance(data, dict):
41
+ raise DefinitionError(f"{path}: expected a mapping")
42
+ name = data.get("workflow") or data.get("name")
43
+ _check_name(name, path)
44
+ assert name is not None # narrowed by _check_name
45
+ steps_raw = data.get("steps")
46
+ if not isinstance(steps_raw, list) or not steps_raw:
47
+ raise DefinitionError(f"{name}: 'steps' must be a non-empty list")
48
+
49
+ steps: list[StepSpec] = []
50
+ seen: set[str] = set()
51
+ for i, s in enumerate(steps_raw):
52
+ if not isinstance(s, dict) or "id" not in s or "type" not in s:
53
+ raise DefinitionError(f"{name}: step[{i}] needs 'id' and 'type'")
54
+ sid = s["id"]
55
+ if sid in seen:
56
+ raise DefinitionError(f"{name}: duplicate step id {sid!r}")
57
+ seen.add(sid)
58
+ needs = tuple(s.get("needs") or ())
59
+ for n in needs:
60
+ if n not in seen:
61
+ raise DefinitionError(
62
+ f"{name}: step {sid!r} needs unknown step {n!r}"
63
+ )
64
+ if_expr = s.get("if") or s.get("if_expr")
65
+ if if_expr is not None:
66
+ try:
67
+ validate_guard(if_expr)
68
+ except Exception as exc:
69
+ raise DefinitionError(
70
+ f"{name}: step {sid!r} has invalid if: {exc}"
71
+ ) from exc
72
+ _check_known_type(s["type"], registry, f"{name} step {sid!r}")
73
+ steps.append(
74
+ StepSpec(
75
+ id=sid,
76
+ type=s["type"],
77
+ input=dict(s.get("input") or {}),
78
+ needs=needs,
79
+ if_expr=if_expr,
80
+ )
81
+ )
82
+ _check_acyclic(steps, name)
83
+ defn = WorkflowDefinition(name=name, steps=tuple(steps))
84
+ registry.register_workflow(defn)
85
+ return defn
86
+
87
+
88
+ def load_state_machine(path: str | Path, registry) -> StateMachineDefinition:
89
+ data = yaml.safe_load(Path(path).read_text())
90
+ if not isinstance(data, dict):
91
+ raise DefinitionError(f"{path}: expected a mapping")
92
+ name = data.get("name") or data.get("state_machine")
93
+ _check_name(name, path)
94
+ assert name is not None
95
+ entity = data.get("entity")
96
+ if not entity:
97
+ raise DefinitionError(f"{name}: missing 'entity'")
98
+ states = tuple(data.get("states") or ())
99
+ if not states:
100
+ raise DefinitionError(f"{name}: 'states' must be a non-empty list")
101
+ transitions = []
102
+ actions: set[str] = set()
103
+ for i, t in enumerate(data.get("transitions") or []):
104
+ for field in ("from", "to", "action"):
105
+ if field not in t:
106
+ raise DefinitionError(
107
+ f"{name}: transition[{i}] missing {field!r}"
108
+ )
109
+ if t["from"] not in states or t["to"] not in states:
110
+ raise DefinitionError(
111
+ f"{name}: transition[{i}] uses undeclared state"
112
+ )
113
+ if t["action"] in actions:
114
+ raise DefinitionError(
115
+ f"{name}: duplicate action {t['action']!r}"
116
+ )
117
+ actions.add(t["action"])
118
+ if t.get("creates_job"):
119
+ _check_known_type(t["creates_job"], registry, f"{name} transition")
120
+ tw = t.get("triggers_workflow")
121
+ trigger = _parse_trigger(tw, name, i)
122
+ if trigger and not _is_registered_workflow(trigger.workflow, registry):
123
+ raise DefinitionError(
124
+ f"{name}: triggers_workflow {trigger.workflow!r} not registered"
125
+ )
126
+ if trigger and trigger.failure_job:
127
+ _check_known_type(trigger.failure_job, registry,
128
+ f"{name} transition trigger")
129
+ transitions.append(
130
+ TransitionSpec(
131
+ from_state=t["from"],
132
+ to_state=t["to"],
133
+ action=t["action"],
134
+ guard=dict(t.get("guard") or {}),
135
+ creates_job=t.get("creates_job"),
136
+ triggers_workflow=trigger,
137
+ notify=t.get("notify"),
138
+ )
139
+ )
140
+ initial = data.get("initial") or states[0]
141
+ if initial not in states:
142
+ raise DefinitionError(f"{name}: initial {initial!r} not in states")
143
+ defn = StateMachineDefinition(
144
+ name=name,
145
+ entity=entity,
146
+ states=states,
147
+ transitions=tuple(transitions),
148
+ initial=initial,
149
+ )
150
+ registry.register_state_machine(defn)
151
+ return defn
152
+
153
+
154
+ def load_definitions(dir_: str | Path, registry) -> dict[str, int]:
155
+ """Load every *.yaml under `workflows/` and `state_machines/` inside `dir_`.
156
+
157
+ Returns counts per kind.
158
+ """
159
+ dir_ = Path(dir_)
160
+ counts = {"workflows": 0, "state_machines": 0}
161
+ for f in sorted((dir_ / "workflows").glob("*.yaml")):
162
+ load_workflow(f, registry)
163
+ counts["workflows"] += 1
164
+ for f in sorted((dir_ / "state_machines").glob("*.yaml")):
165
+ load_state_machine(f, registry)
166
+ counts["state_machines"] += 1
167
+ return counts
168
+
169
+
170
+ # ── helpers ────────────────────────────────────────────────────────────────
171
+
172
+
173
+ def _parse_trigger(tw: Any, name: str, i: int) -> WorkflowTriggerSpec | None:
174
+ """`triggers_workflow` may be a string or a typed mapping."""
175
+ if tw is None:
176
+ return None
177
+ if isinstance(tw, str):
178
+ return WorkflowTriggerSpec(workflow=tw)
179
+ if isinstance(tw, dict):
180
+ unknown = set(tw) - {"workflow", "on_done", "on_rejected",
181
+ "on_failed", "failure_job"}
182
+ if unknown:
183
+ raise DefinitionError(
184
+ f"{name}: transition[{i}] triggers_workflow unknown keys "
185
+ f"{sorted(unknown)}"
186
+ )
187
+ if not tw.get("workflow"):
188
+ raise DefinitionError(
189
+ f"{name}: transition[{i}] triggers_workflow needs 'workflow'"
190
+ )
191
+ return WorkflowTriggerSpec(
192
+ workflow=tw["workflow"],
193
+ on_done=tw.get("on_done"),
194
+ on_rejected=tw.get("on_rejected"),
195
+ on_failed=tw.get("on_failed"),
196
+ failure_job=tw.get("failure_job"),
197
+ )
198
+ raise DefinitionError(
199
+ f"{name}: transition[{i}] triggers_workflow must be str or mapping"
200
+ )
201
+
202
+
203
+ def _check_name(name: Any, path) -> None:
204
+ if not name or not _NAME.match(str(name)):
205
+ raise DefinitionError(f"{path}: name must be 'domain.action@vN', got {name!r}")
206
+
207
+
208
+ def _check_known_type(type_name: str, registry, where: str) -> None:
209
+ try:
210
+ registry.job_type(type_name)
211
+ except KeyError as exc:
212
+ raise DefinitionError(f"{where}: unknown job type {type_name!r}") from exc
213
+
214
+
215
+ def _is_registered_workflow(name: str, registry) -> bool:
216
+ try:
217
+ registry.workflow(name)
218
+ except KeyError:
219
+ return False
220
+ return True
221
+
222
+
223
+ def _check_acyclic(steps: list[StepSpec], name: str) -> None:
224
+ by_id = {s.id: s for s in steps}
225
+ visiting: set[str] = set()
226
+ done: set[str] = set()
227
+
228
+ def visit(sid: str, trail: list[str]) -> None:
229
+ if sid in done:
230
+ return
231
+ if sid in visiting:
232
+ raise DefinitionError(
233
+ f"{name}: cyclic dependency {' → '.join(trail + [sid])}"
234
+ )
235
+ visiting.add(sid)
236
+ for n in by_id[sid].needs:
237
+ visit(n, trail + [sid])
238
+ visiting.discard(sid)
239
+ done.add(sid)
240
+
241
+ for s in steps:
242
+ visit(s.id, [])
@@ -0,0 +1 @@
1
+ """keelflow engine — runtime internals. Apps must not import from here."""
@@ -0,0 +1,65 @@
1
+ """AuditService (T14) — append-only audit writer in the caller's TX.
2
+
3
+ `record` inserts into `keelflow_audit` inside the caller's transaction, so
4
+ an unaudited change is impossible (the insert rolls back with the data).
5
+ `trace` reads the correlation chain for an entity.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ from typing import Any
11
+
12
+ from ..sdk import AuditEvent, AuditService, EntityRef, Transaction
13
+ from .enqueue import _as_cursor
14
+
15
+
16
+ class PostgresAuditService(AuditService):
17
+ async def record(self, event: AuditEvent, tx: Transaction) -> None:
18
+ cursor = _as_cursor(tx)
19
+ await cursor.execute(
20
+ """
21
+ INSERT INTO keelflow_audit
22
+ (actor, action, entity_type, entity_id, changes, source,
23
+ correlation_id)
24
+ VALUES (%s, %s, %s, %s, %s, %s, %s)
25
+ """,
26
+ (
27
+ json.dumps(_actor(event.actor)),
28
+ event.action,
29
+ event.entity.type,
30
+ event.entity.id,
31
+ json.dumps(_changes(event.changes)),
32
+ event.source,
33
+ event.correlation_id or None,
34
+ ),
35
+ )
36
+
37
+ async def trace(self, conn, entity: EntityRef) -> list[dict[str, Any]]: # type: ignore[override]
38
+ cur = await conn.execute(
39
+ "SELECT occurred_at, actor, action, changes, source, correlation_id"
40
+ " FROM keelflow_audit"
41
+ " WHERE entity_type=%s AND entity_id=%s ORDER BY id",
42
+ (entity.type, entity.id),
43
+ )
44
+ return [
45
+ {
46
+ "occurred_at": r[0],
47
+ "actor": r[1],
48
+ "action": r[2],
49
+ "changes": r[3],
50
+ "source": r[4],
51
+ "correlation_id": r[5],
52
+ }
53
+ for r in await cur.fetchall()
54
+ ]
55
+
56
+
57
+ def _actor(actor) -> dict[str, Any]:
58
+ d = {"type": actor.type.value, "id": actor.id}
59
+ if actor.via:
60
+ d["via"] = _actor(actor.via)
61
+ return d
62
+
63
+
64
+ def _changes(changes: dict[str, tuple[Any, Any]]) -> dict[str, Any]:
65
+ return {k: {"before": b, "after": a} for k, (b, a) in changes.items()}