snakemake-logger-plugin-biffix 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,13 @@
1
+ """biffix Snakemake logger plugin (Snakemake >= 9.0).
2
+
3
+ Emits biffix ``EngineEvent`` RunEvents (the producer contract,
4
+ ``schemas/wire.engine.schema.json``) both to the biffix ingest route (live) and to a durable
5
+ ``events.jsonl`` (the archival replay copy). LogRecord-parsing logic is adapted from snakesee's
6
+ logger plugin (MIT; see the NOTICE section of README.md); this package has **no dependency** on
7
+ snakesee.
8
+ """
9
+
10
+ from snakemake_logger_plugin_biffix.handler import LogHandler
11
+ from snakemake_logger_plugin_biffix.settings import LogHandlerSettings
12
+
13
+ __all__ = ["LogHandler", "LogHandlerSettings"]
@@ -0,0 +1,201 @@
1
+ """The biffix ``EngineEvent`` wire shape.
2
+
3
+ These builders produce dicts that validate against ``schemas/wire.engine.schema.json`` (the
4
+ artifact ``biffix-core`` emits). The Rust ``wire::engine`` enum is the source of truth; this
5
+ module binds to it as a wire protocol. ``kind`` is the tag; ``task_key`` is
6
+ ``rule:sorted(wildcards)`` (canonical, order-independent).
7
+
8
+ There is a builder for every kind the Snakemake handler emits: ``run_started``, ``run_progress``,
9
+ ``run_dag``, ``task_created``, ``task_submitted``, ``task_retrying``, ``task_succeeded``,
10
+ ``task_failed``, ``task_outputs``, ``engine_task_verdict``. The other schema kinds are not emitted
11
+ by this plugin:
12
+ ``run_succeeded``/``run_failed`` (the coordinator that wraps the Snakemake invocation emits the run
13
+ terminal) and ``task_cached`` (the logger records carry no cache-hit signal: the cache decision
14
+ happens inside the executor, decoupled from the log events, so a cached rule is reported as
15
+ ``task_succeeded``). The conformance guard (``test_schema_conformance.py``) makes every schema kind
16
+ either have a builder here or be listed there as not emitted, so a new kind cannot slip through.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ SCHEMA_VERSION = 1
22
+
23
+
24
+ def task_key(rule: str, wildcards: dict[str, str] | None) -> str:
25
+ """Canonical stable logical-task identity: ``rule:sorted(wildcards)``.
26
+
27
+ Known limitation: names and values are joined without escaping, so a wildcard value that
28
+ itself contains ``,`` or ``=`` can collide with a different wildcard set (``{"a": "1,b=2"}``
29
+ and ``{"a": "1", "b": "2"}`` both give ``r:a=1,b=2``). The format is shared with every other
30
+ task_key producer and consumer (already-stored keys, the joins on them, the backend's
31
+ rule-from-key parse), so it is kept as is, an accepted limitation until a real workflow hits
32
+ it (escaping would be a contract change every producer and consumer adopts together);
33
+ ``tests/test_events.py`` pins it. Snakemake's default wildcard regex allows these characters,
34
+ so a workflow that needs distinct tasks for such values should constrain its wildcards."""
35
+ if not wildcards:
36
+ return rule
37
+ inner = ",".join(f"{k}={wildcards[k]}" for k in sorted(wildcards))
38
+ return f"{rule}:{inner}"
39
+
40
+
41
+ def _envelope(ts: float, run_id: str, body: dict[str, object]) -> dict[str, object]:
42
+ return {"schema_version": SCHEMA_VERSION, "ts": ts, "run_id": run_id, **body}
43
+
44
+
45
+ def run_started(ts: float, run_id: str) -> dict[str, object]:
46
+ return _envelope(ts, run_id, {"kind": "run_started"})
47
+
48
+
49
+ def run_progress(ts: float, run_id: str, completed: int, total: int) -> dict[str, object]:
50
+ return _envelope(ts, run_id, {"kind": "run_progress", "completed": completed, "total": total})
51
+
52
+
53
+ def task_created(
54
+ ts: float,
55
+ run_id: str,
56
+ tkey: str,
57
+ group: str | None = None,
58
+ deps: list[str] | None = None,
59
+ ) -> dict[str, object]:
60
+ body: dict[str, object] = {"kind": "task_created", "task_key": tkey}
61
+ if group is not None:
62
+ # The DAG node (rule) this task rolls up onto, for the run's graph view.
63
+ body["group"] = group
64
+ if deps:
65
+ # The in-run tasks that produced this task's inputs (its upstream), sorted for stability.
66
+ body["deps"] = sorted(set(deps))
67
+ return _envelope(ts, run_id, body)
68
+
69
+
70
+ def run_dag(
71
+ ts: float, run_id: str, nodes: list[str], edges: list[tuple[str, str]]
72
+ ) -> dict[str, object]:
73
+ """The run's group-level rule graph, emitted once. Folds as a set-union in the reducer."""
74
+ return _envelope(
75
+ ts,
76
+ run_id,
77
+ {
78
+ "kind": "run_dag",
79
+ "dag": {
80
+ "nodes": [{"id": n} for n in nodes],
81
+ "edges": [{"from": f, "to": t} for f, t in edges],
82
+ },
83
+ },
84
+ )
85
+
86
+
87
+ def task_submitted(
88
+ ts: float,
89
+ run_id: str,
90
+ tkey: str,
91
+ attempt: int,
92
+ batch_job_id: str,
93
+ image: str | None = None,
94
+ ) -> dict[str, object]:
95
+ """An attempt submitted to Batch: binds ``batch_job_id`` to the task. ``image`` is the
96
+ container the Batch job runs, as the executor reports it; omitted when unknown."""
97
+ body: dict[str, object] = {
98
+ "kind": "task_submitted",
99
+ "task_key": tkey,
100
+ "attempt": attempt,
101
+ "batch_job_id": batch_job_id,
102
+ }
103
+ if image:
104
+ body["image"] = image
105
+ return _envelope(ts, run_id, body)
106
+
107
+
108
+ # The longest ``reason`` an ``engine_task_verdict`` carries; a longer one is cut.
109
+ REASON_MAX = 1000
110
+
111
+
112
+ def engine_task_verdict(
113
+ ts: float,
114
+ run_id: str,
115
+ tkey: str,
116
+ attempt: int | None = None,
117
+ category: str | None = None,
118
+ reason: str | None = None,
119
+ ) -> dict[str, object]:
120
+ """Snakemake failed a Batch attempt that AWS may report succeeded (its declared outputs are
121
+ missing after the job exited 0): ``verdict: failed``, which outranks AWS's success for the
122
+ task. ``attempt`` is the one its ``task_submitted`` carried; ``category`` is the open
123
+ ``TerminationCategory`` vocabulary (``missing_output``, ...); ``reason`` is Snakemake's message,
124
+ cut to ``REASON_MAX`` characters. Absent fields are omitted."""
125
+ body: dict[str, object] = {"kind": "engine_task_verdict", "task_key": tkey, "verdict": "failed"}
126
+ if attempt is not None:
127
+ body["attempt"] = attempt
128
+ if category is not None:
129
+ body["category"] = category
130
+ if reason:
131
+ body["reason"] = reason if len(reason) <= REASON_MAX else reason[: REASON_MAX - 1] + "…"
132
+ return _envelope(ts, run_id, body)
133
+
134
+
135
+ def _terminal(
136
+ ts: float, run_id: str, kind: str, tkey: str, attempt: int | None
137
+ ) -> dict[str, object]:
138
+ body: dict[str, object] = {"kind": kind, "task_key": tkey}
139
+ if attempt is not None:
140
+ body["attempt"] = attempt
141
+ return _envelope(ts, run_id, body)
142
+
143
+
144
+ def task_succeeded(
145
+ ts: float, run_id: str, tkey: str, attempt: int | None = None
146
+ ) -> dict[str, object]:
147
+ """Engine-reported success for a LOCAL rule (run on the coordinator, never submitted to Batch,
148
+ so it has no Batch terminal). Carries no batch_job_id; the reducer records it as a task-level
149
+ engine terminal. ``attempt`` is Snakemake's attempt number for the rule (it retries local rules
150
+ under ``--retries``): it puts each attempt's terminal under its own dedup key. Batch tasks get
151
+ their terminal from EventBridge, not this."""
152
+ return _terminal(ts, run_id, "task_succeeded", tkey, attempt)
153
+
154
+
155
+ def task_failed(ts: float, run_id: str, tkey: str, attempt: int | None = None) -> dict[str, object]:
156
+ """Engine-reported failure for a LOCAL rule (see [`task_succeeded`]). With ``attempt`` set, a
157
+ retried rule's second failure is not dropped as a redelivery of the first, and the reducer can
158
+ tell the failure a retry replaces from the retry's own failure."""
159
+ return _terminal(ts, run_id, "task_failed", tkey, attempt)
160
+
161
+
162
+ def task_retrying(
163
+ ts: float, run_id: str, tkey: str, prev_attempt: int, next_attempt: int
164
+ ) -> dict[str, object]:
165
+ return _envelope(
166
+ ts,
167
+ run_id,
168
+ {
169
+ "kind": "task_retrying",
170
+ "task_key": tkey,
171
+ "prev_attempt": prev_attempt,
172
+ "next_attempt": next_attempt,
173
+ },
174
+ )
175
+
176
+
177
+ def task_outputs(
178
+ ts: float,
179
+ run_id: str,
180
+ tkey: str,
181
+ outputs: list[str],
182
+ role: str | None = None,
183
+ attempt: int | None = None,
184
+ ) -> dict[str, object]:
185
+ """The set of S3 objects one task attempt produced in one ``role`` (``output``, ``log``,
186
+ ``benchmark``, ...; absent ⇒ ``output``). ``outputs`` are S3 URIs, emitted sorted and
187
+ de-duplicated so the same set always serializes identically (the reducer folds it as a
188
+ set-union keyed on ``s3_uri``).
189
+
190
+ Never sets ``reused``: Snakemake's logger records carry no cache-hit signal, so the plugin
191
+ cannot claim outputs came from a previous run."""
192
+ body: dict[str, object] = {
193
+ "kind": "task_outputs",
194
+ "task_key": tkey,
195
+ "outputs": [{"s3_uri": uri} for uri in sorted(set(outputs))],
196
+ }
197
+ if role is not None:
198
+ body["role"] = role
199
+ if attempt is not None:
200
+ body["attempt"] = attempt
201
+ return _envelope(ts, run_id, body)