snakemake-logger-plugin-biffix 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snakemake_logger_plugin_biffix/__init__.py +13 -0
- snakemake_logger_plugin_biffix/events.py +201 -0
- snakemake_logger_plugin_biffix/handler.py +583 -0
- snakemake_logger_plugin_biffix/poster.py +579 -0
- snakemake_logger_plugin_biffix/py.typed +0 -0
- snakemake_logger_plugin_biffix/remote_adapter.py +68 -0
- snakemake_logger_plugin_biffix/settings.py +48 -0
- snakemake_logger_plugin_biffix/writer.py +63 -0
- snakemake_logger_plugin_biffix-0.2.2.dist-info/METADATA +130 -0
- snakemake_logger_plugin_biffix-0.2.2.dist-info/RECORD +12 -0
- snakemake_logger_plugin_biffix-0.2.2.dist-info/WHEEL +4 -0
- snakemake_logger_plugin_biffix-0.2.2.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""biffix Snakemake logger plugin (Snakemake >= 9.0).
|
|
2
|
+
|
|
3
|
+
Emits biffix ``EngineEvent`` RunEvents (the producer contract,
|
|
4
|
+
``schemas/wire.engine.schema.json``) both to the biffix ingest route (live) and to a durable
|
|
5
|
+
``events.jsonl`` (the archival replay copy). LogRecord-parsing logic is adapted from snakesee's
|
|
6
|
+
logger plugin (MIT; see the NOTICE section of README.md); this package has **no dependency** on
|
|
7
|
+
snakesee.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from snakemake_logger_plugin_biffix.handler import LogHandler
|
|
11
|
+
from snakemake_logger_plugin_biffix.settings import LogHandlerSettings
|
|
12
|
+
|
|
13
|
+
__all__ = ["LogHandler", "LogHandlerSettings"]
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""The biffix ``EngineEvent`` wire shape.
|
|
2
|
+
|
|
3
|
+
These builders produce dicts that validate against ``schemas/wire.engine.schema.json`` (the
|
|
4
|
+
artifact ``biffix-core`` emits). The Rust ``wire::engine`` enum is the source of truth; this
|
|
5
|
+
module binds to it as a wire protocol. ``kind`` is the tag; ``task_key`` is
|
|
6
|
+
``rule:sorted(wildcards)`` (canonical, order-independent).
|
|
7
|
+
|
|
8
|
+
There is a builder for every kind the Snakemake handler emits: ``run_started``, ``run_progress``,
|
|
9
|
+
``run_dag``, ``task_created``, ``task_submitted``, ``task_retrying``, ``task_succeeded``,
|
|
10
|
+
``task_failed``, ``task_outputs``, ``engine_task_verdict``. The other schema kinds are not emitted
|
|
11
|
+
by this plugin:
|
|
12
|
+
``run_succeeded``/``run_failed`` (the coordinator that wraps the Snakemake invocation emits the run
|
|
13
|
+
terminal) and ``task_cached`` (the logger records carry no cache-hit signal: the cache decision
|
|
14
|
+
happens inside the executor, decoupled from the log events, so a cached rule is reported as
|
|
15
|
+
``task_succeeded``). The conformance guard (``test_schema_conformance.py``) makes every schema kind
|
|
16
|
+
either have a builder here or be listed there as not emitted, so a new kind cannot slip through.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
SCHEMA_VERSION = 1
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def task_key(rule: str, wildcards: dict[str, str] | None) -> str:
|
|
25
|
+
"""Canonical stable logical-task identity: ``rule:sorted(wildcards)``.
|
|
26
|
+
|
|
27
|
+
Known limitation: names and values are joined without escaping, so a wildcard value that
|
|
28
|
+
itself contains ``,`` or ``=`` can collide with a different wildcard set (``{"a": "1,b=2"}``
|
|
29
|
+
and ``{"a": "1", "b": "2"}`` both give ``r:a=1,b=2``). The format is shared with every other
|
|
30
|
+
task_key producer and consumer (already-stored keys, the joins on them, the backend's
|
|
31
|
+
rule-from-key parse), so it is kept as is, an accepted limitation until a real workflow hits
|
|
32
|
+
it (escaping would be a contract change every producer and consumer adopts together);
|
|
33
|
+
``tests/test_events.py`` pins it. Snakemake's default wildcard regex allows these characters,
|
|
34
|
+
so a workflow that needs distinct tasks for such values should constrain its wildcards."""
|
|
35
|
+
if not wildcards:
|
|
36
|
+
return rule
|
|
37
|
+
inner = ",".join(f"{k}={wildcards[k]}" for k in sorted(wildcards))
|
|
38
|
+
return f"{rule}:{inner}"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _envelope(ts: float, run_id: str, body: dict[str, object]) -> dict[str, object]:
|
|
42
|
+
return {"schema_version": SCHEMA_VERSION, "ts": ts, "run_id": run_id, **body}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def run_started(ts: float, run_id: str) -> dict[str, object]:
|
|
46
|
+
return _envelope(ts, run_id, {"kind": "run_started"})
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def run_progress(ts: float, run_id: str, completed: int, total: int) -> dict[str, object]:
|
|
50
|
+
return _envelope(ts, run_id, {"kind": "run_progress", "completed": completed, "total": total})
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def task_created(
|
|
54
|
+
ts: float,
|
|
55
|
+
run_id: str,
|
|
56
|
+
tkey: str,
|
|
57
|
+
group: str | None = None,
|
|
58
|
+
deps: list[str] | None = None,
|
|
59
|
+
) -> dict[str, object]:
|
|
60
|
+
body: dict[str, object] = {"kind": "task_created", "task_key": tkey}
|
|
61
|
+
if group is not None:
|
|
62
|
+
# The DAG node (rule) this task rolls up onto, for the run's graph view.
|
|
63
|
+
body["group"] = group
|
|
64
|
+
if deps:
|
|
65
|
+
# The in-run tasks that produced this task's inputs (its upstream), sorted for stability.
|
|
66
|
+
body["deps"] = sorted(set(deps))
|
|
67
|
+
return _envelope(ts, run_id, body)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def run_dag(
|
|
71
|
+
ts: float, run_id: str, nodes: list[str], edges: list[tuple[str, str]]
|
|
72
|
+
) -> dict[str, object]:
|
|
73
|
+
"""The run's group-level rule graph, emitted once. Folds as a set-union in the reducer."""
|
|
74
|
+
return _envelope(
|
|
75
|
+
ts,
|
|
76
|
+
run_id,
|
|
77
|
+
{
|
|
78
|
+
"kind": "run_dag",
|
|
79
|
+
"dag": {
|
|
80
|
+
"nodes": [{"id": n} for n in nodes],
|
|
81
|
+
"edges": [{"from": f, "to": t} for f, t in edges],
|
|
82
|
+
},
|
|
83
|
+
},
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def task_submitted(
|
|
88
|
+
ts: float,
|
|
89
|
+
run_id: str,
|
|
90
|
+
tkey: str,
|
|
91
|
+
attempt: int,
|
|
92
|
+
batch_job_id: str,
|
|
93
|
+
image: str | None = None,
|
|
94
|
+
) -> dict[str, object]:
|
|
95
|
+
"""An attempt submitted to Batch: binds ``batch_job_id`` to the task. ``image`` is the
|
|
96
|
+
container the Batch job runs, as the executor reports it; omitted when unknown."""
|
|
97
|
+
body: dict[str, object] = {
|
|
98
|
+
"kind": "task_submitted",
|
|
99
|
+
"task_key": tkey,
|
|
100
|
+
"attempt": attempt,
|
|
101
|
+
"batch_job_id": batch_job_id,
|
|
102
|
+
}
|
|
103
|
+
if image:
|
|
104
|
+
body["image"] = image
|
|
105
|
+
return _envelope(ts, run_id, body)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# The longest ``reason`` an ``engine_task_verdict`` carries; a longer one is cut.
|
|
109
|
+
REASON_MAX = 1000
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def engine_task_verdict(
|
|
113
|
+
ts: float,
|
|
114
|
+
run_id: str,
|
|
115
|
+
tkey: str,
|
|
116
|
+
attempt: int | None = None,
|
|
117
|
+
category: str | None = None,
|
|
118
|
+
reason: str | None = None,
|
|
119
|
+
) -> dict[str, object]:
|
|
120
|
+
"""Snakemake failed a Batch attempt that AWS may report succeeded (its declared outputs are
|
|
121
|
+
missing after the job exited 0): ``verdict: failed``, which outranks AWS's success for the
|
|
122
|
+
task. ``attempt`` is the one its ``task_submitted`` carried; ``category`` is the open
|
|
123
|
+
``TerminationCategory`` vocabulary (``missing_output``, ...); ``reason`` is Snakemake's message,
|
|
124
|
+
cut to ``REASON_MAX`` characters. Absent fields are omitted."""
|
|
125
|
+
body: dict[str, object] = {"kind": "engine_task_verdict", "task_key": tkey, "verdict": "failed"}
|
|
126
|
+
if attempt is not None:
|
|
127
|
+
body["attempt"] = attempt
|
|
128
|
+
if category is not None:
|
|
129
|
+
body["category"] = category
|
|
130
|
+
if reason:
|
|
131
|
+
body["reason"] = reason if len(reason) <= REASON_MAX else reason[: REASON_MAX - 1] + "…"
|
|
132
|
+
return _envelope(ts, run_id, body)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _terminal(
|
|
136
|
+
ts: float, run_id: str, kind: str, tkey: str, attempt: int | None
|
|
137
|
+
) -> dict[str, object]:
|
|
138
|
+
body: dict[str, object] = {"kind": kind, "task_key": tkey}
|
|
139
|
+
if attempt is not None:
|
|
140
|
+
body["attempt"] = attempt
|
|
141
|
+
return _envelope(ts, run_id, body)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def task_succeeded(
|
|
145
|
+
ts: float, run_id: str, tkey: str, attempt: int | None = None
|
|
146
|
+
) -> dict[str, object]:
|
|
147
|
+
"""Engine-reported success for a LOCAL rule (run on the coordinator, never submitted to Batch,
|
|
148
|
+
so it has no Batch terminal). Carries no batch_job_id; the reducer records it as a task-level
|
|
149
|
+
engine terminal. ``attempt`` is Snakemake's attempt number for the rule (it retries local rules
|
|
150
|
+
under ``--retries``): it puts each attempt's terminal under its own dedup key. Batch tasks get
|
|
151
|
+
their terminal from EventBridge, not this."""
|
|
152
|
+
return _terminal(ts, run_id, "task_succeeded", tkey, attempt)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def task_failed(ts: float, run_id: str, tkey: str, attempt: int | None = None) -> dict[str, object]:
|
|
156
|
+
"""Engine-reported failure for a LOCAL rule (see [`task_succeeded`]). With ``attempt`` set, a
|
|
157
|
+
retried rule's second failure is not dropped as a redelivery of the first, and the reducer can
|
|
158
|
+
tell the failure a retry replaces from the retry's own failure."""
|
|
159
|
+
return _terminal(ts, run_id, "task_failed", tkey, attempt)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def task_retrying(
|
|
163
|
+
ts: float, run_id: str, tkey: str, prev_attempt: int, next_attempt: int
|
|
164
|
+
) -> dict[str, object]:
|
|
165
|
+
return _envelope(
|
|
166
|
+
ts,
|
|
167
|
+
run_id,
|
|
168
|
+
{
|
|
169
|
+
"kind": "task_retrying",
|
|
170
|
+
"task_key": tkey,
|
|
171
|
+
"prev_attempt": prev_attempt,
|
|
172
|
+
"next_attempt": next_attempt,
|
|
173
|
+
},
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def task_outputs(
|
|
178
|
+
ts: float,
|
|
179
|
+
run_id: str,
|
|
180
|
+
tkey: str,
|
|
181
|
+
outputs: list[str],
|
|
182
|
+
role: str | None = None,
|
|
183
|
+
attempt: int | None = None,
|
|
184
|
+
) -> dict[str, object]:
|
|
185
|
+
"""The set of S3 objects one task attempt produced in one ``role`` (``output``, ``log``,
|
|
186
|
+
``benchmark``, ...; absent ⇒ ``output``). ``outputs`` are S3 URIs, emitted sorted and
|
|
187
|
+
de-duplicated so the same set always serializes identically (the reducer folds it as a
|
|
188
|
+
set-union keyed on ``s3_uri``).
|
|
189
|
+
|
|
190
|
+
Never sets ``reused``: Snakemake's logger records carry no cache-hit signal, so the plugin
|
|
191
|
+
cannot claim outputs came from a previous run."""
|
|
192
|
+
body: dict[str, object] = {
|
|
193
|
+
"kind": "task_outputs",
|
|
194
|
+
"task_key": tkey,
|
|
195
|
+
"outputs": [{"s3_uri": uri} for uri in sorted(set(outputs))],
|
|
196
|
+
}
|
|
197
|
+
if role is not None:
|
|
198
|
+
body["role"] = role
|
|
199
|
+
if attempt is not None:
|
|
200
|
+
body["attempt"] = attempt
|
|
201
|
+
return _envelope(ts, run_id, body)
|