snakemake-logger-plugin-biffix 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. snakemake_logger_plugin_biffix-0.2.2/.gitignore +43 -0
  2. snakemake_logger_plugin_biffix-0.2.2/LICENSE +21 -0
  3. snakemake_logger_plugin_biffix-0.2.2/PKG-INFO +130 -0
  4. snakemake_logger_plugin_biffix-0.2.2/README.md +102 -0
  5. snakemake_logger_plugin_biffix-0.2.2/pyproject.toml +87 -0
  6. snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/__init__.py +13 -0
  7. snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/events.py +201 -0
  8. snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/handler.py +583 -0
  9. snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/poster.py +579 -0
  10. snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/py.typed +0 -0
  11. snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/remote_adapter.py +68 -0
  12. snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/settings.py +48 -0
  13. snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/writer.py +63 -0
  14. snakemake_logger_plugin_biffix-0.2.2/tests/conftest.py +94 -0
  15. snakemake_logger_plugin_biffix-0.2.2/tests/test_emit_routing.py +155 -0
  16. snakemake_logger_plugin_biffix-0.2.2/tests/test_events.py +155 -0
  17. snakemake_logger_plugin_biffix-0.2.2/tests/test_ingest_limits.py +60 -0
  18. snakemake_logger_plugin_biffix-0.2.2/tests/test_local_rules.py +422 -0
  19. snakemake_logger_plugin_biffix-0.2.2/tests/test_outputs.py +142 -0
  20. snakemake_logger_plugin_biffix-0.2.2/tests/test_poster.py +748 -0
  21. snakemake_logger_plugin_biffix-0.2.2/tests/test_schema_conformance.py +185 -0
  22. snakemake_logger_plugin_biffix-0.2.2/tests/test_writer.py +151 -0
  23. snakemake_logger_plugin_biffix-0.2.2/uv.lock +660 -0
@@ -0,0 +1,43 @@
1
+ # scratch / handoffs — never committed
2
+ ephemeral/
3
+ scratchpad/
4
+
5
+ # OS / editor cruft
6
+ .DS_Store
7
+ *.swp
8
+ .idea/
9
+ .vscode/
10
+
11
+ # python
12
+ __pycache__/
13
+ *.py[cod]
14
+ .venv/
15
+ .pixi/
16
+
17
+ # rust
18
+ /target/
19
+
20
+ # per-tenant deploy configs — real deployment inputs carrying account/tenant/bucket names, kept
21
+ # local (only the documented infra/biffix-deploy.example.yaml is tracked). Never commit a real one.
22
+ infra/biffix-deploy.yaml
23
+ infra/biffix-deploy-*.yaml
24
+
25
+ # cdk / node
26
+ infra/cdk.out/
27
+ # releases infra/deploy.sh --release downloaded (biffix_infra.release)
28
+ infra/.release/
29
+ node_modules/
30
+
31
+ # svelte / ui
32
+ ui/build/
33
+ ui/.svelte-kit/
34
+
35
+ # python tooling caches
36
+ .pytest_cache/
37
+ .ruff_cache/
38
+
39
+ # ui dev-local env (points at the local mock; not committed)
40
+ ui/.env
41
+
42
+ # docker/coordinator/build-wheels.sh output (the base coordinator image build context)
43
+ /build/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Nils Homer
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,130 @@
1
+ Metadata-Version: 2.5
2
+ Name: snakemake-logger-plugin-biffix
3
+ Version: 0.2.2
4
+ Summary: Snakemake (>=9.0) logger plugin: emits biffix RunEvents to the ingest route + a durable events.jsonl.
5
+ Project-URL: Homepage, https://github.com/nh13/biffix
6
+ Project-URL: Repository, https://github.com/nh13/biffix
7
+ Project-URL: Source, https://github.com/nh13/biffix/tree/main/plugins/snakemake-logger-plugin-biffix
8
+ Project-URL: Issues, https://github.com/nh13/biffix/issues
9
+ Author: Nils Homer
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: aws-batch,biffix,bioinformatics,logger,plugin,snakemake,workflow
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Environment :: Plugins
15
+ Classifier: Intended Audience :: Science/Research
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3 :: Only
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
23
+ Classifier: Topic :: System :: Logging
24
+ Classifier: Typing :: Typed
25
+ Requires-Python: >=3.11
26
+ Requires-Dist: snakemake-interface-logger-plugins<3,>=2
27
+ Description-Content-Type: text/markdown
28
+
29
+ # snakemake-logger-plugin-biffix
30
+
31
+ A [Snakemake](https://snakemake.github.io) (9.0 or later) logger plugin that reports a workflow's
32
+ progress to [biffix](https://github.com/nh13/biffix), the AWS-native control plane for running and
33
+ tracking workflows. It is the Snakemake counterpart of the `nf-biffix` Nextflow plugin in
34
+ `../nf-biffix`, and both report the same wire contract (`schemas/wire.engine.schema.json`).
35
+
36
+ ## What it does
37
+
38
+ The plugin maps Snakemake's log records to biffix engine events (`run_started`, `run_progress`,
39
+ `run_dag`, `task_created`, `task_submitted`, `task_retrying`, `task_succeeded`, `task_failed`,
40
+ `task_outputs` and `engine_task_verdict`) and sends each one to two places:
41
+
42
+ - **The ingest route, live.** Events are buffered and POSTed in batches to
43
+ `{ingest_url}/events`, bearer-authenticated with the run's capability token, by a background
44
+ thread, so Snakemake's logging never waits on the network. Failed requests are retried or
45
+ dropped; a lost POST never fails the workflow.
46
+ - **A durable `events.jsonl`.** Every event is also appended to a local file, which the biffix
47
+ coordinator uploads and replays through ingest when the run exits, so an event the live POST
48
+ dropped still arrives (ingest dedupes the repeats).
49
+
50
+ For a job on AWS Batch, the plugin emits `task_submitted` carrying the Batch job id, read from the
51
+ payload the AWS Batch executor plugin attaches to its log records, and AWS EventBridge supplies the
52
+ job's terminal state. For a local rule, which runs on the coordinator and never reaches Batch, the
53
+ plugin emits the terminal event itself.
54
+
55
+ The plugin has no dependencies beyond `snakemake-interface-logger-plugins`; it uses only the
56
+ standard library for HTTP.
57
+
58
+ ## How biffix uses it
59
+
60
+ biffix builds the plugin into a wheel for the coordinator base image
61
+ (`docker/coordinator/build-wheels.sh`) and installs it into each Snakemake runtime layer
62
+ (`codebuild/runtime/build.sh`). The coordinator (`docker/coordinator/lib/snakemake.sh`) runs
63
+ Snakemake with `--logger biffix` and sets the variables below. The version in `pyproject.toml` must match
64
+ `logger_version` in `crates/biffix-workflow/src/runtime/pins.rs`; a test in `runtime/tests.rs`
65
+ checks it.
66
+
67
+ ## Configuration
68
+
69
+ Enable it with `snakemake --logger biffix`. Each setting comes from an environment variable or a
70
+ command-line option; a non-empty option wins.
71
+
72
+ | Environment variable | Option | Meaning |
73
+ | --- | --- | --- |
74
+ | `BIFFIX_RUN_ID` | `--logger-biffix-run-id` | The biffix run the events belong to. |
75
+ | `BIFFIX_INGEST_URL` | `--logger-biffix-ingest-url` | Base URL of the ingest route. |
76
+ | `BIFFIX_INGEST_TOKEN` | `--logger-biffix-ingest-token` | The run's capability token. It is a secret, so prefer the environment variable. |
77
+ | `BIFFIX_EVENT_FILE` | `--logger-biffix-event-file` | Name of the durable events file, relative to the directory Snakemake starts in (default `.biffix_events.jsonl`). |
78
+
79
+ Snakemake's own derived variables (`SNAKEMAKE_LOGGER_BIFFIX_*`) are not read; the `BIFFIX_*` names
80
+ are the only environment channel.
81
+
82
+ ## Development
83
+
84
+ ```sh
85
+ uv sync --dev
86
+ uv run ruff check . && uv run ruff format --check .
87
+ uv run validate-pyproject pyproject.toml && uv run mypy
88
+ uv run pytest
89
+ ```
90
+
91
+ The tests read files from the biffix repository's `schemas/` directory, so they run from a biffix
92
+ checkout: `test_schema_conformance.py` checks every emitted event against the engine wire schema,
93
+ and `test_ingest_limits.py` checks that the poster's limits (largest batch, largest body, buffer
94
+ size, retries and timeouts) equal `schemas/ingest-limits.json`, which biffix-core writes from the
95
+ constants the ingest route enforces.
96
+
97
+ ## License
98
+
99
+ MIT; see [LICENSE](https://github.com/nh13/biffix/blob/main/plugins/snakemake-logger-plugin-biffix/LICENSE).
100
+
101
+ ## NOTICE
102
+
103
+ Parts of `handler.py`, `remote_adapter.py` and `writer.py` (Snakemake log record parsing, the
104
+ remote executor payload adapter and the JSONL event writer) are adapted from the
105
+ `snakemake-logger-plugin-snakesee` package of [snakesee](https://github.com/fg-labs/snakesee).
106
+ This package does not depend on snakesee. snakesee is distributed under the following license:
107
+
108
+ ```text
109
+ MIT License
110
+
111
+ Copyright (c) 2024 Fulcrum Genomics LLC
112
+
113
+ Permission is hereby granted, free of charge, to any person obtaining a copy
114
+ of this software and associated documentation files (the "Software"), to deal
115
+ in the Software without restriction, including without limitation the rights
116
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
117
+ copies of the Software, and to permit persons to whom the Software is
118
+ furnished to do so, subject to the following conditions:
119
+
120
+ The above copyright notice and this permission notice shall be included in all
121
+ copies or substantial portions of the Software.
122
+
123
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
124
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
125
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
126
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
127
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
128
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
129
+ SOFTWARE.
130
+ ```
@@ -0,0 +1,102 @@
1
+ # snakemake-logger-plugin-biffix
2
+
3
+ A [Snakemake](https://snakemake.github.io) (9.0 or later) logger plugin that reports a workflow's
4
+ progress to [biffix](https://github.com/nh13/biffix), the AWS-native control plane for running and
5
+ tracking workflows. It is the Snakemake counterpart of the `nf-biffix` Nextflow plugin in
6
+ `../nf-biffix`, and both report the same wire contract (`schemas/wire.engine.schema.json`).
7
+
8
+ ## What it does
9
+
10
+ The plugin maps Snakemake's log records to biffix engine events (`run_started`, `run_progress`,
11
+ `run_dag`, `task_created`, `task_submitted`, `task_retrying`, `task_succeeded`, `task_failed`,
12
+ `task_outputs` and `engine_task_verdict`) and sends each one to two places:
13
+
14
+ - **The ingest route, live.** Events are buffered and POSTed in batches to
15
+ `{ingest_url}/events`, bearer-authenticated with the run's capability token, by a background
16
+ thread, so Snakemake's logging never waits on the network. Failed requests are retried or
17
+ dropped; a lost POST never fails the workflow.
18
+ - **A durable `events.jsonl`.** Every event is also appended to a local file, which the biffix
19
+ coordinator uploads and replays through ingest when the run exits, so an event the live POST
20
+ dropped still arrives (ingest dedupes the repeats).
21
+
22
+ For a job on AWS Batch, the plugin emits `task_submitted` carrying the Batch job id, read from the
23
+ payload the AWS Batch executor plugin attaches to its log records, and AWS EventBridge supplies the
24
+ job's terminal state. For a local rule, which runs on the coordinator and never reaches Batch, the
25
+ plugin emits the terminal event itself.
26
+
27
+ The plugin has no dependencies beyond `snakemake-interface-logger-plugins`; it uses only the
28
+ standard library for HTTP.
29
+
30
+ ## How biffix uses it
31
+
32
+ biffix builds the plugin into a wheel for the coordinator base image
33
+ (`docker/coordinator/build-wheels.sh`) and installs it into each Snakemake runtime layer
34
+ (`codebuild/runtime/build.sh`). The coordinator (`docker/coordinator/lib/snakemake.sh`) runs
35
+ Snakemake with `--logger biffix` and sets the variables below. The version in `pyproject.toml` must match
36
+ `logger_version` in `crates/biffix-workflow/src/runtime/pins.rs`; a test in `runtime/tests.rs`
37
+ checks it.
38
+
39
+ ## Configuration
40
+
41
+ Enable it with `snakemake --logger biffix`. Each setting comes from an environment variable or a
42
+ command-line option; a non-empty option wins.
43
+
44
+ | Environment variable | Option | Meaning |
45
+ | --- | --- | --- |
46
+ | `BIFFIX_RUN_ID` | `--logger-biffix-run-id` | The biffix run the events belong to. |
47
+ | `BIFFIX_INGEST_URL` | `--logger-biffix-ingest-url` | Base URL of the ingest route. |
48
+ | `BIFFIX_INGEST_TOKEN` | `--logger-biffix-ingest-token` | The run's capability token. It is a secret, so prefer the environment variable. |
49
+ | `BIFFIX_EVENT_FILE` | `--logger-biffix-event-file` | Name of the durable events file, relative to the directory Snakemake starts in (default `.biffix_events.jsonl`). |
50
+
51
+ Snakemake's own derived variables (`SNAKEMAKE_LOGGER_BIFFIX_*`) are not read; the `BIFFIX_*` names
52
+ are the only environment channel.
53
+
54
+ ## Development
55
+
56
+ ```sh
57
+ uv sync --dev
58
+ uv run ruff check . && uv run ruff format --check .
59
+ uv run validate-pyproject pyproject.toml && uv run mypy
60
+ uv run pytest
61
+ ```
62
+
63
+ The tests read files from the biffix repository's `schemas/` directory, so they run from a biffix
64
+ checkout: `test_schema_conformance.py` checks every emitted event against the engine wire schema,
65
+ and `test_ingest_limits.py` checks that the poster's limits (largest batch, largest body, buffer
66
+ size, retries and timeouts) equal `schemas/ingest-limits.json`, which biffix-core writes from the
67
+ constants the ingest route enforces.
68
+
69
+ ## License
70
+
71
+ MIT; see [LICENSE](https://github.com/nh13/biffix/blob/main/plugins/snakemake-logger-plugin-biffix/LICENSE).
72
+
73
+ ## NOTICE
74
+
75
+ Parts of `handler.py`, `remote_adapter.py` and `writer.py` (Snakemake log record parsing, the
76
+ remote executor payload adapter and the JSONL event writer) are adapted from the
77
+ `snakemake-logger-plugin-snakesee` package of [snakesee](https://github.com/fg-labs/snakesee).
78
+ This package does not depend on snakesee. snakesee is distributed under the following license:
79
+
80
+ ```text
81
+ MIT License
82
+
83
+ Copyright (c) 2024 Fulcrum Genomics LLC
84
+
85
+ Permission is hereby granted, free of charge, to any person obtaining a copy
86
+ of this software and associated documentation files (the "Software"), to deal
87
+ in the Software without restriction, including without limitation the rights
88
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
89
+ copies of the Software, and to permit persons to whom the Software is
90
+ furnished to do so, subject to the following conditions:
91
+
92
+ The above copyright notice and this permission notice shall be included in all
93
+ copies or substantial portions of the Software.
94
+
95
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
96
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
97
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
98
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
99
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
100
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
101
+ SOFTWARE.
102
+ ```
@@ -0,0 +1,87 @@
1
+ [project]
2
+ name = "snakemake-logger-plugin-biffix"
3
+ version = "0.2.2"
4
+ description = "Snakemake (>=9.0) logger plugin: emits biffix RunEvents to the ingest route + a durable events.jsonl."
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ license-files = ["LICENSE"]
8
+ authors = [{ name = "Nils Homer" }]
9
+ requires-python = ">=3.11"
10
+ keywords = ["snakemake", "plugin", "logger", "biffix", "workflow", "aws-batch", "bioinformatics"]
11
+ classifiers = [
12
+ "Development Status :: 4 - Beta",
13
+ "Environment :: Plugins",
14
+ "Intended Audience :: Science/Research",
15
+ "Operating System :: OS Independent",
16
+ "Programming Language :: Python :: 3",
17
+ "Programming Language :: Python :: 3 :: Only",
18
+ "Programming Language :: Python :: 3.11",
19
+ "Programming Language :: Python :: 3.12",
20
+ "Programming Language :: Python :: 3.13",
21
+ "Topic :: Scientific/Engineering :: Bio-Informatics",
22
+ "Topic :: System :: Logging",
23
+ "Typing :: Typed",
24
+ ]
25
+ dependencies = [
26
+ "snakemake-interface-logger-plugins>=2,<3",
27
+ ]
28
+
29
+ [project.urls]
30
+ Homepage = "https://github.com/nh13/biffix"
31
+ Repository = "https://github.com/nh13/biffix"
32
+ Source = "https://github.com/nh13/biffix/tree/main/plugins/snakemake-logger-plugin-biffix"
33
+ Issues = "https://github.com/nh13/biffix/issues"
34
+
35
+ [dependency-groups]
36
+ dev = [
37
+ # Pinned exactly, to the same versions in infra/, docker/coordinator/ and
38
+ # plugins/snakemake-logger-plugin-biffix/, so every project lints alike locally and in CI.
39
+ # Dependabot bumps the three together (.github/dependabot.yml).
40
+ "mypy==2.4.0",
41
+ "pytest==9.1.1",
42
+ "ruff==0.17.1",
43
+ "validate-pyproject[all]==0.26",
44
+ # Required by the producer-conformance guard (test_schema_conformance.py), which fails
45
+ # rather than skips without it.
46
+ "jsonschema>=4",
47
+ "types-jsonschema>=4",
48
+ ]
49
+
50
+ [tool.mypy]
51
+ files = ["src", "tests"]
52
+ python_version = "3.11"
53
+ strict_optional = true
54
+ strict_equality = true
55
+ check_untyped_defs = true
56
+ disallow_any_generics = true
57
+ disallow_untyped_calls = true
58
+ disallow_untyped_decorators = true
59
+ disallow_untyped_defs = true
60
+ no_implicit_optional = true
61
+ warn_no_return = true
62
+ warn_redundant_casts = true
63
+ warn_return_any = true
64
+ warn_unreachable = true
65
+ warn_unused_configs = true
66
+ warn_unused_ignores = true
67
+ enable_error_code = ["ignore-without-code", "possibly-undefined", "exhaustive-match"]
68
+
69
+ [build-system]
70
+ requires = ["hatchling"]
71
+ build-backend = "hatchling.build"
72
+
73
+ [tool.hatch.build.targets.wheel]
74
+ packages = ["src/snakemake_logger_plugin_biffix"]
75
+
76
+ [tool.uv]
77
+ # Resolve only releases at least a week old, so a compromised release has time to be caught before
78
+ # it is locked. Keep it >= the Dependabot cooldown (7 days, .github/dependabot.yml), or Dependabot
79
+ # proposes versions this project refuses to lock.
80
+ exclude-newer = "1 week"
81
+
82
+ [tool.ruff]
83
+ line-length = 100
84
+ target-version = "py311"
85
+
86
+ [tool.ruff.lint]
87
+ select = ["E", "F", "I", "UP", "B", "ARG", "N", "LOG", "RUF008", "RUF012", "RUF024"]
@@ -0,0 +1,13 @@
1
+ """biffix Snakemake logger plugin (Snakemake >= 9.0).
2
+
3
+ Emits biffix ``EngineEvent`` RunEvents (the producer contract,
4
+ ``schemas/wire.engine.schema.json``) both to the biffix ingest route (live) and to a durable
5
+ ``events.jsonl`` (the archival replay copy). LogRecord-parsing logic is adapted from snakesee's
6
+ logger plugin (MIT; see the NOTICE section of README.md); this package has **no dependency** on
7
+ snakesee.
8
+ """
9
+
10
+ from snakemake_logger_plugin_biffix.handler import LogHandler
11
+ from snakemake_logger_plugin_biffix.settings import LogHandlerSettings
12
+
13
+ __all__ = ["LogHandler", "LogHandlerSettings"]
@@ -0,0 +1,201 @@
1
+ """The biffix ``EngineEvent`` wire shape.
2
+
3
+ These builders produce dicts that validate against ``schemas/wire.engine.schema.json`` (the
4
+ artifact ``biffix-core`` emits). The Rust ``wire::engine`` enum is the source of truth; this
5
+ module binds to it as a wire protocol. ``kind`` is the tag; ``task_key`` is
6
+ ``rule:sorted(wildcards)`` (canonical, order-independent).
7
+
8
+ There is a builder for every kind the Snakemake handler emits: ``run_started``, ``run_progress``,
9
+ ``run_dag``, ``task_created``, ``task_submitted``, ``task_retrying``, ``task_succeeded``,
10
+ ``task_failed``, ``task_outputs``, ``engine_task_verdict``. The other schema kinds are not emitted
11
+ by this plugin:
12
+ ``run_succeeded``/``run_failed`` (the coordinator that wraps the Snakemake invocation emits the run
13
+ terminal) and ``task_cached`` (the logger records carry no cache-hit signal: the cache decision
14
+ happens inside the executor, decoupled from the log events, so a cached rule is reported as
15
+ ``task_succeeded``). The conformance guard (``test_schema_conformance.py``) makes every schema kind
16
+ either have a builder here or be listed there as not emitted, so a new kind cannot slip through.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ SCHEMA_VERSION = 1
22
+
23
+
24
+ def task_key(rule: str, wildcards: dict[str, str] | None) -> str:
25
+ """Canonical stable logical-task identity: ``rule:sorted(wildcards)``.
26
+
27
+ Known limitation: names and values are joined without escaping, so a wildcard value that
28
+ itself contains ``,`` or ``=`` can collide with a different wildcard set (``{"a": "1,b=2"}``
29
+ and ``{"a": "1", "b": "2"}`` both give ``r:a=1,b=2``). The format is shared with every other
30
+ task_key producer and consumer (already-stored keys, the joins on them, the backend's
31
+ rule-from-key parse), so it is kept as is, an accepted limitation until a real workflow hits
32
+ it (escaping would be a contract change every producer and consumer adopts together);
33
+ ``tests/test_events.py`` pins it. Snakemake's default wildcard regex allows these characters,
34
+ so a workflow that needs distinct tasks for such values should constrain its wildcards."""
35
+ if not wildcards:
36
+ return rule
37
+ inner = ",".join(f"{k}={wildcards[k]}" for k in sorted(wildcards))
38
+ return f"{rule}:{inner}"
39
+
40
+
41
+ def _envelope(ts: float, run_id: str, body: dict[str, object]) -> dict[str, object]:
42
+ return {"schema_version": SCHEMA_VERSION, "ts": ts, "run_id": run_id, **body}
43
+
44
+
45
+ def run_started(ts: float, run_id: str) -> dict[str, object]:
46
+ return _envelope(ts, run_id, {"kind": "run_started"})
47
+
48
+
49
+ def run_progress(ts: float, run_id: str, completed: int, total: int) -> dict[str, object]:
50
+ return _envelope(ts, run_id, {"kind": "run_progress", "completed": completed, "total": total})
51
+
52
+
53
+ def task_created(
54
+ ts: float,
55
+ run_id: str,
56
+ tkey: str,
57
+ group: str | None = None,
58
+ deps: list[str] | None = None,
59
+ ) -> dict[str, object]:
60
+ body: dict[str, object] = {"kind": "task_created", "task_key": tkey}
61
+ if group is not None:
62
+ # The DAG node (rule) this task rolls up onto, for the run's graph view.
63
+ body["group"] = group
64
+ if deps:
65
+ # The in-run tasks that produced this task's inputs (its upstream), sorted for stability.
66
+ body["deps"] = sorted(set(deps))
67
+ return _envelope(ts, run_id, body)
68
+
69
+
70
+ def run_dag(
71
+ ts: float, run_id: str, nodes: list[str], edges: list[tuple[str, str]]
72
+ ) -> dict[str, object]:
73
+ """The run's group-level rule graph, emitted once. Folds as a set-union in the reducer."""
74
+ return _envelope(
75
+ ts,
76
+ run_id,
77
+ {
78
+ "kind": "run_dag",
79
+ "dag": {
80
+ "nodes": [{"id": n} for n in nodes],
81
+ "edges": [{"from": f, "to": t} for f, t in edges],
82
+ },
83
+ },
84
+ )
85
+
86
+
87
+ def task_submitted(
88
+ ts: float,
89
+ run_id: str,
90
+ tkey: str,
91
+ attempt: int,
92
+ batch_job_id: str,
93
+ image: str | None = None,
94
+ ) -> dict[str, object]:
95
+ """An attempt submitted to Batch: binds ``batch_job_id`` to the task. ``image`` is the
96
+ container the Batch job runs, as the executor reports it; omitted when unknown."""
97
+ body: dict[str, object] = {
98
+ "kind": "task_submitted",
99
+ "task_key": tkey,
100
+ "attempt": attempt,
101
+ "batch_job_id": batch_job_id,
102
+ }
103
+ if image:
104
+ body["image"] = image
105
+ return _envelope(ts, run_id, body)
106
+
107
+
108
+ # The longest ``reason`` an ``engine_task_verdict`` carries; a longer one is cut.
109
+ REASON_MAX = 1000
110
+
111
+
112
+ def engine_task_verdict(
113
+ ts: float,
114
+ run_id: str,
115
+ tkey: str,
116
+ attempt: int | None = None,
117
+ category: str | None = None,
118
+ reason: str | None = None,
119
+ ) -> dict[str, object]:
120
+ """Snakemake failed a Batch attempt that AWS may report succeeded (its declared outputs are
121
+ missing after the job exited 0): ``verdict: failed``, which outranks AWS's success for the
122
+ task. ``attempt`` is the one its ``task_submitted`` carried; ``category`` is the open
123
+ ``TerminationCategory`` vocabulary (``missing_output``, ...); ``reason`` is Snakemake's message,
124
+ cut to ``REASON_MAX`` characters. Absent fields are omitted."""
125
+ body: dict[str, object] = {"kind": "engine_task_verdict", "task_key": tkey, "verdict": "failed"}
126
+ if attempt is not None:
127
+ body["attempt"] = attempt
128
+ if category is not None:
129
+ body["category"] = category
130
+ if reason:
131
+ body["reason"] = reason if len(reason) <= REASON_MAX else reason[: REASON_MAX - 1] + "…"
132
+ return _envelope(ts, run_id, body)
133
+
134
+
135
+ def _terminal(
136
+ ts: float, run_id: str, kind: str, tkey: str, attempt: int | None
137
+ ) -> dict[str, object]:
138
+ body: dict[str, object] = {"kind": kind, "task_key": tkey}
139
+ if attempt is not None:
140
+ body["attempt"] = attempt
141
+ return _envelope(ts, run_id, body)
142
+
143
+
144
+ def task_succeeded(
145
+ ts: float, run_id: str, tkey: str, attempt: int | None = None
146
+ ) -> dict[str, object]:
147
+ """Engine-reported success for a LOCAL rule (run on the coordinator, never submitted to Batch,
148
+ so it has no Batch terminal). Carries no batch_job_id; the reducer records it as a task-level
149
+ engine terminal. ``attempt`` is Snakemake's attempt number for the rule (it retries local rules
150
+ under ``--retries``): it puts each attempt's terminal under its own dedup key. Batch tasks get
151
+ their terminal from EventBridge, not this."""
152
+ return _terminal(ts, run_id, "task_succeeded", tkey, attempt)
153
+
154
+
155
+ def task_failed(ts: float, run_id: str, tkey: str, attempt: int | None = None) -> dict[str, object]:
156
+ """Engine-reported failure for a LOCAL rule (see [`task_succeeded`]). With ``attempt`` set, a
157
+ retried rule's second failure is not dropped as a redelivery of the first, and the reducer can
158
+ tell the failure a retry replaces from the retry's own failure."""
159
+ return _terminal(ts, run_id, "task_failed", tkey, attempt)
160
+
161
+
162
+ def task_retrying(
163
+ ts: float, run_id: str, tkey: str, prev_attempt: int, next_attempt: int
164
+ ) -> dict[str, object]:
165
+ return _envelope(
166
+ ts,
167
+ run_id,
168
+ {
169
+ "kind": "task_retrying",
170
+ "task_key": tkey,
171
+ "prev_attempt": prev_attempt,
172
+ "next_attempt": next_attempt,
173
+ },
174
+ )
175
+
176
+
177
+ def task_outputs(
178
+ ts: float,
179
+ run_id: str,
180
+ tkey: str,
181
+ outputs: list[str],
182
+ role: str | None = None,
183
+ attempt: int | None = None,
184
+ ) -> dict[str, object]:
185
+ """The set of S3 objects one task attempt produced in one ``role`` (``output``, ``log``,
186
+ ``benchmark``, ...; absent ⇒ ``output``). ``outputs`` are S3 URIs, emitted sorted and
187
+ de-duplicated so the same set always serializes identically (the reducer folds it as a
188
+ set-union keyed on ``s3_uri``).
189
+
190
+ Never sets ``reused``: Snakemake's logger records carry no cache-hit signal, so the plugin
191
+ cannot claim outputs came from a previous run."""
192
+ body: dict[str, object] = {
193
+ "kind": "task_outputs",
194
+ "task_key": tkey,
195
+ "outputs": [{"s3_uri": uri} for uri in sorted(set(outputs))],
196
+ }
197
+ if role is not None:
198
+ body["role"] = role
199
+ if attempt is not None:
200
+ body["attempt"] = attempt
201
+ return _envelope(ts, run_id, body)