snakemake-logger-plugin-biffix 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snakemake_logger_plugin_biffix-0.2.2/.gitignore +43 -0
- snakemake_logger_plugin_biffix-0.2.2/LICENSE +21 -0
- snakemake_logger_plugin_biffix-0.2.2/PKG-INFO +130 -0
- snakemake_logger_plugin_biffix-0.2.2/README.md +102 -0
- snakemake_logger_plugin_biffix-0.2.2/pyproject.toml +87 -0
- snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/__init__.py +13 -0
- snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/events.py +201 -0
- snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/handler.py +583 -0
- snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/poster.py +579 -0
- snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/py.typed +0 -0
- snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/remote_adapter.py +68 -0
- snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/settings.py +48 -0
- snakemake_logger_plugin_biffix-0.2.2/src/snakemake_logger_plugin_biffix/writer.py +63 -0
- snakemake_logger_plugin_biffix-0.2.2/tests/conftest.py +94 -0
- snakemake_logger_plugin_biffix-0.2.2/tests/test_emit_routing.py +155 -0
- snakemake_logger_plugin_biffix-0.2.2/tests/test_events.py +155 -0
- snakemake_logger_plugin_biffix-0.2.2/tests/test_ingest_limits.py +60 -0
- snakemake_logger_plugin_biffix-0.2.2/tests/test_local_rules.py +422 -0
- snakemake_logger_plugin_biffix-0.2.2/tests/test_outputs.py +142 -0
- snakemake_logger_plugin_biffix-0.2.2/tests/test_poster.py +748 -0
- snakemake_logger_plugin_biffix-0.2.2/tests/test_schema_conformance.py +185 -0
- snakemake_logger_plugin_biffix-0.2.2/tests/test_writer.py +151 -0
- snakemake_logger_plugin_biffix-0.2.2/uv.lock +660 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# scratch / handoffs — never committed
|
|
2
|
+
ephemeral/
|
|
3
|
+
scratchpad/
|
|
4
|
+
|
|
5
|
+
# OS / editor cruft
|
|
6
|
+
.DS_Store
|
|
7
|
+
*.swp
|
|
8
|
+
.idea/
|
|
9
|
+
.vscode/
|
|
10
|
+
|
|
11
|
+
# python
|
|
12
|
+
__pycache__/
|
|
13
|
+
*.py[cod]
|
|
14
|
+
.venv/
|
|
15
|
+
.pixi/
|
|
16
|
+
|
|
17
|
+
# rust
|
|
18
|
+
/target/
|
|
19
|
+
|
|
20
|
+
# per-tenant deploy configs — real deployment inputs carrying account/tenant/bucket names, kept
|
|
21
|
+
# local (only the documented infra/biffix-deploy.example.yaml is tracked). Never commit a real one.
|
|
22
|
+
infra/biffix-deploy.yaml
|
|
23
|
+
infra/biffix-deploy-*.yaml
|
|
24
|
+
|
|
25
|
+
# cdk / node
|
|
26
|
+
infra/cdk.out/
|
|
27
|
+
# releases infra/deploy.sh --release downloaded (biffix_infra.release)
|
|
28
|
+
infra/.release/
|
|
29
|
+
node_modules/
|
|
30
|
+
|
|
31
|
+
# svelte / ui
|
|
32
|
+
ui/build/
|
|
33
|
+
ui/.svelte-kit/
|
|
34
|
+
|
|
35
|
+
# python tooling caches
|
|
36
|
+
.pytest_cache/
|
|
37
|
+
.ruff_cache/
|
|
38
|
+
|
|
39
|
+
# ui dev-local env (points at the local mock; not committed)
|
|
40
|
+
ui/.env
|
|
41
|
+
|
|
42
|
+
# docker/coordinator/build-wheels.sh output (the base coordinator image build context)
|
|
43
|
+
/build/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nils Homer
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: snakemake-logger-plugin-biffix
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Snakemake (>=9.0) logger plugin: emits biffix RunEvents to the ingest route + a durable events.jsonl.
|
|
5
|
+
Project-URL: Homepage, https://github.com/nh13/biffix
|
|
6
|
+
Project-URL: Repository, https://github.com/nh13/biffix
|
|
7
|
+
Project-URL: Source, https://github.com/nh13/biffix/tree/main/plugins/snakemake-logger-plugin-biffix
|
|
8
|
+
Project-URL: Issues, https://github.com/nh13/biffix/issues
|
|
9
|
+
Author: Nils Homer
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: aws-batch,biffix,bioinformatics,logger,plugin,snakemake,workflow
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Environment :: Plugins
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
23
|
+
Classifier: Topic :: System :: Logging
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Requires-Dist: snakemake-interface-logger-plugins<3,>=2
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# snakemake-logger-plugin-biffix
|
|
30
|
+
|
|
31
|
+
A [Snakemake](https://snakemake.github.io) (9.0 or later) logger plugin that reports a workflow's
|
|
32
|
+
progress to [biffix](https://github.com/nh13/biffix), the AWS-native control plane for running and
|
|
33
|
+
tracking workflows. It is the Snakemake counterpart of the `nf-biffix` Nextflow plugin in
|
|
34
|
+
`../nf-biffix`, and both report the same wire contract (`schemas/wire.engine.schema.json`).
|
|
35
|
+
|
|
36
|
+
## What it does
|
|
37
|
+
|
|
38
|
+
The plugin maps Snakemake's log records to biffix engine events (`run_started`, `run_progress`,
|
|
39
|
+
`run_dag`, `task_created`, `task_submitted`, `task_retrying`, `task_succeeded`, `task_failed`,
|
|
40
|
+
`task_outputs` and `engine_task_verdict`) and sends each one to two places:
|
|
41
|
+
|
|
42
|
+
- **The ingest route, live.** Events are buffered and POSTed in batches to
|
|
43
|
+
`{ingest_url}/events`, bearer-authenticated with the run's capability token, by a background
|
|
44
|
+
thread, so Snakemake's logging never waits on the network. Failed requests are retried or
|
|
45
|
+
dropped; a lost POST never fails the workflow.
|
|
46
|
+
- **A durable `events.jsonl`.** Every event is also appended to a local file, which the biffix
|
|
47
|
+
coordinator uploads and replays through ingest when the run exits, so an event the live POST
|
|
48
|
+
dropped still arrives (ingest dedupes the repeats).
|
|
49
|
+
|
|
50
|
+
For a job on AWS Batch, the plugin emits `task_submitted` carrying the Batch job id, read from the
|
|
51
|
+
payload the AWS Batch executor plugin attaches to its log records, and AWS EventBridge supplies the
|
|
52
|
+
job's terminal state. For a local rule, which runs on the coordinator and never reaches Batch, the
|
|
53
|
+
plugin emits the terminal event itself.
|
|
54
|
+
|
|
55
|
+
The plugin has no dependencies beyond `snakemake-interface-logger-plugins`; it uses only the
|
|
56
|
+
standard library for HTTP.
|
|
57
|
+
|
|
58
|
+
## How biffix uses it
|
|
59
|
+
|
|
60
|
+
biffix builds the plugin into a wheel for the coordinator base image
|
|
61
|
+
(`docker/coordinator/build-wheels.sh`) and installs it into each Snakemake runtime layer
|
|
62
|
+
(`codebuild/runtime/build.sh`). The coordinator (`docker/coordinator/lib/snakemake.sh`) runs
|
|
63
|
+
Snakemake with `--logger biffix` and sets the variables below. The version in `pyproject.toml` must match
|
|
64
|
+
`logger_version` in `crates/biffix-workflow/src/runtime/pins.rs`; a test in `runtime/tests.rs`
|
|
65
|
+
checks it.
|
|
66
|
+
|
|
67
|
+
## Configuration
|
|
68
|
+
|
|
69
|
+
Enable it with `snakemake --logger biffix`. Each setting comes from an environment variable or a
|
|
70
|
+
command-line option; a non-empty option wins.
|
|
71
|
+
|
|
72
|
+
| Environment variable | Option | Meaning |
|
|
73
|
+
| --- | --- | --- |
|
|
74
|
+
| `BIFFIX_RUN_ID` | `--logger-biffix-run-id` | The biffix run the events belong to. |
|
|
75
|
+
| `BIFFIX_INGEST_URL` | `--logger-biffix-ingest-url` | Base URL of the ingest route. |
|
|
76
|
+
| `BIFFIX_INGEST_TOKEN` | `--logger-biffix-ingest-token` | The run's capability token. It is a secret, so prefer the environment variable. |
|
|
77
|
+
| `BIFFIX_EVENT_FILE` | `--logger-biffix-event-file` | Name of the durable events file, relative to the directory Snakemake starts in (default `.biffix_events.jsonl`). |
|
|
78
|
+
|
|
79
|
+
Snakemake's own derived variables (`SNAKEMAKE_LOGGER_BIFFIX_*`) are not read; the `BIFFIX_*` names
|
|
80
|
+
are the only environment channel.
|
|
81
|
+
|
|
82
|
+
## Development
|
|
83
|
+
|
|
84
|
+
```sh
|
|
85
|
+
uv sync --dev
|
|
86
|
+
uv run ruff check . && uv run ruff format --check .
|
|
87
|
+
uv run validate-pyproject pyproject.toml && uv run mypy
|
|
88
|
+
uv run pytest
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
The tests read files from the biffix repository's `schemas/` directory, so they run from a biffix
|
|
92
|
+
checkout: `test_schema_conformance.py` checks every emitted event against the engine wire schema,
|
|
93
|
+
and `test_ingest_limits.py` checks that the poster's limits (largest batch, largest body, buffer
|
|
94
|
+
size, retries and timeouts) equal `schemas/ingest-limits.json`, which biffix-core writes from the
|
|
95
|
+
constants the ingest route enforces.
|
|
96
|
+
|
|
97
|
+
## License
|
|
98
|
+
|
|
99
|
+
MIT; see [LICENSE](https://github.com/nh13/biffix/blob/main/plugins/snakemake-logger-plugin-biffix/LICENSE).
|
|
100
|
+
|
|
101
|
+
## NOTICE
|
|
102
|
+
|
|
103
|
+
Parts of `handler.py`, `remote_adapter.py` and `writer.py` (Snakemake log record parsing, the
|
|
104
|
+
remote executor payload adapter and the JSONL event writer) are adapted from the
|
|
105
|
+
`snakemake-logger-plugin-snakesee` package of [snakesee](https://github.com/fg-labs/snakesee).
|
|
106
|
+
This package does not depend on snakesee. snakesee is distributed under the following license:
|
|
107
|
+
|
|
108
|
+
```text
|
|
109
|
+
MIT License
|
|
110
|
+
|
|
111
|
+
Copyright (c) 2024 Fulcrum Genomics LLC
|
|
112
|
+
|
|
113
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
114
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
115
|
+
in the Software without restriction, including without limitation the rights
|
|
116
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
117
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
118
|
+
furnished to do so, subject to the following conditions:
|
|
119
|
+
|
|
120
|
+
The above copyright notice and this permission notice shall be included in all
|
|
121
|
+
copies or substantial portions of the Software.
|
|
122
|
+
|
|
123
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
124
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
125
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
126
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
127
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
128
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
129
|
+
SOFTWARE.
|
|
130
|
+
```
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# snakemake-logger-plugin-biffix
|
|
2
|
+
|
|
3
|
+
A [Snakemake](https://snakemake.github.io) (9.0 or later) logger plugin that reports a workflow's
|
|
4
|
+
progress to [biffix](https://github.com/nh13/biffix), the AWS-native control plane for running and
|
|
5
|
+
tracking workflows. It is the Snakemake counterpart of the `nf-biffix` Nextflow plugin in
|
|
6
|
+
`../nf-biffix`, and both report the same wire contract (`schemas/wire.engine.schema.json`).
|
|
7
|
+
|
|
8
|
+
## What it does
|
|
9
|
+
|
|
10
|
+
The plugin maps Snakemake's log records to biffix engine events (`run_started`, `run_progress`,
|
|
11
|
+
`run_dag`, `task_created`, `task_submitted`, `task_retrying`, `task_succeeded`, `task_failed`,
|
|
12
|
+
`task_outputs` and `engine_task_verdict`) and sends each one to two places:
|
|
13
|
+
|
|
14
|
+
- **The ingest route, live.** Events are buffered and POSTed in batches to
|
|
15
|
+
`{ingest_url}/events`, bearer-authenticated with the run's capability token, by a background
|
|
16
|
+
thread, so Snakemake's logging never waits on the network. Failed requests are retried or
|
|
17
|
+
dropped; a lost POST never fails the workflow.
|
|
18
|
+
- **A durable `events.jsonl`.** Every event is also appended to a local file, which the biffix
|
|
19
|
+
coordinator uploads and replays through ingest when the run exits, so an event the live POST
|
|
20
|
+
dropped still arrives (ingest dedupes the repeats).
|
|
21
|
+
|
|
22
|
+
For a job on AWS Batch, the plugin emits `task_submitted` carrying the Batch job id, read from the
|
|
23
|
+
payload the AWS Batch executor plugin attaches to its log records, and AWS EventBridge supplies the
|
|
24
|
+
job's terminal state. For a local rule, which runs on the coordinator and never reaches Batch, the
|
|
25
|
+
plugin emits the terminal event itself.
|
|
26
|
+
|
|
27
|
+
The plugin has no dependencies beyond `snakemake-interface-logger-plugins`; it uses only the
|
|
28
|
+
standard library for HTTP.
|
|
29
|
+
|
|
30
|
+
## How biffix uses it
|
|
31
|
+
|
|
32
|
+
biffix builds the plugin into a wheel for the coordinator base image
|
|
33
|
+
(`docker/coordinator/build-wheels.sh`) and installs it into each Snakemake runtime layer
|
|
34
|
+
(`codebuild/runtime/build.sh`). The coordinator (`docker/coordinator/lib/snakemake.sh`) runs
|
|
35
|
+
Snakemake with `--logger biffix` and sets the variables below. The version in `pyproject.toml` must match
|
|
36
|
+
`logger_version` in `crates/biffix-workflow/src/runtime/pins.rs`; a test in `runtime/tests.rs`
|
|
37
|
+
checks it.
|
|
38
|
+
|
|
39
|
+
## Configuration
|
|
40
|
+
|
|
41
|
+
Enable it with `snakemake --logger biffix`. Each setting comes from an environment variable or a
|
|
42
|
+
command-line option; a non-empty option wins.
|
|
43
|
+
|
|
44
|
+
| Environment variable | Option | Meaning |
|
|
45
|
+
| --- | --- | --- |
|
|
46
|
+
| `BIFFIX_RUN_ID` | `--logger-biffix-run-id` | The biffix run the events belong to. |
|
|
47
|
+
| `BIFFIX_INGEST_URL` | `--logger-biffix-ingest-url` | Base URL of the ingest route. |
|
|
48
|
+
| `BIFFIX_INGEST_TOKEN` | `--logger-biffix-ingest-token` | The run's capability token. It is a secret, so prefer the environment variable. |
|
|
49
|
+
| `BIFFIX_EVENT_FILE` | `--logger-biffix-event-file` | Name of the durable events file, relative to the directory Snakemake starts in (default `.biffix_events.jsonl`). |
|
|
50
|
+
|
|
51
|
+
Snakemake's own derived variables (`SNAKEMAKE_LOGGER_BIFFIX_*`) are not read; the `BIFFIX_*` names
|
|
52
|
+
are the only environment channel.
|
|
53
|
+
|
|
54
|
+
## Development
|
|
55
|
+
|
|
56
|
+
```sh
|
|
57
|
+
uv sync --dev
|
|
58
|
+
uv run ruff check . && uv run ruff format --check .
|
|
59
|
+
uv run validate-pyproject pyproject.toml && uv run mypy
|
|
60
|
+
uv run pytest
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
The tests read files from the biffix repository's `schemas/` directory, so they run from a biffix
|
|
64
|
+
checkout: `test_schema_conformance.py` checks every emitted event against the engine wire schema,
|
|
65
|
+
and `test_ingest_limits.py` checks that the poster's limits (largest batch, largest body, buffer
|
|
66
|
+
size, retries and timeouts) equal `schemas/ingest-limits.json`, which biffix-core writes from the
|
|
67
|
+
constants the ingest route enforces.
|
|
68
|
+
|
|
69
|
+
## License
|
|
70
|
+
|
|
71
|
+
MIT; see [LICENSE](https://github.com/nh13/biffix/blob/main/plugins/snakemake-logger-plugin-biffix/LICENSE).
|
|
72
|
+
|
|
73
|
+
## NOTICE
|
|
74
|
+
|
|
75
|
+
Parts of `handler.py`, `remote_adapter.py` and `writer.py` (Snakemake log record parsing, the
|
|
76
|
+
remote executor payload adapter and the JSONL event writer) are adapted from the
|
|
77
|
+
`snakemake-logger-plugin-snakesee` package of [snakesee](https://github.com/fg-labs/snakesee).
|
|
78
|
+
This package does not depend on snakesee. snakesee is distributed under the following license:
|
|
79
|
+
|
|
80
|
+
```text
|
|
81
|
+
MIT License
|
|
82
|
+
|
|
83
|
+
Copyright (c) 2024 Fulcrum Genomics LLC
|
|
84
|
+
|
|
85
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
86
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
87
|
+
in the Software without restriction, including without limitation the rights
|
|
88
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
89
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
90
|
+
furnished to do so, subject to the following conditions:
|
|
91
|
+
|
|
92
|
+
The above copyright notice and this permission notice shall be included in all
|
|
93
|
+
copies or substantial portions of the Software.
|
|
94
|
+
|
|
95
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
96
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
97
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
98
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
99
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
100
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
101
|
+
SOFTWARE.
|
|
102
|
+
```
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "snakemake-logger-plugin-biffix"
|
|
3
|
+
version = "0.2.2"
|
|
4
|
+
description = "Snakemake (>=9.0) logger plugin: emits biffix RunEvents to the ingest route + a durable events.jsonl."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [{ name = "Nils Homer" }]
|
|
9
|
+
requires-python = ">=3.11"
|
|
10
|
+
keywords = ["snakemake", "plugin", "logger", "biffix", "workflow", "aws-batch", "bioinformatics"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Development Status :: 4 - Beta",
|
|
13
|
+
"Environment :: Plugins",
|
|
14
|
+
"Intended Audience :: Science/Research",
|
|
15
|
+
"Operating System :: OS Independent",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
|
22
|
+
"Topic :: System :: Logging",
|
|
23
|
+
"Typing :: Typed",
|
|
24
|
+
]
|
|
25
|
+
dependencies = [
|
|
26
|
+
"snakemake-interface-logger-plugins>=2,<3",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
[project.urls]
|
|
30
|
+
Homepage = "https://github.com/nh13/biffix"
|
|
31
|
+
Repository = "https://github.com/nh13/biffix"
|
|
32
|
+
Source = "https://github.com/nh13/biffix/tree/main/plugins/snakemake-logger-plugin-biffix"
|
|
33
|
+
Issues = "https://github.com/nh13/biffix/issues"
|
|
34
|
+
|
|
35
|
+
[dependency-groups]
|
|
36
|
+
dev = [
|
|
37
|
+
# Pinned exactly, to the same versions in infra/, docker/coordinator/ and
|
|
38
|
+
# plugins/snakemake-logger-plugin-biffix/, so every project lints alike locally and in CI.
|
|
39
|
+
# Dependabot bumps the three together (.github/dependabot.yml).
|
|
40
|
+
"mypy==2.4.0",
|
|
41
|
+
"pytest==9.1.1",
|
|
42
|
+
"ruff==0.17.1",
|
|
43
|
+
"validate-pyproject[all]==0.26",
|
|
44
|
+
# Required by the producer-conformance guard (test_schema_conformance.py), which fails
|
|
45
|
+
# rather than skips without it.
|
|
46
|
+
"jsonschema>=4",
|
|
47
|
+
"types-jsonschema>=4",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[tool.mypy]
|
|
51
|
+
files = ["src", "tests"]
|
|
52
|
+
python_version = "3.11"
|
|
53
|
+
strict_optional = true
|
|
54
|
+
strict_equality = true
|
|
55
|
+
check_untyped_defs = true
|
|
56
|
+
disallow_any_generics = true
|
|
57
|
+
disallow_untyped_calls = true
|
|
58
|
+
disallow_untyped_decorators = true
|
|
59
|
+
disallow_untyped_defs = true
|
|
60
|
+
no_implicit_optional = true
|
|
61
|
+
warn_no_return = true
|
|
62
|
+
warn_redundant_casts = true
|
|
63
|
+
warn_return_any = true
|
|
64
|
+
warn_unreachable = true
|
|
65
|
+
warn_unused_configs = true
|
|
66
|
+
warn_unused_ignores = true
|
|
67
|
+
enable_error_code = ["ignore-without-code", "possibly-undefined", "exhaustive-match"]
|
|
68
|
+
|
|
69
|
+
[build-system]
|
|
70
|
+
requires = ["hatchling"]
|
|
71
|
+
build-backend = "hatchling.build"
|
|
72
|
+
|
|
73
|
+
[tool.hatch.build.targets.wheel]
|
|
74
|
+
packages = ["src/snakemake_logger_plugin_biffix"]
|
|
75
|
+
|
|
76
|
+
[tool.uv]
|
|
77
|
+
# Resolve only releases at least a week old, so a compromised release has time to be caught before
|
|
78
|
+
# it is locked. Keep it >= the Dependabot cooldown (7 days, .github/dependabot.yml), or Dependabot
|
|
79
|
+
# proposes versions this project refuses to lock.
|
|
80
|
+
exclude-newer = "1 week"
|
|
81
|
+
|
|
82
|
+
[tool.ruff]
|
|
83
|
+
line-length = 100
|
|
84
|
+
target-version = "py311"
|
|
85
|
+
|
|
86
|
+
[tool.ruff.lint]
|
|
87
|
+
select = ["E", "F", "I", "UP", "B", "ARG", "N", "LOG", "RUF008", "RUF012", "RUF024"]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""biffix Snakemake logger plugin (Snakemake >= 9.0).
|
|
2
|
+
|
|
3
|
+
Emits biffix ``EngineEvent`` RunEvents (the producer contract,
|
|
4
|
+
``schemas/wire.engine.schema.json``) both to the biffix ingest route (live) and to a durable
|
|
5
|
+
``events.jsonl`` (the archival replay copy). LogRecord-parsing logic is adapted from snakesee's
|
|
6
|
+
logger plugin (MIT; see the NOTICE section of README.md); this package has **no dependency** on
|
|
7
|
+
snakesee.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from snakemake_logger_plugin_biffix.handler import LogHandler
|
|
11
|
+
from snakemake_logger_plugin_biffix.settings import LogHandlerSettings
|
|
12
|
+
|
|
13
|
+
__all__ = ["LogHandler", "LogHandlerSettings"]
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""The biffix ``EngineEvent`` wire shape.
|
|
2
|
+
|
|
3
|
+
These builders produce dicts that validate against ``schemas/wire.engine.schema.json`` (the
|
|
4
|
+
artifact ``biffix-core`` emits). The Rust ``wire::engine`` enum is the source of truth; this
|
|
5
|
+
module binds to it as a wire protocol. ``kind`` is the tag; ``task_key`` is
|
|
6
|
+
``rule:sorted(wildcards)`` (canonical, order-independent).
|
|
7
|
+
|
|
8
|
+
There is a builder for every kind the Snakemake handler emits: ``run_started``, ``run_progress``,
|
|
9
|
+
``run_dag``, ``task_created``, ``task_submitted``, ``task_retrying``, ``task_succeeded``,
|
|
10
|
+
``task_failed``, ``task_outputs``, ``engine_task_verdict``. The other schema kinds are not emitted
|
|
11
|
+
by this plugin:
|
|
12
|
+
``run_succeeded``/``run_failed`` (the coordinator that wraps the Snakemake invocation emits the run
|
|
13
|
+
terminal) and ``task_cached`` (the logger records carry no cache-hit signal: the cache decision
|
|
14
|
+
happens inside the executor, decoupled from the log events, so a cached rule is reported as
|
|
15
|
+
``task_succeeded``). The conformance guard (``test_schema_conformance.py``) makes every schema kind
|
|
16
|
+
either have a builder here or be listed there as not emitted, so a new kind cannot slip through.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
SCHEMA_VERSION = 1
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def task_key(rule: str, wildcards: dict[str, str] | None) -> str:
|
|
25
|
+
"""Canonical stable logical-task identity: ``rule:sorted(wildcards)``.
|
|
26
|
+
|
|
27
|
+
Known limitation: names and values are joined without escaping, so a wildcard value that
|
|
28
|
+
itself contains ``,`` or ``=`` can collide with a different wildcard set (``{"a": "1,b=2"}``
|
|
29
|
+
and ``{"a": "1", "b": "2"}`` both give ``r:a=1,b=2``). The format is shared with every other
|
|
30
|
+
task_key producer and consumer (already-stored keys, the joins on them, the backend's
|
|
31
|
+
rule-from-key parse), so it is kept as is, an accepted limitation until a real workflow hits
|
|
32
|
+
it (escaping would be a contract change every producer and consumer adopts together);
|
|
33
|
+
``tests/test_events.py`` pins it. Snakemake's default wildcard regex allows these characters,
|
|
34
|
+
so a workflow that needs distinct tasks for such values should constrain its wildcards."""
|
|
35
|
+
if not wildcards:
|
|
36
|
+
return rule
|
|
37
|
+
inner = ",".join(f"{k}={wildcards[k]}" for k in sorted(wildcards))
|
|
38
|
+
return f"{rule}:{inner}"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _envelope(ts: float, run_id: str, body: dict[str, object]) -> dict[str, object]:
|
|
42
|
+
return {"schema_version": SCHEMA_VERSION, "ts": ts, "run_id": run_id, **body}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def run_started(ts: float, run_id: str) -> dict[str, object]:
|
|
46
|
+
return _envelope(ts, run_id, {"kind": "run_started"})
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def run_progress(ts: float, run_id: str, completed: int, total: int) -> dict[str, object]:
|
|
50
|
+
return _envelope(ts, run_id, {"kind": "run_progress", "completed": completed, "total": total})
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def task_created(
|
|
54
|
+
ts: float,
|
|
55
|
+
run_id: str,
|
|
56
|
+
tkey: str,
|
|
57
|
+
group: str | None = None,
|
|
58
|
+
deps: list[str] | None = None,
|
|
59
|
+
) -> dict[str, object]:
|
|
60
|
+
body: dict[str, object] = {"kind": "task_created", "task_key": tkey}
|
|
61
|
+
if group is not None:
|
|
62
|
+
# The DAG node (rule) this task rolls up onto, for the run's graph view.
|
|
63
|
+
body["group"] = group
|
|
64
|
+
if deps:
|
|
65
|
+
# The in-run tasks that produced this task's inputs (its upstream), sorted for stability.
|
|
66
|
+
body["deps"] = sorted(set(deps))
|
|
67
|
+
return _envelope(ts, run_id, body)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def run_dag(
|
|
71
|
+
ts: float, run_id: str, nodes: list[str], edges: list[tuple[str, str]]
|
|
72
|
+
) -> dict[str, object]:
|
|
73
|
+
"""The run's group-level rule graph, emitted once. Folds as a set-union in the reducer."""
|
|
74
|
+
return _envelope(
|
|
75
|
+
ts,
|
|
76
|
+
run_id,
|
|
77
|
+
{
|
|
78
|
+
"kind": "run_dag",
|
|
79
|
+
"dag": {
|
|
80
|
+
"nodes": [{"id": n} for n in nodes],
|
|
81
|
+
"edges": [{"from": f, "to": t} for f, t in edges],
|
|
82
|
+
},
|
|
83
|
+
},
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def task_submitted(
|
|
88
|
+
ts: float,
|
|
89
|
+
run_id: str,
|
|
90
|
+
tkey: str,
|
|
91
|
+
attempt: int,
|
|
92
|
+
batch_job_id: str,
|
|
93
|
+
image: str | None = None,
|
|
94
|
+
) -> dict[str, object]:
|
|
95
|
+
"""An attempt submitted to Batch: binds ``batch_job_id`` to the task. ``image`` is the
|
|
96
|
+
container the Batch job runs, as the executor reports it; omitted when unknown."""
|
|
97
|
+
body: dict[str, object] = {
|
|
98
|
+
"kind": "task_submitted",
|
|
99
|
+
"task_key": tkey,
|
|
100
|
+
"attempt": attempt,
|
|
101
|
+
"batch_job_id": batch_job_id,
|
|
102
|
+
}
|
|
103
|
+
if image:
|
|
104
|
+
body["image"] = image
|
|
105
|
+
return _envelope(ts, run_id, body)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# The longest ``reason`` an ``engine_task_verdict`` carries; a longer one is cut.
|
|
109
|
+
REASON_MAX = 1000
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def engine_task_verdict(
|
|
113
|
+
ts: float,
|
|
114
|
+
run_id: str,
|
|
115
|
+
tkey: str,
|
|
116
|
+
attempt: int | None = None,
|
|
117
|
+
category: str | None = None,
|
|
118
|
+
reason: str | None = None,
|
|
119
|
+
) -> dict[str, object]:
|
|
120
|
+
"""Snakemake failed a Batch attempt that AWS may report succeeded (its declared outputs are
|
|
121
|
+
missing after the job exited 0): ``verdict: failed``, which outranks AWS's success for the
|
|
122
|
+
task. ``attempt`` is the one its ``task_submitted`` carried; ``category`` is the open
|
|
123
|
+
``TerminationCategory`` vocabulary (``missing_output``, ...); ``reason`` is Snakemake's message,
|
|
124
|
+
cut to ``REASON_MAX`` characters. Absent fields are omitted."""
|
|
125
|
+
body: dict[str, object] = {"kind": "engine_task_verdict", "task_key": tkey, "verdict": "failed"}
|
|
126
|
+
if attempt is not None:
|
|
127
|
+
body["attempt"] = attempt
|
|
128
|
+
if category is not None:
|
|
129
|
+
body["category"] = category
|
|
130
|
+
if reason:
|
|
131
|
+
body["reason"] = reason if len(reason) <= REASON_MAX else reason[: REASON_MAX - 1] + "…"
|
|
132
|
+
return _envelope(ts, run_id, body)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _terminal(
|
|
136
|
+
ts: float, run_id: str, kind: str, tkey: str, attempt: int | None
|
|
137
|
+
) -> dict[str, object]:
|
|
138
|
+
body: dict[str, object] = {"kind": kind, "task_key": tkey}
|
|
139
|
+
if attempt is not None:
|
|
140
|
+
body["attempt"] = attempt
|
|
141
|
+
return _envelope(ts, run_id, body)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def task_succeeded(
|
|
145
|
+
ts: float, run_id: str, tkey: str, attempt: int | None = None
|
|
146
|
+
) -> dict[str, object]:
|
|
147
|
+
"""Engine-reported success for a LOCAL rule (run on the coordinator, never submitted to Batch,
|
|
148
|
+
so it has no Batch terminal). Carries no batch_job_id; the reducer records it as a task-level
|
|
149
|
+
engine terminal. ``attempt`` is Snakemake's attempt number for the rule (it retries local rules
|
|
150
|
+
under ``--retries``): it puts each attempt's terminal under its own dedup key. Batch tasks get
|
|
151
|
+
their terminal from EventBridge, not this."""
|
|
152
|
+
return _terminal(ts, run_id, "task_succeeded", tkey, attempt)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def task_failed(ts: float, run_id: str, tkey: str, attempt: int | None = None) -> dict[str, object]:
|
|
156
|
+
"""Engine-reported failure for a LOCAL rule (see [`task_succeeded`]). With ``attempt`` set, a
|
|
157
|
+
retried rule's second failure is not dropped as a redelivery of the first, and the reducer can
|
|
158
|
+
tell the failure a retry replaces from the retry's own failure."""
|
|
159
|
+
return _terminal(ts, run_id, "task_failed", tkey, attempt)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def task_retrying(
|
|
163
|
+
ts: float, run_id: str, tkey: str, prev_attempt: int, next_attempt: int
|
|
164
|
+
) -> dict[str, object]:
|
|
165
|
+
return _envelope(
|
|
166
|
+
ts,
|
|
167
|
+
run_id,
|
|
168
|
+
{
|
|
169
|
+
"kind": "task_retrying",
|
|
170
|
+
"task_key": tkey,
|
|
171
|
+
"prev_attempt": prev_attempt,
|
|
172
|
+
"next_attempt": next_attempt,
|
|
173
|
+
},
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def task_outputs(
|
|
178
|
+
ts: float,
|
|
179
|
+
run_id: str,
|
|
180
|
+
tkey: str,
|
|
181
|
+
outputs: list[str],
|
|
182
|
+
role: str | None = None,
|
|
183
|
+
attempt: int | None = None,
|
|
184
|
+
) -> dict[str, object]:
|
|
185
|
+
"""The set of S3 objects one task attempt produced in one ``role`` (``output``, ``log``,
|
|
186
|
+
``benchmark``, ...; absent ⇒ ``output``). ``outputs`` are S3 URIs, emitted sorted and
|
|
187
|
+
de-duplicated so the same set always serializes identically (the reducer folds it as a
|
|
188
|
+
set-union keyed on ``s3_uri``).
|
|
189
|
+
|
|
190
|
+
Never sets ``reused``: Snakemake's logger records carry no cache-hit signal, so the plugin
|
|
191
|
+
cannot claim outputs came from a previous run."""
|
|
192
|
+
body: dict[str, object] = {
|
|
193
|
+
"kind": "task_outputs",
|
|
194
|
+
"task_key": tkey,
|
|
195
|
+
"outputs": [{"s3_uri": uri} for uri in sorted(set(outputs))],
|
|
196
|
+
}
|
|
197
|
+
if role is not None:
|
|
198
|
+
body["role"] = role
|
|
199
|
+
if attempt is not None:
|
|
200
|
+
body["attempt"] = attempt
|
|
201
|
+
return _envelope(ts, run_id, body)
|