snakemake-logger-plugin-panoptes 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snakemake_logger_plugin_panoptes-0.1.1/.github/workflows/ci.yml +63 -0
- snakemake_logger_plugin_panoptes-0.1.1/.github/workflows/python-publish.yml +39 -0
- snakemake_logger_plugin_panoptes-0.1.1/.gitignore +16 -0
- snakemake_logger_plugin_panoptes-0.1.1/LICENSE +21 -0
- snakemake_logger_plugin_panoptes-0.1.1/PKG-INFO +78 -0
- snakemake_logger_plugin_panoptes-0.1.1/README.md +60 -0
- snakemake_logger_plugin_panoptes-0.1.1/pyproject.toml +34 -0
- snakemake_logger_plugin_panoptes-0.1.1/src/snakemake_logger_plugin_panoptes/__init__.py +81 -0
- snakemake_logger_plugin_panoptes-0.1.1/src/snakemake_logger_plugin_panoptes/log_handler.py +232 -0
- snakemake_logger_plugin_panoptes-0.1.1/src/snakemake_logger_plugin_panoptes/py.typed +0 -0
- snakemake_logger_plugin_panoptes-0.1.1/tests/test_plugin.py +170 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on: [push, pull_request]
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
test:
|
|
7
|
+
runs-on: ${{ matrix.os }}
|
|
8
|
+
strategy:
|
|
9
|
+
fail-fast: false
|
|
10
|
+
matrix:
|
|
11
|
+
os: [ubuntu-latest]
|
|
12
|
+
python-version: ['3.11', '3.12', '3.13', '3.14']
|
|
13
|
+
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
|
|
17
|
+
- name: Set up Python ${{ matrix.python-version }}
|
|
18
|
+
uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: ${{ matrix.python-version }}
|
|
21
|
+
|
|
22
|
+
- name: Install plugin + dev deps
|
|
23
|
+
run: |
|
|
24
|
+
python -m pip install --upgrade pip
|
|
25
|
+
pip install -e '.[dev]'
|
|
26
|
+
|
|
27
|
+
- name: Run unit tests
|
|
28
|
+
run: pytest -v
|
|
29
|
+
|
|
30
|
+
- name: Verify plugin is discoverable by Snakemake 9
|
|
31
|
+
run: |
|
|
32
|
+
pip install 'snakemake>=9'
|
|
33
|
+
python -c "
|
|
34
|
+
from snakemake_logger_plugin_panoptes import LogHandler, LogHandlerSettings
|
|
35
|
+
fields = sorted(LogHandlerSettings.__dataclass_fields__)
|
|
36
|
+
assert 'address' in fields and 'timeout' in fields, fields
|
|
37
|
+
print('plugin imports + settings exposed:', fields)
|
|
38
|
+
"
|
|
39
|
+
# Real wiring check: Snakemake should advertise `--logger-panoptes-address`
|
|
40
|
+
# in its help once the plugin is installed.
|
|
41
|
+
snakemake --help 2>&1 | grep -qi 'logger-panoptes-address' && echo 'snakemake sees the panoptes logger plugin'
|
|
42
|
+
|
|
43
|
+
build:
|
|
44
|
+
runs-on: ubuntu-latest
|
|
45
|
+
needs: test
|
|
46
|
+
steps:
|
|
47
|
+
- uses: actions/checkout@v4
|
|
48
|
+
- uses: actions/setup-python@v5
|
|
49
|
+
with:
|
|
50
|
+
python-version: '3.11'
|
|
51
|
+
- name: Build sdist and wheel
|
|
52
|
+
run: |
|
|
53
|
+
python -m pip install --upgrade pip build
|
|
54
|
+
python -m build
|
|
55
|
+
- name: Verify package contents
|
|
56
|
+
run: |
|
|
57
|
+
pip install twine
|
|
58
|
+
twine check dist/*
|
|
59
|
+
- name: Upload artifacts
|
|
60
|
+
uses: actions/upload-artifact@v4
|
|
61
|
+
with:
|
|
62
|
+
name: dist
|
|
63
|
+
path: dist/
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# This workflow will upload a Python Package using Twine when a release is created
|
|
2
|
+
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
|
|
3
|
+
|
|
4
|
+
# This workflow uses actions that are not certified by GitHub.
|
|
5
|
+
# They are provided by a third-party and are governed by
|
|
6
|
+
# separate terms of service, privacy policy, and support
|
|
7
|
+
# documentation.
|
|
8
|
+
|
|
9
|
+
name: Upload Python Package
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
release:
|
|
13
|
+
types: [published]
|
|
14
|
+
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
deploy:
|
|
20
|
+
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
|
|
23
|
+
steps:
|
|
24
|
+
- uses: actions/checkout@v3
|
|
25
|
+
- name: Set up Python
|
|
26
|
+
uses: actions/setup-python@v3
|
|
27
|
+
with:
|
|
28
|
+
python-version: '3.x'
|
|
29
|
+
- name: Install dependencies
|
|
30
|
+
run: |
|
|
31
|
+
python -m pip install --upgrade pip
|
|
32
|
+
pip install build
|
|
33
|
+
- name: Build package
|
|
34
|
+
run: python -m build
|
|
35
|
+
- name: Publish package
|
|
36
|
+
uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
|
|
37
|
+
with:
|
|
38
|
+
user: __token__
|
|
39
|
+
password: ${{ secrets.PYPI_API_TOKEN }}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 panoptes-organization
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: snakemake-logger-plugin-panoptes
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Snakemake logger plugin that forwards workflow events to a panoptes server.
|
|
5
|
+
Project-URL: repository, https://github.com/panoptes-organization/snakemake-logger-plugin-panoptes
|
|
6
|
+
Author: panoptes-organization
|
|
7
|
+
License: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Requires-Dist: requests>=2.22.0
|
|
14
|
+
Requires-Dist: snakemake-interface-logger-plugins<3,>=1.2.0
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest>=7; extra == 'dev'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# snakemake-logger-plugin-panoptes
|
|
20
|
+
|
|
21
|
+
A [Snakemake 9](https://snakemake.readthedocs.io/) logger plugin that forwards
|
|
22
|
+
workflow events to a running [panoptes](https://github.com/panoptes-organization/panoptes)
|
|
23
|
+
server, replacing the legacy `--wms-monitor` integration that was removed in
|
|
24
|
+
Snakemake 9.
|
|
25
|
+
|
|
26
|
+
## Install
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install snakemake-logger-plugin-panoptes
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Use
|
|
33
|
+
|
|
34
|
+
Start a panoptes server (see the panoptes README), then run Snakemake with the
|
|
35
|
+
plugin enabled:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
snakemake \
|
|
39
|
+
--cores 1 \
|
|
40
|
+
--logger panoptes \
|
|
41
|
+
--logger-panoptes-address http://127.0.0.1:5000
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
You can also point at the address via an environment variable:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
export SNAKEMAKE_LOGGER_PANOPTES_ADDRESS=http://127.0.0.1:5000
|
|
48
|
+
snakemake --cores 1 --logger panoptes
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Settings
|
|
52
|
+
|
|
53
|
+
| Flag | Env var | Default | Description |
|
|
54
|
+
| --- | --- | --- | --- |
|
|
55
|
+
| `--logger-panoptes-address` | `SNAKEMAKE_LOGGER_PANOPTES_ADDRESS` | _(required)_ | Base URL of the panoptes server. |
|
|
56
|
+
| `--logger-panoptes-timeout` | — | `10.0` | Per-request HTTP timeout in seconds. |
|
|
57
|
+
|
|
58
|
+
## How it works
|
|
59
|
+
|
|
60
|
+
On workflow start the plugin calls `GET /create_workflow` to register a new
|
|
61
|
+
workflow with the panoptes server and remembers the returned workflow id. It
|
|
62
|
+
then translates Snakemake [`LogEvent`](https://github.com/snakemake/snakemake-interface-logger-plugins)
|
|
63
|
+
records (`JOB_INFO`, `JOB_STARTED`, `JOB_FINISHED`, `JOB_ERROR`, `SHELLCMD`,
|
|
64
|
+
`PROGRESS`, `ERROR`, `RUN_INFO`) into the JSON message format that panoptes'
|
|
65
|
+
`/update_workflow_status` endpoint already understands.
|
|
66
|
+
|
|
67
|
+
Network errors are logged but never crash the workflow.
|
|
68
|
+
|
|
69
|
+
## Development
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pip install -e .[dev] # editable install with test deps
|
|
73
|
+
pytest
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## License
|
|
77
|
+
|
|
78
|
+
MIT
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# snakemake-logger-plugin-panoptes
|
|
2
|
+
|
|
3
|
+
A [Snakemake 9](https://snakemake.readthedocs.io/) logger plugin that forwards
|
|
4
|
+
workflow events to a running [panoptes](https://github.com/panoptes-organization/panoptes)
|
|
5
|
+
server, replacing the legacy `--wms-monitor` integration that was removed in
|
|
6
|
+
Snakemake 9.
|
|
7
|
+
|
|
8
|
+
## Install
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
pip install snakemake-logger-plugin-panoptes
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Use
|
|
15
|
+
|
|
16
|
+
Start a panoptes server (see the panoptes README), then run Snakemake with the
|
|
17
|
+
plugin enabled:
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
snakemake \
|
|
21
|
+
--cores 1 \
|
|
22
|
+
--logger panoptes \
|
|
23
|
+
--logger-panoptes-address http://127.0.0.1:5000
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
You can also point at the address via an environment variable:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
export SNAKEMAKE_LOGGER_PANOPTES_ADDRESS=http://127.0.0.1:5000
|
|
30
|
+
snakemake --cores 1 --logger panoptes
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
### Settings
|
|
34
|
+
|
|
35
|
+
| Flag | Env var | Default | Description |
|
|
36
|
+
| --- | --- | --- | --- |
|
|
37
|
+
| `--logger-panoptes-address` | `SNAKEMAKE_LOGGER_PANOPTES_ADDRESS` | _(required)_ | Base URL of the panoptes server. |
|
|
38
|
+
| `--logger-panoptes-timeout` | — | `10.0` | Per-request HTTP timeout in seconds. |
|
|
39
|
+
|
|
40
|
+
## How it works
|
|
41
|
+
|
|
42
|
+
On workflow start the plugin calls `GET /create_workflow` to register a new
|
|
43
|
+
workflow with the panoptes server and remembers the returned workflow id. It
|
|
44
|
+
then translates Snakemake [`LogEvent`](https://github.com/snakemake/snakemake-interface-logger-plugins)
|
|
45
|
+
records (`JOB_INFO`, `JOB_STARTED`, `JOB_FINISHED`, `JOB_ERROR`, `SHELLCMD`,
|
|
46
|
+
`PROGRESS`, `ERROR`, `RUN_INFO`) into the JSON message format that panoptes'
|
|
47
|
+
`/update_workflow_status` endpoint already understands.
|
|
48
|
+
|
|
49
|
+
Network errors are logged but never crash the workflow.
|
|
50
|
+
|
|
51
|
+
## Development
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install -e .[dev] # editable install with test deps
|
|
55
|
+
pytest
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## License
|
|
59
|
+
|
|
60
|
+
MIT
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "snakemake-logger-plugin-panoptes"
|
|
3
|
+
version = "0.1.1"
|
|
4
|
+
description = "Snakemake logger plugin that forwards workflow events to a panoptes server."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = { text = "MIT" }
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "panoptes-organization" },
|
|
9
|
+
]
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"snakemake-interface-logger-plugins>=1.2.0,<3",
|
|
13
|
+
"requests>=2.22.0",
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.optional-dependencies]
|
|
22
|
+
dev = [
|
|
23
|
+
"pytest>=7",
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
repository = "https://github.com/panoptes-organization/snakemake-logger-plugin-panoptes"
|
|
28
|
+
|
|
29
|
+
[build-system]
|
|
30
|
+
requires = ["hatchling"]
|
|
31
|
+
build-backend = "hatchling.build"
|
|
32
|
+
|
|
33
|
+
[tool.hatch.build.targets.wheel]
|
|
34
|
+
packages = ["src/snakemake_logger_plugin_panoptes"]
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Snakemake logger plugin that forwards workflow events to a panoptes server."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from logging import LogRecord
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
from snakemake_interface_logger_plugins.base import LogHandlerBase
|
|
8
|
+
from snakemake_interface_logger_plugins.settings import LogHandlerSettingsBase
|
|
9
|
+
|
|
10
|
+
from snakemake_logger_plugin_panoptes.log_handler import PanoptesLogHandler
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class LogHandlerSettings(LogHandlerSettingsBase):
|
|
15
|
+
"""Settings exposed via ``--logger-panoptes-*`` on the Snakemake CLI."""
|
|
16
|
+
|
|
17
|
+
address: Optional[str] = field(
|
|
18
|
+
default=None,
|
|
19
|
+
metadata={
|
|
20
|
+
"help": (
|
|
21
|
+
"Base URL of the panoptes server, e.g. http://127.0.0.1:5000. "
|
|
22
|
+
"Required."
|
|
23
|
+
),
|
|
24
|
+
"env_var": True,
|
|
25
|
+
"required": True,
|
|
26
|
+
},
|
|
27
|
+
)
|
|
28
|
+
timeout: Optional[float] = field(
|
|
29
|
+
default=10.0,
|
|
30
|
+
metadata={
|
|
31
|
+
"help": "Per-request HTTP timeout in seconds.",
|
|
32
|
+
"env_var": False,
|
|
33
|
+
"required": False,
|
|
34
|
+
},
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class LogHandler(LogHandlerBase, PanoptesLogHandler):
|
|
39
|
+
"""Snakemake logger plugin entry point."""
|
|
40
|
+
|
|
41
|
+
def __post_init__(self) -> None:
|
|
42
|
+
# ``self.settings`` and ``self.common_settings`` are populated by
|
|
43
|
+
# the Snakemake plugin runtime before __post_init__ is called.
|
|
44
|
+
settings: LogHandlerSettings = self.settings # type: ignore[assignment]
|
|
45
|
+
if not settings.address:
|
|
46
|
+
raise ValueError(
|
|
47
|
+
"snakemake-logger-plugin-panoptes requires --logger-panoptes-address "
|
|
48
|
+
"(or the SNAKEMAKE_LOGGER_PANOPTES_ADDRESS env var) to be set."
|
|
49
|
+
)
|
|
50
|
+
PanoptesLogHandler.__init__(
|
|
51
|
+
self,
|
|
52
|
+
common_settings=self.common_settings,
|
|
53
|
+
address=settings.address,
|
|
54
|
+
timeout=settings.timeout if settings.timeout is not None else 10.0,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
def emit(self, record: LogRecord) -> None:
|
|
58
|
+
PanoptesLogHandler.emit(self, record)
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def writes_to_stream(self) -> bool:
|
|
62
|
+
return False
|
|
63
|
+
|
|
64
|
+
@property
|
|
65
|
+
def writes_to_file(self) -> bool:
|
|
66
|
+
return False
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def has_filter(self) -> bool:
|
|
70
|
+
return False
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def has_formatter(self) -> bool:
|
|
74
|
+
return False
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def needs_rulegraph(self) -> bool:
|
|
78
|
+
return False
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
__all__ = ["LogHandler", "LogHandlerSettings"]
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"""HTTP log handler that forwards Snakemake LogEvents to a panoptes server."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import time
|
|
8
|
+
from logging import Handler, LogRecord
|
|
9
|
+
from typing import Any, Dict, Optional
|
|
10
|
+
|
|
11
|
+
import requests
|
|
12
|
+
from snakemake_interface_logger_plugins.common import LogEvent
|
|
13
|
+
from snakemake_interface_logger_plugins.settings import OutputSettingsLoggerInterface
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
_LOG = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _to_jsonable(value: Any) -> Any:
|
|
20
|
+
"""Best-effort conversion to JSON-serialisable primitives."""
|
|
21
|
+
if value is None or isinstance(value, (bool, int, float, str)):
|
|
22
|
+
return value
|
|
23
|
+
if isinstance(value, dict):
|
|
24
|
+
return {str(k): _to_jsonable(v) for k, v in value.items()}
|
|
25
|
+
if isinstance(value, (list, tuple, set)):
|
|
26
|
+
return [_to_jsonable(v) for v in value]
|
|
27
|
+
# Snakemake's Namedlist / IOFile / Wildcards behave like sequences/mappings;
|
|
28
|
+
# fall back to repr() so the panoptes server still sees something.
|
|
29
|
+
try:
|
|
30
|
+
return list(value) # type: ignore[arg-type]
|
|
31
|
+
except TypeError:
|
|
32
|
+
return str(value)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _resources_to_dict(resources: Any) -> Dict[str, Any]:
|
|
36
|
+
if resources is None:
|
|
37
|
+
return {}
|
|
38
|
+
names = getattr(resources, "_names", None)
|
|
39
|
+
if names is None:
|
|
40
|
+
return _to_jsonable(resources) if isinstance(resources, dict) else {}
|
|
41
|
+
return {
|
|
42
|
+
name: _to_jsonable(value)
|
|
43
|
+
for name, value in zip(names, resources)
|
|
44
|
+
if name not in {"_cores", "_nodes"}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class PanoptesLogHandler(Handler):
|
|
49
|
+
"""Translate Snakemake LogRecords into panoptes HTTP API calls."""
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
common_settings: OutputSettingsLoggerInterface,
|
|
54
|
+
address: str,
|
|
55
|
+
timeout: float = 10.0,
|
|
56
|
+
):
|
|
57
|
+
super().__init__()
|
|
58
|
+
self.common_settings = common_settings
|
|
59
|
+
self.address = address.rstrip("/")
|
|
60
|
+
self.timeout = timeout
|
|
61
|
+
self.session = requests.Session()
|
|
62
|
+
self.workflow_id: Optional[int] = None
|
|
63
|
+
self.workflow_name: Optional[str] = None
|
|
64
|
+
# Map (Snakemake-side) jobid -> rule name, so we can re-attach context
|
|
65
|
+
# to events like JOB_FINISHED / JOB_ERROR that don't carry the rule.
|
|
66
|
+
self._job_rules: Dict[int, str] = {}
|
|
67
|
+
|
|
68
|
+
# ------------------------------------------------------------------ #
|
|
69
|
+
# registration with panoptes
|
|
70
|
+
# ------------------------------------------------------------------ #
|
|
71
|
+
|
|
72
|
+
def _ensure_workflow(self) -> bool:
|
|
73
|
+
if self.workflow_id is not None:
|
|
74
|
+
return True
|
|
75
|
+
try:
|
|
76
|
+
response = self.session.get(
|
|
77
|
+
f"{self.address}/create_workflow", timeout=self.timeout
|
|
78
|
+
)
|
|
79
|
+
response.raise_for_status()
|
|
80
|
+
payload = response.json()
|
|
81
|
+
self.workflow_id = payload.get("id")
|
|
82
|
+
self.workflow_name = payload.get("name")
|
|
83
|
+
return self.workflow_id is not None
|
|
84
|
+
except Exception as exc:
|
|
85
|
+
_LOG.warning("panoptes: could not register workflow: %s", exc)
|
|
86
|
+
return False
|
|
87
|
+
|
|
88
|
+
def _post_message(self, message: Dict[str, Any]) -> None:
|
|
89
|
+
if not self._ensure_workflow():
|
|
90
|
+
return
|
|
91
|
+
payload = {
|
|
92
|
+
"msg": json.dumps(message),
|
|
93
|
+
"timestamp": time.asctime(),
|
|
94
|
+
"id": self.workflow_id,
|
|
95
|
+
}
|
|
96
|
+
try:
|
|
97
|
+
self.session.post(
|
|
98
|
+
f"{self.address}/update_workflow_status",
|
|
99
|
+
data=payload,
|
|
100
|
+
timeout=self.timeout,
|
|
101
|
+
)
|
|
102
|
+
except Exception as exc:
|
|
103
|
+
_LOG.warning("panoptes: failed to deliver event %r: %s", message.get("level"), exc)
|
|
104
|
+
|
|
105
|
+
# ------------------------------------------------------------------ #
|
|
106
|
+
# event translation
|
|
107
|
+
# ------------------------------------------------------------------ #
|
|
108
|
+
|
|
109
|
+
def _on_workflow_started(self, record: LogRecord) -> Optional[Dict[str, Any]]:
|
|
110
|
+
# Make sure panoptes has a workflow row before any other event lands.
|
|
111
|
+
self._ensure_workflow()
|
|
112
|
+
snakefile = getattr(record, "snakefile", None)
|
|
113
|
+
return {
|
|
114
|
+
"level": "info",
|
|
115
|
+
"msg": f"Workflow started: {snakefile}" if snakefile else "Workflow started",
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
def _on_job_info(self, record: LogRecord) -> Dict[str, Any]:
|
|
119
|
+
jobid = int(getattr(record, "jobid", 0) or 0)
|
|
120
|
+
rule_name = getattr(record, "rule_name", "") or ""
|
|
121
|
+
self._job_rules[jobid] = rule_name
|
|
122
|
+
return {
|
|
123
|
+
"level": "job_info",
|
|
124
|
+
"jobid": jobid,
|
|
125
|
+
"name": rule_name,
|
|
126
|
+
"msg": getattr(record, "rule_msg", None),
|
|
127
|
+
"input": _to_jsonable(getattr(record, "input", []) or []),
|
|
128
|
+
"output": _to_jsonable(getattr(record, "output", []) or []),
|
|
129
|
+
"log": _to_jsonable(getattr(record, "log", []) or []),
|
|
130
|
+
"wildcards": _to_jsonable(getattr(record, "wildcards", {}) or {}),
|
|
131
|
+
"is_checkpoint": bool(getattr(record, "is_checkpoint", False)),
|
|
132
|
+
"shellcmd": getattr(record, "shellcmd", None),
|
|
133
|
+
"threads": getattr(record, "threads", None),
|
|
134
|
+
"priority": getattr(record, "priority", None),
|
|
135
|
+
"reason": getattr(record, "reason", None),
|
|
136
|
+
"resources": _resources_to_dict(getattr(record, "resources", None)),
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
def _on_job_started(self, record: LogRecord) -> Optional[Dict[str, Any]]:
|
|
140
|
+
jobs = getattr(record, "jobs", None)
|
|
141
|
+
if jobs is None:
|
|
142
|
+
return None
|
|
143
|
+
if isinstance(jobs, int):
|
|
144
|
+
jobs = [jobs]
|
|
145
|
+
return {"level": "job_started", "jobs": [int(j) for j in jobs]}
|
|
146
|
+
|
|
147
|
+
def _on_job_finished(self, record: LogRecord) -> Optional[Dict[str, Any]]:
|
|
148
|
+
jobid = getattr(record, "job_id", None)
|
|
149
|
+
if jobid is None:
|
|
150
|
+
jobid = getattr(record, "jobid", None)
|
|
151
|
+
if jobid is None:
|
|
152
|
+
return None
|
|
153
|
+
return {"level": "job_finished", "jobid": int(jobid)}
|
|
154
|
+
|
|
155
|
+
def _on_job_error(self, record: LogRecord) -> Dict[str, Any]:
|
|
156
|
+
jobid = int(getattr(record, "jobid", 0) or 0)
|
|
157
|
+
return {
|
|
158
|
+
"level": "job_error",
|
|
159
|
+
"jobid": jobid,
|
|
160
|
+
"name": self._job_rules.get(jobid, ""),
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
def _on_shellcmd(self, record: LogRecord) -> Optional[Dict[str, Any]]:
|
|
164
|
+
shellcmd = getattr(record, "shellcmd", None)
|
|
165
|
+
if not shellcmd:
|
|
166
|
+
return None
|
|
167
|
+
jobid = getattr(record, "jobid", None)
|
|
168
|
+
return {
|
|
169
|
+
"level": "shellcmd",
|
|
170
|
+
"jobid": int(jobid) if jobid is not None else None,
|
|
171
|
+
"msg": shellcmd,
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
def _on_progress(self, record: LogRecord) -> Dict[str, Any]:
|
|
175
|
+
return {
|
|
176
|
+
"level": "progress",
|
|
177
|
+
"done": int(getattr(record, "done", 0) or 0),
|
|
178
|
+
"total": int(getattr(record, "total", 0) or 0),
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
def _on_error(self, record: LogRecord) -> Dict[str, Any]:
|
|
182
|
+
return {
|
|
183
|
+
"level": "error",
|
|
184
|
+
"msg": getattr(record, "exception", None) or record.getMessage(),
|
|
185
|
+
"rule": getattr(record, "rule", None),
|
|
186
|
+
"location": getattr(record, "location", None),
|
|
187
|
+
"traceback": getattr(record, "traceback", None),
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
def _on_run_info(self, record: LogRecord) -> Optional[Dict[str, Any]]:
|
|
191
|
+
stats = getattr(record, "stats", None) or {}
|
|
192
|
+
total = stats.get("total")
|
|
193
|
+
if total is None:
|
|
194
|
+
return None
|
|
195
|
+
return {"level": "progress", "done": 0, "total": int(total)}
|
|
196
|
+
|
|
197
|
+
# ------------------------------------------------------------------ #
|
|
198
|
+
# Handler interface
|
|
199
|
+
# ------------------------------------------------------------------ #
|
|
200
|
+
|
|
201
|
+
_DISPATCH = {
|
|
202
|
+
LogEvent.WORKFLOW_STARTED: "_on_workflow_started",
|
|
203
|
+
LogEvent.JOB_INFO: "_on_job_info",
|
|
204
|
+
LogEvent.JOB_STARTED: "_on_job_started",
|
|
205
|
+
LogEvent.JOB_FINISHED: "_on_job_finished",
|
|
206
|
+
LogEvent.JOB_ERROR: "_on_job_error",
|
|
207
|
+
LogEvent.SHELLCMD: "_on_shellcmd",
|
|
208
|
+
LogEvent.PROGRESS: "_on_progress",
|
|
209
|
+
LogEvent.ERROR: "_on_error",
|
|
210
|
+
LogEvent.RUN_INFO: "_on_run_info",
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
def emit(self, record: LogRecord) -> None:
|
|
214
|
+
try:
|
|
215
|
+
event = getattr(record, "event", None)
|
|
216
|
+
if event is None:
|
|
217
|
+
return
|
|
218
|
+
method_name = self._DISPATCH.get(event)
|
|
219
|
+
if method_name is None:
|
|
220
|
+
return
|
|
221
|
+
message = getattr(self, method_name)(record)
|
|
222
|
+
if message is None:
|
|
223
|
+
return
|
|
224
|
+
self._post_message(message)
|
|
225
|
+
except Exception:
|
|
226
|
+
self.handleError(record)
|
|
227
|
+
|
|
228
|
+
def close(self) -> None:
|
|
229
|
+
try:
|
|
230
|
+
self.session.close()
|
|
231
|
+
finally:
|
|
232
|
+
super().close()
|
|
File without changes
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""Unit tests for the panoptes log handler.
|
|
2
|
+
|
|
3
|
+
The tests stub out the HTTP layer so they run without Snakemake or a live
|
|
4
|
+
panoptes server.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
from types import SimpleNamespace
|
|
12
|
+
from typing import Any, Dict, List
|
|
13
|
+
|
|
14
|
+
import pytest
|
|
15
|
+
|
|
16
|
+
from snakemake_interface_logger_plugins.common import LogEvent
|
|
17
|
+
|
|
18
|
+
from snakemake_logger_plugin_panoptes.log_handler import PanoptesLogHandler
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class _Resp:
|
|
22
|
+
def __init__(self, payload: Dict[str, Any], status: int = 200) -> None:
|
|
23
|
+
self._payload = payload
|
|
24
|
+
self.status_code = status
|
|
25
|
+
|
|
26
|
+
def raise_for_status(self) -> None:
|
|
27
|
+
if not 200 <= self.status_code < 300:
|
|
28
|
+
raise RuntimeError(f"status {self.status_code}")
|
|
29
|
+
|
|
30
|
+
def json(self) -> Dict[str, Any]:
|
|
31
|
+
return self._payload
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class _FakeSession:
|
|
35
|
+
def __init__(self) -> None:
|
|
36
|
+
self.get_calls: List[str] = []
|
|
37
|
+
self.post_calls: List[Dict[str, Any]] = []
|
|
38
|
+
|
|
39
|
+
def get(self, url: str, timeout: float = 0) -> _Resp: # noqa: ARG002
|
|
40
|
+
self.get_calls.append(url)
|
|
41
|
+
return _Resp({"id": 7, "name": "wf-uuid"})
|
|
42
|
+
|
|
43
|
+
def post(self, url: str, data: Dict[str, Any], timeout: float = 0) -> _Resp: # noqa: ARG002
|
|
44
|
+
self.post_calls.append({"url": url, "data": data})
|
|
45
|
+
return _Resp({}, status=200)
|
|
46
|
+
|
|
47
|
+
def close(self) -> None:
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _make_handler() -> PanoptesLogHandler:
|
|
52
|
+
common = SimpleNamespace(dryrun=False)
|
|
53
|
+
h = PanoptesLogHandler(common_settings=common, address="http://example.test")
|
|
54
|
+
h.session = _FakeSession() # type: ignore[assignment]
|
|
55
|
+
return h
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _record(event: LogEvent, **attrs: Any) -> logging.LogRecord:
|
|
59
|
+
record = logging.LogRecord(
|
|
60
|
+
name="snakemake",
|
|
61
|
+
level=logging.INFO,
|
|
62
|
+
pathname="",
|
|
63
|
+
lineno=0,
|
|
64
|
+
msg="",
|
|
65
|
+
args=(),
|
|
66
|
+
exc_info=None,
|
|
67
|
+
)
|
|
68
|
+
record.event = event # type: ignore[attr-defined]
|
|
69
|
+
for key, value in attrs.items():
|
|
70
|
+
setattr(record, key, value)
|
|
71
|
+
return record
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _last_message(handler: PanoptesLogHandler) -> Dict[str, Any]:
|
|
75
|
+
fake = handler.session # type: ignore[assignment]
|
|
76
|
+
assert isinstance(fake, _FakeSession)
|
|
77
|
+
assert fake.post_calls, "expected at least one POST"
|
|
78
|
+
return json.loads(fake.post_calls[-1]["data"]["msg"])
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_workflow_registration_uses_create_workflow() -> None:
|
|
82
|
+
h = _make_handler()
|
|
83
|
+
h.emit(_record(LogEvent.WORKFLOW_STARTED, snakefile="/tmp/Snakefile"))
|
|
84
|
+
assert h.workflow_id == 7
|
|
85
|
+
assert any("/create_workflow" in url for url in h.session.get_calls) # type: ignore[attr-defined]
|
|
86
|
+
msg = _last_message(h)
|
|
87
|
+
assert msg["level"] == "info"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_job_info_payload_shape() -> None:
|
|
91
|
+
h = _make_handler()
|
|
92
|
+
h.emit(
|
|
93
|
+
_record(
|
|
94
|
+
LogEvent.JOB_INFO,
|
|
95
|
+
jobid=3,
|
|
96
|
+
rule_name="samtools_sort",
|
|
97
|
+
input=["samples/c.bam"],
|
|
98
|
+
output=["results/c.sorted.bam"],
|
|
99
|
+
log=["logs/c.log"],
|
|
100
|
+
wildcards={"sample": "c"},
|
|
101
|
+
is_checkpoint=False,
|
|
102
|
+
rule_msg=None,
|
|
103
|
+
shellcmd="samtools sort ...",
|
|
104
|
+
threads=4,
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
msg = _last_message(h)
|
|
108
|
+
assert msg["level"] == "job_info"
|
|
109
|
+
assert msg["jobid"] == 3
|
|
110
|
+
assert msg["name"] == "samtools_sort"
|
|
111
|
+
assert msg["input"] == ["samples/c.bam"]
|
|
112
|
+
assert msg["wildcards"] == {"sample": "c"}
|
|
113
|
+
assert msg["is_checkpoint"] is False
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def test_job_finished_accepts_job_id_or_jobid() -> None:
|
|
117
|
+
h = _make_handler()
|
|
118
|
+
h.emit(_record(LogEvent.JOB_FINISHED, job_id=5))
|
|
119
|
+
assert _last_message(h) == {"level": "job_finished", "jobid": 5}
|
|
120
|
+
|
|
121
|
+
h.emit(_record(LogEvent.JOB_FINISHED, jobid=6))
|
|
122
|
+
assert _last_message(h) == {"level": "job_finished", "jobid": 6}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_progress_payload() -> None:
|
|
126
|
+
h = _make_handler()
|
|
127
|
+
h.emit(_record(LogEvent.PROGRESS, done=2, total=10))
|
|
128
|
+
assert _last_message(h) == {"level": "progress", "done": 2, "total": 10}
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def test_job_error_carries_rule_from_prior_job_info() -> None:
|
|
132
|
+
h = _make_handler()
|
|
133
|
+
h.emit(
|
|
134
|
+
_record(
|
|
135
|
+
LogEvent.JOB_INFO,
|
|
136
|
+
jobid=11,
|
|
137
|
+
rule_name="bwa_map",
|
|
138
|
+
input=[],
|
|
139
|
+
output=[],
|
|
140
|
+
log=[],
|
|
141
|
+
wildcards={},
|
|
142
|
+
is_checkpoint=False,
|
|
143
|
+
)
|
|
144
|
+
)
|
|
145
|
+
h.emit(_record(LogEvent.JOB_ERROR, jobid=11))
|
|
146
|
+
msg = _last_message(h)
|
|
147
|
+
assert msg == {"level": "job_error", "jobid": 11, "name": "bwa_map"}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def test_unknown_event_is_silently_ignored() -> None:
|
|
151
|
+
h = _make_handler()
|
|
152
|
+
rec = _record(LogEvent.RULEGRAPH, rulegraph={})
|
|
153
|
+
h.emit(rec)
|
|
154
|
+
# rulegraph isn't mapped — no POST should be sent (only the auto-registration GET).
|
|
155
|
+
assert h.session.post_calls == [] # type: ignore[attr-defined]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_record_without_event_is_dropped() -> None:
|
|
159
|
+
h = _make_handler()
|
|
160
|
+
rec = logging.LogRecord("x", logging.INFO, "", 0, "msg", (), None)
|
|
161
|
+
h.emit(rec)
|
|
162
|
+
assert h.session.post_calls == [] # type: ignore[attr-defined]
|
|
163
|
+
assert h.session.get_calls == [] # type: ignore[attr-defined]
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def test_run_info_sets_total() -> None:
|
|
167
|
+
h = _make_handler()
|
|
168
|
+
h.emit(_record(LogEvent.RUN_INFO, stats={"total": 14, "bwa": 7, "sort": 7}))
|
|
169
|
+
msg = _last_message(h)
|
|
170
|
+
assert msg == {"level": "progress", "done": 0, "total": 14}
|