airflow-pytest-plugin 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/PKG-INFO +53 -8
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/README.md +52 -7
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/pyproject.toml +1 -1
- airflow_pytest_plugin-0.3.0/src/airflow_pytest_plugin/config.py +79 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/plugin.py +11 -3
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/producer/archiving_parser.py +15 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/sources/base.py +9 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/sources/filesystem.py +99 -19
- airflow_pytest_plugin-0.3.0/src/airflow_pytest_plugin/web/app.py +155 -0
- airflow_pytest_plugin-0.3.0/src/airflow_pytest_plugin/web/routes/__init__.py +20 -0
- airflow_pytest_plugin-0.3.0/src/airflow_pytest_plugin/web/routes/common.py +53 -0
- airflow_pytest_plugin-0.3.0/src/airflow_pytest_plugin/web/routes/compare.py +94 -0
- airflow_pytest_plugin-0.3.0/src/airflow_pytest_plugin/web/routes/failures.py +89 -0
- airflow_pytest_plugin-0.3.0/src/airflow_pytest_plugin/web/routes/flaky.py +106 -0
- airflow_pytest_plugin-0.3.0/src/airflow_pytest_plugin/web/routes/monitoring.py +75 -0
- airflow_pytest_plugin-0.3.0/src/airflow_pytest_plugin/web/routes/reports.py +212 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/web/templates.py +1032 -110
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/conftest.py +29 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/test_archiving_parser.py +14 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/test_config.py +40 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/test_filesystem_source.py +95 -1
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/test_plugin.py +14 -0
- airflow_pytest_plugin-0.3.0/tests/test_web_app.py +648 -0
- airflow_pytest_plugin-0.2.0/src/airflow_pytest_plugin/config.py +0 -39
- airflow_pytest_plugin-0.2.0/src/airflow_pytest_plugin/web/app.py +0 -184
- airflow_pytest_plugin-0.2.0/tests/test_web_app.py +0 -264
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/.gitignore +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/LICENSE +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/examples/example_dag.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/__init__.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/compat/__init__.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/compat/airflow.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/layout.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/models.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/producer/__init__.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/sources/__init__.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/version.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/web/__init__.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/web/__main__.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/test_authz_integration.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/test_compat.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/test_layout.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/test_models.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/test_package.py +0 -0
- {airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/tests/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: airflow-pytest-plugin
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: View airflow-pytest-operator results in the Airflow 3 web UI.
|
|
5
5
|
Project-URL: Homepage, https://github.com/IKrysanov/airflow-pytest-plugin
|
|
6
6
|
Project-URL: Repository, https://github.com/IKrysanov/airflow-pytest-plugin
|
|
@@ -70,7 +70,9 @@ The operator runs a `pytest` suite as an Airflow task and parses the JUnit
|
|
|
70
70
|
report into a structured result. This plugin archives each of those reports —
|
|
71
71
|
keyed by `dag_id / run_id / task_id / try` — and serves a small web UI to browse
|
|
72
72
|
them: pass/fail counts per run, durations, and the per-test breakdown (with
|
|
73
|
-
failure messages) for any run.
|
|
73
|
+
failure messages) for any run. On top of that it adds **cross-run analytics** —
|
|
74
|
+
flaky-test detection, per-test history, run-to-run comparison, a duration
|
|
75
|
+
histogram, and a searchable catalogue of unique tests.
|
|
74
76
|
|
|
75
77
|
It has two halves that share one on-disk layout:
|
|
76
78
|
|
|
@@ -96,17 +98,34 @@ It has two halves that share one on-disk layout:
|
|
|
96
98
|
## Screenshots
|
|
97
99
|
|
|
98
100
|
**Overview** — the run list with the historical chart (per-status legend
|
|
99
|
-
toggles, run numbers, and a carousel beyond 30 runs)
|
|
101
|
+
toggles, run numbers, and a carousel beyond 30 runs) beside a **flaky-tests
|
|
102
|
+
panel**, KPI cards (including a clickable **unique tests** count), and Airflow-
|
|
100
103
|
matched colours and font:
|
|
101
104
|
|
|
102
105
|

|
|
103
106
|
|
|
104
|
-
**A single run** — a clickable success donut (pass-rate
|
|
105
|
-
slice to filter by status),
|
|
106
|
-
|
|
107
|
+
**A single run** — a clickable success donut (pass-rate over the test count;
|
|
108
|
+
click a slice to filter by status), a **test-duration histogram** (10-second
|
|
109
|
+
buckets, scrollable), case search / group-by-module, and every test's captured
|
|
110
|
+
output on expand:
|
|
107
111
|
|
|
108
112
|

|
|
109
113
|
|
|
114
|
+
**Flaky tests & comparison** — from a run, *Flaky tests* lists the tests that
|
|
115
|
+
both pass and fail across recent runs (with a recent-outcome strip), and
|
|
116
|
+
*Compare to previous* diffs it against the prior run; expanding a case offers its
|
|
117
|
+
full **history**:
|
|
118
|
+
|
|
119
|
+

|
|
120
|
+
|
|
121
|
+
**Unique tests & failures** — the *Unique tests* KPI opens the searchable,
|
|
122
|
+
paginated catalogue of every distinct test; the *Failures* KPI lists every
|
|
123
|
+
failed/errored case across the visible runs — a row jumps to that run:
|
|
124
|
+
|
|
125
|
+

|
|
126
|
+
|
|
127
|
+

|
|
128
|
+
|
|
110
129
|
---
|
|
111
130
|
|
|
112
131
|
## Install
|
|
@@ -208,10 +227,16 @@ runtime. Endpoints (relative to the mount):
|
|
|
208
227
|
| `GET /` | the single-page viewer (HTML) |
|
|
209
228
|
| `GET /api/reports?dag_id=&run_id=` | summaries, newest first |
|
|
210
229
|
| `GET /api/reports/{report_id}` | one report with per-case rows |
|
|
230
|
+
| `GET /api/failures?dag_id=&run_id=&task_id=` | failed/errored cases across the visible runs |
|
|
231
|
+
| `GET /api/compare?base=&head=` | per-test diff between two runs (newly failed / fixed / …) |
|
|
232
|
+
| `GET /api/flaky?dag_id=&task_id=&window=` | tests that both pass and fail over recent runs |
|
|
233
|
+
| `GET /api/test-history?dag_id=&task_id=&node_id=&limit=` | one test's outcome per run |
|
|
234
|
+
| `GET /api/unique-tests?dag_id=&task_id=&run_id=&full=` | distinct test count (+ list when `full`) |
|
|
211
235
|
| `DELETE /api/reports/{report_id}` | delete a report (RBAC-gated) |
|
|
212
236
|
| `GET /api/reports/{report_id}/allure.zip` | raw Allure results as a zip (if any) |
|
|
213
|
-
| `GET /api/health` | `
|
|
214
|
-
| `GET /api/
|
|
237
|
+
| `GET /api/health` | liveness + readiness: `status`, `ready`, `reports_root`(+`_exists`), `auth`, `secure_xml` |
|
|
238
|
+
| `GET /api/version` | `{"name": ..., "version": ...}` from package metadata |
|
|
239
|
+
| `GET /api/docs` | OpenAPI docs (Swagger UI) |
|
|
215
240
|
|
|
216
241
|
The reads (`GET`) and the delete are gated by Airflow RBAC — see below.
|
|
217
242
|
|
|
@@ -268,6 +293,26 @@ Download them from a report's detail view, or `GET
|
|
|
268
293
|
| `AIRFLOW_PYTEST_REPORTS_ROOT` (env) | — | report root (highest precedence) |
|
|
269
294
|
| `[pytest_reports] reports_root` (cfg) | — | report root |
|
|
270
295
|
| built-in default | `/opt/airflow/pytest-reports` | fallback |
|
|
296
|
+
| `AIRFLOW_PYTEST_PLUGIN_ENABLE` (env) | `True` | reader on/off — see below |
|
|
297
|
+
| `AIRFLOW_PYTEST_SCAN_CACHE_TTL` (env) | `2.0` | seconds a directory scan is reused (`0` disables) |
|
|
298
|
+
|
|
299
|
+
**Enable / disable the reader.** Set `AIRFLOW_PYTEST_PLUGIN_ENABLE` to a falsey
|
|
300
|
+
value (`0`, `false`, `no`, `off`) to stop the plugin registering its UI and API
|
|
301
|
+
with Airflow; any other value, or leaving it unset, keeps it on (the default).
|
|
302
|
+
This is a kill switch for the reader only — the producer-side
|
|
303
|
+
`ArchivingResultParser` still archives reports regardless. It is read once at
|
|
304
|
+
plugin discovery, so toggling it takes effect on the next API-server restart.
|
|
305
|
+
|
|
306
|
+
```bash
|
|
307
|
+
export AIRFLOW_PYTEST_PLUGIN_ENABLE=false # hide the Pytest Reports UI + API
|
|
308
|
+
```
|
|
309
|
+
|
|
310
|
+
**Scan cache.** Loading the home page hits several summary-driven endpoints (the run
|
|
311
|
+
list, the flaky panel, the unique-tests count), and the filter box queries as you
|
|
312
|
+
type. To avoid walking the report tree once per call, the filesystem source reuses a
|
|
313
|
+
single scan for `AIRFLOW_PYTEST_SCAN_CACHE_TTL` seconds (default `2.0`; deletes
|
|
314
|
+
invalidate it immediately). New runs therefore appear within a couple of seconds (or
|
|
315
|
+
on **Refresh**); set it to `0` for no caching, or higher on a very large tree.
|
|
271
316
|
|
|
272
317
|
## Architecture (SOLID)
|
|
273
318
|
|
|
@@ -26,7 +26,9 @@ The operator runs a `pytest` suite as an Airflow task and parses the JUnit
|
|
|
26
26
|
report into a structured result. This plugin archives each of those reports —
|
|
27
27
|
keyed by `dag_id / run_id / task_id / try` — and serves a small web UI to browse
|
|
28
28
|
them: pass/fail counts per run, durations, and the per-test breakdown (with
|
|
29
|
-
failure messages) for any run.
|
|
29
|
+
failure messages) for any run. On top of that it adds **cross-run analytics** —
|
|
30
|
+
flaky-test detection, per-test history, run-to-run comparison, a duration
|
|
31
|
+
histogram, and a searchable catalogue of unique tests.
|
|
30
32
|
|
|
31
33
|
It has two halves that share one on-disk layout:
|
|
32
34
|
|
|
@@ -52,17 +54,34 @@ It has two halves that share one on-disk layout:
|
|
|
52
54
|
## Screenshots
|
|
53
55
|
|
|
54
56
|
**Overview** — the run list with the historical chart (per-status legend
|
|
55
|
-
toggles, run numbers, and a carousel beyond 30 runs)
|
|
57
|
+
toggles, run numbers, and a carousel beyond 30 runs) beside a **flaky-tests
|
|
58
|
+
panel**, KPI cards (including a clickable **unique tests** count), and Airflow-
|
|
56
59
|
matched colours and font:
|
|
57
60
|
|
|
58
61
|

|
|
59
62
|
|
|
60
|
-
**A single run** — a clickable success donut (pass-rate
|
|
61
|
-
slice to filter by status),
|
|
62
|
-
|
|
63
|
+
**A single run** — a clickable success donut (pass-rate over the test count;
|
|
64
|
+
click a slice to filter by status), a **test-duration histogram** (10-second
|
|
65
|
+
buckets, scrollable), case search / group-by-module, and every test's captured
|
|
66
|
+
output on expand:
|
|
63
67
|
|
|
64
68
|

|
|
65
69
|
|
|
70
|
+
**Flaky tests & comparison** — from a run, *Flaky tests* lists the tests that
|
|
71
|
+
both pass and fail across recent runs (with a recent-outcome strip), and
|
|
72
|
+
*Compare to previous* diffs it against the prior run; expanding a case offers its
|
|
73
|
+
full **history**:
|
|
74
|
+
|
|
75
|
+

|
|
76
|
+
|
|
77
|
+
**Unique tests & failures** — the *Unique tests* KPI opens the searchable,
|
|
78
|
+
paginated catalogue of every distinct test; the *Failures* KPI lists every
|
|
79
|
+
failed/errored case across the visible runs — a row jumps to that run:
|
|
80
|
+
|
|
81
|
+

|
|
82
|
+
|
|
83
|
+

|
|
84
|
+
|
|
66
85
|
---
|
|
67
86
|
|
|
68
87
|
## Install
|
|
@@ -164,10 +183,16 @@ runtime. Endpoints (relative to the mount):
|
|
|
164
183
|
| `GET /` | the single-page viewer (HTML) |
|
|
165
184
|
| `GET /api/reports?dag_id=&run_id=` | summaries, newest first |
|
|
166
185
|
| `GET /api/reports/{report_id}` | one report with per-case rows |
|
|
186
|
+
| `GET /api/failures?dag_id=&run_id=&task_id=` | failed/errored cases across the visible runs |
|
|
187
|
+
| `GET /api/compare?base=&head=` | per-test diff between two runs (newly failed / fixed / …) |
|
|
188
|
+
| `GET /api/flaky?dag_id=&task_id=&window=` | tests that both pass and fail over recent runs |
|
|
189
|
+
| `GET /api/test-history?dag_id=&task_id=&node_id=&limit=` | one test's outcome per run |
|
|
190
|
+
| `GET /api/unique-tests?dag_id=&task_id=&run_id=&full=` | distinct test count (+ list when `full`) |
|
|
167
191
|
| `DELETE /api/reports/{report_id}` | delete a report (RBAC-gated) |
|
|
168
192
|
| `GET /api/reports/{report_id}/allure.zip` | raw Allure results as a zip (if any) |
|
|
169
|
-
| `GET /api/health` | `
|
|
170
|
-
| `GET /api/
|
|
193
|
+
| `GET /api/health` | liveness + readiness: `status`, `ready`, `reports_root`(+`_exists`), `auth`, `secure_xml` |
|
|
194
|
+
| `GET /api/version` | `{"name": ..., "version": ...}` from package metadata |
|
|
195
|
+
| `GET /api/docs` | OpenAPI docs (Swagger UI) |
|
|
171
196
|
|
|
172
197
|
The reads (`GET`) and the delete are gated by Airflow RBAC — see below.
|
|
173
198
|
|
|
@@ -224,6 +249,26 @@ Download them from a report's detail view, or `GET
|
|
|
224
249
|
| `AIRFLOW_PYTEST_REPORTS_ROOT` (env) | — | report root (highest precedence) |
|
|
225
250
|
| `[pytest_reports] reports_root` (cfg) | — | report root |
|
|
226
251
|
| built-in default | `/opt/airflow/pytest-reports` | fallback |
|
|
252
|
+
| `AIRFLOW_PYTEST_PLUGIN_ENABLE` (env) | `True` | reader on/off — see below |
|
|
253
|
+
| `AIRFLOW_PYTEST_SCAN_CACHE_TTL` (env) | `2.0` | seconds a directory scan is reused (`0` disables) |
|
|
254
|
+
|
|
255
|
+
**Enable / disable the reader.** Set `AIRFLOW_PYTEST_PLUGIN_ENABLE` to a falsey
|
|
256
|
+
value (`0`, `false`, `no`, `off`) to stop the plugin registering its UI and API
|
|
257
|
+
with Airflow; any other value, or leaving it unset, keeps it on (the default).
|
|
258
|
+
This is a kill switch for the reader only — the producer-side
|
|
259
|
+
`ArchivingResultParser` still archives reports regardless. It is read once at
|
|
260
|
+
plugin discovery, so toggling it takes effect on the next API-server restart.
|
|
261
|
+
|
|
262
|
+
```bash
|
|
263
|
+
export AIRFLOW_PYTEST_PLUGIN_ENABLE=false # hide the Pytest Reports UI + API
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
**Scan cache.** Loading the home page hits several summary-driven endpoints (the run
|
|
267
|
+
list, the flaky panel, the unique-tests count), and the filter box queries as you
|
|
268
|
+
type. To avoid walking the report tree once per call, the filesystem source reuses a
|
|
269
|
+
single scan for `AIRFLOW_PYTEST_SCAN_CACHE_TTL` seconds (default `2.0`; deletes
|
|
270
|
+
invalidate it immediately). New runs therefore appear within a couple of seconds (or
|
|
271
|
+
on **Refresh**); set it to `0` for no caching, or higher on a very large tree.
|
|
227
272
|
|
|
228
273
|
## Architecture (SOLID)
|
|
229
274
|
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# Copyright 2026 the airflow-pytest-plugin contributors
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Resolve the report root (env, then Airflow config, then default)."""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import os
|
|
20
|
+
|
|
21
|
+
from .compat import get_conf_value
|
|
22
|
+
|
|
23
|
+
ENV_VAR = "AIRFLOW_PYTEST_REPORTS_ROOT"
|
|
24
|
+
CONF_SECTION = "pytest_reports"
|
|
25
|
+
CONF_KEY = "reports_root"
|
|
26
|
+
DEFAULT_ROOT = "/opt/airflow/pytest-reports"
|
|
27
|
+
|
|
28
|
+
#: Toggles whether the reader plugin (UI + API) registers with Airflow. Default on.
|
|
29
|
+
ENABLE_ENV_VAR = "AIRFLOW_PYTEST_PLUGIN_ENABLE"
|
|
30
|
+
_FALSEY = frozenset({"0", "false", "no", "off", "n", "f"})
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def is_plugin_enabled() -> bool:
|
|
34
|
+
"""Whether the reader plugin should register with Airflow.
|
|
35
|
+
|
|
36
|
+
Reads ``AIRFLOW_PYTEST_PLUGIN_ENABLE`` -- ``True`` (the default when unset/empty)
|
|
37
|
+
registers the UI + API; a falsey value (``0``/``false``/``no``/``off``) disables
|
|
38
|
+
it. Only gates the reader; the producer-side parser is unaffected.
|
|
39
|
+
"""
|
|
40
|
+
raw = os.environ.get(ENABLE_ENV_VAR)
|
|
41
|
+
if raw is None or not raw.strip():
|
|
42
|
+
return True
|
|
43
|
+
return raw.strip().lower() not in _FALSEY
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
#: How long (seconds) the filesystem source may reuse a directory scan. A short
|
|
47
|
+
#: window lets the several summary-driven endpoints on one page load (list + flaky +
|
|
48
|
+
#: unique-tests, plus filter typing) share one tree walk instead of rescanning each.
|
|
49
|
+
SCAN_TTL_ENV_VAR = "AIRFLOW_PYTEST_SCAN_CACHE_TTL"
|
|
50
|
+
DEFAULT_SCAN_TTL = 2.0
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def get_scan_cache_ttl() -> float:
|
|
54
|
+
"""Resolve the directory-scan cache TTL in seconds (``0`` disables caching).
|
|
55
|
+
|
|
56
|
+
Reads ``AIRFLOW_PYTEST_SCAN_CACHE_TTL``; falls back to ``DEFAULT_SCAN_TTL``.
|
|
57
|
+
A malformed or negative value falls back to the default.
|
|
58
|
+
"""
|
|
59
|
+
raw = os.environ.get(SCAN_TTL_ENV_VAR)
|
|
60
|
+
if raw is None or not raw.strip():
|
|
61
|
+
return DEFAULT_SCAN_TTL
|
|
62
|
+
try:
|
|
63
|
+
ttl = float(raw.strip())
|
|
64
|
+
except ValueError:
|
|
65
|
+
return DEFAULT_SCAN_TTL
|
|
66
|
+
return ttl if ttl >= 0 else DEFAULT_SCAN_TTL
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def get_reports_root() -> str:
|
|
70
|
+
"""Resolve the report root directory (absolute path)."""
|
|
71
|
+
env = os.environ.get(ENV_VAR)
|
|
72
|
+
if env and env.strip():
|
|
73
|
+
return os.path.abspath(env.strip())
|
|
74
|
+
|
|
75
|
+
conf = get_conf_value(CONF_SECTION, CONF_KEY)
|
|
76
|
+
if conf and conf.strip():
|
|
77
|
+
return os.path.abspath(conf.strip())
|
|
78
|
+
|
|
79
|
+
return os.path.abspath(DEFAULT_ROOT)
|
{airflow_pytest_plugin-0.2.0 → airflow_pytest_plugin-0.3.0}/src/airflow_pytest_plugin/plugin.py
RENAMED
|
@@ -20,6 +20,7 @@ import logging
|
|
|
20
20
|
from typing import TYPE_CHECKING, Any
|
|
21
21
|
|
|
22
22
|
from .compat import get_airflow_plugin_base
|
|
23
|
+
from .config import ENABLE_ENV_VAR, is_plugin_enabled
|
|
23
24
|
|
|
24
25
|
_log = logging.getLogger(__name__)
|
|
25
26
|
|
|
@@ -73,9 +74,16 @@ else:
|
|
|
73
74
|
_Base = object
|
|
74
75
|
|
|
75
76
|
|
|
77
|
+
#: Kill switch: when AIRFLOW_PYTEST_PLUGIN_ENABLE is falsey the reader registers
|
|
78
|
+
#: nothing (no app, no nav link), so the UI/API are unavailable.
|
|
79
|
+
_ENABLED = is_plugin_enabled()
|
|
80
|
+
if not _ENABLED:
|
|
81
|
+
_log.info("Pytest Reports reader disabled via %s; not registering.", ENABLE_ENV_VAR)
|
|
82
|
+
|
|
83
|
+
|
|
76
84
|
class PytestReportsPlugin(_Base):
|
|
77
|
-
"""Exposes the Pytest Reports UI to Airflow."""
|
|
85
|
+
"""Exposes the Pytest Reports UI to Airflow (unless disabled via the env var)."""
|
|
78
86
|
|
|
79
87
|
name = "pytest_reports"
|
|
80
|
-
fastapi_apps = _build_fastapi_apps()
|
|
81
|
-
external_views = _build_external_views()
|
|
88
|
+
fastapi_apps = _build_fastapi_apps() if _ENABLED else []
|
|
89
|
+
external_views = _build_external_views() if _ENABLED else []
|
|
@@ -163,6 +163,9 @@ class ArchivingResultParser(JUnitResultParser): # type: ignore[misc]
|
|
|
163
163
|
"report_file": REPORT_FILENAME,
|
|
164
164
|
"allure": self._archive_allure(out_dir, ref),
|
|
165
165
|
"summary": result.to_xcom(),
|
|
166
|
+
# Compact per-test outcomes [node_id, outcome, duration] so cross-run
|
|
167
|
+
# views (compare/flaky/history) need not re-parse junit.xml.
|
|
168
|
+
"tests": _test_rows(result),
|
|
166
169
|
}
|
|
167
170
|
# Atomic write: a reader scanning concurrently never sees a half file.
|
|
168
171
|
tmp = os.path.join(out_dir, f".{META_FILENAME}.{uuid.uuid4().hex}.tmp")
|
|
@@ -229,6 +232,18 @@ def _first_int(*values: Any, default: int) -> int:
|
|
|
229
232
|
return default
|
|
230
233
|
|
|
231
234
|
|
|
235
|
+
def _test_rows(result: TestRunResult) -> list[list[Any]]:
|
|
236
|
+
"""Compact ``[node_id, outcome, duration]`` rows for cross-run views."""
|
|
237
|
+
rows: list[list[Any]] = []
|
|
238
|
+
for c in getattr(result, "cases", ()) or ():
|
|
239
|
+
try:
|
|
240
|
+
dur = round(float(getattr(c, "time", 0.0) or 0.0), 3)
|
|
241
|
+
except (TypeError, ValueError):
|
|
242
|
+
dur = 0.0
|
|
243
|
+
rows.append([c.node_id, c.outcome, dur])
|
|
244
|
+
return rows
|
|
245
|
+
|
|
246
|
+
|
|
232
247
|
def _logical_date(context: dict[str, Any] | None) -> str | None:
|
|
233
248
|
if not context:
|
|
234
249
|
return None
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
19
|
from abc import ABC, abstractmethod
|
|
20
|
+
from typing import Any
|
|
20
21
|
|
|
21
22
|
from ..models import ReportDetail, ReportRef, ReportSummary
|
|
22
23
|
|
|
@@ -50,3 +51,11 @@ class ReportSource(ABC):
|
|
|
50
51
|
Optional capability (default: unsupported) for exporting to Allure TestOps.
|
|
51
52
|
"""
|
|
52
53
|
return None
|
|
54
|
+
|
|
55
|
+
def test_outcomes(self, ref: ReportRef) -> dict[str, dict[str, Any]] | None:
|
|
56
|
+
"""Map ``node_id -> {"outcome", "duration"}`` for one run, or ``None``.
|
|
57
|
+
|
|
58
|
+
Powers cross-run views (compare/flaky/history). Optional capability
|
|
59
|
+
(default: unsupported).
|
|
60
|
+
"""
|
|
61
|
+
return None
|
|
@@ -21,22 +21,28 @@ import json
|
|
|
21
21
|
import logging
|
|
22
22
|
import os
|
|
23
23
|
import shutil
|
|
24
|
+
import threading
|
|
25
|
+
import time
|
|
24
26
|
import zipfile
|
|
25
27
|
from pathlib import Path
|
|
26
28
|
from typing import Any
|
|
27
29
|
|
|
28
30
|
from airflow_pytest_operator import JUnitResultParser
|
|
29
31
|
|
|
30
|
-
from ..config import get_reports_root
|
|
32
|
+
from ..config import get_reports_root, get_scan_cache_ttl
|
|
31
33
|
from ..layout import ALLURE_DIRNAME, META_FILENAME, REPORT_FILENAME, ReportLayout
|
|
32
34
|
from ..models import CaseView, ReportDetail, ReportRef, ReportSummary
|
|
33
35
|
from .base import ReportSource
|
|
34
36
|
|
|
35
37
|
try: # prefer the hardened parser when present (matches the operator)
|
|
36
38
|
from defusedxml.ElementTree import parse as _xml_parse
|
|
39
|
+
|
|
40
|
+
_SECURE_XML = True
|
|
37
41
|
except Exception: # pragma: no cover - fallback path
|
|
38
42
|
from xml.etree.ElementTree import parse as _xml_parse
|
|
39
43
|
|
|
44
|
+
_SECURE_XML = False
|
|
45
|
+
|
|
40
46
|
_log = logging.getLogger(__name__)
|
|
41
47
|
|
|
42
48
|
#: Cap one case's captured output so a pathological report can't bloat a response.
|
|
@@ -52,42 +58,81 @@ class FileSystemReportSource(ReportSource):
|
|
|
52
58
|
report_root: str | None = None,
|
|
53
59
|
layout: ReportLayout | None = None,
|
|
54
60
|
parser: JUnitResultParser | None = None,
|
|
61
|
+
scan_cache_ttl: float | None = None,
|
|
55
62
|
) -> None:
|
|
56
63
|
self._report_root = os.path.abspath(report_root or get_reports_root())
|
|
57
64
|
self._layout = layout or ReportLayout()
|
|
58
65
|
self._parser = parser or JUnitResultParser()
|
|
66
|
+
self._scan_ttl = (
|
|
67
|
+
get_scan_cache_ttl() if scan_cache_ttl is None else scan_cache_ttl
|
|
68
|
+
)
|
|
69
|
+
# (monotonic_timestamp, summaries); guarded so concurrent requests on a cold
|
|
70
|
+
# cache do a single tree walk rather than one each (single-flight).
|
|
71
|
+
self._scan_cache: tuple[float, list[ReportSummary]] | None = None
|
|
72
|
+
self._scan_lock = threading.Lock()
|
|
59
73
|
|
|
60
74
|
@property
|
|
61
75
|
def report_root(self) -> str:
|
|
62
76
|
return self._report_root
|
|
63
77
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
) -> list[ReportSummary]:
|
|
78
|
+
@property
|
|
79
|
+
def secure_xml(self) -> bool:
|
|
80
|
+
"""Whether JUnit XML is parsed with the hardened ``defusedxml`` parser."""
|
|
81
|
+
return _SECURE_XML
|
|
82
|
+
|
|
83
|
+
def _scan_disk(self) -> list[ReportSummary]:
|
|
84
|
+
"""Walk the tree and build every summary, newest first (uncached)."""
|
|
70
85
|
root = Path(self._report_root)
|
|
71
86
|
if not root.is_dir():
|
|
72
87
|
return []
|
|
73
|
-
|
|
74
|
-
summaries: list[ReportSummary] = []
|
|
88
|
+
out: list[ReportSummary] = []
|
|
75
89
|
for meta_file in root.rglob(META_FILENAME):
|
|
76
90
|
meta = self._load_meta(meta_file)
|
|
77
91
|
if meta is None:
|
|
78
92
|
continue
|
|
79
93
|
summary = self._summary_from_meta(meta)
|
|
80
|
-
if summary is None:
|
|
81
|
-
|
|
82
|
-
if dag_id and dag_id.lower() not in summary.ref.dag_id.lower():
|
|
83
|
-
continue
|
|
84
|
-
if run_id and run_id.lower() not in summary.ref.run_id.lower():
|
|
85
|
-
continue
|
|
86
|
-
summaries.append(summary)
|
|
87
|
-
|
|
94
|
+
if summary is not None:
|
|
95
|
+
out.append(summary)
|
|
88
96
|
# Newest first: ISO-8601 created_at sorts chronologically; missing sorts last.
|
|
89
|
-
|
|
90
|
-
return
|
|
97
|
+
out.sort(key=lambda s: s.created_at or "", reverse=True)
|
|
98
|
+
return out
|
|
99
|
+
|
|
100
|
+
def _all_summaries(self) -> list[ReportSummary]:
|
|
101
|
+
"""The full scan, reused within the TTL so a page's several summary-driven
|
|
102
|
+
endpoints share one tree walk. ``ttl <= 0`` disables caching."""
|
|
103
|
+
if self._scan_ttl <= 0:
|
|
104
|
+
return self._scan_disk()
|
|
105
|
+
cached = self._scan_cache
|
|
106
|
+
if cached is not None and (time.monotonic() - cached[0]) < self._scan_ttl:
|
|
107
|
+
return cached[1]
|
|
108
|
+
with self._scan_lock:
|
|
109
|
+
# Re-check: another thread may have refreshed while we waited on the lock.
|
|
110
|
+
cached = self._scan_cache
|
|
111
|
+
if cached is not None and (time.monotonic() - cached[0]) < self._scan_ttl:
|
|
112
|
+
return cached[1]
|
|
113
|
+
fresh = self._scan_disk()
|
|
114
|
+
self._scan_cache = (time.monotonic(), fresh)
|
|
115
|
+
return fresh
|
|
116
|
+
|
|
117
|
+
def _invalidate_scan(self) -> None:
|
|
118
|
+
self._scan_cache = None
|
|
119
|
+
|
|
120
|
+
def list_summaries(
|
|
121
|
+
self,
|
|
122
|
+
*,
|
|
123
|
+
dag_id: str | None = None,
|
|
124
|
+
run_id: str | None = None,
|
|
125
|
+
) -> list[ReportSummary]:
|
|
126
|
+
d = dag_id.lower() if dag_id else None
|
|
127
|
+
r = run_id.lower() if run_id else None
|
|
128
|
+
summaries = self._all_summaries()
|
|
129
|
+
# Filter into a fresh list -- never hand back (or sort) the cached one.
|
|
130
|
+
return [
|
|
131
|
+
s
|
|
132
|
+
for s in summaries
|
|
133
|
+
if (not d or d in s.ref.dag_id.lower())
|
|
134
|
+
and (not r or r in s.ref.run_id.lower())
|
|
135
|
+
]
|
|
91
136
|
|
|
92
137
|
def get_detail(self, ref: ReportRef) -> ReportDetail | None:
|
|
93
138
|
# Token is attacker-controlled: bound the directory before reading.
|
|
@@ -136,6 +181,40 @@ class FileSystemReportSource(ReportSource):
|
|
|
136
181
|
)
|
|
137
182
|
return ReportDetail(summary=summary, cases=cases)
|
|
138
183
|
|
|
184
|
+
def test_outcomes(self, ref: ReportRef) -> dict[str, dict[str, Any]] | None:
|
|
185
|
+
report_dir = self._safe_dir(ref)
|
|
186
|
+
if report_dir is None:
|
|
187
|
+
return None
|
|
188
|
+
meta = self._load_meta(Path(os.path.join(report_dir, META_FILENAME)))
|
|
189
|
+
rows = meta.get("tests") if isinstance(meta, dict) else None
|
|
190
|
+
if isinstance(rows, list):
|
|
191
|
+
out: dict[str, dict[str, Any]] = {}
|
|
192
|
+
for row in rows:
|
|
193
|
+
if isinstance(row, (list, tuple)) and len(row) >= 2 and row[0]:
|
|
194
|
+
dur = (
|
|
195
|
+
float(row[2])
|
|
196
|
+
if len(row) > 2 and isinstance(row[2], int | float)
|
|
197
|
+
else 0.0
|
|
198
|
+
)
|
|
199
|
+
out[str(row[0])] = {"outcome": str(row[1]), "duration": dur}
|
|
200
|
+
return out
|
|
201
|
+
# Older archive without the per-test map: parse junit.xml on demand.
|
|
202
|
+
report_path = os.path.join(report_dir, REPORT_FILENAME)
|
|
203
|
+
if not os.path.isfile(report_path):
|
|
204
|
+
return None
|
|
205
|
+
try:
|
|
206
|
+
result = self._parser.parse(report_path)
|
|
207
|
+
except Exception:
|
|
208
|
+
_log.exception("Failed to parse JUnit report %s", report_path)
|
|
209
|
+
return None
|
|
210
|
+
return {
|
|
211
|
+
c.node_id: {
|
|
212
|
+
"outcome": c.outcome,
|
|
213
|
+
"duration": float(getattr(c, "time", 0.0) or 0.0),
|
|
214
|
+
}
|
|
215
|
+
for c in result.cases
|
|
216
|
+
}
|
|
217
|
+
|
|
139
218
|
def delete(self, ref: ReportRef) -> bool:
|
|
140
219
|
target = self._safe_dir(ref)
|
|
141
220
|
if target is None or not os.path.isdir(target):
|
|
@@ -145,6 +224,7 @@ class FileSystemReportSource(ReportSource):
|
|
|
145
224
|
self._prune_empty_parents(
|
|
146
225
|
os.path.dirname(target), os.path.realpath(self._report_root)
|
|
147
226
|
)
|
|
227
|
+
self._invalidate_scan() # the deleted run must drop out of the list at once
|
|
148
228
|
_log.info("Deleted report %s", target)
|
|
149
229
|
return True
|
|
150
230
|
|