pytest-test-observer 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_test_observer-0.1.0/PKG-INFO +226 -0
- pytest_test_observer-0.1.0/README.md +212 -0
- pytest_test_observer-0.1.0/pyproject.toml +88 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/__init__.py +1 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/allure_compat.py +66 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/allure_compat_helpers.py +52 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/buffer.py +51 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/constants.py +57 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/context.py +63 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/helper.py +25 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/models.py +30 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/options.py +68 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/plugin.py +179 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/py.typed +0 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/replay.py +201 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/reporter.py +134 -0
- pytest_test_observer-0.1.0/src/pytest_test_observer/schema.py +90 -0
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: pytest-test-observer
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A pytest plugin for observing test execution events.
|
|
5
|
+
Author: Dmitrii Shakhov
|
|
6
|
+
Author-email: Dmitrii Shakhov <sourx.rus@gmail.com>
|
|
7
|
+
License: MIT
|
|
8
|
+
Requires-Dist: pytest>=7.0
|
|
9
|
+
Requires-Dist: clickhouse-connect>=0.6.0
|
|
10
|
+
Requires-Dist: allure-pytest>=2.13 ; extra == 'allure'
|
|
11
|
+
Requires-Python: >=3.9
|
|
12
|
+
Provides-Extra: allure
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
|
|
15
|
+
# pytest-test-observer
|
|
16
|
+
|
|
17
|
+
An easy-to-use pytest plugin to take your test observability to the next level. Compatible with `allure-pytest`.
|
|
18
|
+
|
|
19
|
+
## Install
|
|
20
|
+
|
|
21
|
+
The plugin isn't published to PyPI yet - install from source:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
uv add --editable git+https://github.com/shakhov-dmitrii/pytest-test-observer
|
|
25
|
+
# or for development inside a clone:
|
|
26
|
+
uv sync
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
For Allure support, install the extra:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
uv add "pytest-test-observer[allure]"
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Quick start
|
|
36
|
+
|
|
37
|
+
1. Start a local ClickHouse **and Grafana**:
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
docker compose up -d
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
This brings up:
|
|
44
|
+
- ClickHouse on <http://localhost:8123> (HTTP) and `:9000` (native)
|
|
45
|
+
- Grafana on <http://localhost:3000> (login: `admin` / `admin`) with the ClickHouse datasource auto-provisioned and a starter dashboard pre-loaded.
|
|
46
|
+
|
|
47
|
+
2. Run your tests with the ClickHouse URL:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
pytest --ch-url=localhost:8123 --ch-table=pytest_results
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
3. Inspect the rows - either via SQL:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
docker exec -it pytest-test-observer-clickhouse clickhouse-client \
|
|
57
|
+
-q "SELECT nodeid, status, duration, ci_provider FROM default.pytest_results ORDER BY timestamp DESC LIMIT 20 FORMAT PrettyCompact"
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
or in the dashboard: <http://localhost:3000/d/pytest-test-observer-overview>
|
|
61
|
+
|
|
62
|
+
If `--ch-url` is not provided, it does nothing and adds no overhead.
|
|
63
|
+
|
|
64
|
+
## Configuration
|
|
65
|
+
|
|
66
|
+
Every connection setting can be supplied three ways, resolved in this order (highest priority first):
|
|
67
|
+
|
|
68
|
+
1. **CLI flag** - `pytest --ch-url=...`
|
|
69
|
+
2. **Environment variable** - `PYTEST_OBSERVER_CH_URL=...`
|
|
70
|
+
3. **pyproject.toml** - `ch_url = "..."` under `[tool.pytest.ini_options]` (or any other pytest config file)
|
|
71
|
+
4. Built-in default
|
|
72
|
+
|
|
73
|
+
| CLI flag | Env var | Ini key (`pyproject.toml`) | Default |
|
|
74
|
+
| ------------------ | -------------------------------- | -------------------------- | ---------------- |
|
|
75
|
+
| `--ch-url` | `PYTEST_OBSERVER_CH_URL` | `ch_url` | none |
|
|
76
|
+
| `--ch-user` | `PYTEST_OBSERVER_CH_USER` | `ch_user` | `default` |
|
|
77
|
+
| `--ch-password` | `PYTEST_OBSERVER_CH_PASSWORD` | `ch_password` | `""` |
|
|
78
|
+
| `--ch-db` | `PYTEST_OBSERVER_CH_DB` | `ch_db` | `default` |
|
|
79
|
+
| `--ch-table` | `PYTEST_OBSERVER_CH_TABLE` | `ch_table` | `pytest_results` |
|
|
80
|
+
| `--ch-send-from` | `PYTEST_OBSERVER_CH_SEND_FROM` | `ch_send_from` | `any` |
|
|
81
|
+
| `--ch-auto-migrate`| `PYTEST_OBSERVER_CH_AUTO_MIGRATE`| `ch_auto_migrate` | `true` |
|
|
82
|
+
|
|
83
|
+
### `--ch-send-from`: where rows come from
|
|
84
|
+
|
|
85
|
+
- `any` (default) - send for both local and CI runs.
|
|
86
|
+
- `ci` - only send when a provider-specific CI sentinel is set (`GITHUB_ACTIONS`, `GITLAB_CI`, `CIRCLECI`, or `JENKINS_URL`).
|
|
87
|
+
|
|
88
|
+
### `--ch-auto-migrate`: schema migrations across plugin versions
|
|
89
|
+
|
|
90
|
+
When a new plugin version adds columns, the plugin auto-applies `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` for each missing column. Existing rows get the column's default value. Old data doesn't change.
|
|
91
|
+
|
|
92
|
+
If your team's policy forbids the plugin running DDL, set `ch_auto_migrate = false` in `pyproject.toml` (or the env / CLI equivalent). The plugin will then refuse to migrate and surface the SQL you'd need to run yourself.
|
|
93
|
+
|
|
94
|
+
### Example: defaults into `pyproject.toml`
|
|
95
|
+
|
|
96
|
+
```toml
|
|
97
|
+
[tool.pytest.ini_options]
|
|
98
|
+
ch_url = "clickhouse.internal:8123"
|
|
99
|
+
ch_db = "ci_metrics"
|
|
100
|
+
ch_table = "pytest_results"
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Then a plain `pytest` picks them up - no flags, no env vars.
|
|
104
|
+
|
|
105
|
+
### Example: CI secret via env var
|
|
106
|
+
|
|
107
|
+
```yaml
|
|
108
|
+
# GitHub Actions
|
|
109
|
+
- run: pytest
|
|
110
|
+
env:
|
|
111
|
+
PYTEST_OBSERVER_CH_URL: ${{ secrets.CLICKHOUSE_URL }}
|
|
112
|
+
PYTEST_OBSERVER_CH_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Other environment variables
|
|
116
|
+
|
|
117
|
+
| Variable | Effect |
|
|
118
|
+
| --------------------------- | ---------------------------------------------------------------- |
|
|
119
|
+
| `PYTEST_OBSERVER_RUN_ID` | Override the auto-generated `run_id` (UUID) for the session |
|
|
120
|
+
| `XDG_CACHE_HOME` | Base directory for the disk-buffer fallback |
|
|
121
|
+
|
|
122
|
+
## ClickHouse schema
|
|
123
|
+
|
|
124
|
+
The table is auto-created on first flush:
|
|
125
|
+
|
|
126
|
+
```sql
|
|
127
|
+
CREATE TABLE IF NOT EXISTS pytest_results (
|
|
128
|
+
run_id String,
|
|
129
|
+
timestamp DateTime64(3),
|
|
130
|
+
started_at UInt64,
|
|
131
|
+
finished_at UInt64,
|
|
132
|
+
nodeid String,
|
|
133
|
+
status LowCardinality(String),
|
|
134
|
+
when_phase LowCardinality(String),
|
|
135
|
+
duration Float64,
|
|
136
|
+
markers Array(String),
|
|
137
|
+
worker_id LowCardinality(String),
|
|
138
|
+
ci_provider LowCardinality(String),
|
|
139
|
+
ci_run_id String,
|
|
140
|
+
git_commit String,
|
|
141
|
+
git_branch String,
|
|
142
|
+
allure_id String,
|
|
143
|
+
allure_title String,
|
|
144
|
+
allure_severity LowCardinality(String),
|
|
145
|
+
allure_labels Map(String, Array(String)),
|
|
146
|
+
allure_links Array(Tuple(String, String, String))
|
|
147
|
+
) ENGINE = MergeTree
|
|
148
|
+
ORDER BY (nodeid, timestamp)
|
|
149
|
+
PARTITION BY toYYYYMM(timestamp);
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## Allure compatibility
|
|
153
|
+
|
|
154
|
+
When `allure-pytest` is installed and tests use the standard Allure decorators, the plugin captures:
|
|
155
|
+
|
|
156
|
+
- Labels (`@allure.feature`, `@allure.story`, `@allure.tag`, `@allure.severity`, `@allure.id`, `@allure.epic`, `@allure.suite`, ...) -> `allure_labels`
|
|
157
|
+
- Links (`@allure.link`, `@allure.issue`, `@allure.testcase`) -> `allure_links`
|
|
158
|
+
- `@allure.title(...)` -> `allure_title`
|
|
159
|
+
- `@allure.severity(...)` -> `allure_severity` (also stored in `allure_labels`)
|
|
160
|
+
- `@allure.id(...)` -> `allure_id`
|
|
161
|
+
|
|
162
|
+
## CI / git context detection
|
|
163
|
+
|
|
164
|
+
Detected automatically.
|
|
165
|
+
|
|
166
|
+
| Provider | Env vars used |
|
|
167
|
+
| -------------- | -------------------------------------------------------------------------------------|
|
|
168
|
+
| GitHub Actions | `GITHUB_ACTIONS`, `GITHUB_RUN_ID`, `GITHUB_SHA`, `GITHUB_HEAD_REF`/`GITHUB_REF_NAME` |
|
|
169
|
+
| GitLab CI | `GITLAB_CI`, `CI_PIPELINE_ID`, `CI_COMMIT_SHA`, `CI_COMMIT_REF_NAME` |
|
|
170
|
+
| CircleCI | `CIRCLECI`, `CIRCLE_BUILD_NUM`, `CIRCLE_SHA1`, `CIRCLE_BRANCH` |
|
|
171
|
+
| Jenkins | `JENKINS_URL`, `BUILD_ID`/`BUILD_NUMBER`, `GIT_COMMIT`, `GIT_BRANCH` |
|
|
172
|
+
| Local | `git rev-parse HEAD` and `git rev-parse --abbrev-ref HEAD` (`ci_provider="local"`) |
|
|
173
|
+
|
|
174
|
+
## Disk buffer fallback
|
|
175
|
+
|
|
176
|
+
When ClickHouse is unreachable, slow, or rejects the insert, the batch is written to:
|
|
177
|
+
|
|
178
|
+
```
|
|
179
|
+
$XDG_CACHE_HOME/pytest-test-observer/<run_id>.jsonl # or ~/.cache/... if unset
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
One JSON object per line, keys identical to the ClickHouse columns. The plugin emits a `warnings.warn` with the path. **The pytest exit code is unaffected.**
|
|
183
|
+
|
|
184
|
+
### Replaying buffered files back into ClickHouse
|
|
185
|
+
|
|
186
|
+
Once ClickHouse is reachable again, run:
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
python -m pytest_test_observer.replay --ch-url=localhost:8123
|
|
190
|
+
# or pick specific files:
|
|
191
|
+
python -m pytest_test_observer.replay /path/to/run-abc.jsonl
|
|
192
|
+
# or just see what would happen:
|
|
193
|
+
python -m pytest_test_observer.replay --dry-run
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
All the `PYTEST_OBSERVER_CH_*` env vars from the configuration table are honoured by the replay tool too.
|
|
197
|
+
|
|
198
|
+
## Example queries
|
|
199
|
+
|
|
200
|
+
```sql
|
|
201
|
+
-- Top 10 flakiest tests in the last 30 days
|
|
202
|
+
SELECT
|
|
203
|
+
nodeid,
|
|
204
|
+
countIf(status IN ('failed','broken')) AS non_passes,
|
|
205
|
+
count() AS total,
|
|
206
|
+
non_passes / total AS flakiness
|
|
207
|
+
FROM pytest_results
|
|
208
|
+
WHERE timestamp > now() - INTERVAL 30 DAY
|
|
209
|
+
GROUP BY nodeid
|
|
210
|
+
HAVING total >= 10 AND non_passes > 0
|
|
211
|
+
ORDER BY flakiness DESC
|
|
212
|
+
LIMIT 10;
|
|
213
|
+
|
|
214
|
+
-- Slowest 10 tests (median duration)
|
|
215
|
+
SELECT nodeid, quantileExact(0.5)(duration) AS p50_seconds, count() AS runs
|
|
216
|
+
FROM pytest_results
|
|
217
|
+
WHERE timestamp > now() - INTERVAL 7 DAY AND status = 'passed'
|
|
218
|
+
GROUP BY nodeid
|
|
219
|
+
ORDER BY p50_seconds DESC
|
|
220
|
+
LIMIT 10;
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
## License
|
|
224
|
+
|
|
225
|
+
This project is licensed under the MIT License.
|
|
226
|
+
See the LICENSE file for details.
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
# pytest-test-observer
|
|
2
|
+
|
|
3
|
+
An easy-to-use pytest plugin to take your test observability to the next level. Compatible with `allure-pytest`.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
The plugin isn't published to PyPI yet - install from source:
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
uv add --editable git+https://github.com/shakhov-dmitrii/pytest-test-observer
|
|
11
|
+
# or for development inside a clone:
|
|
12
|
+
uv sync
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
For Allure support, install the extra:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
uv add "pytest-test-observer[allure]"
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Quick start
|
|
22
|
+
|
|
23
|
+
1. Start a local ClickHouse **and Grafana**:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
docker compose up -d
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
This brings up:
|
|
30
|
+
- ClickHouse on <http://localhost:8123> (HTTP) and `:9000` (native)
|
|
31
|
+
- Grafana on <http://localhost:3000> (login: `admin` / `admin`) with the ClickHouse datasource auto-provisioned and a starter dashboard pre-loaded.
|
|
32
|
+
|
|
33
|
+
2. Run your tests with the ClickHouse URL:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pytest --ch-url=localhost:8123 --ch-table=pytest_results
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
3. Inspect the rows - either via SQL:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
docker exec -it pytest-test-observer-clickhouse clickhouse-client \
|
|
43
|
+
-q "SELECT nodeid, status, duration, ci_provider FROM default.pytest_results ORDER BY timestamp DESC LIMIT 20 FORMAT PrettyCompact"
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
or in the dashboard: <http://localhost:3000/d/pytest-test-observer-overview>
|
|
47
|
+
|
|
48
|
+
If `--ch-url` is not provided, it does nothing and adds no overhead.
|
|
49
|
+
|
|
50
|
+
## Configuration
|
|
51
|
+
|
|
52
|
+
Every connection setting can be supplied three ways, resolved in this order (highest priority first):
|
|
53
|
+
|
|
54
|
+
1. **CLI flag** - `pytest --ch-url=...`
|
|
55
|
+
2. **Environment variable** - `PYTEST_OBSERVER_CH_URL=...`
|
|
56
|
+
3. **pyproject.toml** - `ch_url = "..."` under `[tool.pytest.ini_options]` (or any other pytest config file)
|
|
57
|
+
4. Built-in default
|
|
58
|
+
|
|
59
|
+
| CLI flag | Env var | Ini key (`pyproject.toml`) | Default |
|
|
60
|
+
| ------------------ | -------------------------------- | -------------------------- | ---------------- |
|
|
61
|
+
| `--ch-url` | `PYTEST_OBSERVER_CH_URL` | `ch_url` | none |
|
|
62
|
+
| `--ch-user` | `PYTEST_OBSERVER_CH_USER` | `ch_user` | `default` |
|
|
63
|
+
| `--ch-password` | `PYTEST_OBSERVER_CH_PASSWORD` | `ch_password` | `""` |
|
|
64
|
+
| `--ch-db` | `PYTEST_OBSERVER_CH_DB` | `ch_db` | `default` |
|
|
65
|
+
| `--ch-table` | `PYTEST_OBSERVER_CH_TABLE` | `ch_table` | `pytest_results` |
|
|
66
|
+
| `--ch-send-from` | `PYTEST_OBSERVER_CH_SEND_FROM` | `ch_send_from` | `any` |
|
|
67
|
+
| `--ch-auto-migrate`| `PYTEST_OBSERVER_CH_AUTO_MIGRATE`| `ch_auto_migrate` | `true` |
|
|
68
|
+
|
|
69
|
+
### `--ch-send-from`: where rows come from
|
|
70
|
+
|
|
71
|
+
- `any` (default) - send for both local and CI runs.
|
|
72
|
+
- `ci` - only send when a provider-specific CI sentinel is set (`GITHUB_ACTIONS`, `GITLAB_CI`, `CIRCLECI`, or `JENKINS_URL`).
|
|
73
|
+
|
|
74
|
+
### `--ch-auto-migrate`: schema migrations across plugin versions
|
|
75
|
+
|
|
76
|
+
When a new plugin version adds columns, the plugin auto-applies `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` for each missing column. Existing rows get the column's default value. Old data doesn't change.
|
|
77
|
+
|
|
78
|
+
If your team's policy forbids the plugin running DDL, set `ch_auto_migrate = false` in `pyproject.toml` (or the env / CLI equivalent). The plugin will then refuse to migrate and surface the SQL you'd need to run yourself.
|
|
79
|
+
|
|
80
|
+
### Example: defaults into `pyproject.toml`
|
|
81
|
+
|
|
82
|
+
```toml
|
|
83
|
+
[tool.pytest.ini_options]
|
|
84
|
+
ch_url = "clickhouse.internal:8123"
|
|
85
|
+
ch_db = "ci_metrics"
|
|
86
|
+
ch_table = "pytest_results"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Then a plain `pytest` picks them up - no flags, no env vars.
|
|
90
|
+
|
|
91
|
+
### Example: CI secret via env var
|
|
92
|
+
|
|
93
|
+
```yaml
|
|
94
|
+
# GitHub Actions
|
|
95
|
+
- run: pytest
|
|
96
|
+
env:
|
|
97
|
+
PYTEST_OBSERVER_CH_URL: ${{ secrets.CLICKHOUSE_URL }}
|
|
98
|
+
PYTEST_OBSERVER_CH_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }}
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
### Other environment variables
|
|
102
|
+
|
|
103
|
+
| Variable | Effect |
|
|
104
|
+
| --------------------------- | ---------------------------------------------------------------- |
|
|
105
|
+
| `PYTEST_OBSERVER_RUN_ID` | Override the auto-generated `run_id` (UUID) for the session |
|
|
106
|
+
| `XDG_CACHE_HOME` | Base directory for the disk-buffer fallback |
|
|
107
|
+
|
|
108
|
+
## ClickHouse schema
|
|
109
|
+
|
|
110
|
+
The table is auto-created on first flush:
|
|
111
|
+
|
|
112
|
+
```sql
|
|
113
|
+
CREATE TABLE IF NOT EXISTS pytest_results (
|
|
114
|
+
run_id String,
|
|
115
|
+
timestamp DateTime64(3),
|
|
116
|
+
started_at UInt64,
|
|
117
|
+
finished_at UInt64,
|
|
118
|
+
nodeid String,
|
|
119
|
+
status LowCardinality(String),
|
|
120
|
+
when_phase LowCardinality(String),
|
|
121
|
+
duration Float64,
|
|
122
|
+
markers Array(String),
|
|
123
|
+
worker_id LowCardinality(String),
|
|
124
|
+
ci_provider LowCardinality(String),
|
|
125
|
+
ci_run_id String,
|
|
126
|
+
git_commit String,
|
|
127
|
+
git_branch String,
|
|
128
|
+
allure_id String,
|
|
129
|
+
allure_title String,
|
|
130
|
+
allure_severity LowCardinality(String),
|
|
131
|
+
allure_labels Map(String, Array(String)),
|
|
132
|
+
allure_links Array(Tuple(String, String, String))
|
|
133
|
+
) ENGINE = MergeTree
|
|
134
|
+
ORDER BY (nodeid, timestamp)
|
|
135
|
+
PARTITION BY toYYYYMM(timestamp);
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## Allure compatibility
|
|
139
|
+
|
|
140
|
+
When `allure-pytest` is installed and tests use the standard Allure decorators, the plugin captures:
|
|
141
|
+
|
|
142
|
+
- Labels (`@allure.feature`, `@allure.story`, `@allure.tag`, `@allure.severity`, `@allure.id`, `@allure.epic`, `@allure.suite`, ...) -> `allure_labels`
|
|
143
|
+
- Links (`@allure.link`, `@allure.issue`, `@allure.testcase`) -> `allure_links`
|
|
144
|
+
- `@allure.title(...)` -> `allure_title`
|
|
145
|
+
- `@allure.severity(...)` -> `allure_severity` (also stored in `allure_labels`)
|
|
146
|
+
- `@allure.id(...)` -> `allure_id`
|
|
147
|
+
|
|
148
|
+
## CI / git context detection
|
|
149
|
+
|
|
150
|
+
Detected automatically.
|
|
151
|
+
|
|
152
|
+
| Provider | Env vars used |
|
|
153
|
+
| -------------- | -------------------------------------------------------------------------------------|
|
|
154
|
+
| GitHub Actions | `GITHUB_ACTIONS`, `GITHUB_RUN_ID`, `GITHUB_SHA`, `GITHUB_HEAD_REF`/`GITHUB_REF_NAME` |
|
|
155
|
+
| GitLab CI | `GITLAB_CI`, `CI_PIPELINE_ID`, `CI_COMMIT_SHA`, `CI_COMMIT_REF_NAME` |
|
|
156
|
+
| CircleCI | `CIRCLECI`, `CIRCLE_BUILD_NUM`, `CIRCLE_SHA1`, `CIRCLE_BRANCH` |
|
|
157
|
+
| Jenkins | `JENKINS_URL`, `BUILD_ID`/`BUILD_NUMBER`, `GIT_COMMIT`, `GIT_BRANCH` |
|
|
158
|
+
| Local | `git rev-parse HEAD` and `git rev-parse --abbrev-ref HEAD` (`ci_provider="local"`) |
|
|
159
|
+
|
|
160
|
+
## Disk buffer fallback
|
|
161
|
+
|
|
162
|
+
When ClickHouse is unreachable, slow, or rejects the insert, the batch is written to:
|
|
163
|
+
|
|
164
|
+
```
|
|
165
|
+
$XDG_CACHE_HOME/pytest-test-observer/<run_id>.jsonl # or ~/.cache/... if unset
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
One JSON object per line, keys identical to the ClickHouse columns. The plugin emits a `warnings.warn` with the path. **The pytest exit code is unaffected.**
|
|
169
|
+
|
|
170
|
+
### Replaying buffered files back into ClickHouse
|
|
171
|
+
|
|
172
|
+
Once ClickHouse is reachable again, run:
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
python -m pytest_test_observer.replay --ch-url=localhost:8123
|
|
176
|
+
# or pick specific files:
|
|
177
|
+
python -m pytest_test_observer.replay /path/to/run-abc.jsonl
|
|
178
|
+
# or just see what would happen:
|
|
179
|
+
python -m pytest_test_observer.replay --dry-run
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
All the `PYTEST_OBSERVER_CH_*` env vars from the configuration table are honoured by the replay tool too.
|
|
183
|
+
|
|
184
|
+
## Example queries
|
|
185
|
+
|
|
186
|
+
```sql
|
|
187
|
+
-- Top 10 flakiest tests in the last 30 days
|
|
188
|
+
SELECT
|
|
189
|
+
nodeid,
|
|
190
|
+
countIf(status IN ('failed','broken')) AS non_passes,
|
|
191
|
+
count() AS total,
|
|
192
|
+
non_passes / total AS flakiness
|
|
193
|
+
FROM pytest_results
|
|
194
|
+
WHERE timestamp > now() - INTERVAL 30 DAY
|
|
195
|
+
GROUP BY nodeid
|
|
196
|
+
HAVING total >= 10 AND non_passes > 0
|
|
197
|
+
ORDER BY flakiness DESC
|
|
198
|
+
LIMIT 10;
|
|
199
|
+
|
|
200
|
+
-- Slowest 10 tests (median duration)
|
|
201
|
+
SELECT nodeid, quantileExact(0.5)(duration) AS p50_seconds, count() AS runs
|
|
202
|
+
FROM pytest_results
|
|
203
|
+
WHERE timestamp > now() - INTERVAL 7 DAY AND status = 'passed'
|
|
204
|
+
GROUP BY nodeid
|
|
205
|
+
ORDER BY p50_seconds DESC
|
|
206
|
+
LIMIT 10;
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
## License
|
|
210
|
+
|
|
211
|
+
This project is licensed under the MIT License.
|
|
212
|
+
See the LICENSE file for details.
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pytest-test-observer"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "A pytest plugin for observing test execution events."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = { text = "MIT" }
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "Dmitrii Shakhov", email = "sourx.rus@gmail.com" }
|
|
9
|
+
]
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"pytest>=7.0",
|
|
13
|
+
"clickhouse-connect>=0.6.0",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[project.optional-dependencies]
|
|
17
|
+
allure = ["allure-pytest>=2.13"]
|
|
18
|
+
|
|
19
|
+
[project.entry-points.pytest11]
|
|
20
|
+
test_observer = "pytest_test_observer.plugin"
|
|
21
|
+
|
|
22
|
+
[build-system]
|
|
23
|
+
requires = ["uv_build>=0.10.9,<0.11.0"]
|
|
24
|
+
build-backend = "uv_build"
|
|
25
|
+
|
|
26
|
+
[dependency-groups]
|
|
27
|
+
dev = [
|
|
28
|
+
"pytest>=7.0",
|
|
29
|
+
"allure-pytest>=2.13",
|
|
30
|
+
"pytest-xdist>=3.0",
|
|
31
|
+
"pytest-asyncio>=0.23",
|
|
32
|
+
"pytest-cov>=5.0",
|
|
33
|
+
"ruff>=0.6",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[tool.pytest.ini_options]
|
|
37
|
+
testpaths = ["tests"]
|
|
38
|
+
filterwarnings = [
|
|
39
|
+
"ignore:The configuration option \"asyncio_default_fixture_loop_scope\":pytest.PytestDeprecationWarning",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
[tool.coverage.run]
|
|
43
|
+
branch = true
|
|
44
|
+
source = ["src/pytest_test_observer"]
|
|
45
|
+
omit = []
|
|
46
|
+
|
|
47
|
+
[tool.coverage.report]
|
|
48
|
+
show_missing = true
|
|
49
|
+
skip_covered = false
|
|
50
|
+
precision = 1
|
|
51
|
+
fail_under = 90
|
|
52
|
+
exclude_lines = [
|
|
53
|
+
"pragma: no cover",
|
|
54
|
+
"if TYPE_CHECKING:",
|
|
55
|
+
"raise NotImplementedError",
|
|
56
|
+
"if __name__ == .__main__.:",
|
|
57
|
+
"@overload",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
[tool.coverage.html]
|
|
61
|
+
directory = "htmlcov"
|
|
62
|
+
|
|
63
|
+
[tool.ruff]
|
|
64
|
+
line-length = 100
|
|
65
|
+
target-version = "py39"
|
|
66
|
+
extend-exclude = [".venv", "dist", "build"]
|
|
67
|
+
|
|
68
|
+
[tool.ruff.lint]
|
|
69
|
+
select = [
|
|
70
|
+
"E", # pycodestyle errors
|
|
71
|
+
"W", # pycodestyle warnings
|
|
72
|
+
"F", # pyflakes
|
|
73
|
+
"I", # isort
|
|
74
|
+
"B", # bugbear
|
|
75
|
+
"UP", # pyupgrade
|
|
76
|
+
"SIM", # simplify
|
|
77
|
+
"RUF", # ruff-specific
|
|
78
|
+
]
|
|
79
|
+
ignore = [
|
|
80
|
+
"E501", # line length is enforced by formatter, not the linter
|
|
81
|
+
]
|
|
82
|
+
|
|
83
|
+
[tool.ruff.lint.per-file-ignores]
|
|
84
|
+
"tests/*" = ["B011"] # `assert False` is fine in tests
|
|
85
|
+
|
|
86
|
+
[tool.ruff.format]
|
|
87
|
+
quote-style = "double"
|
|
88
|
+
indent-style = "space"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Compatibility layer for allure-pytest — the public API surface."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import TYPE_CHECKING, Any
|
|
6
|
+
|
|
7
|
+
from pytest_test_observer.allure_compat_helpers import (
|
|
8
|
+
extract_labels,
|
|
9
|
+
extract_links,
|
|
10
|
+
first_label_value,
|
|
11
|
+
read_allure_title,
|
|
12
|
+
)
|
|
13
|
+
from pytest_test_observer.constants import ID_LABEL, SEVERITY_LABEL, TestStatus
|
|
14
|
+
from pytest_test_observer.models import AllureMeta
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
import pytest
|
|
18
|
+
|
|
19
|
+
__all__ = ("AllureMeta", "detect_allure", "empty_allure_meta", "extract_allure_meta", "map_status")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def detect_allure(pluginmanager: Any) -> bool:
|
|
23
|
+
return bool(pluginmanager.hasplugin("allure_pytest"))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def map_status(report: pytest.TestReport) -> TestStatus:
|
|
27
|
+
"""Map a pytest TestReport to the allure-style status vocabulary.
|
|
28
|
+
|
|
29
|
+
Rules:
|
|
30
|
+
- skipped → SKIPPED
|
|
31
|
+
- passed → PASSED
|
|
32
|
+
- failed (call phase) → FAILED
|
|
33
|
+
- failed (setup/teardown phase) → BROKEN (allure convention)
|
|
34
|
+
- outcome == 'rerun' (pytest-rerunfailures) → FAILED/BROKEN by phase
|
|
35
|
+
- anything else → UNKNOWN
|
|
36
|
+
"""
|
|
37
|
+
if report.skipped:
|
|
38
|
+
return TestStatus.SKIPPED
|
|
39
|
+
if report.passed:
|
|
40
|
+
return TestStatus.PASSED
|
|
41
|
+
if report.failed or report.outcome == TestStatus.RERUN:
|
|
42
|
+
if report.when == "call":
|
|
43
|
+
return TestStatus.FAILED
|
|
44
|
+
if report.when in ("setup", "teardown"):
|
|
45
|
+
return TestStatus.BROKEN
|
|
46
|
+
return TestStatus.UNKNOWN
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def extract_allure_meta(item: pytest.Item) -> AllureMeta:
|
|
50
|
+
meta = empty_allure_meta()
|
|
51
|
+
meta["labels"] = extract_labels(item)
|
|
52
|
+
meta["links"] = extract_links(item)
|
|
53
|
+
meta["title"] = read_allure_title(item)
|
|
54
|
+
meta["severity"] = first_label_value(meta["labels"], SEVERITY_LABEL)
|
|
55
|
+
meta["allure_id"] = first_label_value(meta["labels"], ID_LABEL)
|
|
56
|
+
return meta
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def empty_allure_meta() -> AllureMeta:
|
|
60
|
+
return {
|
|
61
|
+
"labels": {},
|
|
62
|
+
"links": [],
|
|
63
|
+
"title": "",
|
|
64
|
+
"severity": "",
|
|
65
|
+
"allure_id": "",
|
|
66
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import TYPE_CHECKING
|
|
4
|
+
|
|
5
|
+
from pytest_test_observer.constants import (
|
|
6
|
+
DISPLAY_NAME_ATTR,
|
|
7
|
+
LABEL_MARK,
|
|
8
|
+
LABEL_TYPE_KEY,
|
|
9
|
+
LINK_MARK,
|
|
10
|
+
LINK_NAME_KEY,
|
|
11
|
+
LINK_TYPE_KEY,
|
|
12
|
+
)
|
|
13
|
+
from pytest_test_observer.helper import as_str
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
import pytest
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def extract_labels(item: pytest.Item) -> dict[str, list[str]]:
|
|
20
|
+
labels: dict[str, list[str]] = {}
|
|
21
|
+
for mark in item.iter_markers(LABEL_MARK):
|
|
22
|
+
label_type = mark.kwargs.get(LABEL_TYPE_KEY)
|
|
23
|
+
if label_type is None:
|
|
24
|
+
continue
|
|
25
|
+
type_str = as_str(label_type)
|
|
26
|
+
for value in mark.args:
|
|
27
|
+
labels.setdefault(type_str, []).append(as_str(value))
|
|
28
|
+
return labels
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def extract_links(item: pytest.Item) -> list[tuple[str, str, str]]:
|
|
32
|
+
links: list[tuple[str, str, str]] = []
|
|
33
|
+
for mark in item.iter_markers(LINK_MARK):
|
|
34
|
+
if not mark.args:
|
|
35
|
+
continue
|
|
36
|
+
url = as_str(mark.args[0])
|
|
37
|
+
link_type = as_str(mark.kwargs.get(LINK_TYPE_KEY, ""))
|
|
38
|
+
link_name = as_str(mark.kwargs.get(LINK_NAME_KEY, ""))
|
|
39
|
+
links.append((link_type, url, link_name))
|
|
40
|
+
return links
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def read_allure_title(item: pytest.Item) -> str:
|
|
44
|
+
func = getattr(item, "function", None) or getattr(item, "obj", None)
|
|
45
|
+
if func is None:
|
|
46
|
+
return ""
|
|
47
|
+
return str(getattr(func, DISPLAY_NAME_ATTR, "") or "")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def first_label_value(labels: dict[str, list[str]], key: str) -> str:
|
|
51
|
+
values = labels.get(key, ())
|
|
52
|
+
return values[0] if values else ""
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""On-disk JSONL fallback for batches we couldn't push to ClickHouse."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
_BUFFER_SUBDIR = "pytest-test-observer"
|
|
13
|
+
_EXTENSION = ".jsonl"
|
|
14
|
+
|
|
15
|
+
# Whitelist of characters allowed verbatim in run_id-derived filenames.
|
|
16
|
+
# Anything else (including path separators and `..`) is mapped to '_'.
|
|
17
|
+
_UNSAFE_CHARS = re.compile(r"[^A-Za-z0-9._-]")
|
|
18
|
+
# OS filename limits hover around 255 bytes; leave headroom for the
|
|
19
|
+
# extension and any pathological multi-byte inputs.
|
|
20
|
+
_MAX_RUN_ID = 128
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def buffer_dir() -> Path:
|
|
24
|
+
base = os.environ.get("XDG_CACHE_HOME")
|
|
25
|
+
root = Path(base) if base else Path.home() / ".cache"
|
|
26
|
+
return root / _BUFFER_SUBDIR
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def write_jsonl(rows: list[dict], run_id: str) -> Path:
|
|
30
|
+
safe_id = _sanitize_run_id(run_id)
|
|
31
|
+
path = buffer_dir() / f"{safe_id}{_EXTENSION}"
|
|
32
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
if not rows:
|
|
34
|
+
return path
|
|
35
|
+
content = "\n".join(json.dumps(row, default=_json_default) for row in rows) + "\n"
|
|
36
|
+
with path.open("a", encoding="utf-8") as f:
|
|
37
|
+
f.write(content)
|
|
38
|
+
return path
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _sanitize_run_id(run_id: str) -> str:
|
|
42
|
+
cleaned = _UNSAFE_CHARS.sub("_", run_id or "")[:_MAX_RUN_ID]
|
|
43
|
+
return cleaned or "unknown"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _json_default(obj: Any) -> Any:
|
|
47
|
+
if isinstance(obj, datetime):
|
|
48
|
+
return obj.isoformat(timespec="milliseconds")
|
|
49
|
+
if isinstance(obj, tuple):
|
|
50
|
+
return list(obj)
|
|
51
|
+
raise TypeError(f"Object of type {type(obj).__name__} is not JSON serializable")
|