testrelic-pytest 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- testrelic_pytest-0.1.1/.gitignore +74 -0
- testrelic_pytest-0.1.1/CHANGELOG.md +35 -0
- testrelic_pytest-0.1.1/LICENSE +21 -0
- testrelic_pytest-0.1.1/PKG-INFO +147 -0
- testrelic_pytest-0.1.1/README.md +105 -0
- testrelic_pytest-0.1.1/pyproject.toml +108 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/__init__.py +35 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/_version.py +1 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/ci_detector.py +102 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/cli.py +59 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/cloud/__init__.py +64 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/cloud/artifact.py +260 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/cloud/auth.py +258 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/cloud/client.py +274 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/cloud/queue.py +108 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/cloud/repo_cache.py +85 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/cloud/upload.py +287 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/config.py +185 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/git_metadata.py +105 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/plugin.py +596 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/redaction.py +43 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/reporter.py +258 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/run_type_env.py +30 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/schema.py +146 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/translator.py +281 -0
- testrelic_pytest-0.1.1/src/testrelic_pytest/types.py +22 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# Virtual environments
|
|
30
|
+
.venv/
|
|
31
|
+
venv/
|
|
32
|
+
env/
|
|
33
|
+
ENV/
|
|
34
|
+
|
|
35
|
+
# Test/coverage caches
|
|
36
|
+
.tox/
|
|
37
|
+
.nox/
|
|
38
|
+
.coverage
|
|
39
|
+
.coverage.*
|
|
40
|
+
.cache
|
|
41
|
+
.pytest_cache/
|
|
42
|
+
htmlcov/
|
|
43
|
+
nosetests.xml
|
|
44
|
+
coverage.xml
|
|
45
|
+
*.cover
|
|
46
|
+
*.py,cover
|
|
47
|
+
.hypothesis/
|
|
48
|
+
|
|
49
|
+
# Type/lint caches
|
|
50
|
+
.mypy_cache/
|
|
51
|
+
.ruff_cache/
|
|
52
|
+
.dmypy.json
|
|
53
|
+
dmypy.json
|
|
54
|
+
|
|
55
|
+
# IDE
|
|
56
|
+
.idea/
|
|
57
|
+
.vscode/
|
|
58
|
+
*.swp
|
|
59
|
+
*~
|
|
60
|
+
|
|
61
|
+
# OS
|
|
62
|
+
.DS_Store
|
|
63
|
+
Thumbs.db
|
|
64
|
+
|
|
65
|
+
# TestRelic local state
|
|
66
|
+
.testrelic/
|
|
67
|
+
*.queue.json
|
|
68
|
+
|
|
69
|
+
# Jupyter
|
|
70
|
+
.ipynb_checkpoints
|
|
71
|
+
|
|
72
|
+
# Local env
|
|
73
|
+
.env
|
|
74
|
+
.env.local
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `testrelic-pytest` are documented here.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and the project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [0.1.1] - 2026-05-26
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
- Cloud `/api/v1/runs` rejected 0.1.0 batch uploads. The server validator uses
|
|
12
|
+
Zod's `z.string().datetime()` (Zulu-suffix only) and expects the top-level
|
|
13
|
+
run `duration` as an integer. Additionally, the per-test `duration` must be
|
|
14
|
+
sent as an integer too — Zod accepts float per-test (`z.number()`) but a
|
|
15
|
+
downstream handler returns `500 INTERNAL_ERROR` if it isn't a whole number.
|
|
16
|
+
- Format all timestamps as `...Z` via the new `iso_z()` helper, round run
|
|
17
|
+
duration, and change `PytestResult.duration` to `int` (rounded to ms in the
|
|
18
|
+
translator). All discovered via the first staging smoke test against
|
|
19
|
+
`https://stage.testrelic.ai/api/v1`.
|
|
20
|
+
|
|
21
|
+
## [0.1.0] - 2026-05-25
|
|
22
|
+
|
|
23
|
+
### Added
|
|
24
|
+
- Initial release.
|
|
25
|
+
- pytest11 plugin that captures generic pytest test outcomes and uploads to the
|
|
26
|
+
TestRelic AI cloud via `POST /api/v1/runs` (batch) or the realtime
|
|
27
|
+
`runs/init` → `runs/{id}/tests` → `runs/{id}/finalize` triplet.
|
|
28
|
+
- Outcome translation for `passed`, `failed`, `skipped`, `xfailed`, `xpassed`,
|
|
29
|
+
and setup/teardown `error`.
|
|
30
|
+
- Capture of markers, parametrize ids, keywords, warnings, and skip reasons.
|
|
31
|
+
- Captured stdout/stderr/log embedded inline with redaction and truncation
|
|
32
|
+
(artifact promotion for large streams is planned for v0.2).
|
|
33
|
+
- `pytest-xdist` controller aggregation — workers forward reports through the
|
|
34
|
+
built-in xdist channel; only the controller uploads.
|
|
35
|
+
- `testrelic-pytest drain` CLI for replaying the offline queue.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 TestRelic AI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: testrelic-pytest
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Generic pytest reporter that uploads test results to the TestRelic AI cloud platform.
|
|
5
|
+
Project-URL: Homepage, https://testrelic.ai
|
|
6
|
+
Project-URL: Documentation, https://docs.testrelic.ai/pytest
|
|
7
|
+
Project-URL: Repository, https://github.com/testrelic-ai/testrelic-python-sdk
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/testrelic-ai/testrelic-python-sdk/issues
|
|
9
|
+
Author-email: TestRelic AI <hello@testrelic.ai>
|
|
10
|
+
License: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: analytics,ci,observability,pytest,reporter,test,test-automation,test-results,testing,testrelic
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Framework :: Pytest
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
24
|
+
Classifier: Topic :: Software Development :: Testing
|
|
25
|
+
Requires-Python: >=3.9
|
|
26
|
+
Requires-Dist: httpx<1.0,>=0.27
|
|
27
|
+
Requires-Dist: platformdirs>=4.0
|
|
28
|
+
Requires-Dist: pydantic<3.0,>=2.6
|
|
29
|
+
Requires-Dist: pytest>=7.0
|
|
30
|
+
Requires-Dist: tomli-w>=1.0
|
|
31
|
+
Requires-Dist: tomli>=2.0; python_version < '3.11'
|
|
32
|
+
Requires-Dist: typer<1.0,>=0.12
|
|
33
|
+
Provides-Extra: dev
|
|
34
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
35
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
36
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
37
|
+
Requires-Dist: pytest-xdist>=3.5; extra == 'dev'
|
|
38
|
+
Requires-Dist: respx>=0.21; extra == 'dev'
|
|
39
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
40
|
+
Requires-Dist: twine>=5.1; extra == 'dev'
|
|
41
|
+
Description-Content-Type: text/markdown
|
|
42
|
+
|
|
43
|
+
# testrelic-pytest
|
|
44
|
+
|
|
45
|
+
[](https://pypi.org/project/testrelic-pytest/)
|
|
46
|
+
|
|
47
|
+
Generic [pytest](https://docs.pytest.org/) reporter that ships your test results
|
|
48
|
+
to the [TestRelic AI](https://testrelic.ai/) cloud platform. Works with any
|
|
49
|
+
pytest project — unit tests, API tests, integration tests — and stays out of the
|
|
50
|
+
way when no API key is configured.
|
|
51
|
+
|
|
52
|
+
## Install
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install testrelic-pytest
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The plugin auto-registers via the `pytest11` entry point. Nothing to import.
|
|
59
|
+
|
|
60
|
+
## Configure
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
export TESTRELIC_API_KEY=tr_live_...
|
|
64
|
+
pytest
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
The plugin silently no-ops when `TESTRELIC_API_KEY` is unset, so it is safe to
|
|
68
|
+
add to any project.
|
|
69
|
+
|
|
70
|
+
### Options
|
|
71
|
+
|
|
72
|
+
All options can be set via CLI flag, environment variable, or a JSON file at
|
|
73
|
+
`.testrelic/testrelic-config.json`. Precedence (highest first):
|
|
74
|
+
**CLI flag → environment → JSON file → defaults**.
|
|
75
|
+
|
|
76
|
+
CLI flags use a `--testrelic-pytest-*` prefix so they don't collide with the
|
|
77
|
+
`testrelic-playwright` plugin if both are installed.
|
|
78
|
+
|
|
79
|
+
| CLI flag | Env var | Default | Purpose |
|
|
80
|
+
|---|---|---|---|
|
|
81
|
+
| `--testrelic-pytest-disable` | `TESTRELIC_DISABLE=1` | off | Force-disable the plugin. |
|
|
82
|
+
| `--testrelic-pytest-api-key VALUE` | `TESTRELIC_API_KEY` | — | API key from the TestRelic dashboard. |
|
|
83
|
+
| `--testrelic-pytest-endpoint URL` | `TESTRELIC_CLOUD_ENDPOINT` | `https://platform.testrelic.ai/api/v1` | Cloud base URL. |
|
|
84
|
+
| `--testrelic-pytest-upload-strategy {batch,realtime,both,none}` | `TESTRELIC_UPLOAD_STRATEGY` | `batch` | When to send results. |
|
|
85
|
+
| `--testrelic-pytest-quiet` | `TESTRELIC_QUIET=1` | off | Suppress banner output. |
|
|
86
|
+
| `--testrelic-pytest-run-type {smoke,regression,nightly,ci}` | `TESTRELIC_RUN_TYPE` | — | Dashboard bucket. |
|
|
87
|
+
| `--testrelic-pytest-project-name NAME` | `TESTRELIC_PROJECT_NAME` | git remote or pyproject name | Stable project identity. |
|
|
88
|
+
| `--testrelic-pytest-artifact-threshold-kb N` | — | 32 | Captured output beyond N KB is truncated inline. |
|
|
89
|
+
| `--testrelic-pytest-output PATH` | — | — | Optional JSON dump for debugging. |
|
|
90
|
+
|
|
91
|
+
### Config file (`.testrelic/testrelic-config.json`)
|
|
92
|
+
|
|
93
|
+
```json
|
|
94
|
+
{
|
|
95
|
+
"apiKey": "$TESTRELIC_API_KEY",
|
|
96
|
+
"endpoint": "https://platform.testrelic.ai/api/v1",
|
|
97
|
+
"upload": "batch",
|
|
98
|
+
"projectName": "my-suite",
|
|
99
|
+
"artifactThresholdKb": 32,
|
|
100
|
+
"uploadArtifacts": true,
|
|
101
|
+
"queueDirectory": ".testrelic/queue",
|
|
102
|
+
"metadata": { "team": "platform" }
|
|
103
|
+
}
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`$ENV_VAR` strings are expanded against the process environment at startup.
|
|
107
|
+
|
|
108
|
+
## What gets captured
|
|
109
|
+
|
|
110
|
+
For each test the plugin captures:
|
|
111
|
+
|
|
112
|
+
- `nodeId`, `file`, `className`, `testName`, `parametrizeId`
|
|
113
|
+
- Outcome: `passed | failed | skipped | xfailed | xpassed | error`
|
|
114
|
+
- Duration (ms) and per-phase breakdown (setup/call/teardown)
|
|
115
|
+
- Markers (e.g. `@pytest.mark.smoke`), keywords, parametrize ids
|
|
116
|
+
- Skip reasons and pytest warnings
|
|
117
|
+
- Failure tracebacks and assertion repr
|
|
118
|
+
- Captured stdout / stderr / caplog (inline with redaction; truncated above
|
|
119
|
+
`--testrelic-artifact-threshold-kb`, hard cap 1 MB per stream — artifact
|
|
120
|
+
upload for large streams is planned for v0.2)
|
|
121
|
+
|
|
122
|
+
Test runs include CI metadata auto-detected from GitHub Actions, GitLab CI,
|
|
123
|
+
Jenkins, and CircleCI, plus git branch/commit/author from the working tree.
|
|
124
|
+
|
|
125
|
+
## pytest-xdist
|
|
126
|
+
|
|
127
|
+
Parallel runs via `pytest-xdist` (`pytest -n 4`) are supported out of the box.
|
|
128
|
+
Each worker forwards its `TestReport` objects through the xdist channel; only
|
|
129
|
+
the controller node uploads. The result is exactly one cloud run with all tests
|
|
130
|
+
aggregated.
|
|
131
|
+
|
|
132
|
+
## Offline mode
|
|
133
|
+
|
|
134
|
+
Any upload that fails after retries is written to `~/.testrelic/queue/` as a
|
|
135
|
+
JSON file. Re-run later:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
testrelic-pytest drain
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
The drainer re-authenticates with `TESTRELIC_API_KEY` and replays each entry to
|
|
142
|
+
its original endpoint.
|
|
143
|
+
|
|
144
|
+
## See also
|
|
145
|
+
|
|
146
|
+
- [`testrelic-deepeval`](https://pypi.org/project/testrelic-deepeval/) — DeepEval / LLM evaluation bridge
|
|
147
|
+
- [`testrelic-playwright`](https://pypi.org/project/testrelic-playwright/) — Playwright reporter for pytest
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# testrelic-pytest
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/testrelic-pytest/)
|
|
4
|
+
|
|
5
|
+
Generic [pytest](https://docs.pytest.org/) reporter that ships your test results
|
|
6
|
+
to the [TestRelic AI](https://testrelic.ai/) cloud platform. Works with any
|
|
7
|
+
pytest project — unit tests, API tests, integration tests — and stays out of the
|
|
8
|
+
way when no API key is configured.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install testrelic-pytest
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
The plugin auto-registers via the `pytest11` entry point. Nothing to import.
|
|
17
|
+
|
|
18
|
+
## Configure
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
export TESTRELIC_API_KEY=tr_live_...
|
|
22
|
+
pytest
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
The plugin silently no-ops when `TESTRELIC_API_KEY` is unset, so it is safe to
|
|
26
|
+
add to any project.
|
|
27
|
+
|
|
28
|
+
### Options
|
|
29
|
+
|
|
30
|
+
All options can be set via CLI flag, environment variable, or a JSON file at
|
|
31
|
+
`.testrelic/testrelic-config.json`. Precedence (highest first):
|
|
32
|
+
**CLI flag → environment → JSON file → defaults**.
|
|
33
|
+
|
|
34
|
+
CLI flags use a `--testrelic-pytest-*` prefix so they don't collide with the
|
|
35
|
+
`testrelic-playwright` plugin if both are installed.
|
|
36
|
+
|
|
37
|
+
| CLI flag | Env var | Default | Purpose |
|
|
38
|
+
|---|---|---|---|
|
|
39
|
+
| `--testrelic-pytest-disable` | `TESTRELIC_DISABLE=1` | off | Force-disable the plugin. |
|
|
40
|
+
| `--testrelic-pytest-api-key VALUE` | `TESTRELIC_API_KEY` | — | API key from the TestRelic dashboard. |
|
|
41
|
+
| `--testrelic-pytest-endpoint URL` | `TESTRELIC_CLOUD_ENDPOINT` | `https://platform.testrelic.ai/api/v1` | Cloud base URL. |
|
|
42
|
+
| `--testrelic-pytest-upload-strategy {batch,realtime,both,none}` | `TESTRELIC_UPLOAD_STRATEGY` | `batch` | When to send results. |
|
|
43
|
+
| `--testrelic-pytest-quiet` | `TESTRELIC_QUIET=1` | off | Suppress banner output. |
|
|
44
|
+
| `--testrelic-pytest-run-type {smoke,regression,nightly,ci}` | `TESTRELIC_RUN_TYPE` | — | Dashboard bucket. |
|
|
45
|
+
| `--testrelic-pytest-project-name NAME` | `TESTRELIC_PROJECT_NAME` | git remote or pyproject name | Stable project identity. |
|
|
46
|
+
| `--testrelic-pytest-artifact-threshold-kb N` | — | 32 | Captured output beyond N KB is truncated inline. |
|
|
47
|
+
| `--testrelic-pytest-output PATH` | — | — | Optional JSON dump for debugging. |
|
|
48
|
+
|
|
49
|
+
### Config file (`.testrelic/testrelic-config.json`)
|
|
50
|
+
|
|
51
|
+
```json
|
|
52
|
+
{
|
|
53
|
+
"apiKey": "$TESTRELIC_API_KEY",
|
|
54
|
+
"endpoint": "https://platform.testrelic.ai/api/v1",
|
|
55
|
+
"upload": "batch",
|
|
56
|
+
"projectName": "my-suite",
|
|
57
|
+
"artifactThresholdKb": 32,
|
|
58
|
+
"uploadArtifacts": true,
|
|
59
|
+
"queueDirectory": ".testrelic/queue",
|
|
60
|
+
"metadata": { "team": "platform" }
|
|
61
|
+
}
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
`$ENV_VAR` strings are expanded against the process environment at startup.
|
|
65
|
+
|
|
66
|
+
## What gets captured
|
|
67
|
+
|
|
68
|
+
For each test the plugin captures:
|
|
69
|
+
|
|
70
|
+
- `nodeId`, `file`, `className`, `testName`, `parametrizeId`
|
|
71
|
+
- Outcome: `passed | failed | skipped | xfailed | xpassed | error`
|
|
72
|
+
- Duration (ms) and per-phase breakdown (setup/call/teardown)
|
|
73
|
+
- Markers (e.g. `@pytest.mark.smoke`), keywords, parametrize ids
|
|
74
|
+
- Skip reasons and pytest warnings
|
|
75
|
+
- Failure tracebacks and assertion repr
|
|
76
|
+
- Captured stdout / stderr / caplog (inline with redaction; truncated above
|
|
77
|
+
`--testrelic-artifact-threshold-kb`, hard cap 1 MB per stream — artifact
|
|
78
|
+
upload for large streams is planned for v0.2)
|
|
79
|
+
|
|
80
|
+
Test runs include CI metadata auto-detected from GitHub Actions, GitLab CI,
|
|
81
|
+
Jenkins, and CircleCI, plus git branch/commit/author from the working tree.
|
|
82
|
+
|
|
83
|
+
## pytest-xdist
|
|
84
|
+
|
|
85
|
+
Parallel runs via `pytest-xdist` (`pytest -n 4`) are supported out of the box.
|
|
86
|
+
Each worker forwards its `TestReport` objects through the xdist channel; only
|
|
87
|
+
the controller node uploads. The result is exactly one cloud run with all tests
|
|
88
|
+
aggregated.
|
|
89
|
+
|
|
90
|
+
## Offline mode
|
|
91
|
+
|
|
92
|
+
Any upload that fails after retries is written to `~/.testrelic/queue/` as a
|
|
93
|
+
JSON file. Re-run later:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
testrelic-pytest drain
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
The drainer re-authenticates with `TESTRELIC_API_KEY` and replays each entry to
|
|
100
|
+
its original endpoint.
|
|
101
|
+
|
|
102
|
+
## See also
|
|
103
|
+
|
|
104
|
+
- [`testrelic-deepeval`](https://pypi.org/project/testrelic-deepeval/) — DeepEval / LLM evaluation bridge
|
|
105
|
+
- [`testrelic-playwright`](https://pypi.org/project/testrelic-playwright/) — Playwright reporter for pytest
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.21"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "testrelic-pytest"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Generic pytest reporter that uploads test results to the TestRelic AI cloud platform."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.9"
|
|
13
|
+
authors = [{ name = "TestRelic AI", email = "hello@testrelic.ai" }]
|
|
14
|
+
keywords = [
|
|
15
|
+
"pytest",
|
|
16
|
+
"test",
|
|
17
|
+
"analytics",
|
|
18
|
+
"reporter",
|
|
19
|
+
"test-results",
|
|
20
|
+
"testrelic",
|
|
21
|
+
"ci",
|
|
22
|
+
"testing",
|
|
23
|
+
"test-automation",
|
|
24
|
+
"observability",
|
|
25
|
+
]
|
|
26
|
+
classifiers = [
|
|
27
|
+
"Development Status :: 4 - Beta",
|
|
28
|
+
"Framework :: Pytest",
|
|
29
|
+
"Intended Audience :: Developers",
|
|
30
|
+
"License :: OSI Approved :: MIT License",
|
|
31
|
+
"Operating System :: OS Independent",
|
|
32
|
+
"Programming Language :: Python :: 3",
|
|
33
|
+
"Programming Language :: Python :: 3.9",
|
|
34
|
+
"Programming Language :: Python :: 3.10",
|
|
35
|
+
"Programming Language :: Python :: 3.11",
|
|
36
|
+
"Programming Language :: Python :: 3.12",
|
|
37
|
+
"Topic :: Software Development :: Testing",
|
|
38
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
39
|
+
]
|
|
40
|
+
dependencies = [
|
|
41
|
+
"pytest>=7.0",
|
|
42
|
+
"httpx>=0.27,<1.0",
|
|
43
|
+
"pydantic>=2.6,<3.0",
|
|
44
|
+
"typer>=0.12,<1.0",
|
|
45
|
+
"platformdirs>=4.0",
|
|
46
|
+
"tomli>=2.0; python_version<'3.11'",
|
|
47
|
+
"tomli-w>=1.0",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[project.optional-dependencies]
|
|
51
|
+
dev = [
|
|
52
|
+
"pytest-cov>=5.0",
|
|
53
|
+
"pytest-xdist>=3.5",
|
|
54
|
+
"respx>=0.21",
|
|
55
|
+
"ruff>=0.6",
|
|
56
|
+
"mypy>=1.10",
|
|
57
|
+
"build>=1.2",
|
|
58
|
+
"twine>=5.1",
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
[project.urls]
|
|
62
|
+
Homepage = "https://testrelic.ai"
|
|
63
|
+
Documentation = "https://docs.testrelic.ai/pytest"
|
|
64
|
+
Repository = "https://github.com/testrelic-ai/testrelic-python-sdk"
|
|
65
|
+
"Bug Tracker" = "https://github.com/testrelic-ai/testrelic-python-sdk/issues"
|
|
66
|
+
|
|
67
|
+
[project.scripts]
|
|
68
|
+
testrelic-pytest = "testrelic_pytest.cli:app"
|
|
69
|
+
|
|
70
|
+
[project.entry-points.pytest11]
|
|
71
|
+
testrelic_pytest = "testrelic_pytest.plugin"
|
|
72
|
+
|
|
73
|
+
[tool.hatch.version]
|
|
74
|
+
path = "src/testrelic_pytest/_version.py"
|
|
75
|
+
|
|
76
|
+
[tool.hatch.build.targets.wheel]
|
|
77
|
+
packages = ["src/testrelic_pytest"]
|
|
78
|
+
|
|
79
|
+
[tool.hatch.build.targets.sdist]
|
|
80
|
+
include = [
|
|
81
|
+
"src/testrelic_pytest",
|
|
82
|
+
"README.md",
|
|
83
|
+
"LICENSE",
|
|
84
|
+
"CHANGELOG.md",
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
[tool.ruff]
|
|
88
|
+
line-length = 100
|
|
89
|
+
target-version = "py39"
|
|
90
|
+
|
|
91
|
+
[tool.ruff.lint]
|
|
92
|
+
select = ["E", "F", "I", "B", "UP", "RUF"]
|
|
93
|
+
ignore = [
|
|
94
|
+
"E501", # line length handled by formatter, not lint
|
|
95
|
+
"UP006", # keep `Dict[]`/`List[]` for parity with the other packages
|
|
96
|
+
"UP007", # keep `Optional[X]` / `Union[X, Y]` until we drop py39 runtime support
|
|
97
|
+
"UP045", # keep `Optional[X]` (PEP 604 ban for runtime hints on py39)
|
|
98
|
+
"B008", # typer Option/Argument call in default is idiomatic
|
|
99
|
+
]
|
|
100
|
+
|
|
101
|
+
[tool.mypy]
|
|
102
|
+
python_version = "3.10"
|
|
103
|
+
strict = true
|
|
104
|
+
ignore_missing_imports = true
|
|
105
|
+
|
|
106
|
+
[tool.pytest.ini_options]
|
|
107
|
+
testpaths = ["tests"]
|
|
108
|
+
addopts = "-v --tb=short -p no:testrelic_pytest"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""TestRelic generic pytest reporter — pytest plugin and helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from testrelic_pytest._version import __version__
|
|
6
|
+
from testrelic_pytest.config import (
|
|
7
|
+
CloudConfig,
|
|
8
|
+
PytestPluginConfig,
|
|
9
|
+
resolve_config,
|
|
10
|
+
)
|
|
11
|
+
from testrelic_pytest.schema import (
|
|
12
|
+
CapturedOutput,
|
|
13
|
+
CIMetadata,
|
|
14
|
+
FailureDiagnostic,
|
|
15
|
+
PhaseResult,
|
|
16
|
+
PytestResult,
|
|
17
|
+
PytestRunReport,
|
|
18
|
+
Status,
|
|
19
|
+
Summary,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"CIMetadata",
|
|
24
|
+
"CapturedOutput",
|
|
25
|
+
"CloudConfig",
|
|
26
|
+
"FailureDiagnostic",
|
|
27
|
+
"PhaseResult",
|
|
28
|
+
"PytestPluginConfig",
|
|
29
|
+
"PytestResult",
|
|
30
|
+
"PytestRunReport",
|
|
31
|
+
"Status",
|
|
32
|
+
"Summary",
|
|
33
|
+
"__version__",
|
|
34
|
+
"resolve_config",
|
|
35
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.1"
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Detect CI provider and pull build metadata from environment variables."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
from testrelic_pytest.schema import CIMetadata
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def detect_ci() -> Optional[CIMetadata]:
|
|
12
|
+
"""Return a populated `CIMetadata` if we recognise the CI provider."""
|
|
13
|
+
env = os.environ
|
|
14
|
+
|
|
15
|
+
if env.get("GITHUB_ACTIONS") == "true":
|
|
16
|
+
return CIMetadata(
|
|
17
|
+
provider="github-actions",
|
|
18
|
+
buildId=env.get("GITHUB_RUN_ID"),
|
|
19
|
+
commitSha=env.get("GITHUB_SHA"),
|
|
20
|
+
branch=env.get("GITHUB_REF_NAME"),
|
|
21
|
+
prNumber=_extract_pr_number(env.get("GITHUB_REF")),
|
|
22
|
+
workflow=env.get("GITHUB_WORKFLOW"),
|
|
23
|
+
runUrl=_github_run_url(dict(env)),
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
if env.get("GITLAB_CI") == "true":
|
|
27
|
+
return CIMetadata(
|
|
28
|
+
provider="gitlab-ci",
|
|
29
|
+
buildId=env.get("CI_PIPELINE_ID"),
|
|
30
|
+
commitSha=env.get("CI_COMMIT_SHA"),
|
|
31
|
+
branch=env.get("CI_COMMIT_REF_NAME"),
|
|
32
|
+
prNumber=env.get("CI_MERGE_REQUEST_IID"),
|
|
33
|
+
workflow=env.get("CI_JOB_NAME"),
|
|
34
|
+
runUrl=env.get("CI_PIPELINE_URL"),
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
if env.get("JENKINS_URL"):
|
|
38
|
+
return CIMetadata(
|
|
39
|
+
provider="jenkins",
|
|
40
|
+
buildId=env.get("BUILD_ID") or env.get("BUILD_NUMBER"),
|
|
41
|
+
commitSha=env.get("GIT_COMMIT"),
|
|
42
|
+
branch=env.get("GIT_BRANCH"),
|
|
43
|
+
workflow=env.get("JOB_NAME"),
|
|
44
|
+
runUrl=env.get("BUILD_URL"),
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
if env.get("CIRCLECI") == "true":
|
|
48
|
+
return CIMetadata(
|
|
49
|
+
provider="circleci",
|
|
50
|
+
buildId=env.get("CIRCLE_BUILD_NUM"),
|
|
51
|
+
commitSha=env.get("CIRCLE_SHA1"),
|
|
52
|
+
branch=env.get("CIRCLE_BRANCH"),
|
|
53
|
+
prNumber=_pr_from_url(env.get("CIRCLE_PULL_REQUEST")),
|
|
54
|
+
workflow=env.get("CIRCLE_JOB"),
|
|
55
|
+
runUrl=env.get("CIRCLE_BUILD_URL"),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
if env.get("BITBUCKET_PIPELINE_UUID"):
|
|
59
|
+
return CIMetadata(
|
|
60
|
+
provider="bitbucket-pipelines",
|
|
61
|
+
buildId=env.get("BITBUCKET_BUILD_NUMBER"),
|
|
62
|
+
commitSha=env.get("BITBUCKET_COMMIT"),
|
|
63
|
+
branch=env.get("BITBUCKET_BRANCH"),
|
|
64
|
+
prNumber=env.get("BITBUCKET_PR_ID"),
|
|
65
|
+
workflow=env.get("BITBUCKET_PIPELINE_UUID"),
|
|
66
|
+
runUrl=_bitbucket_run_url(dict(env)),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _extract_pr_number(ref: Optional[str]) -> Optional[str]:
|
|
73
|
+
if not ref:
|
|
74
|
+
return None
|
|
75
|
+
parts = ref.split("/")
|
|
76
|
+
if len(parts) >= 3 and parts[1] == "pull":
|
|
77
|
+
return parts[2]
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _github_run_url(env: dict[str, str]) -> Optional[str]:
|
|
82
|
+
server = env.get("GITHUB_SERVER_URL")
|
|
83
|
+
repo = env.get("GITHUB_REPOSITORY")
|
|
84
|
+
run = env.get("GITHUB_RUN_ID")
|
|
85
|
+
if server and repo and run:
|
|
86
|
+
return f"{server}/{repo}/actions/runs/{run}"
|
|
87
|
+
return None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _bitbucket_run_url(env: dict[str, str]) -> Optional[str]:
|
|
91
|
+
workspace = env.get("BITBUCKET_WORKSPACE")
|
|
92
|
+
repo = env.get("BITBUCKET_REPO_SLUG")
|
|
93
|
+
build = env.get("BITBUCKET_BUILD_NUMBER")
|
|
94
|
+
if workspace and repo and build:
|
|
95
|
+
return f"https://bitbucket.org/{workspace}/{repo}/pipelines/results/{build}"
|
|
96
|
+
return None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _pr_from_url(url: Optional[str]) -> Optional[str]:
|
|
100
|
+
if not url:
|
|
101
|
+
return None
|
|
102
|
+
return url.rstrip("/").split("/")[-1] or None
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""`testrelic-pytest` CLI."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
|
|
11
|
+
from testrelic_pytest._version import __version__
|
|
12
|
+
|
|
13
|
+
app = typer.Typer(
|
|
14
|
+
name="testrelic-pytest",
|
|
15
|
+
help="TestRelic generic pytest reporter CLI",
|
|
16
|
+
add_completion=False,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@app.command()
|
|
21
|
+
def version() -> None:
|
|
22
|
+
"""Print the package version."""
|
|
23
|
+
typer.echo(__version__)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@app.command(name="drain")
|
|
27
|
+
def drain(
|
|
28
|
+
queue_dir: Optional[Path] = typer.Option(
|
|
29
|
+
None,
|
|
30
|
+
"--queue-dir",
|
|
31
|
+
help="Override the queue directory. Defaults to .testrelic/queue/.",
|
|
32
|
+
),
|
|
33
|
+
) -> None:
|
|
34
|
+
"""Retry queued failed uploads."""
|
|
35
|
+
from testrelic_pytest.cloud.queue import drain_queue
|
|
36
|
+
|
|
37
|
+
counts = drain_queue(queue_dir)
|
|
38
|
+
typer.echo(json.dumps(counts, indent=2))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@app.command()
|
|
42
|
+
def status(
|
|
43
|
+
queue_dir: Optional[Path] = typer.Option(
|
|
44
|
+
None,
|
|
45
|
+
"--queue-dir",
|
|
46
|
+
help="Queue directory to inspect. Defaults to .testrelic/queue/.",
|
|
47
|
+
),
|
|
48
|
+
) -> None:
|
|
49
|
+
"""Show how many uploads are pending replay."""
|
|
50
|
+
target = Path(queue_dir) if queue_dir else Path(".testrelic") / "queue"
|
|
51
|
+
if not target.exists():
|
|
52
|
+
typer.echo(json.dumps({"queued": 0, "queue_dir": str(target)}, indent=2))
|
|
53
|
+
return
|
|
54
|
+
queued = len(list(target.glob("*.json")))
|
|
55
|
+
typer.echo(json.dumps({"queued": queued, "queue_dir": str(target)}, indent=2))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if __name__ == "__main__":
|
|
59
|
+
app()
|