spec-tracer 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spec_tracer-0.1.0/LICENSE +21 -0
- spec_tracer-0.1.0/PKG-INFO +10 -0
- spec_tracer-0.1.0/README.md +213 -0
- spec_tracer-0.1.0/pyproject.toml +32 -0
- spec_tracer-0.1.0/setup.cfg +4 -0
- spec_tracer-0.1.0/spec_tracer/__init__.py +3 -0
- spec_tracer-0.1.0/spec_tracer/aggregator.py +188 -0
- spec_tracer-0.1.0/spec_tracer/cli.py +153 -0
- spec_tracer-0.1.0/spec_tracer/collectors.py +44 -0
- spec_tracer-0.1.0/spec_tracer/linker.py +17 -0
- spec_tracer-0.1.0/spec_tracer/models.py +40 -0
- spec_tracer-0.1.0/spec_tracer/parsers.py +177 -0
- spec_tracer-0.1.0/spec_tracer/renderers.py +907 -0
- spec_tracer-0.1.0/spec_tracer/report_model.py +135 -0
- spec_tracer-0.1.0/spec_tracer.egg-info/PKG-INFO +10 -0
- spec_tracer-0.1.0/spec_tracer.egg-info/SOURCES.txt +30 -0
- spec_tracer-0.1.0/spec_tracer.egg-info/dependency_links.txt +1 -0
- spec_tracer-0.1.0/spec_tracer.egg-info/entry_points.txt +2 -0
- spec_tracer-0.1.0/spec_tracer.egg-info/requires.txt +3 -0
- spec_tracer-0.1.0/spec_tracer.egg-info/top_level.txt +4 -0
- spec_tracer-0.1.0/tests/integration/conftest.py +32 -0
- spec_tracer-0.1.0/tests/integration/test_e2e_coverage.py +24 -0
- spec_tracer-0.1.0/tests/integration/test_edge_cases.py +111 -0
- spec_tracer-0.1.0/tests/integration/test_integration_linking.py +58 -0
- spec_tracer-0.1.0/tests/integration/test_json_report.py +64 -0
- spec_tracer-0.1.0/tests/integration/test_missing_required_layer.py +30 -0
- spec_tracer-0.1.0/tests/integration/test_status_badges.py +29 -0
- spec_tracer-0.1.0/tests/integration/test_unit_linking.py +26 -0
- spec_tracer-0.1.0/tests/unit/test_aggregator.py +232 -0
- spec_tracer-0.1.0/tests/unit/test_build_pyramid.py +190 -0
- spec_tracer-0.1.0/tests/unit/test_renderers.py +180 -0
- spec_tracer-0.1.0/tests/unit/test_report_model.py +250 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ampyard
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: spec-tracer
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CLI tool that collates test results from **Unit, Integration, and E2E** test suites into a single HTML report.
|
|
5
|
+
Requires-Python: >=3.12
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Dist: behave-modern-html-report>=2.2.2
|
|
8
|
+
Requires-Dist: Jinja2>=3.1
|
|
9
|
+
Requires-Dist: pytest-html>=4.2.0
|
|
10
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
# SpecTracer
|
|
2
|
+
|
|
3
|
+
<div align="center">
|
|
4
|
+
<img src="docs/hero-image.png" style="align: center; width:50%; height: auto"></img>
|
|
5
|
+
|
|
6
|
+
<p>
|
|
7
|
+
|
|
8
|
+
[](https://lbesson.mit-license.org/) [](https://github.com/ampyard/spec-tracer/actions/workflows/ci.yml)
|
|
9
|
+
|
|
10
|
+
</p>
|
|
11
|
+
</div>
|
|
12
|
+
|
|
13
|
+
A CLI tool that takes your Gherkin `.feature` files as the source of truth for what needs testing, then collates test results from your **Unit**, **Integration**, and **E2E** suites into a single, self-contained HTML report — plus an optional machine-readable JSON twin for CI automation.
|
|
14
|
+
|
|
15
|
+
Feature files define the scope. Tags on scenarios link them to test results across layers. The report shows:
|
|
16
|
+
|
|
17
|
+
- What percentage of scenarios actually have test coverage (the headline metric).
|
|
18
|
+
- Where that coverage exists across layers (per-scenario pass/fail/skip breakdown).
|
|
19
|
+
- The overall test pyramid — test count, duration, and pass rate per layer.
|
|
20
|
+
- Every failure's stack trace, in one place.
|
|
21
|
+
|
|
22
|
+
An optional JSON report (conforming to [`spectracer-report.schema.json`](spectracer-report.schema.json)) mirrors the same data for scripting — PR bots, custom gating, dashboards — without scraping HTML.
|
|
23
|
+
|
|
24
|
+
The tool is tech-stack agnostic: it only needs Gherkin `.feature` files, JUnit XML, and Cucumber JSON, so it works regardless of what languages or frameworks produced them.
|
|
25
|
+
|
|
26
|
+
## Why
|
|
27
|
+
|
|
28
|
+
- **Fragmented visibility** — unit, integration, and E2E tests usually live in different directories or repos, with no single view of overall coverage.
|
|
29
|
+
- **Inverted pyramids** — teams unknowingly accumulate slow E2E tests instead of fast unit tests, and don't notice until CI is painfully slow.
|
|
30
|
+
- **No traceability** — it's hard to know if a specific business scenario is actually tested across all the layers it should be.
|
|
31
|
+
- **Tooling lock-in** — most reporting tools are tied to one framework (Allure for Java, Cypress Dashboard for Cypress). This one isn't.
|
|
32
|
+
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
Requires Python 3.12+.
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
uv sync
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
This installs the tool along with its only runtime dependency, Jinja2.
|
|
42
|
+
|
|
43
|
+
To build a wheel for local testing:
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
uv build
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
The wheel is written to `dist/unified_test_tracer-*.whl`. Install it with `uv pip install dist/*.whl`.
|
|
50
|
+
|
|
51
|
+
## Quick Start
|
|
52
|
+
|
|
53
|
+
1. Write `.feature` files describing your scenarios, tagged so test results can link back to them (see [Tagging Convention](#tagging-convention) below).
|
|
54
|
+
2. Run your test suites and produce JUnit XML (unit/integration) and/or Cucumber JSON (E2E) output.
|
|
55
|
+
3. Create a `specspectracer.config.json` in your project root:
|
|
56
|
+
|
|
57
|
+
```json
|
|
58
|
+
{
|
|
59
|
+
"features": ["./features"],
|
|
60
|
+
"unit": { "": ["./reports/unit.xml"] },
|
|
61
|
+
"integration": { "": ["./reports/integration.xml"] },
|
|
62
|
+
"e2e": ["./reports/e2e.json"],
|
|
63
|
+
"output": "./report.html"
|
|
64
|
+
}
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
4. Run the tool:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
uv run python build_pyramid.py
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
It auto-discovers `spectracer.config.json` in the current directory — no flags needed. Open the generated `report.html` in a browser.
|
|
74
|
+
|
|
75
|
+
To point at a config file with a different name or location, pass it as the only argument:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
uv run python build_pyramid.py path/to/other-config.json
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
> **Note on shells:** use forward slashes (`./spectracer.config.json`) or a bare filename. A leading `.\` (PowerShell-style) can be mangled by POSIX-style shells (Git Bash, WSL), since backslash is their escape character there.
|
|
82
|
+
|
|
83
|
+
## Tagging Convention
|
|
84
|
+
|
|
85
|
+
Feature files and test results connect via **shared tags**. There are two kinds of tags a scenario can carry:
|
|
86
|
+
|
|
87
|
+
```gherkin
|
|
88
|
+
Feature: User Login
|
|
89
|
+
|
|
90
|
+
@FC-42 @regression @require-unit:auth @require-integration:auth @require-e2e
|
|
91
|
+
Scenario: Successful login with valid credentials
|
|
92
|
+
Given the user is on the login page
|
|
93
|
+
When they enter valid credentials
|
|
94
|
+
Then they should be redirected to the dashboard
|
|
95
|
+
|
|
96
|
+
@FC-43
|
|
97
|
+
Scenario: Login with invalid password shows error
|
|
98
|
+
Given the user is on the login page
|
|
99
|
+
When they enter an invalid password
|
|
100
|
+
Then an error message should be displayed
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
- **Linking tags** (`@FC-42`, `@regression`) — shared with test results. Any test result carrying a matching tag links to that scenario.
|
|
104
|
+
- **Layer requirement tags** (`@require-unit`, `@require-integration`, `@require-e2e`) — declare which layers *must* have coverage for this scenario. These are never used for linking, and the tool flags any declared layer that ends up with zero linked results.
|
|
105
|
+
|
|
106
|
+
### Module-scoped requirements
|
|
107
|
+
|
|
108
|
+
`@require-unit` and `@require-integration` accept an optional `:modulename` suffix, e.g. `@require-unit:auth`. This pairs with module-keyed entries in the config file's `unit`/`integration` objects (see below) — a module-scoped requirement is only satisfied by a result registered under that exact module. An unscoped result (config key `""`) never satisfies it, and a bare `@require-unit` (no module) is satisfied by any linked unit result regardless of module.
|
|
109
|
+
|
|
110
|
+
`@require-e2e` does not accept a module suffix, since E2E scenarios typically span multiple modules by nature.
|
|
111
|
+
|
|
112
|
+
### Matching rules
|
|
113
|
+
|
|
114
|
+
- **Exact string match** — `@FC-42` matches `@FC-42` only, not `@FC-4` or `@FC-42-smoke`.
|
|
115
|
+
- **OR logic** — a scenario tagged `[@FC-42, @regression]` links to a test carrying just `@regression`.
|
|
116
|
+
- **Scenario tags only** — tags on the `Feature:` line are **not** inherited by scenarios.
|
|
117
|
+
- **`@require-*` tags** are excluded from linking — no collision with linking tags is possible.
|
|
118
|
+
- **Tag collisions link everywhere** — if two scenarios (in the same or different feature files) share a tag, one matching test result links to both.
|
|
119
|
+
|
|
120
|
+
### Where the tool looks for tags in test results
|
|
121
|
+
|
|
122
|
+
- **JUnit XML (unit/integration):** the `name` attribute, `classname` attribute, or `<properties><property>` elements — whichever your framework populates.
|
|
123
|
+
- **Cucumber JSON (E2E):** the native scenario-level `tags` array.
|
|
124
|
+
|
|
125
|
+
## Configuration File
|
|
126
|
+
|
|
127
|
+
The tool is configured entirely through a JSON file — there are no CLI flags. Default filename is `spectracer.config.json` at the project root; pass an explicit path as the sole CLI argument to use a different one.
|
|
128
|
+
|
|
129
|
+
```json
|
|
130
|
+
{
|
|
131
|
+
"features": ["./features"],
|
|
132
|
+
"unit": {
|
|
133
|
+
"": ["./reports/unit.xml"],
|
|
134
|
+
"billing": ["./reports/billing-unit.xml"]
|
|
135
|
+
},
|
|
136
|
+
"integration": {
|
|
137
|
+
"": ["./reports/integration.xml"]
|
|
138
|
+
},
|
|
139
|
+
"e2e": ["./reports/e2e.json"],
|
|
140
|
+
"output": "./report.html",
|
|
141
|
+
"output_json": "./report.json",
|
|
142
|
+
"error_on_failure": false,
|
|
143
|
+
"health_checks": {
|
|
144
|
+
"progress_threshold_green": 80,
|
|
145
|
+
"progress_threshold_amber": 50,
|
|
146
|
+
"e2e_duration_amber_seconds": 600,
|
|
147
|
+
"e2e_duration_red_seconds": 1800
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
| Key | Required | Description |
|
|
153
|
+
|---|---|---|
|
|
154
|
+
| `features` | Yes | Array of Gherkin `.feature` file or directory paths (directories are searched recursively). |
|
|
155
|
+
| `unit` | No | Object keyed by module name. Each value is an array of JUnit XML file/directory paths. Use `""` as the key for results not tied to any module. Matched against `@require-unit` / `@require-unit:<module>` tags. |
|
|
156
|
+
| `integration` | No | Same shape as `unit`, matched against `@require-integration` / `@require-integration:<module>` tags. |
|
|
157
|
+
| `e2e` | No | Array of Cucumber JSON file/directory paths. E2E results are never module-scoped. |
|
|
158
|
+
| `output` | Yes | Path for the generated HTML report. Created if the parent directory doesn't exist; overwritten if it already exists. |
|
|
159
|
+
| `output_json` | No | Path for a machine-readable JSON report, conforming to [`spectracer-report.schema.json`](spectracer-report.schema.json). Omit to skip JSON output entirely (default). Same directory-creation/overwrite semantics as `output`. |
|
|
160
|
+
| `error_on_failure` | No | If `true`, exit non-zero when any test result is a failure. Default: `false`. Health checks never affect the exit code — this is the only thing that does. |
|
|
161
|
+
| `health_checks` | No | Overrides for the default thresholds shown above. |
|
|
162
|
+
|
|
163
|
+
## The Report
|
|
164
|
+
|
|
165
|
+
The generated HTML is a single self-contained file (all CSS/JS inlined — no external assets, safe to email or archive) with five sections:
|
|
166
|
+
|
|
167
|
+
1. **Coverage Progress Summary** — the headline `Tested: X / Y scenarios (Z%)` metric, plus a per-feature breakdown. Color-coded green/amber/red using the configurable thresholds.
|
|
168
|
+
2. **Global Pyramid Dashboard** — a 3-tier visualization (E2E / Integration / Unit) with test counts, duration, and pass rate per layer, plus health indicators for an inverted pyramid or an E2E layer with excessive runtime.
|
|
169
|
+
3. **Feature Traceability & Scenario Matrix** — a searchable, expandable tree: Feature → Scenario → Layer results, with full Gherkin text, declared layer requirements (✓/✗), and per-test pass/fail/skip status with failure stack traces.
|
|
170
|
+
4. **Detailed Failure Breakdown** — every failed test across all layers, with feature/scenario context and full stack trace on expand.
|
|
171
|
+
5. **Unlinked Tests** — test results whose tags didn't match any scenario, to help catch orphaned or mis-tagged tests.
|
|
172
|
+
|
|
173
|
+
## Machine-Readable JSON Report
|
|
174
|
+
|
|
175
|
+
Setting `output_json` in the config produces a JSON file alongside the HTML report, built from the exact same internal data — the two outputs can never drift apart. It conforms to [`spectracer-report.schema.json`](spectracer-report.schema.json) (Draft 7), which is the authoritative contract; the highlights:
|
|
176
|
+
|
|
177
|
+
- `summary.coverage` / `summary.pyramid` / `summary.health` — the same headline metric, per-layer stats, and health status (`green`/`amber`/`red` with `reasons[]`) shown on the HTML dashboard.
|
|
178
|
+
- `features[].scenarios[].results[]` — every linked test result per scenario, with `duration` (milliseconds) and `failureMessage` **omitted** rather than `null` when not available, and layer requirement satisfaction under `requirements[]`.
|
|
179
|
+
- `unlinkedTests[]` — the same orphaned results shown in the HTML report's "Unlinked Tests" page.
|
|
180
|
+
- `config` — a verbatim echo of the resolved config used to produce the report, for provenance if the JSON is archived independently of the repo.
|
|
181
|
+
|
|
182
|
+
```json
|
|
183
|
+
{
|
|
184
|
+
"features": ["./features"],
|
|
185
|
+
"unit": { "": ["./reports/unit.xml"] },
|
|
186
|
+
"e2e": ["./reports/e2e.json"],
|
|
187
|
+
"output": "./report.html",
|
|
188
|
+
"output_json": "./report.json"
|
|
189
|
+
}
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Useful for PR bots, custom CI gating beyond `error_on_failure`, or feeding coverage numbers into a dashboard — without scraping the HTML.
|
|
193
|
+
|
|
194
|
+
## Behavior Reference
|
|
195
|
+
|
|
196
|
+
| Scenario | Behavior |
|
|
197
|
+
|---|---|
|
|
198
|
+
| No config file found and none specified | Errors out — a config file is mandatory. |
|
|
199
|
+
| Config missing `features` or `output` | Errors out — both are mandatory keys. |
|
|
200
|
+
| Empty or missing test result path | Silently ignored (zero tests for that layer). |
|
|
201
|
+
| Malformed JUnit XML or Cucumber JSON | Aborts with a clear error message. |
|
|
202
|
+
| Test matches no scenario | Listed in "Unlinked Tests". |
|
|
203
|
+
| Scenario matches no test | Shown as "untested". |
|
|
204
|
+
| Scenario has `@require-*` but no matching test | That layer is flagged as missing. |
|
|
205
|
+
| Feature-level tags | Not inherited by scenarios — only scenario-level tags are used for matching. |
|
|
206
|
+
| Scenario Outline / Examples, `Rule:`, `Background:`, non-English dialects | Deferred to whatever your Gherkin/E2E framework does with them — the tool doesn't parse Gherkin syntax beyond `Feature:`, tags, and `Scenario:` lines. |
|
|
207
|
+
| Unicode / special characters | Preserved, HTML-escaped in the report. |
|
|
208
|
+
|
|
209
|
+
## What This Tool Doesn't Do
|
|
210
|
+
|
|
211
|
+
- **Not a test runner** — it only parses results after your tests have already run.
|
|
212
|
+
- **Not a source-code parser** — it never reads your `.java`/`.py`/`.js` files, only `.feature` files and test-result output.
|
|
213
|
+
- **No tag expressions** — matching is exact string equality only; no `not`/`and`/`or` boolean tag logic.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "spec-tracer"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "CLI tool that collates test results from **Unit, Integration, and E2E** test suites into a single HTML report."
|
|
5
|
+
requires-python = ">=3.12"
|
|
6
|
+
dependencies = [
|
|
7
|
+
"behave-modern-html-report>=2.2.2",
|
|
8
|
+
"Jinja2>=3.1",
|
|
9
|
+
"pytest-html>=4.2.0",
|
|
10
|
+
]
|
|
11
|
+
[project.scripts]
|
|
12
|
+
spec-tracer = "spec_tracer.cli:main"
|
|
13
|
+
|
|
14
|
+
[dependency-groups]
|
|
15
|
+
dev = [
|
|
16
|
+
"pytest>=8.3",
|
|
17
|
+
"behave>=1.2",
|
|
18
|
+
"jsonschema>=4.23",
|
|
19
|
+
]
|
|
20
|
+
[tool.setuptools.packages.find]
|
|
21
|
+
exclude = ["features*", "reports*"]
|
|
22
|
+
|
|
23
|
+
[build-system]
|
|
24
|
+
requires = ["setuptools>=68"]
|
|
25
|
+
build-backend = "setuptools.build_meta"
|
|
26
|
+
|
|
27
|
+
[tool.pytest.ini_options]
|
|
28
|
+
testpaths = ["tests"]
|
|
29
|
+
pythonpath = ["."]
|
|
30
|
+
|
|
31
|
+
[tool.behave]
|
|
32
|
+
formatters.modern = "behave_modern_html_report.formatter:ModernHTMLFormatter"
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
from typing import Dict, List
|
|
3
|
+
|
|
4
|
+
from spec_tracer.models import Scenario, ScenarioView, TestResult
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ReportAggregator:
|
|
8
|
+
|
|
9
|
+
LAYER_ORDER = ["e2e", "integration", "unit"]
|
|
10
|
+
|
|
11
|
+
@staticmethod
|
|
12
|
+
def build_views(
|
|
13
|
+
scenarios: List[Scenario],
|
|
14
|
+
links: Dict[int, List[TestResult]],
|
|
15
|
+
) -> List[ScenarioView]:
|
|
16
|
+
layer_rank = {layer: i for i, layer in enumerate(ReportAggregator.LAYER_ORDER)}
|
|
17
|
+
views: List[ScenarioView] = []
|
|
18
|
+
for scenario in scenarios:
|
|
19
|
+
linked = sorted(links.get(id(scenario), []), key=lambda r: layer_rank.get(r.layer, len(layer_rank)))
|
|
20
|
+
by_layer: Dict[str, List[TestResult]] = {}
|
|
21
|
+
for result in linked:
|
|
22
|
+
by_layer.setdefault(result.layer, []).append(result)
|
|
23
|
+
layers = [by_layer.get(layer, []) for layer in ReportAggregator.LAYER_ORDER if by_layer.get(layer)]
|
|
24
|
+
views.append(ScenarioView(scenario=scenario, linked_results=linked, layers=layers))
|
|
25
|
+
return views
|
|
26
|
+
|
|
27
|
+
@staticmethod
|
|
28
|
+
def coverage_stats(views: List[ScenarioView]) -> dict:
|
|
29
|
+
total = len(views)
|
|
30
|
+
tested = sum(1 for v in views if v.is_tested)
|
|
31
|
+
percentage = int(round((tested / total * 100) if total else 0))
|
|
32
|
+
return {"total": total, "tested": tested, "percentage": percentage}
|
|
33
|
+
|
|
34
|
+
@staticmethod
|
|
35
|
+
def feature_breakdown(views: List[ScenarioView]) -> List[dict]:
|
|
36
|
+
features: Dict[str, List[ScenarioView]] = defaultdict(list)
|
|
37
|
+
for view in views:
|
|
38
|
+
features[view.scenario.feature].append(view)
|
|
39
|
+
|
|
40
|
+
breakdown: List[dict] = []
|
|
41
|
+
for feature_name, feature_views in sorted(features.items()):
|
|
42
|
+
feature_tested = sum(1 for v in feature_views if v.is_tested)
|
|
43
|
+
feature_total = len(feature_views)
|
|
44
|
+
feature_pct = int(round((feature_tested / feature_total * 100) if feature_total else 0))
|
|
45
|
+
breakdown.append({
|
|
46
|
+
"name": feature_name,
|
|
47
|
+
"tested": feature_tested,
|
|
48
|
+
"total": feature_total,
|
|
49
|
+
"percentage": feature_pct,
|
|
50
|
+
})
|
|
51
|
+
return breakdown
|
|
52
|
+
|
|
53
|
+
MIN_TIER_WIDTH_PCT = 28
|
|
54
|
+
|
|
55
|
+
@staticmethod
|
|
56
|
+
def layer_stats(views: List[ScenarioView]) -> List[dict]:
|
|
57
|
+
linked_results = [result for view in views for result in view.linked_results]
|
|
58
|
+
metrics: List[dict] = []
|
|
59
|
+
for layer in ReportAggregator.LAYER_ORDER:
|
|
60
|
+
layer_results = [result for result in linked_results if result.layer == layer]
|
|
61
|
+
if not layer_results:
|
|
62
|
+
continue
|
|
63
|
+
passed = sum(1 for result in layer_results if result.status == "passed")
|
|
64
|
+
failed = sum(1 for result in layer_results if result.status == "failed")
|
|
65
|
+
skipped = sum(1 for result in layer_results if result.status == "skipped")
|
|
66
|
+
duration = sum(result.duration for result in layer_results)
|
|
67
|
+
total = len(layer_results)
|
|
68
|
+
metrics.append(
|
|
69
|
+
{
|
|
70
|
+
"name": layer,
|
|
71
|
+
"label": layer.upper(),
|
|
72
|
+
"count": total,
|
|
73
|
+
"passed": passed,
|
|
74
|
+
"failed": failed,
|
|
75
|
+
"skipped": skipped,
|
|
76
|
+
"duration": duration,
|
|
77
|
+
"pass_pct": int(round((passed / total * 100) if total else 0)),
|
|
78
|
+
"fail_pct": int(round((failed / total * 100) if total else 0)),
|
|
79
|
+
"skip_pct": int(round((skipped / total * 100) if total else 0)),
|
|
80
|
+
}
|
|
81
|
+
)
|
|
82
|
+
max_count = max((metric["count"] for metric in metrics), default=0)
|
|
83
|
+
for metric in metrics:
|
|
84
|
+
share = (metric["count"] / max_count * 100) if max_count else 0
|
|
85
|
+
metric["width_pct"] = round(max(share, ReportAggregator.MIN_TIER_WIDTH_PCT if metric["count"] else 0), 1)
|
|
86
|
+
return metrics
|
|
87
|
+
|
|
88
|
+
@staticmethod
|
|
89
|
+
def failure_breakdown(views: List[ScenarioView]) -> List[dict]:
|
|
90
|
+
features: Dict[str, List[ScenarioView]] = defaultdict(list)
|
|
91
|
+
for view in views:
|
|
92
|
+
if any(result.status == "failed" for result in view.linked_results):
|
|
93
|
+
features[view.scenario.feature].append(view)
|
|
94
|
+
|
|
95
|
+
breakdown: List[dict] = []
|
|
96
|
+
for feature_name, feature_views in sorted(features.items()):
|
|
97
|
+
scenarios = []
|
|
98
|
+
for view in feature_views:
|
|
99
|
+
failed_results = [r for r in view.linked_results if r.status == "failed"]
|
|
100
|
+
scenarios.append({"view": view, "failed_results": failed_results})
|
|
101
|
+
breakdown.append({
|
|
102
|
+
"name": feature_name,
|
|
103
|
+
"scenarios": scenarios,
|
|
104
|
+
"failed_count": sum(len(s["failed_results"]) for s in scenarios),
|
|
105
|
+
})
|
|
106
|
+
return breakdown
|
|
107
|
+
|
|
108
|
+
@staticmethod
|
|
109
|
+
def health_checks(
|
|
110
|
+
views: List[ScenarioView],
|
|
111
|
+
layer_stats: List[dict],
|
|
112
|
+
progress_stats: dict,
|
|
113
|
+
unlinked_count: int = 0,
|
|
114
|
+
progress_threshold_green: float = 80,
|
|
115
|
+
progress_threshold_amber: float = 50,
|
|
116
|
+
e2e_duration_amber_seconds: float = 600,
|
|
117
|
+
e2e_duration_red_seconds: float = 1800,
|
|
118
|
+
) -> dict:
|
|
119
|
+
progress_pct = progress_stats["percentage"]
|
|
120
|
+
if progress_pct >= progress_threshold_green:
|
|
121
|
+
progress_status = "pass"
|
|
122
|
+
progress_message = "Progress is healthy."
|
|
123
|
+
elif progress_pct >= progress_threshold_amber:
|
|
124
|
+
progress_status = "warn"
|
|
125
|
+
progress_message = "Progress still needs attention."
|
|
126
|
+
else:
|
|
127
|
+
progress_status = "fail"
|
|
128
|
+
progress_message = "Progress is below the comfort threshold."
|
|
129
|
+
|
|
130
|
+
unit_count = next((metric["count"] for metric in layer_stats if metric["name"] == "unit"), 0)
|
|
131
|
+
integration_count = next((metric["count"] for metric in layer_stats if metric["name"] == "integration"), 0)
|
|
132
|
+
e2e_count = next((metric["count"] for metric in layer_stats if metric["name"] == "e2e"), 0)
|
|
133
|
+
if unit_count > integration_count + e2e_count:
|
|
134
|
+
pyramid_status = "pass"
|
|
135
|
+
pyramid_message = "Unit coverage is strong enough for the pyramid."
|
|
136
|
+
elif unit_count == integration_count + e2e_count:
|
|
137
|
+
pyramid_status = "warn"
|
|
138
|
+
pyramid_message = "Unit coverage is exactly at parity — add more unit tests."
|
|
139
|
+
else:
|
|
140
|
+
pyramid_status = "fail"
|
|
141
|
+
pyramid_message = "The pyramid is inverted and needs more unit coverage."
|
|
142
|
+
|
|
143
|
+
e2e_duration = next((metric["duration"] for metric in layer_stats if metric["name"] == "e2e"), 0.0)
|
|
144
|
+
if e2e_duration <= e2e_duration_amber_seconds:
|
|
145
|
+
e2e_status = "pass"
|
|
146
|
+
e2e_message = "End to end Runtime is within the healthy envelope."
|
|
147
|
+
elif e2e_duration <= e2e_duration_red_seconds:
|
|
148
|
+
e2e_status = "warn"
|
|
149
|
+
e2e_message = "End to end Runtime is getting slow."
|
|
150
|
+
else:
|
|
151
|
+
e2e_status = "fail"
|
|
152
|
+
e2e_message = "End to end Runtime exceeds the configured threshold."
|
|
153
|
+
|
|
154
|
+
if unlinked_count == 0:
|
|
155
|
+
unlinked_status = "pass"
|
|
156
|
+
unlinked_message = "Every parsed result was linked to a scenario."
|
|
157
|
+
elif unlinked_count <= 3:
|
|
158
|
+
unlinked_status = "warn"
|
|
159
|
+
unlinked_message = "A few results didn't link to any scenario."
|
|
160
|
+
else:
|
|
161
|
+
unlinked_status = "fail"
|
|
162
|
+
unlinked_message = "Several results didn't link to any scenario."
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
"Progress": {"status": progress_status, "message": progress_message, "value": f"{progress_stats['tested']}/{progress_stats['total']}"},
|
|
166
|
+
"pyramid": {
|
|
167
|
+
"status": pyramid_status,
|
|
168
|
+
"message": pyramid_message,
|
|
169
|
+
"value": f"e2e {e2e_count} · integration {integration_count} · unit {unit_count}",
|
|
170
|
+
"layers": [
|
|
171
|
+
{"name": "e2e", "count": e2e_count},
|
|
172
|
+
{"name": "integration", "count": integration_count},
|
|
173
|
+
{"name": "unit", "count": unit_count},
|
|
174
|
+
],
|
|
175
|
+
},
|
|
176
|
+
"end_to_end_runtime": {"status": e2e_status, "message": e2e_message, "value": f"{e2e_duration:.1f}s"},
|
|
177
|
+
"unlinked": {"status": unlinked_status, "message": unlinked_message, "value": str(unlinked_count)},
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
@staticmethod
|
|
181
|
+
def unlinked_results(scenarios: List[Scenario], results: List[TestResult]) -> List[TestResult]:
|
|
182
|
+
scenario_tags = {tag for scenario in scenarios for tag in scenario.tags}
|
|
183
|
+
return [
|
|
184
|
+
result
|
|
185
|
+
for result in results
|
|
186
|
+
if not any(tag in scenario_tags for tag in result.tags)
|
|
187
|
+
and any(tag.startswith("@FC-") for tag in result.tags)
|
|
188
|
+
]
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import sys
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Dict, List
|
|
7
|
+
|
|
8
|
+
from spec_tracer.aggregator import ReportAggregator
|
|
9
|
+
from spec_tracer.collectors import FileCollector
|
|
10
|
+
from spec_tracer.linker import ResultLinker
|
|
11
|
+
from spec_tracer.parsers import CucumberParser, FeatureParser, JunitParser
|
|
12
|
+
from spec_tracer.renderers import HtmlRenderer
|
|
13
|
+
from spec_tracer.report_model import build_report
|
|
14
|
+
|
|
15
|
+
DEFAULT_CONFIG_NAME = "spectracer.config.json"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _find_config_path(argv: List[str] | None = None) -> Path:
|
|
19
|
+
args = sys.argv[1:] if argv is None else argv
|
|
20
|
+
if args:
|
|
21
|
+
path = Path(args[0])
|
|
22
|
+
else:
|
|
23
|
+
path = Path(DEFAULT_CONFIG_NAME)
|
|
24
|
+
if not path.exists():
|
|
25
|
+
raise FileNotFoundError(f"Config file not found: {path}")
|
|
26
|
+
return path
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _load_config(path: Path) -> dict:
|
|
30
|
+
config = json.loads(path.read_text(encoding="utf-8"))
|
|
31
|
+
if "features" not in config:
|
|
32
|
+
raise ValueError("Config is missing required key: 'features'")
|
|
33
|
+
if "output" not in config:
|
|
34
|
+
raise ValueError("Config is missing required key: 'output'")
|
|
35
|
+
return config
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _load_logo(config_dir: Path) -> str:
|
|
39
|
+
logo_path = config_dir / "docs" / "logo.png"
|
|
40
|
+
if logo_path.exists():
|
|
41
|
+
data = logo_path.read_bytes()
|
|
42
|
+
encoded = base64.b64encode(data).decode("ascii")
|
|
43
|
+
return f"data:image/png;base64,{encoded}"
|
|
44
|
+
return ""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _collect_and_parse_features(paths: List[str], base_dir: Path) -> tuple:
|
|
48
|
+
files = FileCollector.feature_files(paths)
|
|
49
|
+
if not files:
|
|
50
|
+
raise FileNotFoundError("No feature files were found")
|
|
51
|
+
scenarios = []
|
|
52
|
+
feature_files: Dict[str, str] = {}
|
|
53
|
+
parser = FeatureParser()
|
|
54
|
+
resolved_base = base_dir.resolve()
|
|
55
|
+
for f in files:
|
|
56
|
+
parsed = parser.parse(f)
|
|
57
|
+
relative = os.path.relpath(f, resolved_base)
|
|
58
|
+
for scenario in parsed:
|
|
59
|
+
feature_files.setdefault(scenario.feature, relative.replace("\\", "/"))
|
|
60
|
+
scenarios.extend(parsed)
|
|
61
|
+
return scenarios, feature_files
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _collect_and_parse_junit_results(entries: Dict[str, List[str]], parser: JunitParser, layer: str) -> List:
|
|
65
|
+
results = []
|
|
66
|
+
for module, paths in entries.items():
|
|
67
|
+
files = FileCollector.xml_files(paths)
|
|
68
|
+
results.extend(parser.parse(files, layer=layer, module=module))
|
|
69
|
+
return results
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _collect_and_parse_e2e_results(paths: List[str], parser: CucumberParser) -> List:
|
|
73
|
+
files = FileCollector.json_files(paths)
|
|
74
|
+
return parser.parse(files, layer="e2e")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def main(argv: List[str] | None = None) -> int:
|
|
78
|
+
config_path = _find_config_path(argv)
|
|
79
|
+
config = _load_config(config_path)
|
|
80
|
+
|
|
81
|
+
scenarios, feature_files = _collect_and_parse_features(config["features"], config_path.parent)
|
|
82
|
+
|
|
83
|
+
junit_parser = JunitParser()
|
|
84
|
+
unit_results = _collect_and_parse_junit_results(config.get("unit", {}), junit_parser, "unit")
|
|
85
|
+
integration_results = _collect_and_parse_junit_results(config.get("integration", {}), junit_parser, "integration")
|
|
86
|
+
|
|
87
|
+
cucumber_parser = CucumberParser()
|
|
88
|
+
e2e_results = _collect_and_parse_e2e_results(config.get("e2e", []), cucumber_parser)
|
|
89
|
+
|
|
90
|
+
results = e2e_results + unit_results + integration_results
|
|
91
|
+
|
|
92
|
+
links = ResultLinker.link(scenarios, results)
|
|
93
|
+
views = ReportAggregator.build_views(scenarios, links)
|
|
94
|
+
stats = ReportAggregator.coverage_stats(views)
|
|
95
|
+
breakdown = ReportAggregator.feature_breakdown(views)
|
|
96
|
+
layer_stats = ReportAggregator.layer_stats(views)
|
|
97
|
+
failed_results = [result for result in results if result.status == "failed"]
|
|
98
|
+
unlinked_results = ReportAggregator.unlinked_results(scenarios, results)
|
|
99
|
+
health_check_config = config.get("health_checks", {})
|
|
100
|
+
health_checks = ReportAggregator.health_checks(
|
|
101
|
+
views,
|
|
102
|
+
layer_stats,
|
|
103
|
+
stats,
|
|
104
|
+
unlinked_count=len(unlinked_results),
|
|
105
|
+
progress_threshold_green=health_check_config.get("progress_threshold_green", 80),
|
|
106
|
+
progress_threshold_amber=health_check_config.get("progress_threshold_amber", 50),
|
|
107
|
+
e2e_duration_amber_seconds=health_check_config.get("e2e_duration_amber_seconds", 600),
|
|
108
|
+
e2e_duration_red_seconds=health_check_config.get("e2e_duration_red_seconds", 1800),
|
|
109
|
+
)
|
|
110
|
+
failure_breakdown = ReportAggregator.failure_breakdown(views)
|
|
111
|
+
|
|
112
|
+
renderer = HtmlRenderer()
|
|
113
|
+
html = renderer.render(
|
|
114
|
+
views,
|
|
115
|
+
stats,
|
|
116
|
+
breakdown,
|
|
117
|
+
layer_stats=layer_stats,
|
|
118
|
+
health_checks=health_checks,
|
|
119
|
+
failed_results=failed_results,
|
|
120
|
+
unlinked_results=unlinked_results,
|
|
121
|
+
failure_breakdown=failure_breakdown,
|
|
122
|
+
logo_data_uri=_load_logo(config_path.parent),
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
output_path = Path(config["output"])
|
|
126
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
127
|
+
output_path.write_text(html, encoding="utf-8")
|
|
128
|
+
|
|
129
|
+
if config.get("output_json"):
|
|
130
|
+
report = build_report(
|
|
131
|
+
config,
|
|
132
|
+
views,
|
|
133
|
+
stats,
|
|
134
|
+
layer_stats,
|
|
135
|
+
health_checks,
|
|
136
|
+
unlinked_results,
|
|
137
|
+
feature_files=feature_files,
|
|
138
|
+
)
|
|
139
|
+
output_json_path = Path(config["output_json"])
|
|
140
|
+
output_json_path.parent.mkdir(parents=True, exist_ok=True)
|
|
141
|
+
output_json_path.write_text(json.dumps(report, indent=2), encoding="utf-8")
|
|
142
|
+
|
|
143
|
+
if config.get("error_on_failure", False) and any(result.status == "failed" for result in results):
|
|
144
|
+
return 1
|
|
145
|
+
return 0
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
if __name__ == "__main__":
|
|
149
|
+
try:
|
|
150
|
+
raise SystemExit(main())
|
|
151
|
+
except Exception as exc:
|
|
152
|
+
print(str(exc), file=sys.stderr)
|
|
153
|
+
raise SystemExit(1) from exc
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from typing import List
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class FileCollector:
|
|
6
|
+
|
|
7
|
+
@staticmethod
|
|
8
|
+
def feature_files(paths: List[str]) -> List[Path]:
|
|
9
|
+
files: List[Path] = []
|
|
10
|
+
for raw_path in paths:
|
|
11
|
+
path = Path(raw_path)
|
|
12
|
+
if not path.exists():
|
|
13
|
+
raise FileNotFoundError(f"Feature path does not exist: {path}")
|
|
14
|
+
if path.is_file() and path.suffix == ".feature":
|
|
15
|
+
files.append(path)
|
|
16
|
+
elif path.is_dir():
|
|
17
|
+
files.extend(sorted(path.rglob("*.feature")))
|
|
18
|
+
return sorted({path.resolve() for path in files})
|
|
19
|
+
|
|
20
|
+
@staticmethod
|
|
21
|
+
def xml_files(paths: List[str]) -> List[Path]:
|
|
22
|
+
files: List[Path] = []
|
|
23
|
+
for raw_path in paths:
|
|
24
|
+
path = Path(raw_path)
|
|
25
|
+
if not path.exists():
|
|
26
|
+
continue
|
|
27
|
+
if path.is_file():
|
|
28
|
+
files.append(path)
|
|
29
|
+
elif path.is_dir():
|
|
30
|
+
files.extend(sorted(path.rglob("*.xml")))
|
|
31
|
+
return sorted({path.resolve() for path in files})
|
|
32
|
+
|
|
33
|
+
@staticmethod
|
|
34
|
+
def json_files(paths: List[str]) -> List[Path]:
|
|
35
|
+
files: List[Path] = []
|
|
36
|
+
for raw_path in paths:
|
|
37
|
+
path = Path(raw_path)
|
|
38
|
+
if not path.exists():
|
|
39
|
+
continue
|
|
40
|
+
if path.is_file():
|
|
41
|
+
files.append(path)
|
|
42
|
+
elif path.is_dir():
|
|
43
|
+
files.extend(sorted(path.rglob("*.json")))
|
|
44
|
+
return sorted({path.resolve() for path in files})
|