test-base 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- test_base/__init__.py +20 -0
- test_base/adapters/__init__.py +354 -0
- test_base/adapters/base.py +263 -0
- test_base/adapters/canonical_json.py +1088 -0
- test_base/adapters/junit_xml.py +1272 -0
- test_base/api/__init__.py +0 -0
- test_base/api/app.py +314 -0
- test_base/api/deps.py +178 -0
- test_base/api/routes/__init__.py +0 -0
- test_base/api/routes/failures.py +104 -0
- test_base/api/routes/health.py +35 -0
- test_base/api/routes/imports.py +143 -0
- test_base/api/routes/queries.py +295 -0
- test_base/api/routes/retention.py +130 -0
- test_base/api/routes/runs.py +147 -0
- test_base/api/routes/server.py +65 -0
- test_base/api/routes/stream.py +117 -0
- test_base/api/schemas.py +390 -0
- test_base/api/security.py +183 -0
- test_base/cli/__init__.py +125 -0
- test_base/cli/commands/__init__.py +0 -0
- test_base/cli/commands/db.py +76 -0
- test_base/cli/commands/demo.py +103 -0
- test_base/cli/commands/formats.py +41 -0
- test_base/cli/commands/ingest.py +119 -0
- test_base/cli/commands/init.py +738 -0
- test_base/cli/commands/prune.py +146 -0
- test_base/cli/commands/publish.py +133 -0
- test_base/cli/commands/serve.py +118 -0
- test_base/cli/commands/token.py +97 -0
- test_base/cli/options.py +189 -0
- test_base/cli/render.py +223 -0
- test_base/config.py +519 -0
- test_base/demo/__init__.py +0 -0
- test_base/demo/evidence.py +447 -0
- test_base/demo/seed.py +628 -0
- test_base/domain/__init__.py +61 -0
- test_base/domain/events.py +422 -0
- test_base/domain/failures.py +327 -0
- test_base/domain/identity.py +205 -0
- test_base/domain/metrics.py +176 -0
- test_base/domain/models.py +947 -0
- test_base/domain/status.py +246 -0
- test_base/errors.py +136 -0
- test_base/logging.py +174 -0
- test_base/py.typed +0 -0
- test_base/sdk/__init__.py +28 -0
- test_base/sdk/client.py +607 -0
- test_base/service/__init__.py +0 -0
- test_base/service/bus.py +331 -0
- test_base/service/coalesce.py +133 -0
- test_base/service/imports.py +552 -0
- test_base/service/retention.py +640 -0
- test_base/service/runs.py +394 -0
- test_base/service/runtime.py +123 -0
- test_base/storage/__init__.py +41 -0
- test_base/storage/artifacts.py +347 -0
- test_base/storage/base.py +350 -0
- test_base/storage/db.py +595 -0
- test_base/storage/failures.py +321 -0
- test_base/storage/locks.py +104 -0
- test_base/storage/migrations/env.py +108 -0
- test_base/storage/migrations/script.py.mako +26 -0
- test_base/storage/migrations/versions/0001_initial.py +289 -0
- test_base/storage/migrations/versions/0002_retention_indexes.py +38 -0
- test_base/storage/migrations/versions/0003_postgresql_bigint.py +74 -0
- test_base/storage/migrations/versions/0004_run_shards.py +37 -0
- test_base/storage/migrations/versions/0005_failure_groups.py +206 -0
- test_base/storage/models.py +560 -0
- test_base/storage/repositories.py +2481 -0
- test_base/storage/uow.py +138 -0
- test_base/version.py +17 -0
- test_base/web/assets/css/app.css +3567 -0
- test_base/web/assets/favicon.svg +5 -0
- test_base/web/assets/js/00-util.js +829 -0
- test_base/web/assets/js/05-windows.js +551 -0
- test_base/web/assets/js/10-provider.js +396 -0
- test_base/web/assets/js/12-live-provider.js +804 -0
- test_base/web/assets/js/20-charts.js +715 -0
- test_base/web/assets/js/30-store.js +793 -0
- test_base/web/assets/js/40-components.js +2141 -0
- test_base/web/assets/js/42-timeline.js +629 -0
- test_base/web/assets/js/50-view-overview.js +747 -0
- test_base/web/assets/js/52-view-analytics.js +842 -0
- test_base/web/assets/js/53-view-suites.js +508 -0
- test_base/web/assets/js/54-view-metrics.js +438 -0
- test_base/web/assets/js/55-view-runs.js +244 -0
- test_base/web/assets/js/56-view-evidence.js +703 -0
- test_base/web/assets/js/57-view-failures.js +596 -0
- test_base/web/assets/js/60-view-run.js +788 -0
- test_base/web/assets/js/65-view-test.js +402 -0
- test_base/web/assets/js/70-view-settings.js +524 -0
- test_base/web/assets/js/75-view-help.js +554 -0
- test_base/web/assets/js/80-settings.js +99 -0
- test_base/web/assets/js/85-activity.js +727 -0
- test_base/web/assets/js/90-app.js +595 -0
- test_base/web/assets/manifest.json +70 -0
- test_base/web/index.html +210 -0
- test_base-0.1.1.dist-info/METADATA +285 -0
- test_base-0.1.1.dist-info/RECORD +103 -0
- test_base-0.1.1.dist-info/WHEEL +4 -0
- test_base-0.1.1.dist-info/entry_points.txt +6 -0
- test_base-0.1.1.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,1088 @@
|
|
|
1
|
+
"""Test Base's own JSON format — the reference publisher contract.
|
|
2
|
+
|
|
3
|
+
This adapter is the answer to "my framework has no JUnit writer, how do I get
|
|
4
|
+
results in?". It accepts two shapes, and both of them must say so out loud with
|
|
5
|
+
a ``kind`` discriminator. That requirement is the whole design: a format that
|
|
6
|
+
claimed "any JSON object with a ``tests`` key" would swallow coverage reports,
|
|
7
|
+
ESLint output and half of ``package.json``, and every one of those would arrive
|
|
8
|
+
as a silently wrong run.
|
|
9
|
+
|
|
10
|
+
Shape 1 — ``test-base-report``
|
|
11
|
+
----------------------------
|
|
12
|
+
|
|
13
|
+
The friendly form, written by hand or by a twenty-line script::
|
|
14
|
+
|
|
15
|
+
{
|
|
16
|
+
"kind": "test-base-report",
|
|
17
|
+
"schema_version": 1,
|
|
18
|
+
"run": {
|
|
19
|
+
"id": "6c0e6b4e-...", // optional; re-importing the same
|
|
20
|
+
// id updates that run instead of
|
|
21
|
+
// creating a second one
|
|
22
|
+
"name": "nightly e2e",
|
|
23
|
+
"team": "checkout",
|
|
24
|
+
"component": "checkout-service",
|
|
25
|
+
"test_type": "e2e", // TestType; unknown values warn
|
|
26
|
+
"trigger": "schedule", // TriggerType
|
|
27
|
+
"status": "failed", // advisory only, the server folds
|
|
28
|
+
"started_at": "2026-04-02T01:15:00Z",
|
|
29
|
+
"finished_at": "2026-04-02T01:19:42Z",
|
|
30
|
+
"tags": ["nightly"],
|
|
31
|
+
"metadata": {"shard": "3/8"},
|
|
32
|
+
"context": {
|
|
33
|
+
"framework": "playwright", "framework_version": "1.44.0",
|
|
34
|
+
"language": "javascript", "environment": "staging",
|
|
35
|
+
"branch": "main", "commit_sha": "8f21c0d",
|
|
36
|
+
"ci_provider": "github-actions",
|
|
37
|
+
"pipeline_url": "https://ci.example.test/runs/91"
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"suites": [
|
|
41
|
+
{"path": "specs/checkout", "name": "checkout", "duration_ms": 91000},
|
|
42
|
+
{"path": "specs/checkout/payment", "name": "payment",
|
|
43
|
+
"parent_path": "specs/checkout"}
|
|
44
|
+
],
|
|
45
|
+
"tests": [
|
|
46
|
+
{
|
|
47
|
+
"name": "declines an expired card",
|
|
48
|
+
"class_name": "CheckoutPayment",
|
|
49
|
+
"suite_path": "specs/checkout/payment",
|
|
50
|
+
"params": {"currency": "EUR"}, // dict, list or pre-rendered string
|
|
51
|
+
"status": "failed", // any word normalize_status knows
|
|
52
|
+
"duration_ms": 812,
|
|
53
|
+
"file": "specs/checkout/payment.spec.ts",
|
|
54
|
+
"owner": "payments",
|
|
55
|
+
"tags": ["smoke"], "markers": ["slow"],
|
|
56
|
+
"worker": "worker-2",
|
|
57
|
+
"failure": {"type": "AssertionError", "message": "...",
|
|
58
|
+
"stack_trace": "...", "expected": "402", "actual": "500"},
|
|
59
|
+
"stdout": "...", "stderr": "...", "logs": "...",
|
|
60
|
+
"skip_reason": null,
|
|
61
|
+
"metadata": {"retry_policy": "2x"}
|
|
62
|
+
}
|
|
63
|
+
]
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
``suites`` is optional: any ``suite_path`` a test names is created implicitly.
|
|
67
|
+
Suite and test identities are *derived*, never carried — ``suite_id`` from
|
|
68
|
+
:func:`test_base.domain.identity.suite_id` and the test identity from
|
|
69
|
+
:func:`test_base.domain.identity.test_identity` — so a publisher in any language
|
|
70
|
+
that follows ``docs/identity`` gets the same UUIDs Test Base does.
|
|
71
|
+
|
|
72
|
+
Retries, two ways
|
|
73
|
+
~~~~~~~~~~~~~~~~~
|
|
74
|
+
|
|
75
|
+
Either nest them::
|
|
76
|
+
|
|
77
|
+
{"name": "flaky checkout", "suite_path": "specs/checkout",
|
|
78
|
+
"attempts": [
|
|
79
|
+
{"status": "failed", "duration_ms": 4100, "failure": {...}},
|
|
80
|
+
{"status": "failed", "duration_ms": 3900, "failure": {...}},
|
|
81
|
+
{"status": "passed", "duration_ms": 4300}
|
|
82
|
+
]}
|
|
83
|
+
|
|
84
|
+
or repeat the test with an increasing ``attempt_number``::
|
|
85
|
+
|
|
86
|
+
{"name": "flaky checkout", "suite_path": "specs/checkout",
|
|
87
|
+
"attempt_number": 1, "status": "failed"}
|
|
88
|
+
{"name": "flaky checkout", "suite_path": "specs/checkout",
|
|
89
|
+
"attempt_number": 2, "status": "passed"}
|
|
90
|
+
|
|
91
|
+
Both produce one test with two attempts, because entries are grouped by test
|
|
92
|
+
identity — the *last* attempt is the effective outcome and the disagreement
|
|
93
|
+
between attempts is what makes the test flaky. Attempt numbers are re-sequenced
|
|
94
|
+
densely from 1 and a warning says so if the file's own numbering was gappy.
|
|
95
|
+
|
|
96
|
+
The expansion is::
|
|
97
|
+
|
|
98
|
+
run.queued -> run.started -> suite.started* -> test.finished* ->
|
|
99
|
+
suite.finished* -> run.finished
|
|
100
|
+
|
|
101
|
+
in that block order — every suite is announced before any result arrives, which
|
|
102
|
+
is what a dashboard subscribing mid-import needs in order to draw the tree.
|
|
103
|
+
|
|
104
|
+
Shape 2 — ``test-base-events``
|
|
105
|
+
-----------------------------
|
|
106
|
+
|
|
107
|
+
The raw form, for a publisher that already speaks the event contract::
|
|
108
|
+
|
|
109
|
+
{"kind": "test-base-events", "schema_version": 1,
|
|
110
|
+
"events": [ {CloudEvents envelope}, ... ]}
|
|
111
|
+
|
|
112
|
+
Each event is validated as a :class:`~test_base.domain.events.DomainEvent` and
|
|
113
|
+
passed through unchanged, **sequence numbers preserved** — they are the file's
|
|
114
|
+
statement about ordering and this adapter has no better information. Missing
|
|
115
|
+
``run_id``/``workspace_id``/``source`` are filled from the import request;
|
|
116
|
+
a ``workspace_id`` that disagrees with the importing workspace is overridden
|
|
117
|
+
(with a warning), because a file must never be able to write into another
|
|
118
|
+
tenant.
|
|
119
|
+
|
|
120
|
+
Versioning
|
|
121
|
+
----------
|
|
122
|
+
|
|
123
|
+
``schema_version`` is refused, not coerced, when this build does not speak it.
|
|
124
|
+
An older Test Base reading a newer file cannot know which fields changed meaning,
|
|
125
|
+
and a wrong pass rate is worse than a failed import.
|
|
126
|
+
|
|
127
|
+
Idempotency
|
|
128
|
+
-----------
|
|
129
|
+
|
|
130
|
+
Event ids are UUIDv5 over ``(source, run_id, sequence, type)`` rather than
|
|
131
|
+
random, so importing the same file twice against the same ``run.id`` applies
|
|
132
|
+
zero new events instead of doubling the run.
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
from __future__ import annotations
|
|
136
|
+
|
|
137
|
+
import json
|
|
138
|
+
import re
|
|
139
|
+
import uuid
|
|
140
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
141
|
+
from datetime import UTC, datetime, timedelta
|
|
142
|
+
from typing import IO, Any
|
|
143
|
+
|
|
144
|
+
from pydantic import ValidationError as PydanticValidationError
|
|
145
|
+
|
|
146
|
+
from test_base.adapters.base import SNIFF_BYTES, BaseAdapter, ParseContext, WarningCollector
|
|
147
|
+
from test_base.domain.events import (
|
|
148
|
+
DomainEvent,
|
|
149
|
+
EventType,
|
|
150
|
+
RunFinishedData,
|
|
151
|
+
RunQueuedData,
|
|
152
|
+
RunStartedData,
|
|
153
|
+
SuiteFinishedData,
|
|
154
|
+
SuiteStartedData,
|
|
155
|
+
TestFinishedData,
|
|
156
|
+
make_event,
|
|
157
|
+
)
|
|
158
|
+
from test_base.domain.identity import (
|
|
159
|
+
TEST_BASE_NAMESPACE,
|
|
160
|
+
normalize_params,
|
|
161
|
+
test_identity,
|
|
162
|
+
)
|
|
163
|
+
from test_base.domain.identity import suite_id as suite_identity
|
|
164
|
+
from test_base.domain.models import (
|
|
165
|
+
FailureDetail,
|
|
166
|
+
RunContext,
|
|
167
|
+
TestType,
|
|
168
|
+
TriggerType,
|
|
169
|
+
utcnow,
|
|
170
|
+
)
|
|
171
|
+
from test_base.domain.status import RunStatus, TestStatus, normalize_status
|
|
172
|
+
from test_base.errors import AdapterError, ValidationError
|
|
173
|
+
from test_base.version import SCHEMA_VERSION
|
|
174
|
+
|
|
175
|
+
__all__ = ["EVENTS_KIND", "REPORT_KIND", "CanonicalJsonAdapter"]
|
|
176
|
+
|
|
177
|
+
#: The two values ``kind`` may take.
|
|
178
|
+
REPORT_KIND = "test-base-report"
|
|
179
|
+
EVENTS_KIND = "test-base-events"
|
|
180
|
+
|
|
181
|
+
#: ``LongText``/``ShortText`` in :mod:`test_base.domain.models`. Adapters must
|
|
182
|
+
#: clip to these or the model rejects the event they just built.
|
|
183
|
+
_LONG_TEXT_LIMIT = 64_000
|
|
184
|
+
_SHORT_TEXT_LIMIT = 512
|
|
185
|
+
|
|
186
|
+
#: ``Slug`` in :mod:`test_base.domain.models`. The event payloads type these
|
|
187
|
+
#: fields as plain ``str``, but ``TestRun.team``/``component``/``tags`` and
|
|
188
|
+
#: ``TestCase.tags`` are ``Slug``s — so a value that sails through event
|
|
189
|
+
#: validation can still blow up when the server folds the event into a row.
|
|
190
|
+
#: Adapters have to be the ones that check.
|
|
191
|
+
_SLUG_RE = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._\- ]*$")
|
|
192
|
+
|
|
193
|
+
_KIND_RE = re.compile(rb'"kind"\s*:\s*"(test-base-report|test-base-events)"')
|
|
194
|
+
_RUN_KEY_RE = re.compile(rb'"run"\s*:\s*\{')
|
|
195
|
+
_TESTS_KEY_RE = re.compile(rb'"tests"\s*:\s*\[')
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _clip(value: object, limit: int) -> tuple[str | None, bool]:
|
|
199
|
+
"""Coerce to text and cut it to ``limit`` characters.
|
|
200
|
+
|
|
201
|
+
Returns the truncation flag as well, because a cut stack trace that does not
|
|
202
|
+
say it was cut is how a debugging session gets wasted.
|
|
203
|
+
"""
|
|
204
|
+
if value is None:
|
|
205
|
+
return None, False
|
|
206
|
+
text = value if isinstance(value, str) else str(value)
|
|
207
|
+
if len(text) <= limit:
|
|
208
|
+
return text, False
|
|
209
|
+
return text[:limit], True
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _event_id(source: str, run_id: uuid.UUID, sequence: int, event_type: EventType) -> uuid.UUID:
|
|
213
|
+
"""Deterministic event id, so a re-import of the same file is a no-op."""
|
|
214
|
+
return uuid.uuid5(TEST_BASE_NAMESPACE, f"{source}\x1f{run_id}\x1f{sequence}\x1f{event_type}")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _as_mapping(value: object) -> dict[str, Any]:
|
|
218
|
+
return dict(value) if isinstance(value, Mapping) else {}
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _as_list(value: object) -> list[Any]:
|
|
222
|
+
if isinstance(value, list):
|
|
223
|
+
return value
|
|
224
|
+
return []
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _as_str_list(value: object, limit: int = 64) -> list[str]:
|
|
228
|
+
if isinstance(value, str):
|
|
229
|
+
candidates = [value]
|
|
230
|
+
elif isinstance(value, Sequence):
|
|
231
|
+
candidates = [item for item in value if isinstance(item, str)]
|
|
232
|
+
else:
|
|
233
|
+
return []
|
|
234
|
+
return [item.strip()[:_SHORT_TEXT_LIMIT] for item in candidates if item.strip()][:limit]
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
class CanonicalJsonAdapter(BaseAdapter):
|
|
238
|
+
"""Reads ``test-base-report`` and ``test-base-events`` JSON documents."""
|
|
239
|
+
|
|
240
|
+
name = "testbase-json"
|
|
241
|
+
display_name = "Test Base JSON"
|
|
242
|
+
extensions = (".json",)
|
|
243
|
+
content_types = (
|
|
244
|
+
"application/json",
|
|
245
|
+
"text/json",
|
|
246
|
+
"application/vnd.test-base+json",
|
|
247
|
+
)
|
|
248
|
+
#: Above JUnit's: when a file somehow looks like both, ours is the more
|
|
249
|
+
#: specific claim.
|
|
250
|
+
priority = 100
|
|
251
|
+
description = "Test Base's own report and raw event JSON (kind: test-base-report/-events)."
|
|
252
|
+
|
|
253
|
+
# ------------------------------------------------------------- detect
|
|
254
|
+
|
|
255
|
+
def detect(self, stream: IO[bytes], filename: str | None, content_type: str | None) -> float:
|
|
256
|
+
"""Score the input. Never claims plain JSON without a ``kind``."""
|
|
257
|
+
hint = self.hint_score(filename, content_type)
|
|
258
|
+
try:
|
|
259
|
+
head = self.sniff(stream, SNIFF_BYTES)
|
|
260
|
+
except Exception: # pragma: no cover - unreadable stream
|
|
261
|
+
return hint
|
|
262
|
+
stripped = head.lstrip(b"\xef\xbb\xbf").lstrip()
|
|
263
|
+
if not stripped:
|
|
264
|
+
return hint
|
|
265
|
+
if not stripped.startswith(b"{"):
|
|
266
|
+
# A JSON array, or not JSON at all. Neither shape is ours.
|
|
267
|
+
return 0.0
|
|
268
|
+
if _KIND_RE.search(head):
|
|
269
|
+
return 1.0
|
|
270
|
+
try:
|
|
271
|
+
document = json.loads(stripped.decode("utf-8", errors="strict"))
|
|
272
|
+
except (UnicodeDecodeError, ValueError):
|
|
273
|
+
# Truncated at SNIFF_BYTES, which is the normal case for a real
|
|
274
|
+
# report: fall back to looking for the two keys textually.
|
|
275
|
+
if _RUN_KEY_RE.search(head) and _TESTS_KEY_RE.search(head):
|
|
276
|
+
return 0.6
|
|
277
|
+
return hint
|
|
278
|
+
if isinstance(document, dict):
|
|
279
|
+
if document.get("kind") in (REPORT_KIND, EVENTS_KIND):
|
|
280
|
+
return 1.0
|
|
281
|
+
if isinstance(document.get("run"), dict) and isinstance(document.get("tests"), list):
|
|
282
|
+
return 0.6
|
|
283
|
+
return hint
|
|
284
|
+
|
|
285
|
+
# -------------------------------------------------------------- parse
|
|
286
|
+
|
|
287
|
+
def parse(self, stream: IO[bytes], context: ParseContext) -> Iterator[DomainEvent]:
|
|
288
|
+
document = self._load(stream)
|
|
289
|
+
self._check_version(document)
|
|
290
|
+
kind = document.get("kind")
|
|
291
|
+
if kind == REPORT_KIND:
|
|
292
|
+
yield from _ReportExpander(context).expand(document)
|
|
293
|
+
elif kind == EVENTS_KIND:
|
|
294
|
+
yield from _EventPassthrough(context).expand(document)
|
|
295
|
+
else:
|
|
296
|
+
raise AdapterError(
|
|
297
|
+
"this is JSON, but not a Test Base document: the top-level object must carry "
|
|
298
|
+
f'"kind": "{REPORT_KIND}" or "kind": "{EVENTS_KIND}". '
|
|
299
|
+
f"Found kind={kind!r}.",
|
|
300
|
+
detail={"kind": kind, "adapter": self.name},
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
@staticmethod
|
|
304
|
+
def _load(stream: IO[bytes]) -> dict[str, Any]:
|
|
305
|
+
raw = stream.read()
|
|
306
|
+
if not raw:
|
|
307
|
+
raise AdapterError("the document is empty")
|
|
308
|
+
try:
|
|
309
|
+
document = json.loads(raw.decode("utf-8-sig"))
|
|
310
|
+
except UnicodeDecodeError as exc:
|
|
311
|
+
raise AdapterError(f"the document is not valid UTF-8: {exc}") from exc
|
|
312
|
+
except json.JSONDecodeError as exc:
|
|
313
|
+
raise AdapterError(
|
|
314
|
+
f"the document is not valid JSON: {exc.msg} at line {exc.lineno} column {exc.colno}"
|
|
315
|
+
) from exc
|
|
316
|
+
if not isinstance(document, dict):
|
|
317
|
+
raise AdapterError(
|
|
318
|
+
'a Test Base JSON document must be a JSON object with a "kind" key, '
|
|
319
|
+
f"not a {type(document).__name__}"
|
|
320
|
+
)
|
|
321
|
+
return document
|
|
322
|
+
|
|
323
|
+
@staticmethod
|
|
324
|
+
def _check_version(document: Mapping[str, Any]) -> None:
|
|
325
|
+
version = document.get("schema_version", SCHEMA_VERSION)
|
|
326
|
+
if isinstance(version, bool) or not isinstance(version, int):
|
|
327
|
+
raise ValidationError(
|
|
328
|
+
f"schema_version must be an integer, got {version!r}",
|
|
329
|
+
detail={"schema_version": version},
|
|
330
|
+
)
|
|
331
|
+
if version != SCHEMA_VERSION:
|
|
332
|
+
raise ValidationError(
|
|
333
|
+
f"this build speaks Test Base schema_version {SCHEMA_VERSION}, but the document "
|
|
334
|
+
f"declares {version}. Upgrade Test Base (a newer document) or re-export the "
|
|
335
|
+
f"report with a publisher that targets version {SCHEMA_VERSION}.",
|
|
336
|
+
detail={"document_schema_version": version, "supported": SCHEMA_VERSION},
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
# --------------------------------------------------------------- shape 1
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
class _ReportExpander:
|
|
344
|
+
"""Turns one ``test-base-report`` document into an ordered event stream.
|
|
345
|
+
|
|
346
|
+
A class rather than a function because expansion needs a handful of pieces
|
|
347
|
+
of per-parse state — the sequence counter lives on the context, but the
|
|
348
|
+
"have I already warned about naive timestamps" flag and the derived run
|
|
349
|
+
identity do not.
|
|
350
|
+
"""
|
|
351
|
+
|
|
352
|
+
def __init__(self, context: ParseContext) -> None:
|
|
353
|
+
self.context = context
|
|
354
|
+
self.warnings = context.warnings
|
|
355
|
+
self.project = context.project_key
|
|
356
|
+
self.text_limit = min(context.max_text_bytes, _LONG_TEXT_LIMIT)
|
|
357
|
+
self._warned: set[str] = set()
|
|
358
|
+
self._context_cache: RunContext | None = None
|
|
359
|
+
self.run_id = context.run_id
|
|
360
|
+
self.source = context.source
|
|
361
|
+
|
|
362
|
+
# ----------------------------------------------------------- helpers
|
|
363
|
+
|
|
364
|
+
def warn_once(self, code: str, message: str, **detail: Any) -> None:
|
|
365
|
+
if code in self._warned:
|
|
366
|
+
return
|
|
367
|
+
self._warned.add(code)
|
|
368
|
+
self.warnings.add(code, message, **detail)
|
|
369
|
+
|
|
370
|
+
def slug(self, value: object, field: str, location: str) -> str | None:
|
|
371
|
+
"""Keep a value only if it can survive being stored as a ``Slug``."""
|
|
372
|
+
text, _ = _clip(value, 120)
|
|
373
|
+
if text is None:
|
|
374
|
+
return None
|
|
375
|
+
if not _SLUG_RE.match(text):
|
|
376
|
+
self.warnings.add(
|
|
377
|
+
"invalid_slug",
|
|
378
|
+
f"{field}={text!r} is not a usable key (letters, digits, dot, dash, "
|
|
379
|
+
"underscore and space only); it was dropped",
|
|
380
|
+
location=location,
|
|
381
|
+
)
|
|
382
|
+
return None
|
|
383
|
+
return text
|
|
384
|
+
|
|
385
|
+
def slugs(self, values: Sequence[str], field: str, location: str) -> list[str]:
|
|
386
|
+
return [kept for kept in (self.slug(value, field, location) for value in values) if kept]
|
|
387
|
+
|
|
388
|
+
def datetime_at(self, value: object, location: str) -> datetime | None:
|
|
389
|
+
if value is None:
|
|
390
|
+
return None
|
|
391
|
+
if isinstance(value, datetime):
|
|
392
|
+
parsed = value
|
|
393
|
+
elif isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
394
|
+
parsed = datetime.fromtimestamp(float(value), tz=UTC)
|
|
395
|
+
else:
|
|
396
|
+
text = str(value).strip()
|
|
397
|
+
if not text:
|
|
398
|
+
return None
|
|
399
|
+
try:
|
|
400
|
+
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
401
|
+
except ValueError:
|
|
402
|
+
self.warnings.add(
|
|
403
|
+
"bad_timestamp",
|
|
404
|
+
f"could not read {text!r} as an ISO-8601 timestamp; ignoring it",
|
|
405
|
+
location=location,
|
|
406
|
+
)
|
|
407
|
+
return None
|
|
408
|
+
if parsed.tzinfo is None:
|
|
409
|
+
self.warn_once(
|
|
410
|
+
"naive_timestamp",
|
|
411
|
+
"the report contains timestamps with no UTC offset; they are read as UTC",
|
|
412
|
+
location=location,
|
|
413
|
+
)
|
|
414
|
+
parsed = parsed.replace(tzinfo=UTC)
|
|
415
|
+
return parsed
|
|
416
|
+
|
|
417
|
+
def duration_at(self, block: Mapping[str, Any], location: str) -> int | None:
|
|
418
|
+
"""Milliseconds from ``duration_ms``, or seconds from ``duration``."""
|
|
419
|
+
raw = block.get("duration_ms")
|
|
420
|
+
scale = 1.0
|
|
421
|
+
if raw is None:
|
|
422
|
+
raw = block.get("duration")
|
|
423
|
+
scale = 1000.0
|
|
424
|
+
if raw is None or isinstance(raw, bool):
|
|
425
|
+
return None
|
|
426
|
+
try:
|
|
427
|
+
value = float(raw) * scale
|
|
428
|
+
except (TypeError, ValueError):
|
|
429
|
+
self.warnings.add(
|
|
430
|
+
"bad_duration", f"could not read {raw!r} as a duration", location=location
|
|
431
|
+
)
|
|
432
|
+
return None
|
|
433
|
+
if value < 0:
|
|
434
|
+
self.warnings.add(
|
|
435
|
+
"bad_duration", f"negative duration {raw!r} clamped to 0", location=location
|
|
436
|
+
)
|
|
437
|
+
return 0
|
|
438
|
+
return round(value)
|
|
439
|
+
|
|
440
|
+
def status_at(self, raw: object, location: str) -> tuple[TestStatus, str | None]:
|
|
441
|
+
original = None if raw is None else str(raw)[:_SHORT_TEXT_LIMIT]
|
|
442
|
+
if original is None or not original.strip():
|
|
443
|
+
self.warnings.add(
|
|
444
|
+
"missing_status",
|
|
445
|
+
"an attempt has no status; recording it as unknown",
|
|
446
|
+
location=location,
|
|
447
|
+
)
|
|
448
|
+
return TestStatus.UNKNOWN, original
|
|
449
|
+
status = normalize_status(original)
|
|
450
|
+
if status is TestStatus.UNKNOWN:
|
|
451
|
+
self.warnings.add(
|
|
452
|
+
"unknown_status",
|
|
453
|
+
f"status {original!r} is not a word Test Base knows; recorded as unknown",
|
|
454
|
+
location=location,
|
|
455
|
+
status=original,
|
|
456
|
+
)
|
|
457
|
+
return status, original
|
|
458
|
+
|
|
459
|
+
def failure_at(self, block: object, location: str) -> FailureDetail | None:
|
|
460
|
+
if not isinstance(block, Mapping):
|
|
461
|
+
return None
|
|
462
|
+
type_text, _ = _clip(block.get("type"), _SHORT_TEXT_LIMIT)
|
|
463
|
+
message, cut_message = _clip(block.get("message"), self.text_limit)
|
|
464
|
+
stack, cut_stack = _clip(block.get("stack_trace") or block.get("stack"), self.text_limit)
|
|
465
|
+
expected, cut_expected = _clip(block.get("expected"), self.text_limit)
|
|
466
|
+
actual, cut_actual = _clip(block.get("actual"), self.text_limit)
|
|
467
|
+
truncated = cut_message or cut_stack or cut_expected or cut_actual
|
|
468
|
+
if truncated:
|
|
469
|
+
self.warnings.add(
|
|
470
|
+
"text_truncated",
|
|
471
|
+
f"failure text was longer than {self.text_limit} characters and was cut",
|
|
472
|
+
location=location,
|
|
473
|
+
)
|
|
474
|
+
if not any((type_text, message, stack, expected, actual)):
|
|
475
|
+
return None
|
|
476
|
+
return FailureDetail(
|
|
477
|
+
type=type_text,
|
|
478
|
+
message=message,
|
|
479
|
+
stack_trace=stack,
|
|
480
|
+
expected=expected,
|
|
481
|
+
actual=actual,
|
|
482
|
+
truncated=truncated or bool(block.get("truncated")),
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
def text_at(self, block: Mapping[str, Any], key: str, location: str) -> str | None:
|
|
486
|
+
text, cut = _clip(block.get(key), self.text_limit)
|
|
487
|
+
if cut:
|
|
488
|
+
self.warnings.add(
|
|
489
|
+
"text_truncated",
|
|
490
|
+
f"{key} was longer than {self.text_limit} characters and was cut",
|
|
491
|
+
location=location,
|
|
492
|
+
)
|
|
493
|
+
return text
|
|
494
|
+
|
|
495
|
+
def emit(
|
|
496
|
+
self,
|
|
497
|
+
event_type: EventType,
|
|
498
|
+
data: Any,
|
|
499
|
+
*,
|
|
500
|
+
subject: str | None = None,
|
|
501
|
+
time: datetime | None = None,
|
|
502
|
+
) -> DomainEvent:
|
|
503
|
+
sequence = self.context.next_sequence()
|
|
504
|
+
return make_event(
|
|
505
|
+
event_type,
|
|
506
|
+
run_id=self.run_id,
|
|
507
|
+
workspace_id=self.context.workspace_id,
|
|
508
|
+
sequence=sequence,
|
|
509
|
+
data=data,
|
|
510
|
+
source=self.source,
|
|
511
|
+
event_id=_event_id(self.source, self.run_id, sequence, event_type),
|
|
512
|
+
time=time or utcnow(),
|
|
513
|
+
subject=subject,
|
|
514
|
+
)
|
|
515
|
+
|
|
516
|
+
# ------------------------------------------------------------ expand
|
|
517
|
+
|
|
518
|
+
def expand(self, document: Mapping[str, Any]) -> Iterator[DomainEvent]:
|
|
519
|
+
run_block = document.get("run")
|
|
520
|
+
if run_block is not None and not isinstance(run_block, Mapping):
|
|
521
|
+
raise AdapterError('"run" must be a JSON object')
|
|
522
|
+
run_block = _as_mapping(run_block)
|
|
523
|
+
|
|
524
|
+
declared_id = _coerce_uuid(run_block.get("id"))
|
|
525
|
+
if declared_id is not None:
|
|
526
|
+
self.run_id = declared_id
|
|
527
|
+
file_project = run_block.get("project_key")
|
|
528
|
+
if isinstance(file_project, str) and file_project and file_project != self.project:
|
|
529
|
+
self.warnings.info(
|
|
530
|
+
"project_key_override",
|
|
531
|
+
f"the file says project {file_project!r}; the import said "
|
|
532
|
+
f"{self.project!r} and the import wins",
|
|
533
|
+
)
|
|
534
|
+
|
|
535
|
+
tests = document.get("tests")
|
|
536
|
+
if tests is not None and not isinstance(tests, list):
|
|
537
|
+
raise AdapterError('"tests" must be a JSON array')
|
|
538
|
+
entries = _as_list(tests)
|
|
539
|
+
|
|
540
|
+
suites = self._collect_suites(document, entries)
|
|
541
|
+
groups = self._group_tests(entries, suites)
|
|
542
|
+
|
|
543
|
+
started_at = self.datetime_at(run_block.get("started_at"), "run") or utcnow()
|
|
544
|
+
finished_at = self.datetime_at(run_block.get("finished_at"), "run")
|
|
545
|
+
run_duration = self.duration_at(run_block, "run")
|
|
546
|
+
|
|
547
|
+
yield self.emit(
|
|
548
|
+
EventType.RUN_QUEUED,
|
|
549
|
+
self._run_queued(run_block, expected=len(groups)),
|
|
550
|
+
subject=self.project,
|
|
551
|
+
time=started_at,
|
|
552
|
+
)
|
|
553
|
+
yield self.emit(
|
|
554
|
+
EventType.RUN_STARTED,
|
|
555
|
+
RunStartedData(
|
|
556
|
+
started_at=started_at,
|
|
557
|
+
project_key=self.project,
|
|
558
|
+
context=self._run_context(run_block),
|
|
559
|
+
),
|
|
560
|
+
subject=self.project,
|
|
561
|
+
time=started_at,
|
|
562
|
+
)
|
|
563
|
+
|
|
564
|
+
for suite in suites.values():
|
|
565
|
+
yield self.emit(
|
|
566
|
+
EventType.SUITE_STARTED,
|
|
567
|
+
SuiteStartedData(
|
|
568
|
+
suite_id=suite.identity,
|
|
569
|
+
name=suite.name,
|
|
570
|
+
path=suite.path,
|
|
571
|
+
parent_id=suite.parent_id,
|
|
572
|
+
started_at=suite.started_at,
|
|
573
|
+
metadata=suite.metadata,
|
|
574
|
+
),
|
|
575
|
+
subject=suite.path,
|
|
576
|
+
time=suite.started_at or started_at,
|
|
577
|
+
)
|
|
578
|
+
|
|
579
|
+
for group in groups:
|
|
580
|
+
yield from self._test_events(group)
|
|
581
|
+
|
|
582
|
+
for suite in suites.values():
|
|
583
|
+
yield self.emit(
|
|
584
|
+
EventType.SUITE_FINISHED,
|
|
585
|
+
SuiteFinishedData(
|
|
586
|
+
suite_id=suite.identity,
|
|
587
|
+
finished_at=suite.finished_at,
|
|
588
|
+
duration_ms=suite.duration_ms,
|
|
589
|
+
),
|
|
590
|
+
subject=suite.path,
|
|
591
|
+
time=suite.finished_at or started_at,
|
|
592
|
+
)
|
|
593
|
+
|
|
594
|
+
if run_duration is None and finished_at is not None:
|
|
595
|
+
run_duration = max(0, round((finished_at - started_at).total_seconds() * 1000))
|
|
596
|
+
if finished_at is None:
|
|
597
|
+
finished_at = (
|
|
598
|
+
started_at + timedelta(milliseconds=run_duration)
|
|
599
|
+
if run_duration is not None
|
|
600
|
+
else utcnow()
|
|
601
|
+
)
|
|
602
|
+
yield self.emit(
|
|
603
|
+
EventType.RUN_FINISHED,
|
|
604
|
+
RunFinishedData(
|
|
605
|
+
finished_at=finished_at,
|
|
606
|
+
status=self._run_status(run_block),
|
|
607
|
+
duration_ms=run_duration,
|
|
608
|
+
metadata={},
|
|
609
|
+
),
|
|
610
|
+
subject=self.project,
|
|
611
|
+
time=finished_at,
|
|
612
|
+
)
|
|
613
|
+
|
|
614
|
+
# ------------------------------------------------------------- run
|
|
615
|
+
|
|
616
|
+
def _run_queued(self, run_block: Mapping[str, Any], *, expected: int) -> RunQueuedData:
|
|
617
|
+
metadata = dict(self.context.metadata)
|
|
618
|
+
metadata.update(_as_mapping(run_block.get("metadata")))
|
|
619
|
+
tags = self.slugs(
|
|
620
|
+
list(dict.fromkeys([*self.context.tags, *_as_str_list(run_block.get("tags"))])),
|
|
621
|
+
"tag",
|
|
622
|
+
"run",
|
|
623
|
+
)
|
|
624
|
+
return RunQueuedData(
|
|
625
|
+
project_key=self.project,
|
|
626
|
+
name=self.context.name or _clip(run_block.get("name"), _SHORT_TEXT_LIMIT)[0],
|
|
627
|
+
team=self.context.team or self.slug(run_block.get("team"), "team", "run"),
|
|
628
|
+
component=(
|
|
629
|
+
self.context.component or self.slug(run_block.get("component"), "component", "run")
|
|
630
|
+
),
|
|
631
|
+
test_type=self._enum(
|
|
632
|
+
TestType, run_block.get("test_type"), self.context.test_type, TestType.OTHER, "run"
|
|
633
|
+
),
|
|
634
|
+
trigger=self._enum(
|
|
635
|
+
TriggerType,
|
|
636
|
+
run_block.get("trigger"),
|
|
637
|
+
self.context.trigger,
|
|
638
|
+
TriggerType.UNKNOWN,
|
|
639
|
+
"run",
|
|
640
|
+
),
|
|
641
|
+
context=self._run_context(run_block),
|
|
642
|
+
tags=tags,
|
|
643
|
+
metadata=metadata,
|
|
644
|
+
expected_tests=expected or None,
|
|
645
|
+
)
|
|
646
|
+
|
|
647
|
+
def _enum(
|
|
648
|
+
self,
|
|
649
|
+
enum_type: type[Any],
|
|
650
|
+
raw: object,
|
|
651
|
+
from_request: Any,
|
|
652
|
+
default: Any,
|
|
653
|
+
location: str,
|
|
654
|
+
) -> Any:
|
|
655
|
+
"""Request value wins; the file only fills in a field left at its default."""
|
|
656
|
+
if from_request != default:
|
|
657
|
+
return from_request
|
|
658
|
+
if raw is None:
|
|
659
|
+
return default
|
|
660
|
+
text = str(raw).strip().lower().replace("-", "_")
|
|
661
|
+
try:
|
|
662
|
+
return enum_type(text)
|
|
663
|
+
except ValueError:
|
|
664
|
+
self.warnings.add(
|
|
665
|
+
"unknown_enum",
|
|
666
|
+
f"{enum_type.__name__} {raw!r} is not a value Test Base knows; using {default}",
|
|
667
|
+
location=location,
|
|
668
|
+
)
|
|
669
|
+
return default
|
|
670
|
+
|
|
671
|
+
def _run_context(self, run_block: Mapping[str, Any]) -> RunContext:
|
|
672
|
+
"""Merge the file's context under the importer's — the human wins.
|
|
673
|
+
|
|
674
|
+
Cached: ``run.queued`` and ``run.started`` both carry it, and building it
|
|
675
|
+
twice would report every dropped field twice.
|
|
676
|
+
"""
|
|
677
|
+
if self._context_cache is not None:
|
|
678
|
+
return self._context_cache
|
|
679
|
+
file_context = _as_mapping(run_block.get("context"))
|
|
680
|
+
merged: dict[str, Any] = {}
|
|
681
|
+
request = self.context.context
|
|
682
|
+
for field in RunContext.model_fields:
|
|
683
|
+
chosen = getattr(request, field, None)
|
|
684
|
+
if chosen is None:
|
|
685
|
+
chosen = file_context.get(field)
|
|
686
|
+
if chosen is not None:
|
|
687
|
+
merged[field] = chosen
|
|
688
|
+
self._context_cache = _best_effort_context(merged, self.warnings, "run.context")
|
|
689
|
+
return self._context_cache
|
|
690
|
+
|
|
691
|
+
def _run_status(self, run_block: Mapping[str, Any]) -> RunStatus | None:
|
|
692
|
+
raw = run_block.get("status")
|
|
693
|
+
if raw is None:
|
|
694
|
+
return None
|
|
695
|
+
text = str(raw).strip().lower()
|
|
696
|
+
try:
|
|
697
|
+
return RunStatus(text)
|
|
698
|
+
except ValueError:
|
|
699
|
+
mapped = normalize_status(text)
|
|
700
|
+
if mapped is TestStatus.CANCELLED:
|
|
701
|
+
return RunStatus.CANCELLED
|
|
702
|
+
self.warnings.info(
|
|
703
|
+
"advisory_run_status",
|
|
704
|
+
f"run status {raw!r} is not a RunStatus; the server will fold the attempts instead",
|
|
705
|
+
location="run",
|
|
706
|
+
)
|
|
707
|
+
return None
|
|
708
|
+
|
|
709
|
+
# ----------------------------------------------------------- suites
|
|
710
|
+
|
|
711
|
+
def _collect_suites(
|
|
712
|
+
self, document: Mapping[str, Any], entries: Sequence[Any]
|
|
713
|
+
) -> dict[str, _Suite]:
|
|
714
|
+
suites: dict[str, _Suite] = {}
|
|
715
|
+
declared = document.get("suites")
|
|
716
|
+
if declared is not None and not isinstance(declared, list):
|
|
717
|
+
raise AdapterError('"suites" must be a JSON array')
|
|
718
|
+
for index, item in enumerate(_as_list(declared)):
|
|
719
|
+
block = _as_mapping(item)
|
|
720
|
+
location = f"suites[{index}]"
|
|
721
|
+
path = _clip(block.get("path") or block.get("name"), _SHORT_TEXT_LIMIT)[0]
|
|
722
|
+
if not path:
|
|
723
|
+
self.warnings.add(
|
|
724
|
+
"missing_suite_path",
|
|
725
|
+
"a declared suite has neither path nor name",
|
|
726
|
+
location=location,
|
|
727
|
+
)
|
|
728
|
+
continue
|
|
729
|
+
parent_path = block.get("parent_path") or block.get("parent")
|
|
730
|
+
suites[path] = _Suite(
|
|
731
|
+
path=path,
|
|
732
|
+
name=_clip(block.get("name") or path, _SHORT_TEXT_LIMIT)[0] or path,
|
|
733
|
+
identity=suite_identity(project=self.project, suite_path=path),
|
|
734
|
+
parent_id=(
|
|
735
|
+
suite_identity(project=self.project, suite_path=str(parent_path))
|
|
736
|
+
if parent_path
|
|
737
|
+
else None
|
|
738
|
+
),
|
|
739
|
+
started_at=self.datetime_at(block.get("started_at"), location),
|
|
740
|
+
finished_at=self.datetime_at(block.get("finished_at"), location),
|
|
741
|
+
duration_ms=self.duration_at(block, location),
|
|
742
|
+
metadata=_as_mapping(block.get("metadata")),
|
|
743
|
+
)
|
|
744
|
+
|
|
745
|
+
for index, item in enumerate(entries):
|
|
746
|
+
block = _as_mapping(item)
|
|
747
|
+
path = self._suite_path(block, f"tests[{index}]", warn=False)
|
|
748
|
+
if path not in suites:
|
|
749
|
+
suites[path] = _Suite(
|
|
750
|
+
path=path,
|
|
751
|
+
name=path.rsplit("/", 1)[-1] or path,
|
|
752
|
+
identity=suite_identity(project=self.project, suite_path=path),
|
|
753
|
+
)
|
|
754
|
+
return suites
|
|
755
|
+
|
|
756
|
+
def _suite_path(self, block: Mapping[str, Any], location: str, *, warn: bool = True) -> str:
|
|
757
|
+
raw = block.get("suite_path") or block.get("suite")
|
|
758
|
+
path = _clip(raw, _SHORT_TEXT_LIMIT)[0]
|
|
759
|
+
if path:
|
|
760
|
+
return path
|
|
761
|
+
fallback = _clip(block.get("class_name") or block.get("classname"), _SHORT_TEXT_LIMIT)[0]
|
|
762
|
+
if warn:
|
|
763
|
+
self.warnings.add(
|
|
764
|
+
"missing_suite_path",
|
|
765
|
+
f"a test has no suite_path; grouping it under {fallback or 'tests'!r}",
|
|
766
|
+
location=location,
|
|
767
|
+
)
|
|
768
|
+
return fallback or "tests"
|
|
769
|
+
|
|
770
|
+
# ------------------------------------------------------------ tests
|
|
771
|
+
|
|
772
|
+
def _group_tests(
|
|
773
|
+
self, entries: Sequence[Any], suites: Mapping[str, _Suite]
|
|
774
|
+
) -> list[_TestGroup]:
|
|
775
|
+
"""Fold entries into one group per test identity, ordered by first sight."""
|
|
776
|
+
groups: dict[tuple[str, str, str, str], _TestGroup] = {}
|
|
777
|
+
order: list[tuple[str, str, str, str]] = []
|
|
778
|
+
|
|
779
|
+
for index, item in enumerate(entries):
|
|
780
|
+
if not isinstance(item, Mapping):
|
|
781
|
+
self.warnings.add(
|
|
782
|
+
"bad_test_entry",
|
|
783
|
+
f"tests[{index}] is a {type(item).__name__}, not an object; skipped",
|
|
784
|
+
location=f"tests[{index}]",
|
|
785
|
+
)
|
|
786
|
+
continue
|
|
787
|
+
location = f"tests[{index}]"
|
|
788
|
+
name = _clip(item.get("name") or item.get("test_name"), _SHORT_TEXT_LIMIT)[0]
|
|
789
|
+
if not name:
|
|
790
|
+
self.warnings.add(
|
|
791
|
+
"missing_test_name",
|
|
792
|
+
"a test entry has no name; skipped",
|
|
793
|
+
location=location,
|
|
794
|
+
)
|
|
795
|
+
continue
|
|
796
|
+
class_name = _clip(item.get("class_name") or item.get("classname"), _SHORT_TEXT_LIMIT)[
|
|
797
|
+
0
|
|
798
|
+
]
|
|
799
|
+
suite_path = self._suite_path(item, location)
|
|
800
|
+
raw_params = item.get("params")
|
|
801
|
+
params = normalize_params(raw_params) if raw_params is not None else ""
|
|
802
|
+
key = (suite_path, class_name or "", name, params)
|
|
803
|
+
|
|
804
|
+
group = groups.get(key)
|
|
805
|
+
if group is None:
|
|
806
|
+
group = _TestGroup(
|
|
807
|
+
name=name,
|
|
808
|
+
class_name=class_name,
|
|
809
|
+
suite_path=suite_path,
|
|
810
|
+
params=params or None,
|
|
811
|
+
identity=test_identity(
|
|
812
|
+
project=self.project,
|
|
813
|
+
suite_path=suite_path,
|
|
814
|
+
test_name=name,
|
|
815
|
+
class_name=class_name,
|
|
816
|
+
params=raw_params,
|
|
817
|
+
),
|
|
818
|
+
full_name=_clip(item.get("full_name"), _SHORT_TEXT_LIMIT)[0],
|
|
819
|
+
file=_clip(item.get("file"), _SHORT_TEXT_LIMIT)[0],
|
|
820
|
+
owner=_clip(item.get("owner"), _SHORT_TEXT_LIMIT)[0],
|
|
821
|
+
tags=self.slugs(_as_str_list(item.get("tags")), "tag", location),
|
|
822
|
+
markers=_as_str_list(item.get("markers")),
|
|
823
|
+
suite=suites[suite_path],
|
|
824
|
+
)
|
|
825
|
+
groups[key] = group
|
|
826
|
+
order.append(key)
|
|
827
|
+
|
|
828
|
+
nested = item.get("attempts")
|
|
829
|
+
if isinstance(nested, list) and nested:
|
|
830
|
+
for offset, raw_attempt in enumerate(nested):
|
|
831
|
+
attempt = _as_mapping(raw_attempt)
|
|
832
|
+
number = _coerce_int(attempt.get("attempt_number"), offset + 1)
|
|
833
|
+
group.attempts.append((number, attempt, f"{location}.attempts[{offset}]"))
|
|
834
|
+
declared_status = item.get("status")
|
|
835
|
+
last_status = _as_mapping(nested[-1]).get("status")
|
|
836
|
+
if declared_status is not None and last_status != declared_status:
|
|
837
|
+
self.warnings.info(
|
|
838
|
+
"status_disagrees",
|
|
839
|
+
f"the entry's status {declared_status!r} differs from its last attempt "
|
|
840
|
+
f"{last_status!r}; the last attempt wins",
|
|
841
|
+
location=location,
|
|
842
|
+
)
|
|
843
|
+
else:
|
|
844
|
+
number = _coerce_int(item.get("attempt_number"), len(group.attempts) + 1)
|
|
845
|
+
group.attempts.append((number, dict(item), location))
|
|
846
|
+
|
|
847
|
+
return [groups[key] for key in order]
|
|
848
|
+
|
|
849
|
+
def _test_events(self, group: _TestGroup) -> Iterator[DomainEvent]:
|
|
850
|
+
ordered = sorted(enumerate(group.attempts), key=lambda pair: (pair[1][0], pair[0]))
|
|
851
|
+
numbers = [attempt[0] for _, attempt in ordered]
|
|
852
|
+
if numbers != list(range(1, len(numbers) + 1)):
|
|
853
|
+
self.warnings.add(
|
|
854
|
+
"attempt_renumbered",
|
|
855
|
+
f"attempt numbers {numbers} for {group.name!r} were not 1..n; "
|
|
856
|
+
"they were re-sequenced in the order given",
|
|
857
|
+
location=group.suite_path,
|
|
858
|
+
)
|
|
859
|
+
suite = group.suite
|
|
860
|
+
for position, (_, (_, attempt, location)) in enumerate(ordered, start=1):
|
|
861
|
+
status, original = self.status_at(attempt.get("status"), location)
|
|
862
|
+
started = self.datetime_at(attempt.get("started_at"), location)
|
|
863
|
+
finished = self.datetime_at(attempt.get("finished_at"), location)
|
|
864
|
+
duration = self.duration_at(attempt, location)
|
|
865
|
+
if duration is None and started and finished:
|
|
866
|
+
duration = max(0, round((finished - started).total_seconds() * 1000))
|
|
867
|
+
yield self.emit(
|
|
868
|
+
EventType.TEST_FINISHED,
|
|
869
|
+
TestFinishedData(
|
|
870
|
+
test_id=group.identity,
|
|
871
|
+
suite_id=suite.identity,
|
|
872
|
+
name=group.name,
|
|
873
|
+
class_name=group.class_name,
|
|
874
|
+
suite_path=group.suite_path,
|
|
875
|
+
params=group.params,
|
|
876
|
+
full_name=group.full_name,
|
|
877
|
+
file=group.file,
|
|
878
|
+
owner=group.owner,
|
|
879
|
+
tags=group.tags,
|
|
880
|
+
markers=group.markers,
|
|
881
|
+
attempt_number=position,
|
|
882
|
+
status=status,
|
|
883
|
+
original_status=original,
|
|
884
|
+
started_at=started,
|
|
885
|
+
finished_at=finished,
|
|
886
|
+
duration_ms=duration,
|
|
887
|
+
failure=self.failure_at(attempt.get("failure"), location),
|
|
888
|
+
stdout=self.text_at(attempt, "stdout", location),
|
|
889
|
+
stderr=self.text_at(attempt, "stderr", location),
|
|
890
|
+
logs=self.text_at(attempt, "logs", location),
|
|
891
|
+
skip_reason=_clip(attempt.get("skip_reason"), _SHORT_TEXT_LIMIT)[0],
|
|
892
|
+
worker=_clip(attempt.get("worker"), _SHORT_TEXT_LIMIT)[0],
|
|
893
|
+
metadata=_as_mapping(attempt.get("metadata")),
|
|
894
|
+
),
|
|
895
|
+
subject=group.full_name or f"{group.suite_path}::{group.name}",
|
|
896
|
+
time=finished or utcnow(),
|
|
897
|
+
)
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
# --------------------------------------------------------------- shape 2
|
|
901
|
+
|
|
902
|
+
|
|
903
|
+
class _EventPassthrough:
|
|
904
|
+
"""Validates a raw ``test-base-events`` document and hands it straight on."""
|
|
905
|
+
|
|
906
|
+
def __init__(self, context: ParseContext) -> None:
|
|
907
|
+
self.context = context
|
|
908
|
+
self.warnings = context.warnings
|
|
909
|
+
|
|
910
|
+
def expand(self, document: Mapping[str, Any]) -> Iterator[DomainEvent]:
|
|
911
|
+
raw_events = document.get("events")
|
|
912
|
+
if not isinstance(raw_events, list):
|
|
913
|
+
raise AdapterError('a "test-base-events" document must carry an "events" array')
|
|
914
|
+
if not raw_events:
|
|
915
|
+
raise AdapterError('the "events" array is empty')
|
|
916
|
+
|
|
917
|
+
run_id: uuid.UUID | None = None
|
|
918
|
+
overridden = 0
|
|
919
|
+
for index, item in enumerate(raw_events):
|
|
920
|
+
if not isinstance(item, Mapping):
|
|
921
|
+
raise AdapterError(f"events[{index}] is not an object")
|
|
922
|
+
payload = dict(item)
|
|
923
|
+
payload.pop("workspaceid", None)
|
|
924
|
+
payload["workspace_id"] = str(self.context.workspace_id)
|
|
925
|
+
declared_workspace = item.get("workspace_id") or item.get("workspaceid")
|
|
926
|
+
if declared_workspace and str(declared_workspace) != str(self.context.workspace_id):
|
|
927
|
+
overridden += 1
|
|
928
|
+
if not (payload.get("run_id") or payload.get("runid")):
|
|
929
|
+
payload["run_id"] = str(self.context.run_id)
|
|
930
|
+
payload.setdefault("source", self.context.source)
|
|
931
|
+
|
|
932
|
+
try:
|
|
933
|
+
event = DomainEvent.model_validate(payload)
|
|
934
|
+
except Exception as exc:
|
|
935
|
+
raise AdapterError(
|
|
936
|
+
f"events[{index}] is not a valid Test Base event: {exc}"
|
|
937
|
+
) from exc
|
|
938
|
+
|
|
939
|
+
if run_id is None:
|
|
940
|
+
run_id = event.run_id
|
|
941
|
+
elif event.run_id != run_id:
|
|
942
|
+
raise AdapterError(
|
|
943
|
+
"every event in one document must belong to the same run; "
|
|
944
|
+
f"events[{index}] is for run {event.run_id} but the document started "
|
|
945
|
+
f"with run {run_id}"
|
|
946
|
+
)
|
|
947
|
+
yield event
|
|
948
|
+
|
|
949
|
+
if overridden:
|
|
950
|
+
self.warnings.add(
|
|
951
|
+
"workspace_override",
|
|
952
|
+
f"{overridden} event(s) named a different workspace; they were re-homed into "
|
|
953
|
+
"the importing workspace",
|
|
954
|
+
)
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
# ------------------------------------------------------------- structures
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
class _Suite:
|
|
961
|
+
"""A suite discovered in a report, declared or implied by a test."""
|
|
962
|
+
|
|
963
|
+
__slots__ = (
|
|
964
|
+
"duration_ms",
|
|
965
|
+
"finished_at",
|
|
966
|
+
"identity",
|
|
967
|
+
"metadata",
|
|
968
|
+
"name",
|
|
969
|
+
"parent_id",
|
|
970
|
+
"path",
|
|
971
|
+
"started_at",
|
|
972
|
+
)
|
|
973
|
+
|
|
974
|
+
def __init__(
|
|
975
|
+
self,
|
|
976
|
+
*,
|
|
977
|
+
path: str,
|
|
978
|
+
name: str,
|
|
979
|
+
identity: uuid.UUID,
|
|
980
|
+
parent_id: uuid.UUID | None = None,
|
|
981
|
+
started_at: datetime | None = None,
|
|
982
|
+
finished_at: datetime | None = None,
|
|
983
|
+
duration_ms: int | None = None,
|
|
984
|
+
metadata: dict[str, Any] | None = None,
|
|
985
|
+
) -> None:
|
|
986
|
+
self.path = path
|
|
987
|
+
self.name = name
|
|
988
|
+
self.identity = identity
|
|
989
|
+
self.parent_id = parent_id
|
|
990
|
+
self.started_at = started_at
|
|
991
|
+
self.finished_at = finished_at
|
|
992
|
+
self.duration_ms = duration_ms
|
|
993
|
+
self.metadata = metadata or {}
|
|
994
|
+
|
|
995
|
+
|
|
996
|
+
class _TestGroup:
|
|
997
|
+
"""Every attempt reported for one test identity, in one report."""
|
|
998
|
+
|
|
999
|
+
__slots__ = (
|
|
1000
|
+
"attempts",
|
|
1001
|
+
"class_name",
|
|
1002
|
+
"file",
|
|
1003
|
+
"full_name",
|
|
1004
|
+
"identity",
|
|
1005
|
+
"markers",
|
|
1006
|
+
"name",
|
|
1007
|
+
"owner",
|
|
1008
|
+
"params",
|
|
1009
|
+
"suite",
|
|
1010
|
+
"suite_path",
|
|
1011
|
+
"tags",
|
|
1012
|
+
)
|
|
1013
|
+
|
|
1014
|
+
def __init__(
|
|
1015
|
+
self,
|
|
1016
|
+
*,
|
|
1017
|
+
name: str,
|
|
1018
|
+
class_name: str | None,
|
|
1019
|
+
suite_path: str,
|
|
1020
|
+
params: str | None,
|
|
1021
|
+
identity: uuid.UUID,
|
|
1022
|
+
full_name: str | None,
|
|
1023
|
+
file: str | None,
|
|
1024
|
+
owner: str | None,
|
|
1025
|
+
tags: list[str],
|
|
1026
|
+
markers: list[str],
|
|
1027
|
+
suite: _Suite,
|
|
1028
|
+
) -> None:
|
|
1029
|
+
self.name = name
|
|
1030
|
+
self.class_name = class_name
|
|
1031
|
+
self.suite_path = suite_path
|
|
1032
|
+
self.params = params
|
|
1033
|
+
self.identity = identity
|
|
1034
|
+
self.full_name = full_name
|
|
1035
|
+
self.file = file
|
|
1036
|
+
self.owner = owner
|
|
1037
|
+
self.tags = tags
|
|
1038
|
+
self.markers = markers
|
|
1039
|
+
self.suite = suite
|
|
1040
|
+
self.attempts: list[tuple[int, dict[str, Any], str]] = []
|
|
1041
|
+
|
|
1042
|
+
|
|
1043
|
+
def _best_effort_context(
|
|
1044
|
+
merged: dict[str, Any], warnings: WarningCollector, location: str
|
|
1045
|
+
) -> RunContext:
|
|
1046
|
+
"""Build a :class:`RunContext`, dropping only the fields that will not validate.
|
|
1047
|
+
|
|
1048
|
+
A single bad ``pipeline_url`` in a hand-written report should not cost the
|
|
1049
|
+
branch, the commit and the environment as well.
|
|
1050
|
+
"""
|
|
1051
|
+
try:
|
|
1052
|
+
return RunContext(**merged)
|
|
1053
|
+
except PydanticValidationError as exc:
|
|
1054
|
+
bad = {str(error["loc"][0]) for error in exc.errors() if error.get("loc")}
|
|
1055
|
+
for field in sorted(bad):
|
|
1056
|
+
warnings.add(
|
|
1057
|
+
"invalid_run_context",
|
|
1058
|
+
f"run context field {field}={merged.get(field)!r} did not validate and was "
|
|
1059
|
+
"dropped; the rest of the context was kept",
|
|
1060
|
+
location=location,
|
|
1061
|
+
)
|
|
1062
|
+
remaining = {key: value for key, value in merged.items() if key not in bad}
|
|
1063
|
+
try:
|
|
1064
|
+
return RunContext(**remaining)
|
|
1065
|
+
except PydanticValidationError: # pragma: no cover - belt and braces
|
|
1066
|
+
return RunContext()
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def _coerce_uuid(value: object) -> uuid.UUID | None:
|
|
1070
|
+
if isinstance(value, uuid.UUID):
|
|
1071
|
+
return value
|
|
1072
|
+
if isinstance(value, str) and value.strip():
|
|
1073
|
+
try:
|
|
1074
|
+
return uuid.UUID(value.strip())
|
|
1075
|
+
except ValueError:
|
|
1076
|
+
return None
|
|
1077
|
+
return None
|
|
1078
|
+
|
|
1079
|
+
|
|
1080
|
+
def _coerce_int(value: object, default: int) -> int:
|
|
1081
|
+
"""Read an attempt number, tolerating the string a hand-written file will use."""
|
|
1082
|
+
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
|
1083
|
+
return default
|
|
1084
|
+
try:
|
|
1085
|
+
number = int(value)
|
|
1086
|
+
except (TypeError, ValueError):
|
|
1087
|
+
return default
|
|
1088
|
+
return number if number >= 1 else default
|