pytest-timing 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/ARCHITECTURE.md +34 -16
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/CHANGELOG.md +21 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/PKG-INFO +23 -9
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/README.md +22 -8
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/pyproject.toml +1 -1
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/__init__.py +1 -1
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/admission.py +2 -1
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/collector.py +19 -5
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/model.py +19 -3
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/plugin.py +15 -6
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/render/ascii.py +14 -2
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/schedule.py +54 -17
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/static/report.html +16 -3
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/xdist_scheduler.py +50 -15
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_ascii.py +13 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_collector.py +13 -1
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_html.py +15 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_memory_gate.py +132 -28
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_runtime_admission.py +5 -1
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/uv.lock +1 -1
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/.github/workflows/ci.yml +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/.github/workflows/release.yml +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/.gitignore +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/LICENSE +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/benchmarks/cpu_bench.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/docs/report.png +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/examples/test_demo.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/cli.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/demand.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/fixtures.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/outputs.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/render/__init__.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/render/html.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/render/trace.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/static/__init__.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/telemetry.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/src/pytest_timing/xdist_compat.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/conftest.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/fixtures/legacy_complete_false.json +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/fixtures/legacy_complete_true.json +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/fixtures/legacy_no_flag.json +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_admission.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_benchmarks.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_cli.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_cpu.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_fixtures.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_memory.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_plugin.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_schedule.py +0 -0
- {pytest_timing-0.3.0 → pytest_timing-0.3.1}/tests/test_trace.py +0 -0
|
@@ -388,27 +388,45 @@ happens only when nothing runs in the domain, so nothing else's memory is at ris
|
|
|
388
388
|
Pressure feedback moves only the CPU limit.
|
|
389
389
|
|
|
390
390
|
Demand comes from history, never from declarations. `Estimates.from_run` keeps, per
|
|
391
|
-
test, the largest
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
several).
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
391
|
+
test, the largest need any attempt showed (the worst case is what an out-of-memory
|
|
392
|
+
kill depends on), and per shared fixture key the memory the attempt that paid its
|
|
393
|
+
set-up kept resident (`after - base`, split evenly when one attempt paid for
|
|
394
|
+
several). A payer's rise spans its set-ups, so its own need is `peak - after`: what
|
|
395
|
+
it used beyond what stayed, which is the fixtures'. Session- and package-scoped
|
|
396
|
+
fixtures are the exception: what they keep is every worker's baseline
|
|
397
|
+
(`is_baseline`), never attributed, charged or projected. Every worker sets them up
|
|
398
|
+
once and never lets go, so gating on them cannot spare the host; it can only hold
|
|
399
|
+
tests back, or park a worker until stall resolution shuts it down, which is what a
|
|
400
|
+
large session fixture did before this rule. The first attempt on each worker is a
|
|
401
|
+
warm-up window (lazy imports, caches, the allocator's first growth), so its rise and
|
|
402
|
+
residual count only for a test or fixture with no other attempt.
|
|
403
|
+
|
|
404
|
+
`Costs.charge` adds a `memory` to each `Charge`: the test's need plus what the
|
|
405
|
+
module- and class-scoped fixtures alive around it keep, projected through the lane's
|
|
406
|
+
fixture state exactly as CPU holds are, since workers report nothing about memory at
|
|
407
|
+
run time. A fixture with recorded memory belongs to its tests' families even when its
|
|
408
|
+
set-up was too quick to matter for time. `_need_memory` is the twin of `_need`; an
|
|
409
|
+
idle worker reserves what its fixtures keep, and a worker whose next test could never
|
|
410
|
+
fit next to what the other workers' fixtures keep is parked like one blocked by CPU
|
|
411
|
+
holds. A fixture reached at run time adds its recorded memory to the queued charges
|
|
412
|
+
when its request is granted.
|
|
403
413
|
|
|
404
414
|
The budget is `--timing-memory`: a size, or `auto`, which takes 80% of the smallest
|
|
405
415
|
memory the domain's workers report (physical memory capped by the cgroup limit,
|
|
406
416
|
`memory.max` on v2 and `memory.limit_in_bytes` on v1, read like the CPU quota). The
|
|
407
417
|
share is deliberate: estimates are rises above each worker's footprint, and the
|
|
408
|
-
footprints, the controller and the rest of the host are
|
|
409
|
-
recorded run every estimate is zero and the gate admits
|
|
410
|
-
so. The planner ignores memory: with tests kept apart by
|
|
411
|
-
memory would add little, and the lane plan can still move.
|
|
418
|
+
footprints (session fixtures included), the controller and the rest of the host are
|
|
419
|
+
not in them. Without a recorded run every estimate is zero and the gate admits
|
|
420
|
+
everything; the summary says so. The planner ignores memory: with tests kept apart by
|
|
421
|
+
the gate, balancing lanes by memory would add little, and the lane plan can still move.
|
|
422
|
+
|
|
423
|
+
Every `Admission` has a `kind` (`cpu` or `memory`), and a refusal records which kinds
|
|
424
|
+
refused (`refused`), as does passing a test over for what other workers keep. When
|
|
425
|
+
the worker is admitted, the wait it records (`AdmissionWait.gates`) says which gates
|
|
426
|
+
held it, and the collector puts it on the test's `cpu.wait` or `memory.wait`
|
|
427
|
+
accordingly. A worker that leaves while parked, having run nothing it waited for,
|
|
428
|
+
has no test to carry its wait: `_release_runtime` records it in `parked`, and each
|
|
429
|
+
gate's summary reports the parked time and worker count beside the tests' waits.
|
|
412
430
|
|
|
413
431
|
## Measuring CPU work
|
|
414
432
|
|
|
@@ -1,5 +1,26 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.1
|
|
4
|
+
|
|
5
|
+
- Memory admission no longer charges what session- and package-scoped fixtures keep
|
|
6
|
+
resident. Every worker sets them up once and never lets go, so counting them once
|
|
7
|
+
per lane, and again against every other lane, could only hold tests back or park
|
|
8
|
+
a worker until it was shut down: on a suite with a large session fixture the gate
|
|
9
|
+
left two of eight workers nearly idle and made the run almost three times longer
|
|
10
|
+
while the host had tens of gigabytes free. Their memory is now the worker's
|
|
11
|
+
baseline, which the budget's headroom covers (#7).
|
|
12
|
+
- Memory estimates are what a test needs of its own. An attempt that set up shared
|
|
13
|
+
fixtures was charged its whole rise and, again, what the fixtures kept, twice the
|
|
14
|
+
residual; its own need is now the rise beyond what stayed. The first attempt on
|
|
15
|
+
each worker, whose window also covers the worker's warm-up, counts only for a
|
|
16
|
+
test or fixture with no other attempt (#7).
|
|
17
|
+
- Waits say which gate held the test. A test the memory gate held back carries the
|
|
18
|
+
seconds in `memory.wait`, beside `cpu.wait` for CPU slots, and the summary line
|
|
19
|
+
reports memory waits even when a CPU budget is set. Time a worker sat parked and
|
|
20
|
+
then left without running what it waited for is reported per gate as well, instead
|
|
21
|
+
of vanishing. The HTML report's table gains a "Held" column and its hover details a
|
|
22
|
+
"held" line, and the terminal's slowest-tests rows say how long each was held (#7).
|
|
23
|
+
|
|
3
24
|
## 0.3.0
|
|
4
25
|
|
|
5
26
|
- Estimate a test's cost from the median of its best attempts instead of its
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: pytest-timing
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Record test timings under pytest-xdist and render cargo-style timing reports (ASCII Gantt and HTML).
|
|
5
5
|
Project-URL: Homepage, https://github.com/messense/pytest-timing
|
|
6
6
|
Author-email: messense <messense@icloud.com>
|
|
@@ -128,7 +128,10 @@ of the worker's process tree around each test, in bytes: `base` before its set-u
|
|
|
128
128
|
`peak` during it, and `after` at its teardown. The difference between `peak` and
|
|
129
129
|
`base` is what the test needed on top of the worker's existing footprint; `after`
|
|
130
130
|
minus `base` is what stayed, such as a shared fixture it set up. With
|
|
131
|
-
`--timing-memory`, the run's `memory` describes the budget and how admission went
|
|
131
|
+
`--timing-memory`, the run's `memory` describes the budget and how admission went,
|
|
132
|
+
and a test the memory gate held back carries the seconds it waited in `memory.wait`,
|
|
133
|
+
as `cpu.wait` does for the CPU gate; the HTML report's table and hover details and
|
|
134
|
+
the terminal's slowest-tests rows show them.
|
|
132
135
|
It also records worker
|
|
133
136
|
lifecycles, detected host CPU environments, and how the session ended (`finished`,
|
|
134
137
|
`collect_only`, `interrupted`, `aborted`, `internal_error`, with pytest's reason). It is
|
|
@@ -268,8 +271,8 @@ What to expect:
|
|
|
268
271
|
forces the oldest blocked request if needed. Such forced reservations can exceed
|
|
269
272
|
the limit and are counted in the summary.
|
|
270
273
|
- The summary reports the budget, the number of tests over one slot, and how long
|
|
271
|
-
tests waited for slots in total. Each test's JSON record carries its own wait
|
|
272
|
-
`cpu.runtime_wait` records admission waits inside a running test. These waits
|
|
274
|
+
tests waited for slots in total. Each test's JSON record carries its own wait in
|
|
275
|
+
`cpu.wait`. `cpu.runtime_wait` records admission waits inside a running test. These waits
|
|
273
276
|
are excluded from duration estimates, fixture setup costs and measured CPU rate.
|
|
274
277
|
A runtime request that gets no grant within five minutes cancels the fixture
|
|
275
278
|
setup. It waits to recover the test's original reservation before failing, so
|
|
@@ -305,18 +308,26 @@ pytest -n 4 --timing-json --timing-schedule pytest-timing.json --timing-memory 1
|
|
|
305
308
|
takes 80% of physical memory, capped by the cgroup memory limit. It reads the memory
|
|
306
309
|
each test needed from the run at `--timing-schedule`, so the first run only records;
|
|
307
310
|
the summary says how many tests had recorded memory. A test with no record weighs
|
|
308
|
-
nothing until it has run once. A
|
|
309
|
-
(a loaded model, say) reserved for as long as it is alive, on that
|
|
311
|
+
nothing until it has run once. A module- or class-scoped fixture keeps what its first
|
|
312
|
+
test left resident (a loaded model, say) reserved for as long as it is alive, on that
|
|
313
|
+
worker. What session- and package-scoped fixtures keep is treated as part of every
|
|
314
|
+
worker's footprint instead: each worker sets them up once and never lets go, so
|
|
315
|
+
gating on them could only delay the run, never spare the host.
|
|
310
316
|
|
|
311
317
|
What to expect:
|
|
312
318
|
|
|
313
319
|
- The gate is the same fair waiting line as for CPU: a worker whose next test does
|
|
314
320
|
not fit waits with its fixtures alive, and the oldest request goes first. Memory
|
|
315
321
|
and CPU budgets are checked together; a test starts only when both fit.
|
|
316
|
-
- Estimates are the largest
|
|
322
|
+
- Estimates are the largest need any recorded attempt showed. For an attempt that
|
|
323
|
+
set up shared fixtures, what stayed resident afterwards is the fixtures' (or the
|
|
324
|
+
worker's footprint), and the test's own need is what it used beyond that. The
|
|
325
|
+
first test on each worker also pays for the worker's warm-up, so its record is
|
|
326
|
+
used only for a test or fixture with no other attempt. Estimates are relative to
|
|
317
327
|
the worker's footprint, so `auto` leaves a fifth of the host for the workers
|
|
318
|
-
themselves, the controller and everything else; set a
|
|
319
|
-
machine
|
|
328
|
+
themselves, their session fixtures, the controller and everything else; set a
|
|
329
|
+
smaller budget on a shared machine, and a larger share of headroom when session
|
|
330
|
+
fixtures are big.
|
|
320
331
|
- A test recorded above the budget runs alone, and the summary counts it. Allocator
|
|
321
332
|
behaviour can make an estimate low: a test that reuses heap an earlier test freed
|
|
322
333
|
shows a smaller rise than it needs. Large buffers and subprocesses, the usual
|
|
@@ -324,6 +335,9 @@ What to expect:
|
|
|
324
335
|
- Memory does not enter the planner: lanes are still balanced by duration and fixture
|
|
325
336
|
cost, and the gate only delays starts. Under `--dist load` and `worksteal`, like
|
|
326
337
|
CPU admission.
|
|
338
|
+
- The summary says how many tests waited for memory and for how long, and how long
|
|
339
|
+
workers sat parked at the gate without running what they waited for. Each held
|
|
340
|
+
test's JSON record carries its wait in `memory.wait`.
|
|
327
341
|
|
|
328
342
|
## Re-render or merge saved runs
|
|
329
343
|
|
|
@@ -102,7 +102,10 @@ of the worker's process tree around each test, in bytes: `base` before its set-u
|
|
|
102
102
|
`peak` during it, and `after` at its teardown. The difference between `peak` and
|
|
103
103
|
`base` is what the test needed on top of the worker's existing footprint; `after`
|
|
104
104
|
minus `base` is what stayed, such as a shared fixture it set up. With
|
|
105
|
-
`--timing-memory`, the run's `memory` describes the budget and how admission went
|
|
105
|
+
`--timing-memory`, the run's `memory` describes the budget and how admission went,
|
|
106
|
+
and a test the memory gate held back carries the seconds it waited in `memory.wait`,
|
|
107
|
+
as `cpu.wait` does for the CPU gate; the HTML report's table and hover details and
|
|
108
|
+
the terminal's slowest-tests rows show them.
|
|
106
109
|
It also records worker
|
|
107
110
|
lifecycles, detected host CPU environments, and how the session ended (`finished`,
|
|
108
111
|
`collect_only`, `interrupted`, `aborted`, `internal_error`, with pytest's reason). It is
|
|
@@ -242,8 +245,8 @@ What to expect:
|
|
|
242
245
|
forces the oldest blocked request if needed. Such forced reservations can exceed
|
|
243
246
|
the limit and are counted in the summary.
|
|
244
247
|
- The summary reports the budget, the number of tests over one slot, and how long
|
|
245
|
-
tests waited for slots in total. Each test's JSON record carries its own wait
|
|
246
|
-
`cpu.runtime_wait` records admission waits inside a running test. These waits
|
|
248
|
+
tests waited for slots in total. Each test's JSON record carries its own wait in
|
|
249
|
+
`cpu.wait`. `cpu.runtime_wait` records admission waits inside a running test. These waits
|
|
247
250
|
are excluded from duration estimates, fixture setup costs and measured CPU rate.
|
|
248
251
|
A runtime request that gets no grant within five minutes cancels the fixture
|
|
249
252
|
setup. It waits to recover the test's original reservation before failing, so
|
|
@@ -279,18 +282,26 @@ pytest -n 4 --timing-json --timing-schedule pytest-timing.json --timing-memory 1
|
|
|
279
282
|
takes 80% of physical memory, capped by the cgroup memory limit. It reads the memory
|
|
280
283
|
each test needed from the run at `--timing-schedule`, so the first run only records;
|
|
281
284
|
the summary says how many tests had recorded memory. A test with no record weighs
|
|
282
|
-
nothing until it has run once. A
|
|
283
|
-
(a loaded model, say) reserved for as long as it is alive, on that
|
|
285
|
+
nothing until it has run once. A module- or class-scoped fixture keeps what its first
|
|
286
|
+
test left resident (a loaded model, say) reserved for as long as it is alive, on that
|
|
287
|
+
worker. What session- and package-scoped fixtures keep is treated as part of every
|
|
288
|
+
worker's footprint instead: each worker sets them up once and never lets go, so
|
|
289
|
+
gating on them could only delay the run, never spare the host.
|
|
284
290
|
|
|
285
291
|
What to expect:
|
|
286
292
|
|
|
287
293
|
- The gate is the same fair waiting line as for CPU: a worker whose next test does
|
|
288
294
|
not fit waits with its fixtures alive, and the oldest request goes first. Memory
|
|
289
295
|
and CPU budgets are checked together; a test starts only when both fit.
|
|
290
|
-
- Estimates are the largest
|
|
296
|
+
- Estimates are the largest need any recorded attempt showed. For an attempt that
|
|
297
|
+
set up shared fixtures, what stayed resident afterwards is the fixtures' (or the
|
|
298
|
+
worker's footprint), and the test's own need is what it used beyond that. The
|
|
299
|
+
first test on each worker also pays for the worker's warm-up, so its record is
|
|
300
|
+
used only for a test or fixture with no other attempt. Estimates are relative to
|
|
291
301
|
the worker's footprint, so `auto` leaves a fifth of the host for the workers
|
|
292
|
-
themselves, the controller and everything else; set a
|
|
293
|
-
machine
|
|
302
|
+
themselves, their session fixtures, the controller and everything else; set a
|
|
303
|
+
smaller budget on a shared machine, and a larger share of headroom when session
|
|
304
|
+
fixtures are big.
|
|
294
305
|
- A test recorded above the budget runs alone, and the summary counts it. Allocator
|
|
295
306
|
behaviour can make an estimate low: a test that reuses heap an earlier test freed
|
|
296
307
|
shows a smaller rise than it needs. Large buffers and subprocesses, the usual
|
|
@@ -298,6 +309,9 @@ What to expect:
|
|
|
298
309
|
- Memory does not enter the planner: lanes are still balanced by duration and fixture
|
|
299
310
|
cost, and the gate only delays starts. Under `--dist load` and `worksteal`, like
|
|
300
311
|
CPU admission.
|
|
312
|
+
- The summary says how many tests waited for memory and for how long, and how long
|
|
313
|
+
workers sat parked at the gate without running what they waited for. Each held
|
|
314
|
+
test's JSON record carries its wait in `memory.wait`.
|
|
301
315
|
|
|
302
316
|
## Re-render or merge saved runs
|
|
303
317
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "pytest-timing"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.1"
|
|
8
8
|
description = "Record test timings under pytest-xdist and render cargo-style timing reports (ASCII Gantt and HTML)."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -40,8 +40,9 @@ class Admission:
|
|
|
40
40
|
RECOVER_AFTER = 8
|
|
41
41
|
COOLDOWN = 10.0
|
|
42
42
|
|
|
43
|
-
def __init__(self, budget: int, name: str = "local") -> None:
|
|
43
|
+
def __init__(self, budget: int, name: str = "local", kind: str = "cpu") -> None:
|
|
44
44
|
self.name = name
|
|
45
|
+
self.kind = kind # what the slots are: ``cpu`` or ``memory`` (bytes)
|
|
45
46
|
self.budget = max(1, int(budget))
|
|
46
47
|
self.limit = self.budget
|
|
47
48
|
self.reserved: dict[Hashable, Reservation] = {}
|
|
@@ -14,6 +14,7 @@ correct span; they do not change that public occurrence/attempt numbering.
|
|
|
14
14
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
|
+
from collections.abc import Iterable
|
|
17
18
|
from dataclasses import dataclass
|
|
18
19
|
from typing import Any
|
|
19
20
|
|
|
@@ -83,12 +84,19 @@ class Collector:
|
|
|
83
84
|
worker.items = items
|
|
84
85
|
|
|
85
86
|
def add_wait(
|
|
86
|
-
self,
|
|
87
|
+
self,
|
|
88
|
+
worker_id: str,
|
|
89
|
+
nodeid: str,
|
|
90
|
+
index: int,
|
|
91
|
+
attempt: int,
|
|
92
|
+
seconds: float,
|
|
93
|
+
gates: Iterable[str] = (),
|
|
87
94
|
) -> None:
|
|
88
95
|
"""Add an admission delay to the exact execution that waited.
|
|
89
96
|
|
|
90
97
|
A nodeid can have several selections and retries. Looking up its last span
|
|
91
|
-
at session finish would put every delay on the final one instead.
|
|
98
|
+
at session finish would put every delay on the final one instead. ``gates``
|
|
99
|
+
names what held the test (``cpu``, ``memory``); unnamed, the CPU gate did.
|
|
92
100
|
"""
|
|
93
101
|
span = self._executions.get((worker_id, index, attempt))
|
|
94
102
|
if span is None:
|
|
@@ -98,9 +106,15 @@ class Collector:
|
|
|
98
106
|
span = self._last.get((worker_id, nodeid))
|
|
99
107
|
if span is None or span.outcome != "crashed" or span.phases or span.attempt != attempt:
|
|
100
108
|
return
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
109
|
+
gates = set(gates)
|
|
110
|
+
if "cpu" in gates or not gates:
|
|
111
|
+
if span.cpu is None:
|
|
112
|
+
span.cpu = CpuRecord(elapsed=span.duration)
|
|
113
|
+
span.cpu.wait += seconds
|
|
114
|
+
if "memory" in gates:
|
|
115
|
+
if span.memory is None:
|
|
116
|
+
span.memory = MemoryRecord() # no reading, but the wait is a fact
|
|
117
|
+
span.memory.wait += seconds
|
|
104
118
|
|
|
105
119
|
def worker_down(self, worker_id: str, epoch: float, error: str | None) -> None:
|
|
106
120
|
worker = self._worker(worker_id)
|
|
@@ -212,14 +212,17 @@ class MemoryRecord:
|
|
|
212
212
|
``peak`` the highest reading until the teardown report, ``after`` the reading at
|
|
213
213
|
that report. ``coverage`` says whether descendants were counted (``tree``) or
|
|
214
214
|
only the worker (``self``). A worker's heap rarely shrinks, so ``rise`` is
|
|
215
|
-
what the attempt needed on top of what was already there,
|
|
216
|
-
stayed resident afterwards
|
|
215
|
+
what the attempt needed on top of what was already there, ``retained`` what
|
|
216
|
+
stayed resident afterwards (a shared fixture it set up, or a leak), and
|
|
217
|
+
``transient`` the rest: what it needed beyond what stayed. ``wait`` is how long
|
|
218
|
+
memory admission held the attempt back before it started, in seconds.
|
|
217
219
|
"""
|
|
218
220
|
|
|
219
221
|
base: int = 0
|
|
220
222
|
peak: int = 0
|
|
221
223
|
after: int = 0
|
|
222
224
|
coverage: str = "none"
|
|
225
|
+
wait: float = 0.0
|
|
223
226
|
|
|
224
227
|
@property
|
|
225
228
|
def rise(self) -> int:
|
|
@@ -229,13 +232,25 @@ class MemoryRecord:
|
|
|
229
232
|
def retained(self) -> int:
|
|
230
233
|
return max(0, self.after - self.base)
|
|
231
234
|
|
|
235
|
+
@property
|
|
236
|
+
def transient(self) -> int:
|
|
237
|
+
return max(0, self.peak - self.after)
|
|
238
|
+
|
|
239
|
+
@property
|
|
240
|
+
def measured(self) -> bool:
|
|
241
|
+
"""Did the platform give a reading? A live process is never resident at zero."""
|
|
242
|
+
return self.peak > 0
|
|
243
|
+
|
|
232
244
|
def to_dict(self) -> dict[str, Any]:
|
|
233
|
-
|
|
245
|
+
doc: dict[str, Any] = {
|
|
234
246
|
"base": self.base,
|
|
235
247
|
"peak": self.peak,
|
|
236
248
|
"after": self.after,
|
|
237
249
|
"coverage": self.coverage,
|
|
238
250
|
}
|
|
251
|
+
if self.wait:
|
|
252
|
+
doc["wait"] = _round(self.wait)
|
|
253
|
+
return doc
|
|
239
254
|
|
|
240
255
|
@classmethod
|
|
241
256
|
def from_dict(cls, data: dict[str, Any]) -> MemoryRecord:
|
|
@@ -244,6 +259,7 @@ class MemoryRecord:
|
|
|
244
259
|
peak=int(data.get("peak", 0)),
|
|
245
260
|
after=int(data.get("after", 0)),
|
|
246
261
|
coverage=str(data.get("coverage", "none")),
|
|
262
|
+
wait=float(data.get("wait", 0.0)),
|
|
247
263
|
)
|
|
248
264
|
|
|
249
265
|
|
|
@@ -176,6 +176,18 @@ def format_bytes(size: float) -> str:
|
|
|
176
176
|
return f"{size:.1f} TiB"
|
|
177
177
|
|
|
178
178
|
|
|
179
|
+
def _waited(summary: dict[str, Any], what: str) -> str:
|
|
180
|
+
"""The waiting part of a gate's summary line: tests held back, workers parked."""
|
|
181
|
+
text = ""
|
|
182
|
+
if summary.get("waited_tests"):
|
|
183
|
+
n = summary["waited_tests"]
|
|
184
|
+
text += f"; {n} test{'s' if n != 1 else ''} waited {summary['waited']:.2f}s {what}"
|
|
185
|
+
if summary.get("parked_workers"):
|
|
186
|
+
n = summary["parked_workers"]
|
|
187
|
+
text += f"; {n} worker{'s' if n != 1 else ''} sat {summary['parked']:.2f}s parked {what}"
|
|
188
|
+
return text
|
|
189
|
+
|
|
190
|
+
|
|
179
191
|
class Settings:
|
|
180
192
|
"""Resolve values by CLI, environment, then ini; enable timing if any source asks."""
|
|
181
193
|
|
|
@@ -573,9 +585,7 @@ class TimingPlugin:
|
|
|
573
585
|
line += f", largest {format_bytes(summary['largest'])}"
|
|
574
586
|
else:
|
|
575
587
|
line += "; no recorded memory in the schedule history yet"
|
|
576
|
-
|
|
577
|
-
n = summary["waited_tests"]
|
|
578
|
-
line += f"; {n} test{'s' if n != 1 else ''} waited {summary['waited']:.2f}s for memory"
|
|
588
|
+
line += _waited(summary, "for memory")
|
|
579
589
|
return line
|
|
580
590
|
|
|
581
591
|
def _cpu_summary(self) -> str | None:
|
|
@@ -609,9 +619,7 @@ class TimingPlugin:
|
|
|
609
619
|
return None # no CPU budget: a memory budget alone has its own line
|
|
610
620
|
line = "cpu: " + "; ".join(parts)
|
|
611
621
|
line += f"; {heavy} test{'s' if heavy != 1 else ''} over one slot"
|
|
612
|
-
|
|
613
|
-
n = summary["waited_tests"]
|
|
614
|
-
line += f"; {n} test{'s' if n != 1 else ''} waited {summary['waited']:.2f}s for slots"
|
|
622
|
+
line += _waited(summary, "for slots")
|
|
615
623
|
if summary.get("cancelled"):
|
|
616
624
|
n = summary["cancelled"]
|
|
617
625
|
line += f"; {n} unadmitted test{'s' if n != 1 else ''} withdrawn at shutdown"
|
|
@@ -672,6 +680,7 @@ class TimingPlugin:
|
|
|
672
680
|
wait.index,
|
|
673
681
|
wait.attempt,
|
|
674
682
|
wait.seconds,
|
|
683
|
+
wait.gates,
|
|
675
684
|
)
|
|
676
685
|
self.collector.run.cpu = scheduler.cpu_summary()
|
|
677
686
|
self.collector.run.memory = scheduler.memory_summary()
|
|
@@ -122,8 +122,10 @@ def render_ascii(
|
|
|
122
122
|
bar = f"{_RED}{bar}{_RESET}"
|
|
123
123
|
duration = f"{format_seconds(test.duration):>7}"
|
|
124
124
|
lines.append(f"{test.worker:<{label_width}} {bar} {duration}")
|
|
125
|
-
|
|
126
|
-
|
|
125
|
+
held = _held(test)
|
|
126
|
+
room = width - label_width - 1 - (len(held) + 2 if held else 0)
|
|
127
|
+
nodeid = _fit_nodeid(test.nodeid, room, glyphs.ellipsis)
|
|
128
|
+
lines.append(f"{'':<{label_width}} {nodeid}{' ' + held if held else ''}")
|
|
127
129
|
return "\n".join(lines)
|
|
128
130
|
|
|
129
131
|
|
|
@@ -265,6 +267,16 @@ def _test_bar(test: TestSpan, scale: Scale, glyphs: Glyphs) -> str:
|
|
|
265
267
|
return "".join(row)
|
|
266
268
|
|
|
267
269
|
|
|
270
|
+
def _held(test: TestSpan) -> str:
|
|
271
|
+
"""How long admission held the test back before it started, and at which gate."""
|
|
272
|
+
parts = []
|
|
273
|
+
if test.cpu is not None and test.cpu.wait:
|
|
274
|
+
parts.append(f"waited {format_seconds(test.cpu.wait)} for cpu")
|
|
275
|
+
if test.memory is not None and test.memory.wait:
|
|
276
|
+
parts.append(f"waited {format_seconds(test.memory.wait)} for memory")
|
|
277
|
+
return ", ".join(parts)
|
|
278
|
+
|
|
279
|
+
|
|
268
280
|
def _fit_nodeid(nodeid: str, room: int, ellipsis: str) -> str:
|
|
269
281
|
if room >= len(nodeid) or room < 8 + len(ellipsis):
|
|
270
282
|
return nodeid
|
|
@@ -13,7 +13,8 @@ from pathlib import Path
|
|
|
13
13
|
from statistics import fmean, median
|
|
14
14
|
|
|
15
15
|
from pytest_timing.demand import Declarations, key_base
|
|
16
|
-
from pytest_timing.
|
|
16
|
+
from pytest_timing.fixtures import key_scope
|
|
17
|
+
from pytest_timing.model import Run, TestSpan
|
|
17
18
|
|
|
18
19
|
Family = frozenset[str]
|
|
19
20
|
NO_FIXTURES: Family = frozenset()
|
|
@@ -23,6 +24,29 @@ EPSILON = 1e-9
|
|
|
23
24
|
SPLIT_GAIN = 0.01
|
|
24
25
|
"""A family is split over more workers only for at least this much predicted gain:
|
|
25
26
|
duplicated set-ups are real cost, and gains this small are within estimate noise."""
|
|
27
|
+
BASELINE_SCOPES = frozenset({"session", "package"})
|
|
28
|
+
"""Scopes whose fixtures' memory is part of the worker's footprint, not a test's need:
|
|
29
|
+
every worker sets them up once and keeps them, so gating on them could only delay
|
|
30
|
+
the run, never spare the host. The budget's headroom is what covers them."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def is_baseline(key: str) -> bool:
|
|
34
|
+
"""Does the fixture behind ``key`` keep memory as a per-worker baseline?"""
|
|
35
|
+
return key_scope(key) in BASELINE_SCOPES
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _warm_ups(run: Run) -> set[int]:
|
|
39
|
+
"""Ids of the first attempt each worker ran.
|
|
40
|
+
|
|
41
|
+
Its memory window also covers the worker's warm-up (lazy imports, caches, the
|
|
42
|
+
allocator's first growth), which is neither the test's need nor a fixture's.
|
|
43
|
+
"""
|
|
44
|
+
first: dict[str, TestSpan] = {}
|
|
45
|
+
for test in run.tests:
|
|
46
|
+
current = first.get(test.worker)
|
|
47
|
+
if current is None or test.start < current.start:
|
|
48
|
+
first[test.worker] = test
|
|
49
|
+
return {id(test) for test in first.values()}
|
|
26
50
|
|
|
27
51
|
|
|
28
52
|
@cache
|
|
@@ -89,16 +113,21 @@ class Estimates:
|
|
|
89
113
|
does not grow with the number of attempts the way the longest one does, so a
|
|
90
114
|
history merged from many runs stays comparable to a single run.
|
|
91
115
|
|
|
92
|
-
Memory is the opposite: the largest
|
|
93
|
-
case is what an out-of-memory kill depends on.
|
|
94
|
-
|
|
95
|
-
when it paid for several at once,
|
|
116
|
+
Memory is the opposite: the largest need any attempt showed, since the worst
|
|
117
|
+
case is what an out-of-memory kill depends on. An attempt that paid for shared
|
|
118
|
+
set-ups needed its whole rise, but what stayed resident afterwards is the
|
|
119
|
+
fixtures' (split evenly when it paid for several at once) or, for session and
|
|
120
|
+
package scope, the worker's baseline; the test's own need is the rest
|
|
121
|
+
(``transient``). The first attempt on each worker also covers the worker's
|
|
122
|
+
warm-up, so its memory counts only for a test or fixture that has no other
|
|
123
|
+
attempt.
|
|
96
124
|
"""
|
|
97
125
|
attempts: dict[str, dict[int, list[float]]] = {} # nodeid -> rank -> own seconds
|
|
98
126
|
families: dict[str, set[str]] = {}
|
|
99
127
|
setups: dict[str, list[float]] = {}
|
|
100
|
-
memory: dict[str, int] = {}
|
|
101
|
-
retained: dict[str, int] = {}
|
|
128
|
+
memory: dict[bool, dict[str, int]] = {False: {}, True: {}} # warm-up? -> bytes
|
|
129
|
+
retained: dict[bool, dict[str, int]] = {False: {}, True: {}}
|
|
130
|
+
warm_ups = _warm_ups(run)
|
|
102
131
|
for test in run.tests:
|
|
103
132
|
paid = []
|
|
104
133
|
if test.fixtures:
|
|
@@ -107,12 +136,16 @@ class Estimates:
|
|
|
107
136
|
if seconds:
|
|
108
137
|
setups.setdefault(key, []).append(seconds)
|
|
109
138
|
paid.append(key)
|
|
110
|
-
if test.memory is not None:
|
|
111
|
-
|
|
139
|
+
if test.memory is not None and test.memory.measured:
|
|
140
|
+
warm = id(test) in warm_ups
|
|
141
|
+
rises, keeps = memory[warm], retained[warm]
|
|
142
|
+
need = test.memory.transient if paid else test.memory.rise
|
|
143
|
+
rises[test.nodeid] = max(rises.get(test.nodeid, 0), need)
|
|
112
144
|
if paid and test.memory.retained:
|
|
113
145
|
share = test.memory.retained // len(paid)
|
|
114
146
|
for key in paid:
|
|
115
|
-
|
|
147
|
+
if not is_baseline(key):
|
|
148
|
+
keeps[key] = max(keeps.get(key, 0), share)
|
|
116
149
|
if test.outcome == "crashed":
|
|
117
150
|
continue # the recorded stop is the worker's death, not the test's
|
|
118
151
|
own = test.duration - test.shared_setup
|
|
@@ -132,8 +165,8 @@ class Estimates:
|
|
|
132
165
|
source,
|
|
133
166
|
{nodeid: frozenset(keys) for nodeid, keys in families.items()},
|
|
134
167
|
{key: median(seconds) for key, seconds in setups.items()},
|
|
135
|
-
memory,
|
|
136
|
-
retained,
|
|
168
|
+
{**memory[True], **memory[False]}, # a clean attempt beats a warm-up one
|
|
169
|
+
{**retained[True], **retained[False]},
|
|
137
170
|
)
|
|
138
171
|
|
|
139
172
|
@classmethod
|
|
@@ -168,7 +201,10 @@ class Estimates:
|
|
|
168
201
|
return self.memory.get(nodeid, 0)
|
|
169
202
|
|
|
170
203
|
def kept(self, key: str) -> int:
|
|
171
|
-
"""Bytes a shared fixture instance keeps resident while alive; unknown is zero
|
|
204
|
+
"""Bytes a shared fixture instance keeps resident while alive; unknown is zero,
|
|
205
|
+
and so is a session or package fixture, whose memory is the worker's baseline."""
|
|
206
|
+
if is_baseline(key):
|
|
207
|
+
return 0
|
|
172
208
|
return self.retained.get(key, 0)
|
|
173
209
|
|
|
174
210
|
|
|
@@ -182,10 +218,11 @@ class Costs:
|
|
|
182
218
|
tests' families even when no run has timed it yet; one a recorded run reports a
|
|
183
219
|
test using without naming it (``getfixturevalue``) is matched by definition.
|
|
184
220
|
|
|
185
|
-
Memory comes from history alone: the bytes each test
|
|
186
|
-
|
|
187
|
-
its tests' families like a declared one, so the planner
|
|
188
|
-
lane that pays for it and the gate knows it is there.
|
|
221
|
+
Memory comes from history alone: the bytes each test needs of its own, and the
|
|
222
|
+
bytes each module- or class-scoped fixture instance keeps resident. A fixture kept
|
|
223
|
+
for its memory belongs to its tests' families like a declared one, so the planner
|
|
224
|
+
keeps it alive on the lane that pays for it and the gate knows it is there. What
|
|
225
|
+
session and package fixtures keep is every worker's baseline and is not charged.
|
|
189
226
|
"""
|
|
190
227
|
|
|
191
228
|
def __init__(
|
|
@@ -209,7 +209,18 @@ button.on { background: var(--link); color: #fff; border-color: var(--link); }
|
|
|
209
209
|
t.m = moduleIndex[t.module];
|
|
210
210
|
t.ph = {};
|
|
211
211
|
PHASES.forEach(function (p) { var v = t.phases && t.phases[p]; if (v) t.ph[p] = { start: v[0], stop: v[1], dur: v[2] }; });
|
|
212
|
+
// Admission waits: how long the controller held the test back before it started, per gate.
|
|
213
|
+
t.cpuWait = (t.cpu && t.cpu.wait) || 0;
|
|
214
|
+
t.memWait = (t.memory && t.memory.wait) || 0;
|
|
215
|
+
t.waited = t.cpuWait + t.memWait;
|
|
212
216
|
});
|
|
217
|
+
var ANY_WAITED = TESTS.some(function (t) { return t.waited > 0; });
|
|
218
|
+
function heldText(t) {
|
|
219
|
+
var parts = [];
|
|
220
|
+
if (t.cpuWait) parts.push(fmt(t.cpuWait) + ' for cpu');
|
|
221
|
+
if (t.memWait) parts.push(fmt(t.memWait) + ' for memory');
|
|
222
|
+
return parts.join(', ');
|
|
223
|
+
}
|
|
213
224
|
var laneIds = WORKERS.map(function (w) { return w.id; }), laneIndex = {};
|
|
214
225
|
laneIds.forEach(function (id, i) { laneIndex[id] = i; });
|
|
215
226
|
TESTS.forEach(function (t) { if (!(t.worker in laneIndex)) { laneIndex[t.worker] = laneIds.length; laneIds.push(t.worker); } });
|
|
@@ -470,6 +481,7 @@ button.on { background: var(--link); color: #fff; border-color: var(--link); }
|
|
|
470
481
|
['idx', '#'], ['nodeid', 'Test'], ['worker', 'Worker'], ['dur', 'Total'],
|
|
471
482
|
['setup', 'Setup'], ['call', 'Call'], ['teardown', 'Teardown'], ['outcome', 'Outcome'], ['start', 'Start']
|
|
472
483
|
];
|
|
484
|
+
if (ANY_WAITED) COLS.push(['waited', 'Held']);
|
|
473
485
|
function sortVal(t, key) {
|
|
474
486
|
if (key === 'idx') return t.i;
|
|
475
487
|
if (PHASES.indexOf(key) >= 0) return t.ph[key] ? t.ph[key].dur : 0;
|
|
@@ -485,9 +497,10 @@ button.on { background: var(--link); color: #fff; border-color: var(--link); }
|
|
|
485
497
|
rows.push('<tr><td class="num">' + (n + 1) + '</td><td class="nodeid mono">' + esc(t.nodeid) + (t.attempt ? ' <span class="muted">(attempt ' + (t.attempt + 1) + ')</span>' : '') + '</td><td class="mono">' + esc(t.worker) + '</td>' +
|
|
486
498
|
'<td class="num">' + fmt(t.dur) + '</td>' +
|
|
487
499
|
PHASES.map(function (p) { var ph = t.ph[p]; return '<td class="num">' + (ph ? fmt(ph.dur) + pct(ph.dur, t.dur) : '-') + '</td>'; }).join('') +
|
|
488
|
-
'<td class="o-' + t.outcome + '">' + esc(t.outcome) + '</td><td class="num">' + fmt(t.start) + '</td
|
|
500
|
+
'<td class="o-' + t.outcome + '">' + esc(t.outcome) + '</td><td class="num">' + fmt(t.start) + '</td>' +
|
|
501
|
+
(ANY_WAITED ? '<td class="num" title="' + esc(heldText(t)) + '">' + (t.waited ? fmt(t.waited) : '-') + '</td>' : '') + '</tr>');
|
|
489
502
|
});
|
|
490
|
-
document.querySelector('#tests-table tbody').innerHTML = rows.join('') || '<tr><td colspan="
|
|
503
|
+
document.querySelector('#tests-table tbody').innerHTML = rows.join('') || '<tr><td colspan="' + COLS.length + '" class="empty">No tests match the current filters.</td></tr>';
|
|
491
504
|
document.getElementById('table-note').textContent = tests.length > TABLE_CAP ? '(showing ' + TABLE_CAP + ' of ' + tests.length + ')' : '(' + tests.length + ' of ' + TESTS.length + ')';
|
|
492
505
|
}
|
|
493
506
|
|
|
@@ -528,7 +541,7 @@ button.on { background: var(--link); color: #fff; border-color: var(--link); }
|
|
|
528
541
|
}
|
|
529
542
|
function tipForTest(t) {
|
|
530
543
|
return '<b>' + esc(t.nodeid) + '</b><br>' + esc(t.outcome) + ' on ' + esc(t.worker) + (t.attempt ? ' (attempt ' + (t.attempt + 1) + ')' : '') +
|
|
531
|
-
'<br>total ' + fmt(t.dur) + ', started at ' + fmt(t.start) + phaseRows(t);
|
|
544
|
+
'<br>total ' + fmt(t.dur) + ', started at ' + fmt(t.start) + (t.waited ? '<br>held ' + esc(heldText(t)) : '') + phaseRows(t);
|
|
532
545
|
}
|
|
533
546
|
function tipForCluster(c) {
|
|
534
547
|
var top = c.tests.slice().sort(function (a, b) { return b.dur - a.dur; }).slice(0, 10);
|
|
@@ -74,9 +74,13 @@ class CpuSetup:
|
|
|
74
74
|
@dataclass(frozen=True)
|
|
75
75
|
class AdmissionWait:
|
|
76
76
|
worker: str
|
|
77
|
-
index: int
|
|
77
|
+
index: int # -1 for a worker that was parked and never ran what it waited for
|
|
78
78
|
attempt: int
|
|
79
79
|
seconds: float
|
|
80
|
+
gates: frozenset[str] = frozenset() # the gate kinds that refused: cpu, memory
|
|
81
|
+
|
|
82
|
+
def held_by(self, kind: str) -> bool:
|
|
83
|
+
return kind in self.gates or not self.gates
|
|
80
84
|
|
|
81
85
|
|
|
82
86
|
@dataclass(frozen=True)
|
|
@@ -127,7 +131,9 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
127
131
|
self.domain: dict[WorkerController, str] = {}
|
|
128
132
|
self.hosts: dict[str, dict[str, Any]] = {} # what each domain's workers detected
|
|
129
133
|
self.waiting: dict[WorkerController, float] = {} # since when its head waits
|
|
134
|
+
self.refused: dict[WorkerController, set[str]] = {} # which gate kinds held it
|
|
130
135
|
self.waits: list[AdmissionWait] = []
|
|
136
|
+
self.parked: list[AdmissionWait] = [] # waits of workers that then ran nothing
|
|
131
137
|
self.rates: dict[str, float] = {} # last measured CPU rate per worker id
|
|
132
138
|
self.last_sample = -math.inf
|
|
133
139
|
self.cancelled: dict[WorkerController, set[int]] = {} # withdrawn, never run
|
|
@@ -251,7 +257,7 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
251
257
|
queued = self.node2pending.pop(node)
|
|
252
258
|
lane = self.lanes.pop(node)
|
|
253
259
|
self.dispatched.pop(node, None)
|
|
254
|
-
self._release_runtime(node)
|
|
260
|
+
self._release_runtime(node, idle=not queued)
|
|
255
261
|
self.cancelled.pop(node, None)
|
|
256
262
|
self.finished.discard(node)
|
|
257
263
|
self.domain.pop(node, None)
|
|
@@ -282,13 +288,20 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
282
288
|
if node not in self.node2pending:
|
|
283
289
|
return
|
|
284
290
|
self.finished.add(node)
|
|
285
|
-
self._release_runtime(node)
|
|
291
|
+
self._release_runtime(node, idle=not self.node2pending[node])
|
|
286
292
|
self._admit_waiting()
|
|
287
293
|
|
|
288
|
-
def _release_runtime(self, node: WorkerController) -> None:
|
|
294
|
+
def _release_runtime(self, node: WorkerController, idle: bool = False) -> None:
|
|
295
|
+
"""Forget ``node``'s run-time state. ``idle`` says it held no test: a wait
|
|
296
|
+
it was in is then a parked worker's, with no test to record it on."""
|
|
289
297
|
self.requests.pop(node, None)
|
|
290
298
|
self.live_holds.pop(node, None)
|
|
291
|
-
self.waiting.pop(node, None)
|
|
299
|
+
since = self.waiting.pop(node, None)
|
|
300
|
+
gates = frozenset(self.refused.pop(node, ()))
|
|
301
|
+
if since is not None and idle and self.clock() - since > 0:
|
|
302
|
+
self.parked.append(
|
|
303
|
+
AdmissionWait(xdist_compat.node_id(node), -1, 0, self.clock() - since, gates)
|
|
304
|
+
)
|
|
292
305
|
self.rates.pop(xdist_compat.node_id(node), None)
|
|
293
306
|
for admission in self._gates(node):
|
|
294
307
|
admission.release(node)
|
|
@@ -380,19 +393,25 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
380
393
|
if not transfer(costs, self._live_lanes(), lane, self._slots()):
|
|
381
394
|
break
|
|
382
395
|
position = charge = None
|
|
396
|
+
blocked: set[str] = set() # the gate kinds that passed tests over
|
|
383
397
|
for i, index in enumerate(lane.plan):
|
|
384
398
|
charge = costs.charge(index, lane.fixtures)
|
|
385
399
|
# Could it never run here while the other workers keep what they
|
|
386
400
|
# hold now? Only their exit would make room: pass it over.
|
|
387
|
-
if
|
|
388
|
-
|
|
389
|
-
|
|
401
|
+
if elsewhere and min(charge.peak, limit_slots) + elsewhere > limit_slots:
|
|
402
|
+
blocked.add("cpu")
|
|
403
|
+
elif (
|
|
404
|
+
kept_elsewhere
|
|
405
|
+
and min(charge.memory, limit_bytes) + kept_elsewhere > limit_bytes
|
|
390
406
|
):
|
|
407
|
+
blocked.add("memory")
|
|
408
|
+
else:
|
|
391
409
|
position = i
|
|
392
410
|
break
|
|
393
411
|
if position is None or charge is None:
|
|
394
412
|
if not queue and not out:
|
|
395
413
|
self.waiting.setdefault(node, self.clock()) # parked
|
|
414
|
+
self.refused.setdefault(node, set()).update(blocked)
|
|
396
415
|
break
|
|
397
416
|
if queue and not self._grant(node):
|
|
398
417
|
break
|
|
@@ -458,7 +477,7 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
458
477
|
]
|
|
459
478
|
size = int(min(offered) * AUTO_MEMORY_SHARE) if offered else 0
|
|
460
479
|
if size > 0:
|
|
461
|
-
self.memory_admissions[domain] = Admission(size, domain)
|
|
480
|
+
self.memory_admissions[domain] = Admission(size, domain, kind="memory")
|
|
462
481
|
|
|
463
482
|
@property
|
|
464
483
|
def gated(self) -> bool:
|
|
@@ -602,9 +621,11 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
602
621
|
assert needs
|
|
603
622
|
lane = self.lanes[node]
|
|
604
623
|
rising = [(gate, need) for gate, need in needs if need > gate.held(node)]
|
|
605
|
-
if not
|
|
624
|
+
refusing = [gate.kind for gate, need in rising if not gate.fits(node, need, lane.free)]
|
|
625
|
+
if refusing:
|
|
606
626
|
if wait:
|
|
607
627
|
since = self.waiting.setdefault(node, self.clock())
|
|
628
|
+
self.refused.setdefault(node, set()).update(refusing)
|
|
608
629
|
for gate, need in needs:
|
|
609
630
|
gate.wait(node, need, since)
|
|
610
631
|
return False
|
|
@@ -637,6 +658,7 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
637
658
|
for gate in self._gates(node):
|
|
638
659
|
gate.admitted(node)
|
|
639
660
|
since = self.waiting.pop(node, None)
|
|
661
|
+
gates = frozenset(self.refused.pop(node, ()))
|
|
640
662
|
if since is not None:
|
|
641
663
|
queue = self.node2pending[node]
|
|
642
664
|
waited = self.clock() - since
|
|
@@ -644,7 +666,9 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
644
666
|
request = self.requests.get(node)
|
|
645
667
|
index = request.index if request is not None else queue[0]
|
|
646
668
|
attempt = request.attempt if request is not None else 0
|
|
647
|
-
self.waits.append(
|
|
669
|
+
self.waits.append(
|
|
670
|
+
AdmissionWait(xdist_compat.node_id(node), index, attempt, waited, gates)
|
|
671
|
+
)
|
|
648
672
|
|
|
649
673
|
def _reserve(self, node: WorkerController) -> None:
|
|
650
674
|
"""Recompute ``node``'s reservation from what it may run without another word."""
|
|
@@ -703,6 +727,7 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
703
727
|
self.cancelled.setdefault(node, set()).add(tail)
|
|
704
728
|
self.withdrawn += 1
|
|
705
729
|
self.waiting.pop(node, None) # it never ran: no wait to record
|
|
730
|
+
self.refused.pop(node, None)
|
|
706
731
|
for gate, need in self._needs(node, queue):
|
|
707
732
|
gate.admitted(node)
|
|
708
733
|
gate.assign(node, need, lane.free, busy=len(queue) > 1)
|
|
@@ -951,11 +976,22 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
951
976
|
"gated": self.gated,
|
|
952
977
|
"domains": domains,
|
|
953
978
|
"heavy_tests": heavy,
|
|
954
|
-
"
|
|
955
|
-
"waited_tests": len({(w.worker, w.index, w.attempt) for w in self.waits}),
|
|
979
|
+
**self._waits("cpu"),
|
|
956
980
|
"cancelled": self.withdrawn,
|
|
957
981
|
}
|
|
958
982
|
|
|
983
|
+
def _waits(self, kind: str) -> dict[str, Any]:
|
|
984
|
+
"""How long tests waited at the gates of ``kind``, and how long workers sat
|
|
985
|
+
parked at them without ever running what they waited for."""
|
|
986
|
+
waits = [w for w in self.waits if w.held_by(kind)]
|
|
987
|
+
parked = [w for w in self.parked if w.held_by(kind)]
|
|
988
|
+
return {
|
|
989
|
+
"waited": round(sum(w.seconds for w in waits), 6),
|
|
990
|
+
"waited_tests": len({(w.worker, w.index, w.attempt) for w in waits}),
|
|
991
|
+
"parked": round(sum(w.seconds for w in parked), 6),
|
|
992
|
+
"parked_workers": len({w.worker for w in parked}),
|
|
993
|
+
}
|
|
994
|
+
|
|
959
995
|
def memory_summary(self) -> dict[str, Any] | None:
|
|
960
996
|
"""What happened on the memory side, for the run's metadata and the summary."""
|
|
961
997
|
if self.cpu is None or self.cpu.memory is None:
|
|
@@ -977,8 +1013,7 @@ class DurationScheduling(LoadScheduling): # type: ignore[misc] # xdist ships n
|
|
|
977
1013
|
"tests": total,
|
|
978
1014
|
"known_tests": known,
|
|
979
1015
|
"largest": largest,
|
|
980
|
-
"
|
|
981
|
-
"waited_tests": len({(w.worker, w.index, w.attempt) for w in self.waits}),
|
|
1016
|
+
**self._waits("memory"),
|
|
982
1017
|
}
|
|
983
1018
|
|
|
984
1019
|
def _check_stealing(self, node: WorkerController) -> None:
|
|
@@ -56,6 +56,19 @@ def test_min_duration_filters_slowest(sample_run: Run) -> None:
|
|
|
56
56
|
assert "test_two" in out and "test_four" not in out
|
|
57
57
|
|
|
58
58
|
|
|
59
|
+
def test_slowest_rows_say_how_long_admission_held_a_test(sample_run: Run) -> None:
|
|
60
|
+
from pytest_timing.model import CpuRecord, MemoryRecord
|
|
61
|
+
|
|
62
|
+
two = next(t for t in sample_run.tests if t.nodeid.endswith("test_two"))
|
|
63
|
+
two.memory = MemoryRecord(base=1, peak=2, after=1, coverage="self", wait=0.3)
|
|
64
|
+
two.cpu = CpuRecord(elapsed=1.0, wait=1.25)
|
|
65
|
+
out = render_ascii(sample_run, width=100, top=2)
|
|
66
|
+
assert "tests/test_a.py::test_two waited 1.25s for cpu, waited 0.30s for memory" in out
|
|
67
|
+
for line in out.splitlines():
|
|
68
|
+
assert len(line) <= 100, line
|
|
69
|
+
assert "waited" not in render_ascii(sample_run, width=100, top=0)
|
|
70
|
+
|
|
71
|
+
|
|
59
72
|
def test_color_only_wraps_glyphs(sample_run: Run) -> None:
|
|
60
73
|
plain = render_ascii(sample_run, width=80)
|
|
61
74
|
colored = render_ascii(sample_run, width=80, color=True)
|
|
@@ -27,9 +27,21 @@ def test_waits_use_execution_identity_and_accumulate_across_requests() -> None:
|
|
|
27
27
|
collector.add_wait("gw0", "test.py::test_same", 9, 0, 0.4)
|
|
28
28
|
collector.add_wait("gw1", "test.py::test_same", 4, 0, 0.6)
|
|
29
29
|
collector.add_wait("gw0", "test.py::test_same", 100, 0, 9) # never guess another execution
|
|
30
|
+
collector.add_wait("gw1", "test.py::test_same", 4, 0, 0.7, gates={"memory"})
|
|
31
|
+
collector.add_wait("gw1", "test.py::test_same", 4, 0, 0.2, gates={"cpu", "memory"})
|
|
30
32
|
run = collector.finish(T0 + 5, termination="finished")
|
|
31
|
-
assert [t.cpu.wait for t in run.tests if t.cpu] == pytest.approx([0.5, 0.1, 0.4, 0.
|
|
33
|
+
assert [t.cpu.wait for t in run.tests if t.cpu] == pytest.approx([0.5, 0.1, 0.4, 0.8])
|
|
32
34
|
assert [(t.occurrence, t.attempt) for t in run.tests] == [(0, 0), (0, 1), (1, 0), (0, 0)]
|
|
35
|
+
# A wait at the memory gate goes on the memory record, even without a reading.
|
|
36
|
+
assert [t.memory.wait for t in run.tests if t.memory] == pytest.approx([0.9])
|
|
37
|
+
assert run.tests[-1].memory is not None and not run.tests[-1].memory.measured
|
|
38
|
+
assert run.tests[-1].to_dict()["memory"] == {
|
|
39
|
+
"base": 0,
|
|
40
|
+
"peak": 0,
|
|
41
|
+
"after": 0,
|
|
42
|
+
"coverage": "none",
|
|
43
|
+
"wait": 0.9,
|
|
44
|
+
}
|
|
33
45
|
|
|
34
46
|
|
|
35
47
|
def test_phases_fold_into_one_span() -> None:
|
|
@@ -63,6 +63,21 @@ def test_report_executes_in_a_browser(sample_run: Run, tmp_path: Path) -> None:
|
|
|
63
63
|
assert "Ended:" in dom and "finished" in dom
|
|
64
64
|
|
|
65
65
|
|
|
66
|
+
@needs_chrome
|
|
67
|
+
def test_report_shows_how_long_admission_held_a_test(sample_run: Run, tmp_path: Path) -> None:
|
|
68
|
+
from pytest_timing.model import CpuRecord, MemoryRecord
|
|
69
|
+
|
|
70
|
+
dom = _chrome_dom(sample_run, tmp_path)
|
|
71
|
+
assert "Held</th>" not in dom # nothing waited: no column
|
|
72
|
+
two = next(t for t in sample_run.tests if t.nodeid.endswith("test_two"))
|
|
73
|
+
two.memory = MemoryRecord(base=1, peak=2, after=1, coverage="self", wait=0.3)
|
|
74
|
+
two.cpu = CpuRecord(elapsed=1.0, wait=1.25)
|
|
75
|
+
dom = _chrome_dom(sample_run, tmp_path)
|
|
76
|
+
assert "Held</th>" in dom
|
|
77
|
+
assert 'title="1.25s for cpu, 300.0ms for memory">1.55s</td>' in dom
|
|
78
|
+
assert dom.count('title="">-</td>') == len(sample_run.tests) - 1
|
|
79
|
+
|
|
80
|
+
|
|
66
81
|
@needs_chrome
|
|
67
82
|
def test_large_report_executes_in_a_browser(tmp_path: Path) -> None:
|
|
68
83
|
"""150k tests on one lane: no argument-limit errors, merged bars, capped table."""
|
|
@@ -15,45 +15,100 @@ from pytest_timing.schedule import Costs, Estimates
|
|
|
15
15
|
|
|
16
16
|
MIB = 1024 * 1024
|
|
17
17
|
SESSION = "session::/repo/conftest.py::model"
|
|
18
|
+
PACKAGE = "package:tests:/repo/tests/conftest.py::cache"
|
|
19
|
+
MODULE = "module:t.py:/repo/t.py::db"
|
|
18
20
|
|
|
19
21
|
|
|
20
|
-
def
|
|
22
|
+
def test_estimates_take_the_largest_need_and_attribute_retained_memory_to_fixtures() -> None:
|
|
21
23
|
c = Collector(make_info())
|
|
22
|
-
full_test(c, "t.py::
|
|
23
|
-
c.tests[-1].memory = MemoryRecord(
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
full_test(c, "t.py::
|
|
27
|
-
c.tests[-1].
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
24
|
+
full_test(c, "t.py::warm", 0.0) # first on gw0: its window covers the warm-up
|
|
25
|
+
c.tests[-1].memory = MemoryRecord(
|
|
26
|
+
base=100 * MIB, peak=900 * MIB, after=800 * MIB, coverage="self"
|
|
27
|
+
)
|
|
28
|
+
full_test(c, "t.py::a", 1.0)
|
|
29
|
+
c.tests[-1].memory = MemoryRecord(
|
|
30
|
+
base=100 * MIB, peak=300 * MIB, after=110 * MIB, coverage="self"
|
|
31
|
+
)
|
|
32
|
+
full_test(c, "t.py::a", 2.0) # a second attempt that needed more
|
|
33
|
+
c.tests[-1].memory = MemoryRecord(
|
|
34
|
+
base=110 * MIB, peak=360 * MIB, after=110 * MIB, coverage="self"
|
|
35
|
+
)
|
|
36
|
+
full_test(c, "t.py::b", 3.0) # paid two set-ups and kept 400 MiB: 200 each
|
|
37
|
+
c.tests[-1].fixtures = {SESSION: 0.5, MODULE: 0.2}
|
|
38
|
+
c.tests[-1].memory = MemoryRecord(
|
|
39
|
+
base=100 * MIB, peak=600 * MIB, after=500 * MIB, coverage="self"
|
|
40
|
+
)
|
|
41
|
+
full_test(c, "t.py::c", 4.0) # no record at all
|
|
42
|
+
full_test(c, "t.py::d", 5.0) # a wait, but no reading
|
|
43
|
+
c.tests[-1].memory = MemoryRecord(wait=0.5)
|
|
44
|
+
run = c.finish(make_info().start + 6, termination="finished")
|
|
31
45
|
est = Estimates.from_run(run)
|
|
32
46
|
assert est.rise("t.py::a") == 250 * MIB
|
|
33
|
-
|
|
47
|
+
# The payer needed its whole rise, but what stayed is the fixtures' (or the
|
|
48
|
+
# worker's baseline); its own need is what it used beyond that.
|
|
49
|
+
assert est.rise("t.py::b") == 100 * MIB
|
|
34
50
|
assert est.rise("t.py::c") == 0 and "t.py::c" not in est.memory
|
|
35
|
-
assert est.
|
|
36
|
-
assert est.kept(
|
|
51
|
+
assert est.rise("t.py::d") == 0 and "t.py::d" not in est.memory
|
|
52
|
+
assert est.kept(SESSION) == 0 and SESSION not in est.retained # baseline, not a need
|
|
53
|
+
assert est.kept(MODULE) == 200 * MIB
|
|
37
54
|
assert est.kept("nothing") == 0
|
|
55
|
+
# The warm-up window counts only while nothing better is known.
|
|
56
|
+
assert est.rise("t.py::warm") == 800 * MIB
|
|
57
|
+
full_test(c, "t.py::warm", 6.0, worker="gw1") # not the first on its worker
|
|
58
|
+
c.tests[-1].memory = MemoryRecord(
|
|
59
|
+
base=100 * MIB, peak=150 * MIB, after=100 * MIB, coverage="self"
|
|
60
|
+
)
|
|
61
|
+
full_test(c, "t.py::e", 5.0, worker="gw1") # gw1's warm-up: earlier, if reported later
|
|
62
|
+
c.tests[-1].memory = MemoryRecord(
|
|
63
|
+
base=100 * MIB, peak=700 * MIB, after=600 * MIB, coverage="self"
|
|
64
|
+
)
|
|
65
|
+
est = Estimates.from_run(c.finish(make_info().start + 8, termination="finished"))
|
|
66
|
+
assert est.rise("t.py::warm") == 50 * MIB
|
|
67
|
+
assert est.rise("t.py::e") == 600 * MIB
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_session_and_package_fixture_memory_is_the_workers_baseline() -> None:
|
|
71
|
+
c = Collector(make_info())
|
|
72
|
+
full_test(c, "t.py::a", 0.0)
|
|
73
|
+
full_test(c, "t.py::b", 1.0) # paid session, package and module set-ups: 300 each
|
|
74
|
+
c.tests[-1].fixtures = {SESSION: 0.5, PACKAGE: 0.3, MODULE: 0.2}
|
|
75
|
+
c.tests[-1].memory = MemoryRecord(
|
|
76
|
+
base=100 * MIB, peak=1100 * MIB, after=1000 * MIB, coverage="self"
|
|
77
|
+
)
|
|
78
|
+
est = Estimates.from_run(c.finish(make_info().start + 2, termination="finished"))
|
|
79
|
+
assert est.retained == {MODULE: 300 * MIB}
|
|
80
|
+
assert est.rise("t.py::b") == 100 * MIB
|
|
81
|
+
# Even hand-written estimates never charge them, and they do not make families.
|
|
82
|
+
est = Estimates(
|
|
83
|
+
{"t.py::a": 1.0, "t.py::b": 1.0},
|
|
84
|
+
families={"t.py::a": frozenset({SESSION, PACKAGE})},
|
|
85
|
+
setups={SESSION: 0.0, PACKAGE: 0.0},
|
|
86
|
+
retained={SESSION: 4096 * MIB, PACKAGE: 1024 * MIB},
|
|
87
|
+
)
|
|
88
|
+
assert est.kept(SESSION) == 0 and est.kept(PACKAGE) == 0
|
|
89
|
+
costs = Costs(est, ["t.py::a", "t.py::b"])
|
|
90
|
+
assert not costs.retained and not costs.any_memory
|
|
91
|
+
assert costs.family[0] == frozenset()
|
|
92
|
+
assert costs.charge(0, set()).memory == 0
|
|
38
93
|
|
|
39
94
|
|
|
40
|
-
def
|
|
95
|
+
def test_costs_charge_a_test_with_its_need_and_the_fixtures_alive_around_it() -> None:
|
|
41
96
|
ids = ["t.py::a", "t.py::b", "u.py::c"]
|
|
42
97
|
est = Estimates(
|
|
43
98
|
dict.fromkeys(ids, 1.0),
|
|
44
|
-
families={"t.py::a": frozenset({
|
|
45
|
-
setups={
|
|
99
|
+
families={"t.py::a": frozenset({MODULE}), "t.py::b": frozenset({MODULE})},
|
|
100
|
+
setups={MODULE: 0.0}, # too quick to matter for time, kept for its memory
|
|
46
101
|
memory={"t.py::a": 50 * MIB, "u.py::c": 10 * MIB},
|
|
47
|
-
retained={
|
|
102
|
+
retained={MODULE: 300 * MIB},
|
|
48
103
|
)
|
|
49
104
|
costs = Costs(est, ids, Declarations())
|
|
50
|
-
assert costs.family[0] == frozenset({
|
|
51
|
-
first = costs.charge(0, set()) # sets the fixture up:
|
|
52
|
-
assert first.memory == 350 * MIB and
|
|
53
|
-
second = costs.charge(1, first.after) # no
|
|
105
|
+
assert costs.family[0] == frozenset({MODULE}) and costs.any_memory
|
|
106
|
+
first = costs.charge(0, set()) # sets the fixture up: its need plus what it will keep
|
|
107
|
+
assert first.memory == 350 * MIB and MODULE in first.after
|
|
108
|
+
second = costs.charge(1, first.after) # no need of its own, but the fixture is alive
|
|
54
109
|
assert second.memory == 300 * MIB
|
|
55
|
-
third = costs.charge(2, first.after) #
|
|
56
|
-
assert third.memory ==
|
|
110
|
+
third = costs.charge(2, first.after) # a module fixture is gone in another module
|
|
111
|
+
assert third.memory == 10 * MIB and MODULE not in third.after
|
|
57
112
|
assert costs.kept_memory(first.after) == 300 * MIB
|
|
58
113
|
assert not Costs(Estimates(dict.fromkeys(ids, 1.0)), ids).any_memory
|
|
59
114
|
|
|
@@ -121,6 +176,9 @@ def test_hungry_tests_take_turns_under_a_memory_budget_alone() -> None:
|
|
|
121
176
|
peak = run_checking_memory(sched, nodes, {i: est.estimate(ids[i]) for i in range(6)})
|
|
122
177
|
assert peak <= 1000 * MIB and peak >= 700 * MIB
|
|
123
178
|
assert sched.waits, "one hungry test had to wait for the other"
|
|
179
|
+
assert all(w.gates == {"memory"} for w in sched.waits) # and it was memory that held it
|
|
180
|
+
assert sched.memory_summary()["waited_tests"] == 1
|
|
181
|
+
assert sched.cpu_summary()["waited_tests"] == 0 and not sched.parked
|
|
124
182
|
assert sched.now >= 4.0 # the hungry tests ran one after the other
|
|
125
183
|
|
|
126
184
|
|
|
@@ -148,15 +206,15 @@ def test_a_fixture_that_keeps_memory_counts_against_tests_on_other_workers() ->
|
|
|
148
206
|
ids = ["f1", "f2", "f3", "big"]
|
|
149
207
|
est = Estimates(
|
|
150
208
|
dict.fromkeys(ids, 1.0),
|
|
151
|
-
families={f: frozenset({
|
|
152
|
-
setups={
|
|
209
|
+
families={f: frozenset({MODULE}) for f in ("f1", "f2", "f3")},
|
|
210
|
+
setups={MODULE: 0.5},
|
|
153
211
|
memory={"big": 600 * MIB},
|
|
154
|
-
retained={
|
|
212
|
+
retained={MODULE: 500 * MIB},
|
|
155
213
|
)
|
|
156
214
|
sched = memory_scheduler(est, ids, budget=1000 * MIB)
|
|
157
215
|
nodes = start(sched, ids)
|
|
158
216
|
mem = sched.memory_admissions["local"]
|
|
159
|
-
holder = next(n for n in nodes if
|
|
217
|
+
holder = next(n for n in nodes if MODULE in sched.lanes[n].fixtures)
|
|
160
218
|
other = next(n for n in nodes if n is not holder)
|
|
161
219
|
assert mem.held(holder) >= 500 * MIB
|
|
162
220
|
# ``big`` cannot run next to the fixture: it is parked, or waits, until the
|
|
@@ -165,6 +223,46 @@ def test_a_fixture_that_keeps_memory_counts_against_tests_on_other_workers() ->
|
|
|
165
223
|
other in sched.waiting
|
|
166
224
|
)
|
|
167
225
|
run_checking_memory(sched, nodes, dict.fromkeys(range(4), 1.0))
|
|
226
|
+
assert all(w.gates == {"memory"} for w in sched.waits) and not sched.parked
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@needs_xdist
|
|
230
|
+
def test_a_parked_worker_that_never_runs_keeps_its_wait_in_the_summary() -> None:
|
|
231
|
+
ids = ["f1", "f2", "f3", "f4", "big"]
|
|
232
|
+
est = Estimates(
|
|
233
|
+
dict.fromkeys(ids, 1.0),
|
|
234
|
+
families={f: frozenset({MODULE}) for f in ids[:4]},
|
|
235
|
+
setups={MODULE: 0.5},
|
|
236
|
+
memory={"big": 600 * MIB},
|
|
237
|
+
retained={MODULE: 500 * MIB},
|
|
238
|
+
)
|
|
239
|
+
sched = memory_scheduler(est, ids, workers=3, budget=1000 * MIB)
|
|
240
|
+
nodes = start(sched, ids, workers=3)
|
|
241
|
+
# The fixture family is split over two lanes; ``big`` cannot start next to what
|
|
242
|
+
# they both keep, so the third worker sits parked with it, undispatched.
|
|
243
|
+
parked = [n for n in nodes if n in sched.waiting and not sched.node2pending[n]]
|
|
244
|
+
assert len(parked) == 1 and sched.refused[parked[0]] == {"memory"}
|
|
245
|
+
assert sched.lanes[parked[0]].plan == [ids.index("big")]
|
|
246
|
+
sched.now += 3.0
|
|
247
|
+
sched.remove_node(parked[0]) # gone without a test: nothing to put the wait on
|
|
248
|
+
assert [(w.index, w.seconds, w.gates) for w in sched.parked] == [
|
|
249
|
+
(-1, 3.0, frozenset({"memory"}))
|
|
250
|
+
]
|
|
251
|
+
summary = sched.memory_summary()
|
|
252
|
+
assert summary["parked_workers"] == 1 and summary["parked"] == 3.0
|
|
253
|
+
assert summary["waited_tests"] == 0
|
|
254
|
+
assert sched.cpu_summary()["parked_workers"] == 0
|
|
255
|
+
assert parked[0] not in sched.waiting and parked[0] not in sched.refused
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def test_summary_lines_say_who_waited_and_who_sat_parked() -> None:
|
|
259
|
+
from pytest_timing.plugin import _waited
|
|
260
|
+
|
|
261
|
+
assert _waited({"waited_tests": 0, "parked_workers": 0}, "for memory") == ""
|
|
262
|
+
assert (
|
|
263
|
+
_waited({"waited_tests": 2, "waited": 1.5, "parked_workers": 1, "parked": 3}, "for memory")
|
|
264
|
+
== "; 2 tests waited 1.50s for memory; 1 worker sat 3.00s parked for memory"
|
|
265
|
+
)
|
|
168
266
|
|
|
169
267
|
|
|
170
268
|
@needs_xdist
|
|
@@ -248,7 +346,13 @@ def test_the_second_run_keeps_hungry_tests_apart(pytester: pytest.Pytester, dist
|
|
|
248
346
|
assert memory["domains"]["local"]["budget"] == 300 * MIB
|
|
249
347
|
assert memory["domains"]["local"]["forced"] == 0
|
|
250
348
|
assert memory["known_tests"] == 5 and memory["largest"] >= 190 * MIB
|
|
251
|
-
assert memory["waited_tests"] >= 1
|
|
349
|
+
assert memory["waited_tests"] >= 1 and memory["parked_workers"] == 0
|
|
350
|
+
# The wait is on the test that was held, marked as memory's, and the chart says so.
|
|
351
|
+
held = [t for t in doc["tests"] if t.get("memory", {}).get("wait")]
|
|
352
|
+
assert [t["nodeid"].split("::")[-1] for t in held] in (["test_hungry_a"], ["test_hungry_b"])
|
|
353
|
+
assert "wait" not in held[0].get("cpu", {})
|
|
354
|
+
assert sum(t["memory"]["wait"] for t in held) == pytest.approx(memory["waited"], abs=1e-3)
|
|
355
|
+
second.stdout.fnmatch_lines([f"*{held[0]['nodeid']} waited * for memory"])
|
|
252
356
|
|
|
253
357
|
|
|
254
358
|
@needs_xdist
|
|
@@ -693,11 +693,15 @@ def test_repeated_timeouts_keep_waits_with_the_attempt_that_waited(
|
|
|
693
693
|
@pytest_timing.cpu(2)
|
|
694
694
|
def heavy(): return 42
|
|
695
695
|
def test_a_competing():
|
|
696
|
+
# Hold the slot until the other test has asked for the fixture, then long
|
|
697
|
+
# enough for that request to time out at least once, whatever the host's pace.
|
|
696
698
|
Path("running").touch()
|
|
697
|
-
|
|
699
|
+
wait_for("requested")
|
|
700
|
+
time.sleep(.15)
|
|
698
701
|
@pytest.mark.flaky(reruns=4)
|
|
699
702
|
def test_b_dynamic(request):
|
|
700
703
|
wait_for("running")
|
|
704
|
+
Path("requested").touch()
|
|
701
705
|
assert request.getfixturevalue("heavy") == 42
|
|
702
706
|
"""
|
|
703
707
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|