ci-time-tracker 0.2.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/PKG-INFO +117 -16
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/README.md +116 -15
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/__init__.py +1 -1
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/analyzer.py +40 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/cache.py +27 -1
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/cli.py +159 -3
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/detection.py +25 -4
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/github_api.py +191 -35
- ci_time_tracker-0.3.1/ci_time_tracker/html_report.py +483 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/impact.py +9 -1
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/models.py +12 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/pricing.py +36 -12
- ci_time_tracker-0.3.1/ci_time_tracker/recommendations.py +469 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/reporter.py +66 -6
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker.egg-info/PKG-INFO +117 -16
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker.egg-info/SOURCES.txt +7 -1
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/pyproject.toml +1 -1
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_cli.py +11 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_detection.py +29 -1
- ci_time_tracker-0.3.1/tests/test_field_test_3.py +186 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_github_api.py +85 -10
- ci_time_tracker-0.3.1/tests/test_html_report.py +232 -0
- ci_time_tracker-0.3.1/tests/test_recommendations.py +576 -0
- ci_time_tracker-0.3.1/tests/test_workflow_cache.py +115 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/LICENSE +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/__main__.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/config_parser.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/gates.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker/log_parser.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker.egg-info/dependency_links.txt +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker.egg-info/entry_points.txt +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker.egg-info/requires.txt +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/ci_time_tracker.egg-info/top_level.txt +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/setup.cfg +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_analyzer.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_config_parser.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_gates_markdown.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_impact.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_log_parser.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_metrics.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_packaging.py +0 -0
- {ci_time_tracker-0.2.0 → ci_time_tracker-0.3.1}/tests/test_reporter.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ci-time-tracker
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Find where CI time and money go: cost, flaky steps and regressions from GitHub Actions, and a PR comment action
|
|
5
5
|
Author: ci-time-tracker contributors
|
|
6
6
|
License-Expression: MIT
|
|
@@ -32,29 +32,58 @@ Dynamic: license-file
|
|
|
32
32
|
# ci-time-tracker
|
|
33
33
|
|
|
34
34
|
[](https://github.com/0xWhisp/ci-time-tracker/actions/workflows/ci.yml)
|
|
35
|
+
[](https://pypi.org/project/ci-time-tracker/)
|
|
35
36
|
|
|
36
37
|
Find where your CI time and money go. ci-time-tracker reads your GitHub Actions
|
|
37
38
|
history and reports the steps that cost the most, the ones that are flaky, and
|
|
38
|
-
the ones that got slower, with the commit where it started.
|
|
39
|
-
|
|
39
|
+
the ones that got slower, with the commit where it started. It also recommends
|
|
40
|
+
changes to your workflows, backed by what your runs actually did. Its GitHub
|
|
41
|
+
Action comments on each pull request with how it changes CI time and cost.
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install ci-time-tracker
|
|
45
|
+
ci-time-tracker --github owner/repo --last 300 --format html --output report.html
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
[](https://0xwhisp.github.io/ci-time-tracker/example-report.html)
|
|
49
|
+
|
|
50
|
+
**[See the full example report](https://0xwhisp.github.io/ci-time-tracker/example-report.html)**
|
|
51
|
+
for the last 300 runs of microsoft/TypeScript: three flaky steps, and
|
|
52
|
+
recommendations with up to 135 hours of runner time at stake, such as running
|
|
53
|
+
the Windows legs of three jobs only after merging, since in 87 builds they
|
|
54
|
+
never failed when Linux passed.
|
|
55
|
+
|
|
56
|
+
Set `GITHUB_TOKEN` first (with the GitHub CLI: `export GITHUB_TOKEN=$(gh auth token)`):
|
|
57
|
+
each run costs a request, and without a token GitHub allows 60 an hour.
|
|
40
58
|
|
|
41
59
|
## Features
|
|
42
60
|
|
|
43
|
-
- **
|
|
44
|
-
|
|
61
|
+
- **History from the API**: Reads workflow runs, jobs and steps straight from
|
|
62
|
+
the GitHub Actions API, cached locally so later runs take seconds
|
|
45
63
|
- **Flaky Detection**: Flag steps that both failed and passed on the same commit,
|
|
46
64
|
including runs that only passed on a re-run
|
|
47
|
-
- **Regression Detection**: Flag steps whose duration jumped and stayed up,
|
|
48
|
-
name the commit where it started
|
|
49
|
-
- **
|
|
50
|
-
and
|
|
65
|
+
- **Regression Detection**: Flag steps whose duration jumped and stayed up, well
|
|
66
|
+
beyond their usual variation, and name the commit where it started
|
|
67
|
+
- **Recommendations**: Dependency caching, cancelling superseded pull request
|
|
68
|
+
runs, and matrix legs that never catch anything, each with the time at stake
|
|
69
|
+
- **Cost and Time**: Rank steps by the time they consume, estimate cost per runner
|
|
70
|
+
at GitHub's list prices, and separate queue time from execution time
|
|
51
71
|
- **Pull Request Comments**: A GitHub Action that tells each pull request how it
|
|
52
72
|
changes CI time and cost, and whether its failures are in known flaky steps
|
|
53
|
-
- **
|
|
73
|
+
- **Output Formats**: A self-contained HTML report, Markdown for
|
|
74
|
+
`$GITHUB_STEP_SUMMARY`, text, or JSON for dashboards; `--fail-on` gates
|
|
75
|
+
- **Config and Log Analysis**: Also parses GitHub Actions, GitLab CI and
|
|
76
|
+
CircleCI configurations and build logs, for providers without API support yet
|
|
54
77
|
|
|
55
78
|
## Installation
|
|
56
79
|
|
|
57
|
-
Requires Python 3.10+.
|
|
80
|
+
Requires Python 3.10+.
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
pip install ci-time-tracker
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Or from source:
|
|
58
87
|
|
|
59
88
|
```bash
|
|
60
89
|
git clone https://github.com/0xWhisp/ci-time-tracker.git
|
|
@@ -296,7 +325,7 @@ consumers, and the full step table collapsed.
|
|
|
296
325
|
```yaml
|
|
297
326
|
- name: CI time report
|
|
298
327
|
run: |
|
|
299
|
-
pip install
|
|
328
|
+
pip install ci-time-tracker
|
|
300
329
|
ci-time-tracker --github ${{ github.repository }} --workflow ci.yml --last 100 --group-matrix --format markdown >> "$GITHUB_STEP_SUMMARY"
|
|
301
330
|
env:
|
|
302
331
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
@@ -369,7 +398,7 @@ jobs:
|
|
|
369
398
|
if: github.event.workflow_run.event == 'pull_request'
|
|
370
399
|
runs-on: ubuntu-latest
|
|
371
400
|
steps:
|
|
372
|
-
- uses: 0xWhisp/ci-time-tracker@
|
|
401
|
+
- uses: 0xWhisp/ci-time-tracker@v0.3.1
|
|
373
402
|
```
|
|
374
403
|
|
|
375
404
|
`workflow_run` runs the copy of the workflow on the default branch with the
|
|
@@ -395,6 +424,31 @@ ci-time-tracker --github acme/web --run-id 39102310449 --group-matrix
|
|
|
395
424
|
ci-time-tracker --github acme/web --run-id 39102310449 --comment # post it
|
|
396
425
|
```
|
|
397
426
|
|
|
427
|
+
### HTML report
|
|
428
|
+
|
|
429
|
+
`--format html` writes a single self-contained page: summary cards, build
|
|
430
|
+
duration over time for each of the busiest workflows (failures marked, a
|
|
431
|
+
rolling median, outliers kept from flattening the chart), issues with their
|
|
432
|
+
evidence, recommendations, and where the time goes. It has no scripts and loads
|
|
433
|
+
nothing from elsewhere, so it opens offline and can be attached to an issue or
|
|
434
|
+
kept as a CI artifact. It follows the viewer's light or dark mode.
|
|
435
|
+
|
|
436
|
+
```bash
|
|
437
|
+
ci-time-tracker --github acme/web --group-matrix --format html --output ci-report.html
|
|
438
|
+
```
|
|
439
|
+
|
|
440
|
+
```yaml
|
|
441
|
+
- run: ci-time-tracker --github ${{ github.repository }} --format html --output ci-report.html
|
|
442
|
+
env:
|
|
443
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
444
|
+
- uses: actions/upload-artifact@v4
|
|
445
|
+
with:
|
|
446
|
+
name: ci-report
|
|
447
|
+
path: ci-report.html
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
The JSON output carries the same per-build data under `timeline`.
|
|
451
|
+
|
|
398
452
|
### Exit codes
|
|
399
453
|
|
|
400
454
|
| Code | Meaning |
|
|
@@ -405,13 +459,47 @@ ci-time-tracker --github acme/web --run-id 39102310449 --comment # post it
|
|
|
405
459
|
| 3 | GitHub API error (rate limit, permissions, not found) |
|
|
406
460
|
| 4 | A quality gate failed |
|
|
407
461
|
|
|
462
|
+
## Recommendations
|
|
463
|
+
|
|
464
|
+
With `--github`, ci-time-tracker also reads the workflow files behind the runs
|
|
465
|
+
and suggests changes where the configuration shows a cause and the history
|
|
466
|
+
shows it matters. Neither is enough alone: a slow install step may already be
|
|
467
|
+
cached, and a job without a cache may install in two seconds.
|
|
468
|
+
|
|
469
|
+
| Recommendation | When |
|
|
470
|
+
|---|---|
|
|
471
|
+
| **Cache dependencies** | A job installs dependencies (pip, uv, poetry, npm, pnpm, yarn, bundler, composer, go) without any cache, and the install takes a median of 20s or more |
|
|
472
|
+
| **Cancel superseded pull request runs** | Pull request runs kept running after a newer push to the same branch started another run, and the workflow has no `concurrency` with `cancel-in-progress` |
|
|
473
|
+
| **Run expensive matrix legs less often** | Over at least 10 builds, the Windows or macOS legs of a job never failed when its Linux legs passed |
|
|
474
|
+
|
|
475
|
+
Each one comes with its evidence, a concrete change, and what is at stake: the
|
|
476
|
+
time (and cost) the affected steps or jobs used over the analyzed runs. That is
|
|
477
|
+
an upper bound on the saving, not a promise of it. Findings with less than
|
|
478
|
+
10 minutes at stake are left out, so the list stays worth reading.
|
|
479
|
+
|
|
480
|
+
For example, on pytest-dev/pytest:
|
|
481
|
+
|
|
482
|
+
```
|
|
483
|
+
RECOMMENDATIONS
|
|
484
|
+
1. Run the Windows legs of `build` less often (.github/workflows/test.yml)
|
|
485
|
+
In 79 builds, the Windows legs never failed when the Linux legs passed. They used 68h 08m, 40% of the job's compute.
|
|
486
|
+
Suggestion: Run the Windows legs only on pushes to the default branch or on a schedule, and keep Linux on pull requests. A platform-specific failure then shows up after merging rather than before.
|
|
487
|
+
At stake: 68h 08m
|
|
488
|
+
```
|
|
489
|
+
|
|
490
|
+
Whether a job could start earlier (a `needs` that only orders jobs) is not
|
|
491
|
+
suggested: the configuration cannot tell an ordering from a real dependency on
|
|
492
|
+
the other job's results.
|
|
493
|
+
|
|
408
494
|
## Cost Estimates
|
|
409
495
|
|
|
410
496
|
Costs are estimated from the runner each job ran on, using GitHub's
|
|
411
497
|
[list prices](https://docs.github.com/en/billing/reference/actions-runner-pricing)
|
|
412
498
|
for **private repositories** as of October 2026, with each job rounded up to a
|
|
413
499
|
whole minute. Public repositories pay nothing for standard GitHub-hosted
|
|
414
|
-
runners; larger runners are billed for public repositories too.
|
|
500
|
+
runners; larger runners are billed for public repositories too. When the
|
|
501
|
+
analyzed repository is public (the runs say so), standard runners are priced
|
|
502
|
+
at `$0`, so the estimate counts only what is actually billed.
|
|
415
503
|
|
|
416
504
|
| Runner | USD per minute |
|
|
417
505
|
|---|---:|
|
|
@@ -492,7 +580,11 @@ A step has **regressed** when its duration jumped and stayed up:
|
|
|
492
580
|
2. The later segment's median must be at least **25%** higher
|
|
493
581
|
(`--regression-threshold`) and at least **10 seconds** higher, so `1s -> 2s`
|
|
494
582
|
steps are not reported.
|
|
495
|
-
3. The
|
|
583
|
+
3. The jump must be at least **three times** the step's usual variation (the
|
|
584
|
+
median absolute deviation of the noisier segment). With hundreds of steps
|
|
585
|
+
checked, the best split of a merely noisy one would otherwise clear the
|
|
586
|
+
thresholds above by chance.
|
|
587
|
+
4. The most recent 5 builds must still be that slow, so a spike that already
|
|
496
588
|
recovered is not reported.
|
|
497
589
|
|
|
498
590
|
The report names the commit and build where the regression started. A step
|
|
@@ -500,6 +592,15 @@ needs at least **10 builds** to be judged; `regression_checked_steps` says how
|
|
|
500
592
|
many had enough history. With `--group-matrix`, the legs of a build are
|
|
501
593
|
combined into one sample (their median).
|
|
502
594
|
|
|
595
|
+
Runs that never executed (awaiting a maintainer's approval, skipped by the
|
|
596
|
+
workflow's conditions, or stale) have no timings and are left out; they do not
|
|
597
|
+
count towards `--last`.
|
|
598
|
+
|
|
599
|
+
When the analyzed runs span several branches, regressions are looked for on
|
|
600
|
+
the repository's default branch only (`regression_branch`). Pull request
|
|
601
|
+
branches run different code with different caches; mixed into one history
|
|
602
|
+
they look like changes that never happened.
|
|
603
|
+
|
|
503
604
|
## Supported CI Providers
|
|
504
605
|
|
|
505
606
|
| Provider | Config Format | Log Format |
|
|
@@ -550,7 +651,7 @@ Then, for each release:
|
|
|
550
651
|
1. Bump `version` in `pyproject.toml` and `__version__` in
|
|
551
652
|
`ci_time_tracker/__init__.py` (a test checks they agree), and add the
|
|
552
653
|
release to [CHANGELOG.md](CHANGELOG.md).
|
|
553
|
-
2. Tag and push: `git tag v0.
|
|
654
|
+
2. Tag and push: `git tag v0.3.1 && git push origin v0.3.1`.
|
|
554
655
|
3. Create the GitHub release from the tag, with the changelog entry as its
|
|
555
656
|
notes, and tick "Publish this Action to the GitHub Marketplace".
|
|
556
657
|
|
|
@@ -1,29 +1,58 @@
|
|
|
1
1
|
# ci-time-tracker
|
|
2
2
|
|
|
3
3
|
[](https://github.com/0xWhisp/ci-time-tracker/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/ci-time-tracker/)
|
|
4
5
|
|
|
5
6
|
Find where your CI time and money go. ci-time-tracker reads your GitHub Actions
|
|
6
7
|
history and reports the steps that cost the most, the ones that are flaky, and
|
|
7
|
-
the ones that got slower, with the commit where it started.
|
|
8
|
-
|
|
8
|
+
the ones that got slower, with the commit where it started. It also recommends
|
|
9
|
+
changes to your workflows, backed by what your runs actually did. Its GitHub
|
|
10
|
+
Action comments on each pull request with how it changes CI time and cost.
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install ci-time-tracker
|
|
14
|
+
ci-time-tracker --github owner/repo --last 300 --format html --output report.html
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
[](https://0xwhisp.github.io/ci-time-tracker/example-report.html)
|
|
18
|
+
|
|
19
|
+
**[See the full example report](https://0xwhisp.github.io/ci-time-tracker/example-report.html)**
|
|
20
|
+
for the last 300 runs of microsoft/TypeScript: three flaky steps, and
|
|
21
|
+
recommendations with up to 135 hours of runner time at stake, such as running
|
|
22
|
+
the Windows legs of three jobs only after merging, since in 87 builds they
|
|
23
|
+
never failed when Linux passed.
|
|
24
|
+
|
|
25
|
+
Set `GITHUB_TOKEN` first (with the GitHub CLI: `export GITHUB_TOKEN=$(gh auth token)`):
|
|
26
|
+
each run costs a request, and without a token GitHub allows 60 an hour.
|
|
9
27
|
|
|
10
28
|
## Features
|
|
11
29
|
|
|
12
|
-
- **
|
|
13
|
-
|
|
30
|
+
- **History from the API**: Reads workflow runs, jobs and steps straight from
|
|
31
|
+
the GitHub Actions API, cached locally so later runs take seconds
|
|
14
32
|
- **Flaky Detection**: Flag steps that both failed and passed on the same commit,
|
|
15
33
|
including runs that only passed on a re-run
|
|
16
|
-
- **Regression Detection**: Flag steps whose duration jumped and stayed up,
|
|
17
|
-
name the commit where it started
|
|
18
|
-
- **
|
|
19
|
-
and
|
|
34
|
+
- **Regression Detection**: Flag steps whose duration jumped and stayed up, well
|
|
35
|
+
beyond their usual variation, and name the commit where it started
|
|
36
|
+
- **Recommendations**: Dependency caching, cancelling superseded pull request
|
|
37
|
+
runs, and matrix legs that never catch anything, each with the time at stake
|
|
38
|
+
- **Cost and Time**: Rank steps by the time they consume, estimate cost per runner
|
|
39
|
+
at GitHub's list prices, and separate queue time from execution time
|
|
20
40
|
- **Pull Request Comments**: A GitHub Action that tells each pull request how it
|
|
21
41
|
changes CI time and cost, and whether its failures are in known flaky steps
|
|
22
|
-
- **
|
|
42
|
+
- **Output Formats**: A self-contained HTML report, Markdown for
|
|
43
|
+
`$GITHUB_STEP_SUMMARY`, text, or JSON for dashboards; `--fail-on` gates
|
|
44
|
+
- **Config and Log Analysis**: Also parses GitHub Actions, GitLab CI and
|
|
45
|
+
CircleCI configurations and build logs, for providers without API support yet
|
|
23
46
|
|
|
24
47
|
## Installation
|
|
25
48
|
|
|
26
|
-
Requires Python 3.10+.
|
|
49
|
+
Requires Python 3.10+.
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install ci-time-tracker
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Or from source:
|
|
27
56
|
|
|
28
57
|
```bash
|
|
29
58
|
git clone https://github.com/0xWhisp/ci-time-tracker.git
|
|
@@ -265,7 +294,7 @@ consumers, and the full step table collapsed.
|
|
|
265
294
|
```yaml
|
|
266
295
|
- name: CI time report
|
|
267
296
|
run: |
|
|
268
|
-
pip install
|
|
297
|
+
pip install ci-time-tracker
|
|
269
298
|
ci-time-tracker --github ${{ github.repository }} --workflow ci.yml --last 100 --group-matrix --format markdown >> "$GITHUB_STEP_SUMMARY"
|
|
270
299
|
env:
|
|
271
300
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
@@ -338,7 +367,7 @@ jobs:
|
|
|
338
367
|
if: github.event.workflow_run.event == 'pull_request'
|
|
339
368
|
runs-on: ubuntu-latest
|
|
340
369
|
steps:
|
|
341
|
-
- uses: 0xWhisp/ci-time-tracker@
|
|
370
|
+
- uses: 0xWhisp/ci-time-tracker@v0.3.1
|
|
342
371
|
```
|
|
343
372
|
|
|
344
373
|
`workflow_run` runs the copy of the workflow on the default branch with the
|
|
@@ -364,6 +393,31 @@ ci-time-tracker --github acme/web --run-id 39102310449 --group-matrix
|
|
|
364
393
|
ci-time-tracker --github acme/web --run-id 39102310449 --comment # post it
|
|
365
394
|
```
|
|
366
395
|
|
|
396
|
+
### HTML report
|
|
397
|
+
|
|
398
|
+
`--format html` writes a single self-contained page: summary cards, build
|
|
399
|
+
duration over time for each of the busiest workflows (failures marked, a
|
|
400
|
+
rolling median, outliers kept from flattening the chart), issues with their
|
|
401
|
+
evidence, recommendations, and where the time goes. It has no scripts and loads
|
|
402
|
+
nothing from elsewhere, so it opens offline and can be attached to an issue or
|
|
403
|
+
kept as a CI artifact. It follows the viewer's light or dark mode.
|
|
404
|
+
|
|
405
|
+
```bash
|
|
406
|
+
ci-time-tracker --github acme/web --group-matrix --format html --output ci-report.html
|
|
407
|
+
```
|
|
408
|
+
|
|
409
|
+
```yaml
|
|
410
|
+
- run: ci-time-tracker --github ${{ github.repository }} --format html --output ci-report.html
|
|
411
|
+
env:
|
|
412
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
413
|
+
- uses: actions/upload-artifact@v4
|
|
414
|
+
with:
|
|
415
|
+
name: ci-report
|
|
416
|
+
path: ci-report.html
|
|
417
|
+
```
|
|
418
|
+
|
|
419
|
+
The JSON output carries the same per-build data under `timeline`.
|
|
420
|
+
|
|
367
421
|
### Exit codes
|
|
368
422
|
|
|
369
423
|
| Code | Meaning |
|
|
@@ -374,13 +428,47 @@ ci-time-tracker --github acme/web --run-id 39102310449 --comment # post it
|
|
|
374
428
|
| 3 | GitHub API error (rate limit, permissions, not found) |
|
|
375
429
|
| 4 | A quality gate failed |
|
|
376
430
|
|
|
431
|
+
## Recommendations
|
|
432
|
+
|
|
433
|
+
With `--github`, ci-time-tracker also reads the workflow files behind the runs
|
|
434
|
+
and suggests changes where the configuration shows a cause and the history
|
|
435
|
+
shows it matters. Neither is enough alone: a slow install step may already be
|
|
436
|
+
cached, and a job without a cache may install in two seconds.
|
|
437
|
+
|
|
438
|
+
| Recommendation | When |
|
|
439
|
+
|---|---|
|
|
440
|
+
| **Cache dependencies** | A job installs dependencies (pip, uv, poetry, npm, pnpm, yarn, bundler, composer, go) without any cache, and the install takes a median of 20s or more |
|
|
441
|
+
| **Cancel superseded pull request runs** | Pull request runs kept running after a newer push to the same branch started another run, and the workflow has no `concurrency` with `cancel-in-progress` |
|
|
442
|
+
| **Run expensive matrix legs less often** | Over at least 10 builds, the Windows or macOS legs of a job never failed when its Linux legs passed |
|
|
443
|
+
|
|
444
|
+
Each one comes with its evidence, a concrete change, and what is at stake: the
|
|
445
|
+
time (and cost) the affected steps or jobs used over the analyzed runs. That is
|
|
446
|
+
an upper bound on the saving, not a promise of it. Findings with less than
|
|
447
|
+
10 minutes at stake are left out, so the list stays worth reading.
|
|
448
|
+
|
|
449
|
+
For example, on pytest-dev/pytest:
|
|
450
|
+
|
|
451
|
+
```
|
|
452
|
+
RECOMMENDATIONS
|
|
453
|
+
1. Run the Windows legs of `build` less often (.github/workflows/test.yml)
|
|
454
|
+
In 79 builds, the Windows legs never failed when the Linux legs passed. They used 68h 08m, 40% of the job's compute.
|
|
455
|
+
Suggestion: Run the Windows legs only on pushes to the default branch or on a schedule, and keep Linux on pull requests. A platform-specific failure then shows up after merging rather than before.
|
|
456
|
+
At stake: 68h 08m
|
|
457
|
+
```
|
|
458
|
+
|
|
459
|
+
Whether a job could start earlier (a `needs` that only orders jobs) is not
|
|
460
|
+
suggested: the configuration cannot tell an ordering from a real dependency on
|
|
461
|
+
the other job's results.
|
|
462
|
+
|
|
377
463
|
## Cost Estimates
|
|
378
464
|
|
|
379
465
|
Costs are estimated from the runner each job ran on, using GitHub's
|
|
380
466
|
[list prices](https://docs.github.com/en/billing/reference/actions-runner-pricing)
|
|
381
467
|
for **private repositories** as of October 2026, with each job rounded up to a
|
|
382
468
|
whole minute. Public repositories pay nothing for standard GitHub-hosted
|
|
383
|
-
runners; larger runners are billed for public repositories too.
|
|
469
|
+
runners; larger runners are billed for public repositories too. When the
|
|
470
|
+
analyzed repository is public (the runs say so), standard runners are priced
|
|
471
|
+
at `$0`, so the estimate counts only what is actually billed.
|
|
384
472
|
|
|
385
473
|
| Runner | USD per minute |
|
|
386
474
|
|---|---:|
|
|
@@ -461,7 +549,11 @@ A step has **regressed** when its duration jumped and stayed up:
|
|
|
461
549
|
2. The later segment's median must be at least **25%** higher
|
|
462
550
|
(`--regression-threshold`) and at least **10 seconds** higher, so `1s -> 2s`
|
|
463
551
|
steps are not reported.
|
|
464
|
-
3. The
|
|
552
|
+
3. The jump must be at least **three times** the step's usual variation (the
|
|
553
|
+
median absolute deviation of the noisier segment). With hundreds of steps
|
|
554
|
+
checked, the best split of a merely noisy one would otherwise clear the
|
|
555
|
+
thresholds above by chance.
|
|
556
|
+
4. The most recent 5 builds must still be that slow, so a spike that already
|
|
465
557
|
recovered is not reported.
|
|
466
558
|
|
|
467
559
|
The report names the commit and build where the regression started. A step
|
|
@@ -469,6 +561,15 @@ needs at least **10 builds** to be judged; `regression_checked_steps` says how
|
|
|
469
561
|
many had enough history. With `--group-matrix`, the legs of a build are
|
|
470
562
|
combined into one sample (their median).
|
|
471
563
|
|
|
564
|
+
Runs that never executed (awaiting a maintainer's approval, skipped by the
|
|
565
|
+
workflow's conditions, or stale) have no timings and are left out; they do not
|
|
566
|
+
count towards `--last`.
|
|
567
|
+
|
|
568
|
+
When the analyzed runs span several branches, regressions are looked for on
|
|
569
|
+
the repository's default branch only (`regression_branch`). Pull request
|
|
570
|
+
branches run different code with different caches; mixed into one history
|
|
571
|
+
they look like changes that never happened.
|
|
572
|
+
|
|
472
573
|
## Supported CI Providers
|
|
473
574
|
|
|
474
575
|
| Provider | Config Format | Log Format |
|
|
@@ -519,7 +620,7 @@ Then, for each release:
|
|
|
519
620
|
1. Bump `version` in `pyproject.toml` and `__version__` in
|
|
520
621
|
`ci_time_tracker/__init__.py` (a test checks they agree), and add the
|
|
521
622
|
release to [CHANGELOG.md](CHANGELOG.md).
|
|
522
|
-
2. Tag and push: `git tag v0.
|
|
623
|
+
2. Tag and push: `git tag v0.3.1 && git push origin v0.3.1`.
|
|
523
624
|
3. Create the GitHub release from the tag, with the changelog entry as its
|
|
524
625
|
notes, and tick "Publish this Action to the GitHub Marketplace".
|
|
525
626
|
|
|
@@ -172,6 +172,7 @@ def analyze_logs(
|
|
|
172
172
|
group_matrix: bool = False,
|
|
173
173
|
regression_threshold: float = DEFAULT_REGRESSION_THRESHOLD,
|
|
174
174
|
min_samples: int = DEFAULT_MIN_SAMPLES,
|
|
175
|
+
regression_branch: str | None = None,
|
|
175
176
|
) -> AnalysisResult:
|
|
176
177
|
"""Analyze multiple build logs and compute statistics.
|
|
177
178
|
|
|
@@ -190,6 +191,9 @@ def analyze_logs(
|
|
|
190
191
|
regression_threshold: Minimum relative duration increase to report,
|
|
191
192
|
e.g. 0.25 for +25%
|
|
192
193
|
min_samples: Minimum builds on each side of a regression
|
|
194
|
+
regression_branch: When builds span several branches, the one
|
|
195
|
+
regressions are looked for on (the default branch); defaults to
|
|
196
|
+
the branch with the most builds
|
|
193
197
|
|
|
194
198
|
Returns:
|
|
195
199
|
AnalysisResult in log mode with step statistics and detected issues
|
|
@@ -297,6 +301,17 @@ def analyze_logs(
|
|
|
297
301
|
name: {"method": "failure-rate"} for name in detect_flaky_steps(step_stats)
|
|
298
302
|
}
|
|
299
303
|
|
|
304
|
+
# Regressions are looked for on one branch only. Pull request branches
|
|
305
|
+
# run different code with different caches, and mixed into one series
|
|
306
|
+
# they read as changes that never happened (seen on rubocop, whose
|
|
307
|
+
# 26 master builds were mixed with 74 from pull requests).
|
|
308
|
+
branches = Counter(log.metadata.get("branch") for log in logs if log.metadata.get("branch"))
|
|
309
|
+
if len(branches) > 1:
|
|
310
|
+
if regression_branch not in branches:
|
|
311
|
+
regression_branch = branches.most_common(1)[0][0]
|
|
312
|
+
else:
|
|
313
|
+
regression_branch = None # one branch or none known: all builds compare
|
|
314
|
+
|
|
300
315
|
# Regressions, from one sample per build (the median across matrix legs
|
|
301
316
|
# when they are grouped), in chronological order
|
|
302
317
|
regression_checked = 0
|
|
@@ -306,6 +321,7 @@ def analyze_logs(
|
|
|
306
321
|
samples = [
|
|
307
322
|
(median(values), logs[build_index])
|
|
308
323
|
for build_index, values in sorted(build_durations.items())
|
|
324
|
+
if regression_branch is None or logs[build_index].metadata.get("branch") == regression_branch
|
|
309
325
|
]
|
|
310
326
|
|
|
311
327
|
if len(samples) >= 2 * min_samples:
|
|
@@ -344,18 +360,42 @@ def analyze_logs(
|
|
|
344
360
|
metadata["flaky_detection"] = flaky_method
|
|
345
361
|
metadata["regression_checked_steps"] = regression_checked
|
|
346
362
|
metadata["regression_min_builds"] = 2 * min_samples
|
|
363
|
+
if regression_branch:
|
|
364
|
+
metadata["regression_branch"] = regression_branch
|
|
347
365
|
metadata.update(aggregate_job_metrics(logs, pricing))
|
|
348
366
|
|
|
349
367
|
return AnalysisResult(
|
|
350
368
|
mode="log",
|
|
351
369
|
steps=step_stats,
|
|
352
370
|
total_estimated_duration=None,
|
|
371
|
+
timeline=build_timeline(logs),
|
|
353
372
|
regressed_steps=[stats.name for stats in step_stats if stats.is_regression],
|
|
354
373
|
flaky_steps=[stats.name for stats in step_stats if stats.is_flaky],
|
|
355
374
|
metadata=metadata,
|
|
356
375
|
)
|
|
357
376
|
|
|
358
377
|
|
|
378
|
+
def build_timeline(logs: list[BuildLog]) -> list[dict[str, Any]]:
|
|
379
|
+
"""One entry per build, in the given order, for trend charts.
|
|
380
|
+
|
|
381
|
+
Earlier attempts of re-runs are left out: they are part of the same build.
|
|
382
|
+
"""
|
|
383
|
+
return [
|
|
384
|
+
{
|
|
385
|
+
"build_id": log.build_id,
|
|
386
|
+
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
|
|
387
|
+
"duration": log.total_duration,
|
|
388
|
+
"conclusion": log.conclusion,
|
|
389
|
+
"head_sha": log.head_sha[:7] if log.head_sha else None,
|
|
390
|
+
"workflow": log.metadata.get("workflow_name"),
|
|
391
|
+
"branch": log.metadata.get("branch"),
|
|
392
|
+
"url": log.metadata.get("url"),
|
|
393
|
+
}
|
|
394
|
+
for log in logs
|
|
395
|
+
if not log.metadata.get("superseded_attempt")
|
|
396
|
+
]
|
|
397
|
+
|
|
398
|
+
|
|
359
399
|
def aggregate_job_metrics(
|
|
360
400
|
logs: list[BuildLog],
|
|
361
401
|
pricing: PricingTable | None = None,
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Listing workflow runs is cheap, but every run costs an extra request for its
|
|
4
4
|
jobs. Those payloads no longer change once a run has completed, so they are
|
|
5
|
-
stored in a small SQLite database between invocations.
|
|
5
|
+
stored in a small SQLite database between invocations. Workflow files are
|
|
6
|
+
stored too, keyed by the commit they were read at, which pins their content.
|
|
6
7
|
"""
|
|
7
8
|
|
|
8
9
|
import json
|
|
@@ -21,6 +22,14 @@ CREATE TABLE IF NOT EXISTS run_jobs (
|
|
|
21
22
|
payload TEXT NOT NULL,
|
|
22
23
|
PRIMARY KEY (repo, run_id, run_attempt)
|
|
23
24
|
);
|
|
25
|
+
CREATE TABLE IF NOT EXISTS workflow_files (
|
|
26
|
+
repo TEXT NOT NULL,
|
|
27
|
+
path TEXT NOT NULL,
|
|
28
|
+
ref TEXT NOT NULL,
|
|
29
|
+
fetched_at TEXT NOT NULL,
|
|
30
|
+
content TEXT NOT NULL,
|
|
31
|
+
PRIMARY KEY (repo, path, ref)
|
|
32
|
+
);
|
|
24
33
|
"""
|
|
25
34
|
|
|
26
35
|
|
|
@@ -93,6 +102,23 @@ class RunCache:
|
|
|
93
102
|
)
|
|
94
103
|
self._connection.commit()
|
|
95
104
|
|
|
105
|
+
def get_workflow_file(self, repo: str, path: str, ref: str) -> str | None:
|
|
106
|
+
"""Return a workflow file as read at commit `ref`, or None when not cached."""
|
|
107
|
+
row = self._connection.execute(
|
|
108
|
+
"SELECT content FROM workflow_files WHERE repo = ? AND path = ? AND ref = ?",
|
|
109
|
+
(repo, path, ref),
|
|
110
|
+
).fetchone()
|
|
111
|
+
return None if row is None else row[0]
|
|
112
|
+
|
|
113
|
+
def store_workflow_file(self, repo: str, path: str, ref: str, content: str) -> None:
|
|
114
|
+
"""Cache a workflow file as read at commit `ref`."""
|
|
115
|
+
self._connection.execute(
|
|
116
|
+
"INSERT OR REPLACE INTO workflow_files "
|
|
117
|
+
"(repo, path, ref, fetched_at, content) VALUES (?, ?, ?, ?, ?)",
|
|
118
|
+
(repo, path, ref, datetime.now(timezone.utc).isoformat(), content),
|
|
119
|
+
)
|
|
120
|
+
self._connection.commit()
|
|
121
|
+
|
|
96
122
|
def count(self, repo: str | None = None) -> int:
|
|
97
123
|
"""Number of cached runs, optionally limited to one repository."""
|
|
98
124
|
if repo is None:
|