ratchet-gate 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ratchet_gate-0.1.0/.gitea/workflows/release.yml +56 -0
- ratchet_gate-0.1.0/.gitignore +6 -0
- ratchet_gate-0.1.0/DIVERGENCES.md +49 -0
- ratchet_gate-0.1.0/LICENSE +21 -0
- ratchet_gate-0.1.0/PKG-INFO +183 -0
- ratchet_gate-0.1.0/README.md +137 -0
- ratchet_gate-0.1.0/RELEASING.md +71 -0
- ratchet_gate-0.1.0/examples/quality-gate.toml +5 -0
- ratchet_gate-0.1.0/examples/quality-gate.yml +65 -0
- ratchet_gate-0.1.0/pyproject.toml +68 -0
- ratchet_gate-0.1.0/scripts/verify-against-repo.sh +70 -0
- ratchet_gate-0.1.0/src/quality_gate/__init__.py +26 -0
- ratchet_gate-0.1.0/src/quality_gate/baseline.py +229 -0
- ratchet_gate-0.1.0/src/quality_gate/cli.py +144 -0
- ratchet_gate-0.1.0/src/quality_gate/collect.py +91 -0
- ratchet_gate-0.1.0/src/quality_gate/compare.py +102 -0
- ratchet_gate-0.1.0/src/quality_gate/config.py +112 -0
- ratchet_gate-0.1.0/src/quality_gate/enums.py +123 -0
- ratchet_gate-0.1.0/src/quality_gate/errors.py +29 -0
- ratchet_gate-0.1.0/src/quality_gate/models.py +159 -0
- ratchet_gate-0.1.0/src/quality_gate/parsers/__init__.py +20 -0
- ratchet_gate-0.1.0/src/quality_gate/parsers/global_tools.py +97 -0
- ratchet_gate-0.1.0/src/quality_gate/parsers/js_tools.py +68 -0
- ratchet_gate-0.1.0/src/quality_gate/parsers/kotlin_tools.py +59 -0
- ratchet_gate-0.1.0/src/quality_gate/parsers/python_tools.py +84 -0
- ratchet_gate-0.1.0/src/quality_gate/parsers/registry.py +91 -0
- ratchet_gate-0.1.0/src/quality_gate/parsers/scoping.py +69 -0
- ratchet_gate-0.1.0/src/quality_gate/py.typed +0 -0
- ratchet_gate-0.1.0/src/quality_gate/render.py +90 -0
- ratchet_gate-0.1.0/tests/conftest.py +6 -0
- ratchet_gate-0.1.0/tests/fixtures/baselines/README.md +13 -0
- ratchet_gate-0.1.0/tests/fixtures/baselines/action-items-android.json +138 -0
- ratchet_gate-0.1.0/tests/fixtures/baselines/gitea-review-agent.json +84 -0
- ratchet_gate-0.1.0/tests/fixtures/baselines/mdbin.json +116 -0
- ratchet_gate-0.1.0/tests/fixtures/baselines/nasa.json +98 -0
- ratchet_gate-0.1.0/tests/fixtures/baselines/pot-fauna.json +184 -0
- ratchet_gate-0.1.0/tests/golden/nasa/check_all_unmeasured.txt +1 -0
- ratchet_gate-0.1.0/tests/golden/nasa/check_bad_ref.txt +1 -0
- ratchet_gate-0.1.0/tests/golden/nasa/check_measured_nasa-server.md +23 -0
- ratchet_gate-0.1.0/tests/golden/nasa/check_ref_head.md +27 -0
- ratchet_gate-0.1.0/tests/golden/nasa/check_unknown_pkg.txt +1 -0
- ratchet_gate-0.1.0/tests/harness.py +133 -0
- ratchet_gate-0.1.0/tests/reference/README.md +16 -0
- ratchet_gate-0.1.0/tests/reference/quality_gate_nasa.py +947 -0
- ratchet_gate-0.1.0/tests/repobuilder.py +173 -0
- ratchet_gate-0.1.0/tests/test_config.py +101 -0
- ratchet_gate-0.1.0/tests/test_dependencies.py +65 -0
- ratchet_gate-0.1.0/tests/test_equivalence.py +446 -0
- ratchet_gate-0.1.0/tests/test_equivalence_js.py +149 -0
- ratchet_gate-0.1.0/tests/test_models.py +225 -0
- ratchet_gate-0.1.0/tests/test_parsers.py +125 -0
- ratchet_gate-0.1.0/uv.lock +447 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Publishing is tag-driven so that the artifact on PyPI and the tag in this repo
|
|
4
|
+
# are the same commit by construction, not by anyone remembering to keep them in
|
|
5
|
+
# step. Tag, push, done.
|
|
6
|
+
#
|
|
7
|
+
# Runs on Gitea Actions: PyPI's Trusted Publishing (OIDC) only recognises
|
|
8
|
+
# GitHub, GitLab, Google and ActiveState as publishers, so this uses an API
|
|
9
|
+
# token in the `PYPI_TOKEN` repo secret instead. See RELEASING.md.
|
|
10
|
+
|
|
11
|
+
on:
|
|
12
|
+
push:
|
|
13
|
+
tags: ["v*"]
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
publish:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
|
|
21
|
+
- uses: astral-sh/setup-uv@v5
|
|
22
|
+
|
|
23
|
+
# The gate's own promise is that it never lets a number move the wrong way
|
|
24
|
+
# unnoticed. Publishing a build that fails its own suite would be the one
|
|
25
|
+
# regression nothing is watching, so the tests gate the upload.
|
|
26
|
+
- name: Test
|
|
27
|
+
run: |
|
|
28
|
+
uv venv
|
|
29
|
+
uv pip install -e '.[dev]'
|
|
30
|
+
uv run pytest
|
|
31
|
+
uv run mypy
|
|
32
|
+
|
|
33
|
+
# The tag must name the version the package actually declares. Otherwise
|
|
34
|
+
# `pip install ratchet-gate==0.2.0` and `git checkout v0.2.0` quietly
|
|
35
|
+
# disagree, and every later bisect is built on that.
|
|
36
|
+
- name: Check tag matches project version
|
|
37
|
+
run: |
|
|
38
|
+
tag="${GITHUB_REF_NAME#v}"
|
|
39
|
+
declared="$(uv run python -c 'import tomllib,pathlib; print(tomllib.loads(pathlib.Path("pyproject.toml").read_text())["project"]["version"])')"
|
|
40
|
+
if [ "$tag" != "$declared" ]; then
|
|
41
|
+
echo "::error::tag v$tag does not match pyproject version $declared"
|
|
42
|
+
exit 1
|
|
43
|
+
fi
|
|
44
|
+
|
|
45
|
+
- name: Build
|
|
46
|
+
run: uv build
|
|
47
|
+
|
|
48
|
+
- name: Publish to PyPI
|
|
49
|
+
env:
|
|
50
|
+
UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
|
|
51
|
+
run: |
|
|
52
|
+
if [ -z "${UV_PUBLISH_TOKEN:-}" ]; then
|
|
53
|
+
echo "::error::PYPI_TOKEN secret is not set -- see RELEASING.md"
|
|
54
|
+
exit 1
|
|
55
|
+
fi
|
|
56
|
+
uv publish
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Deliberate divergences from the implementations this package replaces
|
|
2
|
+
|
|
3
|
+
The rule is byte-identical output. Every departure from it is listed here, with
|
|
4
|
+
what changes, why, and what it cost to check. A divergence that is not on this
|
|
5
|
+
list is a bug.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## 1. `npm_audit_findings` renders as an integer, not `31.0`
|
|
10
|
+
|
|
11
|
+
**Repos affected:** `mdbin` (the only one that baselines `npm_audit_findings` as
|
|
12
|
+
a repo-wide metric).
|
|
13
|
+
|
|
14
|
+
**What changes:** one cell of the summary table.
|
|
15
|
+
|
|
16
|
+
```
|
|
17
|
+
- | `(global)` | npm_audit_findings | 31 | 31.0 | +0 | ok |
|
|
18
|
+
+ | `(global)` | npm_audit_findings | 31 | 31 | +0 | ok |
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
**Why the incumbent does it.** mdbin's copy needed a repo-wide adapter around a
|
|
22
|
+
per-package parser, and gave it a `-> float` return annotation:
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
def parse_npm_audit_total(path: Path) -> float:
|
|
26
|
+
"""Global-slot adapter: GLOBAL_PARSERS values are scalars, not dicts."""
|
|
27
|
+
return float(parse_npm_audit_json(path).get("npm_audit_findings", 0))
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
The `float()` is there to satisfy the annotation, not to describe the value. A
|
|
31
|
+
vulnerability count is a count, and the baseline column two cells to the left
|
|
32
|
+
says `31`, so the table contradicts itself about what kind of number this is.
|
|
33
|
+
|
|
34
|
+
**Why not reproduce it.** Reproducing it would mean writing a cast whose only
|
|
35
|
+
purpose is to make the output wrong in the same way, in a package meant to
|
|
36
|
+
outlive all five copies. It would also propagate: a full `promote` writes the
|
|
37
|
+
measured value straight back into the tracked baseline, so the first
|
|
38
|
+
unqualified promote after adoption would turn `31` into `31.0` in the JSON and
|
|
39
|
+
every integer metric would follow as its parser was touched.
|
|
40
|
+
|
|
41
|
+
**What it does not change.** Nothing about the verdict. `31.0 == 31`, so
|
|
42
|
+
`regressed` and `improved` are identical either way, in this run and in every
|
|
43
|
+
other. The equivalence suite confirms exit codes match; only the rendered cell
|
|
44
|
+
differs.
|
|
45
|
+
|
|
46
|
+
**Pinned by:** `tests/test_parsers.py::test_npm_audit_total_is_an_integer`.
|
|
47
|
+
|
|
48
|
+
**On adoption:** mdbin's next summary shows `31` where it showed `31.0`. If that
|
|
49
|
+
repo commits its summary, expect a one-line diff on the first run.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mateus Marim
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: ratchet-gate
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Ratchet quality gate: enforce monotonic improvement of code-quality metrics against a committed baseline.
|
|
5
|
+
Project-URL: Homepage, https://git.marim.dev/mateuscmarim/quality-gate
|
|
6
|
+
Project-URL: Source, https://git.marim.dev/mateuscmarim/quality-gate
|
|
7
|
+
Author: Mateus Marim
|
|
8
|
+
License: MIT License
|
|
9
|
+
|
|
10
|
+
Copyright (c) 2026 Mateus Marim
|
|
11
|
+
|
|
12
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
13
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
14
|
+
in the Software without restriction, including without limitation the rights
|
|
15
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
16
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
17
|
+
furnished to do so, subject to the following conditions:
|
|
18
|
+
|
|
19
|
+
The above copyright notice and this permission notice shall be included in all
|
|
20
|
+
copies or substantial portions of the Software.
|
|
21
|
+
|
|
22
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
23
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
24
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
25
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
26
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
27
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
28
|
+
SOFTWARE.
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Keywords: baseline,ci,code-quality,coverage,linting,ratchet
|
|
31
|
+
Classifier: Development Status :: 4 - Beta
|
|
32
|
+
Classifier: Environment :: Console
|
|
33
|
+
Classifier: Intended Audience :: Developers
|
|
34
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
37
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
38
|
+
Classifier: Topic :: Software Development :: Testing
|
|
39
|
+
Classifier: Typing :: Typed
|
|
40
|
+
Requires-Python: >=3.12
|
|
41
|
+
Requires-Dist: pydantic>=2.9
|
|
42
|
+
Provides-Extra: dev
|
|
43
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
44
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
45
|
+
Description-Content-Type: text/markdown
|
|
46
|
+
|
|
47
|
+
# quality-gate
|
|
48
|
+
|
|
49
|
+
A **ratchet** quality gate. It reads what your linters, type checkers, test
|
|
50
|
+
runners and scanners already wrote, compares every metric to a committed
|
|
51
|
+
baseline, and fails the build if any of them got worse.
|
|
52
|
+
|
|
53
|
+
Not a threshold gate. There is no number you have to reach before it is useful —
|
|
54
|
+
the baseline starts wherever your code is today, and the only rule is that it
|
|
55
|
+
may not move backwards. Improvements are adopted automatically after merge, so
|
|
56
|
+
the bar rises on its own and never falls.
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
## Ratchet Quality Gate
|
|
60
|
+
|
|
61
|
+
| Package | Metric | Baseline | Current | Δ | Status |
|
|
62
|
+
|---|---|---:|---:|---:|:---:|
|
|
63
|
+
| `api` | coverage_lines_pct | 89.09 | 88.22 | -0.87 | FAIL |
|
|
64
|
+
| `api` | complexity_violations | 108 | 108 | +0 | ok |
|
|
65
|
+
| `api` | mypy_errors | 0 | 0 | +0 | ok |
|
|
66
|
+
| `(global)` | secret_findings | 0 | 0 | +0 | ok |
|
|
67
|
+
|
|
68
|
+
**FAIL** — 1 regression(s):
|
|
69
|
+
- `api` / **coverage_lines_pct** 89.09 → 88.22
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Why this repo exists
|
|
73
|
+
|
|
74
|
+
Five repos were running five copies of the same 400–1500 line script. They had
|
|
75
|
+
diverged: different exclusion lists, different report paths, one repo carrying a
|
|
76
|
+
one-line bug fix that never reached the other four because there was no
|
|
77
|
+
mechanism for it to travel by. Comparing them, almost every difference was
|
|
78
|
+
configuration wearing the costume of code.
|
|
79
|
+
|
|
80
|
+
So the program lives here once, and everything that legitimately differs per
|
|
81
|
+
repo lives in that repo's `quality-gate.toml`.
|
|
82
|
+
|
|
83
|
+
## Install
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
uvx ratchet-gate@0.1.0 check
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
No install step in CI, no vendored copy to keep in sync, and the pin says
|
|
90
|
+
exactly which version scored a given run.
|
|
91
|
+
|
|
92
|
+
The distribution is `ratchet-gate`; the command is available as both
|
|
93
|
+
`ratchet-gate` and `quality-gate`, so an adopting workflow can keep whichever
|
|
94
|
+
name it already says.
|
|
95
|
+
|
|
96
|
+
From a machine on the LAN you can install straight from the source of truth
|
|
97
|
+
instead, which is what `scripts/verify-against-repo.sh` does when checking an
|
|
98
|
+
unreleased commit:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
uvx --from git+https://git.marim.dev/mateuscmarim/quality-gate@v0.1.0 quality-gate check
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
`git.marim.dev` has no public DNS record, so that form works from self-hosted
|
|
105
|
+
runners and nowhere else. PyPI is the form to put in a workflow. See
|
|
106
|
+
[RELEASING.md](RELEASING.md).
|
|
107
|
+
|
|
108
|
+
## Use
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
quality-gate check # score this run, exit 1 on regression
|
|
112
|
+
quality-gate check --baseline-ref origin/main # score against the stricter baseline
|
|
113
|
+
quality-gate check --measured api,worker # only these packages ran
|
|
114
|
+
quality-gate promote --monotonic # adopt improvements only (post-merge)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Point it at a directory of tool output — `.quality/<package>/coverage.json`,
|
|
118
|
+
`ruff.json`, `mypy.txt`, and so on — and a `quality-baseline.json`.
|
|
119
|
+
|
|
120
|
+
## Configure
|
|
121
|
+
|
|
122
|
+
`quality-gate.toml` at the repo root. Every key is optional; a repo whose layout
|
|
123
|
+
matches the defaults needs no file at all.
|
|
124
|
+
|
|
125
|
+
```toml
|
|
126
|
+
[gate]
|
|
127
|
+
artifacts_dir = ".quality"
|
|
128
|
+
baseline_path = "quality-baseline.json"
|
|
129
|
+
non_ratcheted_dirs = ["tests", "scripts", "notebooks"]
|
|
130
|
+
default_baseline_ref = "origin/main"
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
`non_ratcheted_dirs` is the one most repos change. Findings in those directories
|
|
134
|
+
stay in the uploaded artifacts — they just do not gate. One-off analysis
|
|
135
|
+
scripts, notebook data collectors and test helpers are read top-to-bottom and
|
|
136
|
+
run by hand; a 200-line `main()` is the right shape for what they are.
|
|
137
|
+
|
|
138
|
+
## Two things it will not do
|
|
139
|
+
|
|
140
|
+
**It will not let a branch set its own bar.** Every number the gate enforces
|
|
141
|
+
comes out of a tracked file on the branch being scored, so the same commit that
|
|
142
|
+
adds 47 lint errors can raise `lint_errors` to 47 and pass. That needs no
|
|
143
|
+
malice: `promote` without `--monotonic` sets baseline = current for *every*
|
|
144
|
+
metric, so a branch that runs it locally to seed one new row rewrites all the
|
|
145
|
+
others. Pass `--baseline-ref origin/main` in CI and each metric is scored
|
|
146
|
+
against the stricter of the two baselines.
|
|
147
|
+
|
|
148
|
+
**It will not go green when it could not measure something.** A missing
|
|
149
|
+
`gitleaks.json` is not a clean secret scan; a crashed `bandit` is not zero
|
|
150
|
+
findings; a jscpd report that never got written is not zero duplication. Every
|
|
151
|
+
one of those is an error that exits 1. All three used to be written the other
|
|
152
|
+
way round, and each one silently disabled the check it was reporting on.
|
|
153
|
+
|
|
154
|
+
## Supported
|
|
155
|
+
|
|
156
|
+
| Language | Tools |
|
|
157
|
+
|---|---|
|
|
158
|
+
| Python | coverage.py, ruff (lint + complexity), mypy, bandit |
|
|
159
|
+
| JavaScript / TypeScript | vitest, biome, tsc, npm audit |
|
|
160
|
+
| Kotlin / Android | Kover, detekt, Android Lint, kotlinc |
|
|
161
|
+
| Repo-wide | jscpd, gitleaks, pip-audit, npm audit |
|
|
162
|
+
|
|
163
|
+
## Developing
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
uv venv && uv pip install -e '.[dev]'
|
|
167
|
+
.venv/bin/python -m pytest # includes the equivalence suite
|
|
168
|
+
.venv/bin/python -m mypy # --strict, src and tests
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
The suite runs the implementation this package replaces side by side with this
|
|
172
|
+
one over identical inputs and asserts their stdout, stderr, exit code and
|
|
173
|
+
written files match exactly. That is what lets a repo switch over without its
|
|
174
|
+
gate changing verdict on the day it does. Every intended departure is listed in
|
|
175
|
+
[DIVERGENCES.md](DIVERGENCES.md); anything else is a bug.
|
|
176
|
+
|
|
177
|
+
To check against a repo's real reports rather than synthetic ones:
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
scripts/verify-against-repo.sh ../../trainwithme/nasa nasa-server
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Design notes live in the Quality Gate project on mddocs.
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# quality-gate
|
|
2
|
+
|
|
3
|
+
A **ratchet** quality gate. It reads what your linters, type checkers, test
|
|
4
|
+
runners and scanners already wrote, compares every metric to a committed
|
|
5
|
+
baseline, and fails the build if any of them got worse.
|
|
6
|
+
|
|
7
|
+
Not a threshold gate. There is no number you have to reach before it is useful —
|
|
8
|
+
the baseline starts wherever your code is today, and the only rule is that it
|
|
9
|
+
may not move backwards. Improvements are adopted automatically after merge, so
|
|
10
|
+
the bar rises on its own and never falls.
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
## Ratchet Quality Gate
|
|
14
|
+
|
|
15
|
+
| Package | Metric | Baseline | Current | Δ | Status |
|
|
16
|
+
|---|---|---:|---:|---:|:---:|
|
|
17
|
+
| `api` | coverage_lines_pct | 89.09 | 88.22 | -0.87 | FAIL |
|
|
18
|
+
| `api` | complexity_violations | 108 | 108 | +0 | ok |
|
|
19
|
+
| `api` | mypy_errors | 0 | 0 | +0 | ok |
|
|
20
|
+
| `(global)` | secret_findings | 0 | 0 | +0 | ok |
|
|
21
|
+
|
|
22
|
+
**FAIL** — 1 regression(s):
|
|
23
|
+
- `api` / **coverage_lines_pct** 89.09 → 88.22
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Why this repo exists
|
|
27
|
+
|
|
28
|
+
Five repos were running five copies of the same 400–1500 line script. They had
|
|
29
|
+
diverged: different exclusion lists, different report paths, one repo carrying a
|
|
30
|
+
one-line bug fix that never reached the other four because there was no
|
|
31
|
+
mechanism for it to travel by. Comparing them, almost every difference was
|
|
32
|
+
configuration wearing the costume of code.
|
|
33
|
+
|
|
34
|
+
So the program lives here once, and everything that legitimately differs per
|
|
35
|
+
repo lives in that repo's `quality-gate.toml`.
|
|
36
|
+
|
|
37
|
+
## Install
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
uvx ratchet-gate@0.1.0 check
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
No install step in CI, no vendored copy to keep in sync, and the pin says
|
|
44
|
+
exactly which version scored a given run.
|
|
45
|
+
|
|
46
|
+
The distribution is `ratchet-gate`; the command is available as both
|
|
47
|
+
`ratchet-gate` and `quality-gate`, so an adopting workflow can keep whichever
|
|
48
|
+
name it already says.
|
|
49
|
+
|
|
50
|
+
From a machine on the LAN you can install straight from the source of truth
|
|
51
|
+
instead, which is what `scripts/verify-against-repo.sh` does when checking an
|
|
52
|
+
unreleased commit:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
uvx --from git+https://git.marim.dev/mateuscmarim/quality-gate@v0.1.0 quality-gate check
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
`git.marim.dev` has no public DNS record, so that form works from self-hosted
|
|
59
|
+
runners and nowhere else. PyPI is the form to put in a workflow. See
|
|
60
|
+
[RELEASING.md](RELEASING.md).
|
|
61
|
+
|
|
62
|
+
## Use
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
quality-gate check # score this run, exit 1 on regression
|
|
66
|
+
quality-gate check --baseline-ref origin/main # score against the stricter baseline
|
|
67
|
+
quality-gate check --measured api,worker # only these packages ran
|
|
68
|
+
quality-gate promote --monotonic # adopt improvements only (post-merge)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Point it at a directory of tool output — `.quality/<package>/coverage.json`,
|
|
72
|
+
`ruff.json`, `mypy.txt`, and so on — and a `quality-baseline.json`.
|
|
73
|
+
|
|
74
|
+
## Configure
|
|
75
|
+
|
|
76
|
+
`quality-gate.toml` at the repo root. Every key is optional; a repo whose layout
|
|
77
|
+
matches the defaults needs no file at all.
|
|
78
|
+
|
|
79
|
+
```toml
|
|
80
|
+
[gate]
|
|
81
|
+
artifacts_dir = ".quality"
|
|
82
|
+
baseline_path = "quality-baseline.json"
|
|
83
|
+
non_ratcheted_dirs = ["tests", "scripts", "notebooks"]
|
|
84
|
+
default_baseline_ref = "origin/main"
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
`non_ratcheted_dirs` is the one most repos change. Findings in those directories
|
|
88
|
+
stay in the uploaded artifacts — they just do not gate. One-off analysis
|
|
89
|
+
scripts, notebook data collectors and test helpers are read top-to-bottom and
|
|
90
|
+
run by hand; a 200-line `main()` is the right shape for what they are.
|
|
91
|
+
|
|
92
|
+
## Two things it will not do
|
|
93
|
+
|
|
94
|
+
**It will not let a branch set its own bar.** Every number the gate enforces
|
|
95
|
+
comes out of a tracked file on the branch being scored, so the same commit that
|
|
96
|
+
adds 47 lint errors can raise `lint_errors` to 47 and pass. That needs no
|
|
97
|
+
malice: `promote` without `--monotonic` sets baseline = current for *every*
|
|
98
|
+
metric, so a branch that runs it locally to seed one new row rewrites all the
|
|
99
|
+
others. Pass `--baseline-ref origin/main` in CI and each metric is scored
|
|
100
|
+
against the stricter of the two baselines.
|
|
101
|
+
|
|
102
|
+
**It will not go green when it could not measure something.** A missing
|
|
103
|
+
`gitleaks.json` is not a clean secret scan; a crashed `bandit` is not zero
|
|
104
|
+
findings; a jscpd report that never got written is not zero duplication. Every
|
|
105
|
+
one of those is an error that exits 1. All three used to be written the other
|
|
106
|
+
way round, and each one silently disabled the check it was reporting on.
|
|
107
|
+
|
|
108
|
+
## Supported
|
|
109
|
+
|
|
110
|
+
| Language | Tools |
|
|
111
|
+
|---|---|
|
|
112
|
+
| Python | coverage.py, ruff (lint + complexity), mypy, bandit |
|
|
113
|
+
| JavaScript / TypeScript | vitest, biome, tsc, npm audit |
|
|
114
|
+
| Kotlin / Android | Kover, detekt, Android Lint, kotlinc |
|
|
115
|
+
| Repo-wide | jscpd, gitleaks, pip-audit, npm audit |
|
|
116
|
+
|
|
117
|
+
## Developing
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
uv venv && uv pip install -e '.[dev]'
|
|
121
|
+
.venv/bin/python -m pytest # includes the equivalence suite
|
|
122
|
+
.venv/bin/python -m mypy # --strict, src and tests
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
The suite runs the implementation this package replaces side by side with this
|
|
126
|
+
one over identical inputs and asserts their stdout, stderr, exit code and
|
|
127
|
+
written files match exactly. That is what lets a repo switch over without its
|
|
128
|
+
gate changing verdict on the day it does. Every intended departure is listed in
|
|
129
|
+
[DIVERGENCES.md](DIVERGENCES.md); anything else is a bug.
|
|
130
|
+
|
|
131
|
+
To check against a repo's real reports rather than synthetic ones:
|
|
132
|
+
|
|
133
|
+
```bash
|
|
134
|
+
scripts/verify-against-repo.sh ../../trainwithme/nasa nasa-server
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Design notes live in the Quality Gate project on mddocs.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# Releasing
|
|
2
|
+
|
|
3
|
+
The package is published to PyPI as [`ratchet-gate`](https://pypi.org/project/ratchet-gate/).
|
|
4
|
+
|
|
5
|
+
## Why PyPI, and not the git tag
|
|
6
|
+
|
|
7
|
+
Installing straight from `git+https://git.marim.dev/...` works, and the tag is
|
|
8
|
+
still the source of truth for *what* was released. It just isn't reachable from
|
|
9
|
+
everywhere: `git.marim.dev` resolves to `192.168.30.1` and has no public DNS
|
|
10
|
+
record, so it works from the LAN and from self-hosted Gitea runners, and not at
|
|
11
|
+
all from GitHub-hosted ones.
|
|
12
|
+
|
|
13
|
+
`nasa` runs its gate on both forges from a single workflow file. PyPI is the one
|
|
14
|
+
place every runner can already reach without a secret, a VPN, or a second copy
|
|
15
|
+
of the workflow.
|
|
16
|
+
|
|
17
|
+
## One-time setup
|
|
18
|
+
|
|
19
|
+
1. Create the project on PyPI by uploading the first release (below). The name
|
|
20
|
+
`ratchet-gate` was unclaimed as of 2026-08-27.
|
|
21
|
+
2. Mint a project-scoped API token at <https://pypi.org/manage/account/token/>.
|
|
22
|
+
3. Add it to this repo as the `PYPI_TOKEN` secret, under
|
|
23
|
+
*Settings → Actions → Secrets*.
|
|
24
|
+
|
|
25
|
+
Until step 3 is done, the release workflow fails loudly rather than skipping the
|
|
26
|
+
upload — a release that silently didn't publish is worse than one that didn't
|
|
27
|
+
run.
|
|
28
|
+
|
|
29
|
+
## Cutting a release
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
# 1. Bump the version. The workflow refuses to publish a tag that disagrees
|
|
33
|
+
# with it, so these two are never allowed to drift.
|
|
34
|
+
$EDITOR pyproject.toml
|
|
35
|
+
|
|
36
|
+
# 2. Prove it against a real repo, not just the suite. This is the check that
|
|
37
|
+
# catches an extraction moving a number.
|
|
38
|
+
uv build
|
|
39
|
+
GATE="uvx --from ./dist/ratchet_gate-*.whl ratchet-gate" \
|
|
40
|
+
scripts/verify-against-repo.sh ../../trainwithme/nasa nasa-server
|
|
41
|
+
|
|
42
|
+
# 3. Tag and push. The workflow tests, checks the tag, builds and publishes.
|
|
43
|
+
git commit -am "Release vX.Y.Z"
|
|
44
|
+
git tag -a vX.Y.Z -m "vX.Y.Z"
|
|
45
|
+
git push origin main --follow-tags
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Publishing by hand
|
|
49
|
+
|
|
50
|
+
If the runner is unavailable:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
uv build
|
|
54
|
+
UV_PUBLISH_TOKEN=pypi-... uv publish
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## Versions are a contract with five workflows
|
|
58
|
+
|
|
59
|
+
Adopting repos pin an exact version (`uvx ratchet-gate@X.Y.Z`), so a release is
|
|
60
|
+
not visible to anyone until their workflow bumps. That is the point: the gate
|
|
61
|
+
must not change what it measures underneath a PR that is already open.
|
|
62
|
+
|
|
63
|
+
- **Patch** — a fix that cannot move a metric. Parser tolerance, error wording,
|
|
64
|
+
packaging.
|
|
65
|
+
- **Minor** — new metrics, new languages, new flags. Existing baselines keep
|
|
66
|
+
scoring the same.
|
|
67
|
+
- **Major** — anything that can change a verdict on an unchanged repo: scoring
|
|
68
|
+
rules, direction defaults, baseline schema.
|
|
69
|
+
|
|
70
|
+
Every release should be able to say which of the three it is, and
|
|
71
|
+
`scripts/verify-against-repo.sh` is how you find out you were wrong.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Reference CI workflow. Two jobs and one rule between them: pull requests are
|
|
2
|
+
# scored against main's baseline, and only main is ever allowed to move it.
|
|
3
|
+
name: quality-gate
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
pull_request:
|
|
7
|
+
push:
|
|
8
|
+
branches: [main]
|
|
9
|
+
|
|
10
|
+
env:
|
|
11
|
+
GATE: git+https://git.marim.dev/marim.dev/quality-gate@v0.1.0
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
check:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
with:
|
|
19
|
+
# --baseline-ref reads origin/main out of git, so it has to be here.
|
|
20
|
+
# A shallow clone makes the reference unreadable, and the gate fails
|
|
21
|
+
# closed rather than falling back -- which is correct, and confusing
|
|
22
|
+
# if you have not seen it before.
|
|
23
|
+
fetch-depth: 0
|
|
24
|
+
|
|
25
|
+
- uses: astral-sh/setup-uv@v5
|
|
26
|
+
|
|
27
|
+
# ... your own steps here: run the tools, write their output under
|
|
28
|
+
# .quality/<package>/. Every one of them under `|| true`, so a tool that
|
|
29
|
+
# exits non-zero on findings does not fail the job before the gate reads
|
|
30
|
+
# it. The gate is what decides the verdict.
|
|
31
|
+
|
|
32
|
+
- name: Gate
|
|
33
|
+
run: |
|
|
34
|
+
uvx --from "$GATE" quality-gate check \
|
|
35
|
+
--baseline-ref origin/main \
|
|
36
|
+
--summary-out "$GITHUB_STEP_SUMMARY"
|
|
37
|
+
|
|
38
|
+
- uses: actions/upload-artifact@v4
|
|
39
|
+
if: always()
|
|
40
|
+
with:
|
|
41
|
+
name: quality-reports
|
|
42
|
+
path: .quality/
|
|
43
|
+
|
|
44
|
+
promote:
|
|
45
|
+
# Only on main, only after a green check. `promote --monotonic` adopts
|
|
46
|
+
# improvements and never loosens, so this can run unattended: a metric that
|
|
47
|
+
# held steady or drifted within tolerance keeps its prior baseline, and a
|
|
48
|
+
# re-trigger finds nothing new to move.
|
|
49
|
+
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
|
|
50
|
+
needs: check
|
|
51
|
+
runs-on: ubuntu-latest
|
|
52
|
+
permissions:
|
|
53
|
+
contents: write
|
|
54
|
+
steps:
|
|
55
|
+
- uses: actions/checkout@v4
|
|
56
|
+
- uses: astral-sh/setup-uv@v5
|
|
57
|
+
# ... run the tools again here, same as above ...
|
|
58
|
+
- name: Ratchet
|
|
59
|
+
run: |
|
|
60
|
+
uvx --from "$GATE" quality-gate promote --monotonic
|
|
61
|
+
git config user.name "quality-gate"
|
|
62
|
+
git config user.email "quality-gate@users.noreply.github.com"
|
|
63
|
+
git add quality-baseline.json
|
|
64
|
+
git diff --cached --quiet || git commit -m "chore: ratchet quality baseline"
|
|
65
|
+
git push
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
# The distribution is `ratchet-gate`, not `quality-gate`. "Quality gate" is
|
|
3
|
+
# SonarQube's term for a pass/fail threshold, and a threshold is precisely what
|
|
4
|
+
# this is not -- it scores against a committed baseline and only ever demands
|
|
5
|
+
# that the number move the right way. The free generic name was not worth the
|
|
6
|
+
# confusion, or the squat.
|
|
7
|
+
name = "ratchet-gate"
|
|
8
|
+
version = "0.1.0"
|
|
9
|
+
description = "Ratchet quality gate: enforce monotonic improvement of code-quality metrics against a committed baseline."
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
requires-python = ">=3.12"
|
|
12
|
+
license = { file = "LICENSE" }
|
|
13
|
+
authors = [{ name = "Mateus Marim" }]
|
|
14
|
+
keywords = ["ci", "code-quality", "ratchet", "baseline", "coverage", "linting"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
"Programming Language :: Python :: 3.13",
|
|
22
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
23
|
+
"Topic :: Software Development :: Testing",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
dependencies = ["pydantic>=2.9"]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://git.marim.dev/mateuscmarim/quality-gate"
|
|
30
|
+
Source = "https://git.marim.dev/mateuscmarim/quality-gate"
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
# Two names for one entry point. `ratchet-gate` matches the distribution, so
|
|
34
|
+
# `uvx ratchet-gate check` needs no `--from`; `quality-gate` is what five repos'
|
|
35
|
+
# workflows already say, and keeping it means adoption is a one-line diff.
|
|
36
|
+
ratchet-gate = "quality_gate.cli:main"
|
|
37
|
+
quality-gate = "quality_gate.cli:main"
|
|
38
|
+
|
|
39
|
+
[project.optional-dependencies]
|
|
40
|
+
dev = ["pytest>=8", "mypy>=1.11"]
|
|
41
|
+
|
|
42
|
+
[build-system]
|
|
43
|
+
requires = ["hatchling"]
|
|
44
|
+
build-backend = "hatchling.build"
|
|
45
|
+
|
|
46
|
+
[tool.hatch.build.targets.wheel]
|
|
47
|
+
packages = ["src/quality_gate"]
|
|
48
|
+
|
|
49
|
+
[tool.mypy]
|
|
50
|
+
python_version = "3.12"
|
|
51
|
+
strict = true
|
|
52
|
+
plugins = ["pydantic.mypy"]
|
|
53
|
+
files = ["src", "tests"]
|
|
54
|
+
# The vendored original is evidence, not source. It does not type-check
|
|
55
|
+
# under --strict, which is itself part of why it was worth replacing.
|
|
56
|
+
exclude = "tests/reference/"
|
|
57
|
+
|
|
58
|
+
[[tool.mypy.overrides]]
|
|
59
|
+
module = "tests.*"
|
|
60
|
+
disallow_untyped_defs = true
|
|
61
|
+
|
|
62
|
+
[tool.pytest.ini_options]
|
|
63
|
+
testpaths = ["tests"]
|
|
64
|
+
addopts = "-q"
|
|
65
|
+
|
|
66
|
+
[tool.ruff]
|
|
67
|
+
line-length = 100
|
|
68
|
+
target-version = "py312"
|