python-wheels 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- python_wheels-0.1.0.dist-info/METADATA +128 -0
- python_wheels-0.1.0.dist-info/RECORD +10 -0
- python_wheels-0.1.0.dist-info/WHEEL +4 -0
- python_wheels-0.1.0.dist-info/entry_points.txt +3 -0
- python_wheels-0.1.0.dist-info/licenses/LICENSE +21 -0
- pywheels/__init__.py +5 -0
- pywheels/cli.py +415 -0
- pywheels/github_release.py +125 -0
- pywheels/overrides.json +6 -0
- pywheels/verify.py +330 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: python-wheels
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Verify Sigstore-attested wheels from python-wheels-builds before installing
|
|
5
|
+
Project-URL: Homepage, https://github.com/patrickryankenneth/python-wheels
|
|
6
|
+
Project-URL: Builds, https://github.com/patrickryankenneth/python-wheels-builds
|
|
7
|
+
Project-URL: Index, https://python-wheels.github.io
|
|
8
|
+
Project-URL: Issues, https://github.com/patrickryankenneth/python-wheels/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/patrickryankenneth/python-wheels/blob/main/CHANGELOG.md
|
|
10
|
+
Author-email: Patrick Ryan <patrickryankenneth@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: attestation,pip,provenance,sigstore,slsa,supply-chain,wheels
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Environment :: Console
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Topic :: Security
|
|
20
|
+
Classifier: Topic :: System :: Software Distribution
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Provides-Extra: sigstore
|
|
23
|
+
Requires-Dist: sigstore<4.6,>=4.0; extra == 'sigstore'
|
|
24
|
+
Provides-Extra: test
|
|
25
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# python-wheels
|
|
29
|
+
|
|
30
|
+
**Status: working - v0.1.0.** The `python-wheels` CLI (command: `pywheels`,
|
|
31
|
+
also installed as `python-wheels`) verifies and installs attested wheels
|
|
32
|
+
today. First attested build shipped: `dbt-oss` for `win_arm64`.
|
|
33
|
+
|
|
34
|
+
## The problem
|
|
35
|
+
|
|
36
|
+
Some Python packages don't ship a wheel for your platform: an
|
|
37
|
+
uncommon architecture, Alpine/musl instead of glibc, an OS upstream doesn't
|
|
38
|
+
target, or just a combination nobody's gotten around to building for. Right
|
|
39
|
+
now the options are "build it from source yourself, every time" or "hope
|
|
40
|
+
someone in the community hosts one somewhere."
|
|
41
|
+
|
|
42
|
+
It's also not always upstream being lazy. PyPI caps a project at 10GB of
|
|
43
|
+
total storage, and wheels for every platform/arch/Python-version
|
|
44
|
+
combination add up fast - that's part of why projects like PyTorch host
|
|
45
|
+
some of their wheels off-PyPI and point people at an extra index instead.
|
|
46
|
+
`python-wheels` is aimed at exactly that situation: popular packages that
|
|
47
|
+
can't or don't publish a wheel for your platform, for any reason.
|
|
48
|
+
|
|
49
|
+
## What it does
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install python-wheels
|
|
53
|
+
pywheels install dbt-oss==2.0.5 \
|
|
54
|
+
--repo patrickryankenneth/python-wheels-builds \
|
|
55
|
+
--tag dbt-oss-v2.0.5 \
|
|
56
|
+
--workflow build-dbt-oss-win-arm64.yml
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
1. **Checks upstream first.** If pip can already resolve real wheels for
|
|
60
|
+
the package - and its whole dependency closure, not just the top-level
|
|
61
|
+
package - on your platform, it says so and prints the plain
|
|
62
|
+
`pip install` command; this tool gets out of the way whenever upstream
|
|
63
|
+
already has you covered. A small per-package override list
|
|
64
|
+
(`overrides.json`) handles the rare case where a package's real wheel
|
|
65
|
+
availability is hidden from pip's normal resolution - e.g. `dbt-oss`,
|
|
66
|
+
whose PyPI sdist is a stub build backend that fetches a real prebuilt
|
|
67
|
+
wheel from GitHub, keyed by platform, with no source to fall back to.
|
|
68
|
+
2. **Falls back, but verifies.** If there's no usable upstream wheel, it
|
|
69
|
+
fetches the matching release from
|
|
70
|
+
[python-wheels-builds](https://github.com/patrickryankenneth/python-wheels-builds)
|
|
71
|
+
and checks, before recommending anything:
|
|
72
|
+
- its **build provenance attestation** (SLSA - built by the expected CI
|
|
73
|
+
workflow, from the expected repo and ref, unmodified since),
|
|
74
|
+
- its **upstream-source attestation** (built from the real, tagged
|
|
75
|
+
upstream release - not a fork or a patched copy),
|
|
76
|
+
- the **source archive's digest** against what that attestation
|
|
77
|
+
recorded, when a source archive is available.
|
|
78
|
+
|
|
79
|
+
Verification runs offline via [sigstore](https://pypi.org/project/sigstore/)
|
|
80
|
+
(`pip install python-wheels[sigstore]`) if it's installed, or falls back
|
|
81
|
+
to the `gh` CLI (no pip dependency, needs network) if it isn't. Run
|
|
82
|
+
`pywheels doctor` to see which backend is usable on your machine.
|
|
83
|
+
3. **Fails loudly, not silently.** `pywheels install` never runs
|
|
84
|
+
`pip install` for you - it only ever prints the exact command that's
|
|
85
|
+
safe to run. If verification fails, or neither backend is available at
|
|
86
|
+
all, it says so plainly and falls back to `pip install
|
|
87
|
+
--no-binary=:all:` (unverified, from source) as the last resort - never
|
|
88
|
+
a silent, unattested install.
|
|
89
|
+
|
|
90
|
+
`pywheels verify` runs the same verification directly against a wheel
|
|
91
|
+
that's already on disk, or a specific release tag - useful for checking a
|
|
92
|
+
build without going through the whole `install` flow.
|
|
93
|
+
|
|
94
|
+
## Where the wheels actually come from
|
|
95
|
+
|
|
96
|
+
This repo is only the installer and verifier. The wheels themselves are
|
|
97
|
+
built and attested in
|
|
98
|
+
[python-wheels-builds](https://github.com/patrickryankenneth/python-wheels-builds),
|
|
99
|
+
and also served as a plain [PEP 503](https://peps.python.org/pep-0503/)
|
|
100
|
+
index at [python-wheels.github.io](https://python-wheels.github.io) for
|
|
101
|
+
anyone who just wants `pip install --extra-index-url
|
|
102
|
+
https://python-wheels.github.io/simple/ <package>` without the
|
|
103
|
+
verification step. `pywheels` is the piece that ties the CLI, the
|
|
104
|
+
release repo, and the attestation checks together so you don't have to run
|
|
105
|
+
`gh attestation verify` by hand.
|
|
106
|
+
|
|
107
|
+
## Longer-term direction
|
|
108
|
+
|
|
109
|
+
The build side is currently one hand-tuned workflow for one package
|
|
110
|
+
(`dbt-oss`/`dbt-core` on Windows ARM64). The goal is to turn that into a
|
|
111
|
+
standardized, reproducible build recipe that can target other popular PyPI
|
|
112
|
+
packages missing wheels for a given platform - starting with the platforms
|
|
113
|
+
that come up most often (Alpine/musl, less-common architectures, newer
|
|
114
|
+
Python versions upstream hasn't built for yet), rather than trying to
|
|
115
|
+
cover everything at once.
|
|
116
|
+
|
|
117
|
+
## Not yet decided
|
|
118
|
+
|
|
119
|
+
- CLI surface beyond `install`/`verify`/`doctor` (list available
|
|
120
|
+
platforms? show why a package fell back? a way to request a new
|
|
121
|
+
package/platform combo?)
|
|
122
|
+
- How package/platform requests get prioritized once this covers more than
|
|
123
|
+
one package
|
|
124
|
+
- Whether to eventually provide a wrapper mode that invokes pip directly after verification,
|
|
125
|
+
versus keeping the strict "print the safe command only" separation.
|
|
126
|
+
|
|
127
|
+
Contributions and issues on any of the above are welcome - this project is
|
|
128
|
+
still early, even with a first attested build shipped.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
pywheels/__init__.py,sha256=TpM3MTXInLX2wP0p91p0IXOxENixCmV8NyW4FHW14ts,206
|
|
2
|
+
pywheels/cli.py,sha256=YxC7b5dSAWTQd19CelVH7dgyGWmeSyzy0B3gZT1BFhY,20214
|
|
3
|
+
pywheels/github_release.py,sha256=X2e7UEfABcsS8tktq6VZQTmpX_TbB7b1qf-y1zUHXdo,4930
|
|
4
|
+
pywheels/overrides.json,sha256=0SkJcZMLUvwC5wcINEn2jgBfqUcgmyWbuAqfYHcuXNs,566
|
|
5
|
+
pywheels/verify.py,sha256=NMxiQBhxnkMcxJJyziIhBPs0P7tts1lY2NVm-kpA2ok,14210
|
|
6
|
+
python_wheels-0.1.0.dist-info/METADATA,sha256=qY5xUSI5j7PLVcuU7ATutEOotPBUUZCZ3rmSJAYAtY0,6261
|
|
7
|
+
python_wheels-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
8
|
+
python_wheels-0.1.0.dist-info/entry_points.txt,sha256=Y_IzU_QVw6XqzTqsf5L-2uUxOslWaZYA4PRWT9NvFBA,81
|
|
9
|
+
python_wheels-0.1.0.dist-info/licenses/LICENSE,sha256=Qzm3fHs1tWQQVJDoxOjYz6Y3ukVfcC58oVHkkM-2SR0,1069
|
|
10
|
+
python_wheels-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Patrick Ryan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
pywheels/__init__.py
ADDED
pywheels/cli.py
ADDED
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
"""pywheels: fetch and verify a wheel from a python-wheels-builds release
|
|
2
|
+
before anything gets near `pip install`.
|
|
3
|
+
|
|
4
|
+
pywheels verify dbt-core --repo you/python-wheels-builds \
|
|
5
|
+
--tag dbt-oss-v2.0.5 --workflow build-dbt-oss-win-arm64.yml
|
|
6
|
+
|
|
7
|
+
pywheels verify dbt-core --repo you/python-wheels-builds \
|
|
8
|
+
--workflow smoke-test-build-dbt-oss.yml --local-dir ./pywheels-sample
|
|
9
|
+
|
|
10
|
+
pywheels doctor
|
|
11
|
+
|
|
12
|
+
Backend selection (--backend auto|sigstore|gh, default auto): try sigstore
|
|
13
|
+
first (offline, needs the optional `sigstore` extra), fall back to the
|
|
14
|
+
`gh` CLI if that's not installed (needs network, needs nothing from pip).
|
|
15
|
+
If neither is available, verification of one of pywheels' own wheels fails
|
|
16
|
+
loudly - see verify.py's VerificationError, not a silent pass-through.
|
|
17
|
+
|
|
18
|
+
--repo defaults to $PYWHEELS_REPO so a single-project install of this CLI
|
|
19
|
+
doesn't have to pass it on every call; set that once in your shell profile,
|
|
20
|
+
or keep passing --repo explicitly if you point pywheels at more than one
|
|
21
|
+
release repo.
|
|
22
|
+
|
|
23
|
+
pywheels install dbt-core==2.0.5 --tag dbt-oss-v2.0.5 \
|
|
24
|
+
--workflow build-dbt-oss-win-arm64.yml
|
|
25
|
+
|
|
26
|
+
v1 `install` does NOT call `pip install` for you. It only tells you which
|
|
27
|
+
`pip install ...` command is safe to run, because "safe" here specifically
|
|
28
|
+
means "verified", and we can only ever verify wheels signed by our own
|
|
29
|
+
--repo - never an arbitrary upstream wheel. Concretely, per package:
|
|
30
|
+
1. Can pip resolve real, non-source wheels for this package and its
|
|
31
|
+
whole dependency closure on your platform? If so, say so and print
|
|
32
|
+
the plain `pip install` command - unverified, since it's not ours to
|
|
33
|
+
attest, but not blocked either.
|
|
34
|
+
NOTE: for a pure-Python package like dbt-core itself this check is
|
|
35
|
+
almost meaningless on its own (dbt-core ships one universal
|
|
36
|
+
py3-none-any wheel for every platform) - the real gap for us is
|
|
37
|
+
always in a *dependency* that ships real per-platform wheels (e.g.
|
|
38
|
+
dbt-extractor, a Rust extension with no win-arm64 wheel on PyPI).
|
|
39
|
+
Because pip resolves the whole tree, not just the top-level package,
|
|
40
|
+
this check does still fail correctly on an unsupported platform -
|
|
41
|
+
it's just resolving the dependency graph, not "does dbt-core have a
|
|
42
|
+
wheel". Nothing here inspects any dependency's own download-a-binary
|
|
43
|
+
logic at import/run time (if one exists) - see the "not yet handled"
|
|
44
|
+
note below.
|
|
45
|
+
2. Otherwise, fetch the matching wheel from --repo/--tag and run it
|
|
46
|
+
through verify_wheel. If it verifies, print the `pip install` command
|
|
47
|
+
for the wheel now sitting in --workdir. If verification fails or the
|
|
48
|
+
backend can't run at all, say so plainly and do NOT print an install
|
|
49
|
+
command for it.
|
|
50
|
+
3. If neither worked, print the `pip install --no-binary=:all:` command
|
|
51
|
+
as the last resort, with no attestation behind it either.
|
|
52
|
+
|
|
53
|
+
Not yet handled generally: some upstream packages hide their real wheel
|
|
54
|
+
availability behind package-specific indirection that plain pip resolution
|
|
55
|
+
can't see through - e.g. dbt-oss, whose PyPI "sdist" is actually a stub
|
|
56
|
+
build backend that downloads a real wheel from GitHub, keyed by platform,
|
|
57
|
+
with no source to fall back to (see overrides.json). Add an entry there for
|
|
58
|
+
each such package as you confirm its actual mechanism (e.g. pytorch, which
|
|
59
|
+
needs a non-default --index-url to see real wheels at all) - don't guess at
|
|
60
|
+
a new package's mechanism speculatively; confirm it the way dbt-oss's was
|
|
61
|
+
confirmed, by reading what its build backend or index actually does.
|
|
62
|
+
|
|
63
|
+
`verify` always assumes you're pointing it at one of our own wheels; the
|
|
64
|
+
"is this actually a python-wheels artifact, or a normal upstream package
|
|
65
|
+
we don't sign" decision lives in `install`, above.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
from __future__ import annotations
|
|
69
|
+
|
|
70
|
+
import argparse
|
|
71
|
+
import json
|
|
72
|
+
import os
|
|
73
|
+
import platform
|
|
74
|
+
import subprocess
|
|
75
|
+
import sys
|
|
76
|
+
import tempfile
|
|
77
|
+
from pathlib import Path
|
|
78
|
+
from typing import Optional
|
|
79
|
+
|
|
80
|
+
from .github_release import fetch_release_assets, find_local_assets, get_latest_release_tag
|
|
81
|
+
from .verify import (
|
|
82
|
+
verify_wheel,
|
|
83
|
+
VerificationError,
|
|
84
|
+
sigstore_available,
|
|
85
|
+
gh_available,
|
|
86
|
+
DEFAULT_UPSTREAM_PREDICATE_TYPE,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
# A single-project install of this CLI is the common case, so --repo can be
|
|
90
|
+
# set once via env var instead of on every call. Still overridable per-call.
|
|
91
|
+
DEFAULT_REPO = os.environ.get("PYWHEELS_REPO")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
95
|
+
parser = argparse.ArgumentParser(prog="pywheels")
|
|
96
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
97
|
+
|
|
98
|
+
v = sub.add_parser("verify", help="verify a wheel's attestations before you trust it")
|
|
99
|
+
v.add_argument("package", help="package name, e.g. dbt-core")
|
|
100
|
+
v.add_argument("--repo", required=DEFAULT_REPO is None, default=DEFAULT_REPO,
|
|
101
|
+
help="owner/repo of the release, e.g. you/python-wheels-builds "
|
|
102
|
+
"(default: $PYWHEELS_REPO if set)")
|
|
103
|
+
v.add_argument("--tag", help="release tag, e.g. dbt-oss-v2.0.5 (ignored with --local-dir)")
|
|
104
|
+
v.add_argument("--workflow", required=True, help="workflow filename that signed it, e.g. build-dbt-oss-win-arm64.yml")
|
|
105
|
+
v.add_argument("--ref", default="refs/heads/main", help="git ref the signing workflow ran from (default: %(default)s)")
|
|
106
|
+
v.add_argument("--upstream-predicate-type", default=DEFAULT_UPSTREAM_PREDICATE_TYPE,
|
|
107
|
+
help="predicate type URL for the upstream-source attestation (default: %(default)s)")
|
|
108
|
+
v.add_argument("--workdir", default=".pywheels-cache", help="where downloaded assets are cached (default: %(default)s)")
|
|
109
|
+
v.add_argument("--local-dir", type=Path, default=None,
|
|
110
|
+
help="skip the GitHub fetch and verify files already on disk under this directory "
|
|
111
|
+
"instead (e.g. output of `gh run download`)")
|
|
112
|
+
v.add_argument("--backend", choices=["auto", "sigstore", "gh"], default="auto",
|
|
113
|
+
help="verification backend (default: %(default)s - sigstore if installed, else gh)")
|
|
114
|
+
v.add_argument("-v", "--verbose", action="count", default=0,
|
|
115
|
+
help="pass through extra logging to the backend. Repeatable (-vv). "
|
|
116
|
+
"sigstore: raises its own -v/-vv debug logging. "
|
|
117
|
+
"gh: has no verbosity flag, so this instead asks for --format=json "
|
|
118
|
+
"(the closest thing gh offers to a detailed result dump).")
|
|
119
|
+
|
|
120
|
+
i = sub.add_parser("install", help="print the safe pip install command for a package: verified ours, or plain upstream, or source")
|
|
121
|
+
i.add_argument("package", help="package name, optionally pinned, e.g. dbt-core or dbt-core==2.0.5")
|
|
122
|
+
i.add_argument("--repo", required=DEFAULT_REPO is None, default=DEFAULT_REPO,
|
|
123
|
+
help="owner/repo to check if upstream has no usable wheel "
|
|
124
|
+
"(default: $PYWHEELS_REPO if set)")
|
|
125
|
+
i.add_argument("--workflow", required=True, help="workflow filename that signs our wheels, e.g. build-dbt-oss-win-arm64.yml")
|
|
126
|
+
i.add_argument("--tag", default=None,
|
|
127
|
+
help="release tag to use from --repo (default: that repo's latest release). "
|
|
128
|
+
"Required if your tag naming doesn't match '<package>-v<version>' - "
|
|
129
|
+
"e.g. dbt-core==2.0.5 needs --tag dbt-oss-v2.0.5 explicitly, since the "
|
|
130
|
+
"release-tag prefix ('dbt-oss') and the pip package name ('dbt-core') differ.")
|
|
131
|
+
i.add_argument("--ref", default="refs/heads/main", help="git ref the signing workflow ran from (default: %(default)s)")
|
|
132
|
+
i.add_argument("--upstream-predicate-type", default=DEFAULT_UPSTREAM_PREDICATE_TYPE,
|
|
133
|
+
help="predicate type URL for the upstream-source attestation (default: %(default)s)")
|
|
134
|
+
i.add_argument("--index-url", default=None, help="pip index to check for upstream wheels (default: pip's own default, i.e. PyPI)")
|
|
135
|
+
i.add_argument("--workdir", default=".pywheels-cache", help="where our downloaded wheel is cached if we fetch one (default: %(default)s)")
|
|
136
|
+
i.add_argument("--backend", choices=["auto", "sigstore", "gh"], default="auto",
|
|
137
|
+
help="verification backend for our own wheel (default: %(default)s - sigstore if installed, else gh)")
|
|
138
|
+
i.add_argument("-v", "--verbose", action="count", default=0, help="pass through extra logging to the verify backend")
|
|
139
|
+
|
|
140
|
+
sub.add_parser("doctor", help="check which verification backends are usable, and how to fix the ones that aren't")
|
|
141
|
+
|
|
142
|
+
return parser
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _cmd_verify(args: argparse.Namespace) -> int:
|
|
146
|
+
if args.local_dir is None and not args.tag:
|
|
147
|
+
print("error: pass either --tag (to fetch a release) or --local-dir (to use files on disk)", file=sys.stderr)
|
|
148
|
+
return 2
|
|
149
|
+
|
|
150
|
+
try:
|
|
151
|
+
if args.local_dir is not None:
|
|
152
|
+
assets = find_local_assets(args.package, args.local_dir)
|
|
153
|
+
else:
|
|
154
|
+
dest = Path(args.workdir)
|
|
155
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
156
|
+
assets = fetch_release_assets(args.repo, args.tag, args.package, dest)
|
|
157
|
+
|
|
158
|
+
report = verify_wheel(
|
|
159
|
+
assets.wheel,
|
|
160
|
+
assets.bundle,
|
|
161
|
+
repo=args.repo,
|
|
162
|
+
workflow_file=args.workflow,
|
|
163
|
+
ref=args.ref,
|
|
164
|
+
upstream_predicate_type=args.upstream_predicate_type,
|
|
165
|
+
source_archive_path=assets.archive,
|
|
166
|
+
backend=args.backend,
|
|
167
|
+
verbose=args.verbose,
|
|
168
|
+
)
|
|
169
|
+
except (FileNotFoundError, VerificationError) as exc:
|
|
170
|
+
print(f"{args.package}: could not verify - {exc}", file=sys.stderr)
|
|
171
|
+
return 2
|
|
172
|
+
|
|
173
|
+
print(f"(using backend: {report.backend})")
|
|
174
|
+
for a in report.attestations:
|
|
175
|
+
status = "ignored" if a.ignored else ("OK" if a.verified else "FAILED")
|
|
176
|
+
print(f"[{status}] {a.predicate_type or '(unknown predicate)'}")
|
|
177
|
+
if not a.verified and not a.ignored:
|
|
178
|
+
print(f" {a.detail}", file=sys.stderr)
|
|
179
|
+
|
|
180
|
+
if report.archive_checked:
|
|
181
|
+
print(f"[{'OK' if report.archive_ok else 'FAILED'}] source archive digest")
|
|
182
|
+
if not report.archive_ok and report.archive_check_error:
|
|
183
|
+
print(f" {report.archive_check_error}", file=sys.stderr)
|
|
184
|
+
|
|
185
|
+
if report.ok:
|
|
186
|
+
print(f"\n{args.package}: verified OK")
|
|
187
|
+
return 0
|
|
188
|
+
|
|
189
|
+
print(f"\n{args.package}: VERIFICATION FAILED - refusing to trust this wheel", file=sys.stderr)
|
|
190
|
+
return 1
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _split_package_spec(spec: str):
|
|
194
|
+
"""'dbt-core==2.0.5' -> ('dbt-core', '2.0.5'); 'dbt-core' -> ('dbt-core', None)."""
|
|
195
|
+
if "==" in spec:
|
|
196
|
+
name, version = spec.split("==", 1)
|
|
197
|
+
return name, version
|
|
198
|
+
return spec, None
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
_OVERRIDES_PATH = Path(__file__).resolve().parent / "overrides.json"
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _load_overrides() -> dict:
|
|
205
|
+
try:
|
|
206
|
+
return json.loads(_OVERRIDES_PATH.read_text())
|
|
207
|
+
except FileNotFoundError:
|
|
208
|
+
return {}
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _pip_can_resolve_wheels(spec: str, index_url: Optional[str]) -> bool:
|
|
212
|
+
"""Default strategy. Dry-run only - downloads (never installs) `spec`
|
|
213
|
+
and its full dependency closure into a throwaway dir with
|
|
214
|
+
--only-binary=:all:, then deletes it. Success means pip found real
|
|
215
|
+
wheels for every package in that closure, not just the top-level one:
|
|
216
|
+
a pure-Python top-level package (like dbt-core's own py3-none-any
|
|
217
|
+
wheel) will "have a wheel" on every platform regardless, so this check
|
|
218
|
+
only means something because pip resolves dependencies too - that's
|
|
219
|
+
where a real per-platform gap (e.g. dbt-extractor having no win-arm64
|
|
220
|
+
wheel) actually shows up. --no-deps would defeat the point; don't add
|
|
221
|
+
it here.
|
|
222
|
+
|
|
223
|
+
Only correct when the package's own PyPI index metadata actually
|
|
224
|
+
reflects what it can serve. Some packages hide real wheel availability
|
|
225
|
+
behind a stub build backend or an alternate index - see overrides.json
|
|
226
|
+
and _pip_can_resolve_via_source_build for the confirmed exception."""
|
|
227
|
+
with tempfile.TemporaryDirectory(prefix="pywheels-check-") as tmp:
|
|
228
|
+
cmd = [sys.executable, "-m", "pip", "download", "--only-binary=:all:", "-d", tmp, spec]
|
|
229
|
+
if index_url:
|
|
230
|
+
cmd += ["--index-url", index_url]
|
|
231
|
+
return subprocess.run(cmd, capture_output=True).returncode == 0
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _pip_can_resolve_via_source_build(spec: str, index_url: Optional[str]) -> bool:
|
|
235
|
+
"""Override strategy "source-build-probe". For a package whose PyPI
|
|
236
|
+
"sdist" never actually compiles anything - it's a stub whose PEP 517
|
|
237
|
+
build backend downloads a real prebuilt wheel from elsewhere, keyed by
|
|
238
|
+
platform (see overrides.json's note for dbt-oss specifically) - the
|
|
239
|
+
only way to know if upstream covers this platform is to actually run
|
|
240
|
+
that backend. --no-deps: we're probing whether THIS package's own
|
|
241
|
+
backend serves this platform, not re-checking its dependencies (which
|
|
242
|
+
the default strategy already handles fine when they're ordinary
|
|
243
|
+
wheels, as dbt-oss's sole dependency `mashumaro` is)."""
|
|
244
|
+
with tempfile.TemporaryDirectory(prefix="pywheels-probe-") as tmp:
|
|
245
|
+
cmd = [sys.executable, "-m", "pip", "download", "--no-binary=:all:", "--no-deps", "-d", tmp, spec]
|
|
246
|
+
if index_url:
|
|
247
|
+
cmd += ["--index-url", index_url]
|
|
248
|
+
return subprocess.run(cmd, capture_output=True).returncode == 0
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
_STRATEGIES = {
|
|
252
|
+
"source-build-probe": _pip_can_resolve_via_source_build,
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _upstream_available(spec: str, package: str, index_url: Optional[str]) -> bool:
|
|
257
|
+
override = _load_overrides().get(package)
|
|
258
|
+
strategy = _STRATEGIES.get(override["strategy"]) if override else None
|
|
259
|
+
check = strategy or _pip_can_resolve_wheels
|
|
260
|
+
return check(spec, index_url)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _print_pip_cmd(*parts: str) -> None:
|
|
264
|
+
print(" " + " ".join(parts))
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _cmd_install(args: argparse.Namespace) -> int:
|
|
268
|
+
package, version = _split_package_spec(args.package)
|
|
269
|
+
spec = args.package
|
|
270
|
+
|
|
271
|
+
print(f"pywheels install {spec}: checking whether pip can resolve real wheels for your platform...")
|
|
272
|
+
if _upstream_available(spec, package, args.index_url):
|
|
273
|
+
print(f"\nupstream has wheels for {spec} and its dependencies on this platform.")
|
|
274
|
+
print("this is NOT attested by us - we only ever verify wheels from our own --repo. safe to run:\n")
|
|
275
|
+
cmd = ["pip", "install", spec]
|
|
276
|
+
if args.index_url:
|
|
277
|
+
cmd += ["--index-url", args.index_url]
|
|
278
|
+
_print_pip_cmd(*cmd)
|
|
279
|
+
return 0
|
|
280
|
+
|
|
281
|
+
print(f"\npip could not resolve real wheels for {spec} and its dependencies on this platform.")
|
|
282
|
+
print(f"checking {args.repo} for an attested build...")
|
|
283
|
+
|
|
284
|
+
if not args.tag and version:
|
|
285
|
+
print(
|
|
286
|
+
f"\ncan't guess a release tag from the version pin ({version}) - this repo's tag "
|
|
287
|
+
f"prefixes don't necessarily match the pip package name (e.g. tag 'dbt-oss-v2.0.5' "
|
|
288
|
+
f"for package 'dbt-core'). Pass --tag explicitly.",
|
|
289
|
+
file=sys.stderr,
|
|
290
|
+
)
|
|
291
|
+
return 2
|
|
292
|
+
|
|
293
|
+
try:
|
|
294
|
+
tag = args.tag or get_latest_release_tag(args.repo)
|
|
295
|
+
dest = Path(args.workdir)
|
|
296
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
297
|
+
assets = fetch_release_assets(args.repo, tag, package, dest)
|
|
298
|
+
except FileNotFoundError as exc:
|
|
299
|
+
print(f"{args.repo}: no wheel available either ({exc})")
|
|
300
|
+
print("nothing we can vouch for. building from source, unverified, is your only option:\n")
|
|
301
|
+
_print_pip_cmd("pip", "install", "--no-binary=:all:", spec)
|
|
302
|
+
return 1
|
|
303
|
+
|
|
304
|
+
try:
|
|
305
|
+
report = verify_wheel(
|
|
306
|
+
assets.wheel,
|
|
307
|
+
assets.bundle,
|
|
308
|
+
repo=args.repo,
|
|
309
|
+
workflow_file=args.workflow,
|
|
310
|
+
ref=args.ref,
|
|
311
|
+
upstream_predicate_type=args.upstream_predicate_type,
|
|
312
|
+
source_archive_path=assets.archive,
|
|
313
|
+
backend=args.backend,
|
|
314
|
+
verbose=args.verbose,
|
|
315
|
+
)
|
|
316
|
+
except VerificationError as exc:
|
|
317
|
+
print(f"{args.package}: could not verify ({exc})", file=sys.stderr)
|
|
318
|
+
print("refusing to recommend an unverifiable wheel. building from source, unverified:\n")
|
|
319
|
+
_print_pip_cmd("pip", "install", "--no-binary=:all:", spec)
|
|
320
|
+
return 2
|
|
321
|
+
|
|
322
|
+
print(f"(using backend: {report.backend})")
|
|
323
|
+
for a in report.attestations:
|
|
324
|
+
status = "ignored" if a.ignored else ("OK" if a.verified else "FAILED")
|
|
325
|
+
print(f"[{status}] {a.predicate_type or '(unknown predicate)'}")
|
|
326
|
+
if report.archive_checked:
|
|
327
|
+
print(f"[{'OK' if report.archive_ok else 'FAILED'}] source archive digest")
|
|
328
|
+
if not report.archive_ok and report.archive_check_error:
|
|
329
|
+
print(f" {report.archive_check_error}", file=sys.stderr)
|
|
330
|
+
|
|
331
|
+
if not report.ok:
|
|
332
|
+
print(f"\n{args.package}: VERIFICATION FAILED for {tag} - refusing to recommend this wheel", file=sys.stderr)
|
|
333
|
+
print("building from source, unverified, is your remaining option:\n")
|
|
334
|
+
_print_pip_cmd("pip", "install", "--no-binary=:all:", spec)
|
|
335
|
+
return 1
|
|
336
|
+
|
|
337
|
+
print(f"\n{args.package}: verified OK ({tag}). safe to run:\n")
|
|
338
|
+
_print_pip_cmd("pip", "install", str(assets.wheel))
|
|
339
|
+
return 0
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
_GH_INSTALL_HINTS = {
|
|
343
|
+
"Linux": "sudo apt install gh (Debian/Ubuntu) | see https://github.com/cli/cli/blob/trunk/docs/install_linux.md for other distros",
|
|
344
|
+
"Darwin": "brew install gh",
|
|
345
|
+
"Windows": "winget install --id GitHub.cli | or: choco install gh",
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _cmd_doctor(_args: argparse.Namespace) -> int:
|
|
350
|
+
print("pywheels doctor")
|
|
351
|
+
print("-" * 44)
|
|
352
|
+
any_backend = False
|
|
353
|
+
|
|
354
|
+
if sigstore_available():
|
|
355
|
+
proc = subprocess.run([sys.executable, "-m", "sigstore", "--version"], capture_output=True, text=True)
|
|
356
|
+
if proc.returncode == 0:
|
|
357
|
+
any_backend = True
|
|
358
|
+
print(f"[OK] sigstore backend: {(proc.stdout or proc.stderr).strip()}")
|
|
359
|
+
else:
|
|
360
|
+
print("[BROKEN] sigstore is installed but `python -m sigstore` did not run cleanly")
|
|
361
|
+
print(f" {(proc.stderr or proc.stdout).strip()}")
|
|
362
|
+
print(" fix: pip install --upgrade --force-reinstall sigstore")
|
|
363
|
+
else:
|
|
364
|
+
print("[missing] sigstore backend (optional)")
|
|
365
|
+
print(" fix: pip install sigstore (or: pip install pywheels[sigstore])")
|
|
366
|
+
|
|
367
|
+
if gh_available():
|
|
368
|
+
proc = subprocess.run(["gh", "--version"], capture_output=True, text=True)
|
|
369
|
+
version = (proc.stdout or proc.stderr).strip().splitlines()[0] if proc.stdout or proc.stderr else "gh"
|
|
370
|
+
any_backend = True
|
|
371
|
+
print(f"[OK] gh backend: {version}")
|
|
372
|
+
auth = subprocess.run(["gh", "auth", "status"], capture_output=True, text=True)
|
|
373
|
+
if auth.returncode != 0:
|
|
374
|
+
print(" note: gh is not authenticated (`gh auth login`) - public-repo verification "
|
|
375
|
+
"should still work, but may hit lower rate limits")
|
|
376
|
+
else:
|
|
377
|
+
print("[missing] gh backend (optional)")
|
|
378
|
+
hint = _GH_INSTALL_HINTS.get(platform.system(), "see https://cli.github.com")
|
|
379
|
+
print(f" fix: {hint}")
|
|
380
|
+
|
|
381
|
+
print("-" * 44)
|
|
382
|
+
if any_backend:
|
|
383
|
+
print("at least one verification backend is usable")
|
|
384
|
+
return 0
|
|
385
|
+
print("NO verification backend is usable - pywheels will refuse to verify any wheel")
|
|
386
|
+
print("install sigstore or gh (see above) before relying on this tool")
|
|
387
|
+
return 1
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def main(argv=None) -> None:
|
|
391
|
+
# stdout is block-buffered (not line-buffered) whenever it isn't a real
|
|
392
|
+
# terminal - which is exactly the case under CI - while stderr is
|
|
393
|
+
# always unbuffered. Left alone, that means every stderr line (our
|
|
394
|
+
# archive_check_error diagnostics, verification failure messages, etc)
|
|
395
|
+
# gets flushed immediately while stdout's lines sit queued until the
|
|
396
|
+
# process exits - so in a captured CI log, the "why it failed" lines
|
|
397
|
+
# appear to jump to the top, ahead of the OK/FAILED context they're
|
|
398
|
+
# explaining, instead of appearing where they were actually printed.
|
|
399
|
+
# Force line-buffering so stdout and stderr interleave in real order.
|
|
400
|
+
sys.stdout.reconfigure(line_buffering=True)
|
|
401
|
+
sys.stderr.reconfigure(line_buffering=True)
|
|
402
|
+
|
|
403
|
+
parser = _build_parser()
|
|
404
|
+
args = parser.parse_args(argv)
|
|
405
|
+
|
|
406
|
+
if args.command == "verify":
|
|
407
|
+
sys.exit(_cmd_verify(args))
|
|
408
|
+
elif args.command == "install":
|
|
409
|
+
sys.exit(_cmd_install(args))
|
|
410
|
+
elif args.command == "doctor":
|
|
411
|
+
sys.exit(_cmd_doctor(args))
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
if __name__ == "__main__":
|
|
415
|
+
main()
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Minimal GitHub Releases client: locate and download the assets pywheels
|
|
2
|
+
needs for one package from a release - the wheel, its attestation bundle,
|
|
3
|
+
and the source archive if one is attached.
|
|
4
|
+
|
|
5
|
+
Also supports finding those same three files in a local directory tree
|
|
6
|
+
(find_local_assets) - useful for testing against artifacts you already
|
|
7
|
+
have on disk (e.g. pulled via `gh run download`) without a live release to
|
|
8
|
+
fetch from.
|
|
9
|
+
|
|
10
|
+
Uses only the standard library (urllib) - two simple GETs don't justify a
|
|
11
|
+
`requests` dependency.
|
|
12
|
+
|
|
13
|
+
v1 scope: single-package repo (dbt-oss). The archive-name match below is
|
|
14
|
+
dbt-oss-specific; generalize it (e.g. read a manifest asset instead of
|
|
15
|
+
guessing a filename pattern) once pywheels serves more than one package.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import fnmatch
|
|
21
|
+
import json
|
|
22
|
+
import urllib.error
|
|
23
|
+
import urllib.request
|
|
24
|
+
from dataclasses import dataclass
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Optional
|
|
27
|
+
|
|
28
|
+
API_ROOT = "https://api.github.com"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class ReleaseAssets:
|
|
33
|
+
wheel: Path
|
|
34
|
+
bundle: Path
|
|
35
|
+
archive: Optional[Path]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _get_release(repo: str, tag: str) -> dict:
|
|
39
|
+
url = f"{API_ROOT}/repos/{repo}/releases/tags/{tag}"
|
|
40
|
+
req = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"})
|
|
41
|
+
try:
|
|
42
|
+
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
43
|
+
return json.loads(resp.read())
|
|
44
|
+
except urllib.error.HTTPError as exc:
|
|
45
|
+
raise FileNotFoundError(f"release {tag!r} not found in {repo} (HTTP {exc.code})") from exc
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def get_latest_release_tag(repo: str) -> str:
|
|
49
|
+
"""Used by `install` when the caller doesn't pin a --tag. `verify` never
|
|
50
|
+
calls this - it always wants a specific, caller-named release."""
|
|
51
|
+
url = f"{API_ROOT}/repos/{repo}/releases/latest"
|
|
52
|
+
req = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json"})
|
|
53
|
+
try:
|
|
54
|
+
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
55
|
+
release = json.loads(resp.read())
|
|
56
|
+
except urllib.error.HTTPError as exc:
|
|
57
|
+
raise FileNotFoundError(f"no releases found in {repo} (HTTP {exc.code})") from exc
|
|
58
|
+
tag = release.get("tag_name")
|
|
59
|
+
if not tag:
|
|
60
|
+
raise FileNotFoundError(f"latest release in {repo} has no tag_name")
|
|
61
|
+
return tag
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _download(url: str, dest: Path) -> Path:
|
|
65
|
+
req = urllib.request.Request(url, headers={"Accept": "application/octet-stream"})
|
|
66
|
+
with urllib.request.urlopen(req, timeout=120) as resp:
|
|
67
|
+
dest.write_bytes(resp.read())
|
|
68
|
+
return dest
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def fetch_release_assets(repo: str, tag: str, package: str, dest_dir: Path) -> ReleaseAssets:
|
|
72
|
+
release = _get_release(repo, tag)
|
|
73
|
+
assets = {a["name"]: a for a in release.get("assets", [])}
|
|
74
|
+
|
|
75
|
+
wheel_pattern = f"{package.replace('-', '_')}-*.whl"
|
|
76
|
+
wheel_name = next((n for n in assets if fnmatch.fnmatch(n, wheel_pattern)), None)
|
|
77
|
+
if wheel_name is None:
|
|
78
|
+
raise FileNotFoundError(f"no wheel matching {wheel_pattern!r} in {repo}@{tag}")
|
|
79
|
+
|
|
80
|
+
bundle_name = f"{wheel_name}.attestations.jsonl"
|
|
81
|
+
if bundle_name not in assets:
|
|
82
|
+
raise FileNotFoundError(f"no attestation bundle {bundle_name!r} in {repo}@{tag}")
|
|
83
|
+
|
|
84
|
+
archive_name = next(
|
|
85
|
+
(n for n in assets if n.startswith("dbt-oss-source-") and n.endswith(".tar.gz")),
|
|
86
|
+
None,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
wheel_path = _download(assets[wheel_name]["browser_download_url"], dest_dir / wheel_name)
|
|
90
|
+
bundle_path = _download(assets[bundle_name]["browser_download_url"], dest_dir / bundle_name)
|
|
91
|
+
archive_path = (
|
|
92
|
+
_download(assets[archive_name]["browser_download_url"], dest_dir / archive_name)
|
|
93
|
+
if archive_name else None
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
return ReleaseAssets(wheel=wheel_path, bundle=bundle_path, archive=archive_path)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def find_local_assets(package: str, local_dir: Path) -> ReleaseAssets:
|
|
100
|
+
"""Same three files as fetch_release_assets, located by searching a
|
|
101
|
+
local directory tree instead of a GitHub release - e.g. the layout
|
|
102
|
+
`gh run download` produces (wheel nested under an artifact-name folder,
|
|
103
|
+
bundle files flat, archive nested under its own artifact folder)."""
|
|
104
|
+
if not local_dir.is_dir():
|
|
105
|
+
raise FileNotFoundError(f"not a directory: {local_dir}")
|
|
106
|
+
|
|
107
|
+
wheel_pattern = f"{package.replace('-', '_')}-*.whl"
|
|
108
|
+
wheel_path = next(
|
|
109
|
+
(p for p in local_dir.rglob("*.whl") if fnmatch.fnmatch(p.name, wheel_pattern)),
|
|
110
|
+
None,
|
|
111
|
+
)
|
|
112
|
+
if wheel_path is None:
|
|
113
|
+
raise FileNotFoundError(f"no wheel matching {wheel_pattern!r} under {local_dir}")
|
|
114
|
+
|
|
115
|
+
bundle_name = f"{wheel_path.name}.attestations.jsonl"
|
|
116
|
+
bundle_path = next((p for p in local_dir.rglob(bundle_name)), None)
|
|
117
|
+
if bundle_path is None:
|
|
118
|
+
raise FileNotFoundError(f"no attestation bundle {bundle_name!r} under {local_dir}")
|
|
119
|
+
|
|
120
|
+
archive_path = next(
|
|
121
|
+
(p for p in local_dir.rglob("*.tar.gz") if p.name.startswith("dbt-oss-source-")),
|
|
122
|
+
None,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
return ReleaseAssets(wheel=wheel_path, bundle=bundle_path, archive=archive_path)
|
pywheels/overrides.json
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
{
|
|
2
|
+
"dbt-oss": {
|
|
3
|
+
"strategy": "source-build-probe",
|
|
4
|
+
"note": "PyPI's dbt-oss sdist is a stub: its custom PEP 517 backend (_dbt_sa_build) downloads a prebuilt wheel from a hardcoded GitHub release URL keyed by platform tag, checked against an embedded sha256 manifest, with no real source to build otherwise. --only-binary=:all: can never see this - pip's index metadata never advertises a real wheel at all, stub or not - so a full source build (which actually invokes the backend) is the only way to know whether upstream truly covers a given platform."
|
|
5
|
+
}
|
|
6
|
+
}
|
pywheels/verify.py
ADDED
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
"""Verification logic for python-wheels-builds artifacts.
|
|
2
|
+
|
|
3
|
+
Two backends, tried in this order unless one is forced:
|
|
4
|
+
|
|
5
|
+
1. sigstore - offline, no network needed at verify time, but requires
|
|
6
|
+
the `sigstore` package (optional dependency) and a local
|
|
7
|
+
attestation bundle.
|
|
8
|
+
2. gh - the GitHub CLI, if it's on PATH. Needs network (it asks
|
|
9
|
+
GitHub for attestations on the wheel's digest directly),
|
|
10
|
+
but needs nothing installed via pip.
|
|
11
|
+
|
|
12
|
+
If neither is available, verification of one of OUR OWN wheels fails
|
|
13
|
+
loudly (VerificationError) rather than silently letting an unverified
|
|
14
|
+
wheel through - this tool exists specifically to not do that. Deciding
|
|
15
|
+
whether a requested package is even one of ours (vs. a plain upstream
|
|
16
|
+
package pywheels doesn't attest, which is fine to install unverified with
|
|
17
|
+
a warning) is an `install`-command concern, not this module's - see the
|
|
18
|
+
docstring in cli.py.
|
|
19
|
+
|
|
20
|
+
For a given wheel, a successful verification means:
|
|
21
|
+
1. Every attestation we actually asked for verifies, scoped to the exact
|
|
22
|
+
signer identity (repo + workflow file) - not just "signed by someone
|
|
23
|
+
using GitHub Actions OIDC".
|
|
24
|
+
2. Among those, there's a SLSA build-provenance attestation AND our own
|
|
25
|
+
upstream-source attestation - both required.
|
|
26
|
+
3. If a source archive is supplied, it hashes to the digest recorded in
|
|
27
|
+
the upstream-source predicate (snapshot_archive_sha256).
|
|
28
|
+
|
|
29
|
+
Attestation types we didn't ask for (IGNORED_PREDICATE_TYPES) are recorded
|
|
30
|
+
but never attempted or counted - see release/v0.2 below.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import base64
|
|
36
|
+
import hashlib
|
|
37
|
+
import importlib.util
|
|
38
|
+
import json
|
|
39
|
+
import shutil
|
|
40
|
+
import subprocess
|
|
41
|
+
import sys
|
|
42
|
+
import tempfile
|
|
43
|
+
from dataclasses import dataclass, field
|
|
44
|
+
from pathlib import Path
|
|
45
|
+
from typing import Optional
|
|
46
|
+
|
|
47
|
+
OIDC_ISSUER = "https://token.actions.githubusercontent.com"
|
|
48
|
+
BUILD_PROVENANCE_PREDICATE = "https://slsa.dev/provenance/v1"
|
|
49
|
+
|
|
50
|
+
# GitHub auto-attaches this the moment a wheel becomes part of a published
|
|
51
|
+
# Release. We never asked for it, its bundle has no Rekor tlog entry (so
|
|
52
|
+
# sigstore-python can't even parse it), and nothing we do depends on it.
|
|
53
|
+
# Record it, don't verify it, don't count it against the wheel.
|
|
54
|
+
IGNORED_PREDICATE_TYPES = {
|
|
55
|
+
"https://in-toto.io/attestation/release/v0.2",
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
DEFAULT_UPSTREAM_PREDICATE_TYPE = "https://patrickryankenneth.github.io/attestations/upstream-source/v1"
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class VerificationError(Exception):
|
|
62
|
+
"""Hard failure: missing files, unreadable bundle, or no usable
|
|
63
|
+
backend at all. A verified-but-failed check is a normal outcome
|
|
64
|
+
reported via VerifyReport.ok, not an exception."""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def sigstore_available() -> bool:
|
|
68
|
+
return importlib.util.find_spec("sigstore") is not None
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def gh_available() -> bool:
|
|
72
|
+
return shutil.which("gh") is not None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class AttestationResult:
|
|
77
|
+
predicate_type: str
|
|
78
|
+
predicate: dict
|
|
79
|
+
verified: bool
|
|
80
|
+
detail: str
|
|
81
|
+
ignored: bool = False
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass
|
|
85
|
+
class VerifyReport:
|
|
86
|
+
wheel: Path
|
|
87
|
+
backend: str = ""
|
|
88
|
+
attestations: list = field(default_factory=list) # list[AttestationResult]
|
|
89
|
+
archive_checked: bool = False
|
|
90
|
+
archive_ok: Optional[bool] = None
|
|
91
|
+
# Set when archive_ok is False *because we never got predicate content
|
|
92
|
+
# to check against* (e.g. `gh attestation download` failed) rather than
|
|
93
|
+
# because we checked it and it genuinely didn't match. Still fails
|
|
94
|
+
# closed either way - see verify_wheel - but this lets callers show an
|
|
95
|
+
# actionable message instead of a bare "FAILED" that looks identical to
|
|
96
|
+
# a real hash mismatch.
|
|
97
|
+
archive_check_error: Optional[str] = None
|
|
98
|
+
|
|
99
|
+
@property
|
|
100
|
+
def ok(self) -> bool:
|
|
101
|
+
checked = [a for a in self.attestations if not a.ignored]
|
|
102
|
+
has_build_provenance = any(
|
|
103
|
+
a.verified and a.predicate_type == BUILD_PROVENANCE_PREDICATE for a in checked
|
|
104
|
+
)
|
|
105
|
+
has_upstream_source = any(
|
|
106
|
+
a.verified and a.predicate_type and a.predicate_type != BUILD_PROVENANCE_PREDICATE
|
|
107
|
+
for a in checked
|
|
108
|
+
)
|
|
109
|
+
archive_ok = self.archive_ok is not False # not checked -> doesn't block
|
|
110
|
+
return has_build_provenance and has_upstream_source and archive_ok
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _signer_identity(repo: str, workflow_file: str, ref: str) -> str:
|
|
114
|
+
return f"https://github.com/{repo}/.github/workflows/{workflow_file}@{ref}"
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _signer_workflow(repo: str, workflow_file: str) -> str:
|
|
118
|
+
return f"{repo}/.github/workflows/{workflow_file}"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _split_bundle_lines(bundle_jsonl: Path) -> list:
|
|
122
|
+
"""Each line of a `gh attestation download` bundle is a complete
|
|
123
|
+
Sigstore Bundle on its own. Split into temp files for per-bundle
|
|
124
|
+
verification."""
|
|
125
|
+
if not bundle_jsonl.exists():
|
|
126
|
+
raise VerificationError(f"attestation bundle not found: {bundle_jsonl}")
|
|
127
|
+
lines = [line for line in bundle_jsonl.read_text().splitlines() if line.strip()]
|
|
128
|
+
if not lines:
|
|
129
|
+
raise VerificationError(f"attestation bundle is empty: {bundle_jsonl}")
|
|
130
|
+
tmpdir = Path(tempfile.mkdtemp(prefix="pywheels-bundle-"))
|
|
131
|
+
paths = []
|
|
132
|
+
for i, line in enumerate(lines):
|
|
133
|
+
p = tmpdir / f"bundle-{i}.json"
|
|
134
|
+
p.write_text(line)
|
|
135
|
+
paths.append(p)
|
|
136
|
+
return paths
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _decode_predicate(bundle_line_path: Path):
|
|
140
|
+
bundle = json.loads(bundle_line_path.read_text())
|
|
141
|
+
payload_b64 = bundle["dsseEnvelope"]["payload"]
|
|
142
|
+
statement = json.loads(base64.b64decode(payload_b64))
|
|
143
|
+
return statement.get("predicateType", ""), statement.get("predicate", {})
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
# ---------------------------------------------------------------- sigstore
|
|
147
|
+
|
|
148
|
+
def _sigstore_verify_identity(target: Path, bundle: Path, cert_identity: str, *, verbose: int = 0):
|
|
149
|
+
# sigstore's -v/-vv is a top-level flag (it sits on the `sigstore`
|
|
150
|
+
# parser, before the subcommand), not an option on `verify identity`
|
|
151
|
+
# itself - see `sigstore --help`.
|
|
152
|
+
cmd = [sys.executable, "-m", "sigstore"] + ["-v"] * verbose + [
|
|
153
|
+
"verify", "identity",
|
|
154
|
+
"--bundle", str(bundle),
|
|
155
|
+
"--cert-identity", cert_identity,
|
|
156
|
+
"--cert-oidc-issuer", OIDC_ISSUER,
|
|
157
|
+
"--offline",
|
|
158
|
+
str(target),
|
|
159
|
+
]
|
|
160
|
+
proc = subprocess.run(cmd, capture_output=True, text=True)
|
|
161
|
+
return proc.returncode == 0, (proc.stdout + proc.stderr).strip()
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _verify_with_sigstore(wheel_path: Path, bundle_jsonl_path: Path, *, repo: str, workflow_file: str, ref: str,
|
|
165
|
+
verbose: int = 0):
|
|
166
|
+
cert_identity = _signer_identity(repo, workflow_file, ref)
|
|
167
|
+
results = []
|
|
168
|
+
for line_path in _split_bundle_lines(bundle_jsonl_path):
|
|
169
|
+
predicate_type, predicate = _decode_predicate(line_path)
|
|
170
|
+
if predicate_type in IGNORED_PREDICATE_TYPES:
|
|
171
|
+
results.append(AttestationResult(
|
|
172
|
+
predicate_type, predicate, verified=False,
|
|
173
|
+
detail="not checked - known GitHub-native attestation, not one we asked for",
|
|
174
|
+
ignored=True,
|
|
175
|
+
))
|
|
176
|
+
continue
|
|
177
|
+
ok, detail = _sigstore_verify_identity(wheel_path, line_path, cert_identity, verbose=verbose)
|
|
178
|
+
results.append(AttestationResult(predicate_type, predicate, verified=ok, detail=detail))
|
|
179
|
+
return results
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# --------------------------------------------------------------------- gh
|
|
183
|
+
|
|
184
|
+
def _gh_verify_identity(wheel_path: Path, repo: str, signer_workflow: str, predicate_type: str, *, verbose: int = 0):
|
|
185
|
+
# `gh attestation verify` has no -v/--verbose of its own. The closest
|
|
186
|
+
# equivalent gh documents is --format=json, which dumps the full
|
|
187
|
+
# verification result instead of the one-line human summary - so
|
|
188
|
+
# that's what verbose asks for here rather than a made-up flag.
|
|
189
|
+
cmd = [
|
|
190
|
+
"gh", "attestation", "verify", str(wheel_path),
|
|
191
|
+
"--repo", repo,
|
|
192
|
+
"--signer-workflow", signer_workflow,
|
|
193
|
+
"--predicate-type", predicate_type,
|
|
194
|
+
]
|
|
195
|
+
if verbose:
|
|
196
|
+
cmd += ["--format", "json"]
|
|
197
|
+
proc = subprocess.run(cmd, capture_output=True, text=True)
|
|
198
|
+
return proc.returncode == 0, (proc.stdout + proc.stderr).strip()
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _gh_download_bundle(wheel_path: Path, repo: str) -> Path:
|
|
202
|
+
tmpdir = Path(tempfile.mkdtemp(prefix="pywheels-gh-"))
|
|
203
|
+
# `gh` runs with cwd=tmpdir (below) so its output bundle lands somewhere
|
|
204
|
+
# we can glob cleanly, isolated from anything else in the caller's cwd.
|
|
205
|
+
# wheel_path may be relative (e.g. the CLI's default --workdir is
|
|
206
|
+
# ".pywheels-cache") - resolved against the CALLER's cwd, not tmpdir.
|
|
207
|
+
# Passing it through unresolved means `gh` looks for it relative to
|
|
208
|
+
# tmpdir instead, where it never exists - it then fails trying to open
|
|
209
|
+
# the wheel to hash it, in a way that reads like an attestation/archive
|
|
210
|
+
# problem but is really just this path bug. Resolve before handing it
|
|
211
|
+
# to a subprocess running in a different directory.
|
|
212
|
+
proc = subprocess.run(
|
|
213
|
+
["gh", "attestation", "download", str(wheel_path.resolve()), "--repo", repo],
|
|
214
|
+
capture_output=True, text=True, cwd=tmpdir,
|
|
215
|
+
)
|
|
216
|
+
if proc.returncode != 0:
|
|
217
|
+
raise VerificationError(f"gh attestation download failed: {(proc.stderr or proc.stdout).strip()}")
|
|
218
|
+
matches = list(tmpdir.glob("*.jsonl"))
|
|
219
|
+
if not matches:
|
|
220
|
+
raise VerificationError("gh attestation download did not produce a bundle file")
|
|
221
|
+
return matches[0]
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _verify_with_gh(wheel_path: Path, *, repo: str, workflow_file: str, upstream_predicate_type: str,
|
|
225
|
+
verbose: int = 0):
|
|
226
|
+
signer_workflow = _signer_workflow(repo, workflow_file)
|
|
227
|
+
results = []
|
|
228
|
+
|
|
229
|
+
ok, detail = _gh_verify_identity(wheel_path, repo, signer_workflow, BUILD_PROVENANCE_PREDICATE, verbose=verbose)
|
|
230
|
+
results.append(AttestationResult(BUILD_PROVENANCE_PREDICATE, {}, verified=ok, detail=detail))
|
|
231
|
+
|
|
232
|
+
ok, detail = _gh_verify_identity(wheel_path, repo, signer_workflow, upstream_predicate_type, verbose=verbose)
|
|
233
|
+
results.append(AttestationResult(upstream_predicate_type, {}, verified=ok, detail=detail))
|
|
234
|
+
|
|
235
|
+
# Bonus, not required: pull predicate content too, purely for the
|
|
236
|
+
# archive-digest check below. A plain download - no sigstore parsing.
|
|
237
|
+
# NOTE: this can fail for reasons that have nothing to do with the
|
|
238
|
+
# wheel itself or with the identity checks above - `gh attestation
|
|
239
|
+
# download` is a separate code path in `gh` from `verify identity` and
|
|
240
|
+
# can fail on its own (network blip, rate limit, a `gh` bug, etc). The
|
|
241
|
+
# identity checks above already ran and stand on their own, so we
|
|
242
|
+
# don't turn this into a VerificationError - but we DO surface *why*
|
|
243
|
+
# it failed instead of swallowing it, so a caller checking the archive
|
|
244
|
+
# digest afterward can tell "we never got to check" apart from "we
|
|
245
|
+
# checked and it didn't match", and can actually see the real reason
|
|
246
|
+
# instead of guessing. See VerifyReport.archive_check_error.
|
|
247
|
+
download_error = None
|
|
248
|
+
try:
|
|
249
|
+
bundle_path = _gh_download_bundle(wheel_path, repo)
|
|
250
|
+
for line_path in _split_bundle_lines(bundle_path):
|
|
251
|
+
predicate_type, predicate = _decode_predicate(line_path)
|
|
252
|
+
for r in results:
|
|
253
|
+
if r.predicate_type == predicate_type and not r.predicate:
|
|
254
|
+
r.predicate = predicate
|
|
255
|
+
except VerificationError as exc:
|
|
256
|
+
download_error = str(exc)
|
|
257
|
+
|
|
258
|
+
return results, download_error
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
# ------------------------------------------------------------------- main
|
|
262
|
+
|
|
263
|
+
def verify_wheel(
|
|
264
|
+
wheel_path: Path,
|
|
265
|
+
bundle_jsonl_path: Optional[Path] = None,
|
|
266
|
+
*,
|
|
267
|
+
repo: str,
|
|
268
|
+
workflow_file: str,
|
|
269
|
+
ref: str = "refs/heads/main",
|
|
270
|
+
upstream_predicate_type: str = DEFAULT_UPSTREAM_PREDICATE_TYPE,
|
|
271
|
+
source_archive_path: Optional[Path] = None,
|
|
272
|
+
backend: str = "auto",
|
|
273
|
+
verbose: int = 0,
|
|
274
|
+
) -> VerifyReport:
|
|
275
|
+
if not wheel_path.exists():
|
|
276
|
+
raise VerificationError(f"wheel not found: {wheel_path}")
|
|
277
|
+
|
|
278
|
+
chosen = backend
|
|
279
|
+
if chosen == "auto":
|
|
280
|
+
if sigstore_available():
|
|
281
|
+
chosen = "sigstore"
|
|
282
|
+
elif gh_available():
|
|
283
|
+
chosen = "gh"
|
|
284
|
+
else:
|
|
285
|
+
raise VerificationError(
|
|
286
|
+
"no verification backend available - install sigstore (`pip install sigstore`) "
|
|
287
|
+
"or the GitHub CLI (`gh`, see https://cli.github.com). Run `pywheels doctor` for "
|
|
288
|
+
"details. Refusing to treat an unverifiable wheel as trusted."
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
report = VerifyReport(wheel=wheel_path, backend=chosen)
|
|
292
|
+
|
|
293
|
+
if chosen == "sigstore":
|
|
294
|
+
if bundle_jsonl_path is None:
|
|
295
|
+
raise VerificationError("sigstore backend requires a local attestation bundle")
|
|
296
|
+
report.attestations = _verify_with_sigstore(
|
|
297
|
+
wheel_path, bundle_jsonl_path, repo=repo, workflow_file=workflow_file, ref=ref, verbose=verbose,
|
|
298
|
+
)
|
|
299
|
+
elif chosen == "gh":
|
|
300
|
+
report.attestations, report.archive_check_error = _verify_with_gh(
|
|
301
|
+
wheel_path, repo=repo, workflow_file=workflow_file, upstream_predicate_type=upstream_predicate_type,
|
|
302
|
+
verbose=verbose,
|
|
303
|
+
)
|
|
304
|
+
else:
|
|
305
|
+
raise VerificationError(f"unknown backend: {chosen!r}")
|
|
306
|
+
|
|
307
|
+
if source_archive_path is not None:
|
|
308
|
+
report.archive_checked = True
|
|
309
|
+
expected = next(
|
|
310
|
+
(a.predicate["snapshot_archive_sha256"] for a in report.attestations
|
|
311
|
+
if a.verified and a.predicate and "snapshot_archive_sha256" in a.predicate),
|
|
312
|
+
None,
|
|
313
|
+
)
|
|
314
|
+
if expected is None:
|
|
315
|
+
# Still fail closed - we have no digest to trust the archive
|
|
316
|
+
# against, whether that's because the predicate genuinely
|
|
317
|
+
# doesn't carry one or because we never fetched it. But make
|
|
318
|
+
# the two cases distinguishable to the caller: if a download
|
|
319
|
+
# error is on record, this ISN'T a real digest mismatch.
|
|
320
|
+
report.archive_ok = False
|
|
321
|
+
if report.archive_check_error is None:
|
|
322
|
+
report.archive_check_error = (
|
|
323
|
+
"no snapshot_archive_sha256 found in any verified attestation's predicate "
|
|
324
|
+
"(predicate content was retrieved, but the digest was genuinely absent)"
|
|
325
|
+
)
|
|
326
|
+
else:
|
|
327
|
+
actual = hashlib.sha256(source_archive_path.read_bytes()).hexdigest()
|
|
328
|
+
report.archive_ok = (actual == expected)
|
|
329
|
+
|
|
330
|
+
return report
|