python-wheels 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- python_wheels-0.1.0/.github/workflows/publish.yml +51 -0
- python_wheels-0.1.0/.github/workflows/smoke-test-pywheels-install.yml +90 -0
- python_wheels-0.1.0/.gitignore +36 -0
- python_wheels-0.1.0/CHANGELOG.md +44 -0
- python_wheels-0.1.0/LICENSE +21 -0
- python_wheels-0.1.0/PKG-INFO +128 -0
- python_wheels-0.1.0/README.md +101 -0
- python_wheels-0.1.0/pyproject.toml +77 -0
- python_wheels-0.1.0/src/pywheels/__init__.py +5 -0
- python_wheels-0.1.0/src/pywheels/cli.py +415 -0
- python_wheels-0.1.0/src/pywheels/github_release.py +125 -0
- python_wheels-0.1.0/src/pywheels/overrides.json +6 -0
- python_wheels-0.1.0/src/pywheels/verify.py +330 -0
- python_wheels-0.1.0/tests/test_pywheels.py +78 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: "3.12"
|
|
19
|
+
|
|
20
|
+
- name: Install build
|
|
21
|
+
run: python -m pip install build
|
|
22
|
+
|
|
23
|
+
- name: Build sdist and wheel
|
|
24
|
+
run: python -m build
|
|
25
|
+
|
|
26
|
+
- name: Tag must match package version
|
|
27
|
+
run: |
|
|
28
|
+
v=$(python -c "import re,pathlib; print(re.search(r'__version__ = \"(.+?)\"', pathlib.Path('src/pywheels/__init__.py').read_text()).group(1))")
|
|
29
|
+
test "v$v" = "$GITHUB_REF_NAME" || { echo "tag $GITHUB_REF_NAME != v$v"; exit 1; }
|
|
30
|
+
|
|
31
|
+
- uses: actions/upload-artifact@v4
|
|
32
|
+
with:
|
|
33
|
+
name: dist
|
|
34
|
+
path: dist/
|
|
35
|
+
|
|
36
|
+
publish:
|
|
37
|
+
needs: build
|
|
38
|
+
runs-on: ubuntu-latest
|
|
39
|
+
environment:
|
|
40
|
+
name: pypi
|
|
41
|
+
url: https://pypi.org/project/python-wheels/
|
|
42
|
+
permissions:
|
|
43
|
+
id-token: write # The only permission PyPI Trusted Publishing requires
|
|
44
|
+
steps:
|
|
45
|
+
- uses: actions/download-artifact@v4
|
|
46
|
+
with:
|
|
47
|
+
name: dist
|
|
48
|
+
path: dist/
|
|
49
|
+
|
|
50
|
+
- name: Publish to PyPI via Trusted Publishing
|
|
51
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
name: smoke-test-pywheels-install
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
push:
|
|
6
|
+
paths:
|
|
7
|
+
- '.github/workflows/smoke-test-pywheels-install.yml'
|
|
8
|
+
- 'pywheels/**'
|
|
9
|
+
- 'src/pywheels/**'
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
install-on-win-arm64:
|
|
13
|
+
runs-on: windows-11-arm
|
|
14
|
+
env:
|
|
15
|
+
PYWHEELS_REPO: patrickryankenneth/python-wheels-builds
|
|
16
|
+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
17
|
+
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
|
|
21
|
+
# ⚡ Restore Python toolcache: takes ~3 seconds instead of 3 minutes
|
|
22
|
+
- name: Cache Python toolcache (Windows ARM64)
|
|
23
|
+
uses: actions/cache@v4
|
|
24
|
+
with:
|
|
25
|
+
path: C:\hostedtoolcache\windows\Python
|
|
26
|
+
key: toolcache-python-3.11-win-arm64
|
|
27
|
+
|
|
28
|
+
- uses: actions/setup-python@v5
|
|
29
|
+
with:
|
|
30
|
+
python-version: '3.11'
|
|
31
|
+
|
|
32
|
+
- name: System info
|
|
33
|
+
shell: pwsh
|
|
34
|
+
run: |
|
|
35
|
+
[System.Environment]::OSVersion
|
|
36
|
+
$env:PROCESSOR_ARCHITECTURE
|
|
37
|
+
python -c "import platform; print(platform.machine(), platform.architecture())"
|
|
38
|
+
|
|
39
|
+
- name: Install pywheels from source (with sigstore extra)
|
|
40
|
+
shell: pwsh
|
|
41
|
+
# The sigstore extra makes `doctor`/`verify_wheel`'s "auto" backend
|
|
42
|
+
# selection prefer sigstore over gh (see verify.py: sigstore_available()
|
|
43
|
+
# is checked first). sigstore verifies fully offline against the
|
|
44
|
+
# bundle file github_release.py already downloaded via plain
|
|
45
|
+
# urllib, so it never calls `gh attestation download` at all - it
|
|
46
|
+
# sidesteps whatever is failing there entirely, whatever the actual
|
|
47
|
+
# cause turns out to be (see verify.py's archive_check_error, which
|
|
48
|
+
# now surfaces that cause instead of swallowing it, for if/when the
|
|
49
|
+
# gh backend needs debugging directly).
|
|
50
|
+
run: pip install -e "."
|
|
51
|
+
|
|
52
|
+
- name: Check backend availability
|
|
53
|
+
shell: pwsh
|
|
54
|
+
run: pywheels doctor
|
|
55
|
+
|
|
56
|
+
- name: pywheels install (should fall through to our repo on this platform)
|
|
57
|
+
shell: pwsh
|
|
58
|
+
run: |
|
|
59
|
+
pywheels install dbt-oss==2.0.5 `
|
|
60
|
+
--tag dbt-oss-v2.0.5 `
|
|
61
|
+
--workflow build-dbt-oss-win-arm64.yml `
|
|
62
|
+
-v | Tee-Object -FilePath install-output.txt
|
|
63
|
+
|
|
64
|
+
- name: Assert the override actually detected the win-arm64 gap
|
|
65
|
+
shell: pwsh
|
|
66
|
+
run: |
|
|
67
|
+
if (Select-String -Path install-output.txt -Pattern "upstream has wheels" -Quiet) {
|
|
68
|
+
Write-Error "expected the upstream check to FAIL on win-arm64 (dbt-oss's manifest has no win_arm64 entry) - it reported success instead. Either the override broke, or upstream now ships a real win_arm64 wheel and this override can be retired."
|
|
69
|
+
exit 1
|
|
70
|
+
}
|
|
71
|
+
if (-not (Select-String -Path install-output.txt -Pattern "verified OK" -Quiet)) {
|
|
72
|
+
Write-Error "expected our repo's wheel to verify OK here - it didn't. See the full output above."
|
|
73
|
+
exit 1
|
|
74
|
+
}
|
|
75
|
+
Write-Host "confirmed: upstream check correctly failed on this platform, our wheel verified instead"
|
|
76
|
+
|
|
77
|
+
- name: Run the exact command pywheels recommended
|
|
78
|
+
shell: pwsh
|
|
79
|
+
run: |
|
|
80
|
+
$line = Select-String -Path install-output.txt -Pattern '^\s\s+pip install' | Select-Object -First 1
|
|
81
|
+
if (-not $line) { Write-Error "no 'pip install' line found in pywheels' output"; exit 1 }
|
|
82
|
+
$cmd = $line.Line.Trim()
|
|
83
|
+
Write-Host "running: $cmd"
|
|
84
|
+
Invoke-Expression $cmd
|
|
85
|
+
|
|
86
|
+
- name: Functional check - does the installed package actually run
|
|
87
|
+
shell: pwsh
|
|
88
|
+
run: |
|
|
89
|
+
python -c "import dbt; print('imported OK from', dbt.__file__)"
|
|
90
|
+
pip show dbt-oss
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# --- Python bytecode / cache ---
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# --- Build artifacts (setuptools and hatchling both use these) ---
|
|
7
|
+
build/
|
|
8
|
+
dist/
|
|
9
|
+
*.egg-info/
|
|
10
|
+
.eggs/
|
|
11
|
+
|
|
12
|
+
# --- pywheels' own local scratch space ---
|
|
13
|
+
# .pywheels-cache/ is cli.py's --workdir default (where `pywheels verify`
|
|
14
|
+
# downloads release assets to when not using --local-dir).
|
|
15
|
+
# release-assets/ looks like the same kind of thing from manual testing -
|
|
16
|
+
# if it's meant to be committed as test fixtures, remove this line and
|
|
17
|
+
# `git add -f` it instead.
|
|
18
|
+
.pywheels-cache/
|
|
19
|
+
release-assets/
|
|
20
|
+
|
|
21
|
+
# --- Virtual environments ---
|
|
22
|
+
# Matches any dir ending in "env" - venv, .venv, env, my-project-env,
|
|
23
|
+
# whatever you or anyone else names theirs - so this never needs a
|
|
24
|
+
# personal/local env name added to it.
|
|
25
|
+
*env/
|
|
26
|
+
*ENV/
|
|
27
|
+
|
|
28
|
+
# --- Editors / OS cruft ---
|
|
29
|
+
.vscode/
|
|
30
|
+
.idea/
|
|
31
|
+
.DS_Store
|
|
32
|
+
|
|
33
|
+
# --- Test/type-check caches, if you add pytest/mypy/ruff later ---
|
|
34
|
+
.pytest_cache/
|
|
35
|
+
.mypy_cache/
|
|
36
|
+
.ruff_cache/
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here.
|
|
4
|
+
|
|
5
|
+
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project uses [Semantic Versioning](https://semver.org/).
|
|
7
|
+
|
|
8
|
+
## [0.1.0] - 2026-09-23
|
|
9
|
+
|
|
10
|
+
Initial release.
|
|
11
|
+
|
|
12
|
+
`pywheels` verifies that a wheel from [python-wheels-builds](https://github.com/patrickryankenneth/python-wheels-builds)
|
|
13
|
+
was actually built by that repo's own GitHub Actions workflow before you
|
|
14
|
+
install it - not just downloaded from somewhere plausible. It checks two
|
|
15
|
+
required attestations (SLSA build provenance, plus our own
|
|
16
|
+
upstream-source attestation) against the exact signer identity, and, when
|
|
17
|
+
a source archive is available, that the archive's digest matches what the
|
|
18
|
+
attestation recorded.
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
|
|
22
|
+
- `pywheels verify` - verify a wheel's attestations directly, either
|
|
23
|
+
fetched from a GitHub release (`--tag`) or already sitting on disk
|
|
24
|
+
(`--local-dir`, e.g. output of `gh run download`).
|
|
25
|
+
- `pywheels install` - the main workflow: check whether pip can already
|
|
26
|
+
resolve real wheels for a package on the current platform; if not,
|
|
27
|
+
fetch and verify our attested build instead; print the exact
|
|
28
|
+
`pip install` command that's safe to run, or fall back to an
|
|
29
|
+
unverified source build as a last resort. Never runs `pip install`
|
|
30
|
+
itself.
|
|
31
|
+
- `pywheels doctor` - reports which verification backend(s) are usable
|
|
32
|
+
and how to fix the ones that aren't.
|
|
33
|
+
- Two independent verification backends: `sigstore` (offline, needs the
|
|
34
|
+
optional `sigstore` extra) and the `gh` CLI (needs network, no pip
|
|
35
|
+
dependency). Falls back from the former to the latter automatically;
|
|
36
|
+
refuses to trust a wheel if neither is available.
|
|
37
|
+
- Per-package build-detection overrides (`overrides.json`) for packages
|
|
38
|
+
whose real wheel availability isn't visible to plain `pip download`
|
|
39
|
+
probing - e.g. `dbt-oss`, whose PyPI sdist is a stub build backend that
|
|
40
|
+
fetches a real prebuilt wheel from GitHub, keyed by platform.
|
|
41
|
+
- Initial attested build: `dbt-oss` for `win_arm64`, filling a real gap
|
|
42
|
+
upstream doesn't cover on that platform yet.
|
|
43
|
+
|
|
44
|
+
[0.1.0]: https://github.com/patrickryankenneth/python-wheels/releases/tag/v0.1.0
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Patrick Ryan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: python-wheels
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Verify Sigstore-attested wheels from python-wheels-builds before installing
|
|
5
|
+
Project-URL: Homepage, https://github.com/patrickryankenneth/python-wheels
|
|
6
|
+
Project-URL: Builds, https://github.com/patrickryankenneth/python-wheels-builds
|
|
7
|
+
Project-URL: Index, https://python-wheels.github.io
|
|
8
|
+
Project-URL: Issues, https://github.com/patrickryankenneth/python-wheels/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/patrickryankenneth/python-wheels/blob/main/CHANGELOG.md
|
|
10
|
+
Author-email: Patrick Ryan <patrickryankenneth@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: attestation,pip,provenance,sigstore,slsa,supply-chain,wheels
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Environment :: Console
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Topic :: Security
|
|
20
|
+
Classifier: Topic :: System :: Software Distribution
|
|
21
|
+
Requires-Python: >=3.9
|
|
22
|
+
Provides-Extra: sigstore
|
|
23
|
+
Requires-Dist: sigstore<4.6,>=4.0; extra == 'sigstore'
|
|
24
|
+
Provides-Extra: test
|
|
25
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# python-wheels
|
|
29
|
+
|
|
30
|
+
**Status: working - v0.1.0.** The `python-wheels` CLI (command: `pywheels`,
|
|
31
|
+
also installed as `python-wheels`) verifies and installs attested wheels
|
|
32
|
+
today. First attested build shipped: `dbt-oss` for `win_arm64`.
|
|
33
|
+
|
|
34
|
+
## The problem
|
|
35
|
+
|
|
36
|
+
Some Python packages don't ship a wheel for your platform: an
|
|
37
|
+
uncommon architecture, Alpine/musl instead of glibc, an OS upstream doesn't
|
|
38
|
+
target, or just a combination nobody's gotten around to building for. Right
|
|
39
|
+
now the options are "build it from source yourself, every time" or "hope
|
|
40
|
+
someone in the community hosts one somewhere."
|
|
41
|
+
|
|
42
|
+
It's also not always upstream being lazy. PyPI caps a project at 10GB of
|
|
43
|
+
total storage, and wheels for every platform/arch/Python-version
|
|
44
|
+
combination add up fast - that's part of why projects like PyTorch host
|
|
45
|
+
some of their wheels off-PyPI and point people at an extra index instead.
|
|
46
|
+
`python-wheels` is aimed at exactly that situation: popular packages that
|
|
47
|
+
can't or don't publish a wheel for your platform, for any reason.
|
|
48
|
+
|
|
49
|
+
## What it does
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install python-wheels
|
|
53
|
+
pywheels install dbt-oss==2.0.5 \
|
|
54
|
+
--repo patrickryankenneth/python-wheels-builds \
|
|
55
|
+
--tag dbt-oss-v2.0.5 \
|
|
56
|
+
--workflow build-dbt-oss-win-arm64.yml
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
1. **Checks upstream first.** If pip can already resolve real wheels for
|
|
60
|
+
the package - and its whole dependency closure, not just the top-level
|
|
61
|
+
package - on your platform, it says so and prints the plain
|
|
62
|
+
`pip install` command; this tool gets out of the way whenever upstream
|
|
63
|
+
already has you covered. A small per-package override list
|
|
64
|
+
(`overrides.json`) handles the rare case where a package's real wheel
|
|
65
|
+
availability is hidden from pip's normal resolution - e.g. `dbt-oss`,
|
|
66
|
+
whose PyPI sdist is a stub build backend that fetches a real prebuilt
|
|
67
|
+
wheel from GitHub, keyed by platform, with no source to fall back to.
|
|
68
|
+
2. **Falls back, but verifies.** If there's no usable upstream wheel, it
|
|
69
|
+
fetches the matching release from
|
|
70
|
+
[python-wheels-builds](https://github.com/patrickryankenneth/python-wheels-builds)
|
|
71
|
+
and checks, before recommending anything:
|
|
72
|
+
- its **build provenance attestation** (SLSA - built by the expected CI
|
|
73
|
+
workflow, from the expected repo and ref, unmodified since),
|
|
74
|
+
- its **upstream-source attestation** (built from the real, tagged
|
|
75
|
+
upstream release - not a fork or a patched copy),
|
|
76
|
+
- the **source archive's digest** against what that attestation
|
|
77
|
+
recorded, when a source archive is available.
|
|
78
|
+
|
|
79
|
+
Verification runs offline via [sigstore](https://pypi.org/project/sigstore/)
|
|
80
|
+
(`pip install python-wheels[sigstore]`) if it's installed, or falls back
|
|
81
|
+
to the `gh` CLI (no pip dependency, needs network) if it isn't. Run
|
|
82
|
+
`pywheels doctor` to see which backend is usable on your machine.
|
|
83
|
+
3. **Fails loudly, not silently.** `pywheels install` never runs
|
|
84
|
+
`pip install` for you - it only ever prints the exact command that's
|
|
85
|
+
safe to run. If verification fails, or neither backend is available at
|
|
86
|
+
all, it says so plainly and falls back to `pip install
|
|
87
|
+
--no-binary=:all:` (unverified, from source) as the last resort - never
|
|
88
|
+
a silent, unattested install.
|
|
89
|
+
|
|
90
|
+
`pywheels verify` runs the same verification directly against a wheel
|
|
91
|
+
that's already on disk, or a specific release tag - useful for checking a
|
|
92
|
+
build without going through the whole `install` flow.
|
|
93
|
+
|
|
94
|
+
## Where the wheels actually come from
|
|
95
|
+
|
|
96
|
+
This repo is only the installer and verifier. The wheels themselves are
|
|
97
|
+
built and attested in
|
|
98
|
+
[python-wheels-builds](https://github.com/patrickryankenneth/python-wheels-builds),
|
|
99
|
+
and also served as a plain [PEP 503](https://peps.python.org/pep-0503/)
|
|
100
|
+
index at [python-wheels.github.io](https://python-wheels.github.io) for
|
|
101
|
+
anyone who just wants `pip install --extra-index-url
|
|
102
|
+
https://python-wheels.github.io/simple/ <package>` without the
|
|
103
|
+
verification step. `pywheels` is the piece that ties the CLI, the
|
|
104
|
+
release repo, and the attestation checks together so you don't have to run
|
|
105
|
+
`gh attestation verify` by hand.
|
|
106
|
+
|
|
107
|
+
## Longer-term direction
|
|
108
|
+
|
|
109
|
+
The build side is currently one hand-tuned workflow for one package
|
|
110
|
+
(`dbt-oss`/`dbt-core` on Windows ARM64). The goal is to turn that into a
|
|
111
|
+
standardized, reproducible build recipe that can target other popular PyPI
|
|
112
|
+
packages missing wheels for a given platform - starting with the platforms
|
|
113
|
+
that come up most often (Alpine/musl, less-common architectures, newer
|
|
114
|
+
Python versions upstream hasn't built for yet), rather than trying to
|
|
115
|
+
cover everything at once.
|
|
116
|
+
|
|
117
|
+
## Not yet decided
|
|
118
|
+
|
|
119
|
+
- CLI surface beyond `install`/`verify`/`doctor` (list available
|
|
120
|
+
platforms? show why a package fell back? a way to request a new
|
|
121
|
+
package/platform combo?)
|
|
122
|
+
- How package/platform requests get prioritized once this covers more than
|
|
123
|
+
one package
|
|
124
|
+
- Whether to eventually provide a wrapper mode that invokes pip directly after verification,
|
|
125
|
+
versus keeping the strict "print the safe command only" separation.
|
|
126
|
+
|
|
127
|
+
Contributions and issues on any of the above are welcome - this project is
|
|
128
|
+
still early, even with a first attested build shipped.
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# python-wheels
|
|
2
|
+
|
|
3
|
+
**Status: working - v0.1.0.** The `python-wheels` CLI (command: `pywheels`,
|
|
4
|
+
also installed as `python-wheels`) verifies and installs attested wheels
|
|
5
|
+
today. First attested build shipped: `dbt-oss` for `win_arm64`.
|
|
6
|
+
|
|
7
|
+
## The problem
|
|
8
|
+
|
|
9
|
+
Some Python packages don't ship a wheel for your platform: an
|
|
10
|
+
uncommon architecture, Alpine/musl instead of glibc, an OS upstream doesn't
|
|
11
|
+
target, or just a combination nobody's gotten around to building for. Right
|
|
12
|
+
now the options are "build it from source yourself, every time" or "hope
|
|
13
|
+
someone in the community hosts one somewhere."
|
|
14
|
+
|
|
15
|
+
It's also not always upstream being lazy. PyPI caps a project at 10GB of
|
|
16
|
+
total storage, and wheels for every platform/arch/Python-version
|
|
17
|
+
combination add up fast - that's part of why projects like PyTorch host
|
|
18
|
+
some of their wheels off-PyPI and point people at an extra index instead.
|
|
19
|
+
`python-wheels` is aimed at exactly that situation: popular packages that
|
|
20
|
+
can't or don't publish a wheel for your platform, for any reason.
|
|
21
|
+
|
|
22
|
+
## What it does
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install python-wheels
|
|
26
|
+
pywheels install dbt-oss==2.0.5 \
|
|
27
|
+
--repo patrickryankenneth/python-wheels-builds \
|
|
28
|
+
--tag dbt-oss-v2.0.5 \
|
|
29
|
+
--workflow build-dbt-oss-win-arm64.yml
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
1. **Checks upstream first.** If pip can already resolve real wheels for
|
|
33
|
+
the package - and its whole dependency closure, not just the top-level
|
|
34
|
+
package - on your platform, it says so and prints the plain
|
|
35
|
+
`pip install` command; this tool gets out of the way whenever upstream
|
|
36
|
+
already has you covered. A small per-package override list
|
|
37
|
+
(`overrides.json`) handles the rare case where a package's real wheel
|
|
38
|
+
availability is hidden from pip's normal resolution - e.g. `dbt-oss`,
|
|
39
|
+
whose PyPI sdist is a stub build backend that fetches a real prebuilt
|
|
40
|
+
wheel from GitHub, keyed by platform, with no source to fall back to.
|
|
41
|
+
2. **Falls back, but verifies.** If there's no usable upstream wheel, it
|
|
42
|
+
fetches the matching release from
|
|
43
|
+
[python-wheels-builds](https://github.com/patrickryankenneth/python-wheels-builds)
|
|
44
|
+
and checks, before recommending anything:
|
|
45
|
+
- its **build provenance attestation** (SLSA - built by the expected CI
|
|
46
|
+
workflow, from the expected repo and ref, unmodified since),
|
|
47
|
+
- its **upstream-source attestation** (built from the real, tagged
|
|
48
|
+
upstream release - not a fork or a patched copy),
|
|
49
|
+
- the **source archive's digest** against what that attestation
|
|
50
|
+
recorded, when a source archive is available.
|
|
51
|
+
|
|
52
|
+
Verification runs offline via [sigstore](https://pypi.org/project/sigstore/)
|
|
53
|
+
(`pip install python-wheels[sigstore]`) if it's installed, or falls back
|
|
54
|
+
to the `gh` CLI (no pip dependency, needs network) if it isn't. Run
|
|
55
|
+
`pywheels doctor` to see which backend is usable on your machine.
|
|
56
|
+
3. **Fails loudly, not silently.** `pywheels install` never runs
|
|
57
|
+
`pip install` for you - it only ever prints the exact command that's
|
|
58
|
+
safe to run. If verification fails, or neither backend is available at
|
|
59
|
+
all, it says so plainly and falls back to `pip install
|
|
60
|
+
--no-binary=:all:` (unverified, from source) as the last resort - never
|
|
61
|
+
a silent, unattested install.
|
|
62
|
+
|
|
63
|
+
`pywheels verify` runs the same verification directly against a wheel
|
|
64
|
+
that's already on disk, or a specific release tag - useful for checking a
|
|
65
|
+
build without going through the whole `install` flow.
|
|
66
|
+
|
|
67
|
+
## Where the wheels actually come from
|
|
68
|
+
|
|
69
|
+
This repo is only the installer and verifier. The wheels themselves are
|
|
70
|
+
built and attested in
|
|
71
|
+
[python-wheels-builds](https://github.com/patrickryankenneth/python-wheels-builds),
|
|
72
|
+
and also served as a plain [PEP 503](https://peps.python.org/pep-0503/)
|
|
73
|
+
index at [python-wheels.github.io](https://python-wheels.github.io) for
|
|
74
|
+
anyone who just wants `pip install --extra-index-url
|
|
75
|
+
https://python-wheels.github.io/simple/ <package>` without the
|
|
76
|
+
verification step. `pywheels` is the piece that ties the CLI, the
|
|
77
|
+
release repo, and the attestation checks together so you don't have to run
|
|
78
|
+
`gh attestation verify` by hand.
|
|
79
|
+
|
|
80
|
+
## Longer-term direction
|
|
81
|
+
|
|
82
|
+
The build side is currently one hand-tuned workflow for one package
|
|
83
|
+
(`dbt-oss`/`dbt-core` on Windows ARM64). The goal is to turn that into a
|
|
84
|
+
standardized, reproducible build recipe that can target other popular PyPI
|
|
85
|
+
packages missing wheels for a given platform - starting with the platforms
|
|
86
|
+
that come up most often (Alpine/musl, less-common architectures, newer
|
|
87
|
+
Python versions upstream hasn't built for yet), rather than trying to
|
|
88
|
+
cover everything at once.
|
|
89
|
+
|
|
90
|
+
## Not yet decided
|
|
91
|
+
|
|
92
|
+
- CLI surface beyond `install`/`verify`/`doctor` (list available
|
|
93
|
+
platforms? show why a package fell back? a way to request a new
|
|
94
|
+
package/platform combo?)
|
|
95
|
+
- How package/platform requests get prioritized once this covers more than
|
|
96
|
+
one package
|
|
97
|
+
- Whether to eventually provide a wrapper mode that invokes pip directly after verification,
|
|
98
|
+
versus keeping the strict "print the safe command only" separation.
|
|
99
|
+
|
|
100
|
+
Contributions and issues on any of the above are welcome - this project is
|
|
101
|
+
still early, even with a first attested build shipped.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "python-wheels"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Verify Sigstore-attested wheels from python-wheels-builds before installing"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
authors = [{ name = "Patrick Ryan", email = "patrickryankenneth@gmail.com" }]
|
|
12
|
+
license = "MIT"
|
|
13
|
+
license-files = ["LICENSE"]
|
|
14
|
+
keywords = ["wheels", "pip", "sigstore", "attestation", "supply-chain", "slsa", "provenance"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Topic :: Security",
|
|
22
|
+
"Topic :: System :: Software Distribution",
|
|
23
|
+
]
|
|
24
|
+
# No required deps, and none planned - the whole design is "verify with
|
|
25
|
+
# whatever's already there" (sigstore if installed, else the `gh` binary,
|
|
26
|
+
# which isn't a pip dependency at all). Nothing here needs a transitive
|
|
27
|
+
# dependency pinned either, for the same reason.
|
|
28
|
+
dependencies = []
|
|
29
|
+
|
|
30
|
+
# sigstore is optional: pywheels falls back to the `gh` CLI (a system
|
|
31
|
+
# binary, not a pip dependency) if it's absent, and fails closed - refuses
|
|
32
|
+
# to treat a wheel as trusted - only if NEITHER is available. See
|
|
33
|
+
# pywheels/verify.py and `pywheels doctor`.
|
|
34
|
+
#
|
|
35
|
+
# Pinned to a range, not left open-ended: 4.0 <= version < 4.6 is the
|
|
36
|
+
# range this scaffold was actually built and tested against (4.5.0, at
|
|
37
|
+
# time of writing). Bump the upper bound deliberately after testing a
|
|
38
|
+
# newer release, rather than always taking latest - a brand-new release
|
|
39
|
+
# hasn't had time to be exercised by the broader ecosystem yet, which
|
|
40
|
+
# matters more for a security-verification dependency than for most
|
|
41
|
+
# others. We can pin our own dependency this way; we can't pin the `gh`
|
|
42
|
+
# CLI or git the same way since they're not pip packages - the closest
|
|
43
|
+
# equivalent there is installing them via your OS's package manager
|
|
44
|
+
# (apt/brew/choco) rather than a bleeding-edge standalone installer, since
|
|
45
|
+
# distro packaging tends to lag upstream by a bit, which is a feature here.
|
|
46
|
+
[project.optional-dependencies]
|
|
47
|
+
sigstore = ["sigstore>=4.0,<4.6"]
|
|
48
|
+
test = ["pytest>=8"]
|
|
49
|
+
|
|
50
|
+
[project.scripts]
|
|
51
|
+
pywheels = "pywheels.cli:main"
|
|
52
|
+
python-wheels = "pywheels.cli:main"
|
|
53
|
+
|
|
54
|
+
[project.urls]
|
|
55
|
+
Homepage = "https://github.com/patrickryankenneth/python-wheels"
|
|
56
|
+
Builds = "https://github.com/patrickryankenneth/python-wheels-builds"
|
|
57
|
+
Index = "https://python-wheels.github.io"
|
|
58
|
+
Issues = "https://github.com/patrickryankenneth/python-wheels/issues"
|
|
59
|
+
Changelog = "https://github.com/patrickryankenneth/python-wheels/blob/main/CHANGELOG.md"
|
|
60
|
+
|
|
61
|
+
[tool.hatch.version]
|
|
62
|
+
path = "src/pywheels/__init__.py"
|
|
63
|
+
|
|
64
|
+
# hatchling's default package discovery looks for a directory matching the
|
|
65
|
+
# normalized project name (python-wheels -> python_wheels). Our importable
|
|
66
|
+
# package is still named `pywheels/` on purpose (see the pyproject/import
|
|
67
|
+
# naming discussion elsewhere), and it now lives under src/ - so it has to
|
|
68
|
+
# be told explicitly where to find it, or the wheel build will fail to
|
|
69
|
+
# locate any package at all. Giving "src/pywheels" (not just "pywheels")
|
|
70
|
+
# is what tells hatchling to strip the src/ prefix - the wheel still ends
|
|
71
|
+
# up with a top-level `pywheels/`, so nothing about the import path or the
|
|
72
|
+
# [project.scripts] entry points below changes.
|
|
73
|
+
[tool.hatch.build.targets.wheel]
|
|
74
|
+
packages = ["src/pywheels"]
|
|
75
|
+
|
|
76
|
+
[tool.pytest.ini_options]
|
|
77
|
+
testpaths = ["tests"]
|