git-buckets 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- git_buckets-0.2.0/PKG-INFO +20 -0
- git_buckets-0.2.0/README.md +5 -0
- git_buckets-0.2.0/pyproject.toml +31 -0
- git_buckets-0.2.0/pyproject.toml.orig +28 -0
- git_buckets-0.2.0/src/git_buckets/__init__.py +0 -0
- git_buckets-0.2.0/src/git_buckets/cli.py +58 -0
- git_buckets-0.2.0/src/git_buckets/keyring/__init__.py +1 -0
- git_buckets-0.2.0/src/git_buckets/keyring/backend.py +120 -0
- git_buckets-0.2.0/src/git_buckets/keyring/proof.py +90 -0
- git_buckets-0.2.0/src/git_buckets/publish.py +352 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: git-buckets
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Git over S3: clone, fetch and push repositories backed by an S3 bucket.
|
|
5
|
+
Author: Full Duplex Media
|
|
6
|
+
Author-email: Full Duplex Media <contact@fullduplex.media>
|
|
7
|
+
License: Apache 2.0
|
|
8
|
+
Requires-Dist: boto3
|
|
9
|
+
Requires-Dist: click
|
|
10
|
+
Requires-Dist: fduplex-git-remote-s3
|
|
11
|
+
Requires-Dist: keyring
|
|
12
|
+
Requires-Dist: packaging
|
|
13
|
+
Requires-Python: >=3.14
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# git-buckets
|
|
17
|
+
|
|
18
|
+
Git over S3: clone, fetch and push repositories backed by an S3 bucket.
|
|
19
|
+
|
|
20
|
+
The end-user surface lands in Increment 3 of the refactor. See the repo root `README.md`.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "git-buckets"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "Git over S3: clone, fetch and push repositories backed by an S3 bucket."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.14"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"boto3",
|
|
9
|
+
"click",
|
|
10
|
+
"fduplex-git-remote-s3",
|
|
11
|
+
"keyring",
|
|
12
|
+
"packaging",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[[project.authors]]
|
|
16
|
+
name = "Full Duplex Media"
|
|
17
|
+
email = "contact@fullduplex.media"
|
|
18
|
+
|
|
19
|
+
[project.license]
|
|
20
|
+
text = "Apache 2.0"
|
|
21
|
+
|
|
22
|
+
[project.entry-points."keyring.backends"]
|
|
23
|
+
git-buckets = "git_buckets.keyring.backend"
|
|
24
|
+
|
|
25
|
+
[project.scripts]
|
|
26
|
+
gb = "git_buckets.cli:cli"
|
|
27
|
+
keyring = "keyring.cli:main"
|
|
28
|
+
|
|
29
|
+
[build-system]
|
|
30
|
+
requires = ["uv_build>=0.11.0,<0.12.0"]
|
|
31
|
+
build-backend = "uv_build"
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "git-buckets"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "Git over S3: clone, fetch and push repositories backed by an S3 bucket."
|
|
5
|
+
authors = [
|
|
6
|
+
{ name = 'Full Duplex Media', email = 'contact@fullduplex.media' }
|
|
7
|
+
]
|
|
8
|
+
readme = 'README.md'
|
|
9
|
+
license = { text = 'Apache 2.0' }
|
|
10
|
+
requires-python = ">=3.14"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"boto3",
|
|
13
|
+
"click",
|
|
14
|
+
"fduplex-git-remote-s3",
|
|
15
|
+
"keyring",
|
|
16
|
+
"packaging",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.entry-points."keyring.backends"]
|
|
20
|
+
git-buckets = "git_buckets.keyring.backend"
|
|
21
|
+
|
|
22
|
+
[project.scripts]
|
|
23
|
+
gb = "git_buckets.cli:cli"
|
|
24
|
+
keyring = "keyring.cli:main"
|
|
25
|
+
|
|
26
|
+
[build-system]
|
|
27
|
+
requires = ['uv_build>=0.11.0,<0.12.0']
|
|
28
|
+
build-backend = 'uv_build'
|
|
File without changes
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
import boto3
|
|
4
|
+
import click
|
|
5
|
+
|
|
6
|
+
from .publish import Distribution, PublishError, default_distributions, parse_distributions, publish, resolve_target
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _session(profile: str | None) -> boto3.Session:
|
|
10
|
+
return boto3.Session(profile_name=profile) if profile else boto3.Session()
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
_profile_option = click.option('--profile', default=None, help='AWS profile to use (overrides AWS_PROFILE).')
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@click.group()
|
|
17
|
+
@click.version_option(package_name='git-buckets')
|
|
18
|
+
def cli() -> None:
|
|
19
|
+
"""Git over S3."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@cli.command(name='publish')
|
|
23
|
+
@click.option('--bucket', 'bucket', default=None, help='Registry bucket alias (<bucket-name>.<web-domain>).')
|
|
24
|
+
@click.option('--repo', 'repo', default=None, help='Repo within the bucket that owns the package.')
|
|
25
|
+
@click.option('--remote', 'remote', default='origin', show_default=True, help='Git remote to read the target from.')
|
|
26
|
+
@_profile_option
|
|
27
|
+
@click.option('--dry-run', is_flag=True, help='Resolve, mint and pre-check for real, but upload nothing.')
|
|
28
|
+
@click.argument('dists', nargs=-1, type=click.Path(path_type=Path))
|
|
29
|
+
def publish_command(
|
|
30
|
+
bucket: str | None,
|
|
31
|
+
repo: str | None,
|
|
32
|
+
remote: str,
|
|
33
|
+
profile: str | None,
|
|
34
|
+
dry_run: bool,
|
|
35
|
+
dists: tuple[Path, ...],
|
|
36
|
+
) -> None:
|
|
37
|
+
"""Publish already-built distributions to a git-bucket's package registry.
|
|
38
|
+
|
|
39
|
+
Not manifest-bound: run it from any Python project. With no flags the target comes from the project's
|
|
40
|
+
s3://[profile@]bucket-alias/repo git remote, and with no paths the batch is dist/*.
|
|
41
|
+
|
|
42
|
+
The AWS profile is resolved in order: --profile if given, else the profile embedded in the remote's
|
|
43
|
+
s3:// URL, else AWS_PROFILE from the environment, else the default credential chain.
|
|
44
|
+
"""
|
|
45
|
+
if bucket is None:
|
|
46
|
+
click.echo(f'Resolving git remote {remote}...')
|
|
47
|
+
try:
|
|
48
|
+
target = resolve_target(bucket=bucket, repo=repo, remote=remote, profile=profile)
|
|
49
|
+
click.echo(f'Bucket: {target.alias}')
|
|
50
|
+
click.echo(f'Repo: {target.repo}')
|
|
51
|
+
click.echo(f'Profile: {target.profile or "(ambient)"}')
|
|
52
|
+
distributions: list[Distribution] = parse_distributions(list(dists) or default_distributions())
|
|
53
|
+
result = publish(_session(target.profile), target, distributions, dry_run=dry_run, report=click.echo)
|
|
54
|
+
except PublishError as x:
|
|
55
|
+
raise click.ClickException(str(x)) from None
|
|
56
|
+
|
|
57
|
+
if result.dry_run:
|
|
58
|
+
click.echo('Dry run: nothing was uploaded.')
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Keyring backend and STS proof builder for git-buckets package registries."""
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""A stateless keyring backend that turns an ambient AWS profile into a git-buckets package token.
|
|
2
|
+
|
|
3
|
+
The backend is inert unless the caller's host is on the ``GB_KEYRING_HOSTS`` allowlist and the username is the
|
|
4
|
+
``gb`` sentinel, so installing it can never change what any other index or tool resolves.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
from urllib.parse import urlsplit
|
|
10
|
+
|
|
11
|
+
import boto3
|
|
12
|
+
from keyring.backend import KeyringBackend
|
|
13
|
+
from keyring.compat import properties
|
|
14
|
+
from keyring.credentials import Credential, SimpleCredential
|
|
15
|
+
from keyring.errors import PasswordDeleteError, PasswordSetError
|
|
16
|
+
|
|
17
|
+
from .proof import mint_token
|
|
18
|
+
|
|
19
|
+
_HOSTS_ENV = 'GB_KEYRING_HOSTS'
|
|
20
|
+
_PROFILE_ENV = 'GB_KEYRING_PROFILE'
|
|
21
|
+
_DEBUG_ENV = 'GB_KEYRING_DEBUG'
|
|
22
|
+
_USERNAME = 'gb'
|
|
23
|
+
|
|
24
|
+
# Above the OS keyrings (5) so it wins inside uv's isolated tool environment; the host allowlist, not the
|
|
25
|
+
# priority, is what keeps it from answering for anything else.
|
|
26
|
+
_PRIORITY = 9.9
|
|
27
|
+
|
|
28
|
+
_tokens: dict[tuple[str, str], str] = {}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _allowed_hosts() -> set[str]:
|
|
32
|
+
raw = os.environ.get(_HOSTS_ENV, '')
|
|
33
|
+
return {host.strip().lower() for host in raw.split(',') if host.strip()}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _profile() -> str:
|
|
37
|
+
return os.environ.get(_PROFILE_ENV, '').strip()
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _debug(message: str) -> None:
|
|
41
|
+
"""Explain a refusal on stderr; stdout carries the password and must stay clean."""
|
|
42
|
+
if os.environ.get(_DEBUG_ENV):
|
|
43
|
+
print(f'git-buckets-keyring: {message}', file=sys.stderr)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _host_of(service: str) -> str | None:
|
|
47
|
+
"""Return the hostname from a full index URL or a bare host, or None when there isn't one."""
|
|
48
|
+
if not service:
|
|
49
|
+
return None
|
|
50
|
+
candidate = service if '://' in service else f'//{service}'
|
|
51
|
+
try:
|
|
52
|
+
return urlsplit(candidate).hostname
|
|
53
|
+
except ValueError:
|
|
54
|
+
return None
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _scheme_of(service: str) -> str:
|
|
58
|
+
"""Return the lowercased scheme when the service string names one explicitly, else ''.
|
|
59
|
+
|
|
60
|
+
A bare host (no ``://``) carries no scheme at all, which is not the same as an insecure one.
|
|
61
|
+
"""
|
|
62
|
+
if not service or '://' not in service:
|
|
63
|
+
return ''
|
|
64
|
+
try:
|
|
65
|
+
return urlsplit(service).scheme.lower()
|
|
66
|
+
except ValueError:
|
|
67
|
+
return ''
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class GitBucketsBackend(KeyringBackend):
|
|
71
|
+
"""Mint a short-lived package token for an allowlisted git-buckets deployment."""
|
|
72
|
+
|
|
73
|
+
@properties.classproperty
|
|
74
|
+
def priority(cls) -> float: # noqa: N805
|
|
75
|
+
return _PRIORITY
|
|
76
|
+
|
|
77
|
+
def get_password(self, service: str, username: str) -> str | None:
|
|
78
|
+
if username != _USERNAME:
|
|
79
|
+
_debug(f'declined: username {username!r} is not {_USERNAME!r}')
|
|
80
|
+
return None
|
|
81
|
+
host = _host_of(service)
|
|
82
|
+
allowed = _allowed_hosts()
|
|
83
|
+
if host is None:
|
|
84
|
+
_debug(f'declined: no host in service {service!r}')
|
|
85
|
+
return None
|
|
86
|
+
if host.lower() not in allowed:
|
|
87
|
+
listed = ', '.join(sorted(allowed)) if allowed else f'empty (set {_HOSTS_ENV})'
|
|
88
|
+
_debug(f'declined: host {host!r} is not on the {_HOSTS_ENV} allowlist: {listed}')
|
|
89
|
+
return None
|
|
90
|
+
scheme = _scheme_of(service)
|
|
91
|
+
if scheme and scheme != 'https':
|
|
92
|
+
_debug(f'declined: scheme {scheme!r} for host {host!r} is not https')
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
profile = _profile()
|
|
96
|
+
cached = _tokens.get((host, profile))
|
|
97
|
+
if cached is not None:
|
|
98
|
+
return cached
|
|
99
|
+
try:
|
|
100
|
+
# Inside the try: a missing or typo'd profile must fail closed like any other mint failure.
|
|
101
|
+
session = boto3.Session(profile_name=profile) if profile else None
|
|
102
|
+
token = mint_token(host, session)
|
|
103
|
+
except Exception as x:
|
|
104
|
+
# A raising backend breaks the client's whole credential lookup; no profile means no answer.
|
|
105
|
+
_debug(f'declined: mint for {host!r} failed ({type(x).__name__}: {x})')
|
|
106
|
+
return None
|
|
107
|
+
_tokens[(host, profile)] = token
|
|
108
|
+
return token
|
|
109
|
+
|
|
110
|
+
def get_credential(self, service: str, username: str | None) -> Credential | None:
|
|
111
|
+
password = self.get_password(service, username or _USERNAME)
|
|
112
|
+
if password is None:
|
|
113
|
+
return None
|
|
114
|
+
return SimpleCredential(_USERNAME, password)
|
|
115
|
+
|
|
116
|
+
def set_password(self, service: str, username: str, password: str) -> None:
|
|
117
|
+
raise PasswordSetError('git-buckets keyring is stateless; tokens are minted, not stored')
|
|
118
|
+
|
|
119
|
+
def delete_password(self, service: str, username: str) -> None:
|
|
120
|
+
raise PasswordDeleteError('git-buckets keyring is stateless; there is nothing to delete')
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Build a SigV4-signed ``sts:GetCallerIdentity`` proof and trade it for a git-buckets package token.
|
|
2
|
+
|
|
3
|
+
The proof is the whole authentication: the server validates the components, rebuilds the call and replays it
|
|
4
|
+
against real STS, so AWS performs the signature check. ``X-Gb-Server-Id`` is added before signing, which is what
|
|
5
|
+
binds the proof to one deployment.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from hashlib import sha256
|
|
10
|
+
from typing import Any
|
|
11
|
+
from urllib.error import HTTPError, URLError
|
|
12
|
+
from urllib.request import Request, urlopen
|
|
13
|
+
|
|
14
|
+
import boto3
|
|
15
|
+
from botocore.auth import SigV4Auth
|
|
16
|
+
from botocore.awsrequest import AWSRequest
|
|
17
|
+
|
|
18
|
+
_STS_BODY = 'Action=GetCallerIdentity&Version=2011-06-15'
|
|
19
|
+
_SERVER_ID_HEADER = 'X-Gb-Server-Id'
|
|
20
|
+
_DEFAULT_REGION = 'us-east-1'
|
|
21
|
+
_TIMEOUT_S = 15
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ProofError(Exception):
|
|
25
|
+
"""The proof could not be built, sent, or exchanged for a token."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def build_proof(session: Any, host: str) -> dict[str, Any]:
|
|
29
|
+
"""Return the signed request bundle ``{'method', 'url', 'headers', 'body'}`` for ``host``.
|
|
30
|
+
|
|
31
|
+
Hand-rolled rather than presigned: boto3 has never supported presigning GetCallerIdentity (boto3#756).
|
|
32
|
+
"""
|
|
33
|
+
credentials = session.get_credentials()
|
|
34
|
+
if credentials is None:
|
|
35
|
+
raise ProofError('no AWS credentials')
|
|
36
|
+
region = session.region_name or _DEFAULT_REGION
|
|
37
|
+
|
|
38
|
+
request = AWSRequest(
|
|
39
|
+
method='POST',
|
|
40
|
+
url=f'https://sts.{region}.amazonaws.com/',
|
|
41
|
+
data=_STS_BODY,
|
|
42
|
+
headers={
|
|
43
|
+
'Content-Type': 'application/x-www-form-urlencoded',
|
|
44
|
+
# Signed, not merely sent: presence inside SignedHeaders is what makes the audience unforgeable.
|
|
45
|
+
_SERVER_ID_HEADER: host,
|
|
46
|
+
},
|
|
47
|
+
)
|
|
48
|
+
SigV4Auth(credentials, 'sts', region).add_auth(request)
|
|
49
|
+
|
|
50
|
+
return {
|
|
51
|
+
'method': 'POST',
|
|
52
|
+
'url': request.url,
|
|
53
|
+
'headers': dict(request.headers),
|
|
54
|
+
'body': _STS_BODY,
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def mint_token(host: str, session: Any = None) -> str:
|
|
59
|
+
"""Exchange a freshly-signed proof for a short-lived package token at ``https://<host>/auth/token``."""
|
|
60
|
+
proof = build_proof(session if session is not None else boto3.Session(), host)
|
|
61
|
+
body = json.dumps(proof).encode()
|
|
62
|
+
request = Request(
|
|
63
|
+
f'https://{host}/auth/token',
|
|
64
|
+
data=body,
|
|
65
|
+
method='POST',
|
|
66
|
+
headers={
|
|
67
|
+
'Content-Type': 'application/json',
|
|
68
|
+
# CloudFront OAC rejects a body-carrying POST without this with a bare edge 403.
|
|
69
|
+
'x-amz-content-sha256': sha256(body).hexdigest(),
|
|
70
|
+
},
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
try:
|
|
74
|
+
with urlopen(request, timeout=_TIMEOUT_S) as response:
|
|
75
|
+
status = response.status
|
|
76
|
+
payload = response.read()
|
|
77
|
+
except HTTPError as x:
|
|
78
|
+
raise ProofError(f'token request failed with status {x.code}') from None
|
|
79
|
+
except (URLError, OSError) as x:
|
|
80
|
+
raise ProofError(f'token request failed: {x}') from None
|
|
81
|
+
|
|
82
|
+
if not 200 <= status < 300:
|
|
83
|
+
raise ProofError(f'token request failed with status {status}')
|
|
84
|
+
try:
|
|
85
|
+
token = json.loads(payload)['token']
|
|
86
|
+
except (ValueError, KeyError, TypeError):
|
|
87
|
+
raise ProofError('token response is malformed') from None
|
|
88
|
+
if not isinstance(token, str) or not token:
|
|
89
|
+
raise ProofError('token response is malformed')
|
|
90
|
+
return token
|
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
"""Transport distributions built elsewhere into a repo's ``<repo>/packages/`` prefix.
|
|
2
|
+
|
|
3
|
+
``gb publish`` is not manifest-bound: its habitat is an arbitrary Python project folder, so everything it needs
|
|
4
|
+
comes from flags or from the project's ``s3://`` git remote. It never builds, never invokes a backend and never
|
|
5
|
+
reimplements packaging semantics — filename grammar is ``packaging``'s problem, and the upload is a direct
|
|
6
|
+
conditional PutObject against Access-Grants-vended credentials rather than a legacy upload API.
|
|
7
|
+
|
|
8
|
+
The registry pre-check runs before any upload and a non-answer is fatal: publish does not proceed on an
|
|
9
|
+
unanswered ownership question.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import subprocess
|
|
13
|
+
from base64 import b64encode
|
|
14
|
+
from collections.abc import Callable
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from json import loads
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
from typing import Any
|
|
19
|
+
from urllib.error import HTTPError, URLError
|
|
20
|
+
from urllib.parse import quote
|
|
21
|
+
from urllib.request import Request, urlopen
|
|
22
|
+
from zipfile import BadZipFile, ZipFile
|
|
23
|
+
|
|
24
|
+
from git_remote_s3 import BucketAliasError, parse_git_url, resolve_bucket_alias
|
|
25
|
+
from git_remote_s3.common import register_s3_access_grants
|
|
26
|
+
from packaging.utils import (
|
|
27
|
+
InvalidSdistFilename,
|
|
28
|
+
InvalidVersion,
|
|
29
|
+
InvalidWheelFilename,
|
|
30
|
+
canonicalize_name,
|
|
31
|
+
parse_sdist_filename,
|
|
32
|
+
parse_wheel_filename,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
from .keyring.proof import ProofError, mint_token
|
|
36
|
+
|
|
37
|
+
_METADATA_SUFFIX = '.metadata'
|
|
38
|
+
_SDIST_SUFFIXES = ('.tar.gz', '.zip')
|
|
39
|
+
_TIMEOUT_S = 30
|
|
40
|
+
# 409 ConditionalRequestConflict is the one conditional-write failure AWS documents as retryable.
|
|
41
|
+
_CONFLICT_ATTEMPTS = 3
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class PublishError(Exception):
|
|
45
|
+
"""Publishing could not proceed. Every instance is a loud, actionable CLI failure."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _silent(_message: str) -> None:
|
|
49
|
+
"""Default progress reporter: publish() is a library call, so it says nothing unless asked to."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class Target:
|
|
54
|
+
"""Where a batch is going: the manifest bucket name, the physical S3 bucket, the host and the repo."""
|
|
55
|
+
|
|
56
|
+
manifest_bucket: str
|
|
57
|
+
bucket: str
|
|
58
|
+
host: str
|
|
59
|
+
repo: str
|
|
60
|
+
profile: str | None
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def alias(self) -> str:
|
|
64
|
+
"""The registry alias the publisher actually knows, rebuilt from the halves ``_split_alias`` produced."""
|
|
65
|
+
return f'{self.manifest_bucket}.{self.host}'
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class Distribution:
|
|
70
|
+
path: Path
|
|
71
|
+
name: str
|
|
72
|
+
version: str
|
|
73
|
+
is_wheel: bool
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def filename(self) -> str:
|
|
77
|
+
return self.path.name
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True)
|
|
81
|
+
class Upload:
|
|
82
|
+
key: str
|
|
83
|
+
body: bytes
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class PublishResult:
|
|
88
|
+
target: Target
|
|
89
|
+
ownership: dict[str, bool]
|
|
90
|
+
keys: list[str]
|
|
91
|
+
dry_run: bool
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _remote_url(remote: str) -> str | None:
|
|
95
|
+
try:
|
|
96
|
+
completed = subprocess.run(
|
|
97
|
+
['git', 'config', '--get', f'remote.{remote}.url'],
|
|
98
|
+
capture_output=True,
|
|
99
|
+
text=True,
|
|
100
|
+
check=False,
|
|
101
|
+
)
|
|
102
|
+
except OSError:
|
|
103
|
+
return None
|
|
104
|
+
if completed.returncode != 0:
|
|
105
|
+
return None
|
|
106
|
+
return completed.stdout.strip() or None
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _no_target_error(remote: str) -> PublishError:
|
|
110
|
+
return PublishError(
|
|
111
|
+
f'cannot resolve where to publish: no --bucket/--repo given and git remote {remote!r} is not an '
|
|
112
|
+
f's3:// git-buckets URL (looked in {Path.cwd()}). Pass --bucket <bucket-name>.<web-domain> and '
|
|
113
|
+
'--repo <repo>, or run from a checkout cloned from a git-bucket.'
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _split_alias(alias: str, remote: str) -> tuple[str, str, str]:
|
|
118
|
+
"""``<manifest-bucket>.<web-domain>`` → (manifest bucket, host, physical bucket).
|
|
119
|
+
|
|
120
|
+
The alias is the record git-buckets itself renders as ``{{ bucket.name }}.{{ web.domain }}``, so the first
|
|
121
|
+
dot splits it exactly. The registry pre-check needs the host and the manifest name; only the upload uses the
|
|
122
|
+
physical bucket the TXT record resolves to.
|
|
123
|
+
"""
|
|
124
|
+
if '.' not in alias:
|
|
125
|
+
raise PublishError(
|
|
126
|
+
f'bucket {alias!r} is a physical bucket name, not a registry alias: publish needs the '
|
|
127
|
+
'<bucket-name>.<web-domain> form (e.g. demos.git.example.com) because the ownership pre-check and '
|
|
128
|
+
'the package token are served by the deployment at that host.'
|
|
129
|
+
)
|
|
130
|
+
manifest_bucket, host = alias.split('.', 1)
|
|
131
|
+
|
|
132
|
+
try:
|
|
133
|
+
bucket = resolve_bucket_alias(alias, remote)
|
|
134
|
+
except BucketAliasError as x:
|
|
135
|
+
raise PublishError(f'cannot resolve bucket alias {alias!r}: {x}') from None
|
|
136
|
+
if '.' in bucket:
|
|
137
|
+
# Manifest bucket names can never contain a dot, so a dotted answer means alias resolution was opted
|
|
138
|
+
# out of (s3.dns-alias / remote.<name>.s3-dns-alias) and the host came back verbatim.
|
|
139
|
+
raise PublishError(
|
|
140
|
+
f'cannot resolve bucket alias {alias!r}: DNS alias resolution is disabled for this remote, so the '
|
|
141
|
+
'alias host would be used as a bucket name. Re-enable it (git config --unset s3.dns-alias, or the '
|
|
142
|
+
f'remote.{remote}.s3-dns-alias key) to publish.'
|
|
143
|
+
)
|
|
144
|
+
return manifest_bucket, host, bucket
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def resolve_target(
|
|
148
|
+
*, bucket: str | None, repo: str | None, remote: str = 'origin', profile: str | None = None
|
|
149
|
+
) -> Target:
|
|
150
|
+
"""Explicit flags, else the project's ``s3://`` git remote, else a loud error naming both options."""
|
|
151
|
+
if (bucket is None) != (repo is None):
|
|
152
|
+
raise PublishError('--bucket and --repo must be given together (or neither, to use the git remote)')
|
|
153
|
+
|
|
154
|
+
url_profile: str | None = None
|
|
155
|
+
if bucket is None:
|
|
156
|
+
_scheme, url_profile, bucket, repo = parse_git_url(_remote_url(remote))
|
|
157
|
+
if bucket is None or not repo:
|
|
158
|
+
raise _no_target_error(remote)
|
|
159
|
+
|
|
160
|
+
manifest_bucket, host, physical = _split_alias(bucket, remote)
|
|
161
|
+
return Target(
|
|
162
|
+
manifest_bucket=manifest_bucket,
|
|
163
|
+
bucket=physical,
|
|
164
|
+
host=host,
|
|
165
|
+
repo=repo.strip('/'),
|
|
166
|
+
profile=profile if profile is not None else url_profile,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def default_distributions(root: Path | None = None) -> list[Path]:
|
|
171
|
+
"""``dist/*`` — the conventional output directory, whatever built it."""
|
|
172
|
+
dist = (root or Path.cwd()) / 'dist'
|
|
173
|
+
if not dist.is_dir():
|
|
174
|
+
raise PublishError(f'no distributions given and no {dist} directory: build first, or pass paths')
|
|
175
|
+
paths = sorted(path for path in dist.iterdir() if path.is_file())
|
|
176
|
+
if not paths:
|
|
177
|
+
raise PublishError(f'no distributions given and {dist} is empty: build first, or pass paths')
|
|
178
|
+
return paths
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def parse_distribution(path: Path) -> Distribution:
|
|
182
|
+
"""One file → (canonical name, version) via ``packaging``; never a hand-rolled filename regex."""
|
|
183
|
+
filename = path.name
|
|
184
|
+
try:
|
|
185
|
+
if filename.endswith('.whl'):
|
|
186
|
+
name, version, _build, _tags = parse_wheel_filename(filename)
|
|
187
|
+
return Distribution(path=path, name=str(name), version=str(version), is_wheel=True)
|
|
188
|
+
if filename.endswith(_SDIST_SUFFIXES):
|
|
189
|
+
name, version = parse_sdist_filename(filename)
|
|
190
|
+
return Distribution(path=path, name=str(name), version=str(version), is_wheel=False)
|
|
191
|
+
except (InvalidWheelFilename, InvalidSdistFilename, InvalidVersion) as x:
|
|
192
|
+
raise PublishError(f'{filename}: not a valid distribution filename ({x})') from None
|
|
193
|
+
raise PublishError(f'{filename}: unknown distribution type (expected a .whl, .tar.gz or .zip)')
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def parse_distributions(paths: list[Path]) -> list[Distribution]:
|
|
197
|
+
"""Parse a batch and reject one that does not belong to a single project."""
|
|
198
|
+
if not paths:
|
|
199
|
+
raise PublishError('no distributions to publish')
|
|
200
|
+
missing = [str(path) for path in paths if not path.is_file()]
|
|
201
|
+
if missing:
|
|
202
|
+
raise PublishError(f'no such distribution file(s): {", ".join(missing)}')
|
|
203
|
+
|
|
204
|
+
distributions = [parse_distribution(path) for path in paths]
|
|
205
|
+
names = sorted({dist.name for dist in distributions})
|
|
206
|
+
if len(names) > 1:
|
|
207
|
+
raise PublishError(
|
|
208
|
+
f'this batch spans {len(names)} projects ({", ".join(names)}): one publish targets one repo, so it '
|
|
209
|
+
'must carry one project. Pass explicit paths, or run gb publish from the member directory (a uv '
|
|
210
|
+
'workspace build drops every member into one shared dist/).'
|
|
211
|
+
)
|
|
212
|
+
return distributions
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def wheel_metadata(path: Path) -> bytes:
|
|
216
|
+
"""The wheel's ``*.dist-info/METADATA`` member, for the PEP 658 sidecar."""
|
|
217
|
+
try:
|
|
218
|
+
with ZipFile(path) as archive:
|
|
219
|
+
members = [
|
|
220
|
+
name for name in archive.namelist() if name.endswith('.dist-info/METADATA') and name.count('/') == 1
|
|
221
|
+
]
|
|
222
|
+
if len(members) != 1:
|
|
223
|
+
raise PublishError(f'{path.name}: expected exactly one .dist-info/METADATA, found {len(members)}')
|
|
224
|
+
return archive.read(members[0])
|
|
225
|
+
except (BadZipFile, OSError) as x:
|
|
226
|
+
raise PublishError(f'{path.name}: cannot read wheel ({x})') from None
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def plan_uploads(target: Target, distributions: list[Distribution]) -> list[Upload]:
|
|
230
|
+
"""The exact objects a batch writes: the distribution, plus a metadata sidecar for wheels only."""
|
|
231
|
+
uploads: list[Upload] = []
|
|
232
|
+
for dist in distributions:
|
|
233
|
+
prefix = f'{target.repo}/packages/{canonicalize_name(dist.name)}/'
|
|
234
|
+
uploads.append(Upload(key=f'{prefix}{dist.filename}', body=dist.path.read_bytes()))
|
|
235
|
+
if dist.is_wheel:
|
|
236
|
+
uploads.append(Upload(key=f'{prefix}{dist.filename}{_METADATA_SUFFIX}', body=wheel_metadata(dist.path)))
|
|
237
|
+
return uploads
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def check_ownership(target: Target, name: str, token: str) -> bool:
|
|
241
|
+
"""The registry's one-bit answer for ``name`` in this bucket. Any non-answer is fatal, never a default."""
|
|
242
|
+
url = (
|
|
243
|
+
f'https://{target.host}/packages/{quote(target.manifest_bucket)}/pypi/{quote(canonicalize_name(name))}/'
|
|
244
|
+
f'?ownership={quote(target.repo, safe="")}'
|
|
245
|
+
)
|
|
246
|
+
# The edge renames Authorization into X-Gb-Authorization; the client only ever sends the standard header.
|
|
247
|
+
credential = b64encode(f'gb:{token}'.encode()).decode()
|
|
248
|
+
request = Request(url, method='GET', headers={'Authorization': f'Basic {credential}'})
|
|
249
|
+
|
|
250
|
+
try:
|
|
251
|
+
with urlopen(request, timeout=_TIMEOUT_S) as response:
|
|
252
|
+
status = response.status
|
|
253
|
+
payload = response.read()
|
|
254
|
+
except HTTPError as x:
|
|
255
|
+
raise PublishError(f'ownership pre-check for {name!r} failed with status {x.code}') from None
|
|
256
|
+
except (URLError, OSError) as x:
|
|
257
|
+
raise PublishError(f'ownership pre-check for {name!r} failed: registry unreachable ({x})') from None
|
|
258
|
+
|
|
259
|
+
if not 200 <= status < 300:
|
|
260
|
+
raise PublishError(f'ownership pre-check for {name!r} failed with status {status}')
|
|
261
|
+
try:
|
|
262
|
+
available = loads(payload)['available']
|
|
263
|
+
except (ValueError, KeyError, TypeError):
|
|
264
|
+
raise PublishError(f'ownership pre-check for {name!r} returned a malformed answer') from None
|
|
265
|
+
return bool(available)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _mint(target: Target, session: Any) -> str:
|
|
269
|
+
try:
|
|
270
|
+
return mint_token(target.host, session)
|
|
271
|
+
except ProofError as x:
|
|
272
|
+
raise PublishError(f'cannot mint a package token for {target.host}: {x}') from None
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _human_size(size: int) -> str:
|
|
276
|
+
value = float(size)
|
|
277
|
+
for unit in ('B', 'KiB', 'MiB', 'GiB'):
|
|
278
|
+
if value < 1024 or unit == 'GiB':
|
|
279
|
+
return f'{int(value)} {unit}' if unit == 'B' else f'{value:.1f} {unit}'
|
|
280
|
+
value /= 1024
|
|
281
|
+
return f'{value:.1f} GiB'
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _put(s3: Any, target: Target, upload: Upload) -> None:
|
|
285
|
+
for attempt in range(1, _CONFLICT_ATTEMPTS + 1):
|
|
286
|
+
try:
|
|
287
|
+
s3.put_object(
|
|
288
|
+
Bucket=target.bucket,
|
|
289
|
+
Key=upload.key,
|
|
290
|
+
Body=upload.body,
|
|
291
|
+
IfNoneMatch='*',
|
|
292
|
+
ChecksumAlgorithm='SHA256',
|
|
293
|
+
)
|
|
294
|
+
return
|
|
295
|
+
except s3.exceptions.ClientError as x:
|
|
296
|
+
code = x.response.get('Error', {}).get('Code')
|
|
297
|
+
if code == 'PreconditionFailed':
|
|
298
|
+
raise PublishError(
|
|
299
|
+
f'{upload.key.rsplit("/", 1)[-1]} is already published at s3://{target.alias}/{upload.key}: '
|
|
300
|
+
'published artifacts are immutable, so bump the version and re-publish.'
|
|
301
|
+
) from None
|
|
302
|
+
if code == 'ConditionalRequestConflict' and attempt < _CONFLICT_ATTEMPTS:
|
|
303
|
+
continue
|
|
304
|
+
raise PublishError(f'upload of {upload.key} failed: {x}') from None
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def publish(
|
|
308
|
+
session: Any,
|
|
309
|
+
target: Target,
|
|
310
|
+
distributions: list[Distribution],
|
|
311
|
+
*,
|
|
312
|
+
dry_run: bool = False,
|
|
313
|
+
report: Callable[[str], None] = _silent,
|
|
314
|
+
) -> PublishResult:
|
|
315
|
+
"""Mint, pre-check every distinct name, then conditionally PUT each object. Nothing uploads on a dry run.
|
|
316
|
+
|
|
317
|
+
``report`` receives one short line per phase as it happens, so a caller can stream progress through the long
|
|
318
|
+
silent stretches (token mint, pre-check, credential vending, the uploads themselves).
|
|
319
|
+
"""
|
|
320
|
+
uploads = plan_uploads(target, distributions)
|
|
321
|
+
|
|
322
|
+
report('Minting a package token...')
|
|
323
|
+
token = _mint(target, session)
|
|
324
|
+
|
|
325
|
+
ownership: dict[str, bool] = {}
|
|
326
|
+
for name in sorted({d.name for d in distributions}):
|
|
327
|
+
report(f'Checking who owns {name}...')
|
|
328
|
+
ownership[name] = check_ownership(target, name, token)
|
|
329
|
+
report(f'Owner: {name}: {"available" if ownership[name] else "claimed"}')
|
|
330
|
+
claimed = [name for name, available in ownership.items() if not available]
|
|
331
|
+
if claimed:
|
|
332
|
+
raise PublishError(
|
|
333
|
+
f'{", ".join(claimed)}: already claimed by another repo in bucket {target.manifest_bucket!r}. '
|
|
334
|
+
'Publish it from the repo that owns the name, or rename the project.'
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
keys = [upload.key for upload in uploads]
|
|
338
|
+
if dry_run:
|
|
339
|
+
report('Would write:')
|
|
340
|
+
for key in keys:
|
|
341
|
+
report(f' s3://{target.alias}/{key}')
|
|
342
|
+
return PublishResult(target=target, ownership=ownership, keys=keys, dry_run=True)
|
|
343
|
+
|
|
344
|
+
report('Vending upload credentials...')
|
|
345
|
+
s3 = session.client('s3')
|
|
346
|
+
register_s3_access_grants(s3, session)
|
|
347
|
+
report('Writing:')
|
|
348
|
+
for upload in uploads:
|
|
349
|
+
report(f' uploading {upload.key.rsplit("/", 1)[-1]} ({_human_size(len(upload.body))})...')
|
|
350
|
+
_put(s3, target, upload)
|
|
351
|
+
report(f' s3://{target.alias}/{upload.key}')
|
|
352
|
+
return PublishResult(target=target, ownership=ownership, keys=keys, dry_run=False)
|