git-buckets 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {git_buckets-0.3.0 → git_buckets-0.4.0}/PKG-INFO +14 -1
- {git_buckets-0.3.0 → git_buckets-0.4.0}/README.md +13 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/pyproject.toml +1 -1
- {git_buckets-0.3.0 → git_buckets-0.4.0}/pyproject.toml.orig +1 -1
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/cli.py +119 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/credential.py +3 -2
- git_buckets-0.4.0/src/git_buckets/package.py +74 -0
- git_buckets-0.4.0/src/git_buckets/wake.py +213 -0
- git_buckets-0.3.0/src/git_buckets/wake.py +0 -127
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/__init__.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/hosts.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/keyring/__init__.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/keyring/backend.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/keyring/proof.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/lfs_agent.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/login.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/net.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/oauth.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/repo.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/secretstore.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/session.py +0 -0
- {git_buckets-0.3.0 → git_buckets-0.4.0}/src/git_buckets/termio.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: git-buckets
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Git over S3: clone, fetch and push repositories backed by an S3 bucket.
|
|
5
5
|
Author: Full Duplex Media
|
|
6
6
|
Author-email: Full Duplex Media <contact@fullduplex.media>
|
|
@@ -123,6 +123,19 @@ gb repo delete my-bucket/tools/cli/my-project --yes # without --yes it only pri
|
|
|
123
123
|
Delete takes the repo's LFS objects, locks, bundles, packages and rendered web tree with it. The undo path is S3
|
|
124
124
|
versioning on the operator's side, not a `gb` command. There is no rename.
|
|
125
125
|
|
|
126
|
+
## Package lifecycle
|
|
127
|
+
|
|
128
|
+
Published files are immutable, so a bad release is yanked rather than replaced: it stays downloadable for anything
|
|
129
|
+
that already pinned it, and every other resolve skips it.
|
|
130
|
+
|
|
131
|
+
```sh
|
|
132
|
+
gb package list # every bucket, grouped by repo
|
|
133
|
+
gb package list -b my-bucket -r tools/cli/my-project # one repo; -v lists every file with size and sha256
|
|
134
|
+
gb package yank my-bucket/tools/cli/my-project my-project 1.2.0 --reason "bad wheel"
|
|
135
|
+
gb package unyank my-bucket/tools/cli/my-project my-project 1.2.0
|
|
136
|
+
gb package delete my-bucket/tools/cli/my-project my-project 1.2.0 --yes # the bytes go; lockfiles that pin it 404
|
|
137
|
+
```
|
|
138
|
+
|
|
126
139
|
## Where the secrets live
|
|
127
140
|
|
|
128
141
|
The refresh token (one year, fixed from sign-in) goes in your OS keyring, or in `~/.config/gb/hosts.yml` at 0600 when no
|
|
@@ -108,6 +108,19 @@ gb repo delete my-bucket/tools/cli/my-project --yes # without --yes it only pri
|
|
|
108
108
|
Delete takes the repo's LFS objects, locks, bundles, packages and rendered web tree with it. The undo path is S3
|
|
109
109
|
versioning on the operator's side, not a `gb` command. There is no rename.
|
|
110
110
|
|
|
111
|
+
## Package lifecycle
|
|
112
|
+
|
|
113
|
+
Published files are immutable, so a bad release is yanked rather than replaced: it stays downloadable for anything
|
|
114
|
+
that already pinned it, and every other resolve skips it.
|
|
115
|
+
|
|
116
|
+
```sh
|
|
117
|
+
gb package list # every bucket, grouped by repo
|
|
118
|
+
gb package list -b my-bucket -r tools/cli/my-project # one repo; -v lists every file with size and sha256
|
|
119
|
+
gb package yank my-bucket/tools/cli/my-project my-project 1.2.0 --reason "bad wheel"
|
|
120
|
+
gb package unyank my-bucket/tools/cli/my-project my-project 1.2.0
|
|
121
|
+
gb package delete my-bucket/tools/cli/my-project my-project 1.2.0 --yes # the bytes go; lockfiles that pin it 404
|
|
122
|
+
```
|
|
123
|
+
|
|
111
124
|
## Where the secrets live
|
|
112
125
|
|
|
113
126
|
The refresh token (one year, fixed from sign-in) goes in your OS keyring, or in `~/.config/gb/hosts.yml` at 0600 when no
|
|
@@ -231,6 +231,125 @@ def repo_unprotect_command(name: str, ref: str, host: str | None) -> None:
|
|
|
231
231
|
_protect('unprotect', name, ref, host)
|
|
232
232
|
|
|
233
233
|
|
|
234
|
+
@cli.group(name='package')
|
|
235
|
+
def package_group() -> None:
|
|
236
|
+
"""Manage published packages: list them, yank a bad release, delete one."""
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _print_versions(versions: list[dict], verbose: bool) -> None:
|
|
240
|
+
for release in versions:
|
|
241
|
+
yanked = release.get('yanked')
|
|
242
|
+
suffix = f' (yanked: {yanked})' if isinstance(yanked, str) and yanked else ' (yanked)' if yanked else ''
|
|
243
|
+
# A file whose name declares no version still lists, grouped under a null version by the API.
|
|
244
|
+
version = release.get('version') or '(unversioned)'
|
|
245
|
+
click.echo(f' {version}{suffix}')
|
|
246
|
+
if verbose:
|
|
247
|
+
for entry in release.get('files', []):
|
|
248
|
+
click.echo(f' {entry["filename"]} {_human_bytes(entry["size"])} {entry["sha256"]}')
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
@package_group.command(name='list')
|
|
252
|
+
@_HOST_OPTION
|
|
253
|
+
@click.option('-b', '--bucket', default=None, help='One bucket; every bucket you can see if omitted.')
|
|
254
|
+
@click.option('-r', '--repo', 'repo_path', default=None, help='One repo inside the bucket; needs --bucket.')
|
|
255
|
+
@click.option('-v', '--verbose', is_flag=True, help='Also list every file with its size and sha256.')
|
|
256
|
+
@click.argument('name', required=False)
|
|
257
|
+
def package_list_command(
|
|
258
|
+
host: str | None, bucket: str | None, repo_path: str | None, verbose: bool, name: str | None
|
|
259
|
+
) -> None:
|
|
260
|
+
"""List published packages: gb package list [-b BUCKET] [-r REPO] [NAME]."""
|
|
261
|
+
from . import package as package_module, repo as repo_module
|
|
262
|
+
|
|
263
|
+
if repo_path and not bucket:
|
|
264
|
+
raise click.UsageError('--repo needs --bucket: packages live in a bucket, and repo paths are not unique.')
|
|
265
|
+
try:
|
|
266
|
+
domain, entry = repo_module.select_host(host)
|
|
267
|
+
labels = [bucket] if bucket else package_module.buckets(domain, entry)
|
|
268
|
+
listings = [(label, package_module.listing(domain, entry, label, repo_path, name)) for label in labels]
|
|
269
|
+
except Exception as x:
|
|
270
|
+
raise click.ClickException(str(x)) from None
|
|
271
|
+
if not listings:
|
|
272
|
+
click.echo('No buckets. Run: gb login <deployment-domain>')
|
|
273
|
+
return
|
|
274
|
+
for label, packages in listings:
|
|
275
|
+
click.echo(label)
|
|
276
|
+
if not packages:
|
|
277
|
+
click.echo(' (no packages)')
|
|
278
|
+
for project in sorted(packages, key=lambda item: (item['repo'], item['name'])):
|
|
279
|
+
click.echo(f' {project["repo"]} {project["name"]}')
|
|
280
|
+
_print_versions(project.get('versions', []), verbose)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
@package_group.command(name='yank')
|
|
284
|
+
@_HOST_OPTION
|
|
285
|
+
@click.option('--reason', default=None, help='Why it was yanked; resolvers and gb package list show it.')
|
|
286
|
+
@click.argument('repo_name', metavar='BUCKET/REPO')
|
|
287
|
+
@click.argument('package', metavar='NAME')
|
|
288
|
+
@click.argument('version')
|
|
289
|
+
def package_yank_command(repo_name: str, package: str, version: str, host: str | None, reason: str | None) -> None:
|
|
290
|
+
"""Yank a release.
|
|
291
|
+
|
|
292
|
+
The files stay downloadable, so anything that already pinned the version keeps resolving; every other
|
|
293
|
+
resolve skips it.
|
|
294
|
+
"""
|
|
295
|
+
from . import package as package_module, repo as repo_module
|
|
296
|
+
|
|
297
|
+
try:
|
|
298
|
+
bucket, path = repo_module.parse_name(repo_name)
|
|
299
|
+
domain, entry = repo_module.select_host(host)
|
|
300
|
+
result = package_module.yank(domain, entry, bucket, path, package, version, reason)
|
|
301
|
+
except Exception as x:
|
|
302
|
+
raise click.ClickException(str(x)) from None
|
|
303
|
+
click.echo(f'Yanked {package} {version} in {repo_name}: {len(result.get("files", []))} file(s).')
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
@package_group.command(name='unyank')
|
|
307
|
+
@_HOST_OPTION
|
|
308
|
+
@click.argument('repo_name', metavar='BUCKET/REPO')
|
|
309
|
+
@click.argument('package', metavar='NAME')
|
|
310
|
+
@click.argument('version')
|
|
311
|
+
def package_unyank_command(repo_name: str, package: str, version: str, host: str | None) -> None:
|
|
312
|
+
"""Put a yanked release back."""
|
|
313
|
+
from . import package as package_module, repo as repo_module
|
|
314
|
+
|
|
315
|
+
try:
|
|
316
|
+
bucket, path = repo_module.parse_name(repo_name)
|
|
317
|
+
domain, entry = repo_module.select_host(host)
|
|
318
|
+
result = package_module.unyank(domain, entry, bucket, path, package, version)
|
|
319
|
+
except Exception as x:
|
|
320
|
+
raise click.ClickException(str(x)) from None
|
|
321
|
+
click.echo(f'Unyanked {package} {version} in {repo_name}: {len(result.get("files", []))} file(s).')
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
@package_group.command(name='delete')
|
|
325
|
+
@_HOST_OPTION
|
|
326
|
+
@click.option('--yes', is_flag=True, help='Actually delete; without it, nothing is touched.')
|
|
327
|
+
@click.argument('repo_name', metavar='BUCKET/REPO')
|
|
328
|
+
@click.argument('package', metavar='NAME')
|
|
329
|
+
@click.argument('version')
|
|
330
|
+
def package_delete_command(repo_name: str, package: str, version: str, host: str | None, yes: bool) -> None:
|
|
331
|
+
"""Delete a release's files."""
|
|
332
|
+
from . import package as package_module, repo as repo_module
|
|
333
|
+
|
|
334
|
+
try:
|
|
335
|
+
bucket, path = repo_module.parse_name(repo_name)
|
|
336
|
+
domain, entry = repo_module.select_host(host)
|
|
337
|
+
except Exception as x:
|
|
338
|
+
raise click.ClickException(str(x)) from None
|
|
339
|
+
if not yes:
|
|
340
|
+
click.echo(f'This deletes {package} {version} from {repo_name}. Lockfiles that pin it will 404.')
|
|
341
|
+
click.echo('Prefer `gb package yank` unless the bytes must not exist.')
|
|
342
|
+
raise click.ClickException('refusing to delete without confirmation; re-run with --yes')
|
|
343
|
+
try:
|
|
344
|
+
result = package_module.delete(domain, entry, bucket, path, package, version)
|
|
345
|
+
except Exception as x:
|
|
346
|
+
raise click.ClickException(str(x)) from None
|
|
347
|
+
deleted = result.get('deleted', [])
|
|
348
|
+
click.echo(f'Deleted {package} {version} from {repo_name}: {len(deleted)} file(s).')
|
|
349
|
+
for filename in deleted:
|
|
350
|
+
click.echo(f' {filename}')
|
|
351
|
+
|
|
352
|
+
|
|
234
353
|
@cli.command(name='credential', hidden=True)
|
|
235
354
|
@click.argument('operation')
|
|
236
355
|
def credential_command(operation: str) -> None:
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
Deliberately importless beyond the client's own small modules: this runs on every git operation against the
|
|
4
4
|
deployment. A warm session token is one cached-file read; a live MicroVM behind it is not something the cache
|
|
5
5
|
can promise, which is what ``wake.probe`` is for -- one cheap authenticated request, silent unless the bucket
|
|
6
|
-
turns out to be cold
|
|
6
|
+
turns out to be cold, and at most once per host per minute so a burst of fills costs one probe.
|
|
7
|
+
``GB_NO_WAKE_PROBE=1`` drops back to the old zero-network warm path.
|
|
7
8
|
"""
|
|
8
9
|
|
|
9
10
|
import sys
|
|
@@ -41,7 +42,7 @@ def get(fields: dict[str, str]) -> str | None:
|
|
|
41
42
|
token = session.token(domain, entry)
|
|
42
43
|
# The bucket's own host, not the deployment domain the token was minted against: that is what git is
|
|
43
44
|
# about to talk to, and the only host the launcher's cold path answers for.
|
|
44
|
-
wake.probe(fields.get('host', domain).partition(':')[0], token)
|
|
45
|
+
wake.probe(fields.get('host', domain).partition(':')[0], token, domain=domain)
|
|
45
46
|
return f'username={_USERNAME}\npassword={token}\n'
|
|
46
47
|
|
|
47
48
|
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""``gb package``: list, yank/unyank and delete releases against a deployment's package API.
|
|
2
|
+
|
|
3
|
+
Click-free like :mod:`repo`, and deliberately built on its helpers: host selection, the session cookie and the
|
|
4
|
+
error mapping are the same ones ``/api/repo/*`` uses, so :class:`~.repo.RepoError` is the error type here too.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
from typing import Any
|
|
9
|
+
from urllib.parse import urlencode
|
|
10
|
+
|
|
11
|
+
from . import net
|
|
12
|
+
from .hosts import Entry
|
|
13
|
+
from .repo import RepoError, _headers, _raise_mapped, list_repos
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def normalize(name: str) -> str:
|
|
17
|
+
"""The PEP 503 normalized project name, which is what the index is keyed by."""
|
|
18
|
+
return re.sub(r'[-_.]+', '-', name).lower()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def buckets(domain: str, entry: Entry) -> list[str]:
|
|
22
|
+
"""Every bucket label the token can see. ``/api/packages`` is per bucket, so listing them all starts here."""
|
|
23
|
+
listing = list_repos(domain, entry)
|
|
24
|
+
return sorted({bucket['bucket'] for bucket in listing.get('buckets', [])})
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def listing(domain: str, entry: Entry, bucket: str, repo: str | None = None, name: str | None = None) -> list[Any]:
|
|
28
|
+
query: dict[str, str] = {'bucket': bucket}
|
|
29
|
+
if repo:
|
|
30
|
+
query['repo'] = repo
|
|
31
|
+
if name:
|
|
32
|
+
query['name'] = normalize(name)
|
|
33
|
+
url = f'https://{domain}/api/packages?{urlencode(query)}'
|
|
34
|
+
try:
|
|
35
|
+
result = net.get_json(url, headers=_headers(domain, entry))
|
|
36
|
+
except net.HttpError as x:
|
|
37
|
+
if x.status == 404:
|
|
38
|
+
raise RepoError(f'{bucket} is not a bucket on {domain}') from None
|
|
39
|
+
if x.status == 403:
|
|
40
|
+
raise RepoError(f'not entitled to list packages in {bucket}') from None
|
|
41
|
+
_raise_mapped(domain, x)
|
|
42
|
+
raise
|
|
43
|
+
return result.get('packages', [])
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _call(
|
|
47
|
+
operation: str, domain: str, entry: Entry, bucket: str, repo: str, name: str, version: str, **extra: str
|
|
48
|
+
) -> dict[str, Any]:
|
|
49
|
+
payload: dict[str, Any] = {'bucket': bucket, 'repo': repo, 'name': normalize(name), 'version': version, **extra}
|
|
50
|
+
try:
|
|
51
|
+
return net.post_json(f'https://{domain}/api/package/{operation}', payload, headers=_headers(domain, entry))
|
|
52
|
+
except net.HttpError as x:
|
|
53
|
+
if x.status == 403:
|
|
54
|
+
raise RepoError(f'not entitled to {operation} packages in {bucket}/{repo}') from None
|
|
55
|
+
if x.status == 404:
|
|
56
|
+
state = 'is not yanked in' if operation == 'unyank' else 'not found in'
|
|
57
|
+
raise RepoError(f'{name} {version} {state} {bucket}/{repo} on {domain}') from None
|
|
58
|
+
_raise_mapped(domain, x)
|
|
59
|
+
raise
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def yank(
|
|
63
|
+
domain: str, entry: Entry, bucket: str, repo: str, name: str, version: str, reason: str | None = None
|
|
64
|
+
) -> dict[str, Any]:
|
|
65
|
+
extra = {'reason': reason} if reason else {}
|
|
66
|
+
return _call('yank', domain, entry, bucket, repo, name, version, **extra)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def unyank(domain: str, entry: Entry, bucket: str, repo: str, name: str, version: str) -> dict[str, Any]:
|
|
70
|
+
return _call('unyank', domain, entry, bucket, repo, name, version)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def delete(domain: str, entry: Entry, bucket: str, repo: str, name: str, version: str) -> dict[str, Any]:
|
|
74
|
+
return _call('delete', domain, entry, bucket, repo, name, version)
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
"""The cold-start wake probe: one cheap authenticated GET, run once before ``get`` hands git a credential.
|
|
2
|
+
|
|
3
|
+
Warm is the common case and it has to cost exactly what it costs today: one request, answered fast, nothing
|
|
4
|
+
printed. Cold is the case this exists for -- a bucket with no live instance yet means the request lands on the
|
|
5
|
+
launcher, which holds the connection for up to its own hold (tens of seconds) before it either answers or
|
|
6
|
+
307s to the same URL with a growing hop count. A plain request with a long timeout would sit there silently
|
|
7
|
+
for that whole stretch, and a user watching a hung ``git fetch`` has no way to tell a cold start from a stall.
|
|
8
|
+
|
|
9
|
+
So the probe is a single request carrying the whole budget, with a timer running alongside it: if the answer
|
|
10
|
+
has not landed by ``warn_after_s`` the timer prints the "waking it up" line and the request keeps going. The
|
|
11
|
+
request is never abandoned and retried -- an abandoned probe is still served, so on a cold bucket it leaves a
|
|
12
|
+
launcher holding the launch lock and the retry starts a second launcher chain for the same wake. Everything
|
|
13
|
+
else -- a network error, a redirect with no ``Location``, the whole deadline elapsing -- is swallowed: this
|
|
14
|
+
fronts for a credential that has to reach git either way, so nothing here may block or fail a clone that
|
|
15
|
+
would otherwise have worked.
|
|
16
|
+
|
|
17
|
+
One 200 also resets the server's idle timer, so a probe is worth suppressing for a short while afterwards:
|
|
18
|
+
``wake-probes.json`` in the deployment's state dir records ``{host: expires_at}`` and collapses a burst of
|
|
19
|
+
credential fills (an LFS lock/verify/locks/unlock sequence, a submodule walk) into one probe.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
import os
|
|
23
|
+
import sys
|
|
24
|
+
from base64 import b64encode
|
|
25
|
+
from contextlib import suppress
|
|
26
|
+
from threading import Lock, Timer
|
|
27
|
+
from time import monotonic, time
|
|
28
|
+
from typing import Any
|
|
29
|
+
|
|
30
|
+
from . import hosts, net
|
|
31
|
+
|
|
32
|
+
HEALTH_PATH = '/_gb/health'
|
|
33
|
+
_NO_PROBE_ENV = 'GB_NO_WAKE_PROBE'
|
|
34
|
+
_TTL_ENV = 'GB_WAKE_PROBE_TTL_S'
|
|
35
|
+
_PROBE_MARKER = 'wake-probes.json'
|
|
36
|
+
_TTL_S = 60.0
|
|
37
|
+
_WARN_AFTER_S = 1.0
|
|
38
|
+
_DEADLINE_S = 90.0
|
|
39
|
+
# Mirrors the launcher's own GB_HOP_BUDGET default; the deadline is what actually stops a runaway chain.
|
|
40
|
+
_MAX_HOPS = 8
|
|
41
|
+
_REDIRECT_STATUSES = (301, 302, 303, 307, 308)
|
|
42
|
+
_WAKE_MESSAGE = 'gb: your git-bucket was asleep, waking it up...'
|
|
43
|
+
_ALREADY_AWAKE_MESSAGE = 'gb: ready (bucket was already awake; slow network)'
|
|
44
|
+
# How much of the server's uptime can predate this probe and still count as "it came up for us".
|
|
45
|
+
_UPTIME_SLACK_S = 2.0
|
|
46
|
+
_BOOT_LABELS = {'launch': 'cold launch', 'resume': 'resumed'}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def probe(
|
|
50
|
+
host: str,
|
|
51
|
+
token: str,
|
|
52
|
+
*,
|
|
53
|
+
domain: str | None = None,
|
|
54
|
+
transport: Any = None,
|
|
55
|
+
clock: Any = monotonic,
|
|
56
|
+
warn_after_s: float = _WARN_AFTER_S,
|
|
57
|
+
deadline_s: float = _DEADLINE_S,
|
|
58
|
+
max_hops: int = _MAX_HOPS,
|
|
59
|
+
stderr: Any = None,
|
|
60
|
+
alarm: Any = None,
|
|
61
|
+
) -> None:
|
|
62
|
+
"""Warm ``host``'s bucket with ``token`` before the caller prints a credential back to git.
|
|
63
|
+
|
|
64
|
+
Skippable with ``GB_NO_WAKE_PROBE=1``, and suppressed for ``GB_WAKE_PROBE_TTL_S`` seconds (default 60,
|
|
65
|
+
0 disables suppression) after a 200 for the same host under ``domain``. Every exception is caught here,
|
|
66
|
+
not just the network ones: a bug in this module must never turn into a broken ``git fetch``. ``transport``,
|
|
67
|
+
``stderr`` and ``alarm`` all default to ``None`` and are resolved inside the call rather than bound as
|
|
68
|
+
defaults, so a caller that patches ``net.request_raw`` or swaps ``sys.stderr`` (as the test suite does with
|
|
69
|
+
capsys) is not shadowed by a reference captured at import time.
|
|
70
|
+
"""
|
|
71
|
+
if os.environ.get(_NO_PROBE_ENV) == '1':
|
|
72
|
+
return
|
|
73
|
+
ttl_s = _TTL_S
|
|
74
|
+
suppressed = False
|
|
75
|
+
# A state dir that cannot even be named is not a reason to skip the probe: anything raising here means
|
|
76
|
+
# "not suppressed", and the probe runs as if no marker had ever been written.
|
|
77
|
+
with suppress(Exception):
|
|
78
|
+
ttl_s = _ttl_s()
|
|
79
|
+
suppressed = bool(domain) and ttl_s > 0 and _suppressed(str(domain), host)
|
|
80
|
+
if suppressed:
|
|
81
|
+
return
|
|
82
|
+
answered = False
|
|
83
|
+
with suppress(Exception):
|
|
84
|
+
answered = _probe(
|
|
85
|
+
host,
|
|
86
|
+
token,
|
|
87
|
+
transport or net.request_raw,
|
|
88
|
+
clock,
|
|
89
|
+
warn_after_s,
|
|
90
|
+
deadline_s,
|
|
91
|
+
max_hops,
|
|
92
|
+
stderr if stderr is not None else sys.stderr,
|
|
93
|
+
alarm or _timer,
|
|
94
|
+
)
|
|
95
|
+
if answered and domain and ttl_s > 0:
|
|
96
|
+
with suppress(Exception):
|
|
97
|
+
_record(domain, host, ttl_s)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _probe(
|
|
101
|
+
host: str,
|
|
102
|
+
token: str,
|
|
103
|
+
transport: Any,
|
|
104
|
+
clock: Any,
|
|
105
|
+
warn_after_s: float,
|
|
106
|
+
deadline_s: float,
|
|
107
|
+
max_hops: int,
|
|
108
|
+
stderr: Any,
|
|
109
|
+
alarm: Any,
|
|
110
|
+
) -> bool:
|
|
111
|
+
start = clock()
|
|
112
|
+
deadline = start + deadline_s
|
|
113
|
+
url = f'https://{host}{HEALTH_PATH}'
|
|
114
|
+
request_headers = {'Authorization': _basic(token)}
|
|
115
|
+
lock = Lock()
|
|
116
|
+
state = {'warned': False, 'done': False}
|
|
117
|
+
status: int | None = None
|
|
118
|
+
response_headers: dict[str, str] = {}
|
|
119
|
+
hops = 0
|
|
120
|
+
|
|
121
|
+
def _warn() -> None:
|
|
122
|
+
# Under the same lock the main thread takes before it announces, so an answer that lands while the
|
|
123
|
+
# timer is already running never gets narrated as if it had been slow.
|
|
124
|
+
with lock:
|
|
125
|
+
if state['done']:
|
|
126
|
+
return
|
|
127
|
+
print(_WAKE_MESSAGE, file=stderr)
|
|
128
|
+
state['warned'] = True
|
|
129
|
+
|
|
130
|
+
countdown = alarm(warn_after_s, _warn)
|
|
131
|
+
countdown.start()
|
|
132
|
+
try:
|
|
133
|
+
while True:
|
|
134
|
+
remaining = deadline - clock()
|
|
135
|
+
if remaining <= 0:
|
|
136
|
+
return False
|
|
137
|
+
try:
|
|
138
|
+
status, response_headers, _ = transport(url, headers=request_headers, timeout=remaining)
|
|
139
|
+
except (net.HttpTimeout, net.HttpError):
|
|
140
|
+
return False
|
|
141
|
+
if status not in _REDIRECT_STATUSES or hops >= max_hops:
|
|
142
|
+
break
|
|
143
|
+
location = response_headers.get('Location') or response_headers.get('location')
|
|
144
|
+
if not location:
|
|
145
|
+
break
|
|
146
|
+
url = location
|
|
147
|
+
hops += 1
|
|
148
|
+
finally:
|
|
149
|
+
countdown.cancel()
|
|
150
|
+
with lock:
|
|
151
|
+
state['done'] = True
|
|
152
|
+
|
|
153
|
+
if status != 200:
|
|
154
|
+
return False
|
|
155
|
+
if state['warned']:
|
|
156
|
+
_announce(clock() - start, response_headers, stderr)
|
|
157
|
+
return True
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _timer(delay: float, callback: Any) -> Timer:
|
|
161
|
+
countdown = Timer(delay, callback)
|
|
162
|
+
countdown.daemon = True
|
|
163
|
+
return countdown
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _announce(elapsed: float, response_headers: dict[str, str], stderr: Any) -> None:
|
|
167
|
+
# A slow DNS lookup or a slow link trips the timer just as well as a cold bucket does, and calling that
|
|
168
|
+
# a wake would be a lie: the server's uptime says whether this process came up during the probe.
|
|
169
|
+
uptime_s = _uptime_s(response_headers)
|
|
170
|
+
if uptime_s is not None and uptime_s > elapsed + _UPTIME_SLACK_S:
|
|
171
|
+
print(_ALREADY_AWAKE_MESSAGE, file=stderr)
|
|
172
|
+
return
|
|
173
|
+
kind = response_headers.get('x-gb-boot') or response_headers.get('X-Gb-Boot') or ''
|
|
174
|
+
label = _BOOT_LABELS.get(kind)
|
|
175
|
+
suffix = f'{label}, {elapsed:.1f}s' if label else f'{elapsed:.1f}s'
|
|
176
|
+
print(f'gb: ready ({suffix})', file=stderr)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _uptime_s(response_headers: dict[str, str]) -> float | None:
|
|
180
|
+
raw = response_headers.get('x-gb-uptime-s') or response_headers.get('X-Gb-Uptime-S')
|
|
181
|
+
if raw is None:
|
|
182
|
+
return None
|
|
183
|
+
try:
|
|
184
|
+
return float(raw)
|
|
185
|
+
except (TypeError, ValueError):
|
|
186
|
+
return None
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _ttl_s() -> float:
|
|
190
|
+
raw = os.environ.get(_TTL_ENV)
|
|
191
|
+
if raw is None:
|
|
192
|
+
return _TTL_S
|
|
193
|
+
try:
|
|
194
|
+
return float(raw)
|
|
195
|
+
except ValueError:
|
|
196
|
+
return _TTL_S
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _suppressed(domain: str, host: str) -> bool:
|
|
200
|
+
expires_at = (hosts.read_cache(domain, _PROBE_MARKER) or {}).get(host)
|
|
201
|
+
return isinstance(expires_at, int | float) and expires_at > time()
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _record(domain: str, host: str, ttl_s: float) -> None:
|
|
205
|
+
now = time()
|
|
206
|
+
marker = hosts.read_cache(domain, _PROBE_MARKER) or {}
|
|
207
|
+
live = {key: value for key, value in marker.items() if isinstance(value, int | float) and value > now}
|
|
208
|
+
live[host] = now + ttl_s
|
|
209
|
+
hosts.write_cache(domain, _PROBE_MARKER, live)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _basic(token: str) -> str:
|
|
213
|
+
return 'Basic ' + b64encode(f'gb:{token}'.encode()).decode()
|
|
@@ -1,127 +0,0 @@
|
|
|
1
|
-
"""The cold-start wake probe: one cheap authenticated GET, run once before ``get`` hands git a credential.
|
|
2
|
-
|
|
3
|
-
Warm is the common case and it has to cost exactly what it costs today: one request, answered fast, nothing
|
|
4
|
-
printed. Cold is the case this exists for -- a bucket with no live instance yet means the request lands on the
|
|
5
|
-
launcher, which holds the connection for up to its own hold (tens of seconds) before it either answers or
|
|
6
|
-
307s to the same URL with a growing hop count. A plain request with a long timeout would sit there silently
|
|
7
|
-
for that whole stretch, and a user watching a hung ``git fetch`` has no way to tell a cold start from a stall.
|
|
8
|
-
|
|
9
|
-
So the probe leads with a short, cheap attempt (``warn_after_s``) whose only job is to notice slowness. A
|
|
10
|
-
timeout there is not a failure -- it is exactly what a cold bucket looks like -- and it is the one signal that
|
|
11
|
-
prints the "waking it up" line before switching to the launcher's own long-poll rhythm for the rest of the
|
|
12
|
-
budget. Everything else -- a network error, a redirect with no ``Location``, the whole deadline elapsing -- is
|
|
13
|
-
swallowed: this fronts for a credential that has to reach git either way, so nothing here may block or fail a
|
|
14
|
-
clone that would otherwise have worked.
|
|
15
|
-
"""
|
|
16
|
-
|
|
17
|
-
import os
|
|
18
|
-
import sys
|
|
19
|
-
from base64 import b64encode
|
|
20
|
-
from contextlib import suppress
|
|
21
|
-
from time import monotonic
|
|
22
|
-
from typing import Any
|
|
23
|
-
|
|
24
|
-
from . import net
|
|
25
|
-
|
|
26
|
-
HEALTH_PATH = '/_gb/health'
|
|
27
|
-
_NO_PROBE_ENV = 'GB_NO_WAKE_PROBE'
|
|
28
|
-
_WARN_AFTER_S = 1.0
|
|
29
|
-
_DEADLINE_S = 90.0
|
|
30
|
-
_MAX_HOPS = 20
|
|
31
|
-
_REDIRECT_STATUSES = (301, 302, 303, 307, 308)
|
|
32
|
-
_WAKE_MESSAGE = 'gb: your git-bucket was asleep, waking it up...'
|
|
33
|
-
_BOOT_LABELS = {'launch': 'cold launch', 'resume': 'resumed'}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def probe(
|
|
37
|
-
host: str,
|
|
38
|
-
token: str,
|
|
39
|
-
*,
|
|
40
|
-
transport: Any = None,
|
|
41
|
-
clock: Any = monotonic,
|
|
42
|
-
warn_after_s: float = _WARN_AFTER_S,
|
|
43
|
-
deadline_s: float = _DEADLINE_S,
|
|
44
|
-
max_hops: int = _MAX_HOPS,
|
|
45
|
-
stderr: Any = None,
|
|
46
|
-
) -> None:
|
|
47
|
-
"""Warm ``host``'s bucket with ``token`` before the caller prints a credential back to git.
|
|
48
|
-
|
|
49
|
-
Skippable with ``GB_NO_WAKE_PROBE=1``. Every exception is caught here, not just the network ones: a bug
|
|
50
|
-
in this module must never turn into a broken ``git fetch``. ``transport`` and ``stderr`` both default to
|
|
51
|
-
``None`` and are resolved inside the call rather than bound as defaults, so a caller that patches
|
|
52
|
-
``net.request_raw`` or swaps ``sys.stderr`` (as the test suite does with capsys) is not shadowed by a
|
|
53
|
-
reference captured at import time.
|
|
54
|
-
"""
|
|
55
|
-
if os.environ.get(_NO_PROBE_ENV) == '1':
|
|
56
|
-
return
|
|
57
|
-
with suppress(Exception):
|
|
58
|
-
_probe(
|
|
59
|
-
host,
|
|
60
|
-
token,
|
|
61
|
-
transport or net.request_raw,
|
|
62
|
-
clock,
|
|
63
|
-
warn_after_s,
|
|
64
|
-
deadline_s,
|
|
65
|
-
max_hops,
|
|
66
|
-
stderr if stderr is not None else sys.stderr,
|
|
67
|
-
)
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
def _probe(
|
|
71
|
-
host: str,
|
|
72
|
-
token: str,
|
|
73
|
-
transport: Any,
|
|
74
|
-
clock: Any,
|
|
75
|
-
warn_after_s: float,
|
|
76
|
-
deadline_s: float,
|
|
77
|
-
max_hops: int,
|
|
78
|
-
stderr: Any,
|
|
79
|
-
) -> None:
|
|
80
|
-
start = clock()
|
|
81
|
-
deadline = start + deadline_s
|
|
82
|
-
url = f'https://{host}{HEALTH_PATH}'
|
|
83
|
-
request_headers = {'Authorization': _basic(token)}
|
|
84
|
-
warned = False
|
|
85
|
-
first = True
|
|
86
|
-
status: int | None = None
|
|
87
|
-
response_headers: dict[str, str] = {}
|
|
88
|
-
hops = 0
|
|
89
|
-
|
|
90
|
-
while True:
|
|
91
|
-
remaining = deadline - clock()
|
|
92
|
-
if remaining <= 0:
|
|
93
|
-
return
|
|
94
|
-
timeout = min(warn_after_s, remaining) if first else remaining
|
|
95
|
-
try:
|
|
96
|
-
status, response_headers, _ = transport(url, headers=request_headers, timeout=timeout)
|
|
97
|
-
except net.HttpTimeout:
|
|
98
|
-
if not first:
|
|
99
|
-
return
|
|
100
|
-
print(_WAKE_MESSAGE, file=stderr)
|
|
101
|
-
warned = True
|
|
102
|
-
first = False
|
|
103
|
-
continue
|
|
104
|
-
except net.HttpError:
|
|
105
|
-
return
|
|
106
|
-
first = False
|
|
107
|
-
if status not in _REDIRECT_STATUSES or hops >= max_hops:
|
|
108
|
-
break
|
|
109
|
-
location = response_headers.get('Location') or response_headers.get('location')
|
|
110
|
-
if not location:
|
|
111
|
-
break
|
|
112
|
-
url = location
|
|
113
|
-
hops += 1
|
|
114
|
-
|
|
115
|
-
if warned and status == 200:
|
|
116
|
-
_announce(clock() - start, response_headers, stderr)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
def _announce(elapsed: float, response_headers: dict[str, str], stderr: Any) -> None:
|
|
120
|
-
kind = response_headers.get('x-gb-boot') or response_headers.get('X-Gb-Boot') or ''
|
|
121
|
-
label = _BOOT_LABELS.get(kind)
|
|
122
|
-
suffix = f'{label}, {elapsed:.1f}s' if label else f'{elapsed:.1f}s'
|
|
123
|
-
print(f'gb: ready ({suffix})', file=stderr)
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
def _basic(token: str) -> str:
|
|
127
|
-
return 'Basic ' + b64encode(f'gb:{token}'.encode()).decode()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|