git-buckets 0.3.0__tar.gz → 0.3.1__tar.gz

{git_buckets-0.3.0 → git_buckets-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: git-buckets
-Version: 0.3.0
+Version: 0.3.1
 Summary: Git over S3: clone, fetch and push repositories backed by an S3 bucket.
 Author: Full Duplex Media
 Author-email: Full Duplex Media <contact@fullduplex.media>

{git_buckets-0.3.0 → git_buckets-0.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "git-buckets"
-version = "0.3.0"
+version = "0.3.1"
 description = "Git over S3: clone, fetch and push repositories backed by an S3 bucket."
 readme = "README.md"
 requires-python = ">=3.14"

{git_buckets-0.3.0 → git_buckets-0.3.1}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "git-buckets"
-version = "0.3.0"
+version = "0.3.1"
 description = "Git over S3: clone, fetch and push repositories backed by an S3 bucket."
 authors = [
   { name = 'Full Duplex Media', email = 'contact@fullduplex.media' }

{git_buckets-0.3.0 → git_buckets-0.3.1}/src/git_buckets/credential.py

@@ -3,7 +3,8 @@
 Deliberately importless beyond the client's own small modules: this runs on every git operation against the
 deployment. A warm session token is one cached-file read; a live MicroVM behind it is not something the cache
 can promise, which is what ``wake.probe`` is for -- one cheap authenticated request, silent unless the bucket
-turns out to be cold. ``GB_NO_WAKE_PROBE=1`` drops back to the old zero-network warm path.
+turns out to be cold, and at most once per host per minute so a burst of fills costs one probe.
+``GB_NO_WAKE_PROBE=1`` drops back to the old zero-network warm path.
 """
 
 import sys
@@ -41,7 +42,7 @@ def get(fields: dict[str, str]) -> str | None:
     token = session.token(domain, entry)
     # The bucket's own host, not the deployment domain the token was minted against: that is what git is
     # about to talk to, and the only host the launcher's cold path answers for.
-    wake.probe(fields.get('host', domain).partition(':')[0], token)
+    wake.probe(fields.get('host', domain).partition(':')[0], token, domain=domain)
     return f'username={_USERNAME}\npassword={token}\n'
 
 

git_buckets-0.3.1/src/git_buckets/wake.py

@@ -0,0 +1,213 @@
+"""The cold-start wake probe: one cheap authenticated GET, run once before ``get`` hands git a credential.
+
+Warm is the common case and it has to cost exactly what it costs today: one request, answered fast, nothing
+printed. Cold is the case this exists for -- a bucket with no live instance yet means the request lands on the
+launcher, which holds the connection for up to its own hold (tens of seconds) before it either answers or
+307s to the same URL with a growing hop count. A plain request with a long timeout would sit there silently
+for that whole stretch, and a user watching a hung ``git fetch`` has no way to tell a cold start from a stall.
+
+So the probe is a single request carrying the whole budget, with a timer running alongside it: if the answer
+has not landed by ``warn_after_s`` the timer prints the "waking it up" line and the request keeps going. The
+request is never abandoned and retried -- an abandoned probe is still served, so on a cold bucket it leaves a
+launcher holding the launch lock and the retry starts a second launcher chain for the same wake. Everything
+else -- a network error, a redirect with no ``Location``, the whole deadline elapsing -- is swallowed: this
+fronts for a credential that has to reach git either way, so nothing here may block or fail a clone that
+would otherwise have worked.
+
+One 200 also resets the server's idle timer, so a probe is worth suppressing for a short while afterwards:
+``wake-probes.json`` in the deployment's state dir records ``{host: expires_at}`` and collapses a burst of
+credential fills (an LFS lock/verify/locks/unlock sequence, a submodule walk) into one probe.
+"""
+
+import os
+import sys
+from base64 import b64encode
+from contextlib import suppress
+from threading import Lock, Timer
+from time import monotonic, time
+from typing import Any
+
+from . import hosts, net
+
+HEALTH_PATH = '/_gb/health'
+_NO_PROBE_ENV = 'GB_NO_WAKE_PROBE'
+_TTL_ENV = 'GB_WAKE_PROBE_TTL_S'
+_PROBE_MARKER = 'wake-probes.json'
+_TTL_S = 60.0
+_WARN_AFTER_S = 1.0
+_DEADLINE_S = 90.0
+# Mirrors the launcher's own GB_HOP_BUDGET default; the deadline is what actually stops a runaway chain.
+_MAX_HOPS = 8
+_REDIRECT_STATUSES = (301, 302, 303, 307, 308)
+_WAKE_MESSAGE = 'gb: your git-bucket was asleep, waking it up...'
+_ALREADY_AWAKE_MESSAGE = 'gb: ready (bucket was already awake; slow network)'
+# How much of the server's uptime can predate this probe and still count as "it came up for us".
+_UPTIME_SLACK_S = 2.0
+_BOOT_LABELS = {'launch': 'cold launch', 'resume': 'resumed'}
+
+
+def probe(
+    host: str,
+    token: str,
+    *,
+    domain: str | None = None,
+    transport: Any = None,
+    clock: Any = monotonic,
+    warn_after_s: float = _WARN_AFTER_S,
+    deadline_s: float = _DEADLINE_S,
+    max_hops: int = _MAX_HOPS,
+    stderr: Any = None,
+    alarm: Any = None,
+) -> None:
+    """Warm ``host``'s bucket with ``token`` before the caller prints a credential back to git.
+
+    Skippable with ``GB_NO_WAKE_PROBE=1``, and suppressed for ``GB_WAKE_PROBE_TTL_S`` seconds (default 60,
+    0 disables suppression) after a 200 for the same host under ``domain``. Every exception is caught here,
+    not just the network ones: a bug in this module must never turn into a broken ``git fetch``. ``transport``,
+    ``stderr`` and ``alarm`` all default to ``None`` and are resolved inside the call rather than bound as
+    defaults, so a caller that patches ``net.request_raw`` or swaps ``sys.stderr`` (as the test suite does with
+    capsys) is not shadowed by a reference captured at import time.
+    """
+    if os.environ.get(_NO_PROBE_ENV) == '1':
+        return
+    ttl_s = _TTL_S
+    suppressed = False
+    # A state dir that cannot even be named is not a reason to skip the probe: anything raising here means
+    # "not suppressed", and the probe runs as if no marker had ever been written.
+    with suppress(Exception):
+        ttl_s = _ttl_s()
+        suppressed = bool(domain) and ttl_s > 0 and _suppressed(str(domain), host)
+    if suppressed:
+        return
+    answered = False
+    with suppress(Exception):
+        answered = _probe(
+            host,
+            token,
+            transport or net.request_raw,
+            clock,
+            warn_after_s,
+            deadline_s,
+            max_hops,
+            stderr if stderr is not None else sys.stderr,
+            alarm or _timer,
+        )
+    if answered and domain and ttl_s > 0:
+        with suppress(Exception):
+            _record(domain, host, ttl_s)
+
+
+def _probe(
+    host: str,
+    token: str,
+    transport: Any,
+    clock: Any,
+    warn_after_s: float,
+    deadline_s: float,
+    max_hops: int,
+    stderr: Any,
+    alarm: Any,
+) -> bool:
+    start = clock()
+    deadline = start + deadline_s
+    url = f'https://{host}{HEALTH_PATH}'
+    request_headers = {'Authorization': _basic(token)}
+    lock = Lock()
+    state = {'warned': False, 'done': False}
+    status: int | None = None
+    response_headers: dict[str, str] = {}
+    hops = 0
+
+    def _warn() -> None:
+        # Under the same lock the main thread takes before it announces, so an answer that lands while the
+        # timer is already running never gets narrated as if it had been slow.
+        with lock:
+            if state['done']:
+                return
+            print(_WAKE_MESSAGE, file=stderr)
+            state['warned'] = True
+
+    countdown = alarm(warn_after_s, _warn)
+    countdown.start()
+    try:
+        while True:
+            remaining = deadline - clock()
+            if remaining <= 0:
+                return False
+            try:
+                status, response_headers, _ = transport(url, headers=request_headers, timeout=remaining)
+            except (net.HttpTimeout, net.HttpError):
+                return False
+            if status not in _REDIRECT_STATUSES or hops >= max_hops:
+                break
+            location = response_headers.get('Location') or response_headers.get('location')
+            if not location:
+                break
+            url = location
+            hops += 1
+    finally:
+        countdown.cancel()
+        with lock:
+            state['done'] = True
+
+    if status != 200:
+        return False
+    if state['warned']:
+        _announce(clock() - start, response_headers, stderr)
+    return True
+
+
+def _timer(delay: float, callback: Any) -> Timer:
+    countdown = Timer(delay, callback)
+    countdown.daemon = True
+    return countdown
+
+
+def _announce(elapsed: float, response_headers: dict[str, str], stderr: Any) -> None:
+    # A slow DNS lookup or a slow link trips the timer just as well as a cold bucket does, and calling that
+    # a wake would be a lie: the server's uptime says whether this process came up during the probe.
+    uptime_s = _uptime_s(response_headers)
+    if uptime_s is not None and uptime_s > elapsed + _UPTIME_SLACK_S:
+        print(_ALREADY_AWAKE_MESSAGE, file=stderr)
+        return
+    kind = response_headers.get('x-gb-boot') or response_headers.get('X-Gb-Boot') or ''
+    label = _BOOT_LABELS.get(kind)
+    suffix = f'{label}, {elapsed:.1f}s' if label else f'{elapsed:.1f}s'
+    print(f'gb: ready ({suffix})', file=stderr)
+
+
+def _uptime_s(response_headers: dict[str, str]) -> float | None:
+    raw = response_headers.get('x-gb-uptime-s') or response_headers.get('X-Gb-Uptime-S')
+    if raw is None:
+        return None
+    try:
+        return float(raw)
+    except (TypeError, ValueError):
+        return None
+
+
+def _ttl_s() -> float:
+    raw = os.environ.get(_TTL_ENV)
+    if raw is None:
+        return _TTL_S
+    try:
+        return float(raw)
+    except ValueError:
+        return _TTL_S
+
+
+def _suppressed(domain: str, host: str) -> bool:
+    expires_at = (hosts.read_cache(domain, _PROBE_MARKER) or {}).get(host)
+    return isinstance(expires_at, int | float) and expires_at > time()
+
+
+def _record(domain: str, host: str, ttl_s: float) -> None:
+    now = time()
+    marker = hosts.read_cache(domain, _PROBE_MARKER) or {}
+    live = {key: value for key, value in marker.items() if isinstance(value, int | float) and value > now}
+    live[host] = now + ttl_s
+    hosts.write_cache(domain, _PROBE_MARKER, live)
+
+
+def _basic(token: str) -> str:
+    return 'Basic ' + b64encode(f'gb:{token}'.encode()).decode()

git_buckets-0.3.0/src/git_buckets/wake.py

@@ -1,127 +0,0 @@
-"""The cold-start wake probe: one cheap authenticated GET, run once before ``get`` hands git a credential.
-
-Warm is the common case and it has to cost exactly what it costs today: one request, answered fast, nothing
-printed. Cold is the case this exists for -- a bucket with no live instance yet means the request lands on the
-launcher, which holds the connection for up to its own hold (tens of seconds) before it either answers or
-307s to the same URL with a growing hop count. A plain request with a long timeout would sit there silently
-for that whole stretch, and a user watching a hung ``git fetch`` has no way to tell a cold start from a stall.
-
-So the probe leads with a short, cheap attempt (``warn_after_s``) whose only job is to notice slowness. A
-timeout there is not a failure -- it is exactly what a cold bucket looks like -- and it is the one signal that
-prints the "waking it up" line before switching to the launcher's own long-poll rhythm for the rest of the
-budget. Everything else -- a network error, a redirect with no ``Location``, the whole deadline elapsing -- is
-swallowed: this fronts for a credential that has to reach git either way, so nothing here may block or fail a
-clone that would otherwise have worked.
-"""
-
-import os
-import sys
-from base64 import b64encode
-from contextlib import suppress
-from time import monotonic
-from typing import Any
-
-from . import net
-
-HEALTH_PATH = '/_gb/health'
-_NO_PROBE_ENV = 'GB_NO_WAKE_PROBE'
-_WARN_AFTER_S = 1.0
-_DEADLINE_S = 90.0
-_MAX_HOPS = 20
-_REDIRECT_STATUSES = (301, 302, 303, 307, 308)
-_WAKE_MESSAGE = 'gb: your git-bucket was asleep, waking it up...'
-_BOOT_LABELS = {'launch': 'cold launch', 'resume': 'resumed'}
-
-
-def probe(
-    host: str,
-    token: str,
-    *,
-    transport: Any = None,
-    clock: Any = monotonic,
-    warn_after_s: float = _WARN_AFTER_S,
-    deadline_s: float = _DEADLINE_S,
-    max_hops: int = _MAX_HOPS,
-    stderr: Any = None,
-) -> None:
-    """Warm ``host``'s bucket with ``token`` before the caller prints a credential back to git.
-
-    Skippable with ``GB_NO_WAKE_PROBE=1``. Every exception is caught here, not just the network ones: a bug
-    in this module must never turn into a broken ``git fetch``. ``transport`` and ``stderr`` both default to
-    ``None`` and are resolved inside the call rather than bound as defaults, so a caller that patches
-    ``net.request_raw`` or swaps ``sys.stderr`` (as the test suite does with capsys) is not shadowed by a
-    reference captured at import time.
-    """
-    if os.environ.get(_NO_PROBE_ENV) == '1':
-        return
-    with suppress(Exception):
-        _probe(
-            host,
-            token,
-            transport or net.request_raw,
-            clock,
-            warn_after_s,
-            deadline_s,
-            max_hops,
-            stderr if stderr is not None else sys.stderr,
-        )
-
-
-def _probe(
-    host: str,
-    token: str,
-    transport: Any,
-    clock: Any,
-    warn_after_s: float,
-    deadline_s: float,
-    max_hops: int,
-    stderr: Any,
-) -> None:
-    start = clock()
-    deadline = start + deadline_s
-    url = f'https://{host}{HEALTH_PATH}'
-    request_headers = {'Authorization': _basic(token)}
-    warned = False
-    first = True
-    status: int | None = None
-    response_headers: dict[str, str] = {}
-    hops = 0
-
-    while True:
-        remaining = deadline - clock()
-        if remaining <= 0:
-            return
-        timeout = min(warn_after_s, remaining) if first else remaining
-        try:
-            status, response_headers, _ = transport(url, headers=request_headers, timeout=timeout)
-        except net.HttpTimeout:
-            if not first:
-                return
-            print(_WAKE_MESSAGE, file=stderr)
-            warned = True
-            first = False
-            continue
-        except net.HttpError:
-            return
-        first = False
-        if status not in _REDIRECT_STATUSES or hops >= max_hops:
-            break
-        location = response_headers.get('Location') or response_headers.get('location')
-        if not location:
-            break
-        url = location
-        hops += 1
-
-    if warned and status == 200:
-        _announce(clock() - start, response_headers, stderr)
-
-
-def _announce(elapsed: float, response_headers: dict[str, str], stderr: Any) -> None:
-    kind = response_headers.get('x-gb-boot') or response_headers.get('X-Gb-Boot') or ''
-    label = _BOOT_LABELS.get(kind)
-    suffix = f'{label}, {elapsed:.1f}s' if label else f'{elapsed:.1f}s'
-    print(f'gb: ready ({suffix})', file=stderr)
-
-
-def _basic(token: str) -> str:
-    return 'Basic ' + b64encode(f'gb:{token}'.encode()).decode()