glab-cli-extras 0.1.0__py3-none-any.whl

glab_cli_extras/gitlab/budget.py

@@ -0,0 +1,135 @@
+"""Per-host request budget, paced on GitLab's `RateLimit-*` response headers.
+
+GitLab reports `RateLimit-Remaining` and `RateLimit-Reset` for the real throttle period, while `RateLimit-Limit` is
+normalized to one minute. The budget therefore paces on Remaining over the time left until Reset, and uses Limit only
+as a per-minute ceiling. The daemon takes `share` of that allowance and leaves the rest to the user's other tools.
+
+A host that sends no rate-limit headers (the throttle is opt-in on self-managed instances) gets a fixed cap.
+
+Time is passed in by the caller (wall-clock seconds, since `RateLimit-Reset` is a Unix timestamp), which keeps the
+class deterministic under test.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+DEFAULT_SHARE = 0.5
+DEFAULT_FIXED_PER_MINUTE = 60.0
+DEFAULT_BURST = 10.0
+NOT_MODIFIED_COST = 0.1
+"""An ETag 304 was free in live tests, but that is undocumented, so it still costs a little."""
+MIN_BACKOFF = 5.0
+MAX_BACKOFF = 300.0
+
+
+def _int_header(headers: Mapping[str, str], name: str) -> int | None:
+    value = headers.get(name)
+    if value is None:
+        return None
+    try:
+        return int(value.strip())
+    except ValueError:
+        return None
+
+
+@dataclass
+class HostBudget:
+    share: float = DEFAULT_SHARE
+    fixed_per_minute: float = DEFAULT_FIXED_PER_MINUTE
+    burst: float = DEFAULT_BURST
+    tokens: float = DEFAULT_BURST
+    last_refill: float | None = None
+    remaining: int | None = None
+    reset_at: float | None = None
+    limit_per_minute: int | None = None
+    blocked_until: float = 0.0
+    backoff: float = MIN_BACKOFF
+    rate_limited_count: int = 0
+    _saw_headers: bool = field(default=False, repr=False)
+
+    def rate(self, now: float) -> float:
+        """Requests per second the daemon may spend on this host right now."""
+        if self.remaining is not None and self.reset_at is not None and self.reset_at > now:
+            rate = self.remaining * self.share / max(1.0, self.reset_at - now)
+            if self.limit_per_minute:
+                rate = min(rate, self.limit_per_minute * self.share / 60.0)
+            return max(rate, 0.01)
+        if self.limit_per_minute:
+            return self.limit_per_minute * self.share / 60.0
+        return self.fixed_per_minute / 60.0
+
+    def _refill(self, now: float) -> None:
+        if self.last_refill is None:
+            self.last_refill = now
+            return
+        elapsed = max(0.0, now - self.last_refill)
+        self.tokens = min(self.burst, self.tokens + elapsed * self.rate(now))
+        self.last_refill = now
+
+    def can_spend(self, cost: float, now: float) -> bool:
+        self._refill(now)
+        return now >= self.blocked_until and self.tokens >= min(cost, self.burst)
+
+    def wait_time(self, cost: float, now: float) -> float:
+        """Seconds until `can_spend(cost)` could become true."""
+        self._refill(now)
+        blocked = max(0.0, self.blocked_until - now)
+        missing = max(0.0, min(cost, self.burst) - self.tokens)
+        return max(blocked, missing / self.rate(now))
+
+    def charge(self, cost: float, now: float) -> None:
+        self._refill(now)
+        self.tokens -= cost
+
+    def observe(self, headers: Mapping[str, str], now: float) -> None:
+        """Update from response headers (lowercase names). A response without rate-limit headers changes nothing."""
+        remaining = _int_header(headers, "ratelimit-remaining")
+        reset = _int_header(headers, "ratelimit-reset")
+        limit = _int_header(headers, "ratelimit-limit")
+        if remaining is None and reset is None and limit is None:
+            return
+        self._saw_headers = True
+        if remaining is not None:
+            self.remaining = remaining
+        if reset is not None:
+            self.reset_at = float(reset)
+        if limit is not None:
+            self.limit_per_minute = limit
+
+    def on_success(self) -> None:
+        self.backoff = MIN_BACKOFF
+
+    def on_rate_limited(self, retry_after: float | None, now: float) -> None:
+        """Block the whole host: honor Retry-After, and back off exponentially on repeated 429s."""
+        wait = max(retry_after or 0.0, self.backoff)
+        self.blocked_until = max(self.blocked_until, now + wait)
+        self.backoff = min(self.backoff * 2, MAX_BACKOFF)
+        self.rate_limited_count += 1
+        self.tokens = min(self.tokens, 0.0)
+
+    def stretch_factor(self, demand_per_second: float, now: float) -> float:
+        """How much every poll interval on this host must grow so the total demand fits the allowance."""
+        rate = self.rate(now)
+        if demand_per_second <= rate:
+            return 1.0
+        return demand_per_second / rate
+
+    @property
+    def premium_level(self) -> bool:
+        """Whether the per-minute limit looks like a paid gitlab.com tier or a generous self-managed throttle."""
+        return (self.limit_per_minute or 0) >= 400
+
+    def describe(self, now: float) -> dict[str, Any]:
+        return {
+            "rate_per_minute": round(self.rate(now) * 60, 1),
+            "tokens": round(self.tokens, 2),
+            "remaining": self.remaining,
+            "reset_in_s": round(self.reset_at - now) if self.reset_at else None,
+            "limit_per_minute": self.limit_per_minute,
+            "blocked_for_s": round(max(0.0, self.blocked_until - now), 1),
+            "rate_limited_count": self.rate_limited_count,
+            "headers_seen": self._saw_headers,
+        }

glab_cli_extras/gitlab/client.py

@@ -0,0 +1,225 @@
+"""GitLab REST and GraphQL access for one host, on top of `glab api -i`.
+
+Every request charges the host budget and feeds its rate-limit headers back into it. A 429 blocks the whole host
+for its Retry-After. Errors are mapped to a small exception hierarchy the pollers and the daemon understand.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections import Counter
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import quote, urlencode
+
+from glab_cli_extras.gitlab.budget import NOT_MODIFIED_COST, HostBudget
+from glab_cli_extras.gitlab.glab import (
+    DEFAULT_TIMEOUT,
+    GlabRunner,
+    HttpResponse,
+    build_api_args,
+    parse_include_output,
+    run_glab,
+    stderr_message,
+)
+
+MAX_CONCURRENT_REQUESTS = 4
+ETAG_RESOURCE_HEADER = "X-GITLAB-GRAPHQL-RESOURCE-ETAG"
+
+
+class ApiError(Exception):
+    def __init__(self, message: str, status: int | None = None) -> None:
+        super().__init__(message)
+        self.status = status
+
+
+class NotFound(ApiError):
+    pass
+
+
+class Unauthorized(ApiError):
+    """401 or 403. glab owns authentication, so the fix is `glab auth login`, never a retry here."""
+
+
+class RateLimited(ApiError):
+    def __init__(self, message: str, retry_after: float | None) -> None:
+        super().__init__(message, 429)
+        self.retry_after = retry_after
+
+
+class TransportError(ApiError):
+    """No HTTP response: network failure, timeout, or glab failed before sending the request."""
+
+
+@dataclass(frozen=True, slots=True)
+class GraphQLResult:
+    not_modified: bool
+    data: Any
+    etag: str | None
+
+
+@dataclass
+class RequestStats:
+    requests: int = 0
+    not_modified: int = 0
+    statuses: Counter[int] = field(default_factory=Counter)
+    transport_errors: int = 0
+    by_kind: Counter[str] = field(default_factory=Counter)
+
+    def describe(self) -> dict[str, Any]:
+        return {
+            "requests": self.requests,
+            "not_modified": self.not_modified,
+            "statuses": {str(k): v for k, v in sorted(self.statuses.items())},
+            "transport_errors": self.transport_errors,
+            "by_kind": dict(sorted(self.by_kind.items())),
+        }
+
+
+def encode_project(project: str) -> str:
+    return quote(project, safe="")
+
+
+def _error_message(response: HttpResponse) -> str:
+    try:
+        payload = response.json()
+    except ValueError:
+        payload = None
+    if isinstance(payload, dict):
+        for key in ("message", "error", "error_description"):
+            if key in payload:
+                return str(payload[key])
+    return f"HTTP {response.status} {response.reason}".strip()
+
+
+class GitLabClient:
+    def __init__(
+        self,
+        host: str,
+        *,
+        runner: GlabRunner = run_glab,
+        budget: HostBudget | None = None,
+        clock: Callable[[], float] = time.time,
+        timeout: float = DEFAULT_TIMEOUT,
+    ) -> None:
+        self.host = host
+        self.budget = budget if budget is not None else HostBudget()
+        self.stats = RequestStats()
+        self._runner = runner
+        self._clock = clock
+        self._timeout = timeout
+        self._semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
+        self._version: tuple[int, int] | None = None
+        self._version_checked = False
+
+    async def _request(self, args: list[str], kind: str) -> HttpResponse:
+        async with self._semaphore:
+            self.stats.requests += 1
+            self.stats.by_kind[kind] += 1
+            try:
+                result = await self._runner(args, self._timeout)
+            except TimeoutError as exc:
+                self.stats.transport_errors += 1
+                self.budget.charge(1.0, self._clock())
+                raise TransportError(f"glab did not finish within {self._timeout:.0f} s") from exc
+        now = self._clock()
+        response = parse_include_output(result.stdout)
+        if response is None:
+            self.stats.transport_errors += 1
+            self.budget.charge(1.0, now)
+            raise TransportError(stderr_message(result))
+        self.stats.statuses[response.status] += 1
+        self.budget.observe(response.headers, now)
+        if response.status == 304:
+            self.stats.not_modified += 1
+            self.budget.charge(NOT_MODIFIED_COST, now)
+            return response
+        self.budget.charge(1.0, now)
+        if response.status == 429:
+            retry_after = response.header("retry-after")
+            seconds = float(retry_after) if retry_after and retry_after.strip().isdigit() else None
+            self.budget.on_rate_limited(seconds, now)
+            raise RateLimited(f"GitLab rate limit hit on {self.host}", seconds)
+        if response.status == 404:
+            raise NotFound(_error_message(response), 404)
+        if response.status in (401, 403):
+            raise Unauthorized(
+                f"{_error_message(response)} (HTTP {response.status}); check `glab auth status --hostname {self.host}`",
+                response.status,
+            )
+        if response.status >= 400:
+            raise ApiError(_error_message(response), response.status)
+        self.budget.on_success()
+        return response
+
+    async def get(
+        self, path: str, params: Mapping[str, str | int] | None = None, *, kind: str = "rest"
+    ) -> HttpResponse:
+        endpoint = path
+        if params:
+            endpoint += ("&" if "?" in path else "?") + urlencode(list(params.items()), safe="[]")
+        return await self._request(build_api_args(self.host, endpoint), kind)
+
+    async def get_json(self, path: str, params: Mapping[str, str | int] | None = None, *, kind: str = "rest") -> Any:
+        response = await self.get(path, params, kind=kind)
+        try:
+            return response.json()
+        except ValueError as exc:
+            raise ApiError(f"GitLab returned invalid JSON for {path}: {exc}", response.status) from exc
+
+    async def graphql(
+        self,
+        query: str,
+        variables: Mapping[str, str | int | bool],
+        *,
+        etag_resource: str | None = None,
+        if_none_match: str | None = None,
+        kind: str = "graphql",
+    ) -> GraphQLResult:
+        """Run a GraphQL query, optionally through GitLab's ETag cache.
+
+        With `etag_resource` (for example `pipelines/id/123`) and `if_none_match`, an unchanged resource returns
+        a 304 from GitLab's cache before the query runs, and the result has `not_modified=True` and no data.
+        """
+        headers: dict[str, str] = {}
+        if etag_resource is not None:
+            headers[ETAG_RESOURCE_HEADER] = f"/api/graphql:{etag_resource}"
+            if if_none_match:
+                headers["If-None-Match"] = if_none_match
+        strings = {k: v for k, v in variables.items() if isinstance(v, str)}
+        typed = {k: v for k, v in variables.items() if not isinstance(v, str)}
+        args = build_api_args(
+            self.host, "graphql", headers=headers, fields={"query": query, **strings}, typed_fields=typed
+        )
+        response = await self._request(args, kind + ("-etag" if etag_resource else ""))
+        etag = response.header("etag")
+        if response.status == 304:
+            return GraphQLResult(True, None, etag or if_none_match)
+        try:
+            payload = response.json()
+        except ValueError as exc:
+            raise ApiError(f"GitLab returned invalid GraphQL JSON: {exc}", response.status) from exc
+        if not isinstance(payload, dict):
+            raise ApiError("GitLab returned an unexpected GraphQL response", response.status)
+        errors = payload.get("errors")
+        if errors:
+            messages = "; ".join(str(e.get("message", e)) if isinstance(e, dict) else str(e) for e in errors)
+            raise ApiError(f"GraphQL error: {messages}", response.status)
+        return GraphQLResult(False, payload.get("data"), etag)
+
+    async def version(self) -> tuple[int, int] | None:
+        """The GitLab version (major, minor), fetched once per client. None when the server does not tell."""
+        if self._version_checked:
+            return self._version
+        self._version_checked = True
+        try:
+            payload = await self.get_json("version", kind="version")
+        except NotFound:
+            return None
+        raw = str(payload.get("version", "")) if isinstance(payload, dict) else ""
+        parts = raw.split(".")
+        if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit():
+            self._version = (int(parts[0]), int(parts[1]))
+        return self._version

glab_cli_extras/gitlab/glab.py

@@ -0,0 +1,175 @@
+"""Run `glab api -i` as a subprocess and parse its output.
+
+glab owns authentication, hosts, the keyring, OAuth refresh, proxies and certificates. This module never sees a token:
+it only builds the `glab api` argument list and reads back what glab prints.
+
+With `-i`, glab prints the HTTP status line, the response headers, a blank line and the body. It exits 1 for every
+non-2xx status, a 304 included, but still prints the status line and the headers. The status is therefore read from the
+output, never from the exit code. Output without a status line means the request never got a response (DNS, TLS,
+connection refused, unknown host); glab explains why on stderr.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import os
+import shutil
+import subprocess
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+DEFAULT_TIMEOUT = 60.0
+
+
+class GlabNotFoundError(RuntimeError):
+    """The `glab` executable is not on PATH."""
+
+
+class GlabTransportError(RuntimeError):
+    """glab produced no HTTP response: the request failed before the server answered, or glab itself failed."""
+
+
+@dataclass(frozen=True, slots=True)
+class ProcessResult:
+    returncode: int
+    stdout: bytes
+    stderr: bytes
+
+
+@dataclass(frozen=True, slots=True)
+class HttpResponse:
+    status: int
+    reason: str
+    headers: Mapping[str, str]
+    """Header names are lowercase. A repeated header keeps its last value."""
+    body: bytes
+
+    def header(self, name: str) -> str | None:
+        return self.headers.get(name.lower())
+
+    def json(self) -> Any:
+        if not self.body.strip():
+            return None
+        return json.loads(self.body)
+
+    def text(self) -> str:
+        return self.body.decode("utf-8", errors="replace")
+
+
+class GlabRunner(Protocol):
+    """Run glab with the given arguments (without the `glab` executable itself)."""
+
+    async def __call__(self, args: Sequence[str], timeout: float) -> ProcessResult: ...
+
+
+def glab_executable() -> str:
+    path = shutil.which("glab")
+    if path is None:
+        raise GlabNotFoundError("The glab CLI is not on PATH. Install it from https://gitlab.com/gitlab-org/cli.")
+    return path
+
+
+def _child_env() -> dict[str, str]:
+    env = dict(os.environ)
+    env["NO_COLOR"] = "1"
+    return env
+
+
+async def run_glab(args: Sequence[str], timeout: float) -> ProcessResult:
+    """The default runner: no shell, stdin from devnull, a hard timeout."""
+    process = await asyncio.create_subprocess_exec(
+        glab_executable(),
+        *args,
+        stdin=asyncio.subprocess.DEVNULL,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+        env=_child_env(),
+    )
+    try:
+        stdout, stderr = await asyncio.wait_for(process.communicate(), timeout)
+    except (TimeoutError, asyncio.CancelledError):
+        with contextlib.suppress(ProcessLookupError):
+            process.kill()
+        await process.wait()
+        raise
+    return ProcessResult(process.returncode if process.returncode is not None else -1, stdout, stderr)
+
+
+def run_glab_sync(args: Sequence[str], timeout: float = DEFAULT_TIMEOUT) -> ProcessResult:
+    """The blocking variant, for the CLI, which resolves refs before any event loop runs."""
+    completed = subprocess.run(
+        [glab_executable(), *args],
+        stdin=subprocess.DEVNULL,
+        capture_output=True,
+        timeout=timeout,
+        env=_child_env(),
+        check=False,
+    )
+    return ProcessResult(completed.returncode, completed.stdout, completed.stderr)
+
+
+def build_api_args(
+    host: str,
+    endpoint: str,
+    *,
+    method: str | None = None,
+    headers: Mapping[str, str] | None = None,
+    fields: Mapping[str, str] | None = None,
+    typed_fields: Mapping[str, str | int | bool] | None = None,
+) -> list[str]:
+    """Build the argument list for `glab api -i`.
+
+    `fields` become `-f name=value` (always strings); `typed_fields` become `-F name=value`, which glab converts to
+    JSON numbers and booleans. For `graphql`, glab sends `query` as the query and every other field as a variable.
+    """
+    args = ["api", "--hostname", host, "-i"]
+    if method is not None:
+        args += ["-X", method]
+    for name, value in (headers or {}).items():
+        args += ["-H", f"{name}: {value}"]
+    for name, value in (fields or {}).items():
+        args += ["-f", f"{name}={value}"]
+    for name, typed in (typed_fields or {}).items():
+        rendered = ("true" if typed else "false") if isinstance(typed, bool) else str(typed)
+        args += ["-F", f"{name}={rendered}"]
+    args.append(endpoint)
+    return args
+
+
+def parse_include_output(stdout: bytes) -> HttpResponse | None:
+    """Parse the output of `glab api -i`. Return None when it holds no HTTP status line."""
+    first_end = stdout.find(b"\n")
+    if first_end < 0:
+        return None
+    status_line = stdout[:first_end].rstrip(b"\r").decode("latin-1")
+    parts = status_line.split(" ", 2)
+    if len(parts) < 2 or not parts[0].startswith("HTTP/") or not parts[1].isdigit():
+        return None
+    status = int(parts[1])
+    reason = parts[2] if len(parts) > 2 else ""
+
+    headers: dict[str, str] = {}
+    position = first_end + 1
+    while position <= len(stdout):
+        line_end = stdout.find(b"\n", position)
+        if line_end < 0:
+            line_end = len(stdout)
+        line = stdout[position:line_end].rstrip(b"\r")
+        position = line_end + 1
+        if not line:
+            break
+        name, sep, value = line.decode("latin-1").partition(":")
+        if sep:
+            headers[name.strip().lower()] = value.strip()
+    body = stdout[position:] if position <= len(stdout) else b""
+    return HttpResponse(status, reason, headers, body)
+
+
+def stderr_message(result: ProcessResult) -> str:
+    """The useful part of glab's stderr: its box-drawing decoration and blank lines removed."""
+    lines = [line.strip() for line in result.stderr.decode("utf-8", errors="replace").splitlines()]
+    text = " ".join(line for line in lines if line and line != "ERROR")
+    return text or f"glab exited with code {result.returncode}"

glab_cli_extras/gitlab/queries/__init__.py

@@ -0,0 +1,11 @@
+"""GraphQL queries shipped as package data."""
+
+from __future__ import annotations
+
+from functools import cache
+from importlib.resources import files
+
+
+@cache
+def load(name: str) -> str:
+    return files(__name__).joinpath(f"{name}.graphql").read_text(encoding="utf-8")

glab_cli_extras/gitlab/queries/branch_pipelines.graphql

@@ -0,0 +1,8 @@
+query BranchPipelines($project: ID!, $ref: String!) {
+  project(fullPath: $project) {
+    id
+    pipelines(ref: $ref, first: 10) {
+      nodes { id iid status sha path createdAt }
+    }
+  }
+}

glab_cli_extras/gitlab/queries/mr.graphql

@@ -0,0 +1,42 @@
+query MergeRequestSnapshot(
+  $project: ID!
+  $iid: String!
+  $withThreads: Boolean!
+  $withPipeline: Boolean!
+  $threadsAfter: String
+) {
+  project(fullPath: $project) {
+    mergeRequest(iid: $iid) {
+      iid
+      state
+      title
+      description
+      draft
+      detailedMergeStatus
+      diffHeadSha
+      sourceBranch
+      targetBranch
+      webUrl
+      approved
+      approvalsLeft
+      approvedBy { nodes { username } }
+      reviewers { nodes { username } }
+      headPipeline @include(if: $withPipeline) {
+        id
+        status
+        project { fullPath }
+      }
+      discussions(first: 100, after: $threadsAfter) @include(if: $withThreads) {
+        pageInfo { hasNextPage endCursor }
+        nodes {
+          id
+          resolvable
+          resolved
+          notes(first: 100) {
+            nodes { id updatedAt system author { username } }
+          }
+        }
+      }
+    }
+  }
+}

glab_cli_extras/gitlab/queries/pipeline.graphql

@@ -0,0 +1,38 @@
+query PipelineTree($project: ID!, $id: CiPipelineID!) {
+  project(fullPath: $project) {
+    pipeline(id: $id) {
+      id
+      iid
+      status
+      ref
+      sha
+      path
+      jobs(first: 100, retried: false) {
+        nodes { ...WatchedJob }
+      }
+      downstream {
+        nodes {
+          id
+          iid
+          status
+          path
+          project { fullPath }
+          jobs(first: 100, retried: false) {
+            nodes { ...WatchedJob }
+          }
+        }
+      }
+    }
+  }
+}
+
+fragment WatchedJob on CiJob {
+  id
+  name
+  status
+  stage { name }
+  allowFailure
+  startedAt
+  finishedAt
+  artifacts { nodes { fileType size } }
+}