libre-devops-helpers 0.4.1__py3-none-any.whl

libre_devops_helpers/core/http.py

@@ -0,0 +1,412 @@
+"""HTTP client for Microsoft APIs: bearer auth, bounded retries, Retry-After, paging.
+
+Built on requests. Retries are decided on the HTTP status code (408, 429, 5xx) and on
+connection errors or timeouts, never on the wording of an error message. The bearer
+token is only ever sent to the client's own https host, and redirects are not followed.
+Every POST this package makes is a read-only query or a token request, so retrying
+one is safe. A 401 is retried once with a fresh token, when the token source can drop
+the one it cached (``core.auth.BearerToken``).
+"""
+
+from __future__ import annotations
+
+import email.utils
+import ipaddress
+import json
+import logging
+import random
+import time
+from collections.abc import Callable, Iterator, Mapping
+from datetime import UTC, datetime
+from typing import Any, Self
+from urllib.parse import quote, urlencode, urlsplit
+
+import requests
+
+from libre_devops_helpers import __version__
+from libre_devops_helpers.core import brand
+from libre_devops_helpers.core.errors import ApiError
+
+log = logging.getLogger(__name__)
+
+RETRY_STATUSES = frozenset({408, 429, 500, 502, 503, 504})
+USER_AGENT = f"{brand.COMMAND}/{__version__}"
+
+
+class ApiClient:
+    """JSON client for one API base URL, usually authenticated with a bearer token.
+
+    ``token`` is called before every request, so a caching provider can refresh a token
+    close to expiry. If it also has a ``refresh()`` method, a 401 answer drops the cached
+    token and the request is sent once more with a new one: that covers a token revoked
+    or expired early. Tokens go in the Authorization header only and are never logged.
+    With ``token=None`` no Authorization header is sent; only then may ``allow_http``
+    permit a plain http base URL on a loopback or link-local host, which is what the
+    managed identity endpoints use.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        token: Callable[[], str] | None,
+        *,
+        name: str = "API",
+        session: requests.Session | None = None,
+        verify: bool | str = True,
+        timeout: float = 30.0,
+        max_attempts: int = 4,
+        backoff: float = 1.0,
+        max_backoff: float = 30.0,
+        max_retry_after: float = 120.0,
+        sleep: Callable[[float], None] = time.sleep,
+        allow_http: bool = False,
+        auth_scheme: str = "Bearer",
+    ) -> None:
+        parts = urlsplit(base_url)
+        if not parts.netloc or parts.scheme not in {"https", "http"}:
+            raise ValueError(f"base_url must be an https URL, got {base_url!r}")
+        if parts.scheme == "http" and not (
+            allow_http and token is None and _is_local(parts.hostname or "")
+        ):
+            raise ValueError(f"base_url must be an https URL, got {base_url!r}")
+        if max_attempts < 1:
+            raise ValueError("max_attempts must be at least 1")
+        self.name = name
+        self.base_url = base_url.rstrip("/")
+        self._scheme = parts.scheme
+        self._host = parts.netloc.lower()
+        self._token = token
+        # "Bearer" for tokens; "Basic" when ``token`` returns base64 "user:password".
+        self._auth_scheme = auth_scheme
+        refresh = getattr(token, "refresh", None)
+        self._refresh_token: Callable[[], None] | None = refresh if callable(refresh) else None
+        self._session = session or requests.Session()
+        self._owns_session = session is None
+        self._verify = verify
+        self._timeout = timeout
+        self._max_attempts = max_attempts
+        self._backoff = backoff
+        self._max_backoff = max_backoff
+        self._max_retry_after = max_retry_after
+        self._sleep = sleep
+
+    def ensure_token(self) -> None:
+        """Get the token now, in this thread.
+
+        Call it on the main thread before fanning requests out to workers, so that a
+        credential which has to ask someone to sign in again asks there.
+        """
+        if self._token is not None:
+            self._token()
+
+    def close(self) -> None:
+        """Close the underlying session if this client created it."""
+        if self._owns_session:
+            self._session.close()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.close()
+
+    def url(self, path: str, params: Mapping[str, str] | None = None) -> str:
+        """Absolute URL for ``path`` with ``params`` percent-encoded.
+
+        ``path`` may be a full URL (an ``@odata.nextLink``); it must use this client's
+        scheme and host, so a token never travels anywhere else.
+        """
+        if "://" in path:
+            parts = urlsplit(path)
+            if parts.scheme != self._scheme or parts.netloc.lower() != self._host:
+                raise ApiError(
+                    f"{self.name}: refusing to send a token to {parts.scheme}://{parts.netloc}"
+                )
+            url = path
+        else:
+            url = f"{self.base_url}/{path.lstrip('/')}"
+        if params:
+            url += ("&" if "?" in url else "?") + urlencode(params, quote_via=quote)
+        return url
+
+    def request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: Mapping[str, str] | None = None,
+        headers: Mapping[str, str] | None = None,
+        json_body: Any = None,
+        form: Mapping[str, str] | None = None,
+        allow_empty: bool = False,
+    ) -> dict[str, Any]:
+        """Send one request and return the JSON object it responds with.
+
+        ``allow_empty`` accepts a success with no body (some actions answer 200 or 204
+        with nothing), returning ``{}`` for it.
+        """
+        url = self.url(path, params)
+        response = self._send(method, url, headers, json_body=json_body, form=form)
+        if allow_empty and not response.content.strip():
+            return {}
+        try:
+            body = response.json()
+        except ValueError:
+            raise ApiError(
+                f"{self.name}: {method} {_path(url)} did not return JSON",
+                status=response.status_code,
+            ) from None
+        if not isinstance(body, dict):
+            raise ApiError(f"{self.name}: {method} {_path(url)} did not return a JSON object")
+        return body
+
+    def get(
+        self,
+        path: str,
+        *,
+        params: Mapping[str, str] | None = None,
+        headers: Mapping[str, str] | None = None,
+    ) -> dict[str, Any]:
+        """GET ``path`` and return the JSON object it responds with."""
+        return self.request("GET", path, params=params, headers=headers)
+
+    def get_text(
+        self,
+        path: str,
+        *,
+        params: Mapping[str, str] | None = None,
+        headers: Mapping[str, str] | None = None,
+    ) -> str:
+        """GET ``path`` and return its body as text, for the few APIs that answer in text."""
+        response = self._send("GET", self.url(path, params), headers)
+        return response.text
+
+    def post(
+        self,
+        path: str,
+        body: Any,
+        *,
+        params: Mapping[str, str] | None = None,
+        headers: Mapping[str, str] | None = None,
+    ) -> dict[str, Any]:
+        """POST ``body`` as JSON to ``path`` and return the JSON object it responds with."""
+        return self.request("POST", path, params=params, headers=headers, json_body=body)
+
+    def get_all(
+        self,
+        path: str,
+        *,
+        params: Mapping[str, str] | None = None,
+        headers: Mapping[str, str] | None = None,
+        next_link: str = "@odata.nextLink",
+    ) -> Iterator[dict[str, Any]]:
+        """Yield every item of a paged collection, following ``next_link``.
+
+        Graph and Defender use ``@odata.nextLink``; Azure Resource Manager and Key Vault
+        use ``nextLink``. Pages are fetched only as items are consumed.
+        """
+        page = self.get(path, params=params, headers=headers)
+        while True:
+            items = page.get("value")
+            if not isinstance(items, list):
+                raise ApiError(f"{self.name}: response has no 'value' array")
+            yield from (item for item in items if isinstance(item, dict))
+            link = page.get(next_link)
+            if not isinstance(link, str) or not link:
+                return
+            page = self.get(link, headers=headers)
+
+    def _send(
+        self,
+        method: str,
+        url: str,
+        headers: Mapping[str, str] | None,
+        *,
+        json_body: Any = None,
+        form: Mapping[str, str] | None = None,
+    ) -> requests.Response:
+        attempt = 0
+        refreshed = False
+        while True:
+            attempt += 1
+            request_headers = {"Accept": "application/json", "User-Agent": USER_AGENT}
+            if self._token is not None:
+                request_headers["Authorization"] = f"{self._auth_scheme} {self._token()}"
+            request_headers.update(headers or {})
+            try:
+                response = self._session.request(
+                    method,
+                    url,
+                    headers=request_headers,
+                    json=json_body,
+                    data=form,
+                    timeout=self._timeout,
+                    verify=self._verify,
+                    allow_redirects=False,
+                )
+            except (requests.ConnectionError, requests.Timeout) as exc:
+                if attempt >= self._max_attempts:
+                    raise ApiError(
+                        f"{self.name}: {method} {_path(url)} failed after {attempt} attempts: "
+                        f"{type(exc).__name__}"
+                    ) from None
+                self._wait(attempt, None, type(exc).__name__)
+                continue
+            except requests.RequestException as exc:
+                raise ApiError(f"{self.name}: {method} {_path(url)} failed: {exc}") from None
+
+            if 200 <= response.status_code < 300:
+                return response
+            if response.status_code == 401 and self._refresh_token and not refreshed:
+                # Once only: a second 401 means the token is not the problem.
+                refreshed = True
+                attempt -= 1
+                log.info("%s: HTTP 401, retrying once with a new token", self.name)
+                self._refresh_token()
+                continue
+            if response.status_code in RETRY_STATUSES and attempt < self._max_attempts:
+                self._wait(attempt, retry_after_seconds(response), f"HTTP {response.status_code}")
+                continue
+            raise error_from_response(self.name, response)
+
+    def _wait(self, attempt: int, retry_after: float | None, reason: str) -> None:
+        # A server-directed Retry-After wins over the backoff (retrying earlier just
+        # throttles again), capped so a broken server cannot stall the run.
+        if retry_after is not None:
+            delay = min(self._max_retry_after, retry_after)
+        else:
+            delay = min(self._max_backoff, self._backoff * 2 ** (attempt - 1))
+            delay += random.uniform(0, self._backoff / 2)
+        log.warning(
+            "%s: %s on attempt %d of %d, retrying in %.1fs",
+            self.name,
+            reason,
+            attempt,
+            self._max_attempts,
+            delay,
+        )
+        self._sleep(delay)
+
+
+def retry_after_seconds(response: requests.Response) -> float | None:
+    """Seconds requested by a Retry-After header (delta-seconds or HTTP date), else None."""
+    value = response.headers.get("Retry-After", "").strip()
+    if not value:
+        return None
+    if value.isdigit():
+        return float(value)
+    try:
+        when = email.utils.parsedate_to_datetime(value)
+    except (TypeError, ValueError):
+        return None
+    if when.tzinfo is None:
+        when = when.replace(tzinfo=UTC)
+    return max(0.0, (when - datetime.now(UTC)).total_seconds())
+
+
+def error_from_response(name: str, response: requests.Response) -> ApiError:
+    """Build an ApiError from a failed response, using the service's error body when present.
+
+    Graph, Defender, ARM and Key Vault reply ``{"error": {"code", "message"}}``; the Entra
+    token endpoint replies ``{"error": "<code>", "error_description": "..."}``.
+    """
+    code: str | None = None
+    message: str | None = None
+    request_id: str | None = None
+    try:
+        body = response.json()
+    except ValueError:
+        body = None
+    if isinstance(body, dict) and isinstance(body.get("error"), dict):
+        error = body["error"]
+        code = error.get("code") if isinstance(error.get("code"), str) else None
+        message = error.get("message") if isinstance(error.get("message"), str) else None
+        inner = error.get("innerError")
+        if isinstance(inner, dict) and isinstance(inner.get("request-id"), str):
+            request_id = inner["request-id"]
+    elif isinstance(body, dict) and isinstance(body.get("error"), str):
+        code = body["error"]
+        description = body.get("error_description")
+        if isinstance(description, str) and description.strip():
+            # The first line carries the AADSTS code and the reason; the rest is trace ids.
+            message = description.strip().splitlines()[0]
+    request_id = (
+        request_id or response.headers.get("request-id") or response.headers.get("x-ms-request-id")
+    )
+
+    status = response.status_code
+    detail = _readable(message) if message else ""
+    text = f"{name}: HTTP {status}"
+    if code:
+        text += f" {code}"
+    text += f": {detail or response.reason or 'request failed'}"
+    if request_id:
+        text += f" (request-id {request_id})"
+    challenge = response.headers.get("WWW-Authenticate", "").lower()
+    if status == 401 and "insufficient_claims" in challenge:
+        # Continuous access evaluation: the token was revoked, or a policy changed.
+        hint: str | None = (
+            "the service revoked the token or wants a sign-in that meets a Conditional "
+            "Access policy (a claims challenge); sign in again"
+        )
+    else:
+        hint = _hint(status, text.lower())
+    return ApiError(text, status=status, code=code, request_id=request_id, hint=hint)
+
+
+def _readable(message: str) -> str:
+    """One readable line from a service error message.
+
+    Some services put a JSON document inside the message (Graph's PIM errors, Intune,
+    which nests two deep); those are unwrapped to their inner code and message. Line
+    breaks are flattened, and the result is capped so one error cannot flood a terminal.
+    """
+    codes: list[str] = []
+    text = message.strip()
+    for _ in range(3):
+        if not text.startswith("{"):
+            break
+        try:
+            inner = json.loads(text)
+        except ValueError:
+            break
+        if not isinstance(inner, dict):
+            break
+        code = inner.get("errorCode") or inner.get("ErrorCode") or inner.get("code")
+        if isinstance(code, str) and code:
+            codes.append(code)
+        found = inner.get("message") or inner.get("Message")
+        if not isinstance(found, str):
+            text = ""
+            break
+        text = found.strip()
+    flat = " ".join(text.split())
+    readable = ": ".join([*codes, flat] if flat else codes)
+    return readable if len(readable) <= 400 else readable[:397] + "..."
+
+
+def _hint(status: int, text: str) -> str | None:
+    if status == 403 and "suspended" in text:
+        return "the service is suspended in this tenant, usually because its licence or trial ended"
+    if status == 403 and "client address is not authorized" in text:
+        return "the resource's firewall does not allow this machine's IP address"
+    if status in {401, 403} and ("forbidden" in text or "permission" in text):
+        return "the token was accepted but lacks the permission this call needs"
+    return {
+        401: "the API rejected the token; check its audience, tenant and expiry",
+        403: "the signed-in identity lacks a role or permission for this call",
+    }.get(status)
+
+
+def _is_local(host: str) -> bool:
+    if host == "localhost":
+        return True
+    try:
+        address = ipaddress.ip_address(host)
+    except ValueError:
+        return False
+    return address.is_loopback or address.is_link_local
+
+
+def _path(url: str) -> str:
+    return urlsplit(url).path

libre_devops_helpers/core/inputs.py

@@ -0,0 +1,193 @@
+"""Read a list of names from arguments, stdin, a text file, a CSV or an Excel workbook.
+
+Commands that take devices, users or vaults accept them however they are to hand:
+``"a,b,c"``, several arguments, ``-`` for stdin, or a file. A text file holds names
+separated by commas, spaces or new lines, with ``#`` comments. A CSV file (a ``.csv``
+suffix, or any file when a column is named) and an Excel workbook (``.xlsx``, ``.xlsm``,
+``.xltx``, ``.xltm``) are read by column header, so a plan, an export from a portal or a
+spreadsheet someone emailed works as it is, title rows above the header and all.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import logging
+from collections.abc import Iterable, Sequence
+from itertools import islice
+from pathlib import Path
+from typing import TextIO
+
+from libre_devops_helpers.core import sheets
+from libre_devops_helpers.core.errors import InputError
+from libre_devops_helpers.core.util import split_names
+
+log = logging.getLogger(__name__)
+
+# With a column named, the header is the first of this many non-blank rows to have it.
+HEADER_SEARCH_ROWS = 25
+
+# Cells of one row, and whether the row is hidden (in Excel; a CSV row never is).
+Row = tuple[Sequence[str], bool]
+
+
+def read_names(
+    values: Sequence[str] = (),
+    *,
+    stdin: TextIO | None = None,
+    from_file: Path | None = None,
+    column: str | None = None,
+    sheet: str | None = None,
+) -> list[str]:
+    """Every name given, in order, with blanks and case-insensitive repeats dropped.
+
+    ``-`` among ``values`` reads ``stdin`` in its place. ``column`` picks a column (by
+    header, case-insensitively) of a CSV or workbook ``from_file``, or of CSV on stdin
+    when there is no file. ``sheet`` picks a workbook's sheet by tab name; without it,
+    the one visible sheet with that column is used, or the first visible sheet when no
+    column is named.
+    """
+    if sheet is not None and (from_file is None or not sheets.is_workbook(from_file)):
+        raise InputError("--sheet applies to an Excel workbook only")
+    collected: list[str] = []
+    for value in values:
+        if value == "-":
+            if stdin is None:
+                raise InputError("'-' reads names from stdin, but there is no stdin")
+            text = stdin.read()
+            collected.extend(_from_csv(text, column, "stdin") if column else _from_text(text))
+        else:
+            collected.extend(split_names([value]))
+    if from_file is not None:
+        collected.extend(_from_file(from_file, column, sheet))
+    return _dedupe(collected)
+
+
+def _from_file(path: Path, column: str | None, sheet: str | None) -> list[str]:
+    sheets.check_readable(path)
+    if sheets.is_workbook(path):
+        return _from_workbook(path, column, sheet)
+    try:
+        text = path.read_text(encoding="utf-8-sig")
+    except UnicodeDecodeError:
+        raise InputError(
+            f"{path} is not a text file", hint="use a text file, a CSV or an Excel workbook"
+        ) from None
+    except OSError as exc:
+        raise InputError(f"cannot read {path}: {exc}") from None
+    if column or path.suffix.lower() == ".csv":
+        return _from_csv(text, column, str(path))
+    return _from_text(text)
+
+
+def _from_text(text: str) -> list[str]:
+    return split_names(line.split("#", 1)[0] for line in text.splitlines())
+
+
+def _dedupe(names: list[str]) -> list[str]:
+    seen: set[str] = set()
+    kept: list[str] = []
+    for name in names:
+        if name.casefold() not in seen:
+            seen.add(name.casefold())
+            kept.append(name)
+    return kept
+
+
+def _from_csv(text: str, column: str | None, source: str) -> list[str]:
+    return _column(((cells, False) for cells in csv.reader(io.StringIO(text))), column, source)
+
+
+def _from_workbook(path: Path, column: str | None, sheet: str | None) -> list[str]:
+    with sheets.open_workbook(path) as book:
+        chosen = book.sheet(sheet) if sheet is not None else _pick_sheet(book, column)
+        return _column(book.rows(chosen), column, f"sheet {chosen.name!r} of {path}")
+
+
+def _pick_sheet(book: sheets.Workbook, column: str | None) -> sheets.Sheet:
+    """The one visible sheet with ``column`` in its header, or the first visible sheet."""
+    visible = [sheet for sheet in book.sheets if not sheet.hidden]
+    if not visible:
+        raise InputError(
+            f"{book.path} has only hidden sheets",
+            hint=f"name one with --sheet ({', '.join(s.name for s in book.sheets)})",
+        )
+    if column is None:
+        return visible[0]
+    having = [sheet for sheet in visible if _has_column(book.rows(sheet), column)]
+    if len(having) == 1:
+        return having[0]
+    names = ", ".join(sheet.name for sheet in having or visible)
+    if having:
+        raise InputError(
+            f"several sheets of {book.path} have a column {column!r}",
+            hint=f"pick one with --sheet ({names})",
+        )
+    raise InputError(
+        f"no sheet of {book.path} has a column {column!r}",
+        hint=f"check the header, or pick a sheet with --sheet ({names})",
+    )
+
+
+def _has_column(rows: Iterable[Row], column: str) -> bool:
+    try:
+        _header(iter(rows), column, "")
+    except InputError:
+        return False
+    return True
+
+
+def _column(rows: Iterable[Row], column: str | None, source: str) -> list[str]:
+    """The non-blank values under the header ``column``, or under the only header."""
+    remaining = iter(rows)
+    index = _header(remaining, column, source)
+    values: list[str] = []
+    hidden = 0
+    for cells, row_hidden in remaining:
+        # Cells are kept whole, so a column may hold names with spaces in them.
+        value = cells[index].strip() if index < len(cells) else ""
+        if value:
+            values.append(value)
+            hidden += row_hidden
+    if hidden:
+        log.warning(
+            "%d of the names in %s are in rows hidden or filtered out in Excel; they are included",
+            hidden,
+            source,
+        )
+    return values
+
+
+def _header(rows: Iterable[Row], column: str | None, source: str) -> int:
+    """Find the header row, consuming rows up to it, and return the column's index.
+
+    Without ``column``, the header is the first non-blank row, and it must name one
+    column only. With it, the header is the first of the first few non-blank rows to
+    have that column, so title rows above the header do no harm.
+    """
+    wanted = column.strip().casefold() if column else None
+    first: list[str] | None = None
+    for row, _ in islice(_non_blank(rows), HEADER_SEARCH_ROWS):
+        cells = [cell.strip() for cell in row]
+        first = first or cells
+        if wanted is None:
+            named = [index for index, cell in enumerate(cells) if cell]
+            if len(named) == 1:
+                return named[0]
+            raise InputError(
+                f"{source} has several columns",
+                hint=f"pick one with --column ({', '.join(cell for cell in cells if cell)})",
+            )
+        for index, cell in enumerate(cells):
+            if cell.casefold() == wanted:
+                return index
+    if first is None:
+        raise InputError(f"{source} has no header row")
+    raise InputError(
+        f"{source} has no column {column!r}",
+        hint=f"columns: {', '.join(cell for cell in first if cell)}",
+    )
+
+
+def _non_blank(rows: Iterable[Row]) -> Iterable[Row]:
+    return (row for row in rows if any(cell.strip() for cell in row[0]))