libre-devops-helpers 0.4.1__py3-none-any.whl

libre_devops_helpers/core/log.py

@@ -0,0 +1,246 @@
+"""Logging setup for the CLI: human text, flat JSON, or the OpenTelemetry wire format.
+
+Library modules only call ``logging.getLogger(__name__)``; they never configure
+handlers. The CLI calls ``configure_logging`` once. Logs go to stderr so they never mix
+with data written to stdout.
+
+``otlp`` writes the OpenTelemetry file format: JSON Lines, each line a complete OTLP/JSON
+``LogsData`` (the same object an ``ExportLogsServiceRequest`` is) holding one record, as
+``Write-LdoLog`` does in LibreDevOpsHelpers. An OpenTelemetry Collector reads it with the
+``otlp_json_file`` receiver, or, from a container's logs, with the ``file_log`` receiver
+and the ``otlp_json`` connector, with no parsing rules either way. The two tools read the
+same variables (``LDO_LOG_FORMAT``, ``LDO_LOG_LEVEL``, ``LDO_SERVICE_NAME`` and the trace
+context), so one pipeline setting controls both. One difference is deliberate: with
+nothing set, this CLI writes readable text.
+
+https://opentelemetry.io/docs/specs/otel/protocol/file-exporter/
+https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sys
+import traceback
+from collections.abc import Mapping
+from typing import Any
+from urllib.parse import unquote
+
+from libre_devops_helpers import __version__
+from libre_devops_helpers.core import brand
+
+LOG_FORMATS = ("text", "json", "otlp")
+
+# LibreDevOpsHelpers' names for formats and levels, so its settings mean the same here.
+_FORMAT_ALIASES = {"jsonindented": "json", "otlpindented": "otlp"}
+_LEVELS = {
+    "trace": logging.DEBUG,
+    "debug": logging.DEBUG,
+    "info": logging.INFO,
+    "success": logging.INFO,
+    "warn": logging.WARNING,
+    "warning": logging.WARNING,
+    "error": logging.ERROR,
+    "fatal": logging.CRITICAL,
+}
+SERVICE_NAME = brand.COMMAND
+
+# https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber
+_SEVERITY = {
+    logging.DEBUG: 5,
+    logging.INFO: 9,
+    logging.WARNING: 13,
+    logging.ERROR: 17,
+    logging.CRITICAL: 21,
+}
+
+
+class JsonFormatter(logging.Formatter):
+    """One flat JSON object per record."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        data: dict[str, Any] = {
+            "time": self.formatTime(record, "%Y-%m-%dT%H:%M:%S") + f".{int(record.msecs):03d}",
+            "level": record.levelname.lower(),
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        if getattr(record, "hint", None):
+            data["hint"] = record.hint
+        if record.exc_info:
+            data["exception"] = self.formatException(record.exc_info)
+        return json.dumps(data)
+
+
+class OtlpFormatter(logging.Formatter):
+    """One OTLP/JSON ``LogsData`` per record, on one line.
+
+    OTLP's JSON rules differ from plain proto3 JSON: 64-bit integers such as timestamps are
+    strings, ``severityNumber`` is the integer, ``traceId`` and ``spanId`` are lowercase
+    hex, and attribute values are typed. Attribute names follow the semantic conventions.
+
+    The resource describes what is logging: ``service.name`` (``LDO_SERVICE_NAME``, else
+    ``OTEL_SERVICE_NAME``, else the command), ``service.version`` and, with
+    ``LDO_DEPLOYMENT_ENVIRONMENT``, ``deployment.environment.name``; anything else comes
+    from ``OTEL_RESOURCE_ATTRIBUTES``. ``LDO_TRACE_ID``, ``LDO_SPAN_ID`` and
+    ``LDO_CORRELATION_ID`` put every record in a trace, so a CI run's logs join up; an id
+    that is not valid hex of the right width is left out rather than sent and rejected.
+    """
+
+    def __init__(
+        self,
+        service_name: str | None = None,
+        service_version: str | None = None,
+        *,
+        environ: Mapping[str, str] | None = None,
+    ):
+        super().__init__()
+        env = os.environ if environ is None else environ
+        self._resource = {"attributes": _resource_attributes(env, service_name, service_version)}
+        correlation = (env.get(brand.env_var("CORRELATION_ID")) or "").strip()
+        self._correlation_id = correlation
+        self._trace_id = otlp_hex_id(env.get(brand.env_var("TRACE_ID")), 32) or otlp_hex_id(
+            correlation, 32
+        )
+        self._span_id = otlp_hex_id(env.get(brand.env_var("SPAN_ID")), 16)
+
+    def format(self, record: logging.LogRecord) -> str:
+        nanos = str(int(record.created * 1_000_000_000))
+        attributes = [
+            _attribute("code.function.name", f"{record.name}.{record.funcName}"),
+            _attribute("code.line.number", record.lineno),
+        ]
+        if self._correlation_id:
+            attributes.append(_attribute("correlation_id", self._correlation_id))
+        if getattr(record, "hint", None):
+            attributes.append(_attribute("hint", str(record.hint)))
+        if record.exc_info and record.exc_info[0] is not None:
+            error_type, error, trace = record.exc_info
+            attributes += [
+                _attribute("exception.type", error_type.__name__),
+                _attribute("exception.message", str(error)),
+                _attribute(
+                    "exception.stacktrace",
+                    "".join(traceback.format_exception(error_type, error, trace)),
+                ),
+            ]
+        log_record: dict[str, Any] = {
+            "timeUnixNano": nanos,
+            "observedTimeUnixNano": nanos,
+            "severityNumber": _SEVERITY.get(record.levelno, 9),
+            "severityText": record.levelname,
+            "body": {"stringValue": record.getMessage()},
+            "attributes": attributes,
+        }
+        if self._trace_id:
+            log_record["traceId"] = self._trace_id
+        if self._span_id:
+            log_record["spanId"] = self._span_id
+        request = {
+            "resourceLogs": [
+                {
+                    "resource": self._resource,
+                    "scopeLogs": [{"scope": {"name": record.name}, "logRecords": [log_record]}],
+                }
+            ]
+        }
+        return json.dumps(request, separators=(",", ":"))
+
+
+def otlp_hex_id(value: str | None, length: int) -> str:
+    """A trace (32) or span (16) id in OTLP's lowercase hex, or "" when ``value`` is not one.
+
+    Dashes are dropped, so a GUID, which is 16 bytes, becomes a trace id: a CI run's id
+    can join its logs up without reformatting. An all-zero id is invalid in W3C trace
+    context, so it is refused too. Refusing means leaving the field out: a collector
+    rejects a whole payload with a bad id, and logging must never lose the record.
+    """
+    candidate = (value or "").strip().replace("-", "").lower()
+    if len(candidate) != length or candidate.strip("0123456789abcdef") or not candidate.strip("0"):
+        return ""
+    return candidate
+
+
+def _resource_attributes(
+    environ: Mapping[str, str], service_name: str | None, service_version: str | None
+) -> list[dict[str, Any]]:
+    extra = _parse_resource_attributes(environ.get("OTEL_RESOURCE_ATTRIBUTES", ""))
+    named = extra.pop("service.name", None)
+    versioned = extra.pop("service.version", None)
+    placed = extra.pop("deployment.environment.name", None)
+    name = (
+        service_name
+        or environ.get(brand.env_var("SERVICE_NAME"))
+        or environ.get("OTEL_SERVICE_NAME")
+        or named
+        or SERVICE_NAME
+    )
+    version = service_version or environ.get(brand.env_var("SERVICE_VERSION")) or versioned
+    environment = environ.get(brand.env_var("DEPLOYMENT_ENVIRONMENT")) or placed
+    attributes = [
+        _attribute("service.name", name),
+        _attribute("service.version", version or __version__),
+    ]
+    if environment:
+        attributes.append(_attribute("deployment.environment.name", environment))
+    attributes.extend(_attribute(key, value) for key, value in extra.items())
+    return attributes
+
+
+def _parse_resource_attributes(text: str) -> dict[str, str]:
+    """``OTEL_RESOURCE_ATTRIBUTES``: ``key=value`` pairs, comma separated, values
+    percent-encoded. A malformed pair is skipped, as the specification allows."""
+    found: dict[str, str] = {}
+    for pair in text.split(","):
+        key, sep, value = pair.partition("=")
+        if sep and key.strip():
+            found[unquote(key.strip())] = unquote(value.strip())
+    return found
+
+
+def _attribute(key: str, value: str | int) -> dict[str, Any]:
+    if isinstance(value, int):
+        return {"key": key, "value": {"intValue": str(value)}}
+    return {"key": key, "value": {"stringValue": value}}
+
+
+def normalise_format(fmt: str | None) -> str:
+    """``text``, ``json`` or ``otlp`` for any spelling LibreDevOpsHelpers accepts."""
+    value = (fmt or "text").strip().lower()
+    value = _FORMAT_ALIASES.get(value, value)
+    return value if value in LOG_FORMATS else "text"
+
+
+def resolve_level(verbosity: int = 0, level_name: str | None = None) -> int:
+    """-v (info) and -vv (debug) win; otherwise ``level_name`` if valid; otherwise warnings."""
+    if verbosity == 1:
+        return logging.INFO
+    if verbosity >= 2:
+        return logging.DEBUG
+    return _LEVELS.get((level_name or "").strip().lower(), logging.WARNING)
+
+
+def configure_logging(
+    verbosity: int = 0, fmt: str | None = "text", level_name: str | None = None
+) -> None:
+    """Send logs to stderr at the resolved level, in the chosen format.
+
+    An unknown format or level falls back to the default, because a logging preference
+    must never stop the command it is logging about.
+    """
+    level = resolve_level(verbosity, level_name)
+    fmt = normalise_format(fmt)
+    handler = logging.StreamHandler(sys.stderr)
+    if fmt == "json":
+        handler.setFormatter(JsonFormatter())
+    elif fmt == "otlp":
+        handler.setFormatter(OtlpFormatter())
+    else:
+        handler.setFormatter(
+            logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s", "%H:%M:%S")
+        )
+    logging.basicConfig(level=level, handlers=[handler], force=True)
+    # urllib3 logs each connection at DEBUG; that is noise unless asked for twice over.
+    logging.getLogger("urllib3").setLevel(logging.DEBUG if verbosity >= 3 else logging.WARNING)

libre_devops_helpers/core/poll.py

@@ -0,0 +1,88 @@
+"""Run a check repeatedly until it is complete or a limit is reached.
+
+The clock and the sleep are injected, so a poll of hours runs in microseconds in tests.
+The poller never sleeps past the timeout: the last wait is cut short so one final pass
+runs at the deadline.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Generic, Literal, TypeVar
+
+from libre_devops_helpers.core.errors import InputError
+
+T = TypeVar("T")
+
+Reason = Literal["complete", "timeout", "max-passes"]
+
+
+@dataclass(frozen=True)
+class PollLimits:
+    """How often to check, and when to give up. ``None`` means no limit of that kind."""
+
+    interval: float
+    timeout: float | None = None
+    max_passes: int | None = None
+
+    def __post_init__(self) -> None:
+        if self.interval <= 0:
+            raise InputError("the poll interval must be more than zero seconds")
+        if self.timeout is not None and self.timeout <= 0:
+            raise InputError("the poll timeout must be more than zero seconds")
+        if self.max_passes is not None and self.max_passes < 1:
+            raise InputError("the poll needs at least one pass")
+
+
+@dataclass(frozen=True)
+class PollOutcome(Generic[T]):
+    """What the poll ended with: the last pass's result and why it stopped."""
+
+    result: T
+    passes: int
+    elapsed: float
+    reason: Reason
+
+    @property
+    def complete(self) -> bool:
+        return self.reason == "complete"
+
+
+def poll(
+    run_pass: Callable[[int], T],
+    is_complete: Callable[[T], bool],
+    limits: PollLimits,
+    *,
+    clock: Callable[[], float] = time.monotonic,
+    sleep: Callable[[float], None] = time.sleep,
+    on_pass: Callable[[int, T], None] | None = None,
+    on_wait: Callable[[float], None] | None = None,
+) -> PollOutcome[T]:
+    """Call ``run_pass(n)`` for n = 1, 2, ... until ``is_complete`` or a limit stops it.
+
+    ``on_pass`` sees each result; ``on_wait`` is told how long the next wait is.
+    Exceptions from ``run_pass`` propagate: the caller decides which ones a pass may
+    absorb (a throttled request) and which end the poll (a rejected token).
+    """
+    started = clock()
+    passes = 0
+    while True:
+        passes += 1
+        result = run_pass(passes)
+        if on_pass is not None:
+            on_pass(passes, result)
+        elapsed = clock() - started
+        if is_complete(result):
+            return PollOutcome(result, passes, elapsed, "complete")
+        if limits.max_passes is not None and passes >= limits.max_passes:
+            return PollOutcome(result, passes, elapsed, "max-passes")
+        if limits.timeout is not None and elapsed >= limits.timeout:
+            return PollOutcome(result, passes, elapsed, "timeout")
+        wait = limits.interval
+        if limits.timeout is not None:
+            wait = min(wait, limits.timeout - elapsed)
+        if on_wait is not None:
+            on_wait(wait)
+        sleep(wait)

libre_devops_helpers/core/process.py

@@ -0,0 +1,106 @@
+"""Run an external command line tool: checked exit codes, a timeout, readable errors.
+
+Vendor layers build on ``CommandRunner`` for their own tools (the Azure CLI today),
+supplying how that tool's errors read and what to suggest. Output is returned to the
+caller but never logged, because it can hold a token.
+"""
+
+from __future__ import annotations
+
+import itertools
+import json
+import logging
+import os
+import shutil
+import subprocess
+from collections.abc import Callable, Mapping
+from typing import Any
+
+from libre_devops_helpers.core.errors import CommandError
+
+log = logging.getLogger(__name__)
+
+Runner = Callable[..., "subprocess.CompletedProcess[str]"]
+
+
+class CommandRunner:
+    """Runs one executable, found on ``PATH`` by ``name`` unless ``executable`` is given.
+
+    ``runner`` defaults to ``subprocess.run``; tests pass a fake so nothing real runs.
+    Subclasses override ``clean_stderr`` and ``hint_for`` for their tool, and
+    ``error_type`` for a more specific error.
+    """
+
+    error_type: type[CommandError] = CommandError
+    # Non-interactive commands are bounded. Interactive ones wait on a person, so are not.
+    timeout_seconds = 120
+
+    def __init__(
+        self,
+        name: str,
+        executable: str | None = None,
+        *,
+        install_hint: str | None = None,
+        runner: Runner = subprocess.run,
+    ) -> None:
+        self.name = name
+        self._executable = executable
+        self._install_hint = install_hint
+        self._runner = runner
+
+    @property
+    def executable(self) -> str:
+        """Path to the tool, resolved on first use so importing never requires it."""
+        if self._executable is None:
+            found = shutil.which(self.name)
+            if found is None:
+                raise self.error_type(f"'{self.name}' is not on PATH", hint=self._install_hint)
+            self._executable = found
+        return self._executable
+
+    def run(
+        self, *args: str, interactive: bool = False, env: Mapping[str, str] | None = None
+    ) -> str:
+        """Run the tool with ``args`` and return stdout. Raises on a non-zero exit.
+
+        ``interactive`` connects the terminal instead of capturing output.
+        """
+        label = " ".join(
+            [self.name, *itertools.takewhile(lambda arg: not arg.startswith("-"), args)]
+        )
+        log.debug("running %s", " ".join([self.name, *args]))
+        try:
+            result = self._runner(
+                [self.executable, *args],
+                capture_output=not interactive,
+                text=True,
+                env={**os.environ, **env} if env else None,
+                timeout=None if interactive else self.timeout_seconds,
+                check=False,
+            )
+        except subprocess.TimeoutExpired:
+            raise self.error_type(f"{label} timed out after {self.timeout_seconds}s") from None
+        except OSError as exc:
+            raise self.error_type(f"cannot run {label}: {exc}") from None
+        if result.returncode != 0:
+            detail = self.clean_stderr(result.stderr) or f"exit code {result.returncode}"
+            raise self.error_type(f"{label} failed: {detail}", hint=self.hint_for(detail))
+        return result.stdout or ""
+
+    def run_json(self, *args: str) -> Any:
+        """Run the tool and parse its stdout as JSON (None for no output)."""
+        out = self.run(*args).strip()
+        if not out:
+            return None
+        try:
+            return json.loads(out)
+        except json.JSONDecodeError:
+            raise self.error_type(f"{self.name} {' '.join(args[:2])} did not return JSON") from None
+
+    def clean_stderr(self, stderr: str | None) -> str:
+        """The error message from stderr, on one line."""
+        return " ".join(line.strip() for line in (stderr or "").splitlines() if line.strip())[:1000]
+
+    def hint_for(self, detail: str) -> str | None:
+        """A next step for a failed command, when the message points to one."""
+        return None