toolforge-logs-cli 0.0.3__tar.gz

toolforge_logs_cli-0.0.3/PKG-INFO

@@ -0,0 +1,16 @@
+Metadata-Version: 2.4
+Name: toolforge-logs-cli
+Version: 0.0.3
+Summary: Toolforge logs API client
+License: GPLv3
+Author: David Caro
+Author-email: dcaro@wikimedia.org
+Requires-Python: >=3.13
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
+Requires-Dist: click (>=8.5.0,<9.0.0)
+Requires-Dist: requests (>=2.34.2,<3.0.0)
+Requires-Dist: toolforge-weld (>=1.6.15,<2.0.0)

toolforge_logs_cli-0.0.3/pyproject.toml

@@ -0,0 +1,46 @@
+[project]
+name = "toolforge-logs-cli"
+version = "0.0.3"
+description = "Toolforge logs API client"
+authors = [
+    {name = "David Caro",email = "dcaro@wikimedia.org"}
+]
+license = {text = "GPLv3"}
+requires-python = ">=3.13"
+dependencies = [
+    "click (>=8.5.0,<9.0.0)",
+    "requests (>=2.34.2,<3.0.0)",
+    "toolforge-weld (>=1.6.15,<2.0.0)"
+]
+
+[tool.poetry.plugins."console_scripts"]
+"toolforge-logs" = "toolforge_logs_cli.cli:main"
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[dependency-groups]
+dev = [
+    "pytest (>=9.1.1,<10.0.0)",
+    "pre-commit (>=4.6.2,<5.0.0)",
+    "ruff (>=0.16.7,<0.17.0)",
+    "mypy (>=2.3.1,<3.0.0)",
+    "pytest-cov (>=7.1.0,<8.0.0)"
+]
+
+[tool.coverage.run]
+include = ["*/toolforge_components_cli/*"]
+
+[tool.ruff.lint]
+# add isort
+extend-select = ["I"]
+
+[tool.mypy]
+exclude = [
+    "^debian/",  # debian packaging files
+    "^.cache",
+    "^build",
+    "^dist",
+    "^.tox",
+]

toolforge_logs_cli-0.0.3/toolforge_logs_cli/cli.py

@@ -0,0 +1,176 @@
+import json
+import logging
+import os
+import subprocess
+import sys
+from collections.abc import Iterator
+from typing import Any
+from urllib.parse import urlencode, urljoin
+
+import click
+from toolforge_weld.errors import (
+    ToolforgeError,
+    ToolforgeUserError,
+    print_error_context,
+)
+
+from toolforge_logs_cli.config import get_loaded_config
+from toolforge_logs_cli.logs import (
+    LogsClient,
+    LogsClientError,
+    UnknownLogFormat,
+    log_to_string,
+)
+
+LOGGER = logging.getLogger("toolforge" if __name__ == "__main__" else __name__)
+
+ERROR_REPORT_MSG = "Please report this issue to the Toolforge admins if it persists: https://w.wiki/6Zuu"
+
+
+def handle_error(e: Exception, debug: bool = False) -> None:
+    user_error = isinstance(e, ToolforgeUserError)
+    prefix = "Error: " if user_error else f"{e.__class__.__name__}: "
+    click.echo(click.style(f"{prefix}{e}", fg="red"), err=True)
+
+    if debug:
+        LOGGER.exception(e)
+        if isinstance(e, ToolforgeError):
+            print_error_context(e)
+    elif not user_error:
+        click.echo(click.style(ERROR_REPORT_MSG, fg="red"), err=True)
+
+
+def _should_prompt() -> bool:
+    return sys.stdin.isatty()
+
+
+@click.version_option(prog_name="Toolforge Logs CLI")
+@click.group(name="toolforge-logs", help="Toolforge Logs command line interface")
+@click.option(
+    "-v",
+    "--verbose",
+    help="Show extra verbose output. NOTE: Do not rely on the format of the verbose output.",
+    is_flag=True,
+    default=(os.environ.get("TOOLFORGE_VERBOSE", "0") == "1"),
+    hidden=(os.environ.get("TOOLFORGE_CLI", "0") == "1"),
+)
+@click.option(
+    "-d",
+    "--debug",
+    help="Show logs to debug the toolforge-logs-* packages. For extra verbose output see --verbose.",
+    is_flag=True,
+    default=(os.environ.get("TOOLFORGE_DEBUG", "0") == "1"),
+    hidden=(os.environ.get("TOOLFORGE_CLI", "0") == "1"),
+)
+@click.pass_context
+def toolforge_logs(ctx, verbose: bool, debug: bool) -> None:
+    ctx.ensure_object(dict)
+    ctx.obj.update(
+        {
+            "verbose": verbose,
+            "debug": debug,
+            "config": get_loaded_config(),
+            "logs_api": LogsClient.from_config(config=get_loaded_config()),
+        }
+    )
+
+
+@toolforge_logs.command(name="get", help="Get tool logs.")
+@click.option(
+    "--json",
+    "as_json",
+    is_flag=True,
+    help="If set, will output in JSON format",
+)
+@click.pass_context
+def get_logs(ctx, as_json: bool) -> None:
+    logs_api: LogsClient = ctx.obj["logs_api"]
+
+    for log in _get_logs_nofollow(logs_api=logs_api):
+        if not as_json:
+            try:
+                log_string = log_to_string(log=log)
+                print(f"{log_string}")
+            except UnknownLogFormat as error:
+                click.echo(
+                    message=click.style(f"Unable to parse log: {error}", fg="red"),
+                    err=True,
+                )
+        else:
+            print(json.dumps(log))
+
+
+def _get_logs_page(
+    logs_api: LogsClient,
+    url: str,
+):
+    logs = []
+    # We’re not using the params kwarg of api.get to avoid overwriting pagination URL params from replies.
+    # Pagination URLs already include all necessary params, set by the server based on what we sent first.
+    response = logs_api.get(url, timeout=None)
+    try:
+        entries = response["data"]["logs"]
+        logs.extend(entries)
+    except KeyError:
+        raise LogsClientError("unexpected response format", {"response": response})
+
+    try:
+        next_url = response["data"]["next"]
+    except KeyError:
+        next_url = None  # no more pages
+    try:
+        cursor_url = next_url["url"]  # ty: ignore[not-subscriptable]
+    except (KeyError, TypeError):  # TypeError raised by None["url"]
+        # logs-api’s OpenAPI spec technically allows next=null
+        if next_url is not None:
+            raise LogsClientError("unexpected cursor format", {"cursor": next_url})
+        next_url = None
+    if not entries and next_url is not None:
+        raise LogsClientError(
+            "bad server: got a pagination cursor but no logs",
+            {"response": response},
+        )
+
+    if next_url is not None:
+        try:
+            next_url = urljoin(url, cursor_url)
+        except Exception:  # noqa: BLE001
+            raise LogsClientError("bad server: invalid cursor URL", {"url": next_url})
+    if next_url == url:
+        raise LogsClientError(
+            "bad server: identical pagination cursor",
+            {"cursor": cursor_url, "url": url},
+        )
+    return next_url, logs
+
+
+def _get_logs_nofollow(logs_api: LogsClient) -> Iterator[dict[str, Any]]:
+    # logs-api returns log entries in pages of bounded size (500 entries at the time of writing).
+    # We get all the logs, not filtering yet.
+    url = "/logs?" + urlencode({"direction": "oldest_first"})
+    while url:
+        url, log_batch = _get_logs_page(logs_api=logs_api, url=url)
+        yield from log_batch
+
+
+def main() -> int:
+    debug = False
+    try:
+        args = sys.argv[1:]
+        debug = "-d" in args or "--debug" in args
+        logging.basicConfig(level=logging.DEBUG if debug else logging.INFO)
+        toolforge_logs(standalone_mode=False)
+        return 0
+    except click.exceptions.ClickException as e:
+        e.show()
+        return e.exit_code
+    except subprocess.CalledProcessError as e:
+        handle_error(e, debug=debug)
+        return e.returncode
+    except Exception as e:  # noqa: BLE001
+        handle_error(e, debug=debug)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

toolforge_logs_cli-0.0.3/toolforge_logs_cli/config.py

@@ -0,0 +1,23 @@
+from dataclasses import dataclass, field
+from functools import cache
+from typing import Any
+
+from toolforge_weld.config import Config, Section, load_config
+
+
+@dataclass
+class LogsConfig(Section):
+    _NAME_: str = field(default="logs", init=False)
+    logs_endpoint: str = "/logs/v1"
+
+    @classmethod
+    def from_dict(cls, my_dict: dict[str, Any]):
+        params = {}
+        if "logs_endpoint" in my_dict:
+            params["logs_endpoint"] = my_dict["logs_endpoint"]
+        return cls(**params)
+
+
+@cache
+def get_loaded_config() -> Config:
+    return load_config(client_name="logs", extra_sections=[LogsConfig])

toolforge_logs_cli-0.0.3/toolforge_logs_cli/logs.py

@@ -0,0 +1,183 @@
+import socket
+from typing import Any
+
+import requests
+from toolforge_weld.api_client import ConnectionError, ToolforgeClient
+from toolforge_weld.errors import ToolforgeError, ToolforgeUserError
+from toolforge_weld.kubernetes_config import Kubeconfig
+
+from toolforge_logs_cli.config import Config
+
+
+class LogsClientError(ToolforgeError):
+    """Raised when an HTTP request fails."""
+
+    def __init__(self, message: str, context: dict[str, Any] | None = None) -> None:
+        super().__init__(message)
+
+        if context:
+            # property is defined in parent class
+            self.context = context
+
+
+class LogsClientUserError(LogsClientError, ToolforgeUserError):
+    """Raised when an HTTP request fails with a 4xx status code."""
+
+
+def handle_http_exception(e: requests.exceptions.HTTPError) -> Exception:
+    if e.response is None:
+        return LogsClientError(message="Got no response", context={})
+
+    error_class: type[LogsClientError | LogsClientUserError]
+    if 400 <= e.response.status_code <= 499:
+        error_class = LogsClientUserError
+    else:
+        error_class = LogsClientError
+
+    context = {}
+    message = e.response.text
+    try:
+        data = e.response.json()
+        if isinstance(data, dict):
+            if "messages" in data and "error" in data["messages"]:
+                message = data["messages"]["error"][0]
+                context = {"messages": data["messages"]}
+        elif isinstance(data, str):
+            message = data
+    except requests.exceptions.InvalidJSONError:
+        pass
+
+    return error_class(message=message, context=context)
+
+
+def handle_connection_error(e: ConnectionError) -> Exception:
+    context = {}
+    if isinstance(e, requests.exceptions.HTTPError) and e.response is not None:
+        context["body"] = e.response.text
+
+    return LogsClientError(
+        message="The logs service seems to be down – please retry in a few minutes.",
+        context=context,
+    )
+
+
+class LogsClient(ToolforgeClient):
+    def __init__(
+        self,
+        kubeconfig: Kubeconfig,
+        server: str,
+        endpoint_prefix: str,
+        user_agent: str,
+    ):
+        super().__init__(
+            kubeconfig=kubeconfig,
+            server=server + endpoint_prefix,
+            user_agent=user_agent,
+            exception_handler=handle_http_exception,
+            connect_exception_handler=handle_connection_error,
+        )
+
+    @classmethod
+    def from_config(cls, config: Config):
+        host = socket.gethostname()
+        kubeconfig = Kubeconfig.load()
+        namespace = kubeconfig.current_namespace
+        user_agent = f"{namespace}@{host}:logs-cli"
+        tool_name = namespace[len("tool-") :]
+        full_endpoint_prefix = f"{config.logs.logs_endpoint}/tool/{tool_name}"
+        return cls(
+            endpoint_prefix=full_endpoint_prefix,
+            kubeconfig=kubeconfig,
+            server=config.api_gateway.url,
+            user_agent=user_agent,
+        )
+
+
+class LogError(Exception):
+    pass
+
+
+class UnknownLogFormat(LogError):
+    pass
+
+
+def log_to_string(log: dict[str, Any]) -> str:
+    if "source" not in log:
+        raise UnknownLogFormat(f"No `source` found in log event: {log}")
+
+    source = log["source"]
+
+    match source:
+        case "run-logs":
+            return _format_run_log(log=log)
+        case "jobs":
+            return _format_job_event(event=log)
+        case "deployments":
+            return _format_deployment_event(event=log)
+        case _:
+            raise UnknownLogFormat(f"Unknown source {source} for log: {log}")
+
+
+def _format_run_log(log: dict[str, Any]) -> str:
+    try:
+        return f"{log['datetime']} [{log['job_name']}] [{log['pod']}|{log['container']}] {log['message']}"
+    except KeyError as error:
+        return f"ERROR: unable to parse run-log, missing property {error}\n original log:\n{log}"
+
+
+def _format_deployment_event(event: dict[str, Any]) -> str:
+    if "event_type" not in event:
+        raise UnknownLogFormat(f"No `event_type` found in deployment event: {event}")
+
+    event_type = event["event_type"]
+    match event_type:
+        case "created":
+            return (
+                f"{event['datetime']} "
+                f"DEPLOYMENT CREATED: deployment '{event['deployment_id']}' ({event['description'] or 'no description'}) "
+                f"created by user '{event['user_name']}', "
+                f"affected builds are {event['builds']} and affected jobs are {event['jobs']}"
+            )
+        case "failed":
+            return (
+                f"{event['datetime']} "
+                f"DEPLOYMENT FAILED: deployment '{event['deployment_id']}' failed ({event['description'] or 'no description'}), "
+                f"affected builds are {event['builds']} and affected jobs are {event['jobs']}. Message (format: {event['message_format']}): '{event['message']}'"
+            )
+        case "succeeded":
+            return (
+                f"{event['datetime']} "
+                f"DEPLOYMENT SUCCEEDED: deployment '{event['deployment_id']}' succeeded ({event['description'] or 'no description'}), "
+                f"affected builds are {event['builds']} and affected jobs are {event['jobs']}"
+            )
+        case _:
+            raise UnknownLogFormat(f"Unknown format {event_type} for event: {event}")
+
+    return ""
+
+
+def _format_job_event(event: dict[str, Any]) -> str:
+    if "event_type" not in event:
+        raise UnknownLogFormat(f"No `event_type` found in job event: {event}")
+
+    event_type = event["event_type"]
+    match event_type:
+        case "created" | "deleted" | "updated":
+            return _format_job_event_with_message(event=event)
+        case "restarted":
+            return _format_job_event_without_message(event=event)
+        case _:
+            raise UnknownLogFormat(f"Unknown format {event_type} for event: {event}")
+
+
+def _format_job_event_without_message(event: dict[str, Any]) -> str:
+    return (
+        f"{event['datetime']} "
+        f"JOB {event['event_type'].upper()}: "
+        f"job '{event['job_name']}' {event['event_type']} by user '{event['user_name']}'"
+    )
+
+
+def _format_job_event_with_message(event: dict[str, Any]) -> str:
+    prefix = _format_job_event_without_message(event=event)
+    return f"{prefix}, with message (format: {event['message_format']}) '{event['message']}'"