contextdb-memory-ci 0.1.0a1__py3-none-any.whl

contextdb_memory_ci/__init__.py

@@ -0,0 +1,55 @@
+"""Public Python client for ContextDB Memory CI."""
+
+from contextdb_memory_ci.client import (
+    DEFAULT_BASE_URL,
+    EXPORT_RESPONSE_CAP,
+    STATUS_RESPONSE_CAP,
+    MemoryCiClient,
+    validate_base_url,
+)
+from contextdb_memory_ci.errors import (
+    ApiError,
+    ConfigError,
+    MalformedResponseError,
+    MemoryCiError,
+    OutputError,
+    ResponseTooLargeError,
+    RunTimeout,
+    TransportError,
+)
+from contextdb_memory_ci.models import (
+    ExportFormat,
+    MemoryCiRun,
+    RegressionState,
+    RunCounts,
+    RunLimits,
+    RunProgress,
+    RunState,
+    TerminalReason,
+)
+
+__all__ = [
+    "DEFAULT_BASE_URL",
+    "EXPORT_RESPONSE_CAP",
+    "STATUS_RESPONSE_CAP",
+    "ApiError",
+    "ConfigError",
+    "ExportFormat",
+    "MalformedResponseError",
+    "MemoryCiClient",
+    "MemoryCiError",
+    "MemoryCiRun",
+    "OutputError",
+    "RegressionState",
+    "ResponseTooLargeError",
+    "RunCounts",
+    "RunLimits",
+    "RunProgress",
+    "RunState",
+    "RunTimeout",
+    "TerminalReason",
+    "TransportError",
+    "validate_base_url",
+]
+
+__version__ = "0.1.0a1"

contextdb_memory_ci/cli.py

@@ -0,0 +1,545 @@
+"""Command-line interface for public ContextDB Memory CI runs."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import math
+import os
+import re
+import sys
+import tempfile
+import time
+import uuid
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import NoReturn, cast
+
+from contextdb_memory_ci.client import DEFAULT_BASE_URL, MemoryCiClient
+from contextdb_memory_ci.errors import (
+    ApiError,
+    ConfigError,
+    MalformedResponseError,
+    MemoryCiError,
+    OutputError,
+    ResponseTooLargeError,
+    RunTimeout,
+    TransportError,
+)
+from contextdb_memory_ci.models import (
+    ExportFormat,
+    MemoryCiRun,
+    RegressionState,
+    RunState,
+    validate_idempotency_key,
+    validate_safe_id,
+)
+
+EXIT_SUCCESS = 0
+EXIT_BEHAVIORAL_FAILURE = 1
+EXIT_OPERATIONAL_FAILURE = 2
+MIN_RUN_TIMEOUT_SECONDS = 1.0
+MAX_RUN_TIMEOUT_SECONDS = 86_400.0
+MIN_POLL_INTERVAL_SECONDS = 0.05
+MAX_POLL_INTERVAL_SECONDS = 60.0
+_TOKEN_LIKE_PATTERN = re.compile(r"cbe_[A-Za-z0-9._~-]+")
+
+
+class _SecretSafeArgumentParser(argparse.ArgumentParser):
+    def error(self, message: str) -> NoReturn:
+        super().error(_TOKEN_LIKE_PATTERN.sub("<redacted>", message))
+
+
+@dataclass(frozen=True)
+class RunConfig:
+    token: str
+    project_id: str
+    suite_id: str
+    base_url: str
+    baseline_run_id: str | None
+    idempotency_key: str
+    timeout_seconds: float
+    poll_interval: float
+    json_output: Path | None
+    junit_output: Path | None
+    github_summary: bool
+    allow_no_baseline: bool
+    cancel_on_timeout: bool
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = _SecretSafeArgumentParser(
+        prog="contextdb-memory-ci",
+        description="Run a hosted Memory CI suite and return a CI-safe exit code.",
+    )
+    subparsers = parser.add_subparsers(
+        dest="command",
+        required=True,
+        parser_class=_SecretSafeArgumentParser,
+    )
+    run = subparsers.add_parser(
+        "run",
+        help="start a run, poll to terminal status, and optionally export it",
+    )
+    run.add_argument("--token", help="evaluation token (or CONTEXTDB_EVAL_TOKEN)")
+    run.add_argument(
+        "--project-id", help="project ID (or CONTEXTDB_PROJECT_ID)"
+    )
+    run.add_argument(
+        "--suite-id", help="suite ID (or CONTEXTDB_EVAL_SUITE_ID)"
+    )
+    run.add_argument(
+        "--base-url",
+        help="API base URL (or CONTEXTDB_EVALS_BASE_URL)",
+    )
+    run.add_argument(
+        "--baseline-run-id",
+        help="explicit baseline run ID; omit to use the pinned baseline",
+    )
+    run.add_argument(
+        "--idempotency-key",
+        help="stable run key (or CONTEXTDB_EVAL_IDEMPOTENCY_KEY)",
+    )
+    run.add_argument(
+        "--timeout-seconds",
+        type=float,
+        default=1_800.0,
+        help="overall polling timeout, from 1 to 86400 seconds",
+    )
+    run.add_argument(
+        "--poll-interval",
+        type=float,
+        default=2.0,
+        help="seconds between status requests, from 0.05 to 60",
+    )
+    run.add_argument("--json-output", metavar="PATH")
+    run.add_argument("--junit-output", metavar="PATH")
+    run.add_argument(
+        "--github-summary",
+        action="store_true",
+        help="append a safe summary to GITHUB_STEP_SUMMARY",
+    )
+    run.add_argument(
+        "--allow-no-baseline",
+        action="store_true",
+        help="allow a passed run without a baseline to exit zero",
+    )
+    run.add_argument(
+        "--no-cancel-on-timeout",
+        action="store_true",
+        help="do not request cancellation after timeout or interruption",
+    )
+    return parser
+
+
+def main(
+    argv: Sequence[str] | None = None,
+    *,
+    environ: Mapping[str, str] | None = None,
+    client_factory: Callable[..., MemoryCiClient] = MemoryCiClient,
+    monotonic: Callable[[], float] = time.monotonic,
+    sleep: Callable[[float], None] = time.sleep,
+    uuid_factory: Callable[[], uuid.UUID] = uuid.uuid4,
+) -> int:
+    env = os.environ if environ is None else environ
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    client: MemoryCiClient | None = None
+    current_run: MemoryCiRun | None = None
+    config: RunConfig | None = None
+    try:
+        config, generated_local_key = _resolve_config(args, env, uuid_factory)
+        if generated_local_key:
+            print(config.idempotency_key, file=sys.stderr)
+
+        client = client_factory(
+            config.base_url,
+            config.token,
+            timeout=min(30.0, config.timeout_seconds),
+        )
+        deadline = monotonic() + config.timeout_seconds
+        current_run = client.create_run(
+            config.project_id,
+            config.suite_id,
+            idempotency_key=config.idempotency_key,
+            baseline_run_id=config.baseline_run_id,
+        )
+        while not current_run.terminal:
+            remaining = deadline - monotonic()
+            if remaining <= 0:
+                raise RunTimeout("run polling timed out")
+            sleep(min(config.poll_interval, remaining))
+            if monotonic() >= deadline:
+                raise RunTimeout("run polling timed out")
+            current_run = client.get_run(config.project_id, current_run.id)
+
+        exit_code = _terminal_exit_code(
+            current_run,
+            allow_no_baseline=config.allow_no_baseline,
+        )
+        _print_terminal_status(current_run)
+        output_failed = _write_requested_outputs(
+            client,
+            config,
+            current_run,
+            env,
+        )
+        if output_failed:
+            return EXIT_OPERATIONAL_FAILURE
+        if (
+            current_run.status is RunState.PASSED
+            and current_run.regression_status is RegressionState.NO_BASELINE
+            and not config.allow_no_baseline
+        ):
+            print(
+                "operational result: no baseline; pass "
+                "--allow-no-baseline to opt in",
+                file=sys.stderr,
+            )
+        return exit_code
+    except RunTimeout:
+        if (
+            client is not None
+            and current_run is not None
+            and config is not None
+            and config.cancel_on_timeout
+        ):
+            _best_effort_cancel(client, config.project_id, current_run.id)
+        print("operational error: run polling timed out", file=sys.stderr)
+        return EXIT_OPERATIONAL_FAILURE
+    except KeyboardInterrupt:
+        if (
+            client is not None
+            and current_run is not None
+            and config is not None
+            and config.cancel_on_timeout
+        ):
+            _best_effort_cancel(client, config.project_id, current_run.id)
+        print("operational error: interrupted", file=sys.stderr)
+        return EXIT_OPERATIONAL_FAILURE
+    except MemoryCiError as error:
+        print(_format_expected_error(error), file=sys.stderr)
+        return EXIT_OPERATIONAL_FAILURE
+    except Exception:
+        print("operational error: unexpected client failure", file=sys.stderr)
+        return EXIT_OPERATIONAL_FAILURE
+    finally:
+        if client is not None:
+            try:
+                client.close()
+            except Exception:
+                pass
+
+
+def _resolve_config(
+    args: argparse.Namespace,
+    env: Mapping[str, str],
+    uuid_factory: Callable[[], uuid.UUID],
+) -> tuple[RunConfig, bool]:
+    token = _required_value(
+        cast(str | None, args.token),
+        env.get("CONTEXTDB_EVAL_TOKEN"),
+        "evaluation token",
+    )
+    project_id = validate_safe_id(
+        _required_value(
+            cast(str | None, args.project_id),
+            env.get("CONTEXTDB_PROJECT_ID"),
+            "project ID",
+        ),
+        "project ID",
+    )
+    suite_id = validate_safe_id(
+        _required_value(
+            cast(str | None, args.suite_id),
+            env.get("CONTEXTDB_EVAL_SUITE_ID"),
+            "suite ID",
+        ),
+        "suite ID",
+    )
+    command_base_url = cast(str | None, args.base_url)
+    if command_base_url is not None:
+        base_url = command_base_url
+    else:
+        base_url = env.get("CONTEXTDB_EVALS_BASE_URL") or DEFAULT_BASE_URL
+    baseline = cast(str | None, args.baseline_run_id)
+    if baseline is not None:
+        baseline = validate_safe_id(baseline, "baseline run ID")
+
+    command_key = cast(str | None, args.idempotency_key)
+    supplied_key = (
+        command_key
+        if command_key is not None
+        else env.get("CONTEXTDB_EVAL_IDEMPOTENCY_KEY")
+    )
+    idempotency_key, generated_local_key = _choose_idempotency_key(
+        supplied_key,
+        env,
+        suite_id,
+        uuid_factory,
+    )
+    timeout_seconds = _bounded_number(
+        cast(float, args.timeout_seconds),
+        "timeout",
+        MIN_RUN_TIMEOUT_SECONDS,
+        MAX_RUN_TIMEOUT_SECONDS,
+    )
+    poll_interval = _bounded_number(
+        cast(float, args.poll_interval),
+        "poll interval",
+        MIN_POLL_INTERVAL_SECONDS,
+        MAX_POLL_INTERVAL_SECONDS,
+    )
+    return (
+        RunConfig(
+            token=token,
+            project_id=project_id,
+            suite_id=suite_id,
+            base_url=base_url,
+            baseline_run_id=baseline,
+            idempotency_key=idempotency_key,
+            timeout_seconds=timeout_seconds,
+            poll_interval=poll_interval,
+            json_output=_optional_path(cast(str | None, args.json_output)),
+            junit_output=_optional_path(cast(str | None, args.junit_output)),
+            github_summary=bool(args.github_summary),
+            allow_no_baseline=bool(args.allow_no_baseline),
+            cancel_on_timeout=not bool(args.no_cancel_on_timeout),
+        ),
+        generated_local_key,
+    )
+
+
+def _required_value(
+    command_value: str | None,
+    environment_value: str | None,
+    label: str,
+) -> str:
+    value = (
+        command_value
+        if command_value is not None
+        else environment_value
+    )
+    if value is None or not value:
+        raise ConfigError(f"{label} is required")
+    return value
+
+
+def _choose_idempotency_key(
+    supplied: str | None,
+    env: Mapping[str, str],
+    suite_id: str,
+    uuid_factory: Callable[[], uuid.UUID],
+) -> tuple[str, bool]:
+    if supplied is not None:
+        return validate_idempotency_key(supplied), False
+    github_run_id = env.get("GITHUB_RUN_ID")
+    github_attempt = env.get("GITHUB_RUN_ATTEMPT")
+    if github_run_id and github_attempt:
+        material = f"{github_run_id}\0{github_attempt}\0{suite_id}".encode()
+        digest = hashlib.sha256(material).hexdigest()[:48]
+        return f"github-{digest}", False
+    return validate_idempotency_key(f"local-{uuid_factory().hex}"), True
+
+
+def _bounded_number(
+    value: float,
+    label: str,
+    minimum: float,
+    maximum: float,
+) -> float:
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, (int, float))
+        or not math.isfinite(value)
+        or not minimum <= value <= maximum
+    ):
+        raise ConfigError(f"{label} must be between {minimum:g} and {maximum:g}")
+    return float(value)
+
+
+def _optional_path(value: str | None) -> Path | None:
+    if value is None:
+        return None
+    if not value or "\0" in value or len(value) > 4_096:
+        raise ConfigError("output path is invalid")
+    return Path(value)
+
+
+def _terminal_exit_code(
+    run: MemoryCiRun,
+    *,
+    allow_no_baseline: bool,
+) -> int:
+    if run.status in {RunState.ERROR, RunState.CANCELLED}:
+        return EXIT_OPERATIONAL_FAILURE
+    if (
+        run.status is RunState.FAILED
+        or run.regression_status is RegressionState.REGRESSED
+    ):
+        return EXIT_BEHAVIORAL_FAILURE
+    if (
+        run.status is RunState.PASSED
+        and run.regression_status is RegressionState.UNCHANGED
+    ):
+        return EXIT_SUCCESS
+    if (
+        run.status is RunState.PASSED
+        and run.regression_status is RegressionState.NO_BASELINE
+    ):
+        return EXIT_SUCCESS if allow_no_baseline else EXIT_OPERATIONAL_FAILURE
+    return EXIT_OPERATIONAL_FAILURE
+
+
+def _write_requested_outputs(
+    client: MemoryCiClient,
+    config: RunConfig,
+    run: MemoryCiRun,
+    env: Mapping[str, str],
+) -> bool:
+    failed = False
+    requested = (
+        (ExportFormat.JSON, config.json_output),
+        (ExportFormat.JUNIT, config.junit_output),
+    )
+    for export_format, output_path in requested:
+        if output_path is None:
+            continue
+        try:
+            payload = client.export_run(
+                config.project_id,
+                run.id,
+                export_format,
+            )
+            _atomic_write(output_path, payload)
+        except MemoryCiError as error:
+            print(_format_expected_error(error), file=sys.stderr)
+            failed = True
+        except Exception:
+            print("output error: safe export could not be written", file=sys.stderr)
+            failed = True
+
+    if config.github_summary:
+        try:
+            summary_value = env.get("GITHUB_STEP_SUMMARY")
+            if not summary_value:
+                raise ConfigError(
+                    "GITHUB_STEP_SUMMARY is required with --github-summary"
+                )
+            summary_path = _optional_path(summary_value)
+            if summary_path is None:
+                raise ConfigError("GitHub summary path is invalid")
+            _atomic_append(summary_path, _github_summary(run).encode("utf-8"))
+        except MemoryCiError as error:
+            print(_format_expected_error(error), file=sys.stderr)
+            failed = True
+        except Exception:
+            print("output error: GitHub summary could not be written", file=sys.stderr)
+            failed = True
+    return failed
+
+
+def _atomic_write(path: Path, payload: bytes) -> None:
+    parent = path.parent
+    if not parent.is_dir():
+        raise OutputError("output parent directory does not exist")
+    temporary_name: str | None = None
+    try:
+        descriptor, temporary_name = tempfile.mkstemp(
+            prefix=f".{path.name}.",
+            suffix=".tmp",
+            dir=parent,
+        )
+        with os.fdopen(descriptor, "wb") as output:
+            output.write(payload)
+            output.flush()
+            os.fsync(output.fileno())
+        os.replace(temporary_name, path)
+        temporary_name = None
+    except OSError:
+        raise OutputError("safe output could not be written atomically") from None
+    finally:
+        if temporary_name is not None:
+            try:
+                os.unlink(temporary_name)
+            except OSError:
+                pass
+
+
+def _atomic_append(path: Path, payload: bytes) -> None:
+    existing = b""
+    try:
+        if path.exists():
+            existing = path.read_bytes()
+    except OSError:
+        raise OutputError("GitHub summary could not be read") from None
+    _atomic_write(path, existing + payload)
+
+
+def _github_summary(run: MemoryCiRun) -> str:
+    reason = (
+        run.terminal_reason.value
+        if run.terminal_reason is not None
+        else "none"
+    )
+    return (
+        "\n## ContextDB Memory CI\n"
+        f"- Run: `{run.id}`\n"
+        f"- Project: `{run.project_id}`\n"
+        f"- Suite: `{run.suite_id}`\n"
+        f"- Status: `{run.status.value}`\n"
+        f"- Regression: `{run.regression_status.value}`\n"
+        f"- Progress: {run.progress.completed}/{run.progress.total}\n"
+        "- Counts: "
+        f"passed={run.counts.passed}, failed={run.counts.failed}, "
+        f"error={run.counts.error}, cancelled={run.counts.cancelled}\n"
+        f"- Terminal reason: `{reason}`\n"
+    )
+
+
+def _print_terminal_status(run: MemoryCiRun) -> None:
+    reason = (
+        run.terminal_reason.value
+        if run.terminal_reason is not None
+        else "none"
+    )
+    print(
+        f"run={run.id} status={run.status.value} "
+        f"regression={run.regression_status.value} "
+        f"passed={run.counts.passed} failed={run.counts.failed} "
+        f"error={run.counts.error} cancelled={run.counts.cancelled} "
+        f"reason={reason}"
+    )
+
+
+def _best_effort_cancel(
+    client: MemoryCiClient,
+    project_id: str,
+    run_id: str,
+) -> None:
+    try:
+        client.cancel_run(project_id, run_id)
+    except BaseException:
+        pass
+
+
+def _format_expected_error(error: MemoryCiError) -> str:
+    if isinstance(error, ConfigError):
+        return f"configuration error: {error}"
+    if isinstance(error, ApiError):
+        return f"API error: {error}"
+    if isinstance(error, TransportError):
+        return "transport error: API request failed"
+    if isinstance(error, ResponseTooLargeError):
+        return "response error: response exceeded the safe size cap"
+    if isinstance(error, MalformedResponseError):
+        return f"response error: {error}"
+    if isinstance(error, OutputError):
+        return f"output error: {error}"
+    return "operational error: Memory CI could not complete"
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())