mxready 1.0.0__py3-none-any.whl

mxready_runner/process.py

@@ -0,0 +1,103 @@
+"""Execute reviewed argument arrays while bounding captured output in memory."""
+
+from __future__ import annotations
+
+import os
+import signal
+import subprocess
+import sys
+from collections.abc import Sequence
+from contextlib import suppress
+from subprocess import CompletedProcess, TimeoutExpired
+from threading import Thread
+from typing import BinaryIO
+
+from mxready_runner.redact import MAX_OUTPUT_BYTES
+
+
+class _Capture:
+    def __init__(self) -> None:
+        self.data = bytearray()
+        self.truncated = False
+
+    def read(self, stream: BinaryIO) -> None:
+        try:
+            while chunk := stream.read(4096):
+                remaining = MAX_OUTPUT_BYTES - len(self.data)
+                self.data.extend(chunk[:remaining])
+                self.truncated |= len(chunk) > remaining
+        finally:
+            stream.close()
+
+    def text(self) -> str:
+        suffix = b"[TRUNCATED]" if self.truncated else b""
+        return (
+            bytes(self.data[: MAX_OUTPUT_BYTES - len(suffix)]).decode(
+                "utf-8",
+                errors="replace",
+            )
+            + suffix.decode()
+        )
+
+
+def _terminate_tree(process: subprocess.Popen[bytes]) -> None:
+    if sys.platform != "win32":
+        with suppress(ProcessLookupError):
+            os.killpg(process.pid, signal.SIGKILL)
+    else:
+        # Fixed system command, never interpolated project input or a shell command.
+        with suppress(OSError, TimeoutExpired):
+            subprocess.run(
+                ["taskkill", "/PID", str(process.pid), "/T", "/F"],
+                shell=False,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                timeout=5,
+                check=False,
+            )
+        if process.poll() is None:
+            process.kill()
+
+
+def run_command(
+    command: Sequence[str],
+    *,
+    timeout: int,
+    cwd: str | os.PathLike[str] | None = None,
+    env: dict[str, str] | None = None,
+) -> CompletedProcess[str]:
+    with subprocess.Popen(
+        list(command),
+        shell=False,
+        cwd=cwd,
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        start_new_session=os.name == "posix",
+        creationflags=getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if os.name == "nt" else 0,
+    ) as process:
+        assert process.stdout is not None and process.stderr is not None
+        stdout, stderr = _Capture(), _Capture()
+        readers = [
+            Thread(target=stdout.read, args=(process.stdout,), daemon=True),
+            Thread(target=stderr.read, args=(process.stderr,), daemon=True),
+        ]
+        for reader in readers:
+            reader.start()
+        timed_out = False
+        try:
+            process.wait(timeout=timeout)
+        except TimeoutExpired:
+            timed_out = True
+            _terminate_tree(process)
+            process.wait()
+        finally:
+            for reader in readers:
+                reader.join(timeout=0.1)
+            if any(reader.is_alive() for reader in readers):
+                _terminate_tree(process)
+                for reader in readers:
+                    reader.join(timeout=5)
+        if timed_out:
+            raise TimeoutExpired(command, timeout, output=stdout.text(), stderr=stderr.text())
+        return CompletedProcess(list(command), process.returncode, stdout.text(), stderr.text())

mxready_runner/redact.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import re
+
+MAX_OUTPUT_BYTES = 16_384
+
+_SENSITIVE_NAME = r"[A-Z0-9_]*(?:TOKEN|PASSWORD|SECRET|KEY|AUTHORIZATION)"
+_BEARER = re.compile(r"(?i)(\bAuthorization\s*:\s*Bearer\s+)\S+")
+_QUOTED_ASSIGNMENT = re.compile(rf"(?i)(\b{_SENSITIVE_NAME}\b\s*[:=]\s*)([\"'])(.*?)(\2)")
+_ASSIGNMENT = re.compile(rf"(?i)(\b{_SENSITIVE_NAME}\b\s*[:=]\s*)([^\s,;]+)")
+_UNIX_HOME = re.compile(r"(?i)(/(?:home|Users)/)[^/\s]+")
+_WINDOWS_HOME = re.compile(r"(?i)([A-Z]:\\Users\\)[^\\\s]+")
+
+
+def redact_text(value: str) -> str:
+    redacted = _BEARER.sub(r"\1[REDACTED]", value)
+    redacted = _QUOTED_ASSIGNMENT.sub(
+        lambda match: f"{match.group(1)}{match.group(2)}[REDACTED]{match.group(2)}",
+        redacted,
+    )
+    redacted = _ASSIGNMENT.sub(r"\1[REDACTED]", redacted)
+    redacted = _UNIX_HOME.sub(r"\1[USER]", redacted)
+    return _WINDOWS_HOME.sub(r"\1[USER]", redacted)
+
+
+def truncate_text(value: str, max_bytes: int = MAX_OUTPUT_BYTES) -> str:
+    encoded = value.encode("utf-8", errors="replace")
+    if len(encoded) <= max_bytes:
+        return value
+    suffix = b"[TRUNCATED]"
+    prefix = encoded[: max(0, max_bytes - len(suffix))]
+    return prefix.decode("utf-8", errors="ignore") + suffix.decode("ascii")
+
+
+def sanitize_output(value: str | None) -> str:
+    return truncate_text(redact_text(value or ""))

mxready_runner/schema.py

@@ -0,0 +1,326 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any, Literal
+from urllib.parse import urlsplit
+from uuid import UUID
+
+_COMMIT = re.compile(r"[0-9a-f]{40}")
+_VERSION = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+")
+_COMMAND_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,63}")
+_REPOSITORY_SEGMENT = re.compile(r"[A-Za-z0-9_.-]+")
+_FORBIDDEN_EXECUTABLES = {
+    "apk",
+    "apt",
+    "apt-get",
+    "bash",
+    "brew",
+    "busybox",
+    "choco",
+    "cmd",
+    "command",
+    "conda",
+    "dnf",
+    "env",
+    "mamba",
+    "nice",
+    "nohup",
+    "npm",
+    "pacman",
+    "pip",
+    "pip3",
+    "pnpm",
+    "powershell",
+    "pwsh",
+    "sh",
+    "sudo",
+    "uv",
+    "xargs",
+    "yum",
+    "yarn",
+    "zsh",
+}
+_FORBIDDEN_PYTHON_MODULES = {
+    "conda",
+    "ensurepip",
+    "pip",
+    "pip3",
+    "poetry",
+    "uv",
+}
+_SHELL_TOKENS = {"&", "&&", ";", "<", "<<", ">", ">>", "|", "||"}
+_REDIRECTION = re.compile(r"(?:[0-9]*)(?:>>?|<<?).+")
+_MAX_MANIFEST_BYTES = 1_048_576
+
+
+class ManifestError(ValueError):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class CommandSpec:
+    id: str
+    command: list[str]
+    timeout_seconds: int
+
+
+@dataclass(frozen=True, slots=True)
+class Manifest:
+    schema_version: str
+    scan_id: str
+    repository_url: str
+    repository_commit: str
+    runner_version: str
+    checks: list[CommandSpec]
+    project_commands: list[CommandSpec]
+
+
+@dataclass(frozen=True, slots=True)
+class CheckResult:
+    id: str
+    command: list[str]
+    status: Literal["passed", "failed", "unavailable"]
+    return_code: int | None
+    stdout: str
+    stderr: str
+    duration_ms: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(frozen=True, slots=True)
+class CommandResult:
+    id: str
+    command: list[str]
+    timeout_seconds: int
+    status: Literal["passed", "failed", "timeout", "cancelled"]
+    return_code: int | None
+    stdout: str
+    stderr: str
+    duration_ms: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(frozen=True, slots=True)
+class RunResult:
+    schema_version: str
+    scan_id: str
+    repository_commit: str
+    runner_version: str
+    environment_fingerprint: str
+    checks: list[CheckResult]
+    commands: list[CommandResult]
+    started_at: str
+    finished_at: str
+    overall_status: Literal["passed", "failed", "cancelled"]
+    repository_state: dict[str, str | bool] | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "scan_id": self.scan_id,
+            "repository_commit": self.repository_commit,
+            "runner_version": self.runner_version,
+            "environment_fingerprint": self.environment_fingerprint,
+            "checks": [item.to_dict() for item in self.checks],
+            "commands": [item.to_dict() for item in self.commands],
+            "started_at": self.started_at,
+            "finished_at": self.finished_at,
+            "overall_status": self.overall_status,
+            "repository_state": self.repository_state,
+        }
+
+    def to_json(self, *, indent: int = 2) -> str:
+        return json.dumps(
+            self.to_dict(),
+            ensure_ascii=False,
+            indent=indent,
+            sort_keys=True,
+        )
+
+
+def load_manifest(path: str | Path) -> Manifest:
+    manifest_path = Path(path)
+    try:
+        if manifest_path.stat().st_size > _MAX_MANIFEST_BYTES:
+            raise ManifestError("manifest is too large")
+        document = json.loads(
+            manifest_path.read_text(encoding="utf-8-sig"),
+            object_pairs_hook=_unique_object,
+        )
+        return _validate_manifest(document)
+    except ManifestError:
+        raise
+    except (OSError, UnicodeError, json.JSONDecodeError, TypeError, ValueError) as error:
+        raise ManifestError("manifest is not valid JSON-compatible YAML") from error
+
+
+def _validate_manifest(document: Any) -> Manifest:
+    if not isinstance(document, dict):
+        raise ManifestError("manifest root must be an object")
+    expected_fields = {
+        "schema_version",
+        "scan_id",
+        "repository_url",
+        "repository_commit",
+        "runner_version",
+        "checks",
+        "project_commands",
+    }
+    if set(document) != expected_fields:
+        raise ManifestError("manifest fields do not match the schema")
+    if document["schema_version"] != "1.0":
+        raise ManifestError("unsupported manifest schema version")
+
+    scan_id = _validate_uuid(document["scan_id"])
+    repository_url = _validate_repository_url(document["repository_url"])
+    repository_commit = document["repository_commit"]
+    if not isinstance(repository_commit, str) or _COMMIT.fullmatch(repository_commit) is None:
+        raise ManifestError("repository commit must be 40 lowercase hexadecimal characters")
+    runner_version = document["runner_version"]
+    if not isinstance(runner_version, str) or _VERSION.fullmatch(runner_version) is None:
+        raise ManifestError("runner version is invalid")
+
+    checks = _validate_command_list(document["checks"], "checks")
+    _validate_environment_checks(checks)
+    project_commands = _validate_command_list(
+        document["project_commands"],
+        "project_commands",
+    )
+    identifiers = [item.id for item in [*checks, *project_commands]]
+    if len(identifiers) != len(set(identifiers)):
+        raise ManifestError("command identifiers must be unique")
+
+    return Manifest(
+        schema_version="1.0",
+        scan_id=scan_id,
+        repository_url=repository_url,
+        repository_commit=repository_commit,
+        runner_version=runner_version,
+        checks=checks,
+        project_commands=project_commands,
+    )
+
+
+def _validate_command_list(value: Any, field_name: str) -> list[CommandSpec]:
+    if not isinstance(value, list) or len(value) > 64:
+        raise ManifestError(f"{field_name} must be a bounded list")
+    return [_validate_command(item) for item in value]
+
+
+def _validate_environment_checks(checks: list[CommandSpec]) -> None:
+    # contracts imports CommandSpec from this module; resolve it after module initialization.
+    from mxready_runner.contracts import DEFAULT_CHECKS
+
+    allowed = {check.id: check for check in DEFAULT_CHECKS}
+    for check in checks:
+        expected = allowed.get(check.id)
+        if (
+            expected is None
+            or check.command != expected.command
+            or check.timeout_seconds > expected.timeout_seconds
+        ):
+            raise ManifestError("environment checks must match the fixed runner contract")
+
+
+def _validate_command(value: Any) -> CommandSpec:
+    if not isinstance(value, dict) or set(value) != {
+        "id",
+        "command",
+        "timeout_seconds",
+    }:
+        raise ManifestError("command fields do not match the schema")
+    identifier = value["id"]
+    if not isinstance(identifier, str) or _COMMAND_ID.fullmatch(identifier) is None:
+        raise ManifestError("command id is invalid")
+    command = value["command"]
+    if (
+        not isinstance(command, list)
+        or not 1 <= len(command) <= 32
+        or any(
+            not isinstance(argument, str)
+            or not argument
+            or len(argument) > 4_096
+            or any(ord(character) < 32 for character in argument)
+            for argument in command
+        )
+    ):
+        raise ManifestError("command must be a bounded array of strings")
+    _validate_command_safety(command)
+
+    timeout = value["timeout_seconds"]
+    if type(timeout) is not int or not 1 <= timeout <= 600:
+        raise ManifestError("command timeout must be between 1 and 600 seconds")
+    return CommandSpec(
+        id=identifier,
+        command=list(command),
+        timeout_seconds=timeout,
+    )
+
+
+def _validate_command_safety(command: list[str]) -> None:
+    executable = command[0].replace("\\", "/").rsplit("/", 1)[-1].casefold()
+    executable = os.path.splitext(executable)[0]
+    if executable in _FORBIDDEN_EXECUTABLES or command[0].startswith("-"):
+        raise ManifestError("privileged, shell, and package-manager commands are forbidden")
+    if (
+        executable.startswith("python")
+        and len(command) >= 3
+        and command[1] == "-m"
+        and command[2].split(".", 1)[0].casefold() in _FORBIDDEN_PYTHON_MODULES
+    ):
+        raise ManifestError("Python package-manager modules are forbidden")
+    if any(argument in _SHELL_TOKENS or _REDIRECTION.fullmatch(argument) for argument in command):
+        raise ManifestError("shell operators and redirection are forbidden")
+
+
+def _validate_uuid(value: Any) -> str:
+    if not isinstance(value, str):
+        raise ManifestError("scan id must be a UUID")
+    try:
+        parsed = UUID(value)
+    except ValueError as error:
+        raise ManifestError("scan id must be a UUID") from error
+    if str(parsed) != value.casefold():
+        raise ManifestError("scan id must use canonical UUID form")
+    return str(parsed)
+
+
+def _validate_repository_url(value: Any) -> str:
+    if not isinstance(value, str) or not value.startswith("https://"):
+        raise ManifestError("repository URL must use HTTPS")
+    try:
+        parsed = urlsplit(value)
+        port = parsed.port
+    except ValueError as error:
+        raise ManifestError("repository URL is invalid") from error
+    if (
+        parsed.scheme != "https"
+        or parsed.hostname not in {"github.com", "gitee.com"}
+        or parsed.username is not None
+        or parsed.password is not None
+        or port is not None
+        or parsed.query
+        or parsed.fragment
+    ):
+        raise ManifestError("repository URL must identify a public GitHub or Gitee repo")
+    parts = parsed.path.removesuffix(".git").strip("/").split("/")
+    if len(parts) != 2 or any(_REPOSITORY_SEGMENT.fullmatch(part) is None for part in parts):
+        raise ManifestError("repository URL path is invalid")
+    return value.removesuffix(".git")
+
+
+def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in result:
+            raise ManifestError("duplicate manifest field")
+        result[key] = value
+    return result