lean-runtime 0.6.0__py3-none-any.whl

lean_runtime/runtime.py

@@ -0,0 +1,430 @@
+"""High-level API for reproducible Lean environments and raw project execution."""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from collections.abc import Mapping, Sequence
+from dataclasses import replace
+from datetime import datetime, timezone
+from pathlib import Path
+
+from .backends import Backend, LocalBackend
+from .diagnostics import error_diagnostic, parse_diagnostics
+from .environments import Environment, EnvironmentManager, ExecutionCapture
+from .errors import ProjectError, SpecificationError, ToolchainError
+from .events import EventCallback, EventEmitter
+from .health import DoctorReport, diagnose
+from .lockfiles import EnvironmentLock
+from .models import ExecutionProvenance, ExecutionResult
+from .policies import ExecutionPolicy
+from .references import PackageReference, discover_package, normalize_references
+from .resolver import EnvironmentResolver
+from .serialization import sha256_id, sha256_text
+from .specs import EnvironmentSpec, GitPackage
+from .store import EnvironmentStore, GarbageCollectionReport, StoreStatus, platform_record
+from .toolchains import ToolchainManager, normalize_toolchain
+
+EnvironmentReference = Environment | EnvironmentSpec | EnvironmentLock | str
+
+
+def project_toolchain(project: str | os.PathLike[str]) -> str:
+    """Read and normalize the toolchain pinned by a Lean project."""
+    root = Path(project).expanduser().resolve()
+    path = root / "lean-toolchain"
+    if not path.is_file():
+        raise ProjectError(f"project has no lean-toolchain file: {root}")
+    return normalize_toolchain(path.read_text(encoding="utf-8"))
+
+
+class Runtime:
+    """Compile environments and execute trusted Lean inputs within them."""
+
+    def __init__(
+        self,
+        *,
+        home: str | os.PathLike[str] | None = None,
+        toolchains: ToolchainManager | None = None,
+        backend: Backend | None = None,
+        on_event: EventCallback | None = None,
+    ) -> None:
+        self.toolchains = toolchains or ToolchainManager(home)
+        self.home = self.toolchains.home
+        self.backend = backend or LocalBackend()
+        self.events = EventEmitter(on_event)
+        self.store = EnvironmentStore(self.home)
+        self.resolver = EnvironmentResolver(self.toolchains, self.store, self.events)
+        self.environments = EnvironmentManager(
+            self.store, self.toolchains, self.backend, self.events
+        )
+
+    def resolve(self, spec: EnvironmentSpec, *, timeout: float = 900) -> EnvironmentLock:
+        return self.resolver.resolve(spec, timeout=timeout)
+
+    def ensure(
+        self,
+        lock: EnvironmentLock,
+        *,
+        name: str | None = None,
+        build_profile: str = "release",
+    ) -> Environment:
+        return self.environments.ensure(lock, name=name, build_profile=build_profile)
+
+    def open(self, identifier: str) -> Environment:
+        """Open a published environment without resolution or network access."""
+        return self.environments.open(identifier)
+
+    def create_environment(
+        self,
+        name: str,
+        *,
+        toolchain: str,
+        packages: Sequence[GitPackage],
+        timeout: float = 900,
+    ) -> Environment:
+        spec = EnvironmentSpec(toolchain, tuple(packages))
+        return self.ensure(self.resolve(spec, timeout=timeout), name=name)
+
+    def spec_from_references(
+        self,
+        packages: Sequence[str | PackageReference],
+        *,
+        toolchain: str | None = None,
+    ) -> EnvironmentSpec:
+        """Discover GitHub-style package references and return an exact specification."""
+        references = normalize_references(tuple(packages))
+        if not references:
+            raise SpecificationError("at least one package reference is required")
+        discovery_root = self.store.home / "resolution" / "references"
+        discovered = []
+        for reference in references:
+            self.events.emit(
+                "package_reference.started",
+                f"Discovering {reference.display}",
+                reference=reference.display,
+            )
+            package = discover_package(
+                reference,
+                directory=discovery_root,
+                toolchains=self.toolchains,
+            )
+            discovered.append(package)
+            self.events.emit(
+                "package_reference.resolved",
+                f"Discovered {package.package.name}",
+                reference=reference.display,
+                package=package.package.name,
+                revision=package.package.rev,
+                toolchain=package.toolchain,
+                root_module=package.package.module,
+            )
+        declared_toolchains = {package.toolchain for package in discovered}
+        if toolchain is None:
+            if len(declared_toolchains) != 1:
+                details = ", ".join(
+                    f"{package.package.name}={package.toolchain}" for package in discovered
+                )
+                raise SpecificationError(
+                    "package references declare different Lean toolchains; "
+                    f"select one explicitly with toolchain=... ({details})"
+                )
+            selected = next(iter(declared_toolchains))
+        else:
+            selected = normalize_toolchain(toolchain)
+            for package in discovered:
+                if package.toolchain != selected:
+                    self.events.emit(
+                        "compatibility.toolchain_override",
+                        f"{package.package.name} declares {package.toolchain}; using {selected}",
+                        package=package.package.name,
+                        declared_toolchain=package.toolchain,
+                        environment_toolchain=selected,
+                    )
+        return EnvironmentSpec(selected, tuple(package.package for package in discovered))
+
+    def resolve_references(
+        self,
+        packages: Sequence[str | PackageReference],
+        *,
+        toolchain: str | None = None,
+        timeout: float = 900,
+    ) -> EnvironmentLock:
+        """Discover package references and resolve their exact Lake graph."""
+        return self.resolve(
+            self.spec_from_references(packages, toolchain=toolchain), timeout=timeout
+        )
+
+    def ensure_references(
+        self,
+        packages: Sequence[str | PackageReference],
+        *,
+        toolchain: str | None = None,
+        name: str | None = None,
+        timeout: float = 900,
+    ) -> Environment:
+        """Build or reopen the environment described by package references."""
+        return self.ensure(
+            self.resolve_references(packages, toolchain=toolchain, timeout=timeout), name=name
+        )
+
+    def check(
+        self,
+        source: str,
+        *,
+        environment: EnvironmentReference | None = None,
+        packages: Sequence[str | PackageReference] = (),
+        toolchain: str | None = None,
+        project: str | os.PathLike[str] | None = None,
+        filename: str = "Main.lean",
+        timeout: float | None = None,
+        policy: ExecutionPolicy | None = None,
+    ) -> ExecutionResult:
+        """Check source in a content-addressed environment or a raw toolchain/project."""
+        selected_policy = policy or ExecutionPolicy(timeout_seconds=timeout or 120)
+        if timeout is not None and policy is not None:
+            selected_policy = replace(policy, timeout_seconds=timeout)
+        if environment is not None and packages:
+            raise SpecificationError("check cannot combine environment= with packages=")
+        if project is not None and packages:
+            raise SpecificationError("check cannot combine project= with packages=")
+        if packages:
+            resolved = self.ensure_references(packages, toolchain=toolchain)
+            return resolved.check(source, filename=filename, policy=selected_policy)
+        if environment is not None:
+            resolved = self._environment(environment)
+            return resolved.check(source, filename=filename, policy=selected_policy)
+        return self._raw_check(
+            source,
+            toolchain=toolchain,
+            project=project,
+            filename=filename,
+            policy=selected_policy,
+        )
+
+    def check_file(
+        self,
+        path: str | os.PathLike[str],
+        *,
+        environment: EnvironmentReference | None = None,
+        packages: Sequence[str | PackageReference] = (),
+        toolchain: str | None = None,
+        project: str | os.PathLike[str] | None = None,
+        timeout: float | None = None,
+        policy: ExecutionPolicy | None = None,
+    ) -> ExecutionResult:
+        source_path = Path(path).expanduser().resolve()
+        return self.check(
+            source_path.read_text(encoding="utf-8"),
+            filename=source_path.name,
+            environment=environment,
+            packages=packages,
+            toolchain=toolchain,
+            project=project,
+            timeout=timeout,
+            policy=policy,
+        )
+
+    def check_files(
+        self,
+        files: Mapping[str, str],
+        *,
+        entrypoint: str = "Main.lean",
+        environment: EnvironmentReference,
+        policy: ExecutionPolicy | None = None,
+    ) -> ExecutionResult:
+        """Check a multi-file request in a managed environment."""
+        return self._environment(environment).check_files(
+            files, entrypoint=entrypoint, policy=policy
+        )
+
+    def build(
+        self,
+        project: str | os.PathLike[str],
+        *,
+        targets: Sequence[str] = (),
+        toolchain: str | None = None,
+        timeout: float = 900,
+    ) -> ExecutionResult:
+        """Build an existing trusted Lake project outside the environment store."""
+        root = Path(project).expanduser().resolve()
+        if not root.is_dir():
+            raise ProjectError(f"project directory does not exist: {root}")
+        selected = normalize_toolchain(toolchain) if toolchain else project_toolchain(root)
+        command = self.toolchains.command(selected, "lake", "build", *targets)
+        return self._raw_result(
+            command,
+            cwd=root,
+            toolchain=selected,
+            source_digest=sha256_text(""),
+            policy=ExecutionPolicy(timeout_seconds=timeout, max_output_bytes=10_000_000),
+        )
+
+    def gc(
+        self, *, dry_run: bool = True, minimum_age_seconds: float = 2_592_000
+    ) -> GarbageCollectionReport:
+        return self.store.gc(dry_run=dry_run, minimum_age_seconds=minimum_age_seconds)
+
+    def doctor(self) -> DoctorReport:
+        return diagnose(self.toolchains, self.store)
+
+    def store_status(self) -> StoreStatus:
+        return self.store.status()
+
+    def list_environments(self) -> tuple[dict[str, object], ...]:
+        aliases = self.store.aliases()
+        names_by_id: dict[str, list[str]] = {}
+        for name, environment_id in aliases.items():
+            names_by_id.setdefault(environment_id, []).append(name)
+        records: list[dict[str, object]] = []
+        for path in sorted(self.store.environments.glob("env_*")):
+            metadata_path = path / "metadata.json"
+            if not metadata_path.is_file():
+                continue
+            try:
+                metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
+            except (OSError, json.JSONDecodeError):
+                continue
+            records.append(
+                {
+                    "environment_id": path.name,
+                    "lock_id": metadata.get("lock_id"),
+                    "toolchain": metadata.get("toolchain"),
+                    "created_at": metadata.get("created_at"),
+                    "status": metadata.get("status"),
+                    "names": sorted(names_by_id.get(path.name, [])),
+                }
+            )
+        return tuple(records)
+
+    def replay_capture(self, capture: ExecutionCapture | str | os.PathLike[str]) -> ExecutionResult:
+        """Materialize a capture's lock if needed and replay its check request."""
+        resolved = (
+            ExecutionCapture.load(capture) if isinstance(capture, (str, os.PathLike)) else capture
+        )
+        if resolved.operation != "check":
+            raise ProjectError(f"unsupported capture operation: {resolved.operation}")
+        environment = self.ensure(resolved.lock)
+        return environment.check_files(
+            resolved.files,
+            entrypoint=resolved.entrypoint,
+            policy=resolved.policy,
+        )
+
+    def _environment(self, value: EnvironmentReference) -> Environment:
+        if isinstance(value, Environment):
+            return value
+        if isinstance(value, EnvironmentSpec):
+            return self.ensure(self.resolve(value))
+        if isinstance(value, EnvironmentLock):
+            return self.ensure(value)
+        return self.open(value)
+
+    def _raw_check(
+        self,
+        source: str,
+        *,
+        toolchain: str | None,
+        project: str | os.PathLike[str] | None,
+        filename: str,
+        policy: ExecutionPolicy,
+    ) -> ExecutionResult:
+        project_root = Path(project).expanduser().resolve() if project else None
+        selected = normalize_toolchain(toolchain) if toolchain else None
+        if selected is None and project_root is not None:
+            selected = project_toolchain(project_root)
+        if selected is None:
+            raise ToolchainError("check requires an environment, toolchain, or pinned project")
+        safe_filename = Path(filename).name
+        if not safe_filename.endswith(".lean"):
+            safe_filename += ".lean"
+        with tempfile.TemporaryDirectory(prefix="raw-check-", dir=self.store.jobs) as raw:
+            source_path = Path(raw) / safe_filename
+            source_path.write_text(source, encoding="utf-8")
+            if project_root is None:
+                command = self.toolchains.command(selected, "lean", str(source_path))
+                cwd = source_path.parent
+            else:
+                if not project_root.is_dir():
+                    raise ProjectError(f"project directory does not exist: {project_root}")
+                command = self.toolchains.command(selected, "lake", "env", "lean", str(source_path))
+                cwd = project_root
+            return self._raw_result(
+                command,
+                cwd=cwd,
+                toolchain=selected,
+                source_digest=sha256_text(source),
+                policy=policy,
+            )
+
+    def _raw_result(
+        self,
+        command: Sequence[str],
+        *,
+        cwd: Path,
+        toolchain: str,
+        source_digest: str,
+        policy: ExecutionPolicy,
+    ) -> ExecutionResult:
+        started_at = datetime.now(timezone.utc).isoformat()
+        logical_command = list(command[3:])
+        if source_digest != sha256_text("") and logical_command:
+            logical_command[-1] = Path(logical_command[-1]).name
+        request_digest = sha256_id(
+            "request",
+            {
+                "environment_id": None,
+                "toolchain": toolchain,
+                "command": logical_command,
+                "source_digest": source_digest,
+                "policy": policy.to_dict(),
+                "backend": self.backend.name,
+            },
+        )
+        execution_id = sha256_id(
+            "execution",
+            {
+                "request_digest": request_digest,
+                "started_at": started_at,
+                "nonce": os.urandom(16).hex(),
+            },
+        )
+        raw = self.backend.execute(
+            command,
+            cwd=cwd,
+            environment=self.toolchains.environment,
+            policy=policy,
+        )
+        output = "\n".join(part for part in (raw.stdout, raw.stderr) if part)
+        diagnostics = parse_diagnostics(output)
+        if raw.timed_out:
+            diagnostics += (error_diagnostic("Lean execution exceeded its time limit"),)
+        provenance = ExecutionProvenance(
+            environment_id=None,
+            execution_id=execution_id,
+            request_digest=request_digest,
+            lock_id=None,
+            toolchain=toolchain,
+            packages=(),
+            platform=platform_record(),
+            backend=self.backend.name,
+            requested_policy=policy.to_dict(),
+            enforced_policy_fields=raw.enforced_policy_fields,
+            source_digest=source_digest,
+            started_at=started_at,
+        )
+        return ExecutionResult(
+            ok=raw.exit_code == 0,
+            exit_code=raw.exit_code,
+            toolchain=toolchain,
+            command=tuple(command),
+            cwd=str(cwd),
+            stdout=raw.stdout,
+            stderr=raw.stderr,
+            elapsed_seconds=raw.elapsed_seconds,
+            timed_out=raw.timed_out,
+            cancelled=raw.cancelled,
+            output_truncated=raw.output_truncated,
+            diagnostics=diagnostics,
+            provenance=provenance,
+        )

lean_runtime/serialization.py

@@ -0,0 +1,51 @@
+"""Canonical serialization and digest helpers."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import uuid
+from pathlib import Path
+from typing import Any
+
+
+def canonical_json_bytes(value: Any) -> bytes:
+    """Encode JSON data with a stable, whitespace-free representation."""
+    return json.dumps(
+        value,
+        ensure_ascii=False,
+        allow_nan=False,
+        sort_keys=True,
+        separators=(",", ":"),
+    ).encode("utf-8")
+
+
+def sha256_id(prefix: str, value: Any) -> str:
+    """Return a namespaced SHA-256 identity for canonical JSON data."""
+    digest = hashlib.sha256(canonical_json_bytes(value)).hexdigest()
+    return f"{prefix}_{digest}"
+
+
+def sha256_text(value: str) -> str:
+    return "sha256:" + hashlib.sha256(value.encode("utf-8")).hexdigest()
+
+
+def write_json_atomic(path: Path, value: Any) -> None:
+    """Atomically publish formatted JSON next to its destination."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp")
+    try:
+        with temporary.open("w", encoding="utf-8") as handle:
+            handle.write(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True) + "\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+        temporary.replace(path)
+        if os.name != "nt":
+            directory = os.open(path.parent, os.O_RDONLY)
+            try:
+                os.fsync(directory)
+            finally:
+                os.close(directory)
+    finally:
+        temporary.unlink(missing_ok=True)

lean_runtime/specs.py

@@ -0,0 +1,210 @@
+"""Declarative Lean environment specifications."""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from collections import Counter
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal, cast
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:  # pragma: no cover - exercised by the Python 3.10 CI job
+    import tomli as tomllib
+
+from .errors import SpecificationError
+from .serialization import sha256_id
+from .toolchains import normalize_toolchain
+
+SPEC_SCHEMA = "lean-runtime-environment-spec/1"
+_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_'-]*")
+_MODULE = re.compile(r"[A-Za-z_][A-Za-z0-9_']*(?:\.[A-Za-z_][A-Za-z0-9_']*)*")
+_COMMIT = re.compile(r"[0-9a-fA-F]{40}")
+_TAG = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/+\-]{0,199}")
+
+
+@dataclass(frozen=True, slots=True)
+class GitPackage:
+    """One exact Git dependency.
+
+    A request may use an exact commit or an explicitly marked tag. Tags are
+    resolved to full commits before they enter an environment lock.
+    ``root_module`` is imported by the generated root library so Lake builds
+    the dependency's Lean artifacts before the environment is published.
+    """
+
+    name: str
+    url: str
+    rev: str
+    root_module: str | None = None
+    subdir: str | None = None
+    artifact_command: tuple[str, ...] = ()
+    revision_kind: Literal["commit", "tag"] = "commit"
+
+    def __post_init__(self) -> None:
+        if _NAME.fullmatch(self.name) is None:
+            raise SpecificationError(f"invalid Lake package name: {self.name!r}")
+        if not self.url or "\n" in self.url or "\r" in self.url:
+            raise SpecificationError(f"invalid Git URL for package {self.name!r}")
+        if self.revision_kind == "commit" and _COMMIT.fullmatch(self.rev) is None:
+            raise SpecificationError(
+                f"package {self.name!r} requires a full 40-character Git commit"
+            )
+        if self.revision_kind == "tag" and _TAG.fullmatch(self.rev) is None:
+            raise SpecificationError(f"invalid Git tag for package {self.name!r}: {self.rev!r}")
+        if self.root_module is not None and _MODULE.fullmatch(self.root_module) is None:
+            raise SpecificationError(
+                f"invalid root module for package {self.name!r}: {self.root_module!r}"
+            )
+        if self.subdir is not None:
+            subdir = Path(self.subdir)
+            if subdir.is_absolute() or ".." in subdir.parts:
+                raise SpecificationError(f"package subdir must be relative: {self.subdir!r}")
+        if any(not item or "\x00" in item for item in self.artifact_command):
+            raise SpecificationError(f"invalid artifact command for package {self.name!r}")
+
+    @property
+    def module(self) -> str:
+        if self.root_module:
+            return self.root_module
+        return self.name[0].upper() + self.name[1:]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "source": "git",
+            "name": self.name,
+            "url": self.url,
+            "rev": self.rev.lower() if self.revision_kind == "commit" else self.rev,
+            "revision_kind": self.revision_kind,
+            "root_module": self.module,
+            "subdir": self.subdir,
+            "artifact_command": list(self.artifact_command),
+        }
+
+    @classmethod
+    def git(
+        cls,
+        name: str,
+        url: str,
+        rev: str,
+        *,
+        root_module: str | None = None,
+        subdir: str | None = None,
+        artifact_command: tuple[str, ...] = (),
+    ) -> GitPackage:
+        """Convenience constructor allowing ``Package.git(...)``."""
+        return cls(
+            name=name,
+            url=url,
+            rev=rev,
+            revision_kind="commit",
+            root_module=root_module,
+            subdir=subdir,
+            artifact_command=artifact_command,
+        )
+
+    @classmethod
+    def tag(
+        cls,
+        name: str,
+        url: str,
+        tag: str,
+        *,
+        root_module: str | None = None,
+        subdir: str | None = None,
+        artifact_command: tuple[str, ...] = (),
+    ) -> GitPackage:
+        """Request a friendly Git tag that will be pinned during resolution."""
+        return cls(
+            name=name,
+            url=url,
+            rev=tag,
+            revision_kind="tag",
+            root_module=root_module,
+            subdir=subdir,
+            artifact_command=artifact_command,
+        )
+
+    @classmethod
+    def from_dict(cls, value: dict[str, Any]) -> GitPackage:
+        if value.get("source", "git") != "git":
+            raise SpecificationError("the initial release supports only Git packages")
+        command = value.get("artifact_command") or []
+        if not isinstance(command, list) or not all(isinstance(item, str) for item in command):
+            raise SpecificationError("artifact_command must be an array of strings")
+        tag = value.get("tag")
+        if tag is not None and "rev" in value:
+            raise SpecificationError("a Git package cannot specify both rev and tag")
+        revision_kind_value = "tag" if tag is not None else value.get("revision_kind", "commit")
+        if revision_kind_value not in {"commit", "tag"}:
+            raise SpecificationError("revision_kind must be 'commit' or 'tag'")
+        revision_kind = cast(Literal["commit", "tag"], revision_kind_value)
+        return cls(
+            name=str(value.get("name", "")),
+            url=str(value.get("url", "")),
+            rev=str(tag if tag is not None else value.get("rev", "")),
+            revision_kind=revision_kind,
+            root_module=value.get("root_module"),
+            subdir=value.get("subdir"),
+            artifact_command=tuple(command),
+        )
+
+
+Package = GitPackage
+
+
+@dataclass(frozen=True, slots=True)
+class EnvironmentSpec:
+    """Canonical inputs to the environment compiler."""
+
+    toolchain: str
+    packages: tuple[GitPackage, ...] = field(default_factory=tuple)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "toolchain", normalize_toolchain(self.toolchain))
+        names = [package.name for package in self.packages]
+        duplicates = sorted(name for name, count in Counter(names).items() if count > 1)
+        if duplicates:
+            raise SpecificationError("duplicate direct package names: " + ", ".join(duplicates))
+
+    @property
+    def spec_digest(self) -> str:
+        return sha256_id("spec", self.to_dict())
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema": SPEC_SCHEMA,
+            "toolchain": self.toolchain,
+            "packages": [
+                package.to_dict() for package in sorted(self.packages, key=lambda item: item.name)
+            ],
+        }
+
+    @classmethod
+    def from_dict(cls, value: dict[str, Any]) -> EnvironmentSpec:
+        schema = value.get("schema", SPEC_SCHEMA)
+        if schema != SPEC_SCHEMA:
+            raise SpecificationError(f"unsupported environment specification schema: {schema!r}")
+        raw_packages = value.get("packages", value.get("package", []))
+        if not isinstance(raw_packages, list):
+            raise SpecificationError("packages must be an array")
+        return cls(
+            toolchain=str(value.get("toolchain", "")),
+            packages=tuple(GitPackage.from_dict(item) for item in raw_packages),
+        )
+
+    @classmethod
+    def load(cls, path: str | Path) -> EnvironmentSpec:
+        source = Path(path)
+        with source.open("rb") as handle:
+            value = (
+                tomllib.load(handle)
+                if source.suffix.lower() == ".toml"
+                else json.loads(handle.read().decode("utf-8"))
+            )
+        if not isinstance(value, dict):
+            raise SpecificationError("environment specification must be an object")
+        return cls.from_dict(value)