lean-runtime 0.6.0__py3-none-any.whl

lean_runtime/references.py

@@ -0,0 +1,246 @@
+"""Ergonomic package references compiled into exact Git package specifications."""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import sys
+import tempfile
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:  # pragma: no cover - exercised by the Python 3.10 CI job
+    import tomli as tomllib
+
+from .errors import ResolutionError, SpecificationError
+from .specs import GitPackage
+from .toolchains import ToolchainManager, normalize_toolchain
+
+_GITHUB = re.compile(
+    r"github:(?P<owner>[A-Za-z0-9](?:[A-Za-z0-9-]{0,38}))/"
+    r"(?P<repo>[A-Za-z0-9_.-]+)@(?P<revision>[A-Za-z0-9][A-Za-z0-9._/+\-]{0,199})"
+)
+_COMMIT = re.compile(r"[0-9a-fA-F]{40}")
+
+
+@dataclass(frozen=True, slots=True)
+class PackageReference:
+    """A human-friendly reference to one Git-hosted Lean package."""
+
+    url: str
+    revision: str
+    display: str
+    revision_kind: Literal["commit", "tag"]
+
+    @classmethod
+    def parse(cls, value: str) -> PackageReference:
+        """Parse ``github:owner/repository@tag-or-commit``."""
+        match = _GITHUB.fullmatch(value)
+        if match is None:
+            raise SpecificationError(
+                "invalid package reference; expected github:owner/repository@tag-or-commit"
+            )
+        owner = match.group("owner")
+        repository = match.group("repo")
+        if repository.endswith(".git"):
+            repository = repository[:-4]
+        if not repository:
+            raise SpecificationError("GitHub package reference has an empty repository name")
+        revision = match.group("revision")
+        kind: Literal["commit", "tag"] = (
+            "commit" if _COMMIT.fullmatch(revision) is not None else "tag"
+        )
+        return cls(
+            url=f"https://github.com/{owner}/{repository}.git",
+            revision=revision,
+            display=f"github:{owner}/{repository}@{revision}",
+            revision_kind=kind,
+        )
+
+    @classmethod
+    def git(cls, url: str, revision: str) -> PackageReference:
+        """Construct a reference for a generic Git URL, primarily for Python callers."""
+        if not url or "\n" in url or "\r" in url:
+            raise SpecificationError("invalid Git package-reference URL")
+        kind: Literal["commit", "tag"] = (
+            "commit" if _COMMIT.fullmatch(revision) is not None else "tag"
+        )
+        # Reuse GitPackage's strict revision validation with a temporary valid name.
+        if kind == "commit":
+            GitPackage.git("package_reference", url, revision)
+        else:
+            GitPackage.tag("package_reference", url, revision)
+        return cls(url=url, revision=revision, display=f"git:{url}@{revision}", revision_kind=kind)
+
+
+@dataclass(frozen=True, slots=True)
+class DiscoveredPackage:
+    """Exact package metadata discovered from a referenced Lake project."""
+
+    reference: PackageReference
+    toolchain: str
+    package: GitPackage
+
+
+def _run_git(arguments: list[str], *, cwd: Path | None = None, timeout: float = 120) -> str:
+    try:
+        process = subprocess.run(
+            ["git", *arguments],
+            cwd=cwd,
+            text=True,
+            capture_output=True,
+            timeout=timeout,
+            check=False,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise ResolutionError(
+            "package-reference Git operation timed out",
+            phase="package-discovery",
+            command=("git", *arguments),
+            exit_code=124,
+            output=str(exc.stdout or "") + str(exc.stderr or ""),
+        ) from exc
+    if process.returncode:
+        raise ResolutionError(
+            "could not acquire package-reference metadata",
+            phase="package-discovery",
+            command=("git", *arguments),
+            exit_code=process.returncode,
+            output=process.stdout + process.stderr,
+        )
+    return process.stdout.strip()
+
+
+def _read_lake_metadata(lakefile: Path) -> tuple[str, str]:
+    """Read the declarative Lake metadata needed by environment resolution."""
+    try:
+        value = tomllib.loads(lakefile.read_text(encoding="utf-8"))
+    except (OSError, tomllib.TOMLDecodeError) as exc:
+        raise SpecificationError(f"could not read package metadata {lakefile.name}: {exc}") from exc
+    if not isinstance(value, dict):
+        raise SpecificationError(f"package metadata {lakefile.name} is not a TOML object")
+    name = value.get("name")
+    if not isinstance(name, str) or not name:
+        raise SpecificationError(f"package metadata {lakefile.name} has no package name")
+    libraries = value.get("lean_lib")
+    if not isinstance(libraries, list) or not libraries:
+        raise SpecificationError(f"package metadata {lakefile.name} declares no [[lean_lib]]")
+    first = libraries[0]
+    if not isinstance(first, dict):
+        raise SpecificationError("package's first [[lean_lib]] is malformed")
+    roots = first.get("roots")
+    root_module: Any = roots[0] if isinstance(roots, list) and roots else first.get("name")
+    if not isinstance(root_module, str) or not root_module:
+        raise SpecificationError("package's first [[lean_lib]] has no importable root")
+    return name, root_module
+
+
+def _lake_metadata(
+    checkout: Path,
+    *,
+    toolchain: str,
+    toolchains: ToolchainManager,
+) -> tuple[str, str]:
+    lakefile = checkout / "lakefile.toml"
+    if lakefile.is_file():
+        return _read_lake_metadata(lakefile)
+
+    lakefile_lean = checkout / "lakefile.lean"
+    if not lakefile_lean.is_file():
+        raise SpecificationError(
+            "referenced package has neither a root lakefile.toml nor lakefile.lean"
+        )
+    translated = checkout / ".lean-runtime-lakefile.toml"
+    try:
+        command = toolchains.command(
+            toolchain,
+            "lake",
+            "translate-config",
+            "toml",
+            str(translated),
+        )
+        process = subprocess.run(
+            command,
+            cwd=checkout,
+            env=toolchains.environment,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            timeout=120,
+            check=False,
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise ResolutionError(
+            "Lake package-metadata translation timed out",
+            phase="package-discovery",
+            command=tuple(command),
+            exit_code=124,
+            output=str(exc.stdout or "") + str(exc.stderr or ""),
+        ) from exc
+    if process.returncode:
+        raise ResolutionError(
+            "Lake could not translate package metadata",
+            phase="package-discovery",
+            command=tuple(command),
+            exit_code=process.returncode,
+            output=process.stdout,
+        )
+    if not translated.is_file():
+        raise ResolutionError(
+            "Lake completed without producing translated package metadata",
+            phase="package-discovery",
+            command=tuple(command),
+            output=process.stdout,
+        )
+    return _read_lake_metadata(translated)
+
+
+def discover_package(
+    reference: PackageReference,
+    *,
+    directory: Path,
+    toolchains: ToolchainManager | None = None,
+) -> DiscoveredPackage:
+    """Acquire a shallow checkout and compile its metadata into an exact package."""
+    directory.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix="package-", dir=directory) as raw:
+        checkout = Path(raw)
+        _run_git(["init", "--quiet"], cwd=checkout)
+        _run_git(["remote", "add", "origin", reference.url], cwd=checkout)
+        target = (
+            reference.revision
+            if reference.revision_kind == "commit"
+            else f"refs/tags/{reference.revision}"
+        )
+        _run_git(["fetch", "--quiet", "--depth", "1", "origin", target], cwd=checkout)
+        _run_git(["checkout", "--quiet", "--detach", "FETCH_HEAD"], cwd=checkout)
+        revision = _run_git(["rev-parse", "HEAD"], cwd=checkout).lower()
+        if _COMMIT.fullmatch(revision) is None:
+            raise ResolutionError(
+                "package reference did not resolve to a full Git commit",
+                phase="package-discovery",
+            )
+        toolchain_path = checkout / "lean-toolchain"
+        if not toolchain_path.is_file():
+            raise SpecificationError("referenced package has no lean-toolchain file")
+        toolchain = normalize_toolchain(toolchain_path.read_text(encoding="utf-8"))
+        manager = toolchains or ToolchainManager(directory.parent)
+        name, root_module = _lake_metadata(
+            checkout,
+            toolchain=toolchain,
+            toolchains=manager,
+        )
+        package = GitPackage.git(name, reference.url, revision, root_module=root_module)
+        return DiscoveredPackage(reference=reference, toolchain=toolchain, package=package)
+
+
+def normalize_references(
+    values: Sequence[str | PackageReference],
+) -> tuple[PackageReference, ...]:
+    return tuple(
+        PackageReference.parse(value) if isinstance(value, str) else value for value in values
+    )

lean_runtime/resolver.py

@@ -0,0 +1,282 @@
+"""Compile exact-Git specifications into portable Lake-backed locks."""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import tempfile
+from dataclasses import replace
+from pathlib import Path
+from typing import Any
+
+from .errors import ResolutionError
+from .events import EventEmitter
+from .lake import ROOT_MODULE, generate_lakefile, generate_root_module
+from .lockfiles import EnvironmentLock, LockedPackage
+from .serialization import sha256_id
+from .specs import EnvironmentSpec, GitPackage
+from .store import EnvironmentStore
+from .toolchains import ToolchainManager, normalize_toolchain
+
+_COMMIT = re.compile(r"[0-9a-fA-F]{40}")
+
+
+def _git(path: Path, *arguments: str) -> str:
+    process = subprocess.run(
+        ["git", "-C", str(path), *arguments],
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    if process.returncode:
+        raise ResolutionError(
+            f"could not inspect resolved Git package at {path}",
+            phase="lock-validation",
+            command=("git", "-C", str(path), *arguments),
+            exit_code=process.returncode,
+            output=process.stdout + process.stderr,
+        )
+    return process.stdout.strip()
+
+
+class EnvironmentResolver:
+    def __init__(
+        self,
+        toolchains: ToolchainManager,
+        store: EnvironmentStore,
+        events: EventEmitter | None = None,
+    ) -> None:
+        self.toolchains = toolchains
+        self.store = store
+        self.events = events or EventEmitter()
+
+    def resolve(self, spec: EnvironmentSpec, *, timeout: float = 900) -> EnvironmentLock:
+        self.events.emit(
+            "resolution.started",
+            "Resolving environment",
+            toolchain=spec.toolchain,
+            packages=len(spec.packages),
+        )
+        toolchain = self.toolchains.ensure(spec.toolchain)
+        self.events.emit("toolchain.ready", "Lean toolchain is ready", toolchain=toolchain)
+        pinned_spec = self._pin_tags(spec)
+        root_lakefile = generate_lakefile(pinned_spec)
+        root_module = generate_root_module(spec)
+        resolution_root = self.store.home / "resolution"
+        resolution_root.mkdir(parents=True, exist_ok=True)
+        with tempfile.TemporaryDirectory(prefix="resolve-", dir=resolution_root) as raw:
+            workspace = Path(raw)
+            (workspace / "lean-toolchain").write_text(toolchain + "\n", encoding="utf-8")
+            (workspace / "lakefile.toml").write_text(root_lakefile, encoding="utf-8")
+            (workspace / f"{ROOT_MODULE}.lean").write_text(root_module, encoding="utf-8")
+            command = self.toolchains.command(toolchain, "lake", "update")
+            self.events.emit("resolution.lake_started", "Lake dependency resolution started")
+            try:
+                process = subprocess.run(
+                    command,
+                    cwd=workspace,
+                    env=self.toolchains.environment,
+                    text=True,
+                    capture_output=True,
+                    timeout=timeout,
+                    check=False,
+                )
+            except subprocess.TimeoutExpired as exc:
+                raise ResolutionError(
+                    f"Lake resolution exceeded {timeout:g} seconds",
+                    command=tuple(command),
+                    exit_code=124,
+                    output=str(exc.stdout or "") + str(exc.stderr or ""),
+                ) from exc
+            if process.returncode:
+                raise ResolutionError(
+                    "Lake could not resolve the environment specification",
+                    command=tuple(command),
+                    exit_code=process.returncode,
+                    output=process.stdout + process.stderr,
+                )
+            manifest_path = workspace / "lake-manifest.json"
+            if not manifest_path.is_file():
+                raise ResolutionError("Lake resolution did not produce lake-manifest.json")
+            manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+            if not isinstance(manifest, dict):
+                raise ResolutionError("Lake manifest is not a JSON object")
+            packages = self._lock_packages(workspace, manifest, spec, toolchain)
+            lock = EnvironmentLock(
+                toolchain=toolchain,
+                spec_digest=spec.spec_digest,
+                root_lakefile=root_lakefile,
+                root_module=root_module,
+                manifest=manifest,
+                packages=packages,
+            )
+            self.store.publish_lock(lock)
+            self.events.emit(
+                "resolution.completed",
+                "Environment lock published",
+                lock_id=lock.lock_id,
+                packages=len(packages),
+            )
+            return lock
+
+    def _pin_tags(self, spec: EnvironmentSpec) -> EnvironmentSpec:
+        pinned: list[GitPackage] = []
+        for package in spec.packages:
+            if package.revision_kind == "commit":
+                pinned.append(package)
+                continue
+            reference = f"refs/tags/{package.rev}"
+            command = ["git", "ls-remote", package.url, reference, f"{reference}^{{}}"]
+            try:
+                process = subprocess.run(
+                    command,
+                    text=True,
+                    capture_output=True,
+                    timeout=60,
+                    check=False,
+                )
+            except subprocess.TimeoutExpired as exc:
+                raise ResolutionError(
+                    f"tag lookup timed out for package {package.name!r}",
+                    phase="tag-resolution",
+                    command=tuple(command),
+                    exit_code=124,
+                    output=str(exc.stdout or "") + str(exc.stderr or ""),
+                ) from exc
+            if process.returncode:
+                raise ResolutionError(
+                    f"could not resolve tag {package.rev!r} for package {package.name!r}",
+                    phase="tag-resolution",
+                    command=tuple(command),
+                    exit_code=process.returncode,
+                    output=process.stdout + process.stderr,
+                )
+            candidates: dict[str, str] = {}
+            for line in process.stdout.splitlines():
+                fields = line.split(maxsplit=1)
+                if len(fields) == 2:
+                    candidates[fields[1]] = fields[0].lower()
+            revision = candidates.get(f"{reference}^{{}}", candidates.get(reference, ""))
+            if _COMMIT.fullmatch(revision) is None:
+                raise ResolutionError(
+                    f"Git tag {package.rev!r} was not found for package {package.name!r}",
+                    phase="tag-resolution",
+                    command=tuple(command),
+                    exit_code=1,
+                    output=process.stdout + process.stderr,
+                )
+            pinned.append(replace(package, rev=revision, revision_kind="commit"))
+            self.events.emit(
+                "tag.resolved",
+                f"Resolved {package.rev} for {package.name}",
+                package=package.name,
+                tag=package.rev,
+                revision=revision,
+            )
+        return EnvironmentSpec(spec.toolchain, tuple(pinned))
+
+    def _lock_packages(
+        self,
+        workspace: Path,
+        manifest: dict[str, Any],
+        spec: EnvironmentSpec,
+        toolchain: str,
+    ) -> tuple[LockedPackage, ...]:
+        direct: dict[str, GitPackage] = {package.name: package for package in spec.packages}
+        packages_dir = workspace / str(manifest.get("packagesDir", ".lake/packages"))
+        raw_packages = manifest.get("packages", [])
+        if not isinstance(raw_packages, list):
+            raise ResolutionError("Lake manifest packages field is not an array")
+        locked: list[LockedPackage] = []
+        for value in raw_packages:
+            if not isinstance(value, dict) or value.get("type") != "git":
+                raise ResolutionError(
+                    "the initial lock schema supports only Git transitive dependencies",
+                    phase="lock-validation",
+                )
+            name = str(value.get("name", ""))
+            revision = str(value.get("rev", "")).lower()
+            if _COMMIT.fullmatch(revision) is None:
+                raise ResolutionError(
+                    f"Lake did not resolve package {name!r} to a full Git commit",
+                    phase="lock-validation",
+                )
+            checkout = packages_dir / name
+            if not checkout.is_dir():
+                raise ResolutionError(
+                    f"resolved package checkout is missing: {name}", phase="acquisition"
+                )
+            head = _git(checkout, "rev-parse", "HEAD").lower()
+            if head != revision:
+                raise ResolutionError(
+                    f"package {name!r} checkout does not match its manifest revision",
+                    phase="lock-validation",
+                )
+            tree = _git(checkout, "rev-parse", "HEAD^{tree}").lower()
+            package_root = checkout
+            subdir = value.get("subDir")
+            if isinstance(subdir, str) and subdir:
+                package_root /= subdir
+            requested = direct.get(name)
+            declared_toolchain = package_root / "lean-toolchain"
+            if declared_toolchain.is_file():
+                declared = declared_toolchain.read_text(encoding="utf-8").strip()
+                if declared and normalize_toolchain(declared) != toolchain:
+                    self.events.emit(
+                        "compatibility.toolchain_difference",
+                        f"{name} declares {declared}; compatibility will be tested by the build",
+                        package=name,
+                        declared_toolchain=declared,
+                        environment_toolchain=toolchain,
+                        direct=requested is not None,
+                    )
+            url = str(value.get("url", ""))
+            source_id = sha256_id(
+                "source", {"source": "git", "url": url, "revision": revision, "tree": tree}
+            )
+            metadata = {
+                "schema": "lean-runtime-git-source/1",
+                "source_id": source_id,
+                "name": name,
+                "url": url,
+                "revision": revision,
+                "tree_hash": tree,
+            }
+            self.store.publish_source(checkout, source_id, metadata)
+            self.events.emit(
+                "source.locked",
+                f"Locked {name}",
+                package=name,
+                revision=revision,
+                source_id=source_id,
+            )
+            locked.append(
+                LockedPackage(
+                    name=name,
+                    url=url,
+                    requested_revision=(
+                        (
+                            requested.rev.lower()
+                            if requested.revision_kind == "commit"
+                            else requested.rev
+                        )
+                        if requested
+                        else value.get("inputRev")
+                    ),
+                    revision=revision,
+                    tree_hash=tree,
+                    source_id=source_id,
+                    inherited=bool(value.get("inherited", False)),
+                    subdir=subdir if isinstance(subdir, str) else None,
+                    root_module=requested.module if requested else None,
+                    artifact_command=requested.artifact_command if requested else (),
+                )
+            )
+        missing = sorted(set(direct) - {package.name for package in locked})
+        if missing:
+            raise ResolutionError(
+                "direct packages missing from Lake manifest: " + ", ".join(missing),
+                phase="lock-validation",
+            )
+        return tuple(sorted(locked, key=lambda package: package.name))