lean-runtime 0.6.0__py3-none-any.whl

lean_runtime/store.py

@@ -0,0 +1,462 @@
+"""Content-addressed storage for locks, sources, and environments."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import platform
+import re
+import shutil
+import subprocess
+import time
+import uuid
+from collections.abc import Iterator
+from contextlib import contextmanager, suppress
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .errors import EnvironmentError
+from .lockfiles import EnvironmentLock
+from .locking import FileLock
+from .serialization import sha256_id, write_json_atomic
+
+STORE_SCHEMA = "lean-runtime-store/1"
+_ALIAS = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}")
+_ENVIRONMENT_ID = re.compile(r"env_[0-9a-f]{64}")
+ALIAS_SCHEMA = "lean-runtime-environment-alias/1"
+SUPPORTED_BUILD_PROFILE = "release"
+
+
+def _snapshot_digest(root: Path) -> str:
+    """Hash checked-out content while excluding Git and runtime metadata."""
+    digest = hashlib.sha256()
+    for directory, directories, filenames in os.walk(root, followlinks=False):
+        current = Path(directory)
+        symlink_directories = [name for name in directories if (current / name).is_symlink()]
+        directories[:] = sorted(
+            name
+            for name in directories
+            if name not in {".git", ".lake"} and name not in symlink_directories
+        )
+        for name in sorted([*filenames, *symlink_directories]):
+            path = current / name
+            relative = path.relative_to(root)
+            if relative.as_posix() == ".lean-runtime-source.json":
+                continue
+            if path.is_symlink():
+                digest.update(b"link\0" + relative.as_posix().encode() + b"\0")
+                digest.update(os.readlink(path).encode())
+            elif path.is_file():
+                mode = path.stat().st_mode & 0o111
+                digest.update(
+                    b"file\0" + relative.as_posix().encode() + b"\0" + str(mode).encode() + b"\0"
+                )
+                with path.open("rb") as handle:
+                    for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+                        digest.update(chunk)
+    return "sha256:" + digest.hexdigest()
+
+
+def platform_record() -> dict[str, str]:
+    return {
+        "system": platform.system().lower(),
+        "machine": platform.machine().lower(),
+        "python_platform": platform.platform(),
+    }
+
+
+def environment_identity(lock: EnvironmentLock, build_profile: str = "release") -> str:
+    if build_profile != SUPPORTED_BUILD_PROFILE:
+        raise EnvironmentError(
+            f"unsupported build profile {build_profile!r}; only 'release' is implemented"
+        )
+    return sha256_id(
+        "env",
+        {
+            "schema": STORE_SCHEMA,
+            "lock_id": lock.lock_id,
+            "platform": platform_record(),
+            "build_profile": build_profile,
+        },
+    )
+
+
+def clone_tree(source: Path, destination: Path) -> None:
+    """Clone a tree using copy-on-write where the host supports it."""
+    if destination.exists():
+        raise EnvironmentError(f"clone destination already exists: {destination}")
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    if platform.system() == "Darwin":
+        result = subprocess.run(
+            ["cp", "-cR", str(source), str(destination)],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
+        if result.returncode == 0:
+            return
+    elif platform.system() == "Linux":
+        result = subprocess.run(
+            ["cp", "--reflink=auto", "-a", str(source), str(destination)],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
+        if result.returncode == 0:
+            return
+    shutil.copytree(source, destination, symlinks=True)
+
+
+@dataclass(frozen=True, slots=True)
+class GarbageCollectionReport:
+    candidates: tuple[str, ...]
+    removed: tuple[str, ...]
+    retained: tuple[str, ...]
+    dry_run: bool
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "candidates": list(self.candidates),
+            "removed": list(self.removed),
+            "retained": list(self.retained),
+            "dry_run": self.dry_run,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class StoreStatus:
+    home: str
+    environments: int
+    locks: int
+    sources: int
+    executions: int
+    aliases: int
+    bytes_used: int
+    bytes_free: int
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "home": self.home,
+            "environments": self.environments,
+            "locks": self.locks,
+            "sources": self.sources,
+            "executions": self.executions,
+            "aliases": self.aliases,
+            "bytes_used": self.bytes_used,
+            "bytes_free": self.bytes_free,
+        }
+
+
+class EnvironmentStore:
+    """Filesystem-backed, atomically published content-addressed store."""
+
+    def __init__(self, home: Path) -> None:
+        self.home = home
+        self.sources = home / "sources" / "git"
+        self.locks = home / "locks"
+        self.environments = home / "environments"
+        self.names = home / "names"
+        self.jobs = home / "jobs"
+        self.executions = home / "executions"
+        self.usage = home / "usage"
+        self.leases = home / "leases"
+        self.lock_dir = home / ".locks"
+        for path in (
+            self.sources,
+            self.locks,
+            self.environments,
+            self.names,
+            self.jobs,
+            self.executions,
+            self.usage,
+            self.leases,
+            self.lock_dir,
+        ):
+            path.mkdir(parents=True, exist_ok=True)
+
+    def lock_path(self, lock_id: str) -> Path:
+        return self.locks / lock_id / "environment.lock.json"
+
+    def publish_lock(self, lock: EnvironmentLock) -> Path:
+        destination = self.lock_path(lock.lock_id)
+        with FileLock(self.lock_dir / f"{lock.lock_id}.lock"):
+            if destination.is_file():
+                EnvironmentLock.load(destination)
+                return destination
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            lock.write(destination)
+        return destination
+
+    def load_lock(self, lock_id: str) -> EnvironmentLock:
+        path = self.lock_path(lock_id)
+        if not path.is_file():
+            raise EnvironmentError(f"environment lock is not present: {lock_id}")
+        return EnvironmentLock.load(path)
+
+    def source_path(self, source_id: str) -> Path:
+        return self.sources / source_id / "source"
+
+    def validate_source(
+        self,
+        source_id: str,
+        *,
+        url: str,
+        revision: str,
+        tree_hash: str,
+    ) -> Path:
+        source = self.source_path(source_id)
+        marker = source / ".lean-runtime-source.json"
+        if not source.is_dir() or not marker.is_file():
+            raise EnvironmentError(f"immutable source snapshot is incomplete: {source_id}")
+        metadata = json.loads(marker.read_text(encoding="utf-8"))
+        expected = {
+            "source_id": source_id,
+            "url": url,
+            "revision": revision,
+            "tree_hash": tree_hash,
+        }
+        if any(metadata.get(key) != value for key, value in expected.items()):
+            raise EnvironmentError(f"immutable source metadata mismatch: {source_id}")
+        content_hash = metadata.get("content_hash")
+        if isinstance(content_hash, str) and _snapshot_digest(source) != content_hash:
+            raise EnvironmentError(f"immutable source content was modified: {source_id}")
+        commands = (("rev-parse", "HEAD"), ("rev-parse", "HEAD^{tree}"))
+        observed = []
+        for arguments in commands:
+            process = subprocess.run(
+                ["git", "-C", str(source), *arguments],
+                text=True,
+                capture_output=True,
+                check=False,
+            )
+            if process.returncode:
+                raise EnvironmentError(f"immutable source Git metadata is invalid: {source_id}")
+            observed.append(process.stdout.strip().lower())
+        if observed != [revision.lower(), tree_hash.lower()]:
+            raise EnvironmentError(f"immutable source content mismatch: {source_id}")
+        return source
+
+    def publish_source(self, checkout: Path, source_id: str, metadata: dict[str, Any]) -> Path:
+        destination = self.source_path(source_id)
+        with FileLock(self.lock_dir / f"{source_id}.lock"):
+            if destination.is_dir():
+                self.validate_source(
+                    source_id,
+                    url=str(metadata["url"]),
+                    revision=str(metadata["revision"]),
+                    tree_hash=str(metadata["tree_hash"]),
+                )
+                return destination
+            parent = destination.parent
+            parent.mkdir(parents=True, exist_ok=True)
+            stage = parent / f".staging-{os.getpid()}-{uuid.uuid4().hex}"
+            try:
+                command = [
+                    "git",
+                    "clone",
+                    "--quiet",
+                    "--no-local",
+                    "--depth",
+                    "1",
+                    "--no-tags",
+                    checkout.resolve().as_uri(),
+                    str(stage),
+                ]
+                process = subprocess.run(
+                    command,
+                    text=True,
+                    capture_output=True,
+                    check=False,
+                )
+                if process.returncode:
+                    raise EnvironmentError(
+                        f"could not create compact source snapshot {source_id}: "
+                        + process.stdout
+                        + process.stderr
+                    )
+                remote = subprocess.run(
+                    [
+                        "git",
+                        "-C",
+                        str(stage),
+                        "remote",
+                        "set-url",
+                        "origin",
+                        str(metadata["url"]),
+                    ],
+                    text=True,
+                    capture_output=True,
+                    check=False,
+                )
+                if remote.returncode:
+                    raise EnvironmentError(
+                        f"could not normalize source remote {source_id}: "
+                        + remote.stdout
+                        + remote.stderr
+                    )
+                snapshot_metadata = {**metadata, "content_hash": _snapshot_digest(stage)}
+                write_json_atomic(stage / ".lean-runtime-source.json", snapshot_metadata)
+                stage.replace(destination)
+            finally:
+                if stage.exists():
+                    shutil.rmtree(stage)
+        return destination
+
+    def environment_path(self, environment_id: str) -> Path:
+        return self.environments / environment_id
+
+    def validate_environment_id(self, environment_id: str) -> str:
+        if _ENVIRONMENT_ID.fullmatch(environment_id) is None:
+            raise EnvironmentError(f"invalid environment identity: {environment_id!r}")
+        return environment_id
+
+    def touch_environment(self, environment_id: str) -> None:
+        self.validate_environment_id(environment_id)
+        write_json_atomic(
+            self.usage / f"{environment_id}.json",
+            {"environment_id": environment_id, "last_used_at": time.time()},
+        )
+
+    @contextmanager
+    def execution_lease(self, environment_id: str) -> Iterator[None]:
+        """Prevent GC during a clone without serializing concurrent clones."""
+        self.validate_environment_id(environment_id)
+        lease_directory = self.leases / environment_id
+        lease = lease_directory / f"{os.getpid()}-{uuid.uuid4().hex}.json"
+        with FileLock(self.lock_dir / f"{environment_id}.lock"):
+            if not self.environment_path(environment_id).is_dir():
+                raise EnvironmentError(
+                    f"environment disappeared before execution: {environment_id}"
+                )
+            lease_directory.mkdir(parents=True, exist_ok=True)
+            self.touch_environment(environment_id)
+            write_json_atomic(
+                lease,
+                {"environment_id": environment_id, "pid": os.getpid(), "created_at": time.time()},
+            )
+        try:
+            yield None
+        finally:
+            lease.unlink(missing_ok=True)
+            with suppress(OSError):
+                lease_directory.rmdir()
+
+    def has_execution_leases(self, environment_id: str) -> bool:
+        return any((self.leases / environment_id).glob("*.json"))
+
+    def validate_alias(self, name: str) -> str:
+        if _ALIAS.fullmatch(name) is None:
+            raise EnvironmentError(f"invalid environment name: {name!r}")
+        return name
+
+    def set_alias(self, name: str, environment_id: str) -> None:
+        self.validate_alias(name)
+        self.validate_environment_id(environment_id)
+        record = {
+            "schema": ALIAS_SCHEMA,
+            "name": name,
+            "environment_id": environment_id,
+        }
+        with FileLock(self.lock_dir / f"name-{name}.lock"):
+            write_json_atomic(self.names / f"{name}.json", record)
+
+    def resolve_identifier(self, identifier: str) -> str:
+        if _ENVIRONMENT_ID.fullmatch(identifier) is not None:
+            direct = self.environment_path(identifier)
+            if direct.is_dir():
+                return identifier
+        alias_path = self.names / f"{self.validate_alias(identifier)}.json"
+        if not alias_path.is_file():
+            raise EnvironmentError(f"unknown environment: {identifier}")
+        environment_id = self._read_alias(alias_path, expected_name=identifier)
+        if not self.environment_path(environment_id).is_dir():
+            raise EnvironmentError(f"environment alias is dangling: {identifier}")
+        return environment_id
+
+    def _read_alias(self, path: Path, *, expected_name: str | None = None) -> str:
+        try:
+            value = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError) as exc:
+            raise EnvironmentError(f"invalid environment alias record: {path.name}") from exc
+        if not isinstance(value, dict) or value.get("schema") != ALIAS_SCHEMA:
+            raise EnvironmentError(f"invalid environment alias record: {path.name}")
+        name = value.get("name")
+        environment_id = value.get("environment_id")
+        if not isinstance(name, str) or name != path.stem or name != (expected_name or name):
+            raise EnvironmentError(f"environment alias name mismatch: {path.name}")
+        self.validate_alias(name)
+        if not isinstance(environment_id, str):
+            raise EnvironmentError(f"environment alias has no identity: {name}")
+        return self.validate_environment_id(environment_id)
+
+    def aliases(self) -> dict[str, str]:
+        result: dict[str, str] = {}
+        for path in sorted(self.names.glob("*.json")):
+            result[path.stem] = self._read_alias(path)
+        return result
+
+    def status(self) -> StoreStatus:
+        bytes_used = 0
+        for path in self.home.rglob("*"):
+            try:
+                if path.is_file() and not path.is_symlink():
+                    bytes_used += path.stat().st_size
+            except OSError:
+                continue
+        return StoreStatus(
+            home=str(self.home),
+            environments=sum(1 for path in self.environments.glob("env_*") if path.is_dir()),
+            locks=sum(1 for path in self.locks.glob("lock_*") if path.is_dir()),
+            sources=sum(1 for path in self.sources.glob("source_*") if path.is_dir()),
+            executions=sum(1 for path in self.executions.glob("execution_*.json")),
+            aliases=len(self.aliases()),
+            bytes_used=bytes_used,
+            bytes_free=shutil.disk_usage(self.home).free,
+        )
+
+    def gc(
+        self, *, dry_run: bool = True, minimum_age_seconds: float = 2_592_000
+    ) -> GarbageCollectionReport:
+        """Remove old environments not reachable through a name.
+
+        Locks and immutable sources are retained in the first store schema.
+        """
+        referenced = set(self.aliases().values())
+        now = time.time()
+        candidates: list[str] = []
+        retained: list[str] = []
+        removed: list[str] = []
+        with FileLock(self.lock_dir / "gc.lock"):
+            for path in sorted(self.environments.glob("env_*")):
+                if _ENVIRONMENT_ID.fullmatch(path.name) is None:
+                    retained.append(path.name)
+                    continue
+                usage = self.usage / f"{path.name}.json"
+                age = now - (usage.stat().st_mtime if usage.exists() else path.stat().st_mtime)
+                if (
+                    path.name in referenced
+                    or age < minimum_age_seconds
+                    or self.has_execution_leases(path.name)
+                ):
+                    retained.append(path.name)
+                    continue
+                candidates.append(path.name)
+                if not dry_run:
+                    with FileLock(self.lock_dir / f"{path.name}.lock"):
+                        referenced = set(self.aliases().values())
+                        age = now - (
+                            usage.stat().st_mtime if usage.exists() else path.stat().st_mtime
+                        )
+                        if (
+                            path.name in referenced
+                            or age < minimum_age_seconds
+                            or self.has_execution_leases(path.name)
+                        ):
+                            retained.append(path.name)
+                            continue
+                        shutil.rmtree(path)
+                        usage.unlink(missing_ok=True)
+                        removed.append(path.name)
+        return GarbageCollectionReport(tuple(candidates), tuple(removed), tuple(retained), dry_run)

lean_runtime/toolchains.py

@@ -0,0 +1,164 @@
+"""Private Elan bootstrap and Lean toolchain management."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import stat
+import subprocess
+import tempfile
+import urllib.request
+from pathlib import Path
+
+from .errors import ToolchainError
+
+ELAN_VERSION = "4.2.3"
+ELAN_INIT_URL = f"https://raw.githubusercontent.com/leanprover/elan/v{ELAN_VERSION}/elan-init.sh"
+ELAN_INIT_SHA256 = "a620ff1641616222c8d37c54845492004bb84d6877cdbc944dd65c1aa685bf53"
+
+
+def default_runtime_home() -> Path:
+    """Return the cache root without depending on a platform-specific package."""
+    override = os.environ.get("LEAN_RUNTIME_HOME")
+    if override:
+        return Path(override).expanduser()
+    if os.name == "nt":
+        base = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
+        return base / "lean-runtime"
+    if sys_platform() == "darwin":
+        return Path.home() / "Library" / "Caches" / "lean-runtime"
+    return Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "lean-runtime"
+
+
+def sys_platform() -> str:
+    """Small indirection used by tests."""
+    import sys
+
+    return sys.platform
+
+
+def normalize_toolchain(value: str) -> str:
+    """Normalize common Lean release spellings to an Elan toolchain name."""
+    value = value.strip()
+    if not value:
+        raise ToolchainError("toolchain must not be empty")
+    if "/" in value or ":" in value:
+        return value
+    if value.startswith("v"):
+        return f"leanprover/lean4:{value}"
+    if value[0].isdigit():
+        return f"leanprover/lean4:v{value}"
+    return value
+
+
+class ToolchainManager:
+    """Own an isolated Elan home and install Lean releases on demand.
+
+    Set ``LEAN_RUNTIME_ELAN`` to use an explicit existing Elan executable.
+    This is useful in CI and development; ordinary users get a private copy.
+    """
+
+    def __init__(self, home: str | os.PathLike[str] | None = None) -> None:
+        self.home = Path(home).expanduser().resolve() if home else default_runtime_home()
+        self.elan_home = self.home / "elan"
+
+    @property
+    def environment(self) -> dict[str, str]:
+        env = os.environ.copy()
+        elan_home = Path(os.environ.get("LEAN_RUNTIME_ELAN_HOME", self.elan_home))
+        env["ELAN_HOME"] = str(elan_home)
+        env["PATH"] = os.pathsep.join((str(elan_home / "bin"), env.get("PATH", "")))
+        return env
+
+    def elan_path(self, *, bootstrap: bool = True) -> Path:
+        override = os.environ.get("LEAN_RUNTIME_ELAN")
+        if override:
+            # Elan is commonly installed as symlinks to one multicall binary.
+            # Resolving the link changes argv[0] to `elan-init`, which selects
+            # the installer interface instead of the toolchain manager.
+            path = Path(override).expanduser().absolute()
+            if path.is_file():
+                return path
+            raise ToolchainError(f"LEAN_RUNTIME_ELAN does not name a file: {path}")
+        name = "elan.exe" if os.name == "nt" else "elan"
+        private = self.elan_home / "bin" / name
+        if private.is_file():
+            return private
+        if not bootstrap:
+            raise ToolchainError("private Elan is not installed")
+        self.bootstrap_elan()
+        if not private.is_file():
+            raise ToolchainError("Elan installer completed without creating the executable")
+        return private
+
+    def bootstrap_elan(self) -> Path:
+        """Install Elan under the runtime cache without touching user defaults."""
+        if os.name == "nt":
+            raise ToolchainError(
+                "automatic Elan bootstrap currently supports macOS and Linux; "
+                "set LEAN_RUNTIME_ELAN on Windows"
+            )
+        self.home.mkdir(parents=True, exist_ok=True)
+        with tempfile.TemporaryDirectory(prefix="elan-bootstrap-", dir=self.home) as raw:
+            script = Path(raw) / "elan-init.sh"
+            try:
+                with urllib.request.urlopen(ELAN_INIT_URL, timeout=30) as response:
+                    installer = response.read()
+            except OSError as exc:
+                raise ToolchainError(f"could not download Elan installer: {exc}") from exc
+            observed = hashlib.sha256(installer).hexdigest()
+            if observed != ELAN_INIT_SHA256:
+                raise ToolchainError("downloaded Elan installer failed its SHA-256 integrity check")
+            script.write_bytes(installer)
+            script.chmod(script.stat().st_mode | stat.S_IXUSR)
+            env = self.environment
+            process = subprocess.run(
+                [str(script), "-y", "--no-modify-path", "--default-toolchain", "none"],
+                env=env,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                check=False,
+            )
+        if process.returncode:
+            raise ToolchainError(f"Elan installer exited {process.returncode}:\n{process.stdout}")
+        return self.elan_path(bootstrap=False)
+
+    def is_installed(self, toolchain: str) -> bool:
+        name = normalize_toolchain(toolchain)
+        process = subprocess.run(
+            [str(self.elan_path()), "toolchain", "list"],
+            env=self.environment,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            check=False,
+        )
+        if process.returncode:
+            raise ToolchainError(f"could not list installed Lean toolchains:\n{process.stdout}")
+        installed = {
+            fields[0] for line in process.stdout.splitlines() if (fields := line.split(maxsplit=1))
+        }
+        return name in installed
+
+    def ensure(self, toolchain: str) -> str:
+        """Install a toolchain if necessary and return its normalized name."""
+        name = normalize_toolchain(toolchain)
+        if self.is_installed(name):
+            return name
+        process = subprocess.run(
+            [str(self.elan_path()), "toolchain", "install", name],
+            env=self.environment,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            check=False,
+        )
+        if process.returncode:
+            raise ToolchainError(f"could not install Lean toolchain {name!r}:\n{process.stdout}")
+        return name
+
+    def command(self, toolchain: str, executable: str, *args: str) -> list[str]:
+        """Construct a command pinned to one toolchain."""
+        name = self.ensure(toolchain)
+        return [str(self.elan_path()), "run", name, executable, *args]