nethackers 0.6.22__py3-none-any.whl

nethackers/__init__.py

@@ -0,0 +1,6 @@
+"""NetHackers distributed symbolic-agent evolution infrastructure."""
+
+from nethackers.constants import BASELINE_COMMIT, BASELINE_REPOSITORY
+
+__all__ = ["BASELINE_COMMIT", "BASELINE_REPOSITORY"]
+__version__ = "0.6.22"

nethackers/_runtime_pin.py

@@ -0,0 +1,3 @@
+"""Generated release pin for the source runtime fetched by the compact wheel."""
+
+SOURCE_RUNTIME_DIGEST = "sha256:826b9a11f2fb1aa48bbd5f7e6d940437ba942adef3d112187f434ff96bfa5bd6"

nethackers/bootstrap.py

@@ -0,0 +1,507 @@
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import tarfile
+import tempfile
+import urllib.error
+import urllib.request
+from pathlib import Path, PurePosixPath
+
+from rich.console import Console
+from rich.panel import Panel
+
+from nethackers import __version__
+from nethackers._runtime_pin import SOURCE_RUNTIME_DIGEST
+
+SOURCE_REPOSITORY = "https://github.com/dunnolab/nethackers.git"
+SOURCE_REVISION = f"v{__version__}"
+RUNTIME_ARCHIVE_URL = (
+    f"https://nethackers.dunnolab.ai/dist/nethackers-runtime-{__version__}.tar.gz"
+)
+_RUNTIME_PIN_PATH = "src/nethackers/_runtime_pin.py"
+_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
+_MAX_RUNTIME_ARCHIVE_BYTES = 16 * 1024 * 1024
+_MAX_RUNTIME_MEMBERS = 512
+_MAX_RUNTIME_UNPACKED_BYTES = 64 * 1024 * 1024
+_COMMANDS = {
+    "ascend",
+    "baseline",
+    "candidate",
+    "evaluate",
+    "join",
+    "login",
+    "logs",
+    "logout",
+    "search",
+    "session",
+    "status",
+    "stop",
+    "verify",
+    "whoami",
+}
+
+
+class BootstrapError(RuntimeError):
+    """Raised when the compact runtime checkout cannot be prepared."""
+
+
+def _is_runtime_checkout(path: Path) -> bool:
+    return all(
+        (path / relative).is_file()
+        for relative in (
+            "roots/autoascend/agent.py",
+            "harness/evolution/problem/task.txt",
+            "template/README.md",
+            "pyproject.toml",
+        )
+    )
+
+
+def _source_checkout() -> Path | None:
+    candidate = Path(__file__).resolve().parents[2]
+    if _is_runtime_checkout(candidate):
+        return candidate.resolve()
+    return None
+
+
+def _runtime_home() -> Path:
+    configured = os.environ.get("NETHACKERS_HOME")
+    if configured:
+        return Path(configured).expanduser().resolve()
+    cache_home = Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache"))
+    return (cache_home / "nethackers").resolve()
+
+
+def _state_home() -> Path:
+    configured = os.environ.get("NETHACKERS_STATE_DIR")
+    if configured:
+        return Path(configured).expanduser().resolve()
+    state_home = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local" / "state"))
+    return (state_home / "nethackers").resolve()
+
+
+def _clone_command(source: str, revision: str, destination: Path) -> list[str]:
+    github_source = source.rstrip("/").removesuffix(".git")
+    gh_authenticated = False
+    if github_source == "https://github.com/dunnolab/nethackers" and shutil.which("gh"):
+        try:
+            status = subprocess.run(
+                ["gh", "auth", "status", "--hostname", "github.com"],
+                check=False,
+                capture_output=True,
+                text=True,
+            )
+            gh_authenticated = status.returncode == 0
+        except OSError:
+            pass
+    if gh_authenticated:
+        return [
+            "gh",
+            "repo",
+            "clone",
+            "dunnolab/nethackers",
+            str(destination),
+            "--",
+            "--branch",
+            revision,
+            "--depth",
+            "1",
+            "--filter=blob:none",
+        ]
+    return [
+        "git",
+        "clone",
+        "--branch",
+        revision,
+        "--depth",
+        "1",
+        "--filter=blob:none",
+        source,
+        str(destination),
+    ]
+
+
+def _run_checked(command: list[str]) -> None:
+    try:
+        result = subprocess.run(command, capture_output=True, text=True, check=False)
+    except OSError as exc:
+        raise BootstrapError(f"could not run {command[0]!r}: {exc}") from exc
+    if result.returncode == 0:
+        return
+    detail = (result.stderr or result.stdout).strip().splitlines()
+    reason = detail[-1] if detail else f"exit status {result.returncode}"
+    raise BootstrapError(f"{command[0]} failed: {reason}")
+
+
+def runtime_digest(path: Path) -> str:
+    """Digest the exact Git index while excluding only the generated pin itself."""
+    try:
+        result = subprocess.run(
+            ["git", "-C", str(path), "ls-files", "--stage", "-z"],
+            capture_output=True,
+            check=False,
+        )
+    except OSError as exc:
+        raise BootstrapError(f"could not inspect the source runtime: {exc}") from exc
+    if result.returncode != 0:
+        detail = result.stderr.decode(errors="replace").strip().splitlines()
+        reason = detail[-1] if detail else f"exit status {result.returncode}"
+        raise BootstrapError(f"could not inspect the source runtime: {reason}")
+    digest = hashlib.sha256()
+    entries = 0
+    for record in result.stdout.split(b"\0"):
+        if not record:
+            continue
+        metadata, separator, path_bytes = record.partition(b"\t")
+        fields = metadata.split()
+        if not separator or len(fields) != 3:
+            raise BootstrapError("source runtime has an invalid Git index")
+        try:
+            relative = path_bytes.decode("utf-8")
+        except UnicodeDecodeError as exc:
+            raise BootstrapError("source runtime contains a non-UTF-8 path") from exc
+        if relative == _RUNTIME_PIN_PATH:
+            continue
+        mode, object_id, stage = fields
+        if stage != b"0" or mode not in {b"100644", b"100755"}:
+            raise BootstrapError("source runtime contains an unsupported Git index entry")
+        digest.update(mode + b"\0" + object_id + b"\0" + path_bytes + b"\0")
+        entries += 1
+    if entries < 1:
+        raise BootstrapError("source runtime Git index is empty")
+    return f"sha256:{digest.hexdigest()}"
+
+
+def runtime_directory_digest(path: Path) -> str:
+    """Digest a release tree with the same entries used by Git's index."""
+    digest = hashlib.sha256()
+    entries = 0
+    for item in sorted(
+        path.rglob("*"),
+        key=lambda value: value.relative_to(path).as_posix(),
+    ):
+        relative = item.relative_to(path).as_posix()
+        if relative == _RUNTIME_PIN_PATH:
+            continue
+        if item.is_symlink():
+            raise BootstrapError(f"source runtime contains a symbolic link: {relative}")
+        if item.is_dir():
+            continue
+        if not item.is_file():
+            raise BootstrapError(f"source runtime contains an unsupported entry: {relative}")
+        try:
+            path_bytes = relative.encode("utf-8")
+        except UnicodeEncodeError as exc:
+            raise BootstrapError("source runtime contains a non-UTF-8 path") from exc
+        payload = item.read_bytes()
+        header = f"blob {len(payload)}\0".encode()
+        object_id = hashlib.sha1(
+            header + payload,
+            usedforsecurity=False,
+        ).hexdigest().encode()
+        mode = b"100755" if item.stat().st_mode & 0o111 else b"100644"
+        digest.update(mode + b"\0" + object_id + b"\0" + path_bytes + b"\0")
+        entries += 1
+    if entries < 1:
+        raise BootstrapError("source runtime directory is empty")
+    return f"sha256:{digest.hexdigest()}"
+
+
+def _download_runtime_archive(source: str, destination: Path) -> None:
+    if not source.startswith("https://"):
+        raise BootstrapError("runtime archive URL must use HTTPS")
+    request = urllib.request.Request(
+        source,
+        headers={"User-Agent": f"nethackers/{__version__}"},
+    )
+    try:
+        response = urllib.request.urlopen(request, timeout=60)
+    except (OSError, urllib.error.URLError) as exc:
+        raise BootstrapError(f"could not download the NetHackers runtime: {exc}") from exc
+    with response:
+        final_url = response.geturl()
+        if not final_url.startswith("https://"):
+            raise BootstrapError("runtime archive redirected away from HTTPS")
+        content_length = response.headers.get("Content-Length")
+        if content_length is not None:
+            try:
+                declared_size = int(content_length)
+            except ValueError as exc:
+                raise BootstrapError("runtime archive has an invalid Content-Length") from exc
+            if not 0 < declared_size <= _MAX_RUNTIME_ARCHIVE_BYTES:
+                raise BootstrapError("runtime archive exceeds the download limit")
+        downloaded = 0
+        with destination.open("xb") as output:
+            while chunk := response.read(1024 * 1024):
+                downloaded += len(chunk)
+                if downloaded > _MAX_RUNTIME_ARCHIVE_BYTES:
+                    raise BootstrapError("runtime archive exceeds the download limit")
+                output.write(chunk)
+    if downloaded == 0:
+        raise BootstrapError("runtime archive is empty")
+
+
+def _archive_relative_path(name: str) -> Path | None:
+    if "\\" in name or "\0" in name or ":" in name:
+        raise BootstrapError("runtime archive contains an unsafe path")
+    pure = PurePosixPath(name)
+    if pure.is_absolute() or ".." in pure.parts:
+        raise BootstrapError("runtime archive contains an unsafe path")
+    parts = tuple(part for part in pure.parts if part not in {"", "."})
+    return Path(*parts) if parts else None
+
+
+def _extract_runtime_archive(archive: Path, destination: Path) -> None:
+    try:
+        destination.mkdir(mode=0o700)
+        with tarfile.open(archive, mode="r|gz") as bundle:
+            member_count = 0
+            unpacked_size = 0
+            normalized: set[Path] = set()
+            for member in bundle:
+                member_count += 1
+                if member_count > _MAX_RUNTIME_MEMBERS:
+                    raise BootstrapError("runtime archive contains too many entries")
+                relative = _archive_relative_path(member.name)
+                if relative is None:
+                    if not member.isdir():
+                        raise BootstrapError("runtime archive contains an invalid root entry")
+                    continue
+                if relative in normalized:
+                    raise BootstrapError("runtime archive contains duplicate paths")
+                normalized.add(relative)
+                target = destination / relative
+                if member.isdir():
+                    target.mkdir(parents=True, exist_ok=True, mode=0o755)
+                    target.chmod(0o755)
+                    continue
+                if not member.isfile() or (member.mode & 0o777) not in {0o644, 0o755}:
+                    raise BootstrapError("runtime archive contains an unsupported entry")
+                unpacked_size += member.size
+                if unpacked_size > _MAX_RUNTIME_UNPACKED_BYTES:
+                    raise BootstrapError("runtime archive exceeds the unpacked size limit")
+                target.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
+                source = bundle.extractfile(member)
+                if source is None:
+                    raise BootstrapError("runtime archive file cannot be read")
+                with source, target.open("xb") as output:
+                    shutil.copyfileobj(source, output)
+                target.chmod(member.mode & 0o777)
+    except BootstrapError:
+        raise
+    except (OSError, tarfile.TarError) as exc:
+        raise BootstrapError(f"runtime archive is invalid: {exc}") from exc
+
+
+def _verify_runtime(path: Path) -> None:
+    if not _DIGEST_RE.fullmatch(SOURCE_RUNTIME_DIGEST):
+        raise BootstrapError("this NetHackers wheel has no immutable source-runtime pin")
+    actual = (
+        runtime_digest(path)
+        if (path / ".git").exists()
+        else runtime_directory_digest(path)
+    )
+    if actual != SOURCE_RUNTIME_DIGEST:
+        raise BootstrapError(
+            f"source runtime digest is {actual}, expected {SOURCE_RUNTIME_DIGEST}"
+        )
+    runtime_pin = path / _RUNTIME_PIN_PATH
+    installed_pin = Path(__file__).with_name("_runtime_pin.py").read_bytes()
+    if not runtime_pin.is_file() or runtime_pin.read_bytes() != installed_pin:
+        raise BootstrapError("source runtime pin does not match the installed wheel")
+
+
+def prepare_runtime(*, console: Console | None = None) -> Path:
+    override = os.environ.get("NETHACKERS_RUNTIME_ROOT")
+    if override:
+        runtime = Path(override).expanduser().resolve()
+        if not _is_runtime_checkout(runtime):
+            raise BootstrapError(f"NETHACKERS_RUNTIME_ROOT is not a NetHackers checkout: {runtime}")
+        return runtime
+
+    source_checkout = _source_checkout()
+    if source_checkout is not None:
+        return source_checkout
+
+    runtime = _runtime_home() / "runtime" / __version__
+    if _is_runtime_checkout(runtime):
+        try:
+            _verify_runtime(runtime)
+        except BootstrapError:
+            shutil.rmtree(runtime)
+        else:
+            return runtime
+    if runtime.exists():
+        shutil.rmtree(runtime)
+
+    source = os.environ.get("NETHACKERS_SOURCE_URL")
+    archive_source = os.environ.get("NETHACKERS_RUNTIME_URL", RUNTIME_ARCHIVE_URL)
+    revision = os.environ.get("NETHACKERS_SOURCE_REVISION", SOURCE_REVISION)
+    runtime.parent.mkdir(parents=True, exist_ok=True)
+    temporary = Path(tempfile.mkdtemp(prefix=f"{__version__}-", dir=runtime.parent))
+    shutil.rmtree(temporary)
+    status = (console or Console()).status(
+        "[bold green]Preparing the compact AutoAscend evolution runtime..."
+    )
+    archive = temporary.with_name(f"{temporary.name}.tar.gz")
+    try:
+        with status:
+            if source is not None:
+                if shutil.which("git") is None:
+                    raise BootstrapError("git is required for NETHACKERS_SOURCE_URL")
+                clone_command = _clone_command(source, revision, temporary)
+                try:
+                    _run_checked(clone_command)
+                except BootstrapError:
+                    if clone_command[0] != "gh":
+                        raise
+                    shutil.rmtree(temporary, ignore_errors=True)
+                    _run_checked(
+                        [
+                            "git",
+                            "clone",
+                            "--branch",
+                            revision,
+                            "--depth",
+                            "1",
+                            "--filter=blob:none",
+                            source,
+                            str(temporary),
+                        ]
+                    )
+            else:
+                _download_runtime_archive(archive_source, archive)
+                _extract_runtime_archive(archive, temporary)
+            if not temporary.exists():
+                raise BootstrapError("runtime preparation produced no checkout")
+            _verify_runtime(temporary)
+            if runtime.exists():
+                shutil.rmtree(temporary)
+            else:
+                temporary.replace(runtime)
+    except Exception:
+        shutil.rmtree(temporary, ignore_errors=True)
+        raise
+    finally:
+        archive.unlink(missing_ok=True)
+    if not _is_runtime_checkout(runtime):
+        raise BootstrapError("the downloaded NetHackers runtime is incomplete")
+    return runtime
+
+
+def _run_without_runtime(arguments: list[str]) -> bool:
+    if not arguments:
+        return False
+    if any(argument in {"-h", "--help"} for argument in arguments):
+        return True
+    if arguments[0] == "session":
+        return len(arguments) > 1 and arguments[1] != "start"
+    return arguments[0] in {
+        "-h",
+        "--help",
+        "--version",
+        "logs",
+        "login",
+        "logout",
+        "search",
+        "status",
+        "stop",
+        "whoami",
+    }
+
+
+def _normalize_arguments(arguments: list[str]) -> list[str]:
+    if not arguments:
+        return ["session", "start"]
+    if arguments[0] in _COMMANDS or arguments[0] in {"-h", "--help", "--version"}:
+        return arguments
+    return ["session", "start", *arguments]
+
+
+def _signal_runtime(process: subprocess.Popen[bytes], signal_number: int) -> None:
+    try:
+        if os.name == "posix":
+            os.killpg(process.pid, signal_number)
+        else:
+            process.send_signal(signal_number)
+    except ProcessLookupError:
+        pass
+
+
+def _wait_for_runtime(process: subprocess.Popen[bytes], console: Console) -> int:
+    try:
+        returncode = process.wait()
+    except KeyboardInterrupt:
+        console.print(
+            "[yellow][STOP][/] Finishing and publishing the best checkpoint; "
+            "press Ctrl+C again to force exit."
+        )
+        _signal_runtime(process, signal.SIGINT)
+    try:
+        returncode = process.wait()
+    except KeyboardInterrupt:
+        console.print("[red][STOP][/] Forcing the NetHackers runtime to exit.")
+        _signal_runtime(process, signal.SIGTERM)
+        try:
+            returncode = process.wait(timeout=10)
+        except subprocess.TimeoutExpired:
+            _signal_runtime(process, signal.SIGKILL)
+            returncode = process.wait()
+        return 130
+    return 130 if returncode < 0 else returncode
+
+
+def main(argv: list[str] | None = None) -> int:
+    if argv is None and "NETHACKERS_LAUNCHER" not in os.environ:
+        resolved_launcher = shutil.which(sys.argv[0])
+        if resolved_launcher is not None:
+            os.environ["NETHACKERS_LAUNCHER"] = str(
+                Path(resolved_launcher).expanduser().resolve()
+            )
+    arguments = _normalize_arguments(list(sys.argv[1:] if argv is None else argv))
+    invocation_directory = Path.cwd().resolve()
+    if _run_without_runtime(arguments):
+        from nethackers.cli import main as cli_main
+
+        return cli_main(arguments)
+
+    source_checkout = _source_checkout()
+    console = Console()
+    try:
+        runtime = prepare_runtime(console=console)
+    except BootstrapError as exc:
+        console.print(Panel(str(exc), title="[bold red]NETHACKERS SETUP FAILED[/]"))
+        return 2
+
+    if source_checkout is not None and runtime == source_checkout:
+        from nethackers.cli import main as cli_main
+
+        return cli_main(arguments)
+
+    environment = dict(os.environ)
+    environment["NETHACKERS_RUNTIME_ROOT"] = str(runtime)
+    environment["NETHACKERS_STATE_DIR"] = str(_state_home())
+    environment.setdefault("NETHACKERS_WORKSPACE", str(invocation_directory))
+    try:
+        process = subprocess.Popen(
+            [sys.executable, "-I", "-m", "nethackers.cli", *arguments],
+            cwd=invocation_directory,
+            env=environment,
+            start_new_session=os.name == "posix",
+        )
+    except OSError as exc:
+        console.print(
+            Panel(str(exc), title="[bold red]NETHACKERS START FAILED[/]")
+        )
+        return 2
+    return _wait_for_runtime(process, console)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())