abench 0.1.0__py3-none-any.whl

abench/Dockerfile

@@ -0,0 +1,10 @@
+ARG PYTHON_IMAGE=python:3.11-slim-bookworm
+FROM ${PYTHON_IMAGE}
+RUN apt-get update && apt-get install -y --no-install-recommends git build-essential libhdf5-dev \
+    && rm -rf /var/lib/apt/lists/*
+COPY build_sources.py dependencies.json /opt/build/
+RUN python /opt/build/build_sources.py /opt/build/dependencies.json
+ENV PYTHONUNBUFFERED=1 OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \
+    NUMEXPR_NUM_THREADS=1 NUMBA_NUM_THREADS=1 PYTHONHASHSEED=0 DASK_SCHEDULER=synchronous
+WORKDIR /model
+ENTRYPOINT ["python", "/benchmark/worker.py"]

abench/__init__.py

@@ -0,0 +1,3 @@
+"""Reproducible ActivitySim experiments in Linux containers."""
+
+__version__ = "0.1.0"

abench/__main__.py

@@ -0,0 +1,3 @@
+from .cli import entrypoint
+
+entrypoint()

abench/attempts.py

@@ -0,0 +1,66 @@
+"""Retry completed runs that prepared new flow signatures, never model errors."""
+
+import shutil
+
+from .common import write_json
+from .failures import BenchmarkFailure, describe_failure
+from .report import load_run
+
+
+def measured_attempts(spec, output, run_phase, publish_cache):
+    """Select only a compilation-free attempt for the report.
+
+    Each attempt starts in a new container with fresh outputs and model caches.
+    Only compiled flows survive; failed preparation timings never enter reports.
+    """
+    spec["attempts"] = []
+    for number in range(1, spec.get("cache_retries", 2) + 2):
+        model_cache = output / "cache/model"
+        if model_cache.exists():
+            shutil.rmtree(model_cache)
+        staging = output / "attempts" / f"attempt-{number:03d}"
+        relative = str(staging.relative_to(output))
+        record = {"number": number, "directory": relative, "status": "running"}
+        spec["attempts"].append(record)
+        spec["measured_directory"] = relative
+        write_json(output / "experiment.json", spec)
+        print(f"Running measured attempt {number}…", flush=True)
+        try:
+            run_phase(relative)
+            # Check all ordinary validity conditions before deciding to retry.
+            if not load_run(output, allow_cache_misses=True)["valid"]:
+                raise BenchmarkFailure(
+                    describe_failure(output, "measured", ignore_cache=True)
+                )
+        except BaseException:
+            record["status"] = "failed"
+            write_json(output / "experiment.json", spec)
+            raise
+        phase = staging
+        misses = sum(
+            len(path.read_text().splitlines())
+            for path in phase.glob("cache-miss-*.txt")
+        )
+        record.update(
+            compilations=misses,
+            status="cache preparation" if misses else "accepted",
+        )
+        publish_cache()
+        write_json(output / "experiment.json", spec)
+        if not misses:
+            # Convenience alias only after containers finish; attempt directories
+            # never move or change identity while Docker may cache their paths.
+            (output / "measured").symlink_to(relative, target_is_directory=True)
+            return
+        if number > spec.get("cache_retries", 2):
+            raise BenchmarkFailure(
+                f"Flow compilation persisted after {number} completed attempts. "
+                "No valid benchmark was produced. Inspect attempts/*/cache-miss-details-*.jsonl "
+                "for changing signatures; increase --cache-retries if appropriate. "
+                f"Diagnostics: {output}"
+            )
+        print(
+            f"Attempt {number} compiled {misses} flow signatures; retained as cache "
+            f"preparation at {staging}. Retrying with a fresh model state…",
+            flush=True,
+        )

abench/cli.py

@@ -0,0 +1,563 @@
+"""Run reproducible ActivitySim benchmarks with model profiles."""
+
+import argparse
+import hashlib
+import json
+import math
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import uuid
+from contextlib import nullcontext
+from datetime import datetime, timezone
+from pathlib import Path
+
+from . import __version__
+from .attempts import measured_attempts
+from .common import read_json, write_json
+from .failures import BenchmarkFailure, describe_failure
+from .flow_cache import publish_flows, reuse_flows
+from .profiles import load_profile, validate_model
+from .report import load_run, report
+from .sources import resolve_sources
+
+PACKAGE = Path(__file__).resolve().parent
+
+
+def command(args, log=None):
+    """Keep build/run output on disk and propagate failures to the caller."""
+    if log:
+        with log.open("w") as stream:
+            subprocess.run(args, stdout=stream, stderr=subprocess.STDOUT, check=True)
+    else:
+        return subprocess.check_output(args, text=True).strip()
+
+
+def git_info(root, args):
+    """Non-Git model directories are supported; snapshots still capture their files."""
+    result = subprocess.run(
+        ["git", "-C", str(root), *args], capture_output=True, text=True
+    )
+    return result.stdout.strip() if result.returncode == 0 else None
+
+
+def commit(value):
+    if not re.fullmatch(r"[0-9a-fA-F]{40}", value):
+        raise argparse.ArgumentTypeError("provide the full 40-character Git commit SHA")
+    return value.lower()
+
+
+def positive(value):
+    value = float(value)
+    if not math.isfinite(value) or value <= 0:
+        raise argparse.ArgumentTypeError("must be finite and positive")
+    return value
+
+
+def parser():
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        epilog="Named experiments: abench experiments.yaml; preflight: abench validate experiments.yaml",
+    )
+    p.add_argument("--version", action="version", version=f"abench {__version__}")
+    p.add_argument("--model-dir", type=Path, default=Path.cwd())
+    p.add_argument("--profile", default="benchmark.yaml")
+    p.add_argument(
+        "--source",
+        action="append",
+        default=[],
+        help="distribution=organization/repository@40-character-SHA",
+    )
+    p.add_argument("--activitysim-commit", type=commit)
+    p.add_argument("--sharrow-commit", type=commit)
+    mode = p.add_mutually_exclusive_group()
+    mode.add_argument("--single-process", dest="multiprocess", action="store_false")
+    mode.add_argument("--multiprocess", action="store_true")
+    p.set_defaults(multiprocess=False)
+    p.add_argument("--processes", type=int, help="required for --multiprocess")
+    p.add_argument("--sharrow", action=argparse.BooleanOptionalAction, default=True)
+    p.add_argument(
+        "--households", type=int, default=1000, help="0 means full population"
+    )
+    p.add_argument(
+        "--warmup-households",
+        type=int,
+        default=5000,
+        help="maximum cache-build households (default: 5000); warmup is always single-process",
+    )
+    p.add_argument(
+        "--cache-retries",
+        type=int,
+        default=2,
+        help="additional attempts after completed runs compile flows (default: 2)",
+    )
+    p.add_argument("--data-dir", type=Path, default=None)
+    p.add_argument(
+        "--config-overlay",
+        type=Path,
+        nargs="+",
+        default=[],
+        help="extra config directories, highest priority first",
+    )
+    p.add_argument(
+        "--flow-cache-dir",
+        type=Path,
+        default=Path.home() / ".cache/abench/flows",
+        help="persistent compiled-flow cache (default: ~/.cache/abench/flows)",
+    )
+    p.add_argument(
+        "--reuse-flows",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="automatically reuse and update compatible flows; warmup still runs",
+    )
+    p.add_argument(
+        "--cache-from",
+        type=Path,
+        help="seed flow cache from an experiment with the same revisions and dependencies",
+    )
+    p.add_argument(
+        "--output-dir",
+        type=Path,
+        required=False,
+        help="new experiment directory, or report HTML with --report-only",
+    )
+    p.add_argument("--label", help="experiment label in comparisons")
+    p.add_argument(
+        "--compare",
+        type=Path,
+        nargs="+",
+        default=[],
+        help="previous experiment directories",
+    )
+    p.add_argument(
+        "--report-only",
+        action="store_true",
+        help="rebuild a comparison of --compare directories without Docker",
+    )
+    p.add_argument(
+        "--interval",
+        type=positive,
+        default=0.5,
+        help="memory sampling interval in seconds",
+    )
+    p.add_argument(
+        "--memory", default="16g", help="Docker memory and memory+swap limit"
+    )
+    p.add_argument(
+        "--shm-size", default="8g", help="/dev/shm capacity; charged against --memory"
+    )
+    p.add_argument(
+        "--platform",
+        choices=("linux/arm64", "linux/amd64"),
+        help="defaults to Docker native architecture",
+    )
+    return p
+
+
+def mount(source, target, readonly=False):
+    source = str(source.resolve())
+    if "," in source:
+        raise ValueError("Docker bind paths cannot contain commas")
+    return [
+        "--mount",
+        f"type=bind,src={source},dst={target}" + (",readonly" if readonly else ""),
+    ]
+
+
+def container_phase(spec, output, data, image, phase_name):
+    """Retain Docker exit/OOM state even when the supervisor cannot finish."""
+    phase = output / phase_name
+    phase.mkdir(parents=True)
+    phase_kind = "warmup" if phase_name == "warmup" else "measured"
+    name = "abench-" + uuid.uuid4().hex[:12]
+    # Each container reads an immutable, uniquely named settings snapshot. Reusing
+    # the mutable root manifest across Docker Desktop mounts can expose stale data.
+    snapshot_name = f"spec-{name}.json"
+    write_json(phase / snapshot_name, spec)
+    args = [
+        "docker",
+        "run",
+        "--name",
+        name,
+        # Native Linux bind mounts preserve ownership. Run as the invoking user
+        # so outputs and caches remain writable between attempts and experiments.
+        "--user",
+        f"{os.getuid()}:{os.getgid()}",
+        "--env",
+        f"BENCH_SPEC_PATH=/results/{phase_name}/{snapshot_name}",
+        "--env",
+        f"BENCH_PHASE_NAME={phase_kind}",
+        "--cgroupns=private",
+        "--memory",
+        spec["memory"],
+        "--memory-swap",
+        spec["memory"],
+        "--shm-size",
+        spec["shm_size"],
+        "--network=none",
+    ]
+    if spec["platform"]:
+        args += ["--platform", spec["platform"]]
+    args += mount(output / "model", "/model", True)
+    args += mount(output / "runner", "/benchmark", True)
+    args += mount(data, "/data", True)
+    args += mount(output, "/results")
+    args += [image, "supervise", f"/results/{phase_name}"]
+    try:
+        command(args, phase / "console.log")
+    finally:
+        try:
+            state = json.loads(
+                command(["docker", "inspect", name, "--format", "{{json .State}}"])
+            )
+            write_json(phase / "docker-state.json", state)
+        finally:
+            subprocess.run(
+                ["docker", "rm", "-f", name],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                check=False,
+            )
+
+
+def main(argv=None):
+    p = parser()
+    argv = list(sys.argv[1:] if argv is None else argv)
+    # A file invocation stays separate from model profiles and ordinary flags.
+    candidate = argv[1:] if argv and argv[0] in ("run", "validate") else argv
+    if (
+        candidate
+        and not candidate[0].startswith("-")
+        and candidate[0] not in ("run", "report", "validate")
+    ):
+        if len(candidate) != 1:
+            p.error(
+                "an experiment file cannot be mixed with command-line overrides; edit its defaults or runs"
+            )
+        from .experiments import run_suite
+
+        return run_suite(Path(candidate[0]), main, validate_only=argv[0] == "validate")
+    action = argv.pop(0) if argv and argv[0] in ("run", "report", "validate") else "run"
+    args = p.parse_args(argv)
+    if action == "report":
+        args.report_only = True
+    if action != "validate" and args.output_dir is None:
+        p.error("--output-dir is required")
+    comparisons = [path.expanduser().resolve() for path in args.compare]
+    for previous in comparisons:
+        load_run(previous)
+    output = args.output_dir.expanduser().resolve() if args.output_dir else None
+    if args.report_only:
+        if output is None:
+            p.error("--output-dir is required for reporting")
+        if not comparisons:
+            p.error("--report-only requires --compare")
+        report(comparisons, output)
+        print(output)
+        return 0
+    root = args.model_dir.expanduser().resolve()
+    try:
+        profile = load_profile(args.profile, root)
+        sources = resolve_sources(
+            profile.get("sources", []),
+            args.source,
+            args.activitysim_commit,
+            args.sharrow_commit,
+        )
+        data = (
+            args.data_dir.expanduser().resolve()
+            if args.data_dir
+            else (root / profile.get("data_dir", "data")).resolve()
+        )
+        validate_model(profile, root, data, args.households)
+    except ValueError as error:
+        p.error(str(error))
+    args.activitysim_commit = next(
+        s["commit"] for s in sources if s["name"] == "activitysim"
+    )
+    args.sharrow_commit = next(
+        (s["commit"] for s in sources if s["name"] == "sharrow"), None
+    )
+    if args.sharrow and not args.sharrow_commit:
+        p.error("Sharrow enabled: provide a sharrow source override")
+    if args.cache_retries < 0:
+        p.error("--cache-retries must be nonnegative")
+    if args.warmup_households < 1:
+        p.error("--warmup-households must be positive")
+    if args.households < 0:
+        p.error("--households must be nonnegative")
+    if args.multiprocess and (args.processes is None or args.processes < 1):
+        p.error("--multiprocess requires --processes >= 1")
+    if not args.multiprocess and args.processes not in (None, 1):
+        p.error("--processes > 1 requires --multiprocess")
+    for value in (args.memory, args.shm_size):
+        if not re.fullmatch(r"[1-9][0-9]*[bkmgBKMG]?", value):
+            p.error("memory sizes must be positive integer Docker sizes, such as 16g")
+    args.data_dir = data
+    overlays = [(root / path.expanduser()).resolve() for path in args.config_overlay]
+    for path in overlays:
+        if not path.is_dir() or (output is not None and output.is_relative_to(path)):
+            p.error("config overlays must be existing directories outside --output-dir")
+    args.flow_cache_dir = args.flow_cache_dir.expanduser().resolve()
+    if args.sharrow and args.reuse_flows and output is not None:
+        if args.flow_cache_dir.is_relative_to(output) or output.is_relative_to(
+            args.flow_cache_dir
+        ):
+            p.error("--flow-cache-dir and --output-dir must be separate directories")
+    seed = args.cache_from.expanduser().resolve() if args.cache_from else None
+    if seed:
+        prior = read_json(seed / "experiment.json", {})
+        for key in ("activitysim_commit", "sharrow_commit"):
+            if prior.get(key) != getattr(args, key):
+                p.error(f"--cache-from must use the same {key}")
+        if not (seed / "cache/flows").is_dir():
+            p.error("--cache-from has no flow cache")
+    for source in profile["snapshot"]:
+        if output is not None and output.is_relative_to(root / source):
+            p.error("--output-dir must be outside snapshot source directories")
+    if seed and prior.get("sources") != sources:
+        p.error("--cache-from must use the same complete source dependency manifest")
+    if "," in str(output) or "," in str(data):
+        p.error("Docker bind paths cannot contain commas")
+    if output is not None and output.exists():
+        p.error(
+            "--output-dir must not already exist; each experiment needs a separate output directory"
+        )
+    docker = json.loads(command(["docker", "info", "--format", "{{json .}}"]))
+    if docker.get("OSType") != "linux" or str(docker.get("CgroupVersion")) != "2":
+        p.error("Docker must run Linux containers using cgroup v2")
+    if action == "validate":
+        print(
+            json.dumps(
+                {
+                    "profile": profile,
+                    "sources": sources,
+                    "data_dir": str(data),
+                    "docker": docker,
+                },
+                indent=2,
+            )
+        )
+        return 0
+    output.mkdir(parents=True)
+    spec = vars(args).copy()
+    spec.update(
+        schema_version=2,
+        abench_version=__version__,
+        profile=profile,
+        profile_name=profile["name"],
+        sources=sources,
+        label=args.label or output.name,
+        processes=args.processes or 1,
+        created_at=datetime.now(timezone.utc).isoformat(),
+        model_commit=git_info(root, ["rev-parse", "HEAD"]),
+        model_git_status=git_info(root, ["status", "--porcelain"]),
+        docker={
+            key: docker.get(key)
+            for key in (
+                "ServerVersion",
+                "Architecture",
+                "NCPU",
+                "MemTotal",
+                "KernelVersion",
+                "CgroupVersion",
+            )
+        },
+    )
+    spec = json.loads(json.dumps(spec, default=str))
+    (output / "model").mkdir()
+    for config in profile["snapshot"]:
+        source, target = root / config, output / "model" / config
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if source.is_dir():
+            shutil.copytree(
+                source, target, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")
+            )
+        else:
+            shutil.copy2(source, target)
+    for i, path in enumerate(overlays):
+        shutil.copytree(path, output / "model" / f"overlay-{i}")
+    shutil.copytree(
+        PACKAGE / "runtime",
+        output / "runner",
+        ignore=shutil.ignore_patterns("__pycache__"),
+    )
+    shutil.copytree(
+        PACKAGE, output / "harness", ignore=shutil.ignore_patterns("__pycache__")
+    )
+    shutil.copy2(PACKAGE / "Dockerfile", output / "production-benchmark.Dockerfile")
+    spec["config_sha256"] = {
+        str(path.relative_to(output / "model")): hashlib.sha256(
+            path.read_bytes()
+        ).hexdigest()
+        for path in sorted((output / "model").rglob("*"))
+        if path.is_file()
+    }
+    spec["harness_sha256"] = {
+        str(path.relative_to(output / "harness")): hashlib.sha256(
+            path.read_bytes()
+        ).hexdigest()
+        for path in sorted((output / "harness").rglob("*"))
+        if path.is_file()
+    }
+    spec["input_files"] = {
+        str(path.relative_to(data)): {
+            "bytes": path.stat().st_size,
+            "mtime_ns": path.stat().st_mtime_ns,
+        }
+        for path in sorted(data.rglob("*"))
+        if path.is_file()
+    }
+    write_json(output / "experiment.json", spec)
+    image = "abench:" + uuid.uuid4().hex[:12]
+    stage = "build"
+    try:
+        print(f"Building pinned packages; log: {output / 'build.log'}", flush=True)
+        with tempfile.TemporaryDirectory() as context:
+            shutil.copy2(
+                output / "production-benchmark.Dockerfile", Path(context) / "Dockerfile"
+            )
+            shutil.copy2(
+                PACKAGE / "runtime" / "build_sources.py",
+                Path(context) / "build_sources.py",
+            )
+            write_json(
+                Path(context) / "dependencies.json",
+                {
+                    "sources": sources,
+                    "requirements": profile.get("requirements", []),
+                    "constraints": profile.get("constraints", []),
+                },
+            )
+            build = [
+                "docker",
+                "build",
+                "-t",
+                image,
+                "--build-arg",
+                f"PYTHON_IMAGE={profile.get('python_image', 'python:3.11-slim-bookworm')}",
+            ]
+            if args.platform:
+                build += ["--platform", args.platform]
+            command(build + [context], output / "build.log")
+        spec["image_id"] = command(
+            ["docker", "image", "inspect", image, "--format", "{{.Id}}"]
+        )
+        write_json(output / "experiment.json", spec)
+        freeze = command(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "--network=none",
+                "--entrypoint",
+                "cat",
+                image,
+                "/opt/pip-freeze.txt",
+            ]
+        )
+        (output / "pip-freeze.txt").write_text(freeze + "\n")
+        provenance = command(
+            [
+                "docker",
+                "run",
+                "--rm",
+                "--network=none",
+                "--entrypoint",
+                "cat",
+                image,
+                "/opt/source-provenance.json",
+            ]
+        )
+        (output / "source-provenance.json").write_text(provenance + "\n")
+        if seed:
+            if (seed / "pip-freeze.txt").read_text().strip() != freeze.strip():
+                raise ValueError("Cannot seed cache: installed dependencies differ")
+            # Only flow artifacts are reused. A small serial warmup prepares flows;
+            # measurement remains responsible for rejecting missing signatures.
+            shutil.copytree(seed / "cache/flows", output / "cache/flows")
+        cache = nullcontext(None)
+        identity = None
+        if args.sharrow:
+            stage = "warmup"
+            print(
+                f"Preparing Sharrow cache in single process (up to {args.warmup_households} households)…",
+                flush=True,
+            )
+            cache = nullcontext(None)
+            if args.reuse_flows:
+                identity = json.loads(
+                    command(
+                        [
+                            "docker",
+                            "run",
+                            "--rm",
+                            "--network=none",
+                            "--entrypoint",
+                            "python",
+                            image,
+                            "-c",
+                            (PACKAGE / "runtime/cache_identity.py").read_text(),
+                        ]
+                    )
+                )
+                write_json(output / "flow-cache-identity.json", identity)
+                cache = reuse_flows(
+                    args.flow_cache_dir, identity, output / "cache/flows"
+                )
+                print(
+                    "Checking compatible flow cache (waiting for any active warmup)…",
+                    flush=True,
+                )
+        with cache as cache_info:
+
+            def publish_cache():
+                if cache_info is not None:
+                    publish_flows(identity, output / "cache/flows", cache_info)
+
+            if cache_info is not None:
+                spec["flow_cache"] = cache_info
+                write_json(output / "experiment.json", spec)
+                print(
+                    f"Reused {cache_info['restored_files']} flow-cache files.",
+                    flush=True,
+                )
+            if args.sharrow:
+                container_phase(spec, output, data, image, "warmup")
+                publish_cache()
+            write_json(output / "experiment.json", spec)
+            stage = "measured"
+            measured_attempts(
+                spec,
+                output,
+                lambda phase_name: container_phase(
+                    spec, output, data, image, phase_name
+                ),
+                publish_cache,
+            )
+    except (Exception, KeyboardInterrupt) as error:
+        if isinstance(error, subprocess.CalledProcessError):
+            error = BenchmarkFailure(describe_failure(output, stage, ignore_cache=True))
+            spec["failure"] = {"phase": stage, "error": str(error)}
+            write_json(output / "experiment.json", spec)
+            raise error from None
+        spec["failure"] = {"phase": stage, "error": str(error)}
+        write_json(output / "experiment.json", spec)
+        raise
+    finally:
+        report(comparisons + [output], output / "report.html")
+        print(f"Report: {output / 'report.html'}", flush=True)
+    return 0 if load_run(output)["valid"] else 1
+
+
+def entrypoint():
+    """Expose CLI errors without an unnecessary Python traceback."""
+    try:
+        sys.exit(main())
+    except (ValueError, OSError, subprocess.CalledProcessError) as error:
+        print(f"Benchmark failed: {error}", file=sys.stderr)
+        sys.exit(1)

abench/common.py

@@ -0,0 +1,22 @@
+"""Small artifact helpers shared by the host tools."""
+
+import json
+import os
+import tempfile
+
+
+def read_json(path, default=None):
+    return json.loads(path.read_text()) if path.exists() else default
+
+
+def write_json(path, value):
+    """Atomically replace metadata so Docker bind readers never see a rewrite."""
+    content = json.dumps(value, indent=2) + "\n"
+    descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}-", dir=path.parent)
+    try:
+        with os.fdopen(descriptor, "w") as stream:
+            stream.write(content)
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)