hypara 0.1.0__py3-none-any.whl

hypara/__init__.py

@@ -0,0 +1,4 @@
+"""hypara: benchmark harness for black-box optimizers speaking JSON Lines."""
+
+PROTOCOL_VERSION = 1
+LOG_SCHEMA_VERSION = 1

hypara/cli.py

@@ -0,0 +1,192 @@
+"""CLI: `hypara list | run | suite | report`.
+
+Suite configs are JSON:
+{
+  "name": "smoke",
+  "problems": ["smooth_hill", ...],
+  "optimizers": ["optimizers/random_search", ...],   // dirs with manifest.json
+  "seeds": [1, 2, 3],
+  "budget_overrides": {"smooth_hill": {"evaluations": 30}},   // optional
+  "ask_timeout_sec": 60                                        // optional
+}
+"""
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import json
+import sys
+import time
+from pathlib import Path
+
+from .metrics import aggregate, format_report, load_results
+from .problems.base import Budget
+from .registry import get_problem, problem_names
+from .runner import ASK_TIMEOUT_SEC, RunSpec, execute_run
+
+
+def load_manifest(optimizer_dir: str | Path) -> tuple[str, list[str], Path]:
+    """Returns (name, command, cwd) from <dir>/manifest.json."""
+    d = Path(optimizer_dir)
+    with open(d / "manifest.json", encoding="utf-8") as f:
+        manifest = json.load(f)
+    name = manifest.get("name", d.name)
+    command = manifest["command"]
+    if not isinstance(command, list) or not command:
+        raise ValueError(f"{d / 'manifest.json'}: command must be a non-empty list")
+    return name, command, d
+
+
+def override_budget(default: Budget, override: dict | None) -> Budget:
+    if not override:
+        return default
+    fields = dataclasses.asdict(default)
+    for key in ("evaluations", "cost_limit", "time_limit_sec"):
+        if key in override:
+            fields[key] = override[key]
+    return Budget(**fields)
+
+
+def cmd_list(_args) -> int:
+    for name in problem_names():
+        problem = get_problem(name)
+        budget = problem.default_budget
+        parts = []
+        if budget.evaluations is not None:
+            parts.append(f"evals={budget.evaluations}")
+        if budget.cost_limit is not None:
+            parts.append(f"cost={budget.cost_limit}")
+        fid = f" fidelities={problem.fidelities}" if problem.fidelities else ""
+        print(f"{name:<22} {', '.join(parts)}{fid}")
+    return 0
+
+
+def cmd_run(args) -> int:
+    problem = get_problem(args.problem)
+    opt_name, command, cwd = load_manifest(args.optimizer)
+    override = {}
+    if args.evaluations is not None:
+        override["evaluations"] = args.evaluations
+    if args.cost_limit is not None:
+        override["cost_limit"] = args.cost_limit
+    if args.time_limit is not None:
+        override["time_limit_sec"] = args.time_limit
+    budget = override_budget(problem.default_budget, override)
+
+    run_id = f"{problem.name}--{opt_name}--s{args.seed}"
+    out_dir = Path(args.out) if args.out else Path("results") / "adhoc" / run_id
+    result = execute_run(
+        RunSpec(
+            problem=problem,
+            optimizer_name=opt_name,
+            command=command,
+            optimizer_cwd=cwd,
+            seed=args.seed,
+            out_dir=out_dir,
+            budget=budget,
+            ask_timeout_sec=args.ask_timeout,
+        )
+    )
+    print(json.dumps({k: v for k, v in result.items() if k != "best_so_far"},
+                     ensure_ascii=False, indent=2))
+    print(f"\nlogs: {out_dir}", file=sys.stderr)
+    return 0 if result["status"] == "completed" else 1
+
+
+def cmd_suite(args) -> int:
+    with open(args.config, encoding="utf-8") as f:
+        config = json.load(f)
+    name = config.get("name", Path(args.config).stem)
+    out_root = (
+        Path(args.out)
+        if args.out
+        else Path("results") / f"{name}-{time.strftime('%Y%m%d-%H%M%S')}"
+    )
+    out_root.mkdir(parents=True, exist_ok=True)
+    with open(out_root / "suite.json", "w", encoding="utf-8") as f:
+        json.dump(config, f, ensure_ascii=False, indent=2)
+
+    overrides = config.get("budget_overrides", {})
+    ask_timeout = config.get("ask_timeout_sec", ASK_TIMEOUT_SEC)
+    failures = 0
+    for problem_name in config["problems"]:
+        for optimizer_dir in config["optimizers"]:
+            opt_name, command, cwd = load_manifest(optimizer_dir)
+            for seed in config["seeds"]:
+                problem = get_problem(problem_name)
+                budget = override_budget(
+                    problem.default_budget, overrides.get(problem_name)
+                )
+                run_id = f"{problem_name}--{opt_name}--s{seed}"
+                result = execute_run(
+                    RunSpec(
+                        problem=problem,
+                        optimizer_name=opt_name,
+                        command=command,
+                        optimizer_cwd=cwd,
+                        seed=seed,
+                        out_dir=out_root / run_id,
+                        budget=budget,
+                        ask_timeout_sec=ask_timeout,
+                    )
+                )
+                best = result["best_score"]
+                best_s = "None" if best is None else f"{best:.4f}"
+                print(
+                    f"{run_id:<48} {result['status']:<18} best={best_s} "
+                    f"evals={result['evaluations_used']} "
+                    f"t={result['wall_time_sec']:.1f}s"
+                )
+                if result["status"] != "completed":
+                    failures += 1
+    print(f"\nsuite dir: {out_root}")
+    if failures:
+        print(f"{failures} run(s) did not complete", file=sys.stderr)
+    return 0
+
+
+def cmd_report(args) -> int:
+    results = load_results(args.dir)
+    if not results:
+        print(f"no result.json found under {args.dir}", file=sys.stderr)
+        return 1
+    report = aggregate(results, baseline=args.baseline)
+    print(format_report(report))
+    with open(Path(args.dir) / "report.json", "w", encoding="utf-8") as f:
+        json.dump(report, f, ensure_ascii=False, indent=2)
+    return 0
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(prog="hypara")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    sub.add_parser("list", help="list registered problems")
+
+    p_run = sub.add_parser("run", help="run one optimizer on one problem")
+    p_run.add_argument("--problem", required=True)
+    p_run.add_argument("--optimizer", required=True, help="optimizer dir with manifest.json")
+    p_run.add_argument("--seed", type=int, default=0)
+    p_run.add_argument("--out")
+    p_run.add_argument("--evaluations", type=int)
+    p_run.add_argument("--cost-limit", type=float)
+    p_run.add_argument("--time-limit", type=float)
+    p_run.add_argument("--ask-timeout", type=float, default=ASK_TIMEOUT_SEC)
+
+    p_suite = sub.add_parser("suite", help="run a suite from a JSON config")
+    p_suite.add_argument("--config", required=True)
+    p_suite.add_argument("--out")
+
+    p_report = sub.add_parser("report", help="aggregate metrics from a suite dir")
+    p_report.add_argument("--dir", required=True)
+    p_report.add_argument("--baseline", default="random_search")
+
+    args = parser.parse_args(argv)
+    return {"list": cmd_list, "run": cmd_run, "suite": cmd_suite, "report": cmd_report}[
+        args.cmd
+    ](args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

hypara/event_log.py

@@ -0,0 +1,44 @@
+"""Append-only JSONL event log for a run.
+
+Metrics are always recomputed from this log; the runner never finalizes
+aggregates during the run. Each row carries schema_version.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+from . import LOG_SCHEMA_VERSION
+
+
+class EventLog:
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._f = open(self.path, "a", encoding="utf-8")
+        self._t0 = time.monotonic()
+
+    def emit(self, event: str, **fields) -> None:
+        row = {
+            "schema_version": LOG_SCHEMA_VERSION,
+            "event": event,
+            "elapsed_sec": round(time.monotonic() - self._t0, 4),
+            **fields,
+        }
+        self._f.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n")
+        self._f.flush()
+
+    def close(self) -> None:
+        self._f.close()
+
+
+def read_events(path: str | Path) -> list[dict]:
+    rows = []
+    with open(path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                rows.append(json.loads(line))
+    return rows

hypara/metrics.py

@@ -0,0 +1,153 @@
+"""Compute per-run and aggregated metrics from saved run results.
+
+Normalization is baseline-relative: within one problem, 0 is the median
+best_score of the baseline optimizer's runs and 1 is the best score observed
+across all runs of that problem. Normalized AUC is the mean of the
+normalized best-so-far value over an evenly spaced budget-fraction grid
+(evaluations axis, or cost axis when the budget is cost-limited); before the
+first counting evaluation the value is 0.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from statistics import mean, median
+
+GRID_POINTS = 101
+
+
+def load_results(suite_dir: str | Path) -> list[dict]:
+    results = []
+    for path in sorted(Path(suite_dir).rglob("result.json")):
+        with open(path, encoding="utf-8") as f:
+            results.append(json.load(f))
+    return results
+
+
+def _budget_axis(result: dict) -> tuple[str, float]:
+    """(axis, total): evaluations axis unless the budget is cost-limited."""
+    budget = result["budget"]
+    if budget.get("evaluations") is not None:
+        return "evaluations", float(budget["evaluations"])
+    return "cost", float(budget["cost_limit"])
+
+
+def _position(row: dict, axis: str) -> float:
+    if axis == "evaluations":
+        return row["eval_index"] + 1.0  # score known after the eval completes
+    return row["cost_used"]
+
+
+def best_so_far_values(result: dict) -> list[float | None]:
+    """Best-so-far sampled on the budget-fraction grid; None before first score."""
+    axis, total = _budget_axis(result)
+    curve = result["best_so_far"]
+    values: list[float | None] = []
+    for i in range(GRID_POINTS):
+        pos = total * i / (GRID_POINTS - 1)
+        best = None
+        for row in curve:
+            if _position(row, axis) <= pos:
+                best = row["score"]
+            else:
+                break
+        values.append(best)
+    return values
+
+
+def normalized(value: float | None, base: float, ref: float) -> float:
+    """Clipped to [0, 1]: worse-than-baseline runs score 0 rather than
+    dragging the mean with unbounded negatives when the spread is tiny."""
+    if value is None:
+        return 0.0
+    if ref <= base:
+        # degenerate problem column (e.g. every run tied): all runs score 0
+        return 0.0
+    return min(1.0, max(0.0, (value - base) / (ref - base)))
+
+
+def aggregate(results: list[dict], baseline: str = "random_search") -> dict:
+    """Aggregate suite results into a per-(problem, optimizer) report dict."""
+    by_problem: dict[str, list[dict]] = {}
+    for r in results:
+        by_problem.setdefault(r["problem"], []).append(r)
+
+    report: dict = {"baseline": baseline, "problems": {}, "overall": {}}
+    per_opt_norm_best: dict[str, list[float]] = {}
+    per_opt_norm_auc: dict[str, list[float]] = {}
+
+    for problem, runs in sorted(by_problem.items()):
+        base_scores = [
+            r["best_score"]
+            for r in runs
+            if r["optimizer"] == baseline and r["best_score"] is not None
+        ]
+        all_scores = [r["best_score"] for r in runs if r["best_score"] is not None]
+        base = median(base_scores) if base_scores else 0.0
+        ref = max(all_scores) if all_scores else 1.0
+
+        by_opt: dict[str, list[dict]] = {}
+        for r in runs:
+            by_opt.setdefault(r["optimizer"], []).append(r)
+
+        rows = {}
+        for opt, opt_runs in sorted(by_opt.items()):
+            norm_bests, norm_aucs, valid_rates = [], [], []
+            statuses: dict[str, int] = {}
+            for r in opt_runs:
+                statuses[r["status"]] = statuses.get(r["status"], 0) + 1
+                norm_bests.append(normalized(r["best_score"], base, ref))
+                grid = best_so_far_values(r)
+                norm_aucs.append(mean(normalized(v, base, ref) for v in grid))
+                total = r["valid_count"] + r["invalid_count"]
+                valid_rates.append(r["valid_count"] / total if total else 0.0)
+            rows[opt] = {
+                "runs": len(opt_runs),
+                "statuses": statuses,
+                "mean_best": mean(
+                    [r["best_score"] for r in opt_runs if r["best_score"] is not None] or [0.0]
+                ),
+                "mean_norm_best": mean(norm_bests),
+                "mean_norm_auc": mean(norm_aucs),
+                "mean_valid_rate": mean(valid_rates),
+            }
+            per_opt_norm_best.setdefault(opt, []).append(rows[opt]["mean_norm_best"])
+            per_opt_norm_auc.setdefault(opt, []).append(rows[opt]["mean_norm_auc"])
+
+        report["problems"][problem] = {
+            "baseline_median": base,
+            "observed_best": ref,
+            "optimizers": rows,
+        }
+
+    for opt in per_opt_norm_best:
+        report["overall"][opt] = {
+            "mean_norm_best": mean(per_opt_norm_best[opt]),
+            "mean_norm_auc": mean(per_opt_norm_auc[opt]),
+        }
+    return report
+
+
+def format_report(report: dict) -> str:
+    lines = []
+    header = f"{'optimizer':<22} {'runs':>4} {'best':>8} {'n_best':>7} {'n_auc':>7} {'valid':>6}  status"
+    for problem, pdata in report["problems"].items():
+        lines.append(f"## {problem}  (baseline_median={pdata['baseline_median']:.4f}, "
+                     f"observed_best={pdata['observed_best']:.4f})")
+        lines.append(header)
+        for opt, row in pdata["optimizers"].items():
+            status = ",".join(f"{k}:{v}" for k, v in sorted(row["statuses"].items()))
+            lines.append(
+                f"{opt:<22} {row['runs']:>4} {row['mean_best']:>8.4f} "
+                f"{row['mean_norm_best']:>7.3f} {row['mean_norm_auc']:>7.3f} "
+                f"{row['mean_valid_rate']:>6.2f}  {status}"
+            )
+        lines.append("")
+    lines.append(f"## overall (mean over problems, normalized vs {report['baseline']})")
+    lines.append(f"{'optimizer':<22} {'n_best':>7} {'n_auc':>7}")
+    for opt, row in sorted(
+        report["overall"].items(), key=lambda kv: -kv[1]["mean_norm_best"]
+    ):
+        lines.append(f"{opt:<22} {row['mean_norm_best']:>7.3f} {row['mean_norm_auc']:>7.3f}")
+    return "\n".join(lines)

hypara/problems/_util.py

@@ -0,0 +1,30 @@
+"""Small math helpers shared by surrogate evaluators (stdlib only)."""
+
+from __future__ import annotations
+
+import math
+from random import Random
+
+
+def bump(dist2: float, width: float) -> float:
+    """Gaussian bump in [0, 1]: 1 at dist2=0, width is the sigma."""
+    return math.exp(-dist2 / (2.0 * width * width))
+
+
+def random_rotations(rng: Random, dim: int, count: int) -> list[tuple[int, int, float, float]]:
+    """Random plane (Givens) rotations; a cheap stand-in for a rotation matrix."""
+    rots = []
+    for _ in range(count):
+        i, j = rng.sample(range(dim), 2)
+        theta = rng.uniform(0, 2 * math.pi)
+        rots.append((i, j, math.cos(theta), math.sin(theta)))
+    return rots
+
+
+def apply_rotations(z: list[float], rots: list[tuple[int, int, float, float]]) -> list[float]:
+    z = list(z)
+    for i, j, c, s in rots:
+        zi, zj = z[i], z[j]
+        z[i] = c * zi - s * zj
+        z[j] = s * zi + c * zj
+    return z

hypara/problems/base.py

@@ -0,0 +1,62 @@
+"""Problem / Evaluator interfaces.
+
+A Problem is the template of one task: description, space, default budget.
+An Evaluator is that template instantiated with an instance seed; it must be
+a pure function of (candidate, noise_rng, fidelity) so runs are reproducible.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from random import Random
+
+from ..space import ParamSpec
+
+
+@dataclass(frozen=True)
+class Budget:
+    evaluations: int | None = None
+    cost_limit: float | None = None
+    time_limit_sec: float | None = 300.0
+
+    def __post_init__(self):
+        if self.evaluations is None and self.cost_limit is None:
+            raise ValueError("budget needs evaluations or cost_limit")
+
+    def to_json(self) -> dict:
+        return {
+            "evaluations": self.evaluations,
+            "cost_limit": self.cost_limit,
+            "time_limit_sec": self.time_limit_sec,
+        }
+
+
+@dataclass(frozen=True)
+class EvalResult:
+    score: float
+    cost: float = 1.0
+
+
+class Evaluator(ABC):
+    @abstractmethod
+    def evaluate(
+        self, candidate: dict, noise_rng: Random, fidelity: str | None = None
+    ) -> EvalResult: ...
+
+
+class Problem(ABC):
+    """Subclasses set the class attributes and implement make_evaluator.
+
+    fidelities, when set, is ordered low to high; the last entry is the top
+    fidelity, and only top-fidelity evaluations count toward best_score.
+    """
+
+    name: str
+    description: str
+    space: list[ParamSpec]
+    default_budget: Budget
+    fidelities: list[str] | None = None
+
+    @abstractmethod
+    def make_evaluator(self, instance_seed: int) -> Evaluator: ...

hypara/problems/conditional_knobs.py

@@ -0,0 +1,91 @@
+"""ConditionalKnobs: the categorical `algo` choice switches which knobs exist."""
+
+from __future__ import annotations
+
+import math
+from random import Random
+
+from .base import Budget, EvalResult, Evaluator, Problem
+from ._util import bump
+from ..space import Condition, ParamSpec
+
+
+class _ConditionalKnobsEvaluator(Evaluator):
+    def __init__(self, instance_seed: int):
+        rng = Random(instance_seed)
+        bases = [0.35, 0.50, 0.65]
+        rng.shuffle(bases)
+        self.base = dict(zip(("alpha", "beta", "gamma"), bases))
+        self.alpha_rate = math.exp(rng.uniform(math.log(1e-4), math.log(1.0)))
+        self.alpha_depth = rng.randint(2, 9)
+        self.beta_temp = rng.uniform(0.2, 1.8)
+        self.beta_mode = rng.choice(["fast", "safe", "hybrid"])
+        self.gamma_k = rng.randint(5, 45)
+        self.gamma_smooth = rng.random() < 0.5
+
+    def evaluate(self, candidate, noise_rng, fidelity=None) -> EvalResult:
+        algo = candidate["algo"]
+        score = self.base[algo]
+        if algo == "alpha":
+            d2 = (
+                ((math.log(candidate["alpha_rate"]) - math.log(self.alpha_rate)) / 1.0) ** 2
+                + ((candidate["alpha_depth"] - self.alpha_depth) / 1.5) ** 2
+            )
+            score += 0.35 * bump(d2, width=0.5)
+        elif algo == "beta":
+            d2 = ((candidate["beta_temp"] - self.beta_temp) / 0.2) ** 2
+            score += 0.25 * bump(d2, width=0.5)
+            if candidate["beta_mode"] == self.beta_mode:
+                score += 0.10
+        else:  # gamma
+            d2 = ((candidate["gamma_k"] - self.gamma_k) / 4.0) ** 2
+            score += 0.25 * bump(d2, width=0.5)
+            if candidate["gamma_smooth"] == self.gamma_smooth:
+                score += 0.10
+        return EvalResult(score=score)
+
+
+class ConditionalKnobs(Problem):
+    name = "conditional_knobs"
+    description = (
+        "You are configuring a service that supports three interchangeable "
+        "backend algorithms: alpha, beta, and gamma. Each algorithm exposes "
+        "its own knobs (see the `condition` fields in the space); knobs of "
+        "unselected algorithms must not appear in a candidate. One of the "
+        "three algorithms is inherently better than the others on this "
+        "workload, and within it the knobs have a single fairly narrow sweet "
+        "spot. Picking the right algorithm with untuned knobs gets you to "
+        "about 0.65; tuning its knobs is worth up to 0.35 more, for a "
+        "maximum of 1.0. No measurement noise."
+    )
+    space = [
+        ParamSpec(name="algo", type="categorical", choices=("alpha", "beta", "gamma")),
+        ParamSpec(
+            name="alpha_rate", type="float", low=1e-4, high=1.0, log=True,
+            condition=Condition("algo", ("alpha",)),
+        ),
+        ParamSpec(
+            name="alpha_depth", type="int", low=1, high=10,
+            condition=Condition("algo", ("alpha",)),
+        ),
+        ParamSpec(
+            name="beta_temp", type="float", low=0.0, high=2.0,
+            condition=Condition("algo", ("beta",)),
+        ),
+        ParamSpec(
+            name="beta_mode", type="categorical", choices=("fast", "safe", "hybrid"),
+            condition=Condition("algo", ("beta",)),
+        ),
+        ParamSpec(
+            name="gamma_k", type="int", low=1, high=50,
+            condition=Condition("algo", ("gamma",)),
+        ),
+        ParamSpec(
+            name="gamma_smooth", type="bool",
+            condition=Condition("algo", ("gamma",)),
+        ),
+    ]
+    default_budget = Budget(evaluations=100)
+
+    def make_evaluator(self, instance_seed: int) -> Evaluator:
+        return _ConditionalKnobsEvaluator(instance_seed)

hypara/problems/cost_aware.py

@@ -0,0 +1,58 @@
+"""CostAware: the candidate itself decides how expensive its evaluation is."""
+
+from __future__ import annotations
+
+import math
+from random import Random
+
+from .base import Budget, EvalResult, Evaluator, Problem
+from ._util import apply_rotations, bump, random_rotations
+from ..space import ParamSpec
+
+_DIM = 2
+
+
+class _CostAwareEvaluator(Evaluator):
+    def __init__(self, instance_seed: int):
+        rng = Random(instance_seed)
+        self.center = [rng.uniform(0.2, 0.8) for _ in range(_DIM)]
+        self.rots = random_rotations(rng, _DIM, count=2)
+        self.scales = [rng.uniform(1.5, 3.0) for _ in range(_DIM)]
+        self.best_method = rng.choice(["quick", "thorough"])
+        self.tau = rng.uniform(15.0, 35.0)
+
+    def evaluate(self, candidate, noise_rng, fidelity=None) -> EvalResult:
+        z = [candidate[f"x{i}"] - self.center[i] for i in range(_DIM)]
+        z = apply_rotations(z, self.rots)
+        d2 = sum((s * v) ** 2 for s, v in zip(self.scales, z))
+        quality = 0.8 * bump(d2, width=0.7) + (
+            0.2 if candidate["method"] == self.best_method else 0.0
+        )
+        saturation = 1.0 - math.exp(-candidate["samples"] / self.tau)
+        cost = 0.5 + candidate["samples"] / 20.0
+        return EvalResult(score=quality * saturation, cost=cost)
+
+
+class CostAware(Problem):
+    name = "cost_aware"
+    description = (
+        "You are tuning a measurement job with two continuous parameters, a "
+        "method choice, and a `samples` count from 1 to 100. The budget is a "
+        "total COST limit, and each evaluation costs about 0.5 + samples/20 "
+        "budget units (so samples=100 costs roughly ten times samples=10). "
+        "The score is an underlying quality (smooth, single optimum, method "
+        "matters) multiplied by a saturating function of `samples`: few "
+        "samples cap the score well below its potential, many samples "
+        "approach it with diminishing returns. The saturation rate is "
+        "unknown. Cheap low-sample evaluations still reveal where quality is "
+        "high; spending everything on max-sample evaluations leaves too few "
+        "probes. Maximum achievable score is just under 1.0."
+    )
+    space = [ParamSpec(name=f"x{i}", type="float", low=0.0, high=1.0) for i in range(_DIM)] + [
+        ParamSpec(name="method", type="categorical", choices=("quick", "thorough")),
+        ParamSpec(name="samples", type="int", low=1, high=100),
+    ]
+    default_budget = Budget(evaluations=None, cost_limit=200.0)
+
+    def make_evaluator(self, instance_seed: int) -> Evaluator:
+        return _CostAwareEvaluator(instance_seed)