loopx 0.4.8__py3-none-any.whl

loopx/capabilities/auto_research/kernel.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Mapping
+from typing import Any
+
+
+LIGHTWEIGHT_AUTO_RESEARCH_RESULT_SCHEMA_VERSION = "auto_research_lightweight_result_v0"
+LIGHTWEIGHT_AUTO_RESEARCH_HYPOTHESIS_SCHEMA_VERSION = "auto_research_lightweight_hypothesis_v0"
+LIGHTWEIGHT_AUTO_RESEARCH_EVIDENCE_SCHEMA_VERSION = "auto_research_lightweight_evidence_v0"
+
+MetricEvaluator = Callable[[dict[str, Any], str], Mapping[str, Any]]
+
+
+def _token(value: object, *, default: str = "unknown") -> str:
+    text = str(value or "").strip()
+    return text or default
+
+
+def _number(value: object, *, default: float | None = None) -> float | None:
+    if value is None:
+        return default
+    try:
+        number = float(value)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"metric value is not numeric: {value!r}") from exc
+    if number != number or number in {float("inf"), float("-inf")}:
+        raise ValueError(f"metric value is not finite: {value!r}")
+    return number
+
+
+def _improved(value: float | None, baseline: float, *, direction: str) -> bool:
+    if value is None:
+        return False
+    if direction == "minimize":
+        return value < baseline
+    if direction != "maximize":
+        raise ValueError("direction must be maximize or minimize")
+    return value > baseline
+
+
+def lightweight_hypothesis(
+    *,
+    hypothesis_id: str,
+    todo_id: str,
+    claimed_by: str,
+    text: str,
+    candidate_key: str,
+) -> dict[str, Any]:
+    return {
+        "schema_version": LIGHTWEIGHT_AUTO_RESEARCH_HYPOTHESIS_SCHEMA_VERSION,
+        "hypothesis_id": _token(hypothesis_id, default="hypothesis"),
+        "todo_id": _token(todo_id, default="todo"),
+        "claimed_by": _token(claimed_by, default="agent"),
+        "hypothesis": _token(text, default="untitled hypothesis"),
+        "candidate_key": _token(candidate_key, default="candidate"),
+    }
+
+
+def _evidence_event(
+    *,
+    hypothesis: dict[str, Any],
+    split: str,
+    result: Mapping[str, Any],
+    baseline: float,
+    direction: str,
+) -> dict[str, Any]:
+    metric = _number(result.get("metric"))
+    exact = bool(result.get("exact", True))
+    clean = bool(result.get("protected_scope_clean", True))
+    status = "improved" if exact and clean and _improved(metric, baseline, direction=direction) else "not_improved"
+    return {
+        "schema_version": LIGHTWEIGHT_AUTO_RESEARCH_EVIDENCE_SCHEMA_VERSION,
+        "hypothesis_id": hypothesis["hypothesis_id"],
+        "todo_id": hypothesis["todo_id"],
+        "claimed_by": hypothesis["claimed_by"],
+        "candidate_key": hypothesis["candidate_key"],
+        "split": _token(split, default="dev"),
+        "metric": metric,
+        "baseline": baseline,
+        "direction": direction,
+        "exact": exact,
+        "protected_scope_clean": clean,
+        "status": status,
+        "result_source": _token(result.get("result_source"), default="metric_evaluator"),
+        "strategy": _token(result.get("strategy"), default=hypothesis["candidate_key"]),
+        "artifact_refs": [
+            _token(item, default="artifact")
+            for item in result.get("artifact_refs", [])
+            if str(item).strip()
+        ][:8],
+    }
+
+
+def _rank_key(event: dict[str, Any]) -> float:
+    value = event["metric"]
+    return float("-inf") if value is None else float(value)
+
+
+def _best_positive(
+    evidence: list[dict[str, Any]],
+    *,
+    direction: str,
+) -> dict[str, Any] | None:
+    positives = [event for event in evidence if event["status"] == "improved"]
+    if not positives:
+        return None
+    if direction == "minimize":
+        return min(positives, key=_rank_key)
+    return max(positives, key=_rank_key)
+
+
+def run_lightweight_auto_research(
+    *,
+    goal_id: str,
+    hypotheses: Iterable[dict[str, Any]],
+    evaluate: MetricEvaluator,
+    baseline: float = 1.0,
+    direction: str = "maximize",
+    dev_split: str = "dev",
+    holdout_split: str = "holdout",
+    max_dev_rounds: int | None = None,
+) -> dict[str, Any]:
+    """Run a small auto-research loop: dev attempts, best candidate, holdout.
+
+    This kernel deliberately avoids board, frontier, launcher, and artifact
+    projection concerns. Those layers can wrap the result without growing the
+    research decision core.
+    """
+
+    base = _number(baseline)
+    if base is None:
+        raise ValueError("baseline is required")
+    if direction not in {"maximize", "minimize"}:
+        raise ValueError("direction must be maximize or minimize")
+    items = list(hypotheses)
+    if not items:
+        raise ValueError("at least one hypothesis is required")
+    limit = len(items) if max_dev_rounds is None else max(0, min(max_dev_rounds, len(items)))
+
+    evidence: list[dict[str, Any]] = []
+    by_id = {item["hypothesis_id"]: item for item in items}
+    for hypothesis in items[:limit]:
+        event = _evidence_event(
+            hypothesis=hypothesis,
+            split=dev_split,
+            result=evaluate(hypothesis, dev_split),
+            baseline=base,
+            direction=direction,
+        )
+        evidence.append(event)
+
+    best_dev = _best_positive(evidence, direction=direction)
+    selected = by_id.get(best_dev["hypothesis_id"]) if best_dev else None
+    holdout_event = None
+    if selected is not None:
+        holdout_event = _evidence_event(
+            hypothesis=selected,
+            split=holdout_split,
+            result=evaluate(selected, holdout_split),
+            baseline=base,
+            direction=direction,
+        )
+        evidence.append(holdout_event)
+
+    holdout_positive = bool(holdout_event and holdout_event["status"] == "improved")
+    dev_positive = best_dev is not None
+    decision = (
+        "validated_positive"
+        if holdout_positive
+        else "dev_supported"
+        if dev_positive
+        else "continue_research"
+    )
+    return {
+        "ok": True,
+        "schema_version": LIGHTWEIGHT_AUTO_RESEARCH_RESULT_SCHEMA_VERSION,
+        "goal_id": _token(goal_id, default="auto-research"),
+        "candidate_count": len(items),
+        "dev_round_count": limit,
+        "evidence_event_count": len(evidence),
+        "selected_hypothesis_id": selected["hypothesis_id"] if selected else None,
+        "decision": decision,
+        "dev_metric": best_dev["metric"] if best_dev else None,
+        "holdout_metric": holdout_event["metric"] if holdout_event else None,
+        "evidence": evidence,
+        "public_boundary": {
+            "raw_logs_recorded": False,
+            "private_artifacts_recorded": False,
+            "raw_source_bodies_recorded": False,
+        },
+    }

loopx/capabilities/auto_research/knn_demo_workspace.py

@@ -0,0 +1,322 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import stat
+from pathlib import Path
+
+
+KNN_DEMO_WORKSPACE_SCHEMA_VERSION = "loopx_knn_demo_workspace_v0"
+
+KNN_DEMO_EDITABLE_SCOPE = ["solution.py"]
+KNN_DEMO_PROTECTED_SCOPE = ["task.py", "eval.py", "eval.sh"]
+KNN_DEMO_CONTRACT_FILE = "research_contract.public.json"
+KNN_DEMO_DEV_EVAL_COMMAND = "bash eval.sh dev"
+KNN_DEMO_HOLDOUT_EVAL_COMMAND = "bash eval.sh test"
+
+
+README_MD = """# LoopX KNN Auto-Research Demo
+
+Goal: improve exact k-nearest-neighbor search speed while preserving correctness.
+
+Editable scope:
+- `solution.py`
+
+Protected scope:
+- `task.py`
+- `eval.py`
+- `eval.sh`
+
+Commands:
+- `bash eval.sh dev`
+- `bash eval.sh test`
+
+Evidence rule:
+- Record the command, split, score, mechanism, and changed file before claiming an improvement.
+- Do not edit protected files to improve the score.
+
+Recording-friendly artifact checklist:
+- Curator: metric, editable/protected boundary, and promotion rule.
+- Hypothesis proposer: two exact-KNN speedup hypotheses with mechanism and risk.
+- Executor: baseline dev score, changed mechanism, post-change dev score, and held-out score when dev improves.
+- Evaluator: split-aware verdict and the next role todo or retry reason.
+"""
+
+
+AGENTS_MD = """# KNN Demo Agent Rules
+
+Only edit `solution.py`.
+
+Do not edit `task.py`, `eval.py`, or `eval.sh`.
+
+Run `bash eval.sh dev` for development evidence and `bash eval.sh test` for held-out evidence.
+
+For visible auto-research, do not stop at a status/tick summary. Leave one compact
+public-safe artifact, todo update, or evidence packet that another role can use.
+"""
+
+
+TASK_PY = '''from __future__ import annotations
+
+from random import Random
+
+
+K = 7
+DIM = 12
+
+
+def _sizes(split: str) -> tuple[int, int, int]:
+    if split == "test":
+        return 4, 1800, 100
+    return 4, 1400, 80
+
+
+def _seed(split: str) -> int:
+    return 9001 if split == "test" else 1009
+
+
+def generate_instances(split: str):
+    split = "test" if str(split).strip() == "test" else "dev"
+    count, database_size, query_size = _sizes(split)
+    rng = Random(_seed(split))
+    for _ in range(count):
+        database = [
+            [rng.uniform(-1.0, 1.0) for _ in range(DIM)]
+            for _ in range(database_size)
+        ]
+        queries = [
+            [rng.uniform(-1.0, 1.0) for _ in range(DIM)]
+            for _ in range(query_size)
+        ]
+        yield {"database": database, "queries": queries, "k": K}
+
+
+def squared_distance(left, right) -> float:
+    return sum((a - b) * (a - b) for a, b in zip(left, right))
+
+
+def reference_solve(problem):
+    database = problem["database"]
+    k = int(problem["k"])
+    output = []
+    for query in problem["queries"]:
+        ranked = sorted(
+            (squared_distance(row, query), index)
+            for index, row in enumerate(database)
+        )
+        output.append([index for _distance, index in ranked[:k]])
+    return output
+
+
+def is_solution(problem, candidate) -> bool:
+    return candidate == reference_solve(problem)
+'''
+
+
+SOLUTION_PY = '''from __future__ import annotations
+
+
+def _squared_distance(left, right) -> float:
+    return sum((a - b) * (a - b) for a, b in zip(left, right))
+
+
+def solve(problem):
+    """Baseline exact KNN: full-sort every query."""
+
+    database = problem["database"]
+    k = int(problem["k"])
+    output = []
+    for query in problem["queries"]:
+        ranked = sorted(
+            (_squared_distance(row, query), index)
+            for index, row in enumerate(database)
+        )
+        output.append([index for _distance, index in ranked[:k]])
+    return output
+'''
+
+
+EVAL_PY = '''from __future__ import annotations
+
+import importlib
+import json
+import statistics
+import sys
+import time
+
+from task import generate_instances, is_solution, reference_solve
+
+
+def _median_runtime(fn, problem, *, trials: int) -> tuple[float, object]:
+    timings = []
+    result = None
+    for _ in range(trials):
+        start = time.perf_counter()
+        result = fn(problem)
+        timings.append(time.perf_counter() - start)
+    return statistics.median(timings), result
+
+
+def evaluate(split: str) -> dict[str, object]:
+    split = "test" if str(split).strip() == "test" else "dev"
+    solution = importlib.import_module("solution")
+    trials = 2 if split == "test" else 3
+    speedups = []
+    for problem in generate_instances(split):
+        reference_runtime, _reference = _median_runtime(
+            reference_solve,
+            problem,
+            trials=trials,
+        )
+        candidate_runtime, candidate = _median_runtime(
+            solution.solve,
+            problem,
+            trials=trials,
+        )
+        if not is_solution(problem, candidate):
+            return {
+                "ok": False,
+                "split": split,
+                "score": 0.0,
+                "metric_name": "speedup",
+                "error": "candidate output does not match exact reference",
+            }
+        speedups.append(reference_runtime / max(candidate_runtime, 1e-12))
+    return {
+        "ok": True,
+        "split": split,
+        "score": round(float(statistics.median(speedups)), 6),
+        "metric_name": "speedup",
+        "direction": "maximize",
+        "baseline_score": 1.0,
+        "editable_scope": ["solution.py"],
+        "protected_scope": ["task.py", "eval.py", "eval.sh"],
+    }
+
+
+def main(argv: list[str]) -> int:
+    payload = evaluate(argv[1] if len(argv) > 1 else "dev")
+    print(f"score: {payload['score']:.6f}")
+    print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
+    return 0 if payload.get("ok") else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv))
+'''
+
+
+EVAL_SH = """#!/usr/bin/env bash
+set -euo pipefail
+python3 eval.py "${1:-dev}"
+"""
+
+
+GITIGNORE = """__pycache__/
+*.pyc
+.loopx/
+"""
+
+
+def _sha256_text(content: str) -> str:
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def _contract_payload(*, goal_id: str | None = None) -> dict[str, object]:
+    payload: dict[str, object] = {
+        "schema_version": "auto_research_benchmark_contract_v0",
+        "preset_id": "knn-demo",
+        "objective": "Improve exact KNN speedup while preserving correctness.",
+        "metric": {
+            "name": "speedup",
+            "direction": "maximize",
+            "baseline": 1.0,
+        },
+        "editable_scope": list(KNN_DEMO_EDITABLE_SCOPE),
+        "protected_scope": list(KNN_DEMO_PROTECTED_SCOPE),
+        "protected_scope_sha256": {
+            "task.py": _sha256_text(TASK_PY),
+            "eval.py": _sha256_text(EVAL_PY),
+            "eval.sh": _sha256_text(EVAL_SH),
+        },
+        "dev_eval_command": KNN_DEMO_DEV_EVAL_COMMAND,
+        "holdout_eval_command": KNN_DEMO_HOLDOUT_EVAL_COMMAND,
+        "claim_boundary": (
+            "Claims require command output from the generated benchmark workspace; "
+            "do not edit protected files."
+        ),
+    }
+    if goal_id:
+        payload["goal_id"] = goal_id
+    return payload
+
+
+def _write_if_missing(path: Path, content: str, *, executable: bool = False) -> bool:
+    if path.exists():
+        if executable:
+            path.chmod(path.stat().st_mode | stat.S_IXUSR)
+        return False
+    path.write_text(content, encoding="utf-8", newline="\n")
+    if executable:
+        path.chmod(path.stat().st_mode | stat.S_IXUSR)
+    return True
+
+
+def materialize_knn_demo_workspace(workspace: str | Path, *, goal_id: str | None = None) -> dict[str, object]:
+    """Create the real KNN demo benchmark workspace if it is missing."""
+
+    workspace_path = Path(workspace).expanduser()
+    created = not workspace_path.exists()
+    workspace_path.mkdir(parents=True, exist_ok=True)
+
+    files = {
+        "README.md": README_MD,
+        "AGENTS.md": AGENTS_MD,
+        "task.py": TASK_PY,
+        "solution.py": SOLUTION_PY,
+        "eval.py": EVAL_PY,
+        "eval.sh": EVAL_SH,
+        ".gitignore": GITIGNORE,
+    }
+    written: list[str] = []
+    kept: list[str] = []
+    for relative_path, content in files.items():
+        path = workspace_path / relative_path
+        did_write = _write_if_missing(
+            path,
+            content,
+            executable=relative_path == "eval.sh",
+        )
+        (written if did_write else kept).append(relative_path)
+
+    contract_path = workspace_path / KNN_DEMO_CONTRACT_FILE
+    contract_path.write_text(
+        json.dumps(_contract_payload(goal_id=goal_id), ensure_ascii=False, indent=2, sort_keys=True)
+        + "\n",
+        encoding="utf-8",
+    )
+    marker_path = workspace_path / ".loopx-knn-demo-workspace.public.json"
+    marker = {
+        "schema_version": KNN_DEMO_WORKSPACE_SCHEMA_VERSION,
+        "preset_id": "knn-demo",
+        "goal_id": goal_id or "",
+        "workspace_created": created,
+        "contract_file": KNN_DEMO_CONTRACT_FILE,
+        "editable_scope": list(KNN_DEMO_EDITABLE_SCOPE),
+        "protected_scope": list(KNN_DEMO_PROTECTED_SCOPE),
+        "dev_eval_command": KNN_DEMO_DEV_EVAL_COMMAND,
+        "holdout_eval_command": KNN_DEMO_HOLDOUT_EVAL_COMMAND,
+        "files_written": written,
+        "files_kept": kept,
+        "public_boundary": {
+            "raw_logs_recorded": False,
+            "private_artifacts_recorded": False,
+            "absolute_paths_recorded": False,
+            "credentials_recorded": False,
+        },
+    }
+    marker_path.write_text(
+        json.dumps(marker, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+    return marker