loopgym 0.1.0__py3-none-any.whl

loopgym/__init__.py

@@ -0,0 +1,7 @@
+"""LoopGym — OpenAI Gym equivalent for LSS-defined agent loops."""
+
+from loopgym.envs.base import LoopEnv, Observation
+from loopgym.registry import list_envs, make
+
+__version__ = "0.1.0"
+__all__ = ["LoopEnv", "Observation", "list_envs", "make"]

loopgym/cli.py

@@ -0,0 +1,47 @@
+"""LoopGym CLI."""
+
+from __future__ import annotations
+
+import argparse
+import json
+
+import loopgym as lg
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="LoopGym — run loop environments")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    list_cmd = sub.add_parser("list", help="List registered environments")
+    list_cmd.set_defaults(func=_cmd_list)
+
+    run_cmd = sub.add_parser("run", help="Run an environment episode")
+    run_cmd.add_argument("env_id", help="Environment ID")
+    run_cmd.add_argument("--task-id", default="default")
+    run_cmd.add_argument("--seed", type=int, default=0)
+    run_cmd.add_argument("--spec-path", default=None)
+    run_cmd.set_defaults(func=_cmd_run)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+def _cmd_list(_args: argparse.Namespace) -> None:
+    for env_id in lg.list_envs():
+        print(env_id)
+
+
+def _cmd_run(args: argparse.Namespace) -> None:
+    env = lg.make(args.env_id, spec_path=args.spec_path, seed=args.seed)
+    if hasattr(env, "run_episode"):
+        result = env.run_episode(task_id=args.task_id, seed=args.seed)
+        print(json.dumps(result, indent=2))
+    else:
+        obs = env.reset(task_id=args.task_id, seed=args.seed)
+        while not env.done:
+            obs, reward, done, info = env.step()
+            print(f"step reward={reward:.3f} quality={obs.quality_score:.3f} done={done}")
+
+
+if __name__ == "__main__":
+    main()

loopgym/envs/__init__.py

@@ -0,0 +1,8 @@
+"""Loop environment implementations."""
+
+from loopgym.envs.base import LoopEnv, Observation
+from loopgym.envs.live import LiveEnv
+from loopgym.envs.replay import ReplayEnv
+from loopgym.envs.sim import SimEnv
+
+__all__ = ["LoopEnv", "LiveEnv", "Observation", "ReplayEnv", "SimEnv"]

loopgym/envs/base.py

@@ -0,0 +1,63 @@
+"""Loop environment base classes."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class Observation:
+    """Gym-style observation returned by reset/step."""
+
+    task_id: str
+    iteration: int
+    output: str
+    quality_score: float
+    objective: str
+    done: bool
+    info: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "task_id": self.task_id,
+            "iteration": self.iteration,
+            "output": self.output,
+            "quality_score": self.quality_score,
+            "objective": self.objective,
+            "done": self.done,
+            "info": self.info,
+        }
+
+
+class LoopEnv(ABC):
+    """Abstract loop environment (Gym-style API)."""
+
+    def __init__(self, env_id: str) -> None:
+        self.env_id = env_id
+        self._done = False
+        self._task_id = ""
+        self._obs: Observation | None = None
+
+    @property
+    def done(self) -> bool:
+        return self._done
+
+    @property
+    def observation(self) -> Observation | None:
+        return self._obs
+
+    @abstractmethod
+    def reset(self, task_id: str = "", seed: int | None = None, **kwargs: Any) -> Observation:
+        """Reset environment for a new episode."""
+
+    @abstractmethod
+    def step(self, action: Any = None) -> tuple[Observation, float, bool, dict[str, Any]]:
+        """Advance one loop iteration. Returns (obs, reward, done, info)."""
+
+    def close(self) -> None:
+        """Release resources."""
+
+    def _reward(self, quality_score: float, success: bool) -> float:
+        return quality_score if success else quality_score - 0.5

loopgym/envs/live.py

@@ -0,0 +1,80 @@
+"""LiveEnv — real LLM APIs (optional, user-provided keys)."""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+from loopgym.envs.base import LoopEnv, Observation
+from loopgym.envs.sim import SimEnv
+from loopgym.runtime.loop_runtime import LLMClient
+
+
+class _OpenAILiveLLM:
+    """Thin wrapper for OpenAI-compatible APIs (optional dependency)."""
+
+    def __init__(self, model: str = "gpt-4.1-mini", api_key: str | None = None) -> None:
+        self.model = model
+        self.api_key = api_key or os.environ.get("OPENAI_API_KEY", "")
+        self.tokens_used = 0
+        if not self.api_key:
+            raise ValueError(
+                "LiveEnv requires OPENAI_API_KEY or api_key= parameter. "
+                "Use SimEnv for keyless testing."
+            )
+        try:
+            import openai  # noqa: PLC0415
+        except ImportError as exc:
+            raise ImportError("Install openai: pip install openai") from exc
+        self._client = openai.OpenAI(api_key=self.api_key)
+
+    def complete(self, prompt: str, role: str = "default") -> str:
+        response = self._client.chat.completions.create(
+            model=self.model,
+            messages=[
+                {"role": "system", "content": f"You are a {role} agent in a loop."},
+                {"role": "user", "content": prompt},
+            ],
+            temperature=0.2,
+        )
+        text = response.choices[0].message.content or ""
+        if response.usage:
+            self.tokens_used += response.usage.total_tokens
+        return text
+
+
+class LiveEnv(LoopEnv):
+    """Live environment using real LLM APIs. Falls back to documenting key requirement."""
+
+    def __init__(
+        self,
+        env_id: str,
+        spec: dict[str, Any],
+        llm: LLMClient | None = None,
+        **sim_kwargs: Any,
+    ) -> None:
+        super().__init__(env_id)
+        self._sim = SimEnv(env_id, spec, **sim_kwargs)
+        self._llm = llm
+        if llm is not None:
+            self._sim._runtime = None  # will be set on reset with custom llm
+
+    def reset(self, task_id: str = "", seed: int | None = None, **kwargs: Any) -> Observation:
+        obs = self._sim.reset(task_id=task_id, seed=seed, **kwargs)
+        if self._llm and self._sim._runtime:
+            self._sim._runtime.llm = self._llm
+        return obs
+
+    def step(self, action: Any = None) -> tuple[Observation, float, bool, dict[str, Any]]:
+        return self._sim.step(action)
+
+    @classmethod
+    def with_openai(
+        cls,
+        env_id: str,
+        spec: dict[str, Any],
+        model: str = "gpt-4.1-mini",
+        api_key: str | None = None,
+        **sim_kwargs: Any,
+    ) -> LiveEnv:
+        return cls(env_id, spec, llm=_OpenAILiveLLM(model=model, api_key=api_key), **sim_kwargs)

loopgym/envs/replay.py

@@ -0,0 +1,197 @@
+"""ReplayEnv — replay LoopNet trajectories from ln/record-v1 JSONL."""
+
+from __future__ import annotations
+
+import json
+import os
+import random
+from pathlib import Path
+from typing import Any
+
+from loopgym.envs.base import LoopEnv, Observation
+
+
+def _default_loopnet_seed_path() -> Path | None:
+    """Resolve LoopNet seed corpus from env, sibling clone, or CI deps layout."""
+    loopgym_root = Path(__file__).resolve().parents[2]
+    candidates: list[Path] = []
+    env_path = os.environ.get("LOOPNET_SEED_PATH")
+    if env_path:
+        candidates.append(Path(env_path))
+    candidates.extend(
+        [
+            loopgym_root.parent / "04-loopnet" / "data" / "seed" / "records.jsonl",
+            loopgym_root.parent / "loopnet" / "data" / "seed" / "records.jsonl",
+            loopgym_root / "deps" / "loopnet" / "data" / "seed" / "records.jsonl",
+        ]
+    )
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    return None
+
+
+def load_loopnet_records(path: Path) -> list[dict[str, Any]]:
+    """Load LoopNet records from JSONL."""
+    records: list[dict[str, Any]] = []
+    with path.open(encoding="utf-8") as handle:
+        for line_no, line in enumerate(handle, start=1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"{path}:{line_no}: invalid JSON — {exc}") from exc
+    return records
+
+
+def record_to_trajectory(record: dict[str, Any]) -> list[dict[str, Any]]:
+    """Map ln/record-v1 trajectory steps to ReplayEnv step dicts."""
+    steps: list[dict[str, Any]] = []
+    objective = str(record.get("objective", ""))
+    for step in record.get("trajectory") or []:
+        goal = float(step.get("goal_score", step.get("primary_quality", 0.0)))
+        steps.append(
+            {
+                "iteration": int(step.get("iteration", len(steps) + 1)),
+                "output": (
+                    f"[loopnet replay] {objective[:80]}… "
+                    f"iter={step.get('iteration')} goal={goal:.3f}"
+                ),
+                "quality_score": goal,
+                "cost_usd": step.get("cost_usd"),
+                "latency_seconds": step.get("latency_seconds"),
+                "failure_codes": list(step.get("failure_codes") or []),
+            }
+        )
+    return steps
+
+
+class ReplayEnv(LoopEnv):
+    """Replay recorded trajectories from LoopNet ln/record-v1 records."""
+
+    def __init__(
+        self,
+        env_id: str,
+        trajectory_path: str | Path | None = None,
+        records_path: str | Path | None = None,
+    ) -> None:
+        super().__init__(env_id)
+        self.trajectory_path = Path(trajectory_path) if trajectory_path else None
+        resolved_records = Path(records_path) if records_path else None
+        if resolved_records and resolved_records.exists():
+            self.records_path = resolved_records
+        else:
+            self.records_path = _default_loopnet_seed_path()
+        self._records: list[dict[str, Any]] = []
+        self._record: dict[str, Any] | None = None
+        self._trajectory: list[dict[str, Any]] = []
+        self._index = 0
+
+    def _load_records_corpus(self) -> None:
+        if self._records or not self.records_path or not self.records_path.exists():
+            return
+        self._records = load_loopnet_records(self.records_path)
+
+    def _select_record(self, task_id: str, seed: int | None) -> dict[str, Any] | None:
+        self._load_records_corpus()
+        if not self._records:
+            return None
+
+        if task_id.startswith("ln-"):
+            for record in self._records:
+                if record.get("record_id") == task_id:
+                    return record
+
+        if task_id and task_id != "default":
+            for record in self._records:
+                if record.get("loop_name") == task_id:
+                    return record
+
+        rng = random.Random(seed if seed is not None else 0)
+        return rng.choice(self._records)
+
+    def reset(self, task_id: str = "", seed: int | None = None, **kwargs: Any) -> Observation:
+        record_id = kwargs.get("record_id")
+        self._task_id = task_id or "default"
+        self._index = 0
+        self._trajectory = []
+        self._done = False
+        self._record = None
+
+        if self.trajectory_path and self.trajectory_path.exists():
+            with self.trajectory_path.open(encoding="utf-8") as handle:
+                data = json.load(handle)
+            self._trajectory = list(data.get("steps") or data.get("history") or [])
+        else:
+            if record_id:
+                self._load_records_corpus()
+                for record in self._records:
+                    if record.get("record_id") == record_id:
+                        self._record = record
+                        break
+            else:
+                self._record = self._select_record(self._task_id, seed)
+
+            if self._record:
+                self._trajectory = record_to_trajectory(self._record)
+                self._task_id = str(self._record.get("record_id", self._task_id))
+
+        if not self._trajectory:
+            self._trajectory = [
+                {"iteration": 1, "output": "replay fallback step 1", "quality_score": 0.5},
+                {"iteration": 2, "output": "replay fallback step 2", "quality_score": 0.85},
+            ]
+
+        step = self._trajectory[0]
+        objective = str((self._record or {}).get("objective", "Replay LoopNet trajectory"))
+        self._obs = Observation(
+            task_id=self._task_id,
+            iteration=int(step.get("iteration", 1)),
+            output=str(step.get("output", "")),
+            quality_score=float(step.get("quality_score", 0.0)),
+            objective=objective,
+            done=False,
+            info={
+                "mode": "replay",
+                "total_steps": len(self._trajectory),
+                "record_id": (self._record or {}).get("record_id"),
+                "outcome": (self._record or {}).get("outcome"),
+            },
+        )
+        return self._obs
+
+    def step(self, action: Any = None) -> tuple[Observation, float, bool, dict[str, Any]]:
+        self._index += 1
+        if self._index >= len(self._trajectory):
+            self._done = True
+            obs = self._obs or Observation(
+                task_id=self._task_id,
+                iteration=0,
+                output="",
+                quality_score=0.0,
+                objective="Replay LoopNet trajectory",
+                done=True,
+            )
+            return obs, 0.0, True, {"reason": "trajectory_exhausted"}
+
+        step = self._trajectory[self._index]
+        quality = float(step.get("quality_score", 0.0))
+        self._done = self._index >= len(self._trajectory) - 1
+        objective = str((self._record or {}).get("objective", "Replay LoopNet trajectory"))
+        self._obs = Observation(
+            task_id=self._task_id,
+            iteration=int(step.get("iteration", self._index + 1)),
+            output=str(step.get("output", "")),
+            quality_score=quality,
+            objective=objective,
+            done=self._done,
+            info={
+                "mode": "replay",
+                "step_index": self._index,
+                "record_id": (self._record or {}).get("record_id"),
+                "failure_codes": step.get("failure_codes", []),
+            },
+        )
+        return self._obs, quality, self._done, {"step_index": self._index}

loopgym/envs/sim.py

@@ -0,0 +1,183 @@
+"""SimEnv — mock LLM + mock oracles (no API keys)."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from loopgym.envs.base import LoopEnv, Observation
+from loopgym.runtime.compiler import compile_lss
+from loopgym.runtime.loop_runtime import LoopRuntime, LoopState, MockLLM, load_lss_spec
+
+
+def _task_input(spec: dict[str, Any], task_id: str, tasks_path: Path | None) -> str:
+    """Resolve task input from tasks.json or spec examples."""
+    if tasks_path and tasks_path.exists():
+        with tasks_path.open(encoding="utf-8") as fh:
+            tasks = json.load(fh)
+        for task in tasks.get("tasks", []):
+            if task.get("id") == task_id:
+                payload = task.get("input") or task
+                return json.dumps(payload) if isinstance(payload, dict) else str(payload)
+    inputs = spec.get("inputs") or {}
+    examples = inputs.get("examples") or []
+    if examples:
+        first = examples[0]
+        if isinstance(first, dict):
+            return json.dumps(first)
+        return str(first)
+    return f"task:{task_id or 'default'}"
+
+
+def _mock_llm_seed(env_seed: int, task_id: str, loop_name: str) -> str:
+    """Deterministic seed for reproducible trajectories."""
+    return f"{env_seed}:{task_id}:{loop_name}"
+
+
+class SimEnv(LoopEnv):
+    """Simulation environment with MockLLM — no API keys required."""
+
+    def __init__(
+        self,
+        env_id: str,
+        spec: dict[str, Any],
+        spec_path: Path | None = None,
+        tasks_path: Path | None = None,
+        seed: int = 0,
+    ) -> None:
+        super().__init__(env_id)
+        self.spec = spec
+        self.spec_path = spec_path
+        self.tasks_path = tasks_path
+        self.seed = seed
+        self._graph = compile_lss(spec)
+        self._runtime: LoopRuntime | None = None
+        self._state: LoopState | None = None
+        self._user_input = ""
+
+    @classmethod
+    def from_spec_file(
+        cls,
+        env_id: str,
+        spec_path: str | Path,
+        tasks_path: str | Path | None = None,
+        seed: int = 0,
+    ) -> SimEnv:
+        path = Path(spec_path)
+        tasks = Path(tasks_path) if tasks_path else path.parent / "tasks.json"
+        return cls(
+            env_id=env_id,
+            spec=load_lss_spec(path),
+            spec_path=path,
+            tasks_path=tasks if tasks.exists() else None,
+            seed=seed,
+        )
+
+    def reset(self, task_id: str = "", seed: int | None = None, **kwargs: Any) -> Observation:
+        if seed is not None:
+            self.seed = seed
+        self._task_id = task_id or "default"
+        self._user_input = _task_input(self.spec, self._task_id, self.tasks_path)
+        llm_seed = _mock_llm_seed(self.seed, self._task_id, self._graph.loop_name)
+        llm = MockLLM(seed=llm_seed)
+        self._runtime = LoopRuntime(self.spec, llm=llm)
+        self._state = LoopState()
+        self._done = False
+
+        self._obs = Observation(
+            task_id=self._task_id,
+            iteration=0,
+            output="",
+            quality_score=0.0,
+            objective=self._graph.objective,
+            done=False,
+            info={
+                "env_id": self.env_id,
+                "seed": self.seed,
+                "loop_name": self._graph.loop_name,
+                "user_input": self._user_input,
+            },
+        )
+        return self._obs
+
+    def step(self, action: Any = None) -> tuple[Observation, float, bool, dict[str, Any]]:
+        if self._runtime is None or self._state is None:
+            raise RuntimeError("Call reset() before step()")
+        if self._done:
+            obs = self._obs or Observation(
+                task_id=self._task_id,
+                iteration=0,
+                output="",
+                quality_score=0.0,
+                objective=self._graph.objective,
+                done=True,
+            )
+            return obs, 0.0, True, {"reason": "already_done"}
+
+        if isinstance(action, dict) and action.get("output"):
+            self._state.output = str(action["output"])
+            self._state.quality_score, feedback = self._runtime._evaluate(self._state, self._user_input)
+            self._state.iteration += 1
+            self._state.history.append(
+                {
+                    "iteration": self._state.iteration,
+                    "output": self._state.output,
+                    "quality_score": self._state.quality_score,
+                    "feedback": feedback,
+                    "agent_override": True,
+                }
+            )
+            self._runtime._check_termination(self._state)
+        else:
+            self._state = self._runtime.step_once(self._state, self._user_input)
+
+        self._done = self._state.terminated
+        success = self._state.quality_score >= self._runtime.quality_threshold
+        reward = self._reward(self._state.quality_score, success)
+
+        info = {
+            "iteration": self._state.iteration,
+            "termination_reason": self._state.termination_reason,
+            "success": success,
+            "history_len": len(self._state.history),
+        }
+        if self._state.history:
+            info["last_feedback"] = self._state.history[-1].get("feedback", "")
+
+        self._obs = Observation(
+            task_id=self._task_id,
+            iteration=self._state.iteration,
+            output=self._state.output,
+            quality_score=self._state.quality_score,
+            objective=self._graph.objective,
+            done=self._done,
+            info=info,
+        )
+        return self._obs, reward, self._done, info
+
+    def run_episode(self, task_id: str = "", seed: int | None = None) -> dict[str, Any]:
+        """Run full episode until done (convenience for benchmarks)."""
+        self.reset(task_id=task_id, seed=seed)
+        total_reward = 0.0
+        steps = 0
+        while not self.done:
+            _, reward, _, info = self.step()
+            total_reward += reward
+            steps += 1
+        return {
+            "task_id": self._task_id,
+            "seed": self.seed,
+            "steps": steps,
+            "total_reward": total_reward,
+            "success": info.get("success", False),
+            "quality_score": self._obs.quality_score if self._obs else 0.0,
+            "trajectory": [
+                {
+                    "iteration": h["iteration"],
+                    "output": h["output"],
+                    "quality_score": h["quality_score"],
+                }
+                for h in (self._state.history if self._state else [])
+            ],
+        }

loopgym/evaluators/__init__.py

@@ -0,0 +1,6 @@
+"""Evaluator plugins for loop execution."""
+
+from loopgym.evaluators.deterministic import run_deterministic
+from loopgym.evaluators.rubric import run_rubric
+
+__all__ = ["run_deterministic", "run_rubric"]

loopgym/evaluators/deterministic.py

@@ -0,0 +1,41 @@
+"""Deterministic evaluator implementations."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def run_deterministic(implementation: str, output: str, context: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Run a deterministic evaluator by implementation reference."""
+    ctx = context or {}
+    if implementation == "evaluators.word_count_max":
+        max_words = int(ctx.get("max_words", 100))
+        count = len(output.split())
+        passed = count <= max_words
+        return {
+            "passed": passed,
+            "score": min(1.0, max_words / max(count, 1)),
+            "word_count": count,
+            "failure_codes": [] if passed else ["fail.false_fail"],
+        }
+    if implementation == "evaluators.test_pass_rate":
+        rate = float(ctx.get("mock_pass_rate", 0.0))
+        passed = rate >= float(ctx.get("threshold", 1.0))
+        return {
+            "passed": passed,
+            "score": rate,
+            "test_pass_rate": rate,
+            "failure_codes": [] if passed else ["fail.false_fail"],
+        }
+    if implementation == "evaluators.citation_count_min":
+        citations = len(re.findall(r"\[[\d]+\]|\(\d{4}\)", output))
+        min_citations = int(ctx.get("min_citations", 3))
+        passed = citations >= min_citations
+        return {
+            "passed": passed,
+            "score": min(1.0, citations / min_citations),
+            "citation_count": citations,
+            "failure_codes": [] if passed else ["fail.false_fail"],
+        }
+    return {"passed": True, "score": 1.0, "failure_codes": []}

loopgym/evaluators/rubric.py

@@ -0,0 +1,45 @@
+"""LLM rubric evaluator helpers."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+
+class RubricLLM(Protocol):
+    def complete(self, prompt: str, role: str = "default") -> str: ...
+
+
+def run_rubric(
+    llm: RubricLLM,
+    output: str,
+    objective: str,
+    rubric: dict[str, Any],
+    role: str = "evaluator",
+) -> dict[str, Any]:
+    """Score output against an LSS rubric using an LLM backend."""
+    threshold = float(rubric.get("pass_threshold", 0.8))
+    dimensions = rubric.get("dimensions") or []
+    dim_names = ", ".join(d.get("name", "quality") for d in dimensions) or "quality"
+
+    feedback = llm.complete(
+        f"Evaluate ({dim_names}) against objective '{objective}':\n{output}",
+        role=role,
+    )
+
+    score = 0.0
+    for token in feedback.split():
+        try:
+            val = float(token.rstrip("."))
+            if 0.0 <= val <= 1.0:
+                score = val
+        except ValueError:
+            pass
+
+    passed = score >= threshold
+    return {
+        "passed": passed,
+        "score": score,
+        "feedback": feedback,
+        "failure_codes": [] if passed else ["fail.self_grade"],
+        "dimension_scores": {d.get("name", "quality"): score for d in dimensions},
+    }