loopx 0.4.8__py3-none-any.whl

loopx/benchmark_adapters/skillsbench_batch.py

@@ -0,0 +1,124 @@
+"""SkillsBench batch runner helpers."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import re
+import time
+from typing import Any
+
+
+BATCH_CASE_INTERNAL_ARG_KEYS = frozenset(
+    {
+        "apt_risk_fail_fast_defaulted",
+        "bootstrap_light_fail_fast_defaulted",
+        "update_current_aggregate",
+        "verifier_bootstrap_fail_fast_defaulted",
+    }
+)
+
+
+def split_task_ids_arg(value: str | None) -> list[str]:
+    return [part for part in re.split(r"[,\s]+", value or "") if part]
+
+
+def safe_batch_suffix(task_id: str, index: int) -> str:
+    slug = re.sub(r"[^A-Za-z0-9_.-]+", "-", task_id).strip("-")
+    return f"{index + 1:02d}-{slug or 'task'}"
+
+
+def batch_task_ids(args: argparse.Namespace) -> list[str]:
+    return split_task_ids_arg(getattr(args, "task_ids", None)) or [
+        str(args.task_id)
+    ]
+
+
+def clone_args_for_batch_case(
+    args: argparse.Namespace,
+    *,
+    task_id: str,
+    index: int,
+    total: int,
+    run_group_id: str,
+) -> argparse.Namespace:
+    case_args = argparse.Namespace(**vars(args))
+    case_args.task_id = task_id
+    case_args.task_ids = None
+    case_args.run_group_id = run_group_id
+    if total > 1:
+        suffix = safe_batch_suffix(task_id, index)
+        if args.job_name:
+            case_args.job_name = f"{args.job_name}-{suffix}"
+        if args.rollout_name:
+            case_args.rollout_name = f"{args.rollout_name}-{suffix}"
+    return case_args
+
+
+def batch_case_args_to_cli(case_args: argparse.Namespace) -> list[str]:
+    cli: list[str] = []
+    for key, value in sorted(vars(case_args).items()):
+        if key == "task_ids":
+            continue
+        if key in BATCH_CASE_INTERNAL_ARG_KEYS:
+            continue
+        if (
+            key == "fail_fast_on_apt_risk"
+            and getattr(case_args, "apt_risk_fail_fast_defaulted", False)
+        ):
+            continue
+        if (
+            key == "fail_fast_on_verifier_bootstrap_risk"
+            and getattr(case_args, "verifier_bootstrap_fail_fast_defaulted", False)
+        ):
+            continue
+        option = "--" + key.replace("_", "-")
+        if key == "parallel_cases":
+            value = 1
+        if value is None:
+            continue
+        if isinstance(value, bool):
+            if value:
+                cli.append(option)
+            continue
+        cli.extend([option, str(value)])
+    return cli
+
+
+def parallel_batch_requires_subprocess_isolation(parallel_cases: int) -> bool:
+    return parallel_cases > 1
+
+
+class BatchCaseStartPacer:
+    """Enforce a minimum gap between starts in a batch run."""
+
+    def __init__(self, start_gap_sec: float | int | None) -> None:
+        self.start_gap_sec = max(0.0, float(start_gap_sec or 0.0))
+        self._lock = asyncio.Lock()
+        self._last_start_monotonic: float | None = None
+
+    async def wait_for_slot(self) -> float:
+        if self.start_gap_sec <= 0:
+            return 0.0
+        async with self._lock:
+            wait_sec = 0.0
+            now = time.monotonic()
+            if self._last_start_monotonic is not None:
+                wait_sec = max(
+                    0.0,
+                    self._last_start_monotonic + self.start_gap_sec - now,
+                )
+                if wait_sec > 0:
+                    await asyncio.sleep(wait_sec)
+            self._last_start_monotonic = time.monotonic()
+            return wait_sec
+
+    def annotate_payload(
+        self,
+        payload: dict[str, Any],
+        wait_sec: float,
+    ) -> dict[str, Any]:
+        if self.start_gap_sec > 0:
+            payload["batch_case_start_gap_sec"] = self.start_gap_sec
+            payload["batch_case_start_wait_sec"] = wait_sec
+        return payload

loopx/benchmark_adapters/skillsbench_bridge_guard.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+
+LOOPX_COMMAND_INSTRUMENTATION_SOURCE = r'''
+_SHELL_COMMAND_PREFIXES = {
+    "if", "then", "elif", "else", "while", "until", "do", "!", "time", "{",
+}
+_SHELL_COMMAND_WRAPPERS = {"command", "exec", "builtin", "nohup"}
+_NESTED_SHELLS = {"sh", "bash", "dash", "zsh", "ksh"}
+
+def _shell_tokens(command: str) -> list[str]:
+    try:
+        lexer = shlex.shlex(
+            command or "",
+            posix=True,
+            punctuation_chars=";&|()" + chr(10),
+        )
+        lexer.whitespace = " " + chr(9) + chr(13)
+        lexer.whitespace_split = True
+        lexer.commenters = ""
+        return list(lexer)
+    except ValueError:
+        return (command or "").replace(chr(10), " ; ").split()
+
+def _shell_basename(token: str) -> str:
+    return token.rsplit("/", 1)[-1]
+
+def _shell_separator(token: str) -> bool:
+    return bool(token) and all(char in ";&|()" or char == chr(10) for char in token)
+
+def _nested_shell_command(tokens: list[str], index: int) -> str | None:
+    cursor = index + 1
+    while cursor < len(tokens) and not _shell_separator(tokens[cursor]):
+        option = tokens[cursor]
+        if option == "--":
+            cursor += 1
+            continue
+        if option == "--command" or (
+            option.startswith("-")
+            and not option.startswith("--")
+            and "c" in option[1:]
+        ):
+            return tokens[cursor + 1] if cursor + 1 < len(tokens) else None
+        if not option.startswith("-"):
+            return None
+        cursor += 1
+    return None
+
+def loopx_invocation_argvs(command: str, *, _depth: int = 0) -> list[list[str]]:
+    tokens = _shell_tokens(command)
+    invocations: list[list[str]] = []
+    assignment_re = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=.*$")
+    expect_command = True
+    index = 0
+    while index < len(tokens):
+        token = tokens[index]
+        if _shell_separator(token):
+            expect_command = True
+            index += 1
+            continue
+        if not expect_command:
+            index += 1
+            continue
+        basename = _shell_basename(token)
+        if token in _SHELL_COMMAND_PREFIXES or assignment_re.match(token):
+            index += 1
+            continue
+        if basename in _SHELL_COMMAND_WRAPPERS:
+            index += 1
+            while index < len(tokens) and tokens[index].startswith("-"):
+                index += 1
+            continue
+        if basename == "env":
+            index += 1
+            while index < len(tokens) and (
+                tokens[index].startswith("-") or assignment_re.match(tokens[index])
+            ):
+                index += 1
+            continue
+        if basename in _NESTED_SHELLS:
+            nested = _nested_shell_command(tokens, index)
+            if nested is not None:
+                if _depth < 4:
+                    invocations.extend(loopx_invocation_argvs(nested, _depth=_depth + 1))
+                elif any(
+                    _shell_basename(item) == "loopx" for item in _shell_tokens(nested)
+                ):
+                    # Depth-limited nested shell requests are ambiguous; reject closed.
+                    invocations.extend([["loopx"], ["loopx"]])
+            expect_command = False
+            index += 1
+            continue
+        if basename == "loopx":
+            end = index + 1
+            while end < len(tokens) and not _shell_separator(tokens[end]):
+                end += 1
+            invocations.append(tokens[index:end])
+        expect_command = False
+        index += 1
+    return invocations
+
+def loopx_subcommands(command: str) -> list[str]:
+    invocations = loopx_invocation_argvs(command)
+    if not invocations:
+        return []
+    tokens = invocations[0]
+    out: list[str] = []
+    skip = False
+    valued_options = {
+        "--goal-id", "--todo-id", "--claimed-by", "--status", "--note",
+        "--evidence", "--classification", "--registry", "--runtime-root",
+        "--slots", "--source", "--format", "--project", "--goal-text",
+        "--agent-id", "--available-capability", "--runtime-profile",
+        "--host-surface", "--scheduler-owner", "--execution-mode",
+        "-H", "-O", "-M", "--role", "--task-class", "--action-kind",
+        "--text",
+    }
+    for token in tokens[1:]:
+        if skip:
+            skip = False
+            continue
+        if token in valued_options:
+            skip = True
+            continue
+        if token.startswith("--"):
+            continue
+        if token.startswith("-"):
+            continue
+        if re.match(r"^[A-Za-z][A-Za-z0-9_-]{0,40}$", token):
+            out.append(token)
+            if len(out) >= 2:
+                break
+    return out
+
+def loopx_invocation_count(command: str) -> int:
+    return len(loopx_invocation_argvs(command))
+
+def enforce_single_loopx_invocation(count, record, append_record) -> None:
+    if count <= 1:
+        return
+    stderr_text = (
+        "bridge request rejected: send exactly one LoopX CLI command per "
+        "operation=exec request" + chr(10)
+    )
+    complete_record = dict(record)
+    complete_record.update({
+        "record_phase": "complete",
+        "returncode": 2,
+        "success": False,
+        "stdout_bytes": 0,
+        "stderr_bytes": len(stderr_text.encode("utf-8")),
+        "failure_category": "multiple_loopx_commands_per_bridge_request",
+    })
+    append_record(complete_record)
+    sys.stderr.write(stderr_text)
+    raise SystemExit(2)
+'''.strip()
+
+
+LOOPX_TODO_OUTPUT_INSTRUMENTATION_SOURCE = r'''
+def successful_todo_add_id_from_bridge_stdout(
+    subcommands: list[str],
+    bridge_stdout: str,
+) -> str:
+    if subcommands[:2] != ["todo", "add"]:
+        return ""
+    try:
+        response = json.loads(bridge_stdout or "")
+    except (json.JSONDecodeError, TypeError, ValueError):
+        return ""
+    if not isinstance(response, dict):
+        return ""
+    if response.get("ok") is False or response.get("success") is False:
+        return ""
+    for field in ("exit_code", "returncode"):
+        value = response.get(field)
+        if isinstance(value, int) and not isinstance(value, bool) and value != 0:
+            return ""
+
+    candidates = [response]
+    nested_stdout = response.get("stdout")
+    if isinstance(nested_stdout, str):
+        try:
+            nested_response = json.loads(nested_stdout)
+        except (json.JSONDecodeError, TypeError, ValueError):
+            nested_response = None
+        if isinstance(nested_response, dict):
+            candidates.append(nested_response)
+
+    todo_ids = {
+        str(candidate.get("todo_id") or "")
+        for candidate in candidates
+        if re.fullmatch(
+            r"todo_[A-Za-z0-9_-]{6,80}",
+            str(candidate.get("todo_id") or ""),
+        )
+    }
+    return next(iter(todo_ids)) if len(todo_ids) == 1 else ""
+
+def attach_successful_todo_add_id(record, subcommands, bridge_stdout) -> None:
+    if record.get("returncode") != 0 or record.get("loopx_todo_id"):
+        return
+    todo_id = successful_todo_add_id_from_bridge_stdout(
+        subcommands,
+        bridge_stdout,
+    )
+    if todo_id:
+        record["loopx_todo_id"] = todo_id
+'''.strip()

loopx/benchmark_adapters/skillsbench_bridge_summary.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+def bridge_summary_has_inflight_operation(path: Path | None) -> bool:
+    if path is None or not path.exists():
+        return False
+    starts = 0
+    completions = 0
+    try:
+        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+    except OSError:
+        return False
+    for line in lines:
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(record, dict):
+            continue
+        phase = str(record.get("record_phase") or "").strip().lower()
+        if phase == "complete" and not bridge_operation_record_interrupted(record):
+            completions += 1
+        elif phase == "start" or record.get("operation_observed") is True:
+            starts += 1
+    return starts > completions
+
+
+def bridge_summary_has_meaningful_agent_progress(
+    path: Path | None,
+    *,
+    allow_loopx_closeout: bool,
+) -> bool:
+    """Return true once the worker has done task work or a real closeout action."""
+
+    if path is None or not path.exists():
+        return False
+    try:
+        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+    except OSError:
+        return False
+    closeout_subcommands = {
+        ("todo", "complete"),
+        ("todo", "update"),
+        ("refresh-state",),
+        ("quota", "spend-slot"),
+    }
+    for line in lines:
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(record, dict):
+            continue
+        phase = str(record.get("record_phase") or "").strip().lower()
+        if phase == "complete" and bridge_operation_record_interrupted(record):
+            continue
+        if record.get("task_facing_operation") is True:
+            return True
+        if allow_loopx_closeout and record.get("loopx_state_write") is True:
+            subcommands = record.get("loopx_subcommands")
+            if isinstance(subcommands, list):
+                key = tuple(str(item) for item in subcommands[:2])
+                if key in closeout_subcommands:
+                    return True
+    return False
+
+
+def bridge_summary_has_successful_task_file_write(path: Path | None) -> bool:
+    """Return true after the worker successfully writes task-facing files."""
+
+    return bridge_summary_has_successful_task_operation(path, operation="write_file")
+
+
+def bridge_summary_has_successful_task_operation(
+    path: Path | None,
+    *,
+    operation: str | None = None,
+) -> bool:
+    """Return true after a successful task-facing bridge operation."""
+
+    if path is None or not path.exists():
+        return False
+    try:
+        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+    except OSError:
+        return False
+    for line in lines:
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(record, dict):
+            continue
+        phase = str(record.get("record_phase") or "").strip().lower()
+        if phase != "complete" or bridge_operation_record_interrupted(record):
+            continue
+        if operation is not None and record.get("operation") != operation:
+            continue
+        if record.get("task_facing_operation") is not True:
+            continue
+        if record.get("success") is True or record.get("returncode") == 0:
+            return True
+    return False
+
+
+def bridge_summary_task_progress_receipt(path: Path | None) -> dict[str, Any]:
+    """Reduce one agent invocation to bounded task-facing content changes."""
+
+    receipt: dict[str, Any] = {
+        "schema_version": "skillsbench_bridge_task_progress_receipt_v0",
+        "status": "no_verified_task_mutation",
+        "task_facing_operation_count": 0,
+        "task_facing_success_count": 0,
+        "successful_task_file_write_count": 0,
+        "successful_task_file_change_count": 0,
+        "raw_material_recorded": False,
+    }
+    if path is None or not path.exists():
+        return receipt
+    try:
+        lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+    except OSError:
+        return receipt
+    for line in lines:
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if not isinstance(record, dict):
+            continue
+        receipt["raw_material_recorded"] = receipt["raw_material_recorded"] or any(
+            record.get(field) is True
+            for field in (
+                "raw_request_recorded",
+                "raw_stdout_recorded",
+                "raw_stderr_recorded",
+                "raw_task_text_recorded",
+                "credential_values_recorded",
+                "host_paths_recorded",
+                "remote_paths_recorded",
+            )
+        )
+        phase = str(record.get("record_phase") or "").strip().lower()
+        if phase != "complete" or bridge_operation_record_interrupted(record):
+            continue
+        if record.get("task_facing_operation") is not True:
+            continue
+        receipt["task_facing_operation_count"] += 1
+        success = record.get("success")
+        successful = (
+            success if isinstance(success, bool) else record.get("returncode") == 0
+        )
+        if not successful:
+            continue
+        receipt["task_facing_success_count"] += 1
+        if (
+            record.get("operation") == "write_file"
+            and record.get("durable_task_write") is True
+        ):
+            receipt["successful_task_file_write_count"] += 1
+            if record.get("durable_task_content_changed") is True:
+                receipt["successful_task_file_change_count"] += 1
+    if (
+        receipt["successful_task_file_change_count"] > 0
+        and receipt["raw_material_recorded"] is False
+    ):
+        receipt["status"] = "verified_task_content_change"
+    return receipt
+
+
+def bridge_operation_record_interrupted(record: dict[str, Any]) -> bool:
+    rc = record.get("returncode")
+    if isinstance(rc, int) and not isinstance(rc, bool) and rc < 0:
+        return True
+    category = str(record.get("failure_category") or "")
+    return record.get("interrupted") is True or category in {
+        "bridge_operation_interrupted",
+        "bridge_controller_interrupted",
+    }
+
+
+def prompt_requires_meaningful_bridge_progress(prompt: str, *, route: str) -> bool:
+    text = prompt or ""
+    if "Private bridge command:" not in text:
+        return False
+    if route == "codex-app-server-goal-baseline":
+        return True
+    lowered = text.lower()
+    return any(
+        marker in lowered
+        for marker in (
+            "--- task instruction ---",
+            "first action required",
+            "mandatory product-mode solver checkpoint",
+            "mandatory host-local bridge recovery checkpoint",
+            "must start with either a task-facing sandbox bridge operation",
+            "task-facing validation or repair operation",
+        )
+    )