loopx 0.4.8__py3-none-any.whl

loopx/benchmarks/read_models/benchmark_result.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from typing import Any
+
+from ...control_plane.runtime.public_safety import (
+    compact_numeric_map,
+    public_safe_compact_list,
+    public_safe_compact_text,
+)
+
+BENCHMARK_RESULT_SCHEMA_VERSION = "benchmark_result_v0"
+CONTROL_PLANE_SCORE_COMPONENTS = (
+    "restartability",
+    "stale_state_avoidance",
+    "evidence_discipline",
+    "boundary_safety",
+    "writeback_quality",
+    "gate_compliance",
+    "failure_attribution",
+    "overhead",
+)
+MAX_BENCHMARK_RESULT_LIST_ITEMS = 5
+
+
+def _benchmark_result_source(run: dict[str, Any]) -> dict[str, Any] | None:
+    nested = run.get("benchmark_result")
+    if (
+        isinstance(nested, dict)
+        and nested.get("schema_version") == BENCHMARK_RESULT_SCHEMA_VERSION
+    ):
+        return nested
+    if run.get("schema_version") == BENCHMARK_RESULT_SCHEMA_VERSION:
+        return run
+    return None
+
+
+def _compact_score_layer(value: Any, *, include_schema: bool = False) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {}
+    compact: dict[str, Any] = {}
+    if include_schema:
+        schema = public_safe_compact_text(value.get("schema_version"), limit=80)
+        if schema:
+            compact["schema_version"] = schema
+    for field in ("kind", "aggregation"):
+        text = public_safe_compact_text(value.get(field), limit=80)
+        if text:
+            compact[field] = text
+    if isinstance(value.get("passed"), bool):
+        compact["passed"] = value["passed"]
+    score_value = value.get("value")
+    if isinstance(score_value, (int, float)) and not isinstance(score_value, bool):
+        compact["value"] = score_value
+    return compact
+
+
+def _compact_control_plane_score(value: Any) -> dict[str, Any]:
+    compact = _compact_score_layer(value, include_schema=True)
+    if not isinstance(value, dict):
+        return compact
+    components = (
+        value.get("components") if isinstance(value.get("components"), dict) else {}
+    )
+    compact_components: dict[str, Any] = {}
+    for component in CONTROL_PLANE_SCORE_COMPONENTS:
+        score = components.get(component)
+        if isinstance(score, (int, float)) and not isinstance(score, bool):
+            compact_components[component] = score
+    if compact_components:
+        compact["components"] = compact_components
+        compact["component_order"] = list(compact_components)
+    return compact
+
+
+def compact_benchmark_result(run: dict[str, Any]) -> dict[str, Any] | None:
+    source = _benchmark_result_source(run)
+    if not source:
+        return None
+
+    compact: dict[str, Any] = {"schema_version": BENCHMARK_RESULT_SCHEMA_VERSION}
+    for field in (
+        "task_id",
+        "scenario_id",
+        "worker_mode",
+        "harness_identity",
+        "worker_surface",
+        "terminal_state",
+        "trace_publicness",
+    ):
+        value = public_safe_compact_text(source.get(field), limit=120)
+        if value:
+            compact[field] = value
+
+    official_score = _compact_score_layer(source.get("official_task_score"))
+    if official_score:
+        compact["official_task_score"] = official_score
+    control_score = _compact_control_plane_score(source.get("control_plane_score"))
+    if control_score:
+        compact["control_plane_score"] = control_score
+
+    count_fields = (
+        "step_count",
+        "wall_time_ms",
+        "validation_pass_count",
+        "validation_fail_count",
+        "changed_file_count",
+        "forbidden_access_count",
+        "stale_state_error_count",
+        "writeback_count",
+        "erroneous_write_count",
+        "human_intervention_count",
+        "spend_count",
+        "spend_before_validation_count",
+        "goal_tick_phase_coverage",
+        "cost_usd",
+    )
+    count_source = (
+        dict(source.get("counts")) if isinstance(source.get("counts"), dict) else {}
+    )
+    for field in count_fields:
+        if field in source:
+            count_source[field] = source[field]
+    counts = compact_numeric_map(count_source, keys=count_fields)
+    if counts:
+        compact["counts"] = counts
+
+    for field in (
+        "open_todo_preserved",
+        "archive_hygiene_passed",
+        "queue_contract_passed",
+        "state_reconstructable",
+        "stop_policy_correct",
+    ):
+        if isinstance(source.get(field), bool):
+            compact[field] = source[field]
+
+    labels = public_safe_compact_list(
+        source.get("failure_attribution_labels"),
+        limit=MAX_BENCHMARK_RESULT_LIST_ITEMS,
+    )
+    if labels:
+        compact["failure_attribution_labels"] = labels
+
+    if set(compact) == {"schema_version"}:
+        return None
+    return compact

loopx/benchmarks/read_models/benchmark_run_execution_contract.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from typing import Any
+
+from ...control_plane.runtime.public_safety import (
+    public_safe_compact_list,
+    public_safe_compact_text,
+)
+
+
+def compact_benchmark_run_execution_contract(
+    source: dict[str, Any],
+    *,
+    max_list_items: int,
+) -> dict[str, Any]:
+    sections = (
+        (
+            "benchmark_loop_contract",
+            _compact_fields(
+                source.get("benchmark_loop_contract"),
+                text_fields=(
+                    "schema_version",
+                    "route",
+                    "protocol_id",
+                    "claim_blocker",
+                ),
+                bool_fields=(
+                    "official_feedback_forwarded",
+                    "official_feedback_blinded",
+                    "blind_loop",
+                    "product_mode",
+                    "strict_treatment_claim_allowed",
+                ),
+                int_fields=("max_rounds_budget",),
+                text_limit=120,
+                max_list_items=max_list_items,
+            ),
+        ),
+        (
+            "claim_boundary",
+            _compact_fields(
+                source.get("claim_boundary"),
+                text_fields=("public_claim_allowed",),
+                bool_fields=(
+                    "bridge_connectivity_claim_allowed",
+                    "case_success_claim_allowed",
+                    "official_score_claim_allowed",
+                    "leaderboard_claim_allowed",
+                ),
+                list_fields=("forbidden_claims",),
+                text_limit=180,
+                max_list_items=max_list_items,
+            ),
+        ),
+        (
+            "agent",
+            _compact_fields(
+                source.get("agent"),
+                text_fields=("name", "import_path", "model"),
+                list_fields=("kwargs_keys",),
+                text_limit=120,
+                max_list_items=max_list_items,
+            ),
+        ),
+        (
+            "model_control",
+            _compact_fields(
+                source.get("model_control"),
+                text_fields=(
+                    "schema_version",
+                    "requested_model",
+                    "reported_model",
+                    "control_method",
+                    "control_status",
+                    "actual_model_source",
+                ),
+                bool_fields=("actual_model_verified",),
+                list_fields=("warning_labels",),
+                text_limit=140,
+                max_list_items=max_list_items,
+            ),
+        ),
+    )
+    return {field: value for field, value in sections if value}
+
+
+def _compact_fields(
+    value: Any,
+    *,
+    text_fields: tuple[str, ...] = (),
+    bool_fields: tuple[str, ...] = (),
+    int_fields: tuple[str, ...] = (),
+    list_fields: tuple[str, ...] = (),
+    text_limit: int,
+    max_list_items: int,
+) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {}
+
+    compact: dict[str, Any] = {}
+    for field in text_fields:
+        text = public_safe_compact_text(value.get(field), limit=text_limit)
+        if text:
+            compact[field] = text
+    for field in bool_fields:
+        if isinstance(value.get(field), bool):
+            compact[field] = value[field]
+    for field in int_fields:
+        number = value.get(field)
+        if isinstance(number, int) and not isinstance(number, bool):
+            compact[field] = number
+    for field in list_fields:
+        items = public_safe_compact_list(value.get(field), limit=max_list_items)
+        if items:
+            compact[field] = items
+    return compact

loopx/benchmarks/read_models/benchmark_run_failure.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+from typing import Any
+
+from ...control_plane.runtime.public_safety import (
+    public_safe_compact_list,
+    public_safe_compact_text,
+)
+
+
+def compact_benchmark_runner_failure(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {}
+
+    compact = _compact_fields(
+        value,
+        text_fields=("schema_version", "exception_type", "failure_class"),
+        bool_fields=(
+            "raw_error_recorded",
+            "raw_logs_read",
+            "raw_task_text_read",
+            "raw_trajectory_read",
+        ),
+    )
+    for field, compactor in (
+        ("controller_cutoff", _compact_controller_cutoff),
+        ("user_loop_recovery", _compact_user_loop_recovery),
+        ("native_goal_worker", _compact_native_goal_worker),
+    ):
+        nested = compactor(value.get(field))
+        if nested:
+            compact[field] = nested
+    return compact
+
+
+def compact_benchmark_runner_failure_fingerprint(
+    value: Any,
+    *,
+    max_list_items: int,
+) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {}
+
+    compact = _compact_fields(
+        value,
+        text_fields=(
+            "schema_version",
+            "error_len_bucket",
+            "fingerprint_confidence",
+            "apt_failure_subtype",
+            "pip_failure_subtype",
+            "retryability",
+        ),
+        int_fields=("line_count",),
+        text_limit=100,
+    )
+    for field, limit in (
+        ("matched_patterns", max_list_items * 4),
+        ("failure_line_dependency_classes", max_list_items * 2),
+        ("failure_reason_codes", max_list_items * 2),
+        ("terminal_failure_dependency_classes", max_list_items * 2),
+        ("terminal_failure_reason_codes", max_list_items * 2),
+        ("failure_dependency_endpoints", max_list_items * 2),
+        ("terminal_failure_dependency_endpoints", max_list_items * 2),
+    ):
+        items = public_safe_compact_list(value.get(field), limit=limit)
+        if items:
+            compact[field] = items
+    compact.update(
+        _compact_fields(
+            value,
+            bool_fields=(
+                "error_present",
+                "has_host_paths",
+                "has_urls",
+                "has_secret_like_tokens",
+                "raw_error_recorded",
+            ),
+        )
+    )
+    return compact
+
+
+def _compact_controller_cutoff(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {}
+    return _compact_fields(
+        value,
+        text_fields=("schema_version", "reason"),
+        bool_fields=("cutoff_before_followup",),
+        int_fields=(
+            "max_rounds_budget",
+            "initial_prompt_count",
+            "followup_prompt_count",
+            "stop_decision_count",
+        ),
+    )
+
+
+def _compact_user_loop_recovery(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {}
+    return _compact_fields(
+        value,
+        text_fields=("schema_version", "stage", "exception_type"),
+        bool_fields=("preserved_final_verify", "raw_error_recorded"),
+        int_fields=("round", "delta_events", "delta_tool_calls"),
+    )
+
+
+def _compact_native_goal_worker(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {}
+    compact = _compact_fields(
+        value,
+        text_fields=(
+            "schema_version",
+            "trace_status",
+            "failure_label",
+            "failure_category",
+            "first_blocker",
+        ),
+        int_fields=("trace_count",),
+    )
+    compact.update(
+        _compact_fields(
+            value,
+            bool_fields=(
+                "raw_transcript_recorded",
+                "raw_assistant_message_recorded",
+            ),
+        )
+    )
+    return compact
+
+
+def _compact_fields(
+    value: dict[str, Any],
+    *,
+    text_fields: tuple[str, ...] = (),
+    bool_fields: tuple[str, ...] = (),
+    int_fields: tuple[str, ...] = (),
+    text_limit: int = 140,
+) -> dict[str, Any]:
+    compact: dict[str, Any] = {}
+    for field in text_fields:
+        text = public_safe_compact_text(value.get(field), limit=text_limit)
+        if text:
+            compact[field] = text
+    for field in bool_fields:
+        if isinstance(value.get(field), bool):
+            compact[field] = value[field]
+    for field in int_fields:
+        number = value.get(field)
+        if isinstance(number, int) and not isinstance(number, bool):
+            compact[field] = number
+    return compact

loopx/benchmarks/read_models/benchmark_run_metrics.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from typing import Any
+
+from ...control_plane.runtime.public_safety import (
+    compact_numeric_map,
+    public_safe_compact_text,
+)
+
+
+def compact_benchmark_round_reward_trace(value: Any) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {}
+
+    compact: dict[str, Any] = {}
+    for field in (
+        "schema_version",
+        "source",
+        "round_index_origin",
+        "loop_score_policy",
+        "official_score_policy",
+    ):
+        text = public_safe_compact_text(value.get(field), limit=100)
+        if text:
+            compact[field] = text
+    for field in (
+        "success_observed",
+        "official_feedback_returned_to_agent",
+        "official_feedback_blinded",
+        "reward_feedback_forwarded",
+        "agent_declared_done",
+        "agent_declared_no_remaining_goals",
+        "product_mode_no_open_todo_below_passing_reward_stop",
+        "product_mode_typed_repair_required",
+        "product_mode_typed_repair_todo_identity_observed",
+        "product_mode_typed_repair_task_or_validation_delta",
+        "product_mode_typed_repair_delta_observed",
+        "product_mode_typed_repair_terminal",
+        "product_mode_typed_repair_terminal_receipt_consistent",
+        "official_score_recovered_from_controller_trace",
+    ):
+        if isinstance(value.get(field), bool):
+            compact[field] = value[field]
+    for field in (
+        "first_success_round",
+        "declared_done_round",
+        "max_rounds_budget",
+        "final_round",
+        "best_reward_round",
+        "official_score_recovered_round",
+        "product_mode_no_open_todo_below_passing_reward_streak",
+        "product_mode_no_open_todo_below_passing_reward_streak_threshold",
+        "product_mode_no_open_todo_below_passing_reward_round",
+        "product_mode_no_open_todo_below_passing_reward_stop_round",
+        "product_mode_no_open_todo_below_passing_reward_open_todo_count_public",
+        "product_mode_typed_repair_trigger_round",
+        "product_mode_typed_repair_round_entered",
+        "product_mode_typed_repair_round_entered_count",
+        "product_mode_typed_repair_resolved_round",
+        "product_mode_typed_repair_task_facing_success_delta",
+        "product_mode_typed_repair_terminal_round",
+        "product_mode_typed_repair_open_todo_count_public",
+    ):
+        raw = value.get(field)
+        if isinstance(raw, int) and not isinstance(raw, bool) and raw >= 0:
+            compact[field] = raw
+    for field in (
+        "final_round_reward",
+        "best_round_reward",
+        "declared_done_score",
+        "product_mode_no_open_todo_below_passing_reward_score",
+    ):
+        raw = value.get(field)
+        if isinstance(raw, (int, float)) and not isinstance(raw, bool):
+            compact[field] = float(raw)
+    score_status = public_safe_compact_text(
+        value.get("product_mode_no_open_todo_below_passing_reward_score_status"),
+        limit=100,
+    )
+    if score_status:
+        compact["product_mode_no_open_todo_below_passing_reward_score_status"] = (
+            score_status
+        )
+    for field in (
+        "product_mode_typed_repair_policy_id",
+        "product_mode_typed_repair_terminal_reason",
+    ):
+        text = public_safe_compact_text(value.get(field), limit=120)
+        if text:
+            compact[field] = text
+    for field in ("final_round_passed", "best_round_passed", "best_round_is_final"):
+        if isinstance(value.get(field), bool):
+            compact[field] = value[field]
+
+    records: list[dict[str, Any]] = []
+    raw_records = value.get("records")
+    if isinstance(raw_records, list):
+        seen_rounds: set[int] = set()
+        for item in raw_records:
+            if not isinstance(item, dict):
+                continue
+            agent_round = item.get("agent_round")
+            if (
+                not isinstance(agent_round, int)
+                or isinstance(agent_round, bool)
+                or agent_round <= 0
+                or agent_round in seen_rounds
+            ):
+                continue
+            seen_rounds.add(agent_round)
+            record: dict[str, Any] = {"agent_round": agent_round}
+            for field in ("reward_present", "passed"):
+                if isinstance(item.get(field), bool):
+                    record[field] = item[field]
+            reward = item.get("reward")
+            if isinstance(reward, (int, float)) and not isinstance(reward, bool):
+                record["reward"] = float(reward)
+            tool_calls = item.get("tool_calls")
+            if (
+                isinstance(tool_calls, int)
+                and not isinstance(tool_calls, bool)
+                and tool_calls >= 0
+            ):
+                record["tool_calls"] = tool_calls
+            records.append(record)
+    if records:
+        compact["records"] = sorted(records, key=lambda record: record["agent_round"])
+    return compact
+
+
+def compact_benchmark_overhead_attribution_counters(
+    value: Any,
+) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        return {}
+
+    compact: dict[str, Any] = {}
+    for field in (
+        "schema_version",
+        "source",
+        "trace_publicness",
+        "attribution_granularity",
+        "worker_step_counter_status",
+        "attribution_caveat",
+        "timeout_tier",
+    ):
+        text = public_safe_compact_text(value.get(field), limit=160)
+        if text:
+            compact[field] = text
+
+    for field in (
+        "raw_logs_read",
+        "raw_trace_recorded",
+        "raw_task_prompt_recorded",
+        "credential_values_recorded",
+        "loopx_worker_cli_bridge_required",
+        "observed_true_long_task_bar_met",
+        "expected_hours_scale_bar_met",
+    ):
+        if isinstance(value.get(field), bool):
+            compact[field] = value[field]
+
+    for field in (
+        "wall_time_seconds",
+        "wall_time_limit_seconds",
+        "input_tokens",
+        "cache_tokens",
+        "output_tokens",
+        "cost_usd",
+        "trial_count",
+        "errored_trial_count",
+        "worker_bridge_event_count",
+        "loopx_prompt_driven_case_cli_call_count",
+        "worker_counter_trace_trial_count",
+        "worker_benchmark_run_file_count",
+        "worker_benchmark_run_schema_ok_count",
+        "worker_self_validation_official_score_mismatch_count",
+        "worker_validation_scope_ambiguous_official_score_failure_count",
+        "worker_bridge_connected_official_score_failure_count",
+        "worker_startup_blocker_count",
+        "worker_setup_diagnostic_file_count",
+        "worker_setup_diagnostic_schema_ok_count",
+        "worker_submit_eligible_mismatch_count",
+        "worker_bridge_writeback_loss_count",
+        "environment_setup_failure_before_worker_count",
+        "pre_worker_agent_setup_failure_count",
+        "codex_runtime_goal_tool_trial_count",
+        "loopx_cli_call_total",
+        "loopx_required_cli_call_total",
+        "loopx_optional_context_cli_call_total",
+        "loopx_state_read_count",
+        "loopx_state_write_count",
+        "append_benchmark_run_success_count",
+        "append_benchmark_run_schema_rejected_count",
+        "codex_runtime_goal_tool_call_total",
+    ):
+        raw = value.get(field)
+        if isinstance(raw, (int, float)) and not isinstance(raw, bool):
+            compact[field] = raw
+
+    mismatch_reason = public_safe_compact_text(
+        value.get("worker_submit_eligible_mismatch_reason"),
+        limit=160,
+    )
+    if mismatch_reason:
+        compact["worker_submit_eligible_mismatch_reason"] = mismatch_reason
+
+    for field in ("loopx_cli_calls", "codex_runtime_goal_tool_calls"):
+        calls = compact_numeric_map(value.get(field))
+        if calls:
+            compact[field] = calls
+
+    return compact