loopx 0.4.8__py3-none-any.whl

loopx/control_plane/runtime/run_history.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Optional
+
+from ...boundary_authority import checkpointed_boundary_authority_summary
+from .run_context_retention import (
+    compact_goal_semantic_history,
+    latest_runs_with_agent_context,
+)
+
+
+LatestRun = Callable[[dict[str, Any]], Optional[dict[str, Any]]]
+GoalLifecycleFields = Callable[[dict[str, Any], Optional[dict[str, Any]]], dict[str, Any]]
+GoalProjection = Callable[[dict[str, Any]], Optional[dict[str, Any]]]
+RunCompactor = Callable[[dict[str, Any]], dict[str, Any]]
+QuotaStatus = Callable[[dict[str, Any]], dict[str, Any]]
+StatusNeutralRun = Callable[[dict[str, Any]], bool]
+
+
+def latest_run(
+    goal: dict[str, Any],
+    *,
+    is_status_neutral_run: StatusNeutralRun,
+) -> dict[str, Any] | None:
+    status_run = goal.get("latest_status_run")
+    if isinstance(status_run, dict) and not is_status_neutral_run(status_run):
+        return status_run
+
+    runs = goal.get("latest_runs")
+    if not isinstance(runs, list) or not runs:
+        return None
+    for run in runs:
+        if not isinstance(run, dict):
+            continue
+        if is_status_neutral_run(run):
+            continue
+        return run
+    return None
+
+
+def compact_goal_coordination(coordination: Any) -> dict[str, Any] | None:
+    if not isinstance(coordination, dict):
+        return None
+    compact = dict(coordination)
+    authority_summary = checkpointed_boundary_authority_summary(coordination)
+    if authority_summary is not None:
+        compact["checkpointed_boundary_authority"] = {
+            key: value
+            for key, value in authority_summary.items()
+            if key != "entries"
+        }
+    elif "checkpointed_boundary_authority" in compact:
+        compact.pop("checkpointed_boundary_authority", None)
+    return compact
+
+
+def build_run_history(
+    history: dict[str, Any],
+    *,
+    latest_run: LatestRun,
+    goal_lifecycle_fields: GoalLifecycleFields,
+    subagent_activity_for_goal: GoalProjection,
+    compact_run: RunCompactor,
+    quota_status: QuotaStatus,
+    display_limit: int | None = None,
+) -> dict[str, Any]:
+    display_limit = None if display_limit is None else max(0, display_limit)
+    goals: list[dict[str, Any]] = []
+    for goal in history.get("goals") or []:
+        if not isinstance(goal, dict):
+            continue
+        current_run = latest_run(goal)
+        lifecycle_fields = goal_lifecycle_fields(goal, current_run)
+        subagent_activity = subagent_activity_for_goal(goal)
+        latest_runs = [
+            compact_run(run)
+            for run in goal.get("latest_runs") or []
+            if isinstance(run, dict)
+        ]
+        if display_limit is not None:
+            latest_runs = latest_runs_with_agent_context(
+                latest_runs,
+                limit=display_limit,
+            )
+        semantic_history = compact_goal_semantic_history(
+            goal.get("semantic_history"),
+            compact_run=compact_run,
+        )
+        goals.append(
+            {
+                "id": goal.get("id"),
+                "domain": goal.get("domain"),
+                "status": goal.get("status"),
+                "lifecycle_phase": lifecycle_fields["lifecycle_phase"],
+                "lifecycle_flags": lifecycle_fields["lifecycle_flags"],
+                "registry_member": goal.get("registry_member"),
+                "legacy_runtime_goal": goal.get("legacy_runtime_goal"),
+                "adapter_kind": goal.get("adapter_kind"),
+                "adapter_status": goal.get("adapter_status"),
+                "coordination": compact_goal_coordination(goal.get("coordination")),
+                "explore_graph": goal.get("explore_graph")
+                if isinstance(goal.get("explore_graph"), dict)
+                else None,
+                "guards": goal.get("guards") if isinstance(goal.get("guards"), list) else [],
+                "next_probe": goal.get("next_probe"),
+                "authority_registry": goal.get("authority_registry"),
+                "quota": quota_status(goal) if goal.get("registry_member") else None,
+                "index_exists": goal.get("index_exists"),
+                "raw_index_records": goal.get("raw_index_records"),
+                "unique_runs": goal.get("unique_runs"),
+                "subagent_activity": subagent_activity,
+                "latest_status_run": compact_run(current_run) if current_run else None,
+                "latest_runs": latest_runs,
+            }
+        )
+        if semantic_history is not None:
+            goals[-1]["semantic_history"] = semantic_history
+
+    recent_runs = [
+        compact_run(run)
+        for run in history.get("runs") or []
+        if isinstance(run, dict)
+    ]
+    if display_limit is not None:
+        recent_runs = recent_runs[:display_limit]
+    return {
+        "available": True,
+        "goal_count": history.get("goal_count"),
+        "run_count": history.get("run_count"),
+        "goals": goals,
+        "recent_runs": recent_runs,
+    }

loopx/control_plane/runtime/run_index_duplicates.py

@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+STRUCTURED_INDEX_KEYS = (
+    "benchmark_run",
+    "benchmark_result",
+    "benchmark_comparison",
+    "benchmark_learning_ledger",
+    "benchmark_experiment_report",
+    "active_user_assisted_pilot",
+)
+REWARD_OVERLAY_IDENTITY_KEYS = (
+    "generated_at",
+    "goal_id",
+    "classification",
+    "json_path",
+    "markdown_path",
+)
+
+
+def index_identity(record: dict[str, Any]) -> tuple[str, str, str]:
+    return (
+        str(record.get("generated_at") or ""),
+        str(record.get("json_path") or ""),
+        str(record.get("markdown_path") or ""),
+    )
+
+
+def structured_index_payload_keys(record: dict[str, Any]) -> list[str]:
+    return [key for key in STRUCTURED_INDEX_KEYS if isinstance(record.get(key), dict)]
+
+
+def has_structured_index_payload(record: dict[str, Any]) -> bool:
+    return bool(structured_index_payload_keys(record))
+
+
+def _normalized_index_record(record: dict[str, Any]) -> dict[str, Any]:
+    return {key: value for key, value in record.items() if key != "human_reward"}
+
+
+def _normalized_key(record: dict[str, Any]) -> str:
+    return json.dumps(record, sort_keys=True, ensure_ascii=False)
+
+
+def is_reward_overlay_bundle(records: list[dict[str, Any]]) -> bool:
+    """Return true when compact reward rows are projections of one base run."""
+
+    base_rows = [
+        record for record in records if not isinstance(record.get("human_reward"), dict)
+    ]
+    reward_rows = [
+        record for record in records if isinstance(record.get("human_reward"), dict)
+    ]
+    if len(base_rows) != 1 or not reward_rows:
+        return False
+
+    base = base_rows[0]
+    for overlay in reward_rows:
+        projected = _normalized_index_record(overlay)
+        if not all(key in projected for key in REWARD_OVERLAY_IDENTITY_KEYS):
+            return False
+        if any(
+            key not in base or base[key] != value for key, value in projected.items()
+        ):
+            return False
+    return True
+
+
+def is_repairable_structured_artifact_duplicate(records: list[dict[str, Any]]) -> bool:
+    classifications = {str(record.get("classification") or "") for record in records}
+    health_checks = {
+        str(record.get("health_check") or "")
+        for record in records
+        if record.get("health_check")
+    }
+    structured_rows = [
+        record for record in records if has_structured_index_payload(record)
+    ]
+    return (
+        len(structured_rows) == 1
+        and len(classifications) == 1
+        and len(health_checks) > 1
+        and all(
+            has_structured_index_payload(record)
+            or all(key not in record for key in STRUCTURED_INDEX_KEYS)
+            for record in records
+        )
+    )
+
+
+def is_structured_artifact_bundle(records: list[dict[str, Any]]) -> bool:
+    """Return true when one artifact intentionally indexes multiple compact rows."""
+
+    if len(records) <= 1:
+        return False
+    if any(isinstance(record.get("human_reward"), dict) for record in records):
+        return False
+    if not all(record.get("health_check") for record in records):
+        return False
+    if not all(len(structured_index_payload_keys(record)) == 1 for record in records):
+        return False
+    payload_fingerprints = {
+        _normalized_key(
+            {key: record[key] for key in structured_index_payload_keys(record)}
+        )
+        for record in records
+    }
+    return len(payload_fingerprints) == len(records)
+
+
+def classify_index_duplicate_records(records: list[dict[str, Any]]) -> dict[str, Any]:
+    normalized_keys = {
+        _normalized_key(_normalized_index_record(record)) for record in records
+    }
+    reward_records = sum(
+        1 for record in records if isinstance(record.get("human_reward"), dict)
+    )
+    if is_reward_overlay_bundle(records):
+        return {
+            "duplicate_kind": "reward_overlay",
+            "severity": "info",
+            "repair_hint": "no index repair needed; reward overlay rows merge into the base run",
+            "action": "preserve_reward_overlay",
+            "repairable": False,
+            "reason": "reward overlay rows are intentionally merged by status checks",
+        }
+
+    if not reward_records and len(normalized_keys) == 1:
+        return {
+            "duplicate_kind": "plain_duplicate",
+            "severity": "warning",
+            "repair_hint": "append-only ledger repair can archive or supersede the extra identical index row",
+            "action": "drop_plain_duplicate_rows",
+            "repairable": True,
+            "reason": "duplicate rows are byte-equivalent after reward fields are ignored",
+        }
+
+    if is_structured_artifact_bundle(records):
+        return {
+            "duplicate_kind": "structured_artifact_bundle",
+            "severity": "info",
+            "repair_hint": "no index repair needed; rows are compact projections from one reviewed artifact bundle",
+            "action": "preserve_structured_artifact_bundle",
+            "repairable": False,
+            "reason": "multiple compact structured rows share one artifact identity by design",
+        }
+
+    if is_repairable_structured_artifact_duplicate(records):
+        return {
+            "duplicate_kind": "artifact_identity_collision",
+            "severity": "warning",
+            "repair_hint": "append-only ledger repair can keep the row with compact structured payload",
+            "action": "keep_structured_artifact_row",
+            "repairable": True,
+            "reason": "one row carries the compact structured artifact payload and siblings only repeat the artifact identity",
+        }
+
+    return {
+        "duplicate_kind": "artifact_identity_collision",
+        "severity": "warning",
+        "repair_hint": (
+            "do not delete blindly; inspect artifacts and append an explicit repair/supersede event "
+            "or rebuild a reviewed index copy"
+        ),
+        "action": "blocked_artifact_identity_collision",
+        "repairable": False,
+        "reason": "artifact identity collision is not auto-repairable without reviewed merge semantics",
+    }
+
+
+def duplicate_repair_decision(
+    records: list[tuple[int, dict[str, Any]]],
+) -> dict[str, Any]:
+    line_numbers = [line_number for line_number, _ in records]
+    payload = classify_index_duplicate_records([record for _, record in records])
+    action = payload.get("action")
+    kept_line_numbers = line_numbers
+    removed_line_numbers: list[int] = []
+    if action == "drop_plain_duplicate_rows":
+        kept_line_numbers = [line_numbers[0]]
+        removed_line_numbers = line_numbers[1:]
+    elif action == "keep_structured_artifact_row":
+        structured_lines = [
+            line_number
+            for line_number, record in records
+            if has_structured_index_payload(record)
+        ]
+        kept_line_numbers = [structured_lines[0]]
+        removed_line_numbers = [
+            line_number
+            for line_number in line_numbers
+            if line_number != structured_lines[0]
+        ]
+
+    return {
+        "action": action,
+        "repairable": payload.get("repairable"),
+        "line_numbers": line_numbers,
+        "kept_line_numbers": kept_line_numbers,
+        "removed_line_numbers": removed_line_numbers,
+        "reason": payload.get("reason"),
+    }

loopx/control_plane/runtime/run_index_rebuild.py

@@ -0,0 +1,263 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from .run_artifacts import run_file_stem
+from .run_index_duplicates import classify_index_duplicate_records, index_identity
+
+
+COLLISION_REBUILD_PLAN_SCHEMA = "artifact_identity_collision_rebuild_plan_v0"
+COLLISION_RECOVERY_ARTIFACT_SCHEMA = "artifact_identity_collision_recovery_v0"
+
+
+def _canonical_json(value: Any) -> str:
+    return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+
+
+def _sha256(value: Any) -> str:
+    return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest()
+
+
+def _event_identity(record: dict[str, Any]) -> dict[str, Any]:
+    return {
+        key: record.get(key)
+        for key in (
+            "classification",
+            "todo_id",
+            "target_key",
+            "agent_id",
+            "material_change",
+        )
+        if record.get(key) is not None
+    }
+
+
+def read_index_rows(index_path: Path) -> tuple[list[str], list[tuple[int, dict[str, Any]]]]:
+    raw_lines = index_path.read_text(encoding="utf-8").splitlines()
+    rows: list[tuple[int, dict[str, Any]]] = []
+    for line_number, line in enumerate(raw_lines, start=1):
+        if not line.strip():
+            continue
+        try:
+            item = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(item, dict):
+            rows.append((line_number, item))
+    return raw_lines, rows
+
+
+def collision_review_groups(index_path: Path, goal_id: str) -> list[dict[str, Any]]:
+    _, rows = read_index_rows(index_path)
+    grouped: dict[tuple[str, str, str], list[tuple[int, dict[str, Any]]]] = {}
+    for line_number, record in rows:
+        grouped.setdefault(index_identity(record), []).append((line_number, record))
+
+    groups: list[dict[str, Any]] = []
+    for identity, records in grouped.items():
+        if len(records) <= 1:
+            continue
+        classification = classify_index_duplicate_records(
+            [record for _, record in records]
+        )
+        if classification.get("action") != "blocked_artifact_identity_collision":
+            continue
+        generated_at, json_path, markdown_path = identity
+        groups.append(
+            {
+                "goal_id": goal_id,
+                "index_path": str(index_path),
+                "source_identity": {
+                    "generated_at": generated_at,
+                    "json_path": json_path,
+                    "markdown_path": markdown_path,
+                },
+                "line_numbers": [line_number for line_number, _ in records],
+                "rows": [
+                    {
+                        "line_number": line_number,
+                        "row_sha256": _sha256(record),
+                        "event_identity": _event_identity(record),
+                    }
+                    for line_number, record in records
+                ],
+                "rebuild_contract": "preserve_every_row_with_a_distinct_recovery_artifact",
+            }
+        )
+    return groups
+
+
+def build_collision_rebuild_plan(
+    groups: list[dict[str, Any]],
+    *,
+    goal_filter: str | None,
+    total_collision_group_count: int,
+    truncated: bool,
+) -> dict[str, Any]:
+    plan_body = {
+        "schema_version": COLLISION_REBUILD_PLAN_SCHEMA,
+        "goal_filter": goal_filter,
+        "groups": groups,
+        "total_collision_group_count": total_collision_group_count,
+        "truncated": truncated,
+        "destructive_row_deletion": False,
+    }
+    return {**plan_body, "plan_sha256": _sha256(plan_body)}
+
+
+def validate_reviewed_collision_plan(
+    reviewed_plan: dict[str, Any],
+    current_plan: dict[str, Any],
+) -> str:
+    if reviewed_plan.get("schema_version") != COLLISION_REBUILD_PLAN_SCHEMA:
+        raise ValueError(
+            f"review plan must have schema_version={COLLISION_REBUILD_PLAN_SCHEMA}"
+        )
+    reviewed_body = {
+        key: reviewed_plan.get(key)
+        for key in (
+            "schema_version",
+            "goal_filter",
+            "groups",
+            "total_collision_group_count",
+            "truncated",
+            "destructive_row_deletion",
+        )
+    }
+    reviewed_digest = _sha256(reviewed_body)
+    if reviewed_plan.get("plan_sha256") != reviewed_digest:
+        raise ValueError("review plan digest does not match its contents")
+    if reviewed_digest != current_plan.get("plan_sha256"):
+        raise ValueError(
+            "review plan is stale or does not match the current collision groups; regenerate and review it"
+        )
+    if reviewed_plan.get("destructive_row_deletion") is not False:
+        raise ValueError("review plan must explicitly keep destructive_row_deletion=false")
+    if reviewed_plan.get("truncated") is not False:
+        raise ValueError(
+            "review plan is truncated; increase --limit until truncated=false before execution"
+        )
+    return reviewed_digest
+
+
+def _write_new_or_verify(path: Path, content: str) -> None:
+    encoded = content.encode("utf-8")
+    try:
+        fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+    except FileExistsError:
+        if path.read_bytes() != encoded:
+            raise ValueError(f"rebuild artifact already exists with different content: {path}")
+        return
+    with os.fdopen(fd, "wb") as stream:
+        stream.write(encoded)
+
+
+def _recovery_markdown(record: dict[str, Any]) -> str:
+    recovery = record["artifact_rebuild"]
+    event_identity = recovery.get("event_identity") or {}
+    lines = [
+        "# LoopX Recovered Run Index Event",
+        "",
+        f"- schema_version: `{COLLISION_RECOVERY_ARTIFACT_SCHEMA}`",
+        f"- goal_id: `{record.get('goal_id')}`",
+        f"- generated_at: `{record.get('generated_at')}`",
+        f"- classification: `{record.get('classification')}`",
+        f"- source_line_number: `{recovery.get('source_line_number')}`",
+        f"- review_plan_sha256: `{recovery.get('review_plan_sha256')}`",
+        "- recovery: `compact index event preserved; ambiguous legacy artifact not claimed`",
+    ]
+    for key in ("todo_id", "target_key", "agent_id", "material_change"):
+        if key in event_identity:
+            lines.append(f"- {key}: `{event_identity[key]}`")
+    return "\n".join(lines) + "\n"
+
+
+def apply_reviewed_collision_rebuild(
+    current_plan: dict[str, Any],
+    *,
+    plan_sha256: str,
+) -> list[dict[str, Any]]:
+    groups_by_index: dict[str, list[dict[str, Any]]] = {}
+    for group in current_plan.get("groups") or []:
+        groups_by_index.setdefault(str(group.get("index_path") or ""), []).append(group)
+
+    rebuilt_indexes: list[dict[str, Any]] = []
+    digest_prefix = plan_sha256[:16]
+    for raw_index_path, groups in groups_by_index.items():
+        index_path = Path(raw_index_path)
+        raw_lines, parsed_rows = read_index_rows(index_path)
+        rows_by_line = dict(parsed_rows)
+        replacements: dict[int, dict[str, Any]] = {}
+        recovery_paths: list[str] = []
+
+        for group in groups:
+            source_identity = group.get("source_identity") or {}
+            generated_at = str(source_identity.get("generated_at") or "")
+            stem = run_file_stem(generated_at)
+            group_token = _sha256(source_identity)[:8]
+            for ordinal, row_spec in enumerate(group.get("rows") or [], start=1):
+                line_number = int(row_spec["line_number"])
+                source_record = rows_by_line.get(line_number)
+                if source_record is None or _sha256(source_record) != row_spec.get("row_sha256"):
+                    raise ValueError(
+                        f"reviewed collision row changed before rebuild: {index_path}:{line_number}"
+                    )
+                recovery_stem = (
+                    f"{stem}-collision-rebuild-{digest_prefix}-{group_token}-{ordinal}"
+                )
+                json_path = index_path.parent / f"{recovery_stem}.json"
+                markdown_path = index_path.parent / f"{recovery_stem}.md"
+                rebuilt_record = dict(source_record)
+                rebuilt_record["json_path"] = str(json_path)
+                rebuilt_record["markdown_path"] = str(markdown_path)
+                rebuilt_record["artifact_rebuild"] = {
+                    "schema_version": COLLISION_RECOVERY_ARTIFACT_SCHEMA,
+                    "review_plan_sha256": plan_sha256,
+                    "source_identity": source_identity,
+                    "source_line_number": line_number,
+                    "event_identity": row_spec.get("event_identity") or {},
+                    "ambiguous_legacy_artifact_claimed": False,
+                }
+                recovery_payload = {
+                    "schema_version": COLLISION_RECOVERY_ARTIFACT_SCHEMA,
+                    "artifact_rebuild": rebuilt_record["artifact_rebuild"],
+                    "index_record": source_record,
+                }
+                _write_new_or_verify(
+                    json_path,
+                    json.dumps(recovery_payload, ensure_ascii=False, indent=2) + "\n",
+                )
+                _write_new_or_verify(markdown_path, _recovery_markdown(rebuilt_record))
+                replacements[line_number] = rebuilt_record
+                recovery_paths.extend((str(json_path), str(markdown_path)))
+
+        backup_path = index_path.with_name(
+            f"index.pre-collision-rebuild-{digest_prefix}.jsonl"
+        )
+        original_content = "".join(line + "\n" for line in raw_lines)
+        _write_new_or_verify(backup_path, original_content)
+        rebuilt_lines = [
+            json.dumps(replacements[line_number], ensure_ascii=False)
+            if line_number in replacements
+            else line
+            for line_number, line in enumerate(raw_lines, start=1)
+        ]
+        tmp_path = index_path.with_name(f"index.collision-rebuild-{digest_prefix}.tmp")
+        tmp_path.write_text(
+            "".join(line + "\n" for line in rebuilt_lines),
+            encoding="utf-8",
+        )
+        tmp_path.replace(index_path)
+        rebuilt_indexes.append(
+            {
+                "index_path": str(index_path),
+                "backup_path": str(backup_path),
+                "preserved_row_count": len(replacements),
+                "recovery_paths": recovery_paths,
+            }
+        )
+    return rebuilt_indexes