engineering-platform 2.2.0__py3-none-any.whl

engineering_platform/execution_lease.py

@@ -0,0 +1,365 @@
+"""Canonical active-run ownership leases backed solely by Engineering SQLite."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+import json
+from pathlib import Path
+import sqlite3
+import socket
+from threading import Event, Thread
+import uuid
+
+from .storage import EngineeringStorageError, open_storage
+from .execution_timing import reconcile_interrupted_phases
+from .provider_process_identity import ProcessIdentity, verify_process_identity
+
+LEASE_VERSION = 1
+HEARTBEAT_INTERVAL_SECONDS = 15
+LEASE_TIMEOUT_SECONDS = 90
+TERMINAL_PHASES = frozenset({"COMPLETE", "BLOCKED", "FAILED"})
+
+
+def _connection(root: Path, central_database: Path | None = None) -> sqlite3.Connection:
+    """Open an explicitly supplied CENTRAL lease authority when available."""
+    if central_database is None:
+        return open_storage(root)
+    database = central_database.resolve()
+    if not database.is_file():
+        raise EngineeringStorageError("CENTRAL lease database is unavailable.")
+    connection = sqlite3.connect(database, isolation_level=None)
+    connection.execute("PRAGMA foreign_keys=ON")
+    connection.execute("PRAGMA busy_timeout=10000")
+    return connection
+
+
+class LeaseConflictError(EngineeringStorageError):
+    """Raised when another non-expired host instance owns the run."""
+
+
+@dataclass(frozen=True)
+class Lease:
+    lease_id: str
+    run_id: str
+    host_identity: str
+    host_instance_id: str
+    acquired_at: str
+    last_heartbeat_at: str
+    expires_at: str
+    lease_state: str
+
+
+def host_identity() -> str:
+    return socket.gethostname()[:120] or "execution-host"
+
+
+def host_instance_id() -> str:
+    return f"{uuid.uuid4()}"
+
+
+def _now() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _event(connection: object, lease_id: str, run_id: str, event: str, outcome: str | None = None) -> None:
+    connection.execute(
+        "INSERT OR IGNORE INTO execution_lease_events(lease_id,run_id,event_type,outcome,recorded_at) VALUES(?,?,?,?,?)",
+        (lease_id, run_id, event, outcome or "", _now().isoformat()),
+    )
+
+
+def _recovery_provider_lease_state(connection: object, run_id: str) -> str:
+    """Read recovery receipts through the caller's lease transaction.
+
+    This avoids opening a second SQLite writer while stale-lease
+    reconciliation owns ``BEGIN IMMEDIATE``.  Only a session-bound, PID reuse
+    protected PROCESS_STARTED receipt can retain an expired host lease.
+    """
+    try:
+        recovery = connection.execute(
+            "SELECT state,replacement_invocation_id,provider_session_id FROM provider_recovery_attempts WHERE run_id=?",
+            (run_id,),
+        ).fetchone()
+        if not recovery or recovery[0] != "RECOVERY_IN_PROGRESS":
+            return "NOT_ACTIVE"
+        rows = connection.execute(
+            "SELECT provider_session_id,process_pid,process_group,process_start_fingerprint,process_executable_identity "
+            "FROM provider_invocation_receipts WHERE run_id=? AND invocation_id=? AND launch_state='PROCESS_STARTED'",
+            (run_id, recovery[1]),
+        ).fetchall()
+    except Exception:
+        return "NOT_ACTIVE"
+    if len(rows) != 1 or not isinstance(recovery[2], str):
+        return "AMBIGUOUS"
+    session_id, pid, group, fingerprint, executable = rows[0]
+    if (
+        session_id != recovery[2] or not isinstance(pid, int) or not isinstance(group, int)
+        or not isinstance(fingerprint, str) or not isinstance(executable, str)
+    ):
+        return "AMBIGUOUS"
+    return "ACTIVE" if verify_process_identity(ProcessIdentity(pid, group, fingerprint, executable)) == "MATCH" else "AMBIGUOUS"
+
+
+class LeaseHeartbeat:
+    """Bounded background heartbeat; lifecycle state remains runner-owned."""
+
+    def __init__(self, root: Path, lease: Lease, *, central_database: Path | None = None, interval_seconds: int = HEARTBEAT_INTERVAL_SECONDS) -> None:
+        self.root = root
+        self.lease = lease
+        self.central_database = central_database
+        self.interval_seconds = interval_seconds
+        self._stop = Event()
+        self._thread: Thread | None = None
+        self.error: Exception | None = None
+
+    def start(self) -> None:
+        if self._thread is not None:
+            return
+        self._thread = Thread(target=self._run, name=f"engineering-lease-{self.lease.run_id}", daemon=True)
+        self._thread.start()
+
+    def _run(self) -> None:
+        while not self._stop.wait(self.interval_seconds):
+            try:
+                self.lease = heartbeat(self.root, self.lease, central_database=self.central_database)
+            except Exception as error:  # The expiry boundary remains fail-closed.
+                self.error = error
+                return
+
+    def stop(self) -> Lease:
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=max(1, self.interval_seconds))
+        return self.lease
+
+
+def acquire(root: Path, run_id: str, *, identity: str, instance_id: str, process_id: int | None = None, timeout_seconds: int = LEASE_TIMEOUT_SECONDS, central_database: Path | None = None) -> Lease:
+    if not 0 < HEARTBEAT_INTERVAL_SECONDS < timeout_seconds:
+        raise EngineeringStorageError("Active-run lease policy is invalid.")
+    now = _now()
+    expiry = now + timedelta(seconds=timeout_seconds)
+    connection = _connection(root, central_database)
+    try:
+        connection.execute("BEGIN IMMEDIATE")
+        connection.execute(
+            "UPDATE execution_run_leases SET lease_state='EXPIRED',updated_at=? WHERE run_id=? AND lease_state='ACTIVE' AND expires_at<?",
+            (now.isoformat(), run_id, now.isoformat()),
+        )
+        active = connection.execute(
+            "SELECT lease_id,host_instance_id FROM execution_run_leases WHERE run_id=? AND lease_state='ACTIVE'", (run_id,)
+        ).fetchone()
+        if active:
+            raise LeaseConflictError("A live Execution Host instance already owns this run.")
+        lease_id = f"lease-{uuid.uuid4()}"
+        values = (lease_id, run_id, identity, instance_id, process_id, now.isoformat(), now.isoformat(), expiry.isoformat(), "ACTIVE", LEASE_VERSION, now.isoformat(), now.isoformat())
+        connection.execute("INSERT INTO execution_run_leases(lease_id,run_id,host_identity,host_instance_id,process_id,acquired_at,last_heartbeat_at,expires_at,lease_state,lease_version,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)", values)
+        _event(connection, lease_id, run_id, "LEASE_ACQUIRED")
+        connection.execute("COMMIT")
+        return Lease(lease_id, run_id, identity, instance_id, now.isoformat(), now.isoformat(), expiry.isoformat(), "ACTIVE")
+    except Exception:
+        connection.execute("ROLLBACK")
+        raise
+    finally:
+        connection.close()
+
+
+def heartbeat(root: Path, lease: Lease, *, timeout_seconds: int = LEASE_TIMEOUT_SECONDS, central_database: Path | None = None) -> Lease:
+    now = _now()
+    expiry = now + timedelta(seconds=timeout_seconds)
+    connection = _connection(root, central_database)
+    try:
+        updated = connection.execute("UPDATE execution_run_leases SET last_heartbeat_at=?,expires_at=?,updated_at=? WHERE lease_id=? AND run_id=? AND host_instance_id=? AND lease_state='ACTIVE'", (now.isoformat(), expiry.isoformat(), now.isoformat(), lease.lease_id, lease.run_id, lease.host_instance_id)).rowcount
+        if updated != 1:
+            raise LeaseConflictError("Execution Host no longer owns the active-run lease.")
+    finally:
+        connection.close()
+    return Lease(lease.lease_id, lease.run_id, lease.host_identity, lease.host_instance_id, lease.acquired_at, now.isoformat(), expiry.isoformat(), "ACTIVE")
+
+
+def release(root: Path, lease: Lease, *, central_database: Path | None = None) -> None:
+    connection = _connection(root, central_database)
+    try:
+        connection.execute("BEGIN IMMEDIATE")
+        changed = connection.execute("UPDATE execution_run_leases SET lease_state='RELEASED',updated_at=? WHERE lease_id=? AND host_instance_id=? AND lease_state='ACTIVE'", (_now().isoformat(), lease.lease_id, lease.host_instance_id)).rowcount
+        if changed:
+            _event(connection, lease.lease_id, lease.run_id, "LEASE_RELEASED")
+        connection.execute("COMMIT")
+    finally:
+        connection.close()
+
+
+def release_terminal_lease(root: Path, run_id: str, *, central_database: Path | None = None) -> bool:
+    """Release a departed host's lease only after its checkpoint is terminal.
+
+    The watcher calls this after its foreground child has exited and after it
+    has persisted the terminal transaction.  It is not a stale-lease shortcut
+    and never decides lifecycle state itself.
+    """
+    connection = _connection(root, central_database)
+    try:
+        connection.execute("BEGIN IMMEDIATE")
+        row = connection.execute(
+            "SELECT phase,payload FROM engineering_transactions WHERE run_id=?", (run_id,)
+        ).fetchone()
+        terminal = False
+        if row and row[0] in TERMINAL_PHASES:
+            try:
+                payload = json.loads(row[1])
+            except (TypeError, json.JSONDecodeError):
+                payload = {}
+            terminal = isinstance(payload, dict) and payload.get("terminal") is True
+        if not terminal:
+            connection.execute("ROLLBACK")
+            return False
+        rows = connection.execute(
+            "SELECT lease_id FROM execution_run_leases WHERE run_id=? AND lease_state='ACTIVE'",
+            (run_id,),
+        ).fetchall()
+        now = _now().isoformat()
+        changed = connection.execute(
+            "UPDATE execution_run_leases SET lease_state='RELEASED',updated_at=? "
+            "WHERE run_id=? AND lease_state='ACTIVE'",
+            (now, run_id),
+        ).rowcount
+        for (lease_id,) in rows:
+            _event(connection, str(lease_id), run_id, "LEASE_RELEASED")
+        connection.execute("COMMIT")
+        return bool(changed)
+    except Exception:
+        connection.execute("ROLLBACK")
+        raise
+    finally:
+        connection.close()
+
+
+def reconcile_stale(root: Path, *, central_database: Path | None = None) -> list[dict[str, str]]:
+    """Reconcile datastore ownership only; never fabricate a terminal lifecycle state."""
+    now = _now().isoformat()
+    outcomes: list[dict[str, str]] = []
+    interrupted_runs: list[str] = []
+    connection = _connection(root, central_database)
+    try:
+        connection.execute("BEGIN IMMEDIATE")
+        rows = connection.execute("SELECT lease_id,run_id,host_instance_id,last_heartbeat_at FROM execution_run_leases WHERE lease_state='ACTIVE' AND expires_at<?", (now,)).fetchall()
+        for lease_id, run_id, instance, heartbeat_at in rows:
+            # Provider execution is independent of the host heartbeat.  A
+            # recovered provider turn can still be authoritatively active
+            # after its parent host disappeared; expiring this run lease
+            # would falsely make repository mutation ownership available.
+            recovery_lease_state = _recovery_provider_lease_state(connection, str(run_id))
+            if recovery_lease_state in {"ACTIVE", "AMBIGUOUS"}:
+                renewed = (_now() + timedelta(seconds=LEASE_TIMEOUT_SECONDS)).isoformat()
+                connection.execute(
+                    "UPDATE execution_run_leases SET last_heartbeat_at=?,expires_at=?,updated_at=? "
+                    "WHERE lease_id=? AND lease_state='ACTIVE'",
+                    (now, renewed, now, lease_id),
+                )
+                outcome = (
+                    "RECOVERY_PROVIDER_STILL_ACTIVE"
+                    if recovery_lease_state == "ACTIVE" else "RECOVERY_PROVIDER_AMBIGUOUS"
+                )
+                _event(connection, str(lease_id), str(run_id), outcome)
+                outcomes.append({
+                    "run_id": str(run_id), "host_instance_id": str(instance),
+                    "last_heartbeat": str(heartbeat_at), "outcome": outcome,
+                })
+                continue
+            transaction = connection.execute(
+                "SELECT phase,payload FROM engineering_transactions WHERE run_id=?", (run_id,)
+            ).fetchone()
+            phase = transaction[0] if transaction else None
+            terminal_phase: str | None = phase if phase in TERMINAL_PHASES else None
+            if transaction and terminal_phase is None:
+                try:
+                    payload = json.loads(transaction[1])
+                except (TypeError, json.JSONDecodeError):
+                    payload = {}
+                candidate = payload.get("phase") if isinstance(payload, dict) and payload.get("terminal") is True else None
+                if candidate in TERMINAL_PHASES:
+                    terminal_phase = candidate
+                    connection.execute(
+                        "UPDATE engineering_transactions SET phase=?,updated_at=? WHERE run_id=?",
+                        (terminal_phase, now, run_id),
+                    )
+            outcome = "TERMINAL_EVIDENCE_PRESENT" if terminal_phase else "RECOVERABLE"
+            connection.execute("UPDATE execution_run_leases SET lease_state='EXPIRED',updated_at=? WHERE lease_id=?", (now, lease_id))
+            _event(connection, lease_id, run_id, "LEASE_EXPIRED")
+            _event(connection, lease_id, run_id, "STALE_DETECTED", outcome)
+            _event(connection, lease_id, run_id, "STALE_RECONCILED", outcome)
+            connection.execute(
+                "INSERT INTO execution_run_reconciliations(run_id,outcome,reason,reconciled_at,updated_at) VALUES(?,?,?,?,?) "
+                "ON CONFLICT(run_id) DO UPDATE SET outcome=excluded.outcome,reason=excluded.reason,reconciled_at=excluded.reconciled_at,updated_at=excluded.updated_at",
+                (run_id, outcome, f"lease_expired;terminal_phase={terminal_phase or 'none'}", now, now),
+            )
+            # Close active spans after this lease transaction commits.
+            interrupted_runs.append(run_id)
+            outcomes.append({"run_id": run_id, "host_instance_id": instance, "last_heartbeat": heartbeat_at, "outcome": outcome})
+        active_runs = connection.execute(
+            "SELECT run_id,phase FROM engineering_transactions WHERE phase NOT IN ('COMPLETE','BLOCKED','FAILED')"
+        ).fetchall()
+        for run_id, phase in active_runs:
+            has_live_lease = connection.execute(
+                "SELECT 1 FROM execution_run_leases WHERE run_id=? AND lease_state='ACTIVE' AND expires_at>=?",
+                (run_id, now),
+            ).fetchone()
+            if has_live_lease:
+                continue
+            prior = connection.execute(
+                "SELECT 1 FROM execution_run_reconciliations WHERE run_id=?", (run_id,)
+            ).fetchone()
+            if prior:
+                continue
+            outcome = "INCONSISTENT" if phase in TERMINAL_PHASES else "OPERATOR_INTERVENTION_REQUIRED"
+            connection.execute(
+                "INSERT INTO execution_run_reconciliations(run_id,outcome,reason,reconciled_at,updated_at) VALUES(?,?,?,?,?)",
+                (run_id, outcome, "active_transaction_without_live_lease", now, now),
+            )
+            outcomes.append({"run_id": run_id, "host_instance_id": "unknown", "last_heartbeat": "unknown", "outcome": outcome})
+        connection.execute("COMMIT")
+    finally:
+        connection.close()
+    for run_id in interrupted_runs:
+        # Completed timing evidence remains immutable.  This deliberately uses
+        # the actual reconciliation boundary rather than an invented expiry.
+        reconcile_interrupted_phases(root, run_id, central_database=central_database)
+    return outcomes
+
+
+def liveness(root: Path, run_id: object, *, central_database: Path | None = None) -> dict[str, object]:
+    """Project canonical liveness without consulting status files or processes."""
+    if not isinstance(run_id, str):
+        return {"state": "UNAVAILABLE"}
+    now = _now().isoformat()
+    connection = _connection(root, central_database)
+    try:
+        row = connection.execute("SELECT host_identity,host_instance_id,last_heartbeat_at,expires_at,lease_state FROM execution_run_leases WHERE run_id=? ORDER BY created_at DESC LIMIT 1", (run_id,)).fetchone()
+        reconciliation = connection.execute(
+            "SELECT outcome,reason,reconciled_at FROM execution_run_reconciliations WHERE run_id=?", (run_id,)
+        ).fetchone()
+    finally:
+        connection.close()
+    if not row:
+        result: dict[str, object] = {"state": "STALE"}
+        if reconciliation:
+            result.update({"reconciliation_outcome": reconciliation[0], "reconciliation_reason": reconciliation[1], "reconciled_at": reconciliation[2]})
+        return result
+    identity, instance, heartbeat_at, expires_at, state = row
+    result = {"state": "LIVE" if state == "ACTIVE" and expires_at >= now else "STALE", "lease_state": state, "host_identity": identity, "host_instance_id": instance, "last_heartbeat": heartbeat_at, "lease_expiry": expires_at}
+    if reconciliation:
+        result.update({"reconciliation_outcome": reconciliation[0], "reconciliation_reason": reconciliation[1], "reconciled_at": reconciliation[2]})
+    return result
+
+
+def history(root: Path, run_id: str, *, central_database: Path | None = None) -> dict[str, object]:
+    """Return bounded canonical lease evidence for reports and history views."""
+    connection = _connection(root, central_database)
+    try:
+        row = connection.execute(
+            "SELECT host_identity,host_instance_id,acquired_at,last_heartbeat_at,expires_at,lease_state "
+            "FROM execution_run_leases WHERE run_id=? ORDER BY created_at DESC LIMIT 1", (run_id,)
+        ).fetchone()
+    finally:
+        connection.close()
+    if not row:
+        return {"state": "UNAVAILABLE"}
+    return {"host_identity": row[0], "host_instance_id": row[1], "acquired_at": row[2], "last_heartbeat": row[3], "lease_expiry": row[4], "lease_state": row[5]}