py-data-engine 0.1.0__py3-none-any.whl

data_engine/hosts/daemon/manager.py

@@ -0,0 +1,272 @@
+"""Client-side daemon state management for Data Engine workspaces."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+import os
+import subprocess
+
+from data_engine.domain.time import parse_utc_text
+from data_engine.hosts.daemon.app import DaemonClientError, daemon_request, is_daemon_live
+from data_engine.hosts.daemon.shared_state import DaemonSharedStateAdapter
+from data_engine.domain import WorkspaceControlState
+from data_engine.platform.workspace_models import WorkspacePaths, machine_id_text
+
+_DEFAULT_SHARED_STATE_ADAPTER = DaemonSharedStateAdapter()
+
+
+def read_lease_metadata(paths: WorkspacePaths) -> dict[str, object] | None:
+    """Compatibility seam for reading lease metadata in daemon-manager tests."""
+    return _DEFAULT_SHARED_STATE_ADAPTER.read_lease_metadata(paths)
+
+
+def recover_stale_workspace(*, paths: WorkspacePaths, machine_id: str, stale_after_seconds: float) -> bool:
+    """Compatibility seam for stale-lease recovery in daemon-manager tests."""
+    return _DEFAULT_SHARED_STATE_ADAPTER.recover_stale_workspace(
+        paths,
+        machine_id=machine_id,
+        stale_after_seconds=stale_after_seconds,
+    )
+
+
+def write_control_request(paths: WorkspacePaths, **kwargs: object) -> None:
+    """Compatibility seam for control-request writes in daemon-manager tests."""
+    _DEFAULT_SHARED_STATE_ADAPTER.write_control_request(paths, **kwargs)
+
+
+def _lease_pid_is_live(metadata: dict[str, object] | None) -> bool:
+    """Return whether the recorded lease owner pid is still alive."""
+    if not isinstance(metadata, dict):
+        return False
+    pid_value = metadata.get("pid")
+    try:
+        pid = int(pid_value)
+    except (TypeError, ValueError):
+        return False
+    if pid <= 0:
+        return False
+    try:
+        os.kill(pid, 0)
+    except OSError:
+        return False
+    try:
+        # Refine the basic PID-exists check by rejecting zombies when process
+        # status inspection is available. If the host blocks `ps` (for example
+        # in a restricted sandbox), keep the conservative answer from os.kill:
+        # treat the PID as live so we don't incorrectly reclaim a genuinely
+        # running local owner.
+        result = subprocess.run(
+            ["ps", "-o", "stat=", "-p", str(pid)],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+    except OSError:
+        return True
+    if result.returncode != 0:
+        return False
+    status_text = result.stdout.strip()
+    if not status_text:
+        return False
+    if status_text.split()[0].startswith("Z"):
+        return False
+    return True
+
+
+@dataclass(frozen=True)
+class WorkspaceDaemonSnapshot:
+    """Normalized client view of one workspace daemon state."""
+
+    live: bool
+    workspace_owned: bool
+    leased_by_machine_id: str | None
+    runtime_active: bool
+    runtime_stopping: bool
+    manual_runs: tuple[str, ...]
+    last_checkpoint_at_utc: str | None
+    source: str
+
+
+class WorkspaceDaemonManager:
+    """Track daemon liveness and normalize fallback state for one workspace."""
+
+    def __init__(
+        self,
+        paths: WorkspacePaths,
+        *,
+        max_sync_misses: int = 3,
+        shared_state_adapter: DaemonSharedStateAdapter | None = None,
+    ) -> None:
+        self.paths = paths
+        self.max_sync_misses = max(max_sync_misses, 1)
+        self.shared_state_adapter = shared_state_adapter or DaemonSharedStateAdapter()
+        self._daemon_live = False
+        self._sync_misses = 0
+        self._last_snapshot: WorkspaceDaemonSnapshot | None = None
+
+    @property
+    def daemon_live(self) -> bool:
+        """Return the most recent daemon liveness result."""
+        return self._daemon_live
+
+    def sync(self) -> WorkspaceDaemonSnapshot:
+        """Return the latest normalized daemon snapshot for one workspace."""
+        try:
+            live = is_daemon_live(self.paths)
+        except Exception:
+            live = False
+        self._daemon_live = live
+        if not live:
+            self._sync_misses += 1
+            if self._sync_misses < self.max_sync_misses and self._last_snapshot is not None:
+                return WorkspaceDaemonSnapshot(
+                    live=False,
+                    workspace_owned=self._last_snapshot.workspace_owned,
+                    leased_by_machine_id=self._last_snapshot.leased_by_machine_id,
+                    runtime_active=self._last_snapshot.runtime_active,
+                    runtime_stopping=self._last_snapshot.runtime_stopping,
+                    manual_runs=self._last_snapshot.manual_runs,
+                    last_checkpoint_at_utc=self._last_snapshot.last_checkpoint_at_utc,
+                    source="cached",
+                )
+            snapshot = self._lease_snapshot()
+            self._last_snapshot = snapshot
+            return snapshot
+        try:
+            response = daemon_request(self.paths, {"command": "daemon_status"}, timeout=0.5)
+        except DaemonClientError:
+            self._sync_misses += 1
+            if self._last_snapshot is not None:
+                return WorkspaceDaemonSnapshot(
+                    live=False,
+                    workspace_owned=self._last_snapshot.workspace_owned,
+                    leased_by_machine_id=self._last_snapshot.leased_by_machine_id,
+                    runtime_active=self._last_snapshot.runtime_active,
+                    runtime_stopping=self._last_snapshot.runtime_stopping,
+                    manual_runs=self._last_snapshot.manual_runs,
+                    last_checkpoint_at_utc=self._last_snapshot.last_checkpoint_at_utc,
+                    source="cached",
+                )
+            snapshot = self._lease_snapshot()
+            self._last_snapshot = snapshot
+            return snapshot
+        status = response.get("status") if response.get("ok") else None
+        if not isinstance(status, dict):
+            snapshot = self._lease_snapshot()
+            self._last_snapshot = snapshot
+            return snapshot
+        self._sync_misses = 0
+        manual_runs = tuple(name for name in status.get("manual_runs", []) if isinstance(name, str))
+        leased_by = status.get("leased_by_machine_id")
+        checkpoint = status.get("last_checkpoint_at_utc")
+        snapshot = WorkspaceDaemonSnapshot(
+            live=True,
+            workspace_owned=bool(status.get("workspace_owned", True)),
+            leased_by_machine_id=str(leased_by) if isinstance(leased_by, str) and leased_by.strip() else None,
+            runtime_active=bool(status.get("engine_active")),
+            runtime_stopping=bool(status.get("engine_stopping")),
+            manual_runs=manual_runs,
+            last_checkpoint_at_utc=str(checkpoint) if isinstance(checkpoint, str) and checkpoint.strip() else None,
+            source="daemon",
+        )
+        self._last_snapshot = snapshot
+        return snapshot
+
+    def _lease_snapshot(self) -> WorkspaceDaemonSnapshot:
+        metadata = read_lease_metadata(self.paths)
+        local_machine_id = machine_id_text()
+        if (
+            isinstance(metadata, dict)
+            and str(metadata.get("machine_id", "")).strip() == local_machine_id
+            and not _lease_pid_is_live(metadata)
+        ):
+            recovered = recover_stale_workspace(
+                paths=self.paths,
+                machine_id=local_machine_id,
+                stale_after_seconds=0.0,
+            )
+            if recovered:
+                metadata = read_lease_metadata(self.paths)
+        owner = metadata.get("machine_id") if isinstance(metadata, dict) else None
+        checkpoint = metadata.get("last_checkpoint_at_utc") if isinstance(metadata, dict) else None
+        checkpoint_text = str(checkpoint) if isinstance(checkpoint, str) and checkpoint.strip() else None
+        if checkpoint_text is not None and parse_utc_text(checkpoint_text) is None:
+            checkpoint_text = None
+        return WorkspaceDaemonSnapshot(
+            live=False,
+            workspace_owned=metadata is None,
+            leased_by_machine_id=str(owner) if isinstance(owner, str) and owner.strip() else None,
+            runtime_active=False,
+            runtime_stopping=False,
+            manual_runs=(),
+            last_checkpoint_at_utc=checkpoint_text,
+            source="lease" if metadata is not None else "none",
+        )
+
+    def control_status_text(
+        self,
+        snapshot: WorkspaceDaemonSnapshot,
+        *,
+        daemon_startup_in_progress: bool = False,
+    ) -> str | None:
+        """Return plain-language control status text for UI/TUI display."""
+        return self.control_state(
+            snapshot,
+            daemon_startup_in_progress=daemon_startup_in_progress,
+        ).control_status_text
+
+    def leased_elsewhere_status_text(self, snapshot: WorkspaceDaemonSnapshot) -> str:
+        """Return plain-language action-blocked status for another owner."""
+        return self.control_state(snapshot).blocked_status_text
+
+    def control_state(
+        self,
+        snapshot: WorkspaceDaemonSnapshot,
+        *,
+        daemon_startup_in_progress: bool = False,
+    ) -> WorkspaceControlState:
+        """Return the structured workspace control state for one snapshot."""
+        return WorkspaceControlState.from_snapshot(
+            snapshot,
+            daemon_live=self.daemon_live,
+            local_machine_id=machine_id_text(),
+            control_request=self.shared_state_adapter.read_control_request(self.paths),
+            daemon_startup_in_progress=daemon_startup_in_progress,
+        )
+
+    def request_control(self) -> str:
+        """Record one control-transfer request for the current workstation."""
+        if self._daemon_live:
+            snapshot = self.sync()
+            if snapshot.workspace_owned:
+                return "This workstation already has control."
+        metadata = read_lease_metadata(self.paths)
+        owner = (
+            str(metadata.get("machine_id")).strip()
+            if isinstance(metadata, dict) and isinstance(metadata.get("machine_id"), str)
+            else ""
+        )
+        local_machine_id = machine_id_text()
+        if owner == local_machine_id and not self._daemon_live:
+            if not _lease_pid_is_live(metadata):
+                recovered = recover_stale_workspace(
+                    paths=self.paths,
+                    machine_id=local_machine_id,
+                    stale_after_seconds=0.0,
+                )
+                if recovered:
+                    return "Recovered local control."
+        write_control_request(
+            self.paths,
+            workspace_id=self.paths.workspace_id,
+            requester_machine_id=local_machine_id,
+            requester_host_name=local_machine_id,
+            requester_pid=os.getpid(),
+            requester_client_kind="ui",
+            requested_at_utc=datetime.now(UTC).isoformat(),
+        )
+        return "Control request sent."
+
+
+__all__ = ["WorkspaceDaemonManager", "WorkspaceDaemonSnapshot"]

data_engine/hosts/daemon/ownership.py

@@ -0,0 +1,126 @@
+"""Workspace ownership helpers for the daemon host."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from data_engine.hosts.daemon.app import DataEngineDaemonService
+
+
+def control_request_metadata(service: "DataEngineDaemonService") -> dict[str, object] | None:
+    metadata = service.shared_state_adapter.read_control_request(service.paths)
+    return metadata if isinstance(metadata, dict) else None
+
+
+def honor_control_request_if_needed(service: "DataEngineDaemonService") -> bool:
+    """Relinquish ownership when another workstation requests control."""
+    with service._state_lock:
+        if not service.host.workspace_owned:
+            return False
+    metadata = control_request_metadata(service)
+    if metadata is None:
+        return False
+    requester = str(metadata.get("requester_machine_id", "")).strip()
+    if not requester or requester == service.machine_id:
+        return False
+    service._debug_log(f"control request received requester={requester}")
+    service._relinquish_workspace_for_control_request(requester)
+    return True
+
+
+def try_claim_requested_control(service: "DataEngineDaemonService") -> bool:
+    """Claim released ownership when this workstation requested control."""
+    with service._state_lock:
+        if service.host.workspace_owned:
+            return True
+    metadata = control_request_metadata(service)
+    if metadata is None:
+        return False
+    requester = str(metadata.get("requester_machine_id", "")).strip()
+    if requester != service.machine_id:
+        return False
+    claimed = try_claim_released_workspace(service)
+    if not claimed:
+        return False
+    service.shared_state_adapter.remove_control_request(service.paths)
+    service._debug_log("control request fulfilled workspace claimed")
+    return True
+
+
+def lease_error_text(service: "DataEngineDaemonService") -> str:
+    with service._state_lock:
+        owner = service.host.leased_by_machine_id or "another machine"
+    return f"Workspace {service.paths.workspace_id!r} is leased by {owner}."
+
+
+def try_claim_released_workspace(service: "DataEngineDaemonService") -> bool:
+    """Try to reclaim an available workspace for this daemon."""
+    with service._state_lock:
+        if service.host.workspace_owned:
+            return True
+    shared_state = service.shared_state_adapter
+    metadata = shared_state.read_lease_metadata(service.paths)
+    if metadata is not None:
+        owner = metadata.get("machine_id")
+        if isinstance(owner, str) and owner.strip():
+            with service._state_lock:
+                service.host.leased_by_machine_id = owner
+        return False
+    try:
+        claimed = shared_state.claim_workspace(service.paths)
+    except Exception:
+        return False
+    if not claimed:
+        metadata = shared_state.read_lease_metadata(service.paths)
+        owner = metadata.get("machine_id") if isinstance(metadata, dict) else None
+        with service._state_lock:
+            service.host.leased_by_machine_id = str(owner) if isinstance(owner, str) and owner.strip() else None
+        return False
+    with service._state_lock:
+        service.state.claim_workspace()
+    try:
+        service._checkpoint_once(status="idle")
+        with service._state_lock:
+            service.state.reset_checkpoint_failures()
+    except Exception:
+        with service._state_lock:
+            service.state.release_workspace()
+        release_workspace_claim(service)
+        return False
+    return True
+
+
+def release_workspace_claim(
+    service: "DataEngineDaemonService",
+    *,
+    leased_by_machine_id: str | None = None,
+    status: str | None = None,
+    update_state: bool = False,
+) -> None:
+    """Release shared ownership and mark the daemon as no longer owning the workspace."""
+    with service._state_lock:
+        workspace_owned = service.host.workspace_owned
+    if workspace_owned:
+        try:
+            service.shared_state_adapter.remove_lease_metadata(service.paths)
+        except Exception:
+            pass
+        try:
+            service.shared_state_adapter.release_workspace(service.paths)
+        except Exception:
+            pass
+    with service._state_lock:
+        service.state.release_workspace(leased_by_machine_id=leased_by_machine_id, status=status)
+    if update_state and status is not None:
+        service._update_daemon_state(status=status)
+
+
+__all__ = [
+    "control_request_metadata",
+    "honor_control_request_if_needed",
+    "lease_error_text",
+    "release_workspace_claim",
+    "try_claim_released_workspace",
+    "try_claim_requested_control",
+]

data_engine/hosts/daemon/runtime_commands.py

@@ -0,0 +1,188 @@
+"""Runtime-control actions for the daemon host."""
+
+from __future__ import annotations
+
+import threading
+import traceback
+from typing import TYPE_CHECKING, Any
+
+from data_engine.domain.time import utcnow_text
+from data_engine.hosts.daemon.ownership import lease_error_text, try_claim_released_workspace
+
+if TYPE_CHECKING:
+    from data_engine.hosts.daemon.app import DataEngineDaemonService
+
+
+class DaemonRuntimeCommandHandler:
+    """Own daemon runtime-control actions."""
+
+    def __init__(self, service: "DataEngineDaemonService") -> None:
+        self.service = service
+
+    def automated_flow_names(self, *, force: bool = False) -> tuple[str, ...]:
+        return tuple(
+            card.name
+            for card in self.service._load_flow_cards(force=force)
+            if card.valid and card.mode in {"poll", "schedule"}
+        )
+
+    def run_flow(self, *, name: str, wait: bool) -> dict[str, Any]:
+        service = self.service
+        if not service.state.workspace_owned and not try_claim_released_workspace(service):
+            return {"ok": False, "error": lease_error_text(service)}
+        cards_by_name = {card.name: card for card in service._load_flow_cards(force=True)}
+        card = cards_by_name.get(name)
+        if card is None:
+            return {"ok": False, "error": f"Unknown flow: {name}"}
+        if not card.valid:
+            return {"ok": False, "error": card.error or f"Flow {name} is invalid."}
+        with service._state_lock:
+            existing_thread = service.state.manual_run_threads.get(name)
+            if (existing_thread is not None and existing_thread.is_alive()) or name in service.state.pending_manual_run_names:
+                return {"ok": False, "error": f"Flow {name} is already running."}
+            active_same_group = next(
+                (
+                    flow_name
+                    for flow_name, thread in service.state.manual_run_threads.items()
+                    if flow_name != name
+                    and thread.is_alive()
+                    and cards_by_name.get(flow_name) is not None
+                    and cards_by_name[flow_name].group == card.group
+                ),
+                None,
+            )
+            if active_same_group is None:
+                active_same_group = next(
+                    (
+                        flow_name
+                        for flow_name in service.state.pending_manual_run_names
+                        if flow_name != name
+                        and cards_by_name.get(flow_name) is not None
+                        and cards_by_name[flow_name].group == card.group
+                    ),
+                    None,
+                )
+            if active_same_group is not None:
+                return {"ok": False, "error": f"Group {card.group} already has {active_same_group} running."}
+            service.state.reserve_manual_run(name)
+        try:
+            flow = service.flow_execution_service.load_flow(name, workspace_root=service.paths.workspace_root)
+        except Exception as exc:
+            with service._state_lock:
+                service.state.clear_manual_run_reservation(name)
+            return {"ok": False, "error": str(exc)}
+
+        stop_event = threading.Event()
+
+        def _target() -> None:
+            try:
+                service.runtime_execution_service.run_manual(
+                    flow,
+                    runtime_ledger=service.runtime_ledger,
+                    flow_stop_event=stop_event,
+                )
+                service._debug_log(f"manual flow completed name={name}")
+            except Exception as exc:
+                service._debug_log(f"manual flow crashed name={name} error={exc!r}")
+                service._debug_log(traceback.format_exc().rstrip())
+                service.runtime_ledger.append_log(
+                    level="ERROR",
+                    message=str(exc),
+                    created_at_utc=utcnow_text(),
+                    flow_name=name,
+                )
+                raise
+            finally:
+                with service._state_lock:
+                    service.state.unregister_manual_run(name)
+
+        thread = threading.Thread(target=_target, daemon=True)
+        with service._state_lock:
+            service.state.register_manual_run(name, thread=thread, stop_event=stop_event)
+        thread.start()
+        if wait:
+            thread.join()
+        return {"ok": True}
+
+    def start_engine(self) -> dict[str, Any]:
+        service = self.service
+        if not service.state.workspace_owned and not try_claim_released_workspace(service):
+            return {"ok": False, "error": lease_error_text(service)}
+        with service._state_lock:
+            if not service.state.reserve_engine_start():
+                return {"ok": True}
+        flow_names = self.automated_flow_names(force=True)
+        if not flow_names:
+            flow_names = self.automated_flow_names(force=True)
+        if not flow_names:
+            with service._state_lock:
+                service.state.clear_engine_start_reservation()
+            return {"ok": False, "error": "No automated flows are available."}
+        try:
+            flows = service.flow_execution_service.load_flows(flow_names, workspace_root=service.paths.workspace_root)
+        except Exception as exc:
+            with service._state_lock:
+                service.state.clear_engine_start_reservation()
+            return {"ok": False, "error": str(exc)}
+        with service._state_lock:
+            runtime_stop_event = threading.Event()
+            flow_stop_event = threading.Event()
+            service.state.set_engine_threads(runtime_stop_event=runtime_stop_event, flow_stop_event=flow_stop_event)
+            service.state.begin_runtime(status="running")
+
+        def _target() -> None:
+            try:
+                service.runtime_execution_service.run_grouped(
+                    flows,
+                    runtime_ledger=service.runtime_ledger,
+                    runtime_stop_event=runtime_stop_event,
+                    flow_stop_event=flow_stop_event,
+                )
+                service._debug_log("engine runtime exited normally")
+            except Exception as exc:
+                service._debug_log(f"engine runtime crashed error={exc!r}")
+                service._debug_log(traceback.format_exc().rstrip())
+                raise
+            finally:
+                with service._state_lock:
+                    service.state.end_runtime(status="idle")
+                service._debug_log(f"engine thread finished status={service.state.status}")
+
+        engine_thread = threading.Thread(target=_target, daemon=True)
+        with service._state_lock:
+            service.state.engine_thread = engine_thread
+        try:
+            engine_thread.start()
+        except Exception:
+            with service._state_lock:
+                service.state.end_runtime(status="idle")
+            raise
+        return {"ok": True}
+
+    def stop_engine(self) -> dict[str, Any]:
+        service = self.service
+        if not service.state.workspace_owned:
+            return {"ok": False, "error": lease_error_text(service)}
+        with service._state_lock:
+            if not service.state.runtime_active:
+                return {"ok": True}
+            service.state.stop_runtime(status="stopping")
+            runtime_stop_event = service.state.engine_runtime_stop_event
+            flow_stop_event = service.state.engine_flow_stop_event
+        runtime_stop_event.set()
+        flow_stop_event.set()
+        return {"ok": True}
+
+    def stop_flow(self, name: str) -> dict[str, Any]:
+        service = self.service
+        if not service.state.workspace_owned:
+            return {"ok": False, "error": lease_error_text(service)}
+        with service._state_lock:
+            stop_event = service.state.manual_stop_events.get(name)
+        if stop_event is None:
+            return {"ok": False, "error": f"Flow {name} is not running."}
+        stop_event.set()
+        return {"ok": True}
+
+
+__all__ = ["DaemonRuntimeCommandHandler"]

data_engine/hosts/daemon/runtime_control.py

@@ -0,0 +1,31 @@
+"""Runtime stop/join helpers for the daemon host."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from data_engine.hosts.daemon.app import DataEngineDaemonService
+
+
+def stop_active_work(service: "DataEngineDaemonService") -> None:
+    """Signal all active runtime work to stop and wait briefly for it to exit."""
+    with service._state_lock:
+        engine_runtime_stop_event = service.state.engine_runtime_stop_event
+        engine_flow_stop_event = service.state.engine_flow_stop_event
+        engine_thread = service.state.engine_thread
+        manual_stop_events = list(service.state.manual_stop_events.values())
+        manual_threads = list(service.state.manual_run_threads.values())
+    engine_runtime_stop_event.set()
+    engine_flow_stop_event.set()
+    for stop_event in manual_stop_events:
+        stop_event.set()
+    if engine_thread is not None:
+        engine_thread.join(timeout=1.5)
+    for thread in manual_threads:
+        thread.join(timeout=1.5)
+    with service._state_lock:
+        service.state.end_runtime()
+
+
+__all__ = ["stop_active_work"]