py-data-engine 0.1.0__py3-none-any.whl

data_engine/hosts/daemon/composition.py

@@ -0,0 +1,310 @@
+"""Composition helpers for the daemon host."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+import threading
+from uuid import uuid4
+from typing import Callable
+
+from data_engine.hosts.daemon.shared_state import DaemonSharedStateAdapter
+from data_engine.platform.workspace_models import WorkspacePaths, machine_id_text
+from data_engine.runtime.runtime_db import RuntimeLedger
+from data_engine.services.flow_catalog import FlowCatalogService
+from data_engine.services.flow_execution import FlowExecutionService
+from data_engine.services.ledger import LedgerService
+from data_engine.services.runtime_execution import RuntimeExecutionService
+
+
+@dataclass(frozen=True)
+class DaemonHostDependencyFactories:
+    """Constructor seam for daemon host collaborators."""
+
+    flow_catalog_service_factory: Callable[[], FlowCatalogService]
+    flow_execution_service_factory: Callable[[], FlowExecutionService]
+    runtime_execution_service_factory: Callable[[], RuntimeExecutionService]
+    shared_state_adapter_factory: Callable[[], DaemonSharedStateAdapter] = field(default=DaemonSharedStateAdapter)
+
+
+def default_daemon_host_dependency_factories() -> DaemonHostDependencyFactories:
+    """Build the default daemon-host constructor bundle."""
+    return DaemonHostDependencyFactories(
+        flow_catalog_service_factory=FlowCatalogService,
+        flow_execution_service_factory=FlowExecutionService,
+        runtime_execution_service_factory=RuntimeExecutionService,
+        shared_state_adapter_factory=DaemonSharedStateAdapter,
+    )
+
+
+@dataclass(frozen=True)
+class DaemonHostDependencies:
+    """Concrete collaborators used by one daemon host instance."""
+
+    runtime_ledger: RuntimeLedger
+    flow_catalog_service: FlowCatalogService
+    flow_execution_service: FlowExecutionService
+    runtime_execution_service: RuntimeExecutionService
+    shared_state_adapter: DaemonSharedStateAdapter
+
+    @classmethod
+    def build_default(
+        cls,
+        paths: WorkspacePaths,
+        *,
+        ledger_service: LedgerService | None = None,
+        factories: DaemonHostDependencyFactories | None = None,
+    ) -> "DaemonHostDependencies":
+        """Build the default dependency bundle for one workspace host."""
+        ledger_service = ledger_service or LedgerService()
+        factories = factories or default_daemon_host_dependency_factories()
+        return cls(
+            runtime_ledger=ledger_service.open_for_workspace(paths.workspace_root),
+            flow_catalog_service=factories.flow_catalog_service_factory(),
+            flow_execution_service=factories.flow_execution_service_factory(),
+            runtime_execution_service=factories.runtime_execution_service_factory(),
+            shared_state_adapter=factories.shared_state_adapter_factory(),
+        )
+
+
+@dataclass(frozen=True)
+class DaemonHostIdentity:
+    """Process and machine identity for one daemon host instance."""
+
+    machine_id: str
+    daemon_id: str
+    pid: int
+
+    @classmethod
+    def current_process(cls) -> "DaemonHostIdentity":
+        """Build the current-process identity for one daemon host."""
+        return cls(
+            machine_id=machine_id_text(),
+            daemon_id=uuid4().hex,
+            pid=os.getpid(),
+        )
+
+
+@dataclass
+class DaemonHostState:
+    """Mutable state for a fresh daemon host instance."""
+
+    status: str
+    last_checkpoint_at_utc: str
+    workspace_owned: bool
+    leased_by_machine_id: str | None
+    runtime_active: bool
+    runtime_stopping: bool
+    engine_starting: bool
+    engine_thread: threading.Thread | None
+    engine_runtime_stop_event: threading.Event
+    engine_flow_stop_event: threading.Event
+    pending_manual_run_names: set[str]
+    manual_run_threads: dict[str, threading.Thread]
+    manual_stop_events: dict[str, threading.Event]
+    shutdown_event: threading.Event
+    checkpoint_thread: threading.Thread | None
+    consecutive_checkpoint_failures: int
+    listener: object | None
+
+    @classmethod
+    def build(cls, *, started_at_utc: str) -> "DaemonHostState":
+        """Build the default mutable state for a fresh daemon host."""
+        return cls(
+            status="starting",
+            last_checkpoint_at_utc=started_at_utc,
+            workspace_owned=False,
+            leased_by_machine_id=None,
+            runtime_active=False,
+            runtime_stopping=False,
+            engine_starting=False,
+            engine_thread=None,
+            engine_runtime_stop_event=threading.Event(),
+            engine_flow_stop_event=threading.Event(),
+            pending_manual_run_names=set(),
+            manual_run_threads={},
+            manual_stop_events={},
+            shutdown_event=threading.Event(),
+            checkpoint_thread=None,
+            consecutive_checkpoint_failures=0,
+            listener=None,
+        )
+
+    def claim_workspace(self) -> None:
+        """Mark the current daemon as owning the workspace."""
+        self.workspace_owned = True
+        self.leased_by_machine_id = None
+        self.status = "idle"
+
+    def release_workspace(self, *, leased_by_machine_id: str | None = None, status: str | None = None) -> None:
+        """Mark the current daemon as no longer owning the workspace."""
+        self.workspace_owned = False
+        self.leased_by_machine_id = leased_by_machine_id
+        if status is not None:
+            self.status = status
+
+    def begin_runtime(self, *, status: str = "running") -> None:
+        """Mark the engine runtime as active and running."""
+        self.engine_starting = False
+        self.runtime_active = True
+        self.runtime_stopping = False
+        self.status = status
+
+    def stop_runtime(self, *, status: str = "stopping") -> None:
+        """Mark the engine runtime as stopping."""
+        self.engine_starting = False
+        self.runtime_stopping = True
+        self.status = status
+
+    def end_runtime(self, *, status: str = "idle") -> None:
+        """Mark the engine runtime as inactive."""
+        self.engine_starting = False
+        self.runtime_active = False
+        self.runtime_stopping = False
+        if self.status != "failed":
+            self.status = status
+
+    def set_checkpoint_time(self, checkpoint_at_utc: str, *, status: str | None = None) -> None:
+        """Update the last successful checkpoint timestamp."""
+        self.last_checkpoint_at_utc = checkpoint_at_utc
+        if status is not None:
+            self.status = status
+
+    def set_leased_by_machine_id(self, machine_id: str | None) -> None:
+        """Update the current lease owner identifier."""
+        self.leased_by_machine_id = machine_id
+
+    def increment_checkpoint_failures(self) -> int:
+        """Increment the repeated-checkpoint failure counter."""
+        self.consecutive_checkpoint_failures += 1
+        return self.consecutive_checkpoint_failures
+
+    def reset_checkpoint_failures(self) -> None:
+        """Reset the repeated-checkpoint failure counter."""
+        self.consecutive_checkpoint_failures = 0
+
+    def set_engine_threads(
+        self,
+        *,
+        runtime_stop_event: threading.Event,
+        flow_stop_event: threading.Event,
+        engine_thread: threading.Thread | None = None,
+    ) -> None:
+        """Replace the active engine coordination objects."""
+        self.engine_runtime_stop_event = runtime_stop_event
+        self.engine_flow_stop_event = flow_stop_event
+        self.engine_thread = engine_thread
+
+    def reserve_engine_start(self) -> bool:
+        """Reserve engine startup so concurrent start requests collapse to one attempt."""
+        if self.runtime_active or self.engine_starting:
+            return False
+        self.engine_starting = True
+        return True
+
+    def clear_engine_start_reservation(self) -> None:
+        """Clear any in-progress engine startup reservation."""
+        self.engine_starting = False
+
+    def reserve_manual_run(self, name: str) -> bool:
+        """Reserve one manual run name before flow loading starts."""
+        if name in self.pending_manual_run_names:
+            return False
+        self.pending_manual_run_names.add(name)
+        return True
+
+    def clear_manual_run_reservation(self, name: str) -> None:
+        """Clear one in-progress manual run reservation."""
+        self.pending_manual_run_names.discard(name)
+
+    def register_manual_run(self, name: str, *, thread: threading.Thread, stop_event: threading.Event) -> None:
+        """Register one manual run and its stop signal."""
+        self.pending_manual_run_names.discard(name)
+        self.manual_run_threads[name] = thread
+        self.manual_stop_events[name] = stop_event
+
+    def unregister_manual_run(self, name: str) -> None:
+        """Remove one completed manual run."""
+        self.pending_manual_run_names.discard(name)
+        self.manual_run_threads.pop(name, None)
+        self.manual_stop_events.pop(name, None)
+
+    def set_listener(self, listener: object | None) -> None:
+        """Update the active listener object."""
+        self.listener = listener
+
+
+class DaemonHostFacade:
+    """Explicit high-level host-state facade over the mutable daemon state object."""
+
+    def __init__(self, state: DaemonHostState) -> None:
+        self.state = state
+
+    @property
+    def status(self) -> str:
+        return self.state.status
+
+    @status.setter
+    def status(self, value: str) -> None:
+        self.state.status = value
+
+    @property
+    def workspace_owned(self) -> bool:
+        return self.state.workspace_owned
+
+    @workspace_owned.setter
+    def workspace_owned(self, value: bool) -> None:
+        self.state.workspace_owned = value
+
+    @property
+    def leased_by_machine_id(self) -> str | None:
+        return self.state.leased_by_machine_id
+
+    @leased_by_machine_id.setter
+    def leased_by_machine_id(self, value: str | None) -> None:
+        self.state.leased_by_machine_id = value
+
+    @property
+    def runtime_active(self) -> bool:
+        return self.state.runtime_active
+
+    @runtime_active.setter
+    def runtime_active(self, value: bool) -> None:
+        self.state.runtime_active = value
+
+    @property
+    def runtime_stopping(self) -> bool:
+        return self.state.runtime_stopping
+
+    @runtime_stopping.setter
+    def runtime_stopping(self, value: bool) -> None:
+        self.state.runtime_stopping = value
+
+    @property
+    def shutdown_event(self) -> threading.Event:
+        return self.state.shutdown_event
+
+    @shutdown_event.setter
+    def shutdown_event(self, value: threading.Event) -> None:
+        self.state.shutdown_event = value
+
+    @property
+    def listener(self) -> object | None:
+        return self.state.listener
+
+    @listener.setter
+    def listener(self, value: object | None) -> None:
+        self.state.listener = value
+
+
+__all__ = [
+    "DaemonHostFacade",
+    "DaemonHostDependencyFactories",
+    "DaemonHostDependencies",
+    "DaemonHostIdentity",
+    "DaemonHostState",
+    "default_daemon_host_dependency_factories",
+]
+
+# Backward-compatible alias while the daemon host moves to a single grouped state object.
+DaemonHostInitialState = DaemonHostState

data_engine/hosts/daemon/constants.py

@@ -0,0 +1,15 @@
+"""Shared daemon host constants."""
+
+APP_VERSION = "0.1.0"
+CHECKPOINT_INTERVAL_SECONDS = 30.0
+CONTROL_REQUEST_POLL_INTERVAL_SECONDS = 1.0
+STALE_AFTER_SECONDS = 90.0
+DAEMON_STARTUP_LOCK_STALE_SECONDS = 15.0
+
+__all__ = [
+    "APP_VERSION",
+    "CHECKPOINT_INTERVAL_SECONDS",
+    "CONTROL_REQUEST_POLL_INTERVAL_SECONDS",
+    "DAEMON_STARTUP_LOCK_STALE_SECONDS",
+    "STALE_AFTER_SECONDS",
+]

data_engine/hosts/daemon/entrypoints.py

@@ -0,0 +1,97 @@
+"""Module entrypoints for launching one workspace daemon process."""
+
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+from typing import Callable
+
+from data_engine.domain import DaemonLifecyclePolicy
+from data_engine.hosts.daemon.server import serve_workspace_daemon as serve_daemon_process
+from data_engine.platform.workspace_models import (
+    DATA_ENGINE_APP_ROOT_ENV_VAR,
+    DATA_ENGINE_WORKSPACE_ID_ENV_VAR,
+    DATA_ENGINE_WORKSPACE_ROOT_ENV_VAR,
+)
+from data_engine.services import WorkspaceService
+
+
+def default_workspace_service_factory() -> WorkspaceService:
+    """Build the default workspace-service collaborator for daemon entrypoints."""
+    return WorkspaceService()
+
+
+def serve_workspace_daemon(
+    service_type,
+    *,
+    workspace_root: Path | None = None,
+    workspace_id: str | None = None,
+    lifecycle_policy: DaemonLifecyclePolicy = DaemonLifecyclePolicy.PERSISTENT,
+    workspace_service: WorkspaceService | None = None,
+    resolve_paths_func=None,
+) -> int:
+    """Start serving one workspace daemon in the current process."""
+    return serve_daemon_process(
+        service_type,
+        workspace_root=workspace_root,
+        workspace_id=workspace_id,
+        lifecycle_policy=lifecycle_policy,
+        workspace_service=workspace_service,
+        resolve_paths_func=resolve_paths_func,
+    )
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the daemon module parser."""
+    parser = argparse.ArgumentParser(description="Run one Data Engine workspace daemon.")
+    parser.add_argument("--workspace", type=Path, required=True, help="Authored workspace root to host.")
+    parser.add_argument("--app-root", type=Path, default=None, help="Data Engine app root used for local artifacts.")
+    parser.add_argument("--workspace-id", default=None, help="Explicit workspace id override.")
+    parser.add_argument(
+        "--lifecycle-policy",
+        choices=tuple(policy.value for policy in DaemonLifecyclePolicy),
+        default=DaemonLifecyclePolicy.PERSISTENT.value,
+        help="Daemon lifetime policy.",
+    )
+    return parser
+
+
+def main(
+    service_type,
+    argv: list[str] | None = None,
+    *,
+    workspace_service: WorkspaceService | None = None,
+    workspace_service_factory: Callable[[], WorkspaceService] | None = None,
+    resolve_paths_func=None,
+    serve_workspace_daemon_func=None,
+) -> int:
+    """Run the daemon module entrypoint for one concrete daemon service type."""
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.app_root is not None:
+        os.environ[DATA_ENGINE_APP_ROOT_ENV_VAR] = str(args.app_root.expanduser().resolve())
+    os.environ[DATA_ENGINE_WORKSPACE_ROOT_ENV_VAR] = str(args.workspace.expanduser().resolve())
+    if args.workspace_id:
+        os.environ[DATA_ENGINE_WORKSPACE_ID_ENV_VAR] = args.workspace_id
+    if resolve_paths_func is None:
+        workspace_service = workspace_service or (workspace_service_factory or default_workspace_service_factory)()
+        resolve_paths_func = workspace_service.resolve_paths
+    paths = resolve_paths_func(workspace_root=args.workspace, workspace_id=args.workspace_id)
+    serve_workspace_daemon_func = serve_workspace_daemon_func or serve_workspace_daemon
+    return serve_workspace_daemon_func(
+        service_type,
+        workspace_root=paths.workspace_root,
+        workspace_id=paths.workspace_id,
+        lifecycle_policy=args.lifecycle_policy,
+        workspace_service=workspace_service,
+        resolve_paths_func=resolve_paths_func,
+    )
+
+
+__all__ = [
+    "build_parser",
+    "default_workspace_service_factory",
+    "main",
+    "serve_workspace_daemon",
+]

data_engine/hosts/daemon/lifecycle.py

@@ -0,0 +1,191 @@
+"""Lifecycle and checkpoint policy for the daemon host."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import time
+import traceback
+from typing import TYPE_CHECKING
+
+from data_engine.domain import DaemonLifecyclePolicy
+from data_engine.hosts.daemon.ownership import (
+    honor_control_request_if_needed,
+    release_workspace_claim,
+    try_claim_requested_control,
+)
+from data_engine.hosts.daemon.constants import (
+    CHECKPOINT_INTERVAL_SECONDS,
+    CONTROL_REQUEST_POLL_INTERVAL_SECONDS,
+)
+from data_engine.hosts.daemon.runtime_control import stop_active_work
+
+if TYPE_CHECKING:
+    from data_engine.hosts.daemon.app import DataEngineDaemonService
+
+
+def checkpoint_loop(service: "DataEngineDaemonService") -> None:
+    next_checkpoint_at = time.monotonic() + CHECKPOINT_INTERVAL_SECONDS
+    while not service.host.shutdown_event.wait(CONTROL_REQUEST_POLL_INTERVAL_SECONDS):
+        if _should_shutdown_for_missing_clients(service):
+            service._debug_log("no live local clients remain; shutting down ephemeral daemon")
+            relinquish_workspace_for_missing_clients(service)
+            break
+        if not service._workspace_root_is_available():
+            service._debug_log("workspace root no longer available; shutting down daemon")
+            relinquish_workspace_for_missing_root(service)
+            break
+        with service._state_lock:
+            workspace_owned = service.host.workspace_owned
+        if not workspace_owned:
+            try:
+                if try_claim_requested_control(service):
+                    next_checkpoint_at = time.monotonic() + CHECKPOINT_INTERVAL_SECONDS
+                    continue
+                service._refresh_observer_snapshot()
+            except Exception:
+                pass
+            continue
+        try:
+            if honor_control_request_if_needed(service):
+                next_checkpoint_at = time.monotonic() + CHECKPOINT_INTERVAL_SECONDS
+                continue
+            if time.monotonic() < next_checkpoint_at:
+                continue
+            with service._state_lock:
+                status = "degraded" if service.state.consecutive_checkpoint_failures >= 1 else service.host.status
+            service._checkpoint_once(status=status)
+            with service._state_lock:
+                service.state.consecutive_checkpoint_failures = 0
+            next_checkpoint_at = time.monotonic() + CHECKPOINT_INTERVAL_SECONDS
+        except Exception:
+            service._debug_log("checkpoint failed")
+            service._debug_log(traceback.format_exc().rstrip())
+            with service._state_lock:
+                failure_count = service.state.increment_checkpoint_failures()
+            if failure_count == 2:
+                with service._state_lock:
+                    service.state.status = "degraded"
+                service._update_daemon_state(status="degraded")
+                service._debug_log("daemon marked degraded after repeated checkpoint failures")
+            if failure_count >= 3:
+                service._debug_log("relinquishing workspace after repeated checkpoint failures")
+                relinquish_workspace_after_checkpoint_failures(service)
+                next_checkpoint_at = time.monotonic() + CHECKPOINT_INTERVAL_SECONDS
+
+
+def relinquish_workspace_after_checkpoint_failures(service: "DataEngineDaemonService") -> None:
+    """Stop active work, release shared ownership, and stop the daemon."""
+    with service._state_lock:
+        service.state.stop_runtime(status="failed")
+    service._debug_log("relinquish workspace starting")
+    stop_active_work(service)
+    release_workspace_claim(service, status="failed", update_state=True)
+    service._debug_log("relinquish workspace complete")
+    shutdown_if_unowned_and_idle(service, reason="checkpoint failures")
+
+
+def relinquish_workspace_for_control_request(service: "DataEngineDaemonService", requester_machine_id: str) -> None:
+    """Stop active work, hand ownership off, and stop this daemon."""
+    with service._state_lock:
+        service.state.stop_runtime(status="stopping flow")
+    service._debug_log(f"relinquish for control request requester={requester_machine_id}")
+    stop_active_work(service)
+    release_workspace_claim(
+        service,
+        leased_by_machine_id=requester_machine_id,
+        status="leased",
+        update_state=True,
+    )
+    service._debug_log("relinquish for control request complete")
+    shutdown_if_unowned_and_idle(service, reason="control request handoff")
+
+
+def relinquish_workspace_for_missing_root(service: "DataEngineDaemonService") -> None:
+    """Stop active work and exit when the authored workspace root disappears."""
+    with service._state_lock:
+        service.state.stop_runtime(status="workspace missing")
+    stop_active_work(service)
+    release_workspace_claim(service, status="workspace missing")
+    service.host.shutdown_event.set()
+    service._wake_listener()
+
+
+def relinquish_workspace_for_missing_clients(service: "DataEngineDaemonService") -> None:
+    """Stop active work and exit when an ephemeral daemon has no live local clients."""
+    with service._state_lock:
+        service.state.stop_runtime(status="client disconnected")
+    stop_active_work(service)
+    release_workspace_claim(service, status="client disconnected")
+    service.host.shutdown_event.set()
+    service._wake_listener()
+
+
+def _should_shutdown_for_missing_clients(service: "DataEngineDaemonService") -> bool:
+    """Return whether an ephemeral daemon should stop because no live clients remain."""
+    if service.lifecycle_policy is not DaemonLifecyclePolicy.EPHEMERAL:
+        return False
+    with service._state_lock:
+        if service.host.runtime_active or service.host.runtime_stopping:
+            return False
+        if service.state.manual_run_threads:
+            return False
+    try:
+        return service.runtime_ledger.count_live_client_sessions(service.paths.workspace_id) == 0
+    except Exception:
+        return False
+
+
+def shutdown_if_unowned_and_idle(service: "DataEngineDaemonService", *, reason: str) -> None:
+    """Exit when this daemon no longer owns the workspace and has no active work."""
+    with service._state_lock:
+        if service.host.workspace_owned:
+            return
+        if service.host.runtime_active or service.host.runtime_stopping:
+            return
+        if service.state.manual_run_threads:
+            return
+    service._debug_log(f"shutdown requested reason={reason}")
+    service.host.shutdown_event.set()
+    service._wake_listener()
+
+
+def shutdown(service: "DataEngineDaemonService") -> None:
+    service._debug_log("shutdown starting")
+    stop_active_work(service)
+    if service.state.checkpoint_thread is not None and service.state.checkpoint_thread.is_alive():
+        service.state.checkpoint_thread.join(timeout=5.0)
+    with service._state_lock:
+        workspace_owned = service.host.workspace_owned
+        status = service.host.status
+    if workspace_owned and status not in {"failed", "workspace missing"}:
+        try:
+            service._checkpoint_once(status="stopping")
+        except Exception:
+            pass
+    release_workspace_claim(service)
+    try:
+        service.runtime_ledger.clear_daemon_state(service.paths.workspace_id)
+    except Exception:
+        pass
+    service.runtime_ledger.close()
+    if service.host.listener is not None:
+        try:
+            service.host.listener.close()
+        except Exception:
+            pass
+    if service.paths.daemon_endpoint_kind == "unix":
+        try:
+            Path(service.paths.daemon_endpoint_path).unlink()
+        except FileNotFoundError:
+            pass
+    service._debug_log("shutdown complete")
+
+
+__all__ = [
+    "checkpoint_loop",
+    "relinquish_workspace_after_checkpoint_failures",
+    "relinquish_workspace_for_control_request",
+    "relinquish_workspace_for_missing_root",
+    "shutdown",
+    "shutdown_if_unowned_and_idle",
+]