py-data-engine 0.1.0__py3-none-any.whl

data_engine/hosts/daemon/server.py

@@ -0,0 +1,84 @@
+"""Listener loop and host serving helpers for the daemon process."""
+
+from __future__ import annotations
+
+from multiprocessing import AuthenticationError
+from multiprocessing.connection import Listener
+from pathlib import Path
+import threading
+import traceback
+from typing import TYPE_CHECKING, Any
+
+from data_engine.domain import DaemonLifecyclePolicy
+from data_engine.hosts.daemon.client import (
+    _decode_message,
+    _encode_message,
+    _remove_stale_unix_endpoint,
+    daemon_authkey,
+    endpoint_address,
+    endpoint_family,
+)
+from data_engine.services import WorkspaceService
+
+if TYPE_CHECKING:
+    from data_engine.hosts.daemon.app import DataEngineDaemonService
+
+
+def serve_forever(service: "DataEngineDaemonService") -> None:
+    """Run the workspace daemon listener loop until shutdown."""
+    try:
+        service.initialize()
+        service.state.checkpoint_thread = threading.Thread(target=service._checkpoint_loop, daemon=True)
+        service.state.checkpoint_thread.start()
+        _remove_stale_unix_endpoint(service.paths)
+        listener = Listener(
+            endpoint_address(service.paths),
+            family=endpoint_family(service.paths),
+            authkey=daemon_authkey(service.paths),
+        )
+        service.host.listener = listener
+        service._debug_log(f"listener ready endpoint={service.paths.daemon_endpoint_path}")
+        while not service.host.shutdown_event.is_set():
+            try:
+                connection = listener.accept()
+            except (AuthenticationError, OSError, EOFError):
+                if service.host.shutdown_event.is_set():
+                    break
+                service._debug_log("listener accept failed but daemon remains alive")
+                continue
+            with connection:
+                try:
+                    payload = _decode_message(connection.recv_bytes())
+                    response = service._handle_command(payload)
+                except Exception as exc:  # pragma: no cover - defensive daemon boundary
+                    service._debug_log(f"command handling error: {exc!r}")
+                    response = {"ok": False, "error": str(exc)}
+                connection.send_bytes(_encode_message(response))
+    except Exception as exc:
+        service._debug_log(f"serve_forever fatal error: {exc!r}")
+        service._debug_log(traceback.format_exc().rstrip())
+        raise
+    finally:
+        service._shutdown()
+
+
+def serve_workspace_daemon(
+    service_type: type["DataEngineDaemonService"],
+    *,
+    workspace_root: Path | None = None,
+    workspace_id: str | None = None,
+    lifecycle_policy: DaemonLifecyclePolicy = DaemonLifecyclePolicy.PERSISTENT,
+    workspace_service: WorkspaceService | None = None,
+    resolve_paths_func=None,
+) -> int:
+    """Start serving one workspace daemon in the current process."""
+    if resolve_paths_func is None:
+        workspace_service = workspace_service or WorkspaceService()
+        resolve_paths_func = workspace_service.resolve_paths
+    paths = resolve_paths_func(workspace_root=workspace_root, workspace_id=workspace_id)
+    service = service_type(paths, lifecycle_policy=lifecycle_policy)
+    service.serve_forever()
+    return 0
+
+
+__all__ = ["serve_forever", "serve_workspace_daemon"]

data_engine/hosts/daemon/shared_state.py

@@ -0,0 +1,147 @@
+"""Host-owned adapter over shared workspace lease and snapshot operations."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from data_engine.platform.workspace_models import WorkspacePaths
+from data_engine.runtime.runtime_db import RuntimeLedger
+from data_engine.runtime.shared_state import (
+    checkpoint_workspace_state as checkpoint_runtime_workspace_state,
+    claim_workspace as claim_runtime_workspace,
+    hydrate_local_runtime_state,
+    initialize_workspace_state,
+    lease_is_stale,
+    read_control_request,
+    read_lease_metadata,
+    recover_stale_workspace,
+    release_workspace,
+    remove_control_request,
+    remove_lease_metadata,
+    write_control_request,
+    write_lease_metadata,
+)
+
+
+class DaemonSharedStateAdapter:
+    """Own host-facing access to shared lease, control-request, and snapshot state."""
+
+    def initialize_workspace(self, paths: WorkspacePaths) -> None:
+        initialize_workspace_state(paths)
+
+    def claim_workspace(self, paths: WorkspacePaths) -> bool:
+        return claim_runtime_workspace(paths)
+
+    def release_workspace(self, paths: WorkspacePaths) -> None:
+        release_workspace(paths)
+
+    def recover_stale_workspace(
+        self,
+        paths: WorkspacePaths,
+        *,
+        machine_id: str,
+        stale_after_seconds: float,
+        reclaim: bool = True,
+        ) -> bool:
+        return recover_stale_workspace(
+            paths,
+            machine_id=machine_id,
+            stale_after_seconds=stale_after_seconds,
+            reclaim=reclaim,
+        )
+
+    def lease_is_stale(self, paths: WorkspacePaths, *, stale_after_seconds: float) -> bool:
+        return lease_is_stale(paths, stale_after_seconds=stale_after_seconds)
+
+    def hydrate_local_runtime(self, paths: WorkspacePaths, ledger: RuntimeLedger) -> None:
+        hydrate_local_runtime_state(paths, ledger)
+
+    def checkpoint_workspace_state(
+        self,
+        paths: WorkspacePaths,
+        ledger: RuntimeLedger,
+        *,
+        workspace_id: str,
+        machine_id: str,
+        daemon_id: str,
+        pid: int,
+        status: str,
+        started_at_utc: str,
+        last_checkpoint_at_utc: str,
+        app_version: str | None,
+    ) -> None:
+        checkpoint_runtime_workspace_state(
+            paths,
+            ledger,
+            workspace_id=workspace_id,
+            machine_id=machine_id,
+            daemon_id=daemon_id,
+            pid=pid,
+            status=status,
+            started_at_utc=started_at_utc,
+            last_checkpoint_at_utc=last_checkpoint_at_utc,
+            app_version=app_version,
+        )
+
+    def read_lease_metadata(self, paths: WorkspacePaths) -> dict[str, Any] | None:
+        metadata = read_lease_metadata(paths)
+        return metadata if isinstance(metadata, dict) else None
+
+    def write_lease_metadata(
+        self,
+        paths: WorkspacePaths,
+        *,
+        workspace_id: str,
+        machine_id: str,
+        daemon_id: str,
+        pid: int,
+        status: str,
+        started_at_utc: str,
+        last_checkpoint_at_utc: str,
+        app_version: str | None,
+    ) -> None:
+        write_lease_metadata(
+            paths,
+            workspace_id=workspace_id,
+            machine_id=machine_id,
+            daemon_id=daemon_id,
+            pid=pid,
+            status=status,
+            started_at_utc=started_at_utc,
+            last_checkpoint_at_utc=last_checkpoint_at_utc,
+            app_version=app_version,
+        )
+
+    def remove_lease_metadata(self, paths: WorkspacePaths) -> None:
+        remove_lease_metadata(paths)
+
+    def read_control_request(self, paths: WorkspacePaths) -> dict[str, Any] | None:
+        metadata = read_control_request(paths)
+        return metadata if isinstance(metadata, dict) else None
+
+    def write_control_request(
+        self,
+        paths: WorkspacePaths,
+        *,
+        workspace_id: str,
+        requester_machine_id: str,
+        requester_host_name: str,
+        requester_pid: int,
+        requester_client_kind: str,
+        requested_at_utc: str,
+    ) -> None:
+        write_control_request(
+            paths,
+            workspace_id=workspace_id,
+            requester_machine_id=requester_machine_id,
+            requester_host_name=requester_host_name,
+            requester_pid=requester_pid,
+            requester_client_kind=requester_client_kind,
+            requested_at_utc=requested_at_utc,
+        )
+
+    def remove_control_request(self, paths: WorkspacePaths) -> None:
+        remove_control_request(paths)
+
+
+__all__ = ["DaemonSharedStateAdapter"]

data_engine/hosts/daemon/state_sync.py

@@ -0,0 +1,101 @@
+"""Daemon state publication and observer-sync helpers."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from data_engine.domain.time import utcnow_text
+from data_engine.hosts.daemon.constants import APP_VERSION
+from data_engine.views.models import QtFlowCard
+
+if TYPE_CHECKING:
+    from data_engine.hosts.daemon.app import DataEngineDaemonService
+
+
+class DaemonStateSyncHandler:
+    """Own daemon status payloads, checkpoint publication, and observer sync."""
+
+    def __init__(self, service: "DataEngineDaemonService") -> None:
+        self.service = service
+
+    def load_flow_cards(self, *, force: bool = False) -> tuple[QtFlowCard, ...]:
+        return self.service._load_flow_cards(force=force)
+
+    def status_payload(self) -> dict[str, Any]:
+        service = self.service
+        with service._state_lock:
+            state = service.state
+            status = state.status
+            workspace_owned = state.workspace_owned
+            leased_by_machine_id = state.leased_by_machine_id
+            runtime_active = state.runtime_active
+            runtime_stopping = state.runtime_stopping
+            manual_runs = sorted(state.manual_run_threads)
+            last_checkpoint_at_utc = state.last_checkpoint_at_utc
+        return {
+            "workspace_id": service.paths.workspace_id,
+            "workspace_root": str(service.paths.workspace_root),
+            "machine_id": service.machine_id,
+            "daemon_id": service.daemon_id,
+            "pid": service.pid,
+            "status": status,
+            "workspace_owned": workspace_owned,
+            "leased_by_machine_id": leased_by_machine_id,
+            "engine_active": runtime_active,
+            "engine_stopping": runtime_stopping,
+            "manual_runs": manual_runs,
+            "last_checkpoint_at_utc": last_checkpoint_at_utc,
+        }
+
+    def checkpoint_once(self, *, status: str) -> None:
+        service = self.service
+        checkpoint_time = utcnow_text()
+        service.shared_state_adapter.checkpoint_workspace_state(
+            service.paths,
+            service.runtime_ledger,
+            workspace_id=service.paths.workspace_id,
+            machine_id=service.machine_id,
+            daemon_id=service.daemon_id,
+            pid=service.pid,
+            status=status,
+            started_at_utc=service.started_at_utc,
+            last_checkpoint_at_utc=checkpoint_time,
+            app_version=APP_VERSION,
+        )
+        with service._state_lock:
+            service.state.set_checkpoint_time(checkpoint_time)
+        self.update_daemon_state(status=status)
+
+    def refresh_observer_snapshot(self) -> None:
+        service = self.service
+        service.shared_state_adapter.hydrate_local_runtime(service.paths, service.runtime_ledger)
+        metadata = service.shared_state_adapter.read_lease_metadata(service.paths)
+        with service._state_lock:
+            service.state.set_leased_by_machine_id(
+                str(metadata.get("machine_id"))
+                if metadata is not None and metadata.get("machine_id") is not None
+                else None
+            )
+        if metadata is None:
+            self.update_daemon_state(status="available")
+            service._shutdown_if_unowned_and_idle(reason="lease released")
+            return
+        self.update_daemon_state(status="leased")
+
+    def update_daemon_state(self, *, status: str) -> None:
+        service = self.service
+        service.runtime_ledger.upsert_daemon_state(
+            workspace_id=service.paths.workspace_id,
+            pid=service.pid,
+            endpoint_kind=service.paths.daemon_endpoint_kind,
+            endpoint_path=service.paths.daemon_endpoint_path,
+            started_at_utc=service.started_at_utc,
+            last_checkpoint_at_utc=service.state.last_checkpoint_at_utc,
+            status=status,
+            app_root=str(service.paths.app_root),
+            workspace_root=str(service.paths.workspace_root),
+            version_text=APP_VERSION,
+        )
+
+
+__all__ = ["DaemonStateSyncHandler"]

data_engine/platform/__init__.py

@@ -0,0 +1 @@
+"""Application identity, paths, local settings, and theming."""

data_engine/platform/identity.py

@@ -0,0 +1,35 @@
+"""Centralized application identity and naming helpers."""
+
+from __future__ import annotations
+
+from typing import Final
+
+APP_INTERNAL_ID: Final[str] = "data_engine"
+APP_DISTRIBUTION_NAME: Final[str] = "py-data-engine"
+APP_DISPLAY_NAME: Final[str] = "Data Engine"
+APP_ENV_PREFIX: Final[str] = "DATA_ENGINE"
+APP_CACHE_DIR_NAME: Final[str] = "data_engine"
+APP_RUNTIME_NAMESPACE: Final[str] = "data_engine"
+APP_ARTIFACTS_DIR_NAME: Final[str] = "artifacts"
+WORKSPACE_CACHE_DIR_NAME: Final[str] = "workspace_cache"
+RUNTIME_STATE_DIR_NAME: Final[str] = "runtime_state"
+
+
+def env_var(name: str) -> str:
+    """Return one application-scoped environment variable name."""
+    normalized = name.strip().upper()
+    return f"{APP_ENV_PREFIX}_{normalized}"
+
+
+__all__ = [
+    "APP_CACHE_DIR_NAME",
+    "APP_DISPLAY_NAME",
+    "APP_DISTRIBUTION_NAME",
+    "APP_ENV_PREFIX",
+    "APP_INTERNAL_ID",
+    "APP_RUNTIME_NAMESPACE",
+    "APP_ARTIFACTS_DIR_NAME",
+    "RUNTIME_STATE_DIR_NAME",
+    "WORKSPACE_CACHE_DIR_NAME",
+    "env_var",
+]

data_engine/platform/local_settings.py

@@ -0,0 +1,146 @@
+"""Machine-local app settings persisted in a local SQLite database."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import sqlite3
+
+from data_engine.platform.identity import APP_CACHE_DIR_NAME, env_var
+
+
+DATA_ENGINE_APP_ROOT_ENV_VAR = env_var("app_root")
+DATA_ENGINE_STATE_ROOT_ENV_VAR = env_var("state_root")
+
+
+def default_state_root(*, app_root: Path | None = None) -> Path:
+    """Return the platform-local mutable state root for the app."""
+    env_value = os.environ.get(DATA_ENGINE_STATE_ROOT_ENV_VAR)
+    if env_value and env_value.strip():
+        return Path(env_value).expanduser().resolve()
+
+    home = Path.home()
+    if os.name == "nt":
+        base = Path(os.environ.get("LOCALAPPDATA") or home / "AppData" / "Local")
+        return base / APP_CACHE_DIR_NAME
+    if sys_platform() == "darwin":
+        return home / "Library" / "Application Support" / APP_CACHE_DIR_NAME
+    xdg_state = os.environ.get("XDG_STATE_HOME")
+    if xdg_state and xdg_state.strip():
+        return Path(xdg_state).expanduser().resolve() / APP_CACHE_DIR_NAME
+    xdg_data = os.environ.get("XDG_DATA_HOME")
+    if xdg_data and xdg_data.strip():
+        return Path(xdg_data).expanduser().resolve() / APP_CACHE_DIR_NAME
+    return home / ".local" / "share" / APP_CACHE_DIR_NAME
+
+
+def default_settings_db_path(*, app_root: Path | None = None) -> Path:
+    """Return the default machine-local settings database path."""
+    return default_state_root(app_root=app_root) / "settings" / "app_settings.sqlite"
+
+
+def sys_platform() -> str:
+    """Return the normalized platform identifier."""
+    import sys
+
+    return sys.platform
+
+
+class LocalSettingsStore:
+    """Persist simple machine-local UI settings in SQLite."""
+
+    def __init__(self, db_path: Path) -> None:
+        self.db_path = Path(db_path).expanduser().resolve()
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._initialize()
+
+    @classmethod
+    def open_default(cls, *, app_root: Path | None = None) -> "LocalSettingsStore":
+        return cls(default_settings_db_path(app_root=app_root))
+
+    def _connection(self) -> sqlite3.Connection:
+        # Recreate the parent on every open so repeated temp-root churn during
+        # long test sessions cannot strand the settings store on a deleted path.
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        connection = sqlite3.connect(self.db_path)
+        connection.row_factory = sqlite3.Row
+        self._ensure_schema(connection)
+        return connection
+
+    @staticmethod
+    def _ensure_schema(connection: sqlite3.Connection) -> None:
+        connection.execute(
+            """
+            CREATE TABLE IF NOT EXISTS settings (
+                key TEXT PRIMARY KEY,
+                value TEXT
+            )
+            """
+        )
+
+    def _initialize(self) -> None:
+        with self._connection():
+            pass
+
+    def get(self, key: str) -> str | None:
+        with self._connection() as connection:
+            row = connection.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
+        if row is None:
+            return None
+        value = row["value"]
+        return str(value) if value is not None else None
+
+    def set(self, key: str, value: str | None) -> None:
+        with self._connection() as connection:
+            if value is None or not str(value).strip():
+                connection.execute("DELETE FROM settings WHERE key = ?", (key,))
+            else:
+                connection.execute(
+                    """
+                    INSERT INTO settings (key, value)
+                    VALUES (?, ?)
+                    ON CONFLICT(key) DO UPDATE SET value = excluded.value
+                    """,
+                    (key, str(value)),
+                )
+
+    def workspace_collection_root(self) -> Path | None:
+        value = self.get("workspace_collection_root")
+        if value is None or not value.strip():
+            return None
+        return Path(value).expanduser().resolve()
+
+    def set_workspace_collection_root(self, value: Path | str | None) -> None:
+        if value is None:
+            self.set("workspace_collection_root", None)
+            return
+        self.set("workspace_collection_root", str(Path(value).expanduser().resolve()))
+
+    def default_workspace_id(self) -> str | None:
+        value = self.get("default_workspace_id")
+        if value is None or not value.strip():
+            return None
+        return value.strip()
+
+    def set_default_workspace_id(self, value: str | None) -> None:
+        self.set("default_workspace_id", value.strip() if value is not None else None)
+
+    def runtime_root(self) -> Path | None:
+        value = self.get("runtime_root")
+        if value is None or not value.strip():
+            return None
+        return Path(value).expanduser().resolve()
+
+    def set_runtime_root(self, value: Path | str | None) -> None:
+        if value is None:
+            self.set("runtime_root", None)
+            return
+        self.set("runtime_root", str(Path(value).expanduser().resolve()))
+
+
+__all__ = [
+    "DATA_ENGINE_STATE_ROOT_ENV_VAR",
+    "LocalSettingsStore",
+    "default_settings_db_path",
+    "default_state_root",
+]