py-data-engine 0.1.0__py3-none-any.whl

data_engine/services/flow_catalog.py

@@ -0,0 +1,102 @@
+"""Flow catalog loading services."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+
+from data_engine.authoring.builder import Flow, _title_case_words
+from data_engine.domain import FlowCatalogEntry, default_flow_state, flow_category
+from data_engine.authoring.model import FlowValidationError
+from data_engine.flow_modules.flow_module_loader import FlowModuleDefinition, discover_flow_module_definitions
+from data_engine.platform.workspace_models import path_display
+
+
+def _flow_paths(flow: Flow) -> tuple[str, str]:
+    trigger = flow.trigger
+    source = getattr(trigger, "source", None) if trigger is not None else None
+    target = getattr(flow.mirror_spec, "root", None)
+    return path_display(source), path_display(target)
+
+
+def _flow_interval(flow: Flow) -> str:
+    trigger = flow.trigger
+    if trigger is None:
+        return "-"
+    if getattr(trigger, "interval", None) is not None:
+        return str(trigger.interval)
+    times = getattr(trigger, "times", ())
+    if times:
+        return ", ".join(str(value) for value in times)
+    if getattr(trigger, "time", None) is not None:
+        return str(trigger.time)
+    return "-"
+
+
+def flow_catalog_entry_from_flow(flow: Flow, *, description: str | None) -> FlowCatalogEntry:
+    source_root, target_root = _flow_paths(flow)
+    operation_items = tuple(step.label for step in flow.steps)
+    operations = " -> ".join(operation_items) or "(no steps)"
+    mode = flow.mode
+    derived_title = flow.label or _title_case_words(flow.name or "", empty="Flow")
+    return FlowCatalogEntry(
+        name=flow.name,
+        group=flow.group,
+        title=derived_title,
+        description=description or "",
+        source_root=source_root,
+        target_root=target_root,
+        mode=mode,
+        interval=_flow_interval(flow),
+        operations=operations,
+        operation_items=operation_items,
+        state=default_flow_state(mode),
+        valid=True,
+        category=flow_category(mode),
+    )
+
+
+def _invalid_entry(definition: FlowModuleDefinition, exc: Exception) -> FlowCatalogEntry:
+    return FlowCatalogEntry(
+        name=definition.name,
+        group=None,
+        title=_title_case_words(definition.name, empty="Flow"),
+        description=definition.description or "",
+        source_root="(not set)",
+        target_root="(not set)",
+        mode="manual",
+        interval="-",
+        operations="Unavailable",
+        operation_items=(),
+        state="invalid",
+        valid=False,
+        category="manual",
+        error=str(exc),
+    )
+
+
+class FlowCatalogService:
+    """Own flow catalog loading through an explicit discovery dependency."""
+
+    def __init__(
+        self,
+        *,
+        discover_definitions_func: Callable[..., tuple[FlowModuleDefinition, ...]] = discover_flow_module_definitions,
+    ) -> None:
+        self._discover_definitions = discover_definitions_func
+
+    def load_entries(self, *, workspace_root: Path | None = None) -> tuple[FlowCatalogEntry, ...]:
+        """Return discovered flow catalog entries for the requested workspace root."""
+        entries: list[FlowCatalogEntry] = []
+        definitions = self._discover_definitions(data_root=workspace_root)
+        if not definitions:
+            raise FlowValidationError("No flow modules discovered.")
+        for definition in definitions:
+            try:
+                entries.append(flow_catalog_entry_from_flow(definition.build(), description=definition.description))
+            except Exception as exc:
+                entries.append(_invalid_entry(definition, exc))
+        return tuple(sorted(entries, key=lambda entry: entry.name))
+
+
+__all__ = ["FlowCatalogService", "flow_catalog_entry_from_flow"]

data_engine/services/flow_execution.py

@@ -0,0 +1,48 @@
+"""Executable flow loading services."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from data_engine.flow_modules.flow_module_loader import discover_flow_module_definitions, load_flow_module_definition
+
+if TYPE_CHECKING:
+    from data_engine.authoring.flow import Flow
+
+
+def _default_load_flow(name: str, *, data_root: Path | None = None) -> "Flow":
+    return load_flow_module_definition(name, data_root=data_root).build()
+
+
+def _default_discover_flows(*, data_root: Path | None = None) -> tuple["Flow", ...]:
+    return tuple(definition.build() for definition in discover_flow_module_definitions(data_root=data_root))
+
+
+class FlowExecutionService:
+    """Own executable flow loading through an explicit loader dependency."""
+
+    def __init__(
+        self,
+        *,
+        load_flow_func: Callable[..., "Flow"] = _default_load_flow,
+        discover_flows_func: Callable[..., tuple["Flow", ...]] = _default_discover_flows,
+    ) -> None:
+        self._load_flow = load_flow_func
+        self._discover_flows = discover_flows_func
+
+    def load_flow(self, name: str, *, workspace_root: Path | None = None) -> "Flow":
+        """Return one executable flow definition by name."""
+        return self._load_flow(name, data_root=workspace_root)
+
+    def load_flows(self, names: tuple[str, ...], *, workspace_root: Path | None = None) -> tuple["Flow", ...]:
+        """Return executable flow definitions for the requested names."""
+        return tuple(self.load_flow(name, workspace_root=workspace_root) for name in names)
+
+    def discover_flows(self, *, workspace_root: Path | None = None) -> tuple["Flow", ...]:
+        """Return all executable flow definitions for the requested workspace root."""
+        return self._discover_flows(data_root=workspace_root)
+
+
+__all__ = ["FlowExecutionService"]

data_engine/services/ledger.py

@@ -0,0 +1,85 @@
+"""Runtime ledger services."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Callable
+
+from data_engine.platform.workspace_policy import RuntimeLayoutPolicy
+from data_engine.runtime.runtime_db import RuntimeLedger
+
+
+class LedgerService:
+    """Own workspace-local runtime ledger access and client session bookkeeping."""
+
+    def __init__(
+        self,
+        open_ledger_func: Callable[[Path], RuntimeLedger] | None = None,
+        *,
+        runtime_layout_policy: RuntimeLayoutPolicy | None = None,
+    ) -> None:
+        self.runtime_layout_policy = runtime_layout_policy or RuntimeLayoutPolicy()
+        self._open_ledger_func = open_ledger_func or self._open_default_ledger
+
+    def _open_default_ledger(self, workspace_root: Path) -> RuntimeLedger:
+        paths = self.runtime_layout_policy.resolve_paths(data_root=workspace_root)
+        return RuntimeLedger(paths.runtime_db_path)
+
+    def open_for_workspace(self, workspace_root: Path) -> RuntimeLedger:
+        """Open the configured runtime ledger for one workspace root."""
+        return self._open_ledger_func(Path(workspace_root).expanduser().resolve())
+
+    def close(self, ledger: RuntimeLedger) -> None:
+        """Close one runtime ledger connection."""
+        ledger.close()
+
+    def register_client_session(
+        self,
+        ledger: RuntimeLedger,
+        *,
+        client_id: str,
+        workspace_id: str,
+        client_kind: str,
+        pid: int,
+    ) -> None:
+        """Register or refresh one active local client session."""
+        ledger.upsert_client_session(
+            client_id=client_id,
+            workspace_id=workspace_id,
+            client_kind=client_kind,
+            pid=pid,
+        )
+
+    def remove_client_session(self, ledger: RuntimeLedger, client_id: str) -> None:
+        """Remove one active local client session row."""
+        ledger.remove_client_session(client_id)
+
+    def purge_process_client_sessions(
+        self,
+        ledger: RuntimeLedger,
+        *,
+        workspace_id: str,
+        client_kind: str,
+        pid: int,
+    ) -> None:
+        """Remove all client sessions for one workspace/client-kind/process tuple."""
+        ledger.remove_client_sessions_for_process(
+            workspace_id=workspace_id,
+            client_kind=client_kind,
+            pid=pid,
+        )
+
+    def count_live_client_sessions(
+        self,
+        ledger: RuntimeLedger,
+        workspace_id: str,
+        *,
+        exclude_client_id: str | None = None,
+    ) -> int:
+        """Return the number of currently live client sessions for one workspace."""
+        if exclude_client_id is None:
+            return ledger.count_live_client_sessions(workspace_id)
+        return ledger.count_live_client_sessions(workspace_id, exclude_client_id=exclude_client_id)
+
+
+__all__ = ["LedgerService"]

data_engine/services/logs.py

@@ -0,0 +1,65 @@
+"""Operator log history services."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from data_engine.domain import FlowLogEntry, FlowRunState
+from data_engine.domain.logs import parse_runtime_message
+from data_engine.runtime.runtime_db import RuntimeLedger
+from data_engine.views.logs import FlowLogStore
+
+
+class LogService:
+    """Own operator log-store construction and log history queries."""
+
+    def create_store(self, runtime_ledger: RuntimeLedger | None = None) -> FlowLogStore:
+        """Create one log store backed by the given runtime ledger."""
+        store = FlowLogStore(self._hydrate_entries(runtime_ledger))
+        store._runtime_ledger = runtime_ledger
+        return store
+
+    def reload(self, store: FlowLogStore) -> None:
+        """Reload one log store from its attached runtime ledger."""
+        entries = self._hydrate_entries(getattr(store, "_runtime_ledger", None))
+        store.clear()
+        for entry in entries:
+            store.append_entry(entry)
+
+    def append_entry(self, store: FlowLogStore, entry: FlowLogEntry) -> None:
+        """Append one log entry to the current store."""
+        store.append_entry(entry)
+
+    def clear_flow(self, store: FlowLogStore, flow_name: str | None) -> None:
+        """Clear one flow's visible log history from the current store."""
+        store.clear_flow(flow_name)
+
+    def all_entries(self, store: FlowLogStore) -> tuple[FlowLogEntry, ...]:
+        """Return every entry currently held in the store."""
+        return tuple(store._entries)
+
+    def entries_for_flow(self, store: FlowLogStore, flow_name: str | None) -> tuple[FlowLogEntry, ...]:
+        """Return flow-scoped entries for one selected flow."""
+        return store.entries_for_flow(flow_name)
+
+    def runs_for_flow(self, store: FlowLogStore, flow_name: str | None) -> tuple[FlowRunState, ...]:
+        """Return grouped run history for one selected flow."""
+        return store.runs_for_flow(flow_name)
+
+    def _hydrate_entries(self, runtime_ledger: RuntimeLedger | None) -> tuple[FlowLogEntry, ...]:
+        """Build in-memory flow log entries from one runtime ledger."""
+        if runtime_ledger is None:
+            return ()
+        return tuple(
+            FlowLogEntry(
+                line=FlowLogEntry.format_runtime_message(entry.message),
+                kind="flow" if entry.flow_name is not None else "system",
+                event=parse_runtime_message(entry.message),
+                flow_name=entry.flow_name,
+                created_at_utc=datetime.fromisoformat(entry.created_at_utc),
+            )
+            for entry in runtime_ledger.list_logs()
+        )
+
+
+__all__ = ["LogService"]

data_engine/services/runtime_binding.py

@@ -0,0 +1,105 @@
+"""Workspace runtime binding services for operator surfaces."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from os import getpid
+
+from data_engine.hosts.daemon.manager import WorkspaceDaemonManager
+from data_engine.platform.workspace_models import WorkspacePaths
+from data_engine.runtime.runtime_db import RuntimeLedger
+from data_engine.services.daemon_state import DaemonStateService
+from data_engine.services.ledger import LedgerService
+from data_engine.services.logs import LogService
+from data_engine.views.logs import FlowLogStore
+
+
+@dataclass(frozen=True)
+class WorkspaceRuntimeBinding:
+    """Concrete runtime resources bound to one selected workspace."""
+
+    workspace_paths: WorkspacePaths
+    runtime_ledger: RuntimeLedger
+    log_store: FlowLogStore
+    daemon_manager: WorkspaceDaemonManager
+
+
+class WorkspaceRuntimeBindingService:
+    """Own concrete runtime binding lifecycle for GUI/TUI surfaces."""
+
+    def __init__(
+        self,
+        *,
+        ledger_service: LedgerService,
+        log_service: LogService,
+        daemon_state_service: DaemonStateService,
+    ) -> None:
+        self.ledger_service = ledger_service
+        self.log_service = log_service
+        self.daemon_state_service = daemon_state_service
+
+    def open_binding(self, workspace_paths: WorkspacePaths) -> WorkspaceRuntimeBinding:
+        """Open one concrete runtime binding for a workspace selection."""
+        runtime_ledger = self.ledger_service.open_for_workspace(workspace_paths.workspace_root)
+        return WorkspaceRuntimeBinding(
+            workspace_paths=workspace_paths,
+            runtime_ledger=runtime_ledger,
+            log_store=self.log_service.create_store(runtime_ledger),
+            daemon_manager=self.daemon_state_service.create_manager(workspace_paths),
+        )
+
+    def close_binding(self, binding: WorkspaceRuntimeBinding) -> None:
+        """Close one concrete runtime binding."""
+        self.ledger_service.close(binding.runtime_ledger)
+
+    def register_client_session(
+        self,
+        binding: WorkspaceRuntimeBinding,
+        *,
+        client_id: str,
+        client_kind: str,
+        pid: int | None = None,
+    ) -> None:
+        """Register or refresh one local client session for the binding workspace."""
+        self.ledger_service.register_client_session(
+            binding.runtime_ledger,
+            client_id=client_id,
+            workspace_id=binding.workspace_paths.workspace_id,
+            client_kind=client_kind,
+            pid=getpid() if pid is None else pid,
+        )
+
+    def remove_client_session(self, binding: WorkspaceRuntimeBinding, client_id: str) -> None:
+        """Remove one active local client session row."""
+        self.ledger_service.remove_client_session(binding.runtime_ledger, client_id)
+
+    def purge_process_client_sessions(
+        self,
+        binding: WorkspaceRuntimeBinding,
+        *,
+        client_kind: str,
+        pid: int | None = None,
+    ) -> None:
+        """Remove all client sessions for this workspace/client-kind/process tuple."""
+        self.ledger_service.purge_process_client_sessions(
+            binding.runtime_ledger,
+            workspace_id=binding.workspace_paths.workspace_id,
+            client_kind=client_kind,
+            pid=getpid() if pid is None else pid,
+        )
+
+    def count_live_client_sessions(
+        self,
+        binding: WorkspaceRuntimeBinding,
+        *,
+        exclude_client_id: str | None = None,
+    ) -> int:
+        """Return the number of live local client sessions for the binding workspace."""
+        return self.ledger_service.count_live_client_sessions(
+            binding.runtime_ledger,
+            binding.workspace_paths.workspace_id,
+            exclude_client_id=exclude_client_id,
+        )
+
+
+__all__ = ["WorkspaceRuntimeBinding", "WorkspaceRuntimeBindingService"]

data_engine/services/runtime_execution.py

@@ -0,0 +1,126 @@
+"""Runtime execution services for flow runs and grouped engine runs."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from threading import Event
+from typing import TYPE_CHECKING
+
+from data_engine.authoring.execution import _FlowRuntime, _GroupedFlowRuntime
+from data_engine.runtime.runtime_db import RuntimeLedger
+
+if TYPE_CHECKING:
+    from data_engine.authoring.flow import Flow
+
+
+class RuntimeExecutionService:
+    """Own executable runtime construction for manual and grouped runs."""
+
+    def __init__(
+        self,
+        *,
+        flow_runtime_type: type[_FlowRuntime] = _FlowRuntime,
+        grouped_runtime_type: type[_GroupedFlowRuntime] = _GroupedFlowRuntime,
+    ) -> None:
+        self._flow_runtime_type = flow_runtime_type
+        self._grouped_runtime_type = grouped_runtime_type
+
+    def run_once(
+        self,
+        flow: "Flow",
+        *,
+        runtime_ledger: RuntimeLedger | None = None,
+        flow_stop_event: Event | None = None,
+    ) -> object:
+        """Run one flow as a one-shot execution."""
+        runtime = self._flow_runtime_type(
+            (flow,),
+            continuous=False,
+            flow_stop_event=flow_stop_event,
+            runtime_ledger=runtime_ledger,
+        )
+        return runtime.run()
+
+    def preview(
+        self,
+        flow: "Flow",
+        *,
+        use: str | None = None,
+        runtime_ledger: RuntimeLedger | None = None,
+    ) -> object:
+        """Preview one flow through the one-shot runtime path."""
+        runtime = self._flow_runtime_type(
+            (flow,),
+            continuous=False,
+            runtime_ledger=runtime_ledger,
+        )
+        return runtime.preview(use=use)
+
+    def run_manual(
+        self,
+        flow: "Flow",
+        *,
+        runtime_ledger: RuntimeLedger,
+        flow_stop_event: Event,
+    ) -> object:
+        """Run one flow as a manual one-shot execution."""
+        return self.run_once(
+            flow,
+            runtime_ledger=runtime_ledger,
+            flow_stop_event=flow_stop_event,
+        )
+
+    def run_continuous(
+        self,
+        flow: "Flow",
+        *,
+        runtime_ledger: RuntimeLedger | None = None,
+        flow_stop_event: Event | None = None,
+    ) -> object:
+        """Run one flow continuously."""
+        runtime = self._flow_runtime_type(
+            (flow,),
+            continuous=True,
+            flow_stop_event=flow_stop_event,
+            runtime_ledger=runtime_ledger,
+        )
+        return runtime.run()
+
+    def run_grouped(
+        self,
+        flows: tuple["Flow", ...],
+        *,
+        runtime_ledger: RuntimeLedger,
+        runtime_stop_event: Event,
+        flow_stop_event: Event,
+    ) -> object:
+        """Run grouped automated flows continuously."""
+        runtime = self._grouped_runtime_type(
+            flows,
+            continuous=True,
+            runtime_stop_event=runtime_stop_event,
+            flow_stop_event=flow_stop_event,
+            runtime_ledger=runtime_ledger,
+        )
+        return runtime.run()
+
+    def run_grouped_continuous(
+        self,
+        flows: tuple["Flow", ...],
+        *,
+        runtime_ledger: RuntimeLedger | None = None,
+        runtime_stop_event: Event | None = None,
+        flow_stop_event: Event | None = None,
+    ) -> object:
+        """Run grouped automated flows continuously with optional runtime controls."""
+        runtime = self._grouped_runtime_type(
+            flows,
+            continuous=True,
+            runtime_stop_event=runtime_stop_event,
+            flow_stop_event=flow_stop_event,
+            runtime_ledger=runtime_ledger,
+        )
+        return runtime.run()
+
+
+__all__ = ["RuntimeExecutionService"]

data_engine/services/runtime_history.py

@@ -0,0 +1,62 @@
+"""Runtime history query services."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from data_engine.domain import FlowCatalogLike, FlowLogEntry, FlowRunState, StepOutputIndex
+from data_engine.runtime.runtime_db import RuntimeLedger
+
+
+class RuntimeHistoryService:
+    """Own persisted run/step history queries used by operator surfaces."""
+
+    def rebuild_step_outputs(
+        self,
+        ledger: RuntimeLedger,
+        flow_cards: dict[str, FlowCatalogLike],
+    ) -> StepOutputIndex:
+        """Rebuild latest successful per-step output paths for visible flows."""
+        rebuilt: dict[str, dict[str, Path]] = {}
+        for flow_name, card in flow_cards.items():
+            outputs: dict[str, Path] = {}
+            for run in ledger.list_runs(flow_name=flow_name):
+                for step_run in ledger.list_step_runs(run.run_id):
+                    if step_run.status != "success" or not step_run.output_path:
+                        continue
+                    if step_run.step_label not in card.operation_items or step_run.step_label in outputs:
+                        continue
+                    output_path = Path(str(step_run.output_path))
+                    if output_path.exists():
+                        outputs[step_run.step_label] = output_path
+                if len(outputs) == len(card.operation_items):
+                    break
+            rebuilt[flow_name] = outputs
+        return StepOutputIndex.from_mapping(rebuilt)
+
+    def error_text_for_entry(
+        self,
+        ledger: RuntimeLedger,
+        run_group: FlowRunState,
+        entry: FlowLogEntry,
+    ) -> tuple[str, str | None]:
+        """Return one user-facing error title and persisted error text for a failed entry."""
+        run_id = run_group.key[1]
+        event = entry.event
+        detail_text: str | None = None
+        title = "Run Error"
+        if event is not None and event.step_name is not None:
+            for step_run in ledger.list_step_runs(run_id):
+                if step_run.step_label == event.step_name and step_run.status == "failed":
+                    detail_text = step_run.error_text
+                    title = f"{event.step_name} Error"
+                    break
+        if detail_text is None:
+            for run in ledger.list_runs(flow_name=run_group.key[0]):
+                if run.run_id == run_id:
+                    detail_text = run.error_text
+                    break
+        return title, detail_text
+
+
+__all__ = ["RuntimeHistoryService"]

data_engine/services/settings.py

@@ -0,0 +1,58 @@
+"""Machine-local settings services."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Callable
+
+from data_engine.platform.local_settings import LocalSettingsStore, default_settings_db_path
+
+
+class SettingsService:
+    """Own machine-local settings persistence for operator surfaces."""
+
+    def __init__(self, store: LocalSettingsStore) -> None:
+        self._store = store
+
+    @classmethod
+    def default_store(cls, *, app_root: Path | None = None) -> LocalSettingsStore:
+        """Open the default local settings store for one app root."""
+        return LocalSettingsStore(default_settings_db_path(app_root=app_root))
+
+    @classmethod
+    def open_default(
+        cls,
+        *,
+        app_root: Path | None = None,
+        store_factory: Callable[[Path | None], LocalSettingsStore] | None = None,
+    ) -> "SettingsService":
+        """Open the default local settings store for the current app root."""
+        store_factory = store_factory or (lambda root: cls.default_store(app_root=root))
+        return cls(store_factory(app_root))
+
+    def workspace_collection_root(self) -> Path | None:
+        """Return the saved local workspace collection root override, when present."""
+        return self._store.workspace_collection_root()
+
+    def set_workspace_collection_root(self, value: Path | str | None) -> None:
+        """Persist the local workspace collection root override."""
+        self._store.set_workspace_collection_root(value)
+
+    def default_workspace_id(self) -> str | None:
+        """Return the saved default workspace id, when present."""
+        return self._store.default_workspace_id()
+
+    def set_default_workspace_id(self, value: str | None) -> None:
+        """Persist the default workspace id."""
+        self._store.set_default_workspace_id(value)
+
+    def runtime_root(self) -> Path | None:
+        """Return the saved runtime/artifact root override, when present."""
+        return self._store.runtime_root()
+
+    def set_runtime_root(self, value: Path | str | None) -> None:
+        """Persist the runtime/artifact root override."""
+        self._store.set_runtime_root(value)
+
+
+__all__ = ["SettingsService"]

data_engine/services/shared_state.py

@@ -0,0 +1,28 @@
+"""Shared workspace snapshot and hydration services."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from data_engine.platform.workspace_models import WorkspacePaths
+from data_engine.runtime.runtime_db import RuntimeLedger
+from data_engine.runtime.shared_state import hydrate_local_runtime_state, lease_is_stale, read_lease_metadata
+
+
+class SharedStateService:
+    """Own lease-based shared snapshot hydration for operator surfaces."""
+
+    def hydrate_local_runtime(self, paths: WorkspacePaths, ledger: RuntimeLedger) -> None:
+        """Replace one local runtime ledger from the shared workspace snapshots."""
+        hydrate_local_runtime_state(paths, ledger)
+
+    def read_lease_metadata(self, paths: WorkspacePaths) -> dict[str, Any] | None:
+        """Return current workspace lease metadata, if present."""
+        return read_lease_metadata(paths)
+
+    def lease_is_stale(self, paths: WorkspacePaths, *, stale_after_seconds: float) -> bool:
+        """Return whether current workspace lease metadata is stale."""
+        return lease_is_stale(paths, stale_after_seconds=stale_after_seconds)
+
+
+__all__ = ["SharedStateService"]