py-data-engine 0.1.0__py3-none-any.whl

data_engine/platform/workspace_policy.py

@@ -0,0 +1,333 @@
+"""Workspace state, discovery, and runtime-layout policy services."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from pathlib import Path
+
+from data_engine.platform.identity import (
+    APP_ARTIFACTS_DIR_NAME,
+    APP_RUNTIME_NAMESPACE,
+    RUNTIME_STATE_DIR_NAME,
+    WORKSPACE_CACHE_DIR_NAME,
+)
+from data_engine.platform.local_settings import LocalSettingsStore, default_settings_db_path, default_state_root
+from data_engine.platform.workspace_models import (
+    APP_ROOT_PATH,
+    DATA_ENGINE_APP_ROOT_ENV_VAR,
+    DATA_ENGINE_RUNTIME_ROOT_ENV_VAR,
+    DATA_ENGINE_WORKSPACE_COLLECTION_ROOT_ENV_VAR,
+    DATA_ENGINE_WORKSPACE_ID_ENV_VAR,
+    DATA_ENGINE_WORKSPACE_ROOT_ENV_VAR,
+    DiscoveredWorkspace,
+    WORKSPACE_AVAILABLE_MARKERS_DIR_NAME,
+    WORKSPACE_CONFIG_DIR_NAME,
+    WORKSPACE_CONTROL_REQUESTS_DIR_NAME,
+    WORKSPACE_DATABASES_DIR_NAME,
+    WORKSPACE_FLOW_MODULES_DIR_NAME,
+    WORKSPACE_LEASED_MARKERS_DIR_NAME,
+    WORKSPACE_LEASE_METADATA_DIR_NAME,
+    WORKSPACE_SHARED_FILE_STATE_DIR_NAME,
+    WORKSPACE_SHARED_LOGS_DIR_NAME,
+    WORKSPACE_SHARED_RUNS_DIR_NAME,
+    WORKSPACE_SHARED_STATE_DIR_NAME,
+    WORKSPACE_SHARED_STEP_RUNS_DIR_NAME,
+    WORKSPACE_STALE_MARKERS_DIR_NAME,
+    WORKSPACE_STATE_DIR_NAME,
+    WorkspacePaths,
+    WorkspaceSettings,
+    normalized_path_text,
+    validate_workspace_id,
+)
+
+
+class AppStatePolicy:
+    """Resolve machine-local app settings and mutable-state roots."""
+
+    def effective_app_root(self, *, app_root: Path | None = None) -> Path:
+        """Resolve the effective project/app root from an explicit argument or env."""
+        if app_root is not None:
+            return app_root.resolve()
+        env_value = os.environ.get(DATA_ENGINE_APP_ROOT_ENV_VAR)
+        if env_value and env_value.strip():
+            return Path(env_value).expanduser().resolve()
+        return APP_ROOT_PATH
+
+    def settings_path(self, *, app_root: Path | None = None) -> Path:
+        """Return the machine-local app settings database path."""
+        return default_settings_db_path(app_root=app_root)
+
+    def load_settings(self, *, app_root: Path | None = None) -> WorkspaceSettings:
+        """Load machine-local workspace settings or synthesize defaults."""
+        root = self.effective_app_root(app_root=app_root)
+        settings_path = self.settings_path(app_root=root)
+        state_root = default_state_root(app_root=root)
+        store = LocalSettingsStore(settings_path)
+        env_collection = os.environ.get(DATA_ENGINE_WORKSPACE_COLLECTION_ROOT_ENV_VAR)
+        env_workspace_root = os.environ.get(DATA_ENGINE_WORKSPACE_ROOT_ENV_VAR)
+        env_runtime_root = os.environ.get(DATA_ENGINE_RUNTIME_ROOT_ENV_VAR)
+        runtime_root = (
+            Path(env_runtime_root).expanduser().resolve()
+            if env_runtime_root and env_runtime_root.strip()
+            else store.runtime_root() or state_root / APP_ARTIFACTS_DIR_NAME
+        )
+        if env_collection and env_collection.strip():
+            collection_root = Path(env_collection).expanduser().resolve()
+            default_selected = os.environ.get(DATA_ENGINE_WORKSPACE_ID_ENV_VAR) or store.default_workspace_id()
+            return WorkspaceSettings(root, settings_path, state_root, runtime_root, collection_root, default_selected)
+        if env_workspace_root and env_workspace_root.strip():
+            explicit_root = Path(env_workspace_root).expanduser().resolve()
+            collection_root = explicit_root.parent
+            default_selected = os.environ.get(DATA_ENGINE_WORKSPACE_ID_ENV_VAR) or explicit_root.name or store.default_workspace_id()
+            return WorkspaceSettings(root, settings_path, state_root, runtime_root, collection_root, default_selected)
+        return WorkspaceSettings(
+            app_root=root,
+            settings_path=settings_path,
+            state_root=state_root,
+            runtime_root=runtime_root,
+            workspace_collection_root=store.workspace_collection_root(),
+            default_selected=store.default_workspace_id(),
+        )
+
+    def write_settings(self, settings: WorkspaceSettings) -> None:
+        """Persist machine-local settings through the SQLite settings store."""
+        store = LocalSettingsStore(settings.settings_path)
+        store.set_workspace_collection_root(settings.workspace_collection_root)
+        store.set_default_workspace_id(settings.default_selected)
+        store.set_runtime_root(settings.runtime_root)
+
+
+class WorkspaceDiscoveryPolicy:
+    """Resolve authored workspace discovery and selection."""
+
+    PLACEHOLDER_WORKSPACE_ROOT_NAME = ".workspace_unconfigured"
+    PLACEHOLDER_WORKSPACE_ID = "unconfigured"
+
+    def __init__(self, *, app_state_policy: AppStatePolicy | None = None) -> None:
+        self.app_state_policy = app_state_policy or AppStatePolicy()
+
+    def _normalize_workspace_id(self, candidate: str, *, fallback: str | None = None) -> str:
+        value = str(candidate).strip()
+        if value == self.PLACEHOLDER_WORKSPACE_ROOT_NAME:
+            value = self.PLACEHOLDER_WORKSPACE_ID
+        if not value and fallback is not None:
+            value = fallback
+        return validate_workspace_id(value)
+
+    def _placeholder_workspace(self, *, app_root: Path, preferred_id: str | None = None) -> tuple[DiscoveredWorkspace, Path, bool]:
+        placeholder_root = app_root / self.PLACEHOLDER_WORKSPACE_ROOT_NAME
+        placeholder_id = (
+            self._normalize_workspace_id(preferred_id, fallback=self.PLACEHOLDER_WORKSPACE_ID)
+            if preferred_id is not None
+            else self.PLACEHOLDER_WORKSPACE_ID
+        )
+        return (
+            DiscoveredWorkspace(workspace_id=placeholder_id, workspace_root=placeholder_root),
+            placeholder_root.parent,
+            False,
+        )
+
+    def discover(
+        self,
+        *,
+        app_root: Path | None = None,
+        workspace_collection_root: Path | None = None,
+        explicit_workspace_root: Path | None = None,
+    ) -> tuple[DiscoveredWorkspace, ...]:
+        """Discover valid workspaces beneath the collection root."""
+        if explicit_workspace_root is not None:
+            root = Path(explicit_workspace_root).expanduser().resolve()
+            workspace_id = os.environ.get(DATA_ENGINE_WORKSPACE_ID_ENV_VAR) or root.name or "default"
+            return (DiscoveredWorkspace(workspace_id=workspace_id, workspace_root=root),)
+
+        settings = self.app_state_policy.load_settings(app_root=app_root)
+        collection_root = (
+            Path(workspace_collection_root).expanduser().resolve()
+            if workspace_collection_root is not None
+            else settings.workspace_collection_root
+        )
+        if collection_root is None:
+            return ()
+        if not collection_root.exists():
+            return ()
+        if (collection_root / WORKSPACE_FLOW_MODULES_DIR_NAME).is_dir():
+            workspace_id = os.environ.get(DATA_ENGINE_WORKSPACE_ID_ENV_VAR) or "default"
+            return (DiscoveredWorkspace(workspace_id=workspace_id, workspace_root=collection_root.resolve()),)
+        discovered: list[DiscoveredWorkspace] = []
+        for candidate in sorted(path for path in collection_root.iterdir() if path.is_dir()):
+            if (candidate / WORKSPACE_FLOW_MODULES_DIR_NAME).is_dir():
+                discovered.append(DiscoveredWorkspace(workspace_id=candidate.name, workspace_root=candidate.resolve()))
+        return tuple(discovered)
+
+    def select_workspace(
+        self,
+        *,
+        app_root: Path,
+        workspace_id: str | None,
+        workspace_root: Path | None,
+        workspace_collection_root: Path | None,
+        data_root: Path | None,
+    ) -> tuple[DiscoveredWorkspace, Path, bool]:
+        """Select one authored workspace and collection root from the current policy."""
+        explicit_root = workspace_root if workspace_root is not None else data_root
+        env_workspace_root = os.environ.get(DATA_ENGINE_WORKSPACE_ROOT_ENV_VAR)
+        if explicit_root is not None:
+            discovered_root = Path(explicit_root).expanduser().resolve()
+            fallback_id = self.PLACEHOLDER_WORKSPACE_ID if discovered_root.name == self.PLACEHOLDER_WORKSPACE_ROOT_NAME else (discovered_root.name or "default")
+            selected_id = self._normalize_workspace_id(workspace_id) if workspace_id is not None else fallback_id
+            collection_root = (
+                Path(workspace_collection_root).expanduser().resolve()
+                if workspace_collection_root is not None
+                else discovered_root.parent
+            )
+            return DiscoveredWorkspace(workspace_id=selected_id, workspace_root=discovered_root), collection_root, True
+        if env_workspace_root and env_workspace_root.strip():
+            discovered_root = Path(env_workspace_root).expanduser().resolve()
+            env_workspace_id = os.environ.get(DATA_ENGINE_WORKSPACE_ID_ENV_VAR)
+            if workspace_id is not None:
+                selected_id = self._normalize_workspace_id(workspace_id)
+            elif env_workspace_id and env_workspace_id.strip():
+                selected_id = self._normalize_workspace_id(env_workspace_id)
+            else:
+                fallback_id = self.PLACEHOLDER_WORKSPACE_ID if discovered_root.name == self.PLACEHOLDER_WORKSPACE_ROOT_NAME else (discovered_root.name or "default")
+                selected_id = fallback_id
+            collection_root = (
+                Path(workspace_collection_root).expanduser().resolve()
+                if workspace_collection_root is not None
+                else discovered_root.parent
+            )
+            return DiscoveredWorkspace(workspace_id=selected_id, workspace_root=discovered_root), collection_root, True
+
+        settings = self.app_state_policy.load_settings(app_root=app_root)
+        collection_root = (
+            Path(workspace_collection_root).expanduser().resolve()
+            if workspace_collection_root is not None
+            else settings.workspace_collection_root
+        )
+        if collection_root is None:
+            return self._placeholder_workspace(
+                app_root=app_root,
+                preferred_id=workspace_id or settings.default_selected,
+            )
+        discovered_all = self.discover(app_root=app_root, workspace_collection_root=collection_root)
+        if not discovered_all:
+            if workspace_id is not None:
+                selected_id = self._normalize_workspace_id(workspace_id)
+            else:
+                env_workspace_id = os.environ.get(DATA_ENGINE_WORKSPACE_ID_ENV_VAR)
+                if env_workspace_id and env_workspace_id.strip():
+                    selected_id = self._normalize_workspace_id(env_workspace_id)
+                else:
+                    selected_id = (
+                        self._normalize_workspace_id(settings.default_selected, fallback=self.PLACEHOLDER_WORKSPACE_ID)
+                        if settings.default_selected is not None
+                        else self.PLACEHOLDER_WORKSPACE_ID
+                    )
+            discovered = DiscoveredWorkspace(workspace_id=selected_id, workspace_root=(collection_root / selected_id).resolve())
+            return discovered, collection_root, True
+
+        if workspace_id is not None:
+            requested_id = self._normalize_workspace_id(workspace_id)
+        else:
+            env_workspace_id = os.environ.get(DATA_ENGINE_WORKSPACE_ID_ENV_VAR)
+            requested_id = env_workspace_id if env_workspace_id and env_workspace_id.strip() else settings.default_selected
+        if requested_id is None:
+            return discovered_all[0], collection_root, True
+        requested_id = self._normalize_workspace_id(requested_id, fallback=self.PLACEHOLDER_WORKSPACE_ID)
+        by_id = {item.workspace_id: item for item in discovered_all}
+        return by_id.get(requested_id, discovered_all[0]), collection_root, True
+
+
+class RuntimeLayoutPolicy:
+    """Resolve local runtime and artifact layout for one workspace."""
+
+    def __init__(self, *, app_state_policy: AppStatePolicy | None = None, discovery_policy: WorkspaceDiscoveryPolicy | None = None) -> None:
+        self.app_state_policy = app_state_policy or AppStatePolicy()
+        self.discovery_policy = discovery_policy or WorkspaceDiscoveryPolicy(app_state_policy=self.app_state_policy)
+
+    def resolve_paths(
+        self,
+        *,
+        workspace_id: str | None = None,
+        workspace_root: Path | None = None,
+        workspace_collection_root: Path | None = None,
+        data_root: Path | None = None,
+        app_root: Path | None = None,
+    ) -> WorkspacePaths:
+        """Resolve authored, shared, and local-artifact paths for one selected workspace."""
+        resolved_app_root = self.app_state_policy.effective_app_root(app_root=app_root)
+        discovered, collection_root, workspace_configured = self.discovery_policy.select_workspace(
+            app_root=resolved_app_root,
+            workspace_id=workspace_id,
+            workspace_root=workspace_root,
+            workspace_collection_root=workspace_collection_root,
+            data_root=data_root,
+        )
+        settings = self.app_state_policy.load_settings(app_root=resolved_app_root)
+        workspace_state_dir = discovered.workspace_root / WORKSPACE_STATE_DIR_NAME
+        shared_state_dir = workspace_state_dir / WORKSPACE_SHARED_STATE_DIR_NAME
+        lease_metadata_dir = workspace_state_dir / WORKSPACE_LEASE_METADATA_DIR_NAME
+        control_requests_dir = workspace_state_dir / WORKSPACE_CONTROL_REQUESTS_DIR_NAME
+        local_namespace = self.local_workspace_namespace(discovered.workspace_root, discovered.workspace_id)
+        artifacts_dir = settings.runtime_root
+        workspace_cache_dir = artifacts_dir / WORKSPACE_CACHE_DIR_NAME / local_namespace
+        runtime_state_dir = artifacts_dir / RUNTIME_STATE_DIR_NAME / local_namespace
+        daemon_endpoint_kind, daemon_endpoint_path = self.daemon_endpoint(
+            runtime_state_dir=runtime_state_dir,
+            workspace_id=discovered.workspace_id,
+        )
+        return WorkspacePaths(
+            app_root=resolved_app_root,
+            workspace_collection_root=collection_root,
+            workspace_id=discovered.workspace_id,
+            workspace_root=discovered.workspace_root,
+            config_dir=discovered.workspace_root / WORKSPACE_CONFIG_DIR_NAME,
+            flow_modules_dir=discovered.workspace_root / WORKSPACE_FLOW_MODULES_DIR_NAME,
+            databases_dir=discovered.workspace_root / WORKSPACE_DATABASES_DIR_NAME,
+            workspace_state_dir=workspace_state_dir,
+            available_markers_dir=workspace_state_dir / WORKSPACE_AVAILABLE_MARKERS_DIR_NAME,
+            leased_markers_dir=workspace_state_dir / WORKSPACE_LEASED_MARKERS_DIR_NAME,
+            stale_markers_dir=workspace_state_dir / WORKSPACE_STALE_MARKERS_DIR_NAME,
+            lease_metadata_dir=lease_metadata_dir,
+            lease_metadata_path=lease_metadata_dir / f"{discovered.workspace_id}.parquet",
+            control_requests_dir=control_requests_dir,
+            control_request_path=control_requests_dir / f"{discovered.workspace_id}.parquet",
+            shared_state_dir=shared_state_dir,
+            shared_runs_path=shared_state_dir / WORKSPACE_SHARED_RUNS_DIR_NAME / f"{discovered.workspace_id}.parquet",
+            shared_step_runs_path=shared_state_dir / WORKSPACE_SHARED_STEP_RUNS_DIR_NAME / f"{discovered.workspace_id}.parquet",
+            shared_logs_path=shared_state_dir / WORKSPACE_SHARED_LOGS_DIR_NAME / f"{discovered.workspace_id}.parquet",
+            shared_file_state_path=shared_state_dir / WORKSPACE_SHARED_FILE_STATE_DIR_NAME / f"{discovered.workspace_id}.parquet",
+            artifacts_dir=artifacts_dir,
+            workspace_cache_dir=workspace_cache_dir,
+            compiled_flow_modules_dir=workspace_cache_dir / "compiled_flow_modules",
+            runtime_state_dir=runtime_state_dir,
+            runtime_db_path=runtime_state_dir / "runtime_ledger.sqlite",
+            daemon_log_path=runtime_state_dir / "daemon.log",
+            documentation_dir=artifacts_dir / "documentation",
+            daemon_endpoint_kind=daemon_endpoint_kind,
+            daemon_endpoint_path=daemon_endpoint_path,
+            sphinx_source_dir=resolved_app_root / "src" / "data_engine" / "docs" / "sphinx_source",
+            workspace_configured=workspace_configured,
+        )
+
+    @staticmethod
+    def local_workspace_namespace(workspace_root: Path | str, workspace_id: str) -> str:
+        """Return the machine-local namespace for one workspace root."""
+        return f"{validate_workspace_id(workspace_id)}_{hashlib.sha1(normalized_path_text(Path(workspace_root).expanduser().resolve()).encode('utf-8')).hexdigest()[:12]}"
+
+    @staticmethod
+    def daemon_endpoint(*, runtime_state_dir: Path, workspace_id: str) -> tuple[str, str]:
+        """Return the cross-platform local IPC endpoint."""
+        workspace_id = validate_workspace_id(workspace_id)
+        digest = hashlib.sha1(normalized_path_text(runtime_state_dir).encode("utf-8")).hexdigest()[:12]
+        if os.name == "nt":
+            return "pipe", rf"\\.\pipe\{APP_RUNTIME_NAMESPACE}_{workspace_id}_{digest}"
+        return "unix", normalized_path_text(Path("/tmp") / f"{APP_RUNTIME_NAMESPACE}_{workspace_id}_{digest}.sock")
+
+
+__all__ = [
+    "AppStatePolicy",
+    "RuntimeLayoutPolicy",
+    "WorkspaceDiscoveryPolicy",
+]

data_engine/runtime/__init__.py

@@ -0,0 +1 @@
+"""Runtime execution, file watching, persistence, and shared state."""

data_engine/runtime/file_watch.py

@@ -0,0 +1,185 @@
+"""Filesystem discovery and polling services used by the flow runtime."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Iterable, Protocol, runtime_checkable
+
+from data_engine.authoring.model import FlowValidationError
+from data_engine.platform.workspace_models import normalized_path_text
+
+
+def _normalize_path(value: Path) -> Path:
+    """Normalize a path for stable comparisons across filesystems."""
+    return Path(normalized_path_text(value))
+
+
+def _normalized_name(value: str) -> str:
+    """Normalize a filename for case-insensitive temporary-file checks."""
+    return normalized_path_text(value).casefold()
+
+
+def _queue_key(path: Path) -> str:
+    """Return a stable sort key for a filesystem path."""
+    return _normalize_path(path).as_posix().casefold()
+
+
+def _normalize_extensions(extensions: tuple[str, ...] | None) -> tuple[str, ...] | None:
+    """Normalize extension filters to lowercase dotted forms."""
+    if extensions is None:
+        return None
+    normalized: list[str] = []
+    for ext in extensions:
+        value = str(ext).strip().lower()
+        if not value:
+            raise FlowValidationError("Empty extension is not allowed.")
+        if not value.startswith("."):
+            value = f".{value}"
+        normalized.append(value)
+    return tuple(normalized)
+
+
+def is_temporary_file_path(path: Path) -> bool:
+    """Return whether a path looks like a transient temp or download file."""
+    name = _normalized_name(path.name)
+    if name.startswith(".~lock.") and name.endswith("#"):
+        return True
+    if name.startswith(".") or name.startswith("~$") or name.startswith("._"):
+        return True
+    if name.endswith("~"):
+        return True
+    if any(name.endswith(suffix) for suffix in (".tmp", ".temp", ".part", ".partial", ".crdownload", ".download", ".swp")):
+        return True
+    return False
+
+
+def iter_candidate_paths(
+    input_root: Path,
+    *,
+    extensions: tuple[str, ...] | None = None,
+    recursive: bool = True,
+    allow_missing: bool = False,
+) -> Iterable[Path]:
+    """Yield candidate files from one root using optional extension filters."""
+    if not input_root.exists():
+        if allow_missing:
+            return
+        raise FlowValidationError(f"Input root not found: {input_root}")
+
+    normalized_extensions = _normalize_extensions(extensions)
+    if input_root.is_file():
+        candidates: Iterable[Path] = (input_root,)
+    else:
+        globber = input_root.rglob("*") if recursive else input_root.glob("*")
+        candidates = sorted(globber, key=_queue_key)
+
+    for path in candidates:
+        if not path.is_file():
+            continue
+        if normalized_extensions is not None and path.suffix.lower() not in normalized_extensions:
+            continue
+        if is_temporary_file_path(path):
+            continue
+        yield path
+
+
+@runtime_checkable
+class IFileWatcher(Protocol):
+    """Common interface for runtime file watchers."""
+
+    def start(self) -> None:
+        """Begin watching for filesystem changes."""
+
+    def stop(self) -> None:
+        """Stop watching for filesystem changes."""
+
+    def drain_events(self) -> list[Path]:
+        """Return any queued filesystem events observed since the last drain."""
+
+
+class PollingWatcher:
+    """Filesystem polling watcher that reacts to changes observed after startup."""
+
+    def __init__(
+        self,
+        input_root: Path,
+        *,
+        recursive: bool = True,
+        extensions: tuple[str, ...] | None = None,
+        settle: int = 1,
+    ) -> None:
+        """Initialize a polling watcher for one file or directory root."""
+        if settle < 0:
+            raise FlowValidationError("settle must be zero or greater.")
+        self.input_root = input_root
+        self.recursive = recursive
+        self.extensions = _normalize_extensions(extensions)
+        self.settle = settle
+        self._seen: dict[Path, tuple[int, int, int]] = {}
+        self._stable_counts: dict[Path, int] = {}
+        self._emitted: dict[Path, tuple[int, int, int]] = {}
+        self._running = False
+
+    def start(self) -> None:
+        """Capture an initial filesystem snapshot and begin watching."""
+        self._seen = self._snapshot()
+        self._stable_counts = {path: 0 for path in self._seen}
+        self._emitted = dict(self._seen)
+        self._running = True
+
+    def stop(self) -> None:
+        """Stop watching for new filesystem events."""
+        self._running = False
+
+    def drain_events(self) -> list[Path]:
+        """Return newly stable files observed since the last poll."""
+        if not self._running:
+            return []
+
+        current = self._snapshot()
+        events: list[Path] = []
+        stable_counts: dict[Path, int] = {}
+
+        for path, signature in current.items():
+            prior_signature = self._seen.get(path)
+            if prior_signature == signature:
+                stable_counts[path] = self._stable_counts.get(path, 0) + 1
+            else:
+                stable_counts[path] = 0
+
+            if self._emitted.get(path) == signature:
+                continue
+            if stable_counts[path] < self.settle:
+                continue
+            events.append(path)
+            self._emitted[path] = signature
+
+        self._stable_counts = stable_counts
+        self._seen = current
+        return events
+
+    def _snapshot(self) -> dict[Path, tuple[int, int, int]]:
+        """Capture the current file signatures for all candidate paths."""
+        result: dict[Path, tuple[int, int, int]] = {}
+        for path in iter_candidate_paths(self.input_root, extensions=self.extensions, recursive=self.recursive, allow_missing=True):
+            try:
+                stat = path.stat()
+                result[path] = (
+                    stat.st_mtime_ns,
+                    stat.st_size,
+                    getattr(stat, "st_ctime_ns", 0),
+                )
+            except FileNotFoundError:
+                continue
+
+        self._prune_removed_paths(result)
+        return result
+
+    def _prune_removed_paths(self, current: dict[Path, tuple[int, int, int]]) -> None:
+        """Drop removed paths from watcher state maps."""
+        current_paths = set(current)
+        self._stable_counts = {path: count for path, count in self._stable_counts.items() if path in current_paths}
+        self._emitted = {path: sig for path, sig in self._emitted.items() if path in current_paths}
+
+
+__all__ = ["IFileWatcher", "PollingWatcher", "iter_candidate_paths", "is_temporary_file_path"]

data_engine/runtime/ledger_models.py

@@ -0,0 +1,116 @@
+"""Persisted runtime ledger record models."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from data_engine.domain.time import parse_utc_text
+
+
+def elapsed_seconds(started_at_utc: str, finished_at_utc: str | None) -> float | None:
+    """Return elapsed seconds derived from persisted UTC text timestamps."""
+    started = parse_utc_text(started_at_utc)
+    finished = parse_utc_text(finished_at_utc)
+    if started is None or finished is None:
+        return None
+    return max((finished - started).total_seconds(), 0.0)
+
+
+@dataclass(frozen=True)
+class PersistedRun:
+    """One persisted runtime run summary."""
+
+    run_id: str
+    flow_name: str
+    group_name: str
+    source_path: str | None
+    status: str
+    started_at_utc: str
+    finished_at_utc: str | None
+    error_text: str | None
+
+    @property
+    def elapsed_seconds(self) -> float | None:
+        return elapsed_seconds(self.started_at_utc, self.finished_at_utc)
+
+
+@dataclass(frozen=True)
+class PersistedStepRun:
+    """One persisted runtime step execution."""
+
+    id: int
+    run_id: str
+    flow_name: str
+    step_label: str
+    status: str
+    started_at_utc: str
+    finished_at_utc: str | None
+    elapsed_ms: int | None
+    error_text: str | None
+    output_path: str | None
+
+
+@dataclass(frozen=True)
+class PersistedLogEntry:
+    """One persisted runtime log line."""
+
+    id: int
+    run_id: str | None
+    flow_name: str | None
+    step_label: str | None
+    level: str
+    message: str
+    created_at_utc: str
+
+
+@dataclass(frozen=True)
+class PersistedFileState:
+    """One persisted current file freshness row."""
+
+    flow_name: str
+    source_path: str
+    mtime_ns: int
+    size_bytes: int
+    last_success_run_id: str | None
+    last_success_at_utc: str | None
+    last_status: str
+    last_error_text: str | None
+
+
+@dataclass(frozen=True)
+class PersistedDaemonState:
+    """One persisted daemon metadata row."""
+
+    workspace_id: str
+    pid: int
+    endpoint_kind: str
+    endpoint_path: str
+    started_at_utc: str
+    last_checkpoint_at_utc: str
+    status: str
+    app_root: str
+    workspace_root: str
+    version_text: str | None
+
+
+@dataclass(frozen=True)
+class PersistedClientSession:
+    """One persisted local client session row."""
+
+    client_id: str
+    workspace_id: str
+    client_kind: str
+    pid: int
+    started_at_utc: str
+    updated_at_utc: str
+
+
+__all__ = [
+    "PersistedClientSession",
+    "PersistedDaemonState",
+    "PersistedFileState",
+    "PersistedLogEntry",
+    "PersistedRun",
+    "PersistedStepRun",
+    "elapsed_seconds",
+]