py-data-engine 0.1.0__py3-none-any.whl

data_engine/authoring/execution/continuous.py

@@ -0,0 +1,176 @@
+"""Continuous watch/schedule loop for one sequential flow runtime."""
+
+from __future__ import annotations
+
+from collections import deque
+from datetime import datetime
+from time import monotonic, sleep
+from typing import TYPE_CHECKING
+
+from data_engine.authoring.execution.context import _QueuedJob
+from data_engine.authoring.primitives import WatchSpec
+from data_engine.runtime.file_watch import PollingWatcher
+
+if TYPE_CHECKING:
+    from data_engine.authoring.execution.single import _FlowRuntime
+    from data_engine.authoring.primitives import FlowContext
+
+
+class ContinuousRuntimeLoop:
+    """Own the watch/schedule loop for one sequential runtime."""
+
+    def __init__(self, runtime: "_FlowRuntime") -> None:
+        self.runtime = runtime
+
+    def run(self) -> list["FlowContext"]:
+        results: list["FlowContext"] = []
+        queue: deque[_QueuedJob] = deque()
+        queued_keys: set[tuple[str, str | None]] = set()
+        watch_entries: list[dict[str, object]] = []
+        schedule_entries: list[dict[str, object]] = []
+        now = monotonic()
+
+        for flow in self.runtime.flows:
+            if flow.trigger is None:
+                for source_path in self.runtime.polling.startup_sources(flow):
+                    self.runtime.polling.enqueue_job(queue, queued_keys, flow, source_path)
+                continue
+            if isinstance(flow.trigger, WatchSpec) and flow.trigger.mode == "poll":
+                watcher = self.runtime.polling.make_watcher(flow.trigger)
+                watcher.start()
+                for source_path in self.runtime.polling.stale_poll_sources(flow):
+                    batch_signatures = self.runtime.polling.stale_batch_poll_signatures(flow) if source_path is None else ()
+                    self.runtime.polling.enqueue_job(queue, queued_keys, flow, source_path, batch_signatures=batch_signatures)
+                watch_entries.append(
+                    {
+                        "flow": flow,
+                        "interval": flow.trigger.interval_seconds,
+                        "next_poll": now + float(flow.trigger.interval_seconds),
+                        "watcher": watcher,
+                    }
+                )
+            elif isinstance(flow.trigger, WatchSpec) and flow.trigger.mode == "schedule":
+                if flow.trigger.interval_seconds is not None:
+                    schedule_entries.append(
+                        {
+                            "flow": flow,
+                            "kind": "every",
+                            "interval": flow.trigger.interval_seconds,
+                            "next_due": now + float(flow.trigger.interval_seconds),
+                            "pending": False,
+                        }
+                    )
+                else:
+                    for hour, minute in flow.trigger.time_slots:
+                        schedule_entries.append(
+                            {
+                                "flow": flow,
+                                "kind": "at",
+                                "hour": hour,
+                                "minute": minute,
+                                "pending": False,
+                                "last_run_date": None,
+                            }
+                        )
+
+        self.runtime._emit_status("Watcher/scheduler running.")
+        try:
+            while True:
+                if self.runtime.runtime_stop_event is not None and self.runtime.runtime_stop_event.is_set():
+                    self.runtime._emit_status("Watcher/scheduler stopped.")
+                    break
+                now = monotonic()
+                self._poll_watch_entries(watch_entries, queue, queued_keys, now)
+                self._update_schedule_entries(schedule_entries, now)
+
+                job = queue.popleft() if queue else None
+                if job is not None:
+                    queued_keys.discard(self.runtime.polling.job_key(job.flow, job.source_path))
+                    try:
+                        results.append(
+                            self.runtime.run_executor.run_one(
+                                job.flow,
+                                job.source_path,
+                                batch_signatures=job.batch_signatures,
+                            )
+                        )
+                    except FlowStoppedError:
+                        if self.runtime.flow_stop_event is not None:
+                            self.runtime.flow_stop_event.clear()
+                    except Exception:
+                        continue
+                    continue
+
+                scheduled = next((entry for entry in schedule_entries if entry["pending"]), None)
+                if scheduled is not None:
+                    scheduled["pending"] = False
+                    scheduled_flow = scheduled["flow"]
+                    try:
+                        for source_path in self.runtime.polling.startup_sources(scheduled_flow):
+                            results.append(self.runtime.run_executor.run_one(scheduled_flow, source_path))
+                    except FlowStoppedError:
+                        if self.runtime.flow_stop_event is not None:
+                            self.runtime.flow_stop_event.clear()
+                    except Exception:
+                        continue
+                    continue
+
+                sleep(0.05)
+        finally:
+            for entry in watch_entries:
+                watcher = entry["watcher"]
+                if isinstance(watcher, PollingWatcher):
+                    watcher.stop()
+        return results
+
+    def _poll_watch_entries(
+        self,
+        watch_entries: list[dict[str, object]],
+        queue: deque[_QueuedJob],
+        queued_keys: set[tuple[str, str | None]],
+        now: float,
+    ) -> None:
+        for entry in watch_entries:
+            if now < entry["next_poll"]:
+                continue
+            watched_flow = entry["flow"]
+            watcher = entry["watcher"]
+            assert isinstance(watcher, PollingWatcher)
+            for path in watcher.drain_events():
+                watched_trigger = watched_flow.trigger
+                assert isinstance(watched_trigger, WatchSpec)
+                if watched_trigger.run_as == "batch" and watched_trigger.source is not None and watched_trigger.source.is_dir():
+                    signature = self.runtime.polling.poll_source_signature(watched_flow, path)
+                    self.runtime.polling.enqueue_job(
+                        queue,
+                        queued_keys,
+                        watched_flow,
+                        None,
+                        batch_signatures=(signature,) if signature is not None else (),
+                    )
+                    break
+                if self.runtime.polling.is_poll_source_stale(watched_flow, path):
+                    self.runtime.polling.enqueue_job(queue, queued_keys, watched_flow, path)
+            entry["next_poll"] = now + float(entry["interval"])
+
+    def _update_schedule_entries(self, schedule_entries: list[dict[str, object]], now: float) -> None:
+        for entry in schedule_entries:
+            if entry["kind"] == "every":
+                if now >= entry["next_due"]:
+                    entry["pending"] = True
+                    entry["next_due"] = now + float(entry["interval"])
+                continue
+        current_dt = datetime.now()
+        for entry in schedule_entries:
+            if entry["kind"] != "at":
+                continue
+            if entry["last_run_date"] == current_dt.date():
+                continue
+            if (current_dt.hour, current_dt.minute) >= (entry["hour"], entry["minute"]):
+                entry["pending"] = True
+                entry["last_run_date"] = current_dt.date()
+
+
+from data_engine.authoring.model import FlowStoppedError
+
+__all__ = ["ContinuousRuntimeLoop"]

data_engine/authoring/execution/grouped.py

@@ -0,0 +1,106 @@
+"""Grouped runtime orchestration for authored flows."""
+
+from __future__ import annotations
+
+from queue import Queue
+import threading
+from typing import TYPE_CHECKING, Callable
+
+from data_engine.authoring.primitives import FlowContext
+from data_engine.authoring.execution.single import _FlowRuntime, RuntimeLedgerService, default_runtime_ledger_service
+from data_engine.runtime.runtime_db import RuntimeLedger
+
+if TYPE_CHECKING:
+    from data_engine.authoring.flow import Flow
+
+
+class _GroupedFlowRuntime:
+    """Grouped orchestrator: sequential within a group, parallel across groups."""
+
+    def __init__(
+        self,
+        flows: tuple["Flow", ...],
+        *,
+        continuous: bool,
+        runtime_stop_event: threading.Event | None = None,
+        flow_stop_event: threading.Event | None = None,
+        status_callback: Callable[[str], None] | None = None,
+        runtime_ledger: RuntimeLedger | None = None,
+        runtime_ledger_service: RuntimeLedgerService | None = None,
+        runtime_ledger_factory: Callable[[], RuntimeLedger] | None = None,
+    ) -> None:
+        self.flows = tuple(flows)
+        self.continuous = continuous
+        self.runtime_stop_event = runtime_stop_event
+        self.flow_stop_event = flow_stop_event
+        self.status_callback = status_callback
+        self._runtime_ledger_service = runtime_ledger_service or default_runtime_ledger_service()
+        self._runtime_ledger_factory = runtime_ledger_factory or self._runtime_ledger_service.open_runtime_ledger
+        self.runtime_ledger = runtime_ledger or self._runtime_ledger_factory()
+
+    def run(self) -> list[FlowContext]:
+        grouped = self._grouped_flows()
+        if len(grouped) <= 1:
+            only = next(iter(grouped.values()), ())
+            return _FlowRuntime(
+                tuple(only),
+                continuous=self.continuous,
+                runtime_stop_event=self.runtime_stop_event,
+                flow_stop_event=self.flow_stop_event,
+                status_callback=self.status_callback,
+                runtime_ledger=self.runtime_ledger,
+                runtime_ledger_service=self._runtime_ledger_service,
+            ).run()
+
+        results_by_group: dict[str, list[FlowContext]] = {name: [] for name in grouped}
+        errors: Queue[tuple[str, Exception]] = Queue()
+        threads: list[threading.Thread] = []
+        internal_runtime_stop = self.runtime_stop_event or threading.Event()
+        internal_flow_stop = self.flow_stop_event or threading.Event()
+
+        def run_group(group_name: str, group_flows: tuple["Flow", ...]) -> None:
+            try:
+                runtime = _FlowRuntime(
+                    group_flows,
+                    continuous=self.continuous,
+                    runtime_stop_event=internal_runtime_stop,
+                    flow_stop_event=internal_flow_stop,
+                    status_callback=self.status_callback,
+                    runtime_ledger=self.runtime_ledger,
+                    runtime_ledger_service=self._runtime_ledger_service,
+                    runtime_ledger_factory=self._runtime_ledger_factory,
+                )
+                results_by_group[group_name] = runtime.run()
+            except Exception as exc:  # pragma: no cover
+                errors.put((group_name, exc))
+                if not self.continuous:
+                    internal_runtime_stop.set()
+            finally:
+                self.runtime_ledger.close()
+
+        for group_name, group_flows in grouped.items():
+            thread = threading.Thread(target=run_group, args=(group_name, group_flows), daemon=True)
+            threads.append(thread)
+            thread.start()
+
+        for thread in threads:
+            thread.join()
+
+        if not self.continuous and not errors.empty():
+            _, exc = errors.get()
+            raise exc
+
+        ordered_results: list[FlowContext] = []
+        for group_name in grouped:
+            ordered_results.extend(results_by_group[group_name])
+        return ordered_results
+
+    def _grouped_flows(self) -> dict[str, tuple["Flow", ...]]:
+        grouped: dict[str, list["Flow"]] = {}
+        for index, flow in enumerate(self.flows):
+            key = flow.group or f"group-{index}"
+            grouped.setdefault(key, []).append(flow)
+        return {name: tuple(group_flows) for name, group_flows in grouped.items()}
+
+
+__all__ = ["_GroupedFlowRuntime"]

data_engine/authoring/execution/logging.py

@@ -0,0 +1,83 @@
+"""Runtime log and ledger emission helpers for authored flows."""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from data_engine.domain.time import utcnow_text
+from data_engine.runtime.runtime_db import RuntimeLedger
+
+LOGGER = logging.getLogger(__name__)
+
+
+class RuntimeLogEmitter:
+    """Own runtime log persistence and logger emission."""
+
+    def __init__(self, runtime_ledger: RuntimeLedger) -> None:
+        self.runtime_ledger = runtime_ledger
+
+    def log_runtime_message(
+        self,
+        message: str,
+        *,
+        level: str,
+        run_id: str | None,
+        flow_name: str | None,
+        step_label: str | None = None,
+        exc_info: bool = False,
+    ) -> None:
+        created_at_utc = utcnow_text()
+        self.runtime_ledger.append_log(
+            level=level.upper(),
+            message=message,
+            created_at_utc=created_at_utc,
+            run_id=run_id,
+            flow_name=flow_name,
+            step_label=step_label,
+        )
+        logger_method = LOGGER.error if level == "error" else LOGGER.info
+        logger_method(message, exc_info=exc_info)
+
+    def log_flow_event(
+        self,
+        run_id: str,
+        flow_name: str,
+        source_path: Path | None,
+        *,
+        status: str,
+        elapsed: float | None = None,
+        level: str = "info",
+        exc_info: bool = False,
+    ) -> None:
+        message = f"run={run_id} flow={flow_name} source={source_path} status={status}"
+        if elapsed is not None:
+            message = f"{message} elapsed={elapsed:.6f}"
+        self.log_runtime_message(message, level=level, run_id=run_id, flow_name=flow_name, exc_info=exc_info)
+
+    def log_step_event(
+        self,
+        run_id: str,
+        flow_name: str,
+        step_label: str,
+        source_path: Path | None,
+        *,
+        status: str,
+        elapsed: float | None = None,
+        level: str = "info",
+        exc_info: bool = False,
+    ) -> None:
+        message = f"run={run_id} flow={flow_name} step={step_label} source={source_path} status={status}"
+        if elapsed is not None:
+            message = f"{message} elapsed={elapsed:.6f}"
+        self.log_runtime_message(
+            message,
+            level=level,
+            run_id=run_id,
+            flow_name=flow_name,
+            step_label=step_label,
+            exc_info=exc_info,
+        )
+
+
+__all__ = ["RuntimeLogEmitter"]

data_engine/authoring/execution/polling.py

@@ -0,0 +1,135 @@
+"""Polling, scheduling, and source-queue helpers for authored flows."""
+
+from __future__ import annotations
+
+from collections import deque
+from pathlib import Path
+
+from data_engine.authoring.primitives import WatchSpec
+from data_engine.authoring.execution.context import _QueuedJob
+from data_engine.domain.source_state import SourceSignature
+from data_engine.runtime.file_watch import PollingWatcher, iter_candidate_paths
+from data_engine.runtime.runtime_db import RuntimeLedger
+
+
+class RuntimePollingSupport:
+    """Own watcher creation, queueing, and stale-source detection."""
+
+    def __init__(self, runtime_ledger: RuntimeLedger) -> None:
+        self.runtime_ledger = runtime_ledger
+
+    def make_watcher(self, trigger: WatchSpec) -> PollingWatcher:
+        return PollingWatcher(trigger.source, recursive=True, extensions=trigger.extensions, settle=trigger.settle)
+
+    def startup_sources(self, flow: "Flow", *, allow_missing: bool = False) -> list[Path | None]:
+        trigger = flow.trigger
+        if not isinstance(trigger, WatchSpec) or trigger.source is None:
+            return [None]
+        if not trigger.source.exists():
+            return [None]
+        if trigger.source.is_file():
+            return [trigger.source]
+        if trigger.run_as == "batch":
+            return [None]
+        return list(
+            iter_candidate_paths(
+                trigger.source,
+                extensions=trigger.extensions,
+                recursive=True,
+                allow_missing=allow_missing,
+            )
+        )
+
+    def enqueue_job(
+        self,
+        queue: deque[_QueuedJob],
+        queued_keys: set[tuple[str, str | None]],
+        flow: "Flow",
+        source_path: Path | None,
+        *,
+        batch_signatures: tuple[SourceSignature, ...] = (),
+    ) -> None:
+        key = self.job_key(flow, source_path)
+        if key in queued_keys:
+            if batch_signatures:
+                for index, job in enumerate(queue):
+                    if self.job_key(job.flow, job.source_path) != key:
+                        continue
+                    merged = {signature.source_path: signature for signature in job.batch_signatures}
+                    for signature in batch_signatures:
+                        merged[signature.source_path] = signature
+                    queue[index] = _QueuedJob(
+                        flow=job.flow,
+                        source_path=job.source_path,
+                        batch_signatures=tuple(merged[path] for path in sorted(merged)),
+                    )
+                    break
+            return
+        queue.append(_QueuedJob(flow, source_path, batch_signatures))
+        queued_keys.add(key)
+
+    def job_key(self, flow: "Flow", source_path: Path | None) -> tuple[str, str | None]:
+        return (flow.name, str(source_path) if source_path is not None else None)
+
+    def stale_poll_sources(self, flow: "Flow") -> list[Path | None]:
+        current_source_paths: set[str] = set()
+        stale: list[Path | None] = []
+        trigger = flow.trigger
+        if not isinstance(trigger, WatchSpec) or trigger.mode != "poll" or trigger.source is None:
+            return stale
+        if not trigger.source.exists():
+            return [None]
+        if trigger.run_as == "batch" and trigger.source.is_dir():
+            for source_path in iter_candidate_paths(trigger.source, extensions=trigger.extensions, recursive=True, allow_missing=True):
+                current_source_paths.add(self.runtime_ledger.normalize_source_path(source_path))
+                if self.is_poll_source_stale(flow, source_path):
+                    stale.append(None)
+                    break
+            self.runtime_ledger.prune_missing_file_state(flow_name=flow.name, current_source_paths=current_source_paths)
+            return stale
+        for source_path in self.startup_sources(flow, allow_missing=True):
+            if source_path is None:
+                stale.append(None)
+                continue
+            current_source_paths.add(self.runtime_ledger.normalize_source_path(source_path))
+            if self.is_poll_source_stale(flow, source_path):
+                stale.append(source_path)
+        self.runtime_ledger.prune_missing_file_state(flow_name=flow.name, current_source_paths=current_source_paths)
+        return stale
+
+    def stale_batch_poll_signatures(self, flow: "Flow") -> tuple[SourceSignature, ...]:
+        trigger = flow.trigger
+        if not isinstance(trigger, WatchSpec) or trigger.mode != "poll" or trigger.source is None or not trigger.source.is_dir():
+            return ()
+        signatures: dict[str, SourceSignature] = {}
+        for source_path in iter_candidate_paths(trigger.source, extensions=trigger.extensions, recursive=True, allow_missing=True):
+            if not self.is_poll_source_stale(flow, source_path):
+                continue
+            signature = self.poll_source_signature(flow, source_path)
+            if signature is not None:
+                signatures[signature.source_path] = signature
+        return tuple(signatures[path] for path in sorted(signatures))
+
+    def is_poll_source_stale(self, flow: "Flow", source_path: Path | None) -> bool:
+        trigger = flow.trigger
+        if not isinstance(trigger, WatchSpec) or trigger.mode != "poll":
+            return False
+        if source_path is None or not source_path.exists():
+            return True
+        signature = self.poll_source_signature(flow, source_path)
+        if signature is None and trigger.source is not None and trigger.source.exists() and trigger.source.is_file():
+            return True
+        return self.runtime_ledger.is_poll_source_stale(flow.name, signature)
+
+    def poll_source_signature(self, flow: "Flow", source_path: Path | None) -> SourceSignature | None:
+        if source_path is None or not isinstance(flow.trigger, WatchSpec) or flow.trigger.mode != "poll":
+            return None
+        return self.runtime_ledger.source_signature_for_path(source_path)
+
+    def normalized_source_path(self, source_path: Path | None) -> str | None:
+        if source_path is None:
+            return None
+        return self.runtime_ledger.normalize_source_path(source_path)
+
+
+__all__ = ["RuntimePollingSupport"]