py-data-engine 0.1.0__py3-none-any.whl

data_engine/authoring/execution/runner.py

@@ -0,0 +1,210 @@
+"""Single flow-run execution lifecycle for authored flows."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from time import monotonic
+from typing import TYPE_CHECKING
+
+from data_engine.authoring.model import FlowExecutionError, FlowStoppedError, FlowValidationError
+from data_engine.authoring.primitives import FlowContext, WatchSpec
+from data_engine.domain.time import utcnow_text
+
+if TYPE_CHECKING:
+    from data_engine.authoring.execution.single import _FlowRuntime
+    from data_engine.authoring.flow import Flow
+    from data_engine.authoring.primitives import StepSpec
+
+
+class FlowRunExecutor:
+    """Own one-run lifecycle, step execution, and ledger/log updates."""
+
+    def __init__(self, runtime: "_FlowRuntime") -> None:
+        self.runtime = runtime
+
+    def run_one(self, flow: "Flow", source_path: "Path | None", *, batch_signatures=()) -> FlowContext:
+        self.runtime._check_flow_stop()
+        run_id = self.runtime.context_builder.new_run_id()
+        context = self.runtime.context_builder.build(flow, source_path, run_id=run_id)
+        run_started = monotonic()
+        signature = self.runtime.polling.poll_source_signature(flow, source_path)
+        effective_signatures = batch_signatures or ((signature,) if signature is not None else ())
+        started_at_utc = str(context.metadata["started_at_utc"])
+        normalized_source_path = signature.source_path if signature is not None else self.runtime.polling.normalized_source_path(source_path)
+        self.runtime.runtime_ledger.record_run_started(
+            run_id=run_id,
+            flow_name=context.flow_name,
+            group_name=context.group,
+            source_path=normalized_source_path,
+            started_at_utc=started_at_utc,
+        )
+        for effective_signature in effective_signatures:
+            self.runtime.runtime_ledger.upsert_file_state(flow_name=context.flow_name, signature=effective_signature, status="started")
+        self.runtime.log_emitter.log_flow_event(run_id, context.flow_name, source_path, status="started")
+        try:
+            self._ensure_runtime_sources_available(flow, context, source_path)
+            for step in flow.steps:
+                self.runtime._check_flow_stop()
+                self._load_current_for_step(context, step)
+                step_started = monotonic()
+                step_started_at_utc = utcnow_text()
+                step_run_id = self.runtime.runtime_ledger.record_step_started(
+                    run_id=run_id,
+                    flow_name=context.flow_name,
+                    step_label=step.label,
+                    started_at_utc=step_started_at_utc,
+                )
+                self.runtime.log_emitter.log_step_event(run_id, context.flow_name, step.label, source_path, status="started")
+                try:
+                    result = step.fn(context)
+                except FlowStoppedError:
+                    raise
+                except Exception as exc:
+                    failure = FlowExecutionError(
+                        flow_name=context.flow_name,
+                        phase="step",
+                        step_label=step.label,
+                        function_name=step.function_name,
+                        source_path=source_path,
+                        detail=f"{type(exc).__name__}: {exc}",
+                    )
+                    elapsed_ms = max(int((monotonic() - step_started) * 1000), 0)
+                    self.runtime.runtime_ledger.record_step_finished(
+                        step_run_id=step_run_id,
+                        status="failed",
+                        finished_at_utc=utcnow_text(),
+                        elapsed_ms=elapsed_ms,
+                        error_text=str(failure),
+                    )
+                    self.runtime.log_emitter.log_runtime_message(
+                        str(failure),
+                        level="error",
+                        run_id=run_id,
+                        flow_name=context.flow_name,
+                        step_label=step.label,
+                    )
+                    raise failure from exc
+                context.current = result
+                if step.save_as is not None:
+                    context.objects[step.save_as] = result
+                if isinstance(result, Path) and result.exists():
+                    step_outputs = context.metadata.setdefault("step_outputs", {})
+                    if isinstance(step_outputs, dict):
+                        step_outputs[step.label] = result
+                elapsed = monotonic() - step_started
+                elapsed_ms = max(int(elapsed * 1000), 0)
+                self.runtime.runtime_ledger.record_step_finished(
+                    step_run_id=step_run_id,
+                    status="success",
+                    finished_at_utc=utcnow_text(),
+                    elapsed_ms=elapsed_ms,
+                    output_path=str(result.resolve()) if isinstance(result, Path) and result.exists() else None,
+                )
+                self.runtime.log_emitter.log_step_event(run_id, context.flow_name, step.label, source_path, status="success", elapsed=elapsed)
+        except FlowStoppedError as exc:
+            finished_at_utc = utcnow_text()
+            self.runtime.runtime_ledger.record_run_finished(run_id=run_id, status="stopped", finished_at_utc=finished_at_utc, error_text=str(exc))
+            for effective_signature in effective_signatures:
+                self.runtime.runtime_ledger.upsert_file_state(
+                    flow_name=context.flow_name,
+                    signature=effective_signature,
+                    status="stopped",
+                    error_text=str(exc),
+                )
+            self.runtime.log_emitter.log_flow_event(run_id, context.flow_name, source_path, status="stopped", elapsed=monotonic() - run_started)
+            raise
+        except Exception as exc:
+            elapsed = monotonic() - run_started
+            finished_at_utc = utcnow_text()
+            failed_step = step.label if "step" in locals() else None
+            failure_text = str(exc)
+            self.runtime.runtime_ledger.record_run_finished(run_id=run_id, status="failed", finished_at_utc=finished_at_utc, error_text=failure_text)
+            for effective_signature in effective_signatures:
+                self.runtime.runtime_ledger.upsert_file_state(
+                    flow_name=context.flow_name,
+                    signature=effective_signature,
+                    status="failed",
+                    error_text=failure_text,
+                )
+            if failed_step is None:
+                self.runtime.log_emitter.log_runtime_message(failure_text, level="error", run_id=run_id, flow_name=context.flow_name)
+                self.runtime.log_emitter.log_flow_event(run_id, context.flow_name, source_path, status="failed", elapsed=elapsed, level="error", exc_info=True)
+            else:
+                self.runtime.log_emitter.log_step_event(
+                    run_id,
+                    context.flow_name,
+                    failed_step,
+                    source_path,
+                    status="failed",
+                    elapsed=elapsed,
+                    level="error",
+                    exc_info=True,
+                )
+                self.runtime.log_emitter.log_flow_event(run_id, context.flow_name, source_path, status="failed", elapsed=elapsed)
+            raise
+        total = monotonic() - run_started
+        finished_at_utc = utcnow_text()
+        self.runtime.runtime_ledger.record_run_finished(run_id=run_id, status="success", finished_at_utc=finished_at_utc)
+        for effective_signature in effective_signatures:
+            self.runtime.runtime_ledger.upsert_file_state(
+                flow_name=context.flow_name,
+                signature=effective_signature,
+                status="success",
+                run_id=run_id,
+                finished_at_utc=finished_at_utc,
+            )
+        self.runtime.log_emitter.log_flow_event(run_id, context.flow_name, source_path, status="success", elapsed=total)
+        return context
+
+    def preview_one(self, flow: "Flow", source_path: "Path | None", *, use: str | None) -> FlowContext:
+        self.runtime._check_flow_stop()
+        context = self.runtime.context_builder.build(flow, source_path, run_id="preview")
+        self._ensure_runtime_sources_available(flow, context, source_path)
+        for step in flow.steps:
+            self.runtime._check_flow_stop()
+            self._load_current_for_step(context, step)
+            try:
+                result = step.fn(context)
+            except FlowStoppedError:
+                raise
+            except Exception as exc:
+                raise FlowExecutionError(
+                    flow_name=context.flow_name,
+                    phase="step",
+                    step_label=step.label,
+                    function_name=step.function_name,
+                    source_path=source_path,
+                    detail=f"{type(exc).__name__}: {exc}",
+                ) from exc
+            context.current = result
+            if step.save_as is not None:
+                context.objects[step.save_as] = result
+                if use is not None and step.save_as == use:
+                    context.current = result
+                    return context
+        return context
+
+    def _ensure_runtime_sources_available(self, flow: "Flow", context: FlowContext, source_path: "Path | None") -> None:
+        trigger = flow.trigger
+        if not isinstance(trigger, WatchSpec) or trigger.source is None:
+            return
+        if not trigger.source.exists():
+            raise FlowValidationError(f"Source path not found: {trigger.source}")
+        if trigger.source.is_file():
+            source_path = context.source.path if context.source is not None else source_path
+            if source_path is None or not source_path.exists():
+                raise FlowValidationError(f"Source file not found: {trigger.source}")
+            if not source_path.is_file():
+                raise FlowValidationError(f"Source file is not a file: {trigger.source}")
+        elif not trigger.source.is_dir():
+            raise FlowValidationError(f"Source path is neither a file nor a directory: {trigger.source}")
+
+    def _load_current_for_step(self, context: FlowContext, step: "StepSpec") -> None:
+        if step.use is None or step.use == "current":
+            return
+        if step.use not in context.objects:
+            raise FlowValidationError(f"Step {step.label!r} requested missing object {step.use!r}.")
+        context.current = context.objects[step.use]
+
+
+__all__ = ["FlowRunExecutor"]

data_engine/authoring/execution/single.py

@@ -0,0 +1,171 @@
+"""Single-runtime orchestration for authored flows."""
+
+from __future__ import annotations
+
+from pathlib import Path
+import threading
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Callable
+
+from data_engine.authoring.model import FlowStoppedError, FlowValidationError
+from data_engine.authoring.primitives import FlowContext, WatchSpec
+from data_engine.authoring.execution.continuous import ContinuousRuntimeLoop
+from data_engine.authoring.execution.context import RuntimeContextBuilder
+from data_engine.authoring.execution.logging import RuntimeLogEmitter
+from data_engine.authoring.execution.polling import RuntimePollingSupport
+from data_engine.authoring.execution.runner import FlowRunExecutor
+from data_engine.runtime.file_watch import PollingWatcher
+from data_engine.runtime.runtime_db import RuntimeLedger
+
+if TYPE_CHECKING:
+    from data_engine.authoring.flow import Flow
+    from data_engine.authoring.primitives import StepSpec
+
+
+def _open_default_runtime_ledger() -> RuntimeLedger:
+    """Open the default runtime ledger for authored flow execution."""
+    return RuntimeLedger.open_default()
+
+
+@dataclass(frozen=True)
+class RuntimeLedgerService:
+    """Own how authored flow execution opens its runtime ledger."""
+
+    open_runtime_ledger_func: Callable[[], RuntimeLedger]
+
+    def open_runtime_ledger(self) -> RuntimeLedger:
+        """Open one runtime ledger for authored flow execution."""
+        return self.open_runtime_ledger_func()
+
+
+def default_runtime_ledger_service() -> RuntimeLedgerService:
+    """Build the default runtime-ledger service for authored flows."""
+    return RuntimeLedgerService(open_runtime_ledger_func=_open_default_runtime_ledger)
+
+
+class _FlowRuntime:
+    """Sequential runtime that executes one or more configured flows."""
+
+    def __init__(
+        self,
+        flows: tuple["Flow", ...],
+        *,
+        continuous: bool,
+        runtime_stop_event: threading.Event | None = None,
+        flow_stop_event: threading.Event | None = None,
+        status_callback: Callable[[str], None] | None = None,
+        runtime_ledger: RuntimeLedger | None = None,
+        runtime_ledger_service: RuntimeLedgerService | None = None,
+        runtime_ledger_factory: Callable[[], RuntimeLedger] | None = None,
+    ) -> None:
+        self.flows = tuple(flows)
+        self.continuous = continuous
+        self.runtime_stop_event = runtime_stop_event
+        self.flow_stop_event = flow_stop_event
+        self.status_callback = status_callback
+        runtime_ledger_service = runtime_ledger_service or default_runtime_ledger_service()
+        self._runtime_ledger_factory = runtime_ledger_factory or runtime_ledger_service.open_runtime_ledger
+        self._owns_runtime_ledger = runtime_ledger is None
+        self.runtime_ledger = runtime_ledger or self._runtime_ledger_factory()
+        self.context_builder = RuntimeContextBuilder()
+        self.log_emitter = RuntimeLogEmitter(self.runtime_ledger)
+        self.polling = RuntimePollingSupport(self.runtime_ledger)
+        self.run_executor = FlowRunExecutor(self)
+        self.continuous_loop = ContinuousRuntimeLoop(self)
+
+    def run(self) -> list[FlowContext]:
+        try:
+            self._validate()
+            if not self.continuous or all(flow.mode == "manual" for flow in self.flows):
+                return self._run_once_all()
+            return self.continuous_loop.run()
+        finally:
+            self._close_owned_runtime_ledger()
+
+    def preview(self, *, use: str | None = None):
+        """Run exactly one flow for notebook-style inspection and return one object."""
+        try:
+            self._validate()
+            if len(self.flows) != 1:
+                raise FlowValidationError("preview() requires exactly one flow.")
+            flow = self.flows[0]
+            startup_sources = self.polling.startup_sources(flow)
+            if not startup_sources:
+                raise FlowValidationError("preview() could not determine a startup source.")
+            context = self.run_executor.preview_one(flow, startup_sources[0], use=use)
+            if use is None or use == "current":
+                return context.current
+            if use not in context.objects:
+                raise FlowValidationError(f"preview() could not find saved object {use!r}.")
+            return context.objects[use]
+        finally:
+            self._close_owned_runtime_ledger()
+
+    def _close_owned_runtime_ledger(self) -> None:
+        """Close the runtime ledger when this runtime opened it implicitly."""
+        if not self._owns_runtime_ledger:
+            return
+        self.runtime_ledger.close()
+
+    def _validate(self) -> None:
+        names = [flow.name for flow in self.flows]
+        if any(name is None or not str(name).strip() for name in names):
+            raise FlowValidationError("Flow names must be set before execution.")
+        if len(set(names)) != len(names):
+            raise FlowValidationError("Flow names must be unique within one runtime.")
+        for flow in self.flows:
+            if not flow.steps:
+                raise FlowValidationError(f"Flow {flow.name!r} must define at least one step.")
+
+    def _run_once_all(self) -> list[FlowContext]:
+        results: list[FlowContext] = []
+        for flow in self.flows:
+            for source_path in self.polling.startup_sources(flow):
+                batch_signatures = ()
+                trigger = flow.trigger
+                if (
+                    source_path is None
+                    and isinstance(trigger, WatchSpec)
+                    and trigger.mode == "poll"
+                    and trigger.run_as == "batch"
+                    and trigger.source is not None
+                    and trigger.source.is_dir()
+                ):
+                    batch_signatures = self.polling.stale_batch_poll_signatures(flow)
+                results.append(self.run_executor.run_one(flow, source_path, batch_signatures=batch_signatures))
+        return results
+
+    def _preview_one(self, flow: "Flow", source_path: "Path | None", *, use: str | None) -> FlowContext:
+        return self.run_executor.preview_one(flow, source_path, use=use)
+
+    def _make_watcher(self, trigger: WatchSpec) -> PollingWatcher:
+        return self.polling.make_watcher(trigger)
+
+    def _startup_sources(self, flow: "Flow", *, allow_missing: bool = False):
+        return self.polling.startup_sources(flow, allow_missing=allow_missing)
+
+    def _stale_poll_sources(self, flow: "Flow"):
+        return self.polling.stale_poll_sources(flow)
+
+    def _stale_batch_poll_signatures(self, flow: "Flow"):
+        return self.polling.stale_batch_poll_signatures(flow)
+
+    def _is_poll_source_stale(self, flow: "Flow", source_path: "Path | None") -> bool:
+        return self.polling.is_poll_source_stale(flow, source_path)
+
+    def _poll_source_signature(self, flow: "Flow", source_path: "Path | None"):
+        return self.polling.poll_source_signature(flow, source_path)
+
+    def _normalized_source_path(self, source_path: "Path | None"):
+        return self.polling.normalized_source_path(source_path)
+
+    def _check_flow_stop(self) -> None:
+        if self.flow_stop_event is not None and self.flow_stop_event.is_set():
+            raise FlowStoppedError("Flow stop requested by operator.")
+
+    def _emit_status(self, message: str) -> None:
+        if self.status_callback is not None:
+            self.status_callback(message)
+
+
+__all__ = ["RuntimeLedgerService", "_FlowRuntime", "default_runtime_ledger_service"]