py-data-engine 0.1.3__tar.gz → 0.2.1__tar.gz

{py_data_engine-0.1.3/src/py_data_engine.egg-info → py_data_engine-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-data-engine
-Version: 0.1.3
+Version: 0.2.1
 Summary: Workbook-driven workflow jobs with strict filename parsing and DuckDB-backed processing.
 Author: Data Engine contributors
 License-Expression: MIT
@@ -18,6 +18,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: APScheduler>=3.11.0
 Requires-Dist: duckdb>=1.5.0
 Requires-Dist: fastexcel>=0.19.0
 Requires-Dist: polars>=1.39.0

{py_data_engine-0.1.3 → py_data_engine-0.2.1}/pyproject.toml

@@ -2,6 +2,8 @@
 requires = [
   "setuptools>=68",
   "wheel",
+  "APScheduler>=3.11.0",
+  "duckdb>=1.5.0",
   "polars>=1.39.0",
   "myst-parser>=4.0.0",
   "Sphinx>=9.1.0",
@@ -31,6 +33,7 @@ classifiers = [
   "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [
+  "APScheduler>=3.11.0",
   "duckdb>=1.5.0",
   "fastexcel>=0.19.0",
   "polars>=1.39.0",

{py_data_engine-0.1.3 → py_data_engine-0.2.1}/src/data_engine/__init__.py

@@ -7,9 +7,9 @@ from typing import TYPE_CHECKING
 if TYPE_CHECKING:
     from data_engine.authoring.flow import Flow
     from data_engine.authoring.flow import discover_flows, load_flow, run
-    from data_engine.authoring.primitives import Batch
-    from data_engine.authoring.primitives import FileRef
-    from data_engine.authoring.primitives import FlowContext
+    from data_engine.core.primitives import Batch
+    from data_engine.core.primitives import FileRef
+    from data_engine.core.primitives import FlowContext
 
 __all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_flow", "run"]
 
@@ -21,9 +21,9 @@ def __getattr__(name: str):
         from data_engine.authoring.flow import discover_flows
         from data_engine.authoring.flow import load_flow
         from data_engine.authoring.flow import run
-        from data_engine.authoring.primitives import Batch
-        from data_engine.authoring.primitives import FileRef
-        from data_engine.authoring.primitives import FlowContext
+        from data_engine.core.primitives import Batch
+        from data_engine.core.primitives import FileRef
+        from data_engine.core.primitives import FlowContext
 
         return {
             "Batch": Batch,

{py_data_engine-0.1.3 → py_data_engine-0.2.1}/src/data_engine/application/catalog.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 from pathlib import Path
 
-from data_engine.authoring.model import FlowValidationError
+from data_engine.core.model import FlowValidationError
 from data_engine.domain import FlowCatalogEntry, FlowCatalogLike, FlowCatalogState
 from data_engine.platform.workspace_models import WorkspacePaths
 from data_engine.services import FlowCatalogService
@@ -86,7 +86,10 @@ class FlowCatalogApplication:
         """Load discovered entries and merge them into one catalog state."""
         base = current_state or FlowCatalogState.empty()
         entries = self.flow_catalog_service.load_entries(workspace_root=workspace_root)
-        return base.with_entries(entries).with_empty_message("")
+        state = base.with_entries(entries).with_empty_message("")
+        if base.selected_flow_name is not None and state.selected_flow_name == base.selected_flow_name:
+            return state
+        return state.with_selected_flow_name(_first_grouped_entry_name(state.entries))
 
     def empty_state(
         self,
@@ -149,3 +152,11 @@ class FlowCatalogApplication:
                 error_text=message,
             )
         return FlowCatalogLoadResult(catalog_state=catalog_state, loaded=True)
+
+
+def _first_grouped_entry_name(entries: tuple[FlowCatalogEntry, ...]) -> str | None:
+    """Return the first entry name in the same grouped order used by operator surfaces."""
+    for bucket in group_cards(entries):
+        if bucket.entries:
+            return bucket.entries[0].name
+    return None

{py_data_engine-0.1.3 → py_data_engine-0.2.1}/src/data_engine/application/runtime.py

@@ -7,7 +7,7 @@ from datetime import UTC, datetime
 from typing import Any
 from collections.abc import Iterable
 
-from data_engine.authoring.model import FlowStoppedError
+from data_engine.core.model import FlowStoppedError
 from data_engine.domain import (
     DaemonLifecyclePolicy,
     DaemonStatusState,

{py_data_engine-0.1.3 → py_data_engine-0.2.1}/src/data_engine/authoring/__init__.py

@@ -13,5 +13,5 @@ def __getattr__(name: str):
     if name in {"Flow", "discover_flows", "load_flow", "run"}:
         module = import_module("data_engine.authoring.flow")
         return getattr(module, name)
-    module = import_module("data_engine.authoring.primitives")
+    module = import_module("data_engine.core.primitives")
     return getattr(module, name)

py_data_engine-0.2.1/src/data_engine/authoring/flow.py

@@ -0,0 +1,198 @@
+"""Flow DSL and public authoring entrypoints."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from data_engine.core.flow import Flow as _CoreFlow
+from data_engine.core.helpers import _validate_slot_name
+from data_engine.core.model import FlowValidationError
+from data_engine.core.primitives import FlowContext
+
+if TYPE_CHECKING:
+    from data_engine.authoring.services import AuthoringServices
+    from data_engine.services.flow_execution import FlowExecutionService
+    from data_engine.services.runtime_execution import RuntimeExecutionService
+
+
+def _resolve_authoring_services(
+    *,
+    authoring_services: AuthoringServices | None = None,
+    runtime_execution_service: RuntimeExecutionService | None = None,
+    flow_execution_service: FlowExecutionService | None = None,
+) -> AuthoringServices:
+    """Return one authoring collaborator bundle with explicit overrides applied."""
+    from data_engine.authoring.services import build_authoring_services, default_authoring_services
+
+    services = authoring_services or default_authoring_services()
+    if runtime_execution_service is None and flow_execution_service is None:
+        return services
+    return build_authoring_services(
+        runtime_execution_service=runtime_execution_service or services.runtime_execution_service,
+        flow_execution_service=flow_execution_service or services.flow_execution_service,
+    )
+
+
+class Flow(_CoreFlow):
+    """Public authoring flow with execution conveniences layered over core definitions."""
+
+    def run_once(
+        self,
+        *,
+        authoring_services: AuthoringServices | None = None,
+        runtime_execution_service: RuntimeExecutionService | None = None,
+    ) -> list[FlowContext]:
+        """Run this flow once and return completed runtime contexts.
+
+        Parameters
+        ----------
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        list[FlowContext]
+            One context per executed source.
+        """
+        service = _resolve_authoring_services(
+            authoring_services=authoring_services,
+            runtime_execution_service=runtime_execution_service,
+        ).runtime_execution_service
+        return service.run_once(self)
+
+    def preview(
+        self,
+        *,
+        use: str | None = None,
+        authoring_services: AuthoringServices | None = None,
+        runtime_execution_service: RuntimeExecutionService | None = None,
+    ) -> object:
+        """Run this flow in preview mode and return one preview value.
+
+        Parameters
+        ----------
+        use : str | None
+            Optional named object slot to preview instead of the final current
+            value.
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        object
+            Preview value returned by the runtime execution service.
+
+        Raises
+        ------
+        FlowValidationError
+            If preview is requested from inside a compiled flow module.
+        """
+        from data_engine.flow_modules.flow_module_loader import in_compiled_flow_module_context
+
+        if in_compiled_flow_module_context():
+            raise FlowValidationError("preview() is not available inside compiled flow modules.")
+        normalized_use = _validate_slot_name(method_name="preview", slot_name="use", value=use)
+        service = _resolve_authoring_services(
+            authoring_services=authoring_services,
+            runtime_execution_service=runtime_execution_service,
+        ).runtime_execution_service
+        return service.preview(self, use=normalized_use)
+
+    def show(self) -> object:
+        """Run this flow once and return the single final current value.
+
+        Returns
+        -------
+        object
+            Final ``context.current`` value.
+
+        Raises
+        ------
+        FlowValidationError
+            If called from a compiled flow module or the flow produces anything
+            other than one result.
+        """
+        from data_engine.flow_modules.flow_module_loader import in_compiled_flow_module_context
+
+        if in_compiled_flow_module_context():
+            raise FlowValidationError("show() is not available inside compiled flow modules.")
+        results = self.run_once()
+        if len(results) != 1:
+            raise FlowValidationError(f"show() requires exactly one result, found {len(results)}.")
+        return results[0].current
+
+    def run(
+        self,
+        *,
+        authoring_services: AuthoringServices | None = None,
+        runtime_execution_service: RuntimeExecutionService | None = None,
+    ) -> list[FlowContext]:
+        """Run this flow continuously according to its trigger.
+
+        Parameters
+        ----------
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        list[FlowContext]
+            Completed contexts collected before the runtime exits.
+        """
+        service = _resolve_authoring_services(
+            authoring_services=authoring_services,
+            runtime_execution_service=runtime_execution_service,
+        ).runtime_execution_service
+        return service.run_continuous(self)
+
+
+def load_flow(
+    name: str,
+    *,
+    data_root: Path | None = None,
+    authoring_services: AuthoringServices | None = None,
+    flow_execution_service: FlowExecutionService | None = None,
+) -> Flow:
+    """Load one code-defined flow by flow-module name."""
+    service = _resolve_authoring_services(
+        authoring_services=authoring_services,
+        flow_execution_service=flow_execution_service,
+    ).flow_execution_service
+    return service.load_flow(name, workspace_root=data_root)
+
+
+def discover_flows(
+    *,
+    data_root: Path | None = None,
+    authoring_services: AuthoringServices | None = None,
+    flow_execution_service: FlowExecutionService | None = None,
+) -> tuple[Flow, ...]:
+    """Discover and build all code-defined flows from compiled flow modules."""
+    service = _resolve_authoring_services(
+        authoring_services=authoring_services,
+        flow_execution_service=flow_execution_service,
+    ).flow_execution_service
+    return service.discover_flows(workspace_root=data_root)
+
+
+def run(
+    *flows: Flow,
+    authoring_services: AuthoringServices | None = None,
+    runtime_execution_service: RuntimeExecutionService | None = None,
+) -> list[FlowContext]:
+    """Run multiple flows with sequential execution per group and parallel groups."""
+    service = _resolve_authoring_services(
+        authoring_services=authoring_services,
+        runtime_execution_service=runtime_execution_service,
+    ).runtime_execution_service
+    return service.run_grouped_continuous(tuple(flows))
+
+
+__all__ = ["Flow", "discover_flows", "load_flow", "run"]

{py_data_engine-0.1.3 → py_data_engine-0.2.1}/src/data_engine/authoring/services.py

@@ -4,9 +4,11 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 from functools import lru_cache
+from typing import TYPE_CHECKING
 
-from data_engine.services.flow_execution import FlowExecutionService
-from data_engine.services.runtime_execution import RuntimeExecutionService
+if TYPE_CHECKING:
+    from data_engine.services.flow_execution import FlowExecutionService
+    from data_engine.services.runtime_execution import RuntimeExecutionService
 
 
 @dataclass(frozen=True)
@@ -23,6 +25,9 @@ def build_authoring_services(
     flow_execution_service: FlowExecutionService | None = None,
 ) -> AuthoringServices:
     """Build one authoring collaborator bundle with optional overrides."""
+    from data_engine.services.flow_execution import FlowExecutionService
+    from data_engine.services.runtime_execution import RuntimeExecutionService
+
     return AuthoringServices(
         runtime_execution_service=runtime_execution_service or RuntimeExecutionService(),
         flow_execution_service=flow_execution_service or FlowExecutionService(),

py_data_engine-0.2.1/src/data_engine/core/__init__.py

@@ -0,0 +1,35 @@
+"""Core flow definitions and runtime primitives."""
+
+from data_engine.core.flow import Flow
+from data_engine.core.model import FlowExecutionError, FlowStoppedError, FlowValidationError
+from data_engine.core.primitives import (
+    Batch,
+    FileRef,
+    FlowContext,
+    MirrorContext,
+    MirrorSpec,
+    SourceContext,
+    SourceMetadata,
+    StepSpec,
+    WatchSpec,
+    WorkspaceConfigContext,
+    collect_files,
+)
+
+__all__ = [
+    "Batch",
+    "FileRef",
+    "Flow",
+    "FlowContext",
+    "FlowExecutionError",
+    "FlowStoppedError",
+    "FlowValidationError",
+    "MirrorContext",
+    "MirrorSpec",
+    "SourceContext",
+    "SourceMetadata",
+    "StepSpec",
+    "WatchSpec",
+    "WorkspaceConfigContext",
+    "collect_files",
+]