py-data-engine 0.1.2__tar.gz → 0.2.1__tar.gz

{py_data_engine-0.1.2/src/py_data_engine.egg-info → py_data_engine-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-data-engine
-Version: 0.1.2
+Version: 0.2.1
 Summary: Workbook-driven workflow jobs with strict filename parsing and DuckDB-backed processing.
 Author: Data Engine contributors
 License-Expression: MIT
@@ -18,6 +18,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: APScheduler>=3.11.0
 Requires-Dist: duckdb>=1.5.0
 Requires-Dist: fastexcel>=0.19.0
 Requires-Dist: polars>=1.39.0
@@ -57,10 +58,12 @@ Requires-Dist: notebook>=7.4.0; extra == "dev"
 Requires-Dist: numpy>=2.4.0; extra == "dev"
 Requires-Dist: openpyxl>=3.1.0; extra == "dev"
 Requires-Dist: pandas>=3.0.0; extra == "dev"
+Requires-Dist: pydoclint>=0.7.6; extra == "dev"
 Requires-Dist: pydocstyle>=6.3.0; extra == "dev"
 Requires-Dist: pyinstaller>=6.15.0; extra == "dev"
 Requires-Dist: pytest>=9.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=7.0.0; extra == "dev"
+Requires-Dist: ruff>=0.15.10; extra == "dev"
 Requires-Dist: Sphinx>=9.1.0; extra == "dev"
 Requires-Dist: sphinx_rtd_theme>=3.1.0; extra == "dev"
 Dynamic: license-file

{py_data_engine-0.1.2 → py_data_engine-0.2.1}/pyproject.toml

@@ -2,6 +2,8 @@
 requires = [
   "setuptools>=68",
   "wheel",
+  "APScheduler>=3.11.0",
+  "duckdb>=1.5.0",
   "polars>=1.39.0",
   "myst-parser>=4.0.0",
   "Sphinx>=9.1.0",
@@ -11,7 +13,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "py-data-engine"
-version = "0.1.2"
+dynamic = ["version"]
 description = "Workbook-driven workflow jobs with strict filename parsing and DuckDB-backed processing."
 readme = "README.md"
 requires-python = ">=3.14"
@@ -31,6 +33,7 @@ classifiers = [
   "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [
+  "APScheduler>=3.11.0",
   "duckdb>=1.5.0",
   "fastexcel>=0.19.0",
   "polars>=1.39.0",
@@ -87,10 +90,12 @@ dev = [
   "numpy>=2.4.0",
   "openpyxl>=3.1.0",
   "pandas>=3.0.0",
+  "pydoclint>=0.7.6",
   "pydocstyle>=6.3.0",
   "pyinstaller>=6.15.0",
   "pytest>=9.0.0",
   "pytest-cov>=7.0.0",
+  "ruff>=0.15.10",
   "Sphinx>=9.1.0",
   "sphinx_rtd_theme>=3.1.0",
 ]
@@ -114,6 +119,9 @@ data_engine = [
   "docs/html/**/*",
 ]
 
+[tool.setuptools.dynamic]
+version = {attr = "data_engine.platform.identity.APP_VERSION"}
+
 [tool.pytest.ini_options]
 pythonpath = ["src"]
 testpaths = ["tests"]

{py_data_engine-0.1.2 → py_data_engine-0.2.1}/setup.py

@@ -12,11 +12,22 @@ from setuptools.command.sdist import sdist as _sdist
 ROOT = Path(__file__).resolve().parent
 DOCS_SOURCE_DIR = ROOT / "src" / "data_engine" / "docs" / "sphinx_source"
 DOCS_OUTPUT_DIR = ROOT / "src" / "data_engine" / "docs" / "html"
+PROJECT_AST_MAP_SCRIPT = ROOT / "src" / "data_engine" / "devtools" / "project_ast_map.py"
 
 
 def _build_packaged_docs() -> None:
     if not DOCS_SOURCE_DIR.is_dir():
         raise FileNotFoundError(f"Packaged docs source directory is missing: {DOCS_SOURCE_DIR}")
+    subprocess.run(
+        [
+            sys.executable,
+            str(PROJECT_AST_MAP_SCRIPT),
+            str(ROOT / "src" / "data_engine"),
+            "--write-docs",
+            str(DOCS_SOURCE_DIR / "guides"),
+        ],
+        check=True,
+    )
     subprocess.run(
         [
             sys.executable,

{py_data_engine-0.1.2 → py_data_engine-0.2.1}/src/data_engine/__init__.py

@@ -5,11 +5,11 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from data_engine.authoring.builder import Batch
-    from data_engine.authoring.builder import FileRef
-    from data_engine.authoring.builder import Flow
-    from data_engine.authoring.builder import FlowContext
-    from data_engine.authoring.builder import discover_flows, load_flow, run
+    from data_engine.authoring.flow import Flow
+    from data_engine.authoring.flow import discover_flows, load_flow, run
+    from data_engine.core.primitives import Batch
+    from data_engine.core.primitives import FileRef
+    from data_engine.core.primitives import FlowContext
 
 __all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_flow", "run"]
 
@@ -17,13 +17,13 @@ __all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_fl
 def __getattr__(name: str):
     """Lazy-load runtime symbols so lightweight helpers can import package submodules safely."""
     if name in {"Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_flow", "run"}:
-        from data_engine.authoring.builder import Batch
-        from data_engine.authoring.builder import FileRef
-        from data_engine.authoring.builder import Flow
-        from data_engine.authoring.builder import FlowContext
-        from data_engine.authoring.builder import discover_flows
-        from data_engine.authoring.builder import load_flow
-        from data_engine.authoring.builder import run
+        from data_engine.authoring.flow import Flow
+        from data_engine.authoring.flow import discover_flows
+        from data_engine.authoring.flow import load_flow
+        from data_engine.authoring.flow import run
+        from data_engine.core.primitives import Batch
+        from data_engine.core.primitives import FileRef
+        from data_engine.core.primitives import FlowContext
 
         return {
             "Batch": Batch,

{py_data_engine-0.1.2 → py_data_engine-0.2.1}/src/data_engine/application/catalog.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 from dataclasses import dataclass
 from pathlib import Path
 
-from data_engine.authoring.model import FlowValidationError
+from data_engine.core.model import FlowValidationError
 from data_engine.domain import FlowCatalogEntry, FlowCatalogLike, FlowCatalogState
 from data_engine.platform.workspace_models import WorkspacePaths
 from data_engine.services import FlowCatalogService
@@ -86,7 +86,10 @@ class FlowCatalogApplication:
         """Load discovered entries and merge them into one catalog state."""
         base = current_state or FlowCatalogState.empty()
         entries = self.flow_catalog_service.load_entries(workspace_root=workspace_root)
-        return base.with_entries(entries).with_empty_message("")
+        state = base.with_entries(entries).with_empty_message("")
+        if base.selected_flow_name is not None and state.selected_flow_name == base.selected_flow_name:
+            return state
+        return state.with_selected_flow_name(_first_grouped_entry_name(state.entries))
 
     def empty_state(
         self,
@@ -149,3 +152,11 @@ class FlowCatalogApplication:
                 error_text=message,
             )
         return FlowCatalogLoadResult(catalog_state=catalog_state, loaded=True)
+
+
+def _first_grouped_entry_name(entries: tuple[FlowCatalogEntry, ...]) -> str | None:
+    """Return the first entry name in the same grouped order used by operator surfaces."""
+    for bucket in group_cards(entries):
+        if bucket.entries:
+            return bucket.entries[0].name
+    return None

{py_data_engine-0.1.2 → py_data_engine-0.2.1}/src/data_engine/application/runtime.py

@@ -7,7 +7,7 @@ from datetime import UTC, datetime
 from typing import Any
 from collections.abc import Iterable
 
-from data_engine.authoring.model import FlowStoppedError
+from data_engine.core.model import FlowStoppedError
 from data_engine.domain import (
     DaemonLifecyclePolicy,
     DaemonStatusState,

{py_data_engine-0.1.2 → py_data_engine-0.2.1}/src/data_engine/authoring/__init__.py

@@ -10,5 +10,8 @@ __all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_fl
 def __getattr__(name: str):
     if name not in __all__:
         raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-    builder = import_module("data_engine.authoring.builder")
-    return getattr(builder, name)
+    if name in {"Flow", "discover_flows", "load_flow", "run"}:
+        module = import_module("data_engine.authoring.flow")
+        return getattr(module, name)
+    module = import_module("data_engine.core.primitives")
+    return getattr(module, name)

py_data_engine-0.2.1/src/data_engine/authoring/flow.py

@@ -0,0 +1,198 @@
+"""Flow DSL and public authoring entrypoints."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from data_engine.core.flow import Flow as _CoreFlow
+from data_engine.core.helpers import _validate_slot_name
+from data_engine.core.model import FlowValidationError
+from data_engine.core.primitives import FlowContext
+
+if TYPE_CHECKING:
+    from data_engine.authoring.services import AuthoringServices
+    from data_engine.services.flow_execution import FlowExecutionService
+    from data_engine.services.runtime_execution import RuntimeExecutionService
+
+
+def _resolve_authoring_services(
+    *,
+    authoring_services: AuthoringServices | None = None,
+    runtime_execution_service: RuntimeExecutionService | None = None,
+    flow_execution_service: FlowExecutionService | None = None,
+) -> AuthoringServices:
+    """Return one authoring collaborator bundle with explicit overrides applied."""
+    from data_engine.authoring.services import build_authoring_services, default_authoring_services
+
+    services = authoring_services or default_authoring_services()
+    if runtime_execution_service is None and flow_execution_service is None:
+        return services
+    return build_authoring_services(
+        runtime_execution_service=runtime_execution_service or services.runtime_execution_service,
+        flow_execution_service=flow_execution_service or services.flow_execution_service,
+    )
+
+
+class Flow(_CoreFlow):
+    """Public authoring flow with execution conveniences layered over core definitions."""
+
+    def run_once(
+        self,
+        *,
+        authoring_services: AuthoringServices | None = None,
+        runtime_execution_service: RuntimeExecutionService | None = None,
+    ) -> list[FlowContext]:
+        """Run this flow once and return completed runtime contexts.
+
+        Parameters
+        ----------
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        list[FlowContext]
+            One context per executed source.
+        """
+        service = _resolve_authoring_services(
+            authoring_services=authoring_services,
+            runtime_execution_service=runtime_execution_service,
+        ).runtime_execution_service
+        return service.run_once(self)
+
+    def preview(
+        self,
+        *,
+        use: str | None = None,
+        authoring_services: AuthoringServices | None = None,
+        runtime_execution_service: RuntimeExecutionService | None = None,
+    ) -> object:
+        """Run this flow in preview mode and return one preview value.
+
+        Parameters
+        ----------
+        use : str | None
+            Optional named object slot to preview instead of the final current
+            value.
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        object
+            Preview value returned by the runtime execution service.
+
+        Raises
+        ------
+        FlowValidationError
+            If preview is requested from inside a compiled flow module.
+        """
+        from data_engine.flow_modules.flow_module_loader import in_compiled_flow_module_context
+
+        if in_compiled_flow_module_context():
+            raise FlowValidationError("preview() is not available inside compiled flow modules.")
+        normalized_use = _validate_slot_name(method_name="preview", slot_name="use", value=use)
+        service = _resolve_authoring_services(
+            authoring_services=authoring_services,
+            runtime_execution_service=runtime_execution_service,
+        ).runtime_execution_service
+        return service.preview(self, use=normalized_use)
+
+    def show(self) -> object:
+        """Run this flow once and return the single final current value.
+
+        Returns
+        -------
+        object
+            Final ``context.current`` value.
+
+        Raises
+        ------
+        FlowValidationError
+            If called from a compiled flow module or the flow produces anything
+            other than one result.
+        """
+        from data_engine.flow_modules.flow_module_loader import in_compiled_flow_module_context
+
+        if in_compiled_flow_module_context():
+            raise FlowValidationError("show() is not available inside compiled flow modules.")
+        results = self.run_once()
+        if len(results) != 1:
+            raise FlowValidationError(f"show() requires exactly one result, found {len(results)}.")
+        return results[0].current
+
+    def run(
+        self,
+        *,
+        authoring_services: AuthoringServices | None = None,
+        runtime_execution_service: RuntimeExecutionService | None = None,
+    ) -> list[FlowContext]:
+        """Run this flow continuously according to its trigger.
+
+        Parameters
+        ----------
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        list[FlowContext]
+            Completed contexts collected before the runtime exits.
+        """
+        service = _resolve_authoring_services(
+            authoring_services=authoring_services,
+            runtime_execution_service=runtime_execution_service,
+        ).runtime_execution_service
+        return service.run_continuous(self)
+
+
+def load_flow(
+    name: str,
+    *,
+    data_root: Path | None = None,
+    authoring_services: AuthoringServices | None = None,
+    flow_execution_service: FlowExecutionService | None = None,
+) -> Flow:
+    """Load one code-defined flow by flow-module name."""
+    service = _resolve_authoring_services(
+        authoring_services=authoring_services,
+        flow_execution_service=flow_execution_service,
+    ).flow_execution_service
+    return service.load_flow(name, workspace_root=data_root)
+
+
+def discover_flows(
+    *,
+    data_root: Path | None = None,
+    authoring_services: AuthoringServices | None = None,
+    flow_execution_service: FlowExecutionService | None = None,
+) -> tuple[Flow, ...]:
+    """Discover and build all code-defined flows from compiled flow modules."""
+    service = _resolve_authoring_services(
+        authoring_services=authoring_services,
+        flow_execution_service=flow_execution_service,
+    ).flow_execution_service
+    return service.discover_flows(workspace_root=data_root)
+
+
+def run(
+    *flows: Flow,
+    authoring_services: AuthoringServices | None = None,
+    runtime_execution_service: RuntimeExecutionService | None = None,
+) -> list[FlowContext]:
+    """Run multiple flows with sequential execution per group and parallel groups."""
+    service = _resolve_authoring_services(
+        authoring_services=authoring_services,
+        runtime_execution_service=runtime_execution_service,
+    ).runtime_execution_service
+    return service.run_grouped_continuous(tuple(flows))
+
+
+__all__ = ["Flow", "discover_flows", "load_flow", "run"]

{py_data_engine-0.1.2 → py_data_engine-0.2.1}/src/data_engine/authoring/services.py

@@ -4,9 +4,11 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 from functools import lru_cache
+from typing import TYPE_CHECKING
 
-from data_engine.services.flow_execution import FlowExecutionService
-from data_engine.services.runtime_execution import RuntimeExecutionService
+if TYPE_CHECKING:
+    from data_engine.services.flow_execution import FlowExecutionService
+    from data_engine.services.runtime_execution import RuntimeExecutionService
 
 
 @dataclass(frozen=True)
@@ -23,6 +25,9 @@ def build_authoring_services(
     flow_execution_service: FlowExecutionService | None = None,
 ) -> AuthoringServices:
     """Build one authoring collaborator bundle with optional overrides."""
+    from data_engine.services.flow_execution import FlowExecutionService
+    from data_engine.services.runtime_execution import RuntimeExecutionService
+
     return AuthoringServices(
         runtime_execution_service=runtime_execution_service or RuntimeExecutionService(),
         flow_execution_service=flow_execution_service or FlowExecutionService(),

py_data_engine-0.2.1/src/data_engine/core/__init__.py

@@ -0,0 +1,35 @@
+"""Core flow definitions and runtime primitives."""
+
+from data_engine.core.flow import Flow
+from data_engine.core.model import FlowExecutionError, FlowStoppedError, FlowValidationError
+from data_engine.core.primitives import (
+    Batch,
+    FileRef,
+    FlowContext,
+    MirrorContext,
+    MirrorSpec,
+    SourceContext,
+    SourceMetadata,
+    StepSpec,
+    WatchSpec,
+    WorkspaceConfigContext,
+    collect_files,
+)
+
+__all__ = [
+    "Batch",
+    "FileRef",
+    "Flow",
+    "FlowContext",
+    "FlowExecutionError",
+    "FlowStoppedError",
+    "FlowValidationError",
+    "MirrorContext",
+    "MirrorSpec",
+    "SourceContext",
+    "SourceMetadata",
+    "StepSpec",
+    "WatchSpec",
+    "WorkspaceConfigContext",
+    "collect_files",
+]