py-data-engine 0.1.2__tar.gz → 0.1.3__tar.gz

{py_data_engine-0.1.2/src/py_data_engine.egg-info → py_data_engine-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-data-engine
-Version: 0.1.2
+Version: 0.1.3
 Summary: Workbook-driven workflow jobs with strict filename parsing and DuckDB-backed processing.
 Author: Data Engine contributors
 License-Expression: MIT
@@ -57,10 +57,12 @@ Requires-Dist: notebook>=7.4.0; extra == "dev"
 Requires-Dist: numpy>=2.4.0; extra == "dev"
 Requires-Dist: openpyxl>=3.1.0; extra == "dev"
 Requires-Dist: pandas>=3.0.0; extra == "dev"
+Requires-Dist: pydoclint>=0.7.6; extra == "dev"
 Requires-Dist: pydocstyle>=6.3.0; extra == "dev"
 Requires-Dist: pyinstaller>=6.15.0; extra == "dev"
 Requires-Dist: pytest>=9.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=7.0.0; extra == "dev"
+Requires-Dist: ruff>=0.15.10; extra == "dev"
 Requires-Dist: Sphinx>=9.1.0; extra == "dev"
 Requires-Dist: sphinx_rtd_theme>=3.1.0; extra == "dev"
 Dynamic: license-file

{py_data_engine-0.1.2 → py_data_engine-0.1.3}/pyproject.toml

@@ -11,7 +11,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "py-data-engine"
-version = "0.1.2"
+dynamic = ["version"]
 description = "Workbook-driven workflow jobs with strict filename parsing and DuckDB-backed processing."
 readme = "README.md"
 requires-python = ">=3.14"
@@ -87,10 +87,12 @@ dev = [
   "numpy>=2.4.0",
   "openpyxl>=3.1.0",
   "pandas>=3.0.0",
+  "pydoclint>=0.7.6",
   "pydocstyle>=6.3.0",
   "pyinstaller>=6.15.0",
   "pytest>=9.0.0",
   "pytest-cov>=7.0.0",
+  "ruff>=0.15.10",
   "Sphinx>=9.1.0",
   "sphinx_rtd_theme>=3.1.0",
 ]
@@ -114,6 +116,9 @@ data_engine = [
   "docs/html/**/*",
 ]
 
+[tool.setuptools.dynamic]
+version = {attr = "data_engine.platform.identity.APP_VERSION"}
+
 [tool.pytest.ini_options]
 pythonpath = ["src"]
 testpaths = ["tests"]

{py_data_engine-0.1.2 → py_data_engine-0.1.3}/setup.py

@@ -12,11 +12,22 @@ from setuptools.command.sdist import sdist as _sdist
 ROOT = Path(__file__).resolve().parent
 DOCS_SOURCE_DIR = ROOT / "src" / "data_engine" / "docs" / "sphinx_source"
 DOCS_OUTPUT_DIR = ROOT / "src" / "data_engine" / "docs" / "html"
+PROJECT_AST_MAP_SCRIPT = ROOT / "src" / "data_engine" / "devtools" / "project_ast_map.py"
 
 
 def _build_packaged_docs() -> None:
     if not DOCS_SOURCE_DIR.is_dir():
         raise FileNotFoundError(f"Packaged docs source directory is missing: {DOCS_SOURCE_DIR}")
+    subprocess.run(
+        [
+            sys.executable,
+            str(PROJECT_AST_MAP_SCRIPT),
+            str(ROOT / "src" / "data_engine"),
+            "--write-docs",
+            str(DOCS_SOURCE_DIR / "guides"),
+        ],
+        check=True,
+    )
     subprocess.run(
         [
             sys.executable,

{py_data_engine-0.1.2 → py_data_engine-0.1.3}/src/data_engine/__init__.py

@@ -5,11 +5,11 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from data_engine.authoring.builder import Batch
-    from data_engine.authoring.builder import FileRef
-    from data_engine.authoring.builder import Flow
-    from data_engine.authoring.builder import FlowContext
-    from data_engine.authoring.builder import discover_flows, load_flow, run
+    from data_engine.authoring.flow import Flow
+    from data_engine.authoring.flow import discover_flows, load_flow, run
+    from data_engine.authoring.primitives import Batch
+    from data_engine.authoring.primitives import FileRef
+    from data_engine.authoring.primitives import FlowContext
 
 __all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_flow", "run"]
 
@@ -17,13 +17,13 @@ __all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_fl
 def __getattr__(name: str):
     """Lazy-load runtime symbols so lightweight helpers can import package submodules safely."""
     if name in {"Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_flow", "run"}:
-        from data_engine.authoring.builder import Batch
-        from data_engine.authoring.builder import FileRef
-        from data_engine.authoring.builder import Flow
-        from data_engine.authoring.builder import FlowContext
-        from data_engine.authoring.builder import discover_flows
-        from data_engine.authoring.builder import load_flow
-        from data_engine.authoring.builder import run
+        from data_engine.authoring.flow import Flow
+        from data_engine.authoring.flow import discover_flows
+        from data_engine.authoring.flow import load_flow
+        from data_engine.authoring.flow import run
+        from data_engine.authoring.primitives import Batch
+        from data_engine.authoring.primitives import FileRef
+        from data_engine.authoring.primitives import FlowContext
 
         return {
             "Batch": Batch,

{py_data_engine-0.1.2 → py_data_engine-0.1.3}/src/data_engine/authoring/__init__.py

@@ -10,5 +10,8 @@ __all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_fl
 def __getattr__(name: str):
     if name not in __all__:
         raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-    builder = import_module("data_engine.authoring.builder")
-    return getattr(builder, name)
+    if name in {"Flow", "discover_flows", "load_flow", "run"}:
+        module = import_module("data_engine.authoring.flow")
+        return getattr(module, name)
+    module = import_module("data_engine.authoring.primitives")
+    return getattr(module, name)

{py_data_engine-0.1.2 → py_data_engine-0.1.3}/src/data_engine/authoring/execution/context.py

@@ -5,12 +5,16 @@ from __future__ import annotations
 from dataclasses import dataclass
 import hashlib
 from pathlib import Path
+from typing import TYPE_CHECKING
 from uuid import uuid4
 
 from data_engine.authoring.primitives import FlowContext, MirrorContext, SourceContext, WatchSpec, WorkspaceConfigContext
 from data_engine.domain.source_state import SourceSignature
 from data_engine.domain.time import utcnow_text
 
+if TYPE_CHECKING:
+    from data_engine.authoring.flow import Flow
+
 
 @dataclass(frozen=True)
 class _QueuedJob:

{py_data_engine-0.1.2 → py_data_engine-0.1.3}/src/data_engine/authoring/execution/continuous.py

@@ -8,6 +8,7 @@ from time import monotonic, sleep
 from typing import TYPE_CHECKING
 
 from data_engine.authoring.execution.context import _QueuedJob
+from data_engine.authoring.model import FlowStoppedError
 from data_engine.authoring.primitives import WatchSpec
 from data_engine.runtime.file_watch import PollingWatcher
 
@@ -170,7 +171,4 @@ class ContinuousRuntimeLoop:
                 entry["pending"] = True
                 entry["last_run_date"] = current_dt.date()
 
-
-from data_engine.authoring.model import FlowStoppedError
-
 __all__ = ["ContinuousRuntimeLoop"]

{py_data_engine-0.1.2 → py_data_engine-0.1.3}/src/data_engine/authoring/execution/polling.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 from collections import deque
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from data_engine.authoring.primitives import WatchSpec
 from data_engine.authoring.execution.context import _QueuedJob
@@ -11,6 +12,9 @@ from data_engine.domain.source_state import SourceSignature
 from data_engine.runtime.file_watch import PollingWatcher, iter_candidate_paths
 from data_engine.runtime.runtime_db import RuntimeLedger
 
+if TYPE_CHECKING:
+    from data_engine.authoring.flow import Flow
+
 
 class RuntimePollingSupport:
     """Own watcher creation, queueing, and stale-source detection."""

{py_data_engine-0.1.2 → py_data_engine-0.1.3}/src/data_engine/authoring/execution/single.py

@@ -19,7 +19,6 @@ from data_engine.runtime.runtime_db import RuntimeLedger
 
 if TYPE_CHECKING:
     from data_engine.authoring.flow import Flow
-    from data_engine.authoring.primitives import StepSpec
 
 
 def _open_default_runtime_ledger() -> RuntimeLedger:

{py_data_engine-0.1.2 → py_data_engine-0.1.3}/src/data_engine/authoring/flow.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 from dataclasses import dataclass, replace
 import inspect
 from pathlib import Path
+from typing import Callable
 
 from data_engine.authoring.helpers import (
     _callable_identifier,
@@ -45,7 +46,28 @@ def _resolve_authoring_services(
 
 @dataclass(frozen=True)
 class Flow:
-    """Immutable fluent builder for generic runtime flows."""
+    """Immutable fluent builder for generic runtime flows.
+
+    Attributes
+    ----------
+    group : str
+        Display group used by the GUI, TUI, and runtime summaries.
+    name : str | None
+        Stable flow identifier. When omitted in a flow module, the module loader
+        can derive it from the module name.
+    label : str | None
+        Optional human-readable display label.
+    trigger : WatchSpec | None
+        Runtime trigger configuration.
+    mirror_spec : MirrorSpec | None
+        Mirrored output path configuration.
+    steps : tuple[StepSpec, ...]
+        Ordered callable steps to run.
+
+    Notes
+    -----
+    Builder methods return a new ``Flow`` instance, so calls can be chained.
+    """
 
     group: str
     name: str | None = None
@@ -80,6 +102,38 @@ class Flow:
         extensions: tuple[str, ...] | list[str] | set[str] | None = None,
         settle: int = 1,
     ) -> "Flow":
+        """Configure how this flow is triggered.
+
+        Parameters
+        ----------
+        mode : str
+            Trigger mode: ``"manual"``, ``"poll"``, or ``"schedule"``.
+        run_as : str
+            ``"individual"`` to run once per source file, or ``"batch"`` to run
+            once for the full source set.
+        source : str | Path | None
+            File or directory watched by poll/schedule triggers.
+        interval : str | None
+            Duration string such as ``"10s"`` or ``"5m"`` for poll intervals or
+            recurring schedules.
+        time : str | tuple[str, ...] | list[str] | set[str] | None
+            One or more ``HH:MM`` daily schedule times.
+        extensions : tuple[str, ...] | list[str] | set[str] | None
+            Optional file extensions used when discovering source files.
+        settle : int
+            Polling settle window in seconds before a changed file is queued.
+
+        Returns
+        -------
+        Flow
+            A new flow with the trigger configuration attached.
+
+        Raises
+        ------
+        FlowValidationError
+            If the trigger mode, source, schedule, or polling options are
+            inconsistent.
+        """
         normalized_mode = str(mode).strip().lower()
         if normalized_mode not in {"manual", "poll", "schedule"}:
             raise FlowValidationError("watch() mode must be one of 'manual', 'poll', or 'schedule'.")
@@ -157,17 +211,53 @@ class Flow:
         )
 
     def mirror(self, *, root: str | Path) -> "Flow":
-        """Bind a mirrored output namespace rooted at one directory."""
+        """Bind a mirrored output namespace rooted at one directory.
+
+        Parameters
+        ----------
+        root : str | Path
+            Directory used by ``context.mirror`` helpers when writing outputs.
+
+        Returns
+        -------
+        Flow
+            A new flow with mirror output helpers enabled.
+        """
         return self._clone(mirror_spec=MirrorSpec(root=_resolve_flow_path(root)))
 
     def step(
         self,
-        fn,
+        fn: Callable[[FlowContext], object],
         *,
         use: str | None = None,
         save_as: str | None = None,
         label: str | None = None,
     ) -> "Flow":
+        """Append one callable step to the flow.
+
+        Parameters
+        ----------
+        fn : Callable[[FlowContext], object]
+            Callable that accepts a single ``FlowContext`` and returns the next
+            value for ``context.current``.
+        use : str | None
+            Optional named object slot to load into ``context.current`` before
+            the step runs.
+        save_as : str | None
+            Optional named object slot to store the step result in.
+        label : str | None
+            Optional display label for logs and UI step summaries.
+
+        Returns
+        -------
+        Flow
+            A new flow with the step appended.
+
+        Raises
+        ------
+        FlowValidationError
+            If ``fn`` is not callable or does not accept exactly one argument.
+        """
         if not callable(fn):
             raise FlowValidationError("step() fn must be callable")
         normalized_use = _validate_slot_name(method_name="step", slot_name="use", value=use)
@@ -188,12 +278,36 @@ class Flow:
 
     def map(
         self,
-        fn,
+        fn: Callable[..., object],
         *,
         use: str | None = None,
         save_as: str | None = None,
         label: str | None = None,
     ) -> "Flow":
+        """Append a step that maps a callable over the current iterable value.
+
+        Parameters
+        ----------
+        fn : Callable[..., object]
+            Callable accepting either ``item`` or ``context, item``.
+        use : str | None
+            Optional named object slot to map instead of the current value.
+        save_as : str | None
+            Optional named object slot to store the mapped ``Batch`` result in.
+        label : str | None
+            Optional display label for logs and UI step summaries.
+
+        Returns
+        -------
+        Flow
+            A new flow with the mapping step appended.
+
+        Raises
+        ------
+        FlowValidationError
+            If ``fn`` is not callable, has an unsupported signature, or the
+            mapped current value is not iterable.
+        """
         if not callable(fn):
             raise FlowValidationError("map() fn must be callable")
         normalized_use = _validate_slot_name(method_name="map", slot_name="use", value=use)
@@ -241,6 +355,28 @@ class Flow:
         save_as: str | None = None,
         label: str | None = None,
     ) -> "Flow":
+        """Append a step that collects source files into a ``Batch``.
+
+        Parameters
+        ----------
+        extensions : tuple[str, ...] | list[str] | set[str]
+            File extensions to include, such as ``(".xlsx", ".csv")``.
+        root : str | Path | None
+            Optional search root. Defaults to the active source root.
+        recursive : bool
+            Whether to search child directories.
+        use : str | None
+            Optional named object slot to load before collecting.
+        save_as : str | None
+            Optional named object slot to store the collected batch in.
+        label : str | None
+            Optional display label for logs and UI step summaries.
+
+        Returns
+        -------
+        Flow
+            A new flow with the collection step appended.
+        """
         normalized_use = _validate_slot_name(method_name="collect", slot_name="use", value=use)
         normalized_save_as = _validate_slot_name(method_name="collect", slot_name="save_as", value=save_as)
         normalized_label = _validate_label(method_name="collect", label=label)
@@ -253,12 +389,30 @@ class Flow:
 
     def step_each(
         self,
-        fn,
+        fn: Callable[..., object],
         *,
         use: str | None = None,
         save_as: str | None = None,
         label: str | None = None,
     ) -> "Flow":
+        """Alias for ``map`` that reads naturally in step chains.
+
+        Parameters
+        ----------
+        fn : Callable[..., object]
+            Callable accepting either ``item`` or ``context, item``.
+        use : str | None
+            Optional named object slot to map instead of the current value.
+        save_as : str | None
+            Optional named object slot to store the mapped ``Batch`` result in.
+        label : str | None
+            Optional display label for logs and UI step summaries.
+
+        Returns
+        -------
+        Flow
+            A new flow with the per-item step appended.
+        """
         return self.map(fn, use=use, save_as=save_as, label=label)
 
     @property
@@ -273,6 +427,20 @@ class Flow:
         authoring_services: AuthoringServices | None = None,
         runtime_execution_service: RuntimeExecutionService | None = None,
     ) -> list[FlowContext]:
+        """Run this flow once and return completed runtime contexts.
+
+        Parameters
+        ----------
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        list[FlowContext]
+            One context per executed source.
+        """
         service = _resolve_authoring_services(
             authoring_services=authoring_services,
             runtime_execution_service=runtime_execution_service,
@@ -285,7 +453,29 @@ class Flow:
         use: str | None = None,
         authoring_services: AuthoringServices | None = None,
         runtime_execution_service: RuntimeExecutionService | None = None,
-    ):
+    ) -> object:
+        """Run this flow in preview mode and return one preview value.
+
+        Parameters
+        ----------
+        use : str | None
+            Optional named object slot to preview instead of the final current
+            value.
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        object
+            Preview value returned by the runtime execution service.
+
+        Raises
+        ------
+        FlowValidationError
+            If preview is requested from inside a compiled flow module.
+        """
         if in_compiled_flow_module_context():
             raise FlowValidationError("preview() is not available inside compiled flow modules.")
         normalized_use = _validate_slot_name(method_name="preview", slot_name="use", value=use)
@@ -295,7 +485,20 @@ class Flow:
         ).runtime_execution_service
         return service.preview(self, use=normalized_use)
 
-    def show(self):
+    def show(self) -> object:
+        """Run this flow once and return the single final current value.
+
+        Returns
+        -------
+        object
+            Final ``context.current`` value.
+
+        Raises
+        ------
+        FlowValidationError
+            If called from a compiled flow module or the flow produces anything
+            other than one result.
+        """
         if in_compiled_flow_module_context():
             raise FlowValidationError("show() is not available inside compiled flow modules.")
         results = self.run_once()
@@ -309,6 +512,20 @@ class Flow:
         authoring_services: AuthoringServices | None = None,
         runtime_execution_service: RuntimeExecutionService | None = None,
     ) -> list[FlowContext]:
+        """Run this flow continuously according to its trigger.
+
+        Parameters
+        ----------
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        list[FlowContext]
+            Completed contexts collected before the runtime exits.
+        """
         service = _resolve_authoring_services(
             authoring_services=authoring_services,
             runtime_execution_service=runtime_execution_service,

{py_data_engine-0.1.2 → py_data_engine-0.1.3}/src/data_engine/authoring/primitives.py

@@ -276,7 +276,27 @@ class SourceContext:
 
 @dataclass
 class FlowContext:
-    """Mutable runtime state shared across steps during one flow execution."""
+    """Mutable runtime state shared across steps during one flow execution.
+
+    Attributes
+    ----------
+    flow_name : str
+        Name of the flow currently executing.
+    group : str
+        Display group for the active flow.
+    source : SourceContext | None
+        Source-file path helpers when the flow is bound to a source.
+    mirror : MirrorContext | None
+        Output path helpers when the flow configured ``Flow.mirror(...)``.
+    current : object | None
+        Value passed from the previous step to the current step.
+    objects : dict[str, object]
+        Named values saved by steps with ``save_as=``.
+    metadata : dict[str, object]
+        Runtime metadata such as run ids and source hashes.
+    config : WorkspaceConfigContext
+        Workspace-local TOML config accessor.
+    """
 
     flow_name: str
     group: str
@@ -288,7 +308,14 @@ class FlowContext:
     config: WorkspaceConfigContext = field(default_factory=WorkspaceConfigContext)
 
     def source_metadata(self) -> SourceMetadata | None:
-        """Return filesystem metadata for the current source file when available."""
+        """Return filesystem metadata for the current source file when available.
+
+        Returns
+        -------
+        SourceMetadata | None
+            File path, name, size, and modified timestamp, or ``None`` when the
+            runtime is not executing a concrete source file.
+        """
         source_path = self.source.path if self.source is not None else None
         if source_path is None:
             return None
@@ -301,7 +328,24 @@ class FlowContext:
         )
 
     def database(self, name: str | Path) -> Path:
-        """Return a write-ready path beneath the authored workspace databases directory."""
+        """Return a write-ready path beneath the workspace databases directory.
+
+        Parameters
+        ----------
+        name : str | Path
+            Relative database file name, for example ``"warehouse.duckdb"``.
+
+        Returns
+        -------
+        Path
+            Absolute path under ``<workspace>/databases`` with parent
+            directories created.
+
+        Raises
+        ------
+        FlowValidationError
+            If no authored workspace is active or ``name`` is empty or absolute.
+        """
         if self.config.workspace_root is None:
             raise FlowValidationError("context.database() is only available for authored workspace flows.")
         candidate = Path(name)