py-data-engine 0.1.1__tar.gz → 0.1.3__tar.gz

{py_data_engine-0.1.1/src/py_data_engine.egg-info → py_data_engine-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-data-engine
-Version: 0.1.1
+Version: 0.1.3
 Summary: Workbook-driven workflow jobs with strict filename parsing and DuckDB-backed processing.
 Author: Data Engine contributors
 License-Expression: MIT
@@ -20,12 +20,11 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: duckdb>=1.5.0
 Requires-Dist: fastexcel>=0.19.0
+Requires-Dist: polars>=1.39.0
 Requires-Dist: pyarrow>=23.0.1
 Requires-Dist: PySide6>=6.9.0
 Requires-Dist: textual>=0.74.0
 Requires-Dist: xlsxwriter>=3.2.9
-Provides-Extra: polars
-Requires-Dist: polars>=1.39.0; extra == "polars"
 Provides-Extra: polars-lts
 Requires-Dist: polars-lts-cpu>=1.33.1; extra == "polars-lts"
 Provides-Extra: synthetic
@@ -58,10 +57,12 @@ Requires-Dist: notebook>=7.4.0; extra == "dev"
 Requires-Dist: numpy>=2.4.0; extra == "dev"
 Requires-Dist: openpyxl>=3.1.0; extra == "dev"
 Requires-Dist: pandas>=3.0.0; extra == "dev"
+Requires-Dist: pydoclint>=0.7.6; extra == "dev"
 Requires-Dist: pydocstyle>=6.3.0; extra == "dev"
 Requires-Dist: pyinstaller>=6.15.0; extra == "dev"
 Requires-Dist: pytest>=9.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=7.0.0; extra == "dev"
+Requires-Dist: ruff>=0.15.10; extra == "dev"
 Requires-Dist: Sphinx>=9.1.0; extra == "dev"
 Requires-Dist: sphinx_rtd_theme>=3.1.0; extra == "dev"
 Dynamic: license-file
@@ -86,20 +87,14 @@ Use the installer that matches your environment:
 - Windows: [INSTALL/INSTALL WINDOWS.bat](INSTALL/INSTALL%20WINDOWS.bat)
 - Windows VM / CPU-safe Polars test path: [INSTALL/INSTALL WINDOWS_VM.bat](INSTALL/INSTALL%20WINDOWS_VM.bat)
 
-The macOS and standard Windows installers install the regular Polars package. The Windows VM installer installs the `polars-lts-cpu` variant instead.
+The macOS and standard Windows installers install the normal base runtime, which now includes Polars directly.
 
 ### Manual install
 
-Data Engine now uses explicit Polars extras so you can choose the runtime package:
+Polars is now part of the regular install:
 
 ```bash
-python -m pip install -e ".[dev,polars]"
-```
-
-For the CPU-safe LTS variant:
-
-```bash
-python -m pip install -e ".[dev,polars-lts]"
+python -m pip install -e ".[dev]"
 ```
 
 Launch the GUI with:

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/README.md

@@ -18,20 +18,14 @@ Use the installer that matches your environment:
 - Windows: [INSTALL/INSTALL WINDOWS.bat](INSTALL/INSTALL%20WINDOWS.bat)
 - Windows VM / CPU-safe Polars test path: [INSTALL/INSTALL WINDOWS_VM.bat](INSTALL/INSTALL%20WINDOWS_VM.bat)
 
-The macOS and standard Windows installers install the regular Polars package. The Windows VM installer installs the `polars-lts-cpu` variant instead.
+The macOS and standard Windows installers install the normal base runtime, which now includes Polars directly.
 
 ### Manual install
 
-Data Engine now uses explicit Polars extras so you can choose the runtime package:
+Polars is now part of the regular install:
 
 ```bash
-python -m pip install -e ".[dev,polars]"
-```
-
-For the CPU-safe LTS variant:
-
-```bash
-python -m pip install -e ".[dev,polars-lts]"
+python -m pip install -e ".[dev]"
 ```
 
 Launch the GUI with:

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/pyproject.toml

@@ -11,7 +11,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "py-data-engine"
-version = "0.1.1"
+dynamic = ["version"]
 description = "Workbook-driven workflow jobs with strict filename parsing and DuckDB-backed processing."
 readme = "README.md"
 requires-python = ">=3.14"
@@ -33,6 +33,7 @@ classifiers = [
 dependencies = [
   "duckdb>=1.5.0",
   "fastexcel>=0.19.0",
+  "polars>=1.39.0",
   "pyarrow>=23.0.1",
   "PySide6>=6.9.0",
   "textual>=0.74.0",
@@ -48,9 +49,6 @@ Issues = "https://github.com/bj-data-eng/data-engine/issues"
 data-engine = "data_engine.ui.cli.app:main"
 
 [project.optional-dependencies]
-polars = [
-  "polars>=1.39.0",
-]
 polars-lts = [
   "polars-lts-cpu>=1.33.1",
 ]
@@ -89,10 +87,12 @@ dev = [
   "numpy>=2.4.0",
   "openpyxl>=3.1.0",
   "pandas>=3.0.0",
+  "pydoclint>=0.7.6",
   "pydocstyle>=6.3.0",
   "pyinstaller>=6.15.0",
   "pytest>=9.0.0",
   "pytest-cov>=7.0.0",
+  "ruff>=0.15.10",
   "Sphinx>=9.1.0",
   "sphinx_rtd_theme>=3.1.0",
 ]
@@ -116,6 +116,9 @@ data_engine = [
   "docs/html/**/*",
 ]
 
+[tool.setuptools.dynamic]
+version = {attr = "data_engine.platform.identity.APP_VERSION"}
+
 [tool.pytest.ini_options]
 pythonpath = ["src"]
 testpaths = ["tests"]

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/setup.py

@@ -12,11 +12,22 @@ from setuptools.command.sdist import sdist as _sdist
 ROOT = Path(__file__).resolve().parent
 DOCS_SOURCE_DIR = ROOT / "src" / "data_engine" / "docs" / "sphinx_source"
 DOCS_OUTPUT_DIR = ROOT / "src" / "data_engine" / "docs" / "html"
+PROJECT_AST_MAP_SCRIPT = ROOT / "src" / "data_engine" / "devtools" / "project_ast_map.py"
 
 
 def _build_packaged_docs() -> None:
     if not DOCS_SOURCE_DIR.is_dir():
         raise FileNotFoundError(f"Packaged docs source directory is missing: {DOCS_SOURCE_DIR}")
+    subprocess.run(
+        [
+            sys.executable,
+            str(PROJECT_AST_MAP_SCRIPT),
+            str(ROOT / "src" / "data_engine"),
+            "--write-docs",
+            str(DOCS_SOURCE_DIR / "guides"),
+        ],
+        check=True,
+    )
     subprocess.run(
         [
             sys.executable,

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/src/data_engine/__init__.py

@@ -5,11 +5,11 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from data_engine.authoring.builder import Batch
-    from data_engine.authoring.builder import FileRef
-    from data_engine.authoring.builder import Flow
-    from data_engine.authoring.builder import FlowContext
-    from data_engine.authoring.builder import discover_flows, load_flow, run
+    from data_engine.authoring.flow import Flow
+    from data_engine.authoring.flow import discover_flows, load_flow, run
+    from data_engine.authoring.primitives import Batch
+    from data_engine.authoring.primitives import FileRef
+    from data_engine.authoring.primitives import FlowContext
 
 __all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_flow", "run"]
 
@@ -17,13 +17,13 @@ __all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_fl
 def __getattr__(name: str):
     """Lazy-load runtime symbols so lightweight helpers can import package submodules safely."""
     if name in {"Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_flow", "run"}:
-        from data_engine.authoring.builder import Batch
-        from data_engine.authoring.builder import FileRef
-        from data_engine.authoring.builder import Flow
-        from data_engine.authoring.builder import FlowContext
-        from data_engine.authoring.builder import discover_flows
-        from data_engine.authoring.builder import load_flow
-        from data_engine.authoring.builder import run
+        from data_engine.authoring.flow import Flow
+        from data_engine.authoring.flow import discover_flows
+        from data_engine.authoring.flow import load_flow
+        from data_engine.authoring.flow import run
+        from data_engine.authoring.primitives import Batch
+        from data_engine.authoring.primitives import FileRef
+        from data_engine.authoring.primitives import FlowContext
 
         return {
             "Batch": Batch,

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/src/data_engine/authoring/__init__.py

@@ -10,5 +10,8 @@ __all__ = ["Batch", "FileRef", "Flow", "FlowContext", "discover_flows", "load_fl
 def __getattr__(name: str):
     if name not in __all__:
         raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-    builder = import_module("data_engine.authoring.builder")
-    return getattr(builder, name)
+    if name in {"Flow", "discover_flows", "load_flow", "run"}:
+        module = import_module("data_engine.authoring.flow")
+        return getattr(module, name)
+    module = import_module("data_engine.authoring.primitives")
+    return getattr(module, name)

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/src/data_engine/authoring/execution/context.py

@@ -5,12 +5,16 @@ from __future__ import annotations
 from dataclasses import dataclass
 import hashlib
 from pathlib import Path
+from typing import TYPE_CHECKING
 from uuid import uuid4
 
 from data_engine.authoring.primitives import FlowContext, MirrorContext, SourceContext, WatchSpec, WorkspaceConfigContext
 from data_engine.domain.source_state import SourceSignature
 from data_engine.domain.time import utcnow_text
 
+if TYPE_CHECKING:
+    from data_engine.authoring.flow import Flow
+
 
 @dataclass(frozen=True)
 class _QueuedJob:

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/src/data_engine/authoring/execution/continuous.py

@@ -8,6 +8,7 @@ from time import monotonic, sleep
 from typing import TYPE_CHECKING
 
 from data_engine.authoring.execution.context import _QueuedJob
+from data_engine.authoring.model import FlowStoppedError
 from data_engine.authoring.primitives import WatchSpec
 from data_engine.runtime.file_watch import PollingWatcher
 
@@ -170,7 +171,4 @@ class ContinuousRuntimeLoop:
                 entry["pending"] = True
                 entry["last_run_date"] = current_dt.date()
 
-
-from data_engine.authoring.model import FlowStoppedError
-
 __all__ = ["ContinuousRuntimeLoop"]

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/src/data_engine/authoring/execution/grouped.py

@@ -36,6 +36,7 @@ class _GroupedFlowRuntime:
         self.status_callback = status_callback
         self._runtime_ledger_service = runtime_ledger_service or default_runtime_ledger_service()
         self._runtime_ledger_factory = runtime_ledger_factory or self._runtime_ledger_service.open_runtime_ledger
+        self._owns_runtime_ledger = runtime_ledger is None
         self.runtime_ledger = runtime_ledger or self._runtime_ledger_factory()
 
     def run(self) -> list[FlowContext]:
@@ -75,26 +76,27 @@ class _GroupedFlowRuntime:
                 errors.put((group_name, exc))
                 if not self.continuous:
                     internal_runtime_stop.set()
-            finally:
+        try:
+            for group_name, group_flows in grouped.items():
+                thread = threading.Thread(target=run_group, args=(group_name, group_flows), daemon=True)
+                threads.append(thread)
+                thread.start()
+
+            for thread in threads:
+                thread.join()
+
+            if not self.continuous and not errors.empty():
+                _, exc = errors.get()
+                raise exc
+
+            ordered_results: list[FlowContext] = []
+            for group_name in grouped:
+                ordered_results.extend(results_by_group[group_name])
+            return ordered_results
+        finally:
+            if self._owns_runtime_ledger:
                 self.runtime_ledger.close()
 
-        for group_name, group_flows in grouped.items():
-            thread = threading.Thread(target=run_group, args=(group_name, group_flows), daemon=True)
-            threads.append(thread)
-            thread.start()
-
-        for thread in threads:
-            thread.join()
-
-        if not self.continuous and not errors.empty():
-            _, exc = errors.get()
-            raise exc
-
-        ordered_results: list[FlowContext] = []
-        for group_name in grouped:
-            ordered_results.extend(results_by_group[group_name])
-        return ordered_results
-
     def _grouped_flows(self) -> dict[str, tuple["Flow", ...]]:
         grouped: dict[str, list["Flow"]] = {}
         for index, flow in enumerate(self.flows):

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/src/data_engine/authoring/execution/polling.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 from collections import deque
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from data_engine.authoring.primitives import WatchSpec
 from data_engine.authoring.execution.context import _QueuedJob
@@ -11,6 +12,9 @@ from data_engine.domain.source_state import SourceSignature
 from data_engine.runtime.file_watch import PollingWatcher, iter_candidate_paths
 from data_engine.runtime.runtime_db import RuntimeLedger
 
+if TYPE_CHECKING:
+    from data_engine.authoring.flow import Flow
+
 
 class RuntimePollingSupport:
     """Own watcher creation, queueing, and stale-source detection."""

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/src/data_engine/authoring/execution/single.py

@@ -19,7 +19,6 @@ from data_engine.runtime.runtime_db import RuntimeLedger
 
 if TYPE_CHECKING:
     from data_engine.authoring.flow import Flow
-    from data_engine.authoring.primitives import StepSpec
 
 
 def _open_default_runtime_ledger() -> RuntimeLedger:

{py_data_engine-0.1.1 → py_data_engine-0.1.3}/src/data_engine/authoring/flow.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 from dataclasses import dataclass, replace
 import inspect
 from pathlib import Path
+from typing import Callable
 
 from data_engine.authoring.helpers import (
     _callable_identifier,
@@ -45,7 +46,28 @@ def _resolve_authoring_services(
 
 @dataclass(frozen=True)
 class Flow:
-    """Immutable fluent builder for generic runtime flows."""
+    """Immutable fluent builder for generic runtime flows.
+
+    Attributes
+    ----------
+    group : str
+        Display group used by the GUI, TUI, and runtime summaries.
+    name : str | None
+        Stable flow identifier. When omitted in a flow module, the module loader
+        can derive it from the module name.
+    label : str | None
+        Optional human-readable display label.
+    trigger : WatchSpec | None
+        Runtime trigger configuration.
+    mirror_spec : MirrorSpec | None
+        Mirrored output path configuration.
+    steps : tuple[StepSpec, ...]
+        Ordered callable steps to run.
+
+    Notes
+    -----
+    Builder methods return a new ``Flow`` instance, so calls can be chained.
+    """
 
     group: str
     name: str | None = None
@@ -80,6 +102,38 @@ class Flow:
         extensions: tuple[str, ...] | list[str] | set[str] | None = None,
         settle: int = 1,
     ) -> "Flow":
+        """Configure how this flow is triggered.
+
+        Parameters
+        ----------
+        mode : str
+            Trigger mode: ``"manual"``, ``"poll"``, or ``"schedule"``.
+        run_as : str
+            ``"individual"`` to run once per source file, or ``"batch"`` to run
+            once for the full source set.
+        source : str | Path | None
+            File or directory watched by poll/schedule triggers.
+        interval : str | None
+            Duration string such as ``"10s"`` or ``"5m"`` for poll intervals or
+            recurring schedules.
+        time : str | tuple[str, ...] | list[str] | set[str] | None
+            One or more ``HH:MM`` daily schedule times.
+        extensions : tuple[str, ...] | list[str] | set[str] | None
+            Optional file extensions used when discovering source files.
+        settle : int
+            Polling settle window in seconds before a changed file is queued.
+
+        Returns
+        -------
+        Flow
+            A new flow with the trigger configuration attached.
+
+        Raises
+        ------
+        FlowValidationError
+            If the trigger mode, source, schedule, or polling options are
+            inconsistent.
+        """
         normalized_mode = str(mode).strip().lower()
         if normalized_mode not in {"manual", "poll", "schedule"}:
             raise FlowValidationError("watch() mode must be one of 'manual', 'poll', or 'schedule'.")
@@ -157,17 +211,53 @@ class Flow:
         )
 
     def mirror(self, *, root: str | Path) -> "Flow":
-        """Bind a mirrored output namespace rooted at one directory."""
+        """Bind a mirrored output namespace rooted at one directory.
+
+        Parameters
+        ----------
+        root : str | Path
+            Directory used by ``context.mirror`` helpers when writing outputs.
+
+        Returns
+        -------
+        Flow
+            A new flow with mirror output helpers enabled.
+        """
         return self._clone(mirror_spec=MirrorSpec(root=_resolve_flow_path(root)))
 
     def step(
         self,
-        fn,
+        fn: Callable[[FlowContext], object],
         *,
         use: str | None = None,
         save_as: str | None = None,
         label: str | None = None,
     ) -> "Flow":
+        """Append one callable step to the flow.
+
+        Parameters
+        ----------
+        fn : Callable[[FlowContext], object]
+            Callable that accepts a single ``FlowContext`` and returns the next
+            value for ``context.current``.
+        use : str | None
+            Optional named object slot to load into ``context.current`` before
+            the step runs.
+        save_as : str | None
+            Optional named object slot to store the step result in.
+        label : str | None
+            Optional display label for logs and UI step summaries.
+
+        Returns
+        -------
+        Flow
+            A new flow with the step appended.
+
+        Raises
+        ------
+        FlowValidationError
+            If ``fn`` is not callable or does not accept exactly one argument.
+        """
         if not callable(fn):
             raise FlowValidationError("step() fn must be callable")
         normalized_use = _validate_slot_name(method_name="step", slot_name="use", value=use)
@@ -188,12 +278,36 @@ class Flow:
 
     def map(
         self,
-        fn,
+        fn: Callable[..., object],
         *,
         use: str | None = None,
         save_as: str | None = None,
         label: str | None = None,
     ) -> "Flow":
+        """Append a step that maps a callable over the current iterable value.
+
+        Parameters
+        ----------
+        fn : Callable[..., object]
+            Callable accepting either ``item`` or ``context, item``.
+        use : str | None
+            Optional named object slot to map instead of the current value.
+        save_as : str | None
+            Optional named object slot to store the mapped ``Batch`` result in.
+        label : str | None
+            Optional display label for logs and UI step summaries.
+
+        Returns
+        -------
+        Flow
+            A new flow with the mapping step appended.
+
+        Raises
+        ------
+        FlowValidationError
+            If ``fn`` is not callable, has an unsupported signature, or the
+            mapped current value is not iterable.
+        """
         if not callable(fn):
             raise FlowValidationError("map() fn must be callable")
         normalized_use = _validate_slot_name(method_name="map", slot_name="use", value=use)
@@ -241,6 +355,28 @@ class Flow:
         save_as: str | None = None,
         label: str | None = None,
     ) -> "Flow":
+        """Append a step that collects source files into a ``Batch``.
+
+        Parameters
+        ----------
+        extensions : tuple[str, ...] | list[str] | set[str]
+            File extensions to include, such as ``(".xlsx", ".csv")``.
+        root : str | Path | None
+            Optional search root. Defaults to the active source root.
+        recursive : bool
+            Whether to search child directories.
+        use : str | None
+            Optional named object slot to load before collecting.
+        save_as : str | None
+            Optional named object slot to store the collected batch in.
+        label : str | None
+            Optional display label for logs and UI step summaries.
+
+        Returns
+        -------
+        Flow
+            A new flow with the collection step appended.
+        """
         normalized_use = _validate_slot_name(method_name="collect", slot_name="use", value=use)
         normalized_save_as = _validate_slot_name(method_name="collect", slot_name="save_as", value=save_as)
         normalized_label = _validate_label(method_name="collect", label=label)
@@ -253,12 +389,30 @@ class Flow:
 
     def step_each(
         self,
-        fn,
+        fn: Callable[..., object],
         *,
         use: str | None = None,
         save_as: str | None = None,
         label: str | None = None,
     ) -> "Flow":
+        """Alias for ``map`` that reads naturally in step chains.
+
+        Parameters
+        ----------
+        fn : Callable[..., object]
+            Callable accepting either ``item`` or ``context, item``.
+        use : str | None
+            Optional named object slot to map instead of the current value.
+        save_as : str | None
+            Optional named object slot to store the mapped ``Batch`` result in.
+        label : str | None
+            Optional display label for logs and UI step summaries.
+
+        Returns
+        -------
+        Flow
+            A new flow with the per-item step appended.
+        """
         return self.map(fn, use=use, save_as=save_as, label=label)
 
     @property
@@ -273,6 +427,20 @@ class Flow:
         authoring_services: AuthoringServices | None = None,
         runtime_execution_service: RuntimeExecutionService | None = None,
     ) -> list[FlowContext]:
+        """Run this flow once and return completed runtime contexts.
+
+        Parameters
+        ----------
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        list[FlowContext]
+            One context per executed source.
+        """
         service = _resolve_authoring_services(
             authoring_services=authoring_services,
             runtime_execution_service=runtime_execution_service,
@@ -285,7 +453,29 @@ class Flow:
         use: str | None = None,
         authoring_services: AuthoringServices | None = None,
         runtime_execution_service: RuntimeExecutionService | None = None,
-    ):
+    ) -> object:
+        """Run this flow in preview mode and return one preview value.
+
+        Parameters
+        ----------
+        use : str | None
+            Optional named object slot to preview instead of the final current
+            value.
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        object
+            Preview value returned by the runtime execution service.
+
+        Raises
+        ------
+        FlowValidationError
+            If preview is requested from inside a compiled flow module.
+        """
         if in_compiled_flow_module_context():
             raise FlowValidationError("preview() is not available inside compiled flow modules.")
         normalized_use = _validate_slot_name(method_name="preview", slot_name="use", value=use)
@@ -295,7 +485,20 @@ class Flow:
         ).runtime_execution_service
         return service.preview(self, use=normalized_use)
 
-    def show(self):
+    def show(self) -> object:
+        """Run this flow once and return the single final current value.
+
+        Returns
+        -------
+        object
+            Final ``context.current`` value.
+
+        Raises
+        ------
+        FlowValidationError
+            If called from a compiled flow module or the flow produces anything
+            other than one result.
+        """
         if in_compiled_flow_module_context():
             raise FlowValidationError("show() is not available inside compiled flow modules.")
         results = self.run_once()
@@ -309,6 +512,20 @@ class Flow:
         authoring_services: AuthoringServices | None = None,
         runtime_execution_service: RuntimeExecutionService | None = None,
     ) -> list[FlowContext]:
+        """Run this flow continuously according to its trigger.
+
+        Parameters
+        ----------
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
+        runtime_execution_service : RuntimeExecutionService | None
+            Optional runtime execution service override.
+
+        Returns
+        -------
+        list[FlowContext]
+            Completed contexts collected before the runtime exits.
+        """
         service = _resolve_authoring_services(
             authoring_services=authoring_services,
             runtime_execution_service=runtime_execution_service,