py-data-engine 0.3.11__tar.gz → 0.4.0__tar.gz

py_data_engine-0.4.0/PKG-INFO

@@ -0,0 +1,230 @@
+Metadata-Version: 2.4
+Name: py-data-engine
+Version: 0.4.0
+Summary: GUI orchestrator for python-based dataframe transform pipelines.
+Author: Data Engine contributors
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/bj-data-eng/data-engine
+Project-URL: Repository, https://github.com/bj-data-eng/data-engine
+Project-URL: Issues, https://github.com/bj-data-eng/data-engine/issues
+Keywords: duckdb,excel,parquet,flow,polars,notebook
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database
+Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: APScheduler==3.11.3
+Requires-Dist: duckdb==1.5.4
+Requires-Dist: fastexcel==0.20.2
+Requires-Dist: numpy==2.5.1
+Requires-Dist: openpyxl==3.1.5
+Requires-Dist: polars==1.42.1
+Requires-Dist: pyarrow==25.0.0
+Requires-Dist: PySide6==6.11.1
+Requires-Dist: xlsxwriter==3.2.9
+Provides-Extra: synthetic
+Requires-Dist: openpyxl==3.1.5; extra == "synthetic"
+Requires-Dist: pandas==3.0.3; extra == "synthetic"
+Provides-Extra: notebook
+Requires-Dist: ipykernel==7.3.0; extra == "notebook"
+Requires-Dist: ipython==9.15.0; extra == "notebook"
+Requires-Dist: jupyterlab==4.6.1; extra == "notebook"
+Requires-Dist: notebook==7.6.0; extra == "notebook"
+Provides-Extra: docs
+Requires-Dist: myst-parser==5.1.0; extra == "docs"
+Requires-Dist: Sphinx==9.1.0; extra == "docs"
+Requires-Dist: sphinx_rtd_theme==3.1.0; extra == "docs"
+Provides-Extra: package
+Requires-Dist: build==1.5.1; extra == "package"
+Requires-Dist: pyinstaller==6.21.0; extra == "package"
+Requires-Dist: twine==6.2.0; extra == "package"
+Provides-Extra: test
+Requires-Dist: openpyxl==3.1.5; extra == "test"
+Requires-Dist: pandas==3.0.3; extra == "test"
+Requires-Dist: pydocstyle==6.3.0; extra == "test"
+Requires-Dist: pytest==9.1.1; extra == "test"
+Requires-Dist: pytest-cov==7.1.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: ipykernel==7.3.0; extra == "dev"
+Requires-Dist: ipython==9.15.0; extra == "dev"
+Requires-Dist: jupyterlab==4.6.1; extra == "dev"
+Requires-Dist: myst-parser==5.1.0; extra == "dev"
+Requires-Dist: notebook==7.6.0; extra == "dev"
+Requires-Dist: numpy==2.5.1; extra == "dev"
+Requires-Dist: openpyxl==3.1.5; extra == "dev"
+Requires-Dist: pandas==3.0.3; extra == "dev"
+Requires-Dist: pydoclint==0.9.1; extra == "dev"
+Requires-Dist: pydocstyle==6.3.0; extra == "dev"
+Requires-Dist: pyinstaller==6.21.0; extra == "dev"
+Requires-Dist: pytest==9.1.1; extra == "dev"
+Requires-Dist: pytest-cov==7.1.0; extra == "dev"
+Requires-Dist: ruff==0.15.21; extra == "dev"
+Requires-Dist: Sphinx==9.1.0; extra == "dev"
+Requires-Dist: sphinx_rtd_theme==3.1.0; extra == "dev"
+Requires-Dist: viztracer==1.1.1; extra == "dev"
+Dynamic: license-file
+
+# Data Engine
+
+Data Engine is a GUI orchestrator for Python dataframe workflows. It lets you
+author flows as plain Python modules, run them manually or automatically, and
+inspect parquet-first outputs from the desktop app.
+
+The runtime is built around:
+
+- workspace-based flow discovery
+- manual, poll, and schedule execution modes
+- Polars and DuckDB-friendly flow steps
+- mirrored output paths for source-driven runs
+- saved run, log, and dataframe inspection state
+- a desktop operator surface
+
+## Install
+
+Use the installer for your environment:
+
+- macOS: [INSTALL/INSTALL MAC.command](INSTALL/INSTALL%20MAC.command)
+- Windows: [INSTALL/INSTALL WINDOWS.bat](INSTALL/INSTALL%20WINDOWS.bat)
+
+For local development:
+
+```bash
+python -m pip install --constraint requirements/constraints.txt -e ".[dev]"
+```
+
+For a published package install:
+
+```bash
+python -m pip install py-data-engine
+```
+
+Data Engine requires Python `>=3.14`.
+
+Dependency pinning, constrained installs, and hash-locked runtime installs are
+documented in [SECURITY.md](SECURITY.md).
+
+## Start
+
+Desktop GUI:
+
+```bash
+data-engine start gui
+```
+
+Headless commands:
+
+```bash
+data-engine list
+data-engine show example_summary
+data-engine run --once example_summary
+data-engine run
+```
+
+## Minimal Flow
+
+```python
+from data_engine import Flow
+import polars as pl
+
+
+def read_docs(context):
+    return pl.read_excel(context.source.path)
+
+
+def keep_open(context):
+    return context.current.filter(pl.col("status") == "OPEN")
+
+
+def write_parquet(context):
+    output = context.mirror.with_suffix(".parquet")
+    context.current.write_parquet(output)
+    return output
+
+
+def build():
+    return (
+        Flow(group="Docs")
+        .watch(
+            mode="poll",
+            source="../../../example_data/Input/docs_flat",
+            interval="5s",
+            extensions=[".xlsx", ".xls", ".xlsm"],
+            settle=1,
+        )
+        .mirror(root="../../../example_data/Output/example_mirror")
+        .step(read_docs, save_as="raw_df")
+        .step(keep_open, use="raw_df", save_as="filtered_df")
+        .step(write_parquet, use="filtered_df")
+    )
+```
+
+Each authored flow module exports `build() -> Flow`. The module filename is the
+flow identity.
+
+## Workspaces
+
+Data Engine discovers workspaces from a collection root. Each workspace keeps
+authored flows under:
+
+```text
+workspaces/<workspace_id>/flow_modules/
+```
+
+Shared workspace state lives under:
+
+```text
+workspaces/<workspace_id>/.workspace_state/
+```
+
+Machine-local runtime artifacts are stored outside the authored workspace.
+
+## Useful APIs
+
+```python
+from data_engine import Flow, FlowContext, discover_flows, load_flow, run
+```
+
+Common `Flow` methods:
+
+- `.watch(...)`
+- `.mirror(...)`
+- `.date_range_input(...)`
+- `.step(...)`
+- `.collect(...)`
+- `.map(...)`
+- `.step_each(...)`
+- `.preview(...)`
+- `.run_once()`
+- `.run()`
+
+Common `FlowContext` values:
+
+- `context.source`
+- `context.mirror`
+- `context.current`
+- `context.objects`
+- `context.metadata`
+- `context.database("analytics.duckdb")`
+- `context.template("reports/base.xlsx")`
+- `context.debug`
+
+The full authoring guide and helper reference live in
+`src/data_engine/docs/sphinx_source/guides/`.
+
+## Testing
+
+```bash
+python -m pytest -q
+python -m build
+python -m twine check dist/*
+```
+
+## Status
+
+This project is pre-alpha. Internal architecture is still moving quickly, and
+backwards compatibility is not a current goal.

py_data_engine-0.4.0/README.md

@@ -0,0 +1,159 @@
+# Data Engine
+
+Data Engine is a GUI orchestrator for Python dataframe workflows. It lets you
+author flows as plain Python modules, run them manually or automatically, and
+inspect parquet-first outputs from the desktop app.
+
+The runtime is built around:
+
+- workspace-based flow discovery
+- manual, poll, and schedule execution modes
+- Polars and DuckDB-friendly flow steps
+- mirrored output paths for source-driven runs
+- saved run, log, and dataframe inspection state
+- a desktop operator surface
+
+## Install
+
+Use the installer for your environment:
+
+- macOS: [INSTALL/INSTALL MAC.command](INSTALL/INSTALL%20MAC.command)
+- Windows: [INSTALL/INSTALL WINDOWS.bat](INSTALL/INSTALL%20WINDOWS.bat)
+
+For local development:
+
+```bash
+python -m pip install --constraint requirements/constraints.txt -e ".[dev]"
+```
+
+For a published package install:
+
+```bash
+python -m pip install py-data-engine
+```
+
+Data Engine requires Python `>=3.14`.
+
+Dependency pinning, constrained installs, and hash-locked runtime installs are
+documented in [SECURITY.md](SECURITY.md).
+
+## Start
+
+Desktop GUI:
+
+```bash
+data-engine start gui
+```
+
+Headless commands:
+
+```bash
+data-engine list
+data-engine show example_summary
+data-engine run --once example_summary
+data-engine run
+```
+
+## Minimal Flow
+
+```python
+from data_engine import Flow
+import polars as pl
+
+
+def read_docs(context):
+    return pl.read_excel(context.source.path)
+
+
+def keep_open(context):
+    return context.current.filter(pl.col("status") == "OPEN")
+
+
+def write_parquet(context):
+    output = context.mirror.with_suffix(".parquet")
+    context.current.write_parquet(output)
+    return output
+
+
+def build():
+    return (
+        Flow(group="Docs")
+        .watch(
+            mode="poll",
+            source="../../../example_data/Input/docs_flat",
+            interval="5s",
+            extensions=[".xlsx", ".xls", ".xlsm"],
+            settle=1,
+        )
+        .mirror(root="../../../example_data/Output/example_mirror")
+        .step(read_docs, save_as="raw_df")
+        .step(keep_open, use="raw_df", save_as="filtered_df")
+        .step(write_parquet, use="filtered_df")
+    )
+```
+
+Each authored flow module exports `build() -> Flow`. The module filename is the
+flow identity.
+
+## Workspaces
+
+Data Engine discovers workspaces from a collection root. Each workspace keeps
+authored flows under:
+
+```text
+workspaces/<workspace_id>/flow_modules/
+```
+
+Shared workspace state lives under:
+
+```text
+workspaces/<workspace_id>/.workspace_state/
+```
+
+Machine-local runtime artifacts are stored outside the authored workspace.
+
+## Useful APIs
+
+```python
+from data_engine import Flow, FlowContext, discover_flows, load_flow, run
+```
+
+Common `Flow` methods:
+
+- `.watch(...)`
+- `.mirror(...)`
+- `.date_range_input(...)`
+- `.step(...)`
+- `.collect(...)`
+- `.map(...)`
+- `.step_each(...)`
+- `.preview(...)`
+- `.run_once()`
+- `.run()`
+
+Common `FlowContext` values:
+
+- `context.source`
+- `context.mirror`
+- `context.current`
+- `context.objects`
+- `context.metadata`
+- `context.database("analytics.duckdb")`
+- `context.template("reports/base.xlsx")`
+- `context.debug`
+
+The full authoring guide and helper reference live in
+`src/data_engine/docs/sphinx_source/guides/`.
+
+## Testing
+
+```bash
+python -m pytest -q
+python -m build
+python -m twine check dist/*
+```
+
+## Status
+
+This project is pre-alpha. Internal architecture is still moving quickly, and
+backwards compatibility is not a current goal.

{py_data_engine-0.3.11 → py_data_engine-0.4.0}/pyproject.toml

@@ -1,15 +1,15 @@
 [build-system]
 requires = [
-  "setuptools>=68",
-  "wheel",
-  "APScheduler>=3.11.0",
-  "duckdb>=1.5.0",
-  "numpy>=2.4.0",
-  "polars>=1.39.0",
-  "myst-parser>=4.0.0",
-  "Sphinx>=9.1.0",
-  "sphinx_rtd_theme>=3.1.0",
-  "xlsxwriter>=3.2.9",
+  "setuptools==83.0.0",
+  "APScheduler==3.11.3",
+  "duckdb==1.5.4",
+  "myst-parser==5.1.0",
+  "numpy==2.5.1",
+  "openpyxl==3.1.5",
+  "polars==1.42.1",
+  "Sphinx==9.1.0",
+  "sphinx_rtd_theme==3.1.0",
+  "xlsxwriter==3.2.9",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -35,16 +35,15 @@ classifiers = [
   "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [
-  "APScheduler>=3.11.0",
-  "duckdb>=1.5.0",
-  "fastexcel>=0.19.0",
-  "numpy>=2.4.0",
-  "openpyxl>=3.1.0",
-  "polars>=1.39.0",
-  "pyarrow>=23.0.1",
-  "PySide6>=6.9.0",
-  "textual>=0.74.0",
-  "xlsxwriter>=3.2.9",
+  "APScheduler==3.11.3",
+  "duckdb==1.5.4",
+  "fastexcel==0.20.2",
+  "numpy==2.5.1",
+  "openpyxl==3.1.5",
+  "polars==1.42.1",
+  "pyarrow==25.0.0",
+  "PySide6==6.11.1",
+  "xlsxwriter==3.2.9",
 ]
 
 [project.urls]
@@ -56,52 +55,51 @@ Issues = "https://github.com/bj-data-eng/data-engine/issues"
 data-engine = "data_engine.ui.cli.app:main"
 
 [project.optional-dependencies]
-polars-lts = [
-  "polars-lts-cpu>=1.33.1",
-]
 synthetic = [
-  "openpyxl>=3.1.0",
-  "pandas>=3.0.0",
+  "openpyxl==3.1.5",
+  "pandas==3.0.3",
 ]
 notebook = [
-  "ipykernel>=7.0.0",
-  "ipython>=9.0.0",
-  "jupyterlab>=4.4.0",
-  "notebook>=7.4.0",
+  "ipykernel==7.3.0",
+  "ipython==9.15.0",
+  "jupyterlab==4.6.1",
+  "notebook==7.6.0",
 ]
 docs = [
-  "myst-parser>=4.0.0",
-  "Sphinx>=9.1.0",
-  "sphinx_rtd_theme>=3.1.0",
+  "myst-parser==5.1.0",
+  "Sphinx==9.1.0",
+  "sphinx_rtd_theme==3.1.0",
 ]
 package = [
-  "pyinstaller>=6.15.0",
+  "build==1.5.1",
+  "pyinstaller==6.21.0",
+  "twine==6.2.0",
 ]
 test = [
-  "openpyxl>=3.1.0",
-  "pandas>=3.0.0",
-  "pydocstyle>=6.3.0",
-  "pytest>=9.0.0",
-  "pytest-cov>=7.0.0",
+  "openpyxl==3.1.5",
+  "pandas==3.0.3",
+  "pydocstyle==6.3.0",
+  "pytest==9.1.1",
+  "pytest-cov==7.1.0",
 ]
 dev = [
-  "ipykernel>=7.0.0",
-  "ipython>=9.0.0",
-  "jupyterlab>=4.4.0",
-  "myst-parser>=4.0.0",
-  "notebook>=7.4.0",
-  "numpy>=2.4.0",
-  "openpyxl>=3.1.0",
-  "pandas>=3.0.0",
-  "pydoclint>=0.7.6",
-  "pydocstyle>=6.3.0",
-  "pyinstaller>=6.15.0",
-  "pytest>=9.0.0",
-  "pytest-cov>=7.0.0",
-  "ruff>=0.15.10",
-  "Sphinx>=9.1.0",
-  "sphinx_rtd_theme>=3.1.0",
-  "viztracer>=1.1.0",
+  "ipykernel==7.3.0",
+  "ipython==9.15.0",
+  "jupyterlab==4.6.1",
+  "myst-parser==5.1.0",
+  "notebook==7.6.0",
+  "numpy==2.5.1",
+  "openpyxl==3.1.5",
+  "pandas==3.0.3",
+  "pydoclint==0.9.1",
+  "pydocstyle==6.3.0",
+  "pyinstaller==6.21.0",
+  "pytest==9.1.1",
+  "pytest-cov==7.1.0",
+  "ruff==0.15.21",
+  "Sphinx==9.1.0",
+  "sphinx_rtd_theme==3.1.0",
+  "viztracer==1.1.1",
 ]
 
 [tool.setuptools]

{py_data_engine-0.3.11 → py_data_engine-0.4.0}/src/data_engine/application/actions.py

@@ -18,10 +18,8 @@ class ActionStateApplication:
         active_flow_states,
         engine_state: str = "idle",
         live_runs: dict[str, object] | None = None,
-        has_logs: bool,
         has_automated_flows: bool,
         workspace_available: bool = True,
-        selected_run_group_present: bool = False,
         local_request_pending: bool = False,
         overlay: PendingWorkspaceActionOverlay | None = None,
     ) -> OperatorActionContext:
@@ -32,7 +30,6 @@ class ActionStateApplication:
             runtime_session=runtime_session,
             flow_groups_by_name=flow_groups_by_name,
             active_flow_states=active_flow_states,
-            has_logs=has_logs,
             live_runs=live_runs,
         )
         return OperatorActionContext(
@@ -41,7 +38,6 @@ class ActionStateApplication:
             has_automated_flows=has_automated_flows,
             engine_state=engine_state,
             workspace_available=workspace_available,
-            selected_run_group_present=selected_run_group_present,
             local_request_pending=local_request_pending,
             overlay=PendingWorkspaceActionOverlay() if overlay is None else overlay,
         )

{py_data_engine-0.3.11 → py_data_engine-0.4.0}/src/data_engine/application/runtime.py

@@ -24,6 +24,9 @@ from data_engine.platform.workspace_models import WorkspacePaths, authored_works
 from data_engine.services import DaemonService, DaemonStateService, SharedStateService
 
 
+_FLOW_RESET_TIMEOUT_SECONDS = 120.0
+
+
 @dataclass(frozen=True)
 class DaemonCommandResult:
     """Normalized outcome of one daemon command request."""
@@ -278,6 +281,16 @@ class RuntimeApplication:
         """Request one manual flow stop through the daemon."""
         return self._request(paths, {"command": "stop_flow", "name": name}, timeout=timeout)
 
+    def reset_flow(
+        self,
+        paths: WorkspacePaths,
+        *,
+        name: str,
+        timeout: float = _FLOW_RESET_TIMEOUT_SECONDS,
+    ) -> DaemonCommandResult:
+        """Reset one flow through the daemon that owns shared snapshot publication."""
+        return self._request(paths, {"command": "reset_flow", "name": name}, timeout=timeout)
+
     def daemon_status(self, paths: WorkspacePaths, *, timeout: float = 0.0) -> DaemonCommandResult:
         """Request raw daemon status for host/CLI inspection."""
         return self._request(paths, {"command": "daemon_status"}, timeout=timeout)

{py_data_engine-0.3.11 → py_data_engine-0.4.0}/src/data_engine/authoring/flow.py

@@ -94,10 +94,10 @@ class Flow(_CoreFlow):
 
         Parameters
         ----------
-        authoring_services : AuthoringServices | None
-            Optional service bundle used by tests or embedded hosts.
         inputs : dict[str, object] | None
             Submitted manual-run input values for flows that declare them.
+        authoring_services : AuthoringServices | None
+            Optional service bundle used by tests or embedded hosts.
         runtime_execution_service : RuntimeExecutionService | None
             Optional runtime execution service override.
 

{py_data_engine-0.3.11 → py_data_engine-0.4.0}/src/data_engine/core/flow.py

@@ -47,7 +47,7 @@ class Flow:
     Attributes
     ----------
     group : str
-        Display group used by the GUI, TUI, and runtime summaries.
+        Display group used by the GUI and runtime summaries.
     name : str | None
         Stable flow identifier. When omitted in a flow module, the module loader
         can derive it from the module name.