py-data-engine 0.3.10__tar.gz → 0.3.12__tar.gz

py_data_engine-0.3.12/PKG-INFO

@@ -0,0 +1,240 @@
+Metadata-Version: 2.4
+Name: py-data-engine
+Version: 0.3.12
+Summary: GUI orchestrator for python-based dataframe transform pipelines.
+Author: Data Engine contributors
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/bj-data-eng/data-engine
+Project-URL: Repository, https://github.com/bj-data-eng/data-engine
+Project-URL: Issues, https://github.com/bj-data-eng/data-engine/issues
+Keywords: duckdb,excel,parquet,flow,polars,notebook
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database
+Classifier: Topic :: Office/Business :: Financial :: Spreadsheet
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: APScheduler==3.11.2
+Requires-Dist: duckdb==1.5.1
+Requires-Dist: fastexcel==0.19.0
+Requires-Dist: numpy==2.4.4
+Requires-Dist: openpyxl==3.1.5
+Requires-Dist: polars==1.39.3
+Requires-Dist: pyarrow==23.0.1
+Requires-Dist: PySide6==6.11.0
+Requires-Dist: textual==8.2.3
+Requires-Dist: xlsxwriter==3.2.9
+Provides-Extra: polars-lts
+Requires-Dist: polars-lts-cpu==1.33.1; extra == "polars-lts"
+Provides-Extra: synthetic
+Requires-Dist: openpyxl==3.1.5; extra == "synthetic"
+Requires-Dist: pandas==3.0.2; extra == "synthetic"
+Provides-Extra: notebook
+Requires-Dist: ipykernel==7.2.0; extra == "notebook"
+Requires-Dist: ipython==9.12.0; extra == "notebook"
+Requires-Dist: jupyterlab==4.5.7; extra == "notebook"
+Requires-Dist: notebook==7.5.6; extra == "notebook"
+Provides-Extra: docs
+Requires-Dist: myst-parser==5.0.0; extra == "docs"
+Requires-Dist: Sphinx==9.1.0; extra == "docs"
+Requires-Dist: sphinx_rtd_theme==3.1.0; extra == "docs"
+Provides-Extra: package
+Requires-Dist: build==1.4.2; extra == "package"
+Requires-Dist: pyinstaller==6.19.0; extra == "package"
+Requires-Dist: twine==6.2.0; extra == "package"
+Provides-Extra: test
+Requires-Dist: openpyxl==3.1.5; extra == "test"
+Requires-Dist: pandas==3.0.2; extra == "test"
+Requires-Dist: pydocstyle==6.3.0; extra == "test"
+Requires-Dist: pytest==9.0.3; extra == "test"
+Requires-Dist: pytest-cov==7.1.0; extra == "test"
+Provides-Extra: dev
+Requires-Dist: ipykernel==7.2.0; extra == "dev"
+Requires-Dist: ipython==9.12.0; extra == "dev"
+Requires-Dist: jupyterlab==4.5.7; extra == "dev"
+Requires-Dist: myst-parser==5.0.0; extra == "dev"
+Requires-Dist: notebook==7.5.6; extra == "dev"
+Requires-Dist: numpy==2.4.4; extra == "dev"
+Requires-Dist: openpyxl==3.1.5; extra == "dev"
+Requires-Dist: pandas==3.0.2; extra == "dev"
+Requires-Dist: pydoclint==0.8.3; extra == "dev"
+Requires-Dist: pydocstyle==6.3.0; extra == "dev"
+Requires-Dist: pyinstaller==6.19.0; extra == "dev"
+Requires-Dist: pytest==9.0.3; extra == "dev"
+Requires-Dist: pytest-cov==7.1.0; extra == "dev"
+Requires-Dist: ruff==0.15.10; extra == "dev"
+Requires-Dist: Sphinx==9.1.0; extra == "dev"
+Requires-Dist: sphinx_rtd_theme==3.1.0; extra == "dev"
+Requires-Dist: viztracer==1.1.1; extra == "dev"
+Dynamic: license-file
+
+# Data Engine
+
+Data Engine is a GUI orchestrator for Python dataframe workflows. It lets you
+author flows as plain Python modules, run them manually or automatically, and
+inspect parquet-first outputs from the desktop app.
+
+The runtime is built around:
+
+- workspace-based flow discovery
+- manual, poll, and schedule execution modes
+- Polars and DuckDB-friendly flow steps
+- mirrored output paths for source-driven runs
+- saved run, log, and dataframe inspection state
+- desktop and terminal operator surfaces
+
+## Install
+
+Use the installer for your environment:
+
+- macOS: [INSTALL/INSTALL MAC.command](INSTALL/INSTALL%20MAC.command)
+- Windows: [INSTALL/INSTALL WINDOWS.bat](INSTALL/INSTALL%20WINDOWS.bat)
+- Windows VM / CPU-safe Polars path: [INSTALL/INSTALL WINDOWS_VM.bat](INSTALL/INSTALL%20WINDOWS_VM.bat)
+
+For local development:
+
+```bash
+python -m pip install --constraint requirements/constraints.txt -e ".[dev]"
+```
+
+For a published package install:
+
+```bash
+python -m pip install py-data-engine
+```
+
+Data Engine requires Python `>=3.14`.
+
+Dependency pinning, constrained installs, and hash-locked runtime installs are
+documented in [SECURITY.md](SECURITY.md).
+
+## Start
+
+Desktop GUI:
+
+```bash
+data-engine start gui
+```
+
+Terminal UI:
+
+```bash
+data-engine start tui
+```
+
+Headless commands:
+
+```bash
+data-engine list
+data-engine show example_summary
+data-engine run --once example_summary
+data-engine run
+```
+
+## Minimal Flow
+
+```python
+from data_engine import Flow
+import polars as pl
+
+
+def read_docs(context):
+    return pl.read_excel(context.source.path)
+
+
+def keep_open(context):
+    return context.current.filter(pl.col("status") == "OPEN")
+
+
+def write_parquet(context):
+    output = context.mirror.with_suffix(".parquet")
+    context.current.write_parquet(output)
+    return output
+
+
+def build():
+    return (
+        Flow(group="Docs")
+        .watch(
+            mode="poll",
+            source="../../../example_data/Input/docs_flat",
+            interval="5s",
+            extensions=[".xlsx", ".xls", ".xlsm"],
+            settle=1,
+        )
+        .mirror(root="../../../example_data/Output/example_mirror")
+        .step(read_docs, save_as="raw_df")
+        .step(keep_open, use="raw_df", save_as="filtered_df")
+        .step(write_parquet, use="filtered_df")
+    )
+```
+
+Each authored flow module exports `build() -> Flow`. The module filename is the
+flow identity.
+
+## Workspaces
+
+Data Engine discovers workspaces from a collection root. Each workspace keeps
+authored flows under:
+
+```text
+workspaces/<workspace_id>/flow_modules/
+```
+
+Shared workspace state lives under:
+
+```text
+workspaces/<workspace_id>/.workspace_state/
+```
+
+Machine-local runtime artifacts are stored outside the authored workspace.
+
+## Useful APIs
+
+```python
+from data_engine import Flow, FlowContext, discover_flows, load_flow, run
+```
+
+Common `Flow` methods:
+
+- `.watch(...)`
+- `.mirror(...)`
+- `.date_range_input(...)`
+- `.step(...)`
+- `.collect(...)`
+- `.map(...)`
+- `.step_each(...)`
+- `.preview(...)`
+- `.run_once()`
+- `.run()`
+
+Common `FlowContext` values:
+
+- `context.source`
+- `context.mirror`
+- `context.current`
+- `context.objects`
+- `context.metadata`
+- `context.database("analytics.duckdb")`
+- `context.template("reports/base.xlsx")`
+- `context.debug`
+
+The full authoring guide and helper reference live in
+`src/data_engine/docs/sphinx_source/guides/`.
+
+## Testing
+
+```bash
+python -m pytest -q
+python -m build
+python -m twine check dist/*
+```
+
+## Status
+
+This project is pre-alpha. Internal architecture is still moving quickly, and
+backwards compatibility is not a current goal.

py_data_engine-0.3.12/README.md

@@ -0,0 +1,166 @@
+# Data Engine
+
+Data Engine is a GUI orchestrator for Python dataframe workflows. It lets you
+author flows as plain Python modules, run them manually or automatically, and
+inspect parquet-first outputs from the desktop app.
+
+The runtime is built around:
+
+- workspace-based flow discovery
+- manual, poll, and schedule execution modes
+- Polars and DuckDB-friendly flow steps
+- mirrored output paths for source-driven runs
+- saved run, log, and dataframe inspection state
+- desktop and terminal operator surfaces
+
+## Install
+
+Use the installer for your environment:
+
+- macOS: [INSTALL/INSTALL MAC.command](INSTALL/INSTALL%20MAC.command)
+- Windows: [INSTALL/INSTALL WINDOWS.bat](INSTALL/INSTALL%20WINDOWS.bat)
+- Windows VM / CPU-safe Polars path: [INSTALL/INSTALL WINDOWS_VM.bat](INSTALL/INSTALL%20WINDOWS_VM.bat)
+
+For local development:
+
+```bash
+python -m pip install --constraint requirements/constraints.txt -e ".[dev]"
+```
+
+For a published package install:
+
+```bash
+python -m pip install py-data-engine
+```
+
+Data Engine requires Python `>=3.14`.
+
+Dependency pinning, constrained installs, and hash-locked runtime installs are
+documented in [SECURITY.md](SECURITY.md).
+
+## Start
+
+Desktop GUI:
+
+```bash
+data-engine start gui
+```
+
+Terminal UI:
+
+```bash
+data-engine start tui
+```
+
+Headless commands:
+
+```bash
+data-engine list
+data-engine show example_summary
+data-engine run --once example_summary
+data-engine run
+```
+
+## Minimal Flow
+
+```python
+from data_engine import Flow
+import polars as pl
+
+
+def read_docs(context):
+    return pl.read_excel(context.source.path)
+
+
+def keep_open(context):
+    return context.current.filter(pl.col("status") == "OPEN")
+
+
+def write_parquet(context):
+    output = context.mirror.with_suffix(".parquet")
+    context.current.write_parquet(output)
+    return output
+
+
+def build():
+    return (
+        Flow(group="Docs")
+        .watch(
+            mode="poll",
+            source="../../../example_data/Input/docs_flat",
+            interval="5s",
+            extensions=[".xlsx", ".xls", ".xlsm"],
+            settle=1,
+        )
+        .mirror(root="../../../example_data/Output/example_mirror")
+        .step(read_docs, save_as="raw_df")
+        .step(keep_open, use="raw_df", save_as="filtered_df")
+        .step(write_parquet, use="filtered_df")
+    )
+```
+
+Each authored flow module exports `build() -> Flow`. The module filename is the
+flow identity.
+
+## Workspaces
+
+Data Engine discovers workspaces from a collection root. Each workspace keeps
+authored flows under:
+
+```text
+workspaces/<workspace_id>/flow_modules/
+```
+
+Shared workspace state lives under:
+
+```text
+workspaces/<workspace_id>/.workspace_state/
+```
+
+Machine-local runtime artifacts are stored outside the authored workspace.
+
+## Useful APIs
+
+```python
+from data_engine import Flow, FlowContext, discover_flows, load_flow, run
+```
+
+Common `Flow` methods:
+
+- `.watch(...)`
+- `.mirror(...)`
+- `.date_range_input(...)`
+- `.step(...)`
+- `.collect(...)`
+- `.map(...)`
+- `.step_each(...)`
+- `.preview(...)`
+- `.run_once()`
+- `.run()`
+
+Common `FlowContext` values:
+
+- `context.source`
+- `context.mirror`
+- `context.current`
+- `context.objects`
+- `context.metadata`
+- `context.database("analytics.duckdb")`
+- `context.template("reports/base.xlsx")`
+- `context.debug`
+
+The full authoring guide and helper reference live in
+`src/data_engine/docs/sphinx_source/guides/`.
+
+## Testing
+
+```bash
+python -m pytest -q
+python -m build
+python -m twine check dist/*
+```
+
+## Status
+
+This project is pre-alpha. Internal architecture is still moving quickly, and
+backwards compatibility is not a current goal.

{py_data_engine-0.3.10 → py_data_engine-0.3.12}/pyproject.toml

@@ -1,15 +1,15 @@
 [build-system]
 requires = [
-  "setuptools>=68",
-  "wheel",
-  "APScheduler>=3.11.0",
-  "duckdb>=1.5.0",
-  "numpy>=2.4.0",
-  "polars>=1.39.0",
-  "myst-parser>=4.0.0",
-  "Sphinx>=9.1.0",
-  "sphinx_rtd_theme>=3.1.0",
-  "xlsxwriter>=3.2.9",
+  "setuptools==82.0.1",
+  "APScheduler==3.11.2",
+  "duckdb==1.5.1",
+  "myst-parser==5.0.0",
+  "numpy==2.4.4",
+  "openpyxl==3.1.5",
+  "polars==1.39.3",
+  "Sphinx==9.1.0",
+  "sphinx_rtd_theme==3.1.0",
+  "xlsxwriter==3.2.9",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -35,16 +35,16 @@ classifiers = [
   "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [
-  "APScheduler>=3.11.0",
-  "duckdb>=1.5.0",
-  "fastexcel>=0.19.0",
-  "numpy>=2.4.0",
-  "openpyxl>=3.1.0",
-  "polars>=1.39.0",
-  "pyarrow>=23.0.1",
-  "PySide6>=6.9.0",
-  "textual>=0.74.0",
-  "xlsxwriter>=3.2.9",
+  "APScheduler==3.11.2",
+  "duckdb==1.5.1",
+  "fastexcel==0.19.0",
+  "numpy==2.4.4",
+  "openpyxl==3.1.5",
+  "polars==1.39.3",
+  "pyarrow==23.0.1",
+  "PySide6==6.11.0",
+  "textual==8.2.3",
+  "xlsxwriter==3.2.9",
 ]
 
 [project.urls]
@@ -57,51 +57,53 @@ data-engine = "data_engine.ui.cli.app:main"
 
 [project.optional-dependencies]
 polars-lts = [
-  "polars-lts-cpu>=1.33.1",
+  "polars-lts-cpu==1.33.1",
 ]
 synthetic = [
-  "openpyxl>=3.1.0",
-  "pandas>=3.0.0",
+  "openpyxl==3.1.5",
+  "pandas==3.0.2",
 ]
 notebook = [
-  "ipykernel>=7.0.0",
-  "ipython>=9.0.0",
-  "jupyterlab>=4.4.0",
-  "notebook>=7.4.0",
+  "ipykernel==7.2.0",
+  "ipython==9.12.0",
+  "jupyterlab==4.5.7",
+  "notebook==7.5.6",
 ]
 docs = [
-  "myst-parser>=4.0.0",
-  "Sphinx>=9.1.0",
-  "sphinx_rtd_theme>=3.1.0",
+  "myst-parser==5.0.0",
+  "Sphinx==9.1.0",
+  "sphinx_rtd_theme==3.1.0",
 ]
 package = [
-  "pyinstaller>=6.15.0",
+  "build==1.4.2",
+  "pyinstaller==6.19.0",
+  "twine==6.2.0",
 ]
 test = [
-  "openpyxl>=3.1.0",
-  "pandas>=3.0.0",
-  "pydocstyle>=6.3.0",
-  "pytest>=9.0.0",
-  "pytest-cov>=7.0.0",
+  "openpyxl==3.1.5",
+  "pandas==3.0.2",
+  "pydocstyle==6.3.0",
+  "pytest==9.0.3",
+  "pytest-cov==7.1.0",
 ]
 dev = [
-  "ipykernel>=7.0.0",
-  "ipython>=9.0.0",
-  "jupyterlab>=4.4.0",
-  "myst-parser>=4.0.0",
-  "notebook>=7.4.0",
-  "numpy>=2.4.0",
-  "openpyxl>=3.1.0",
-  "pandas>=3.0.0",
-  "pydoclint>=0.7.6",
-  "pydocstyle>=6.3.0",
-  "pyinstaller>=6.15.0",
-  "pytest>=9.0.0",
-  "pytest-cov>=7.0.0",
-  "ruff>=0.15.10",
-  "Sphinx>=9.1.0",
-  "sphinx_rtd_theme>=3.1.0",
-  "viztracer>=1.1.0",
+  "ipykernel==7.2.0",
+  "ipython==9.12.0",
+  "jupyterlab==4.5.7",
+  "myst-parser==5.0.0",
+  "notebook==7.5.6",
+  "numpy==2.4.4",
+  "openpyxl==3.1.5",
+  "pandas==3.0.2",
+  "pydoclint==0.8.3",
+  "pydocstyle==6.3.0",
+  "pyinstaller==6.19.0",
+  "pytest==9.0.3",
+  "pytest-cov==7.1.0",
+  "ruff==0.15.10",
+  "Sphinx==9.1.0",
+  "sphinx_rtd_theme==3.1.0",
+  "viztracer==1.1.1",
 ]
 
 [tool.setuptools]
@@ -115,9 +117,6 @@ include = ["data_engine*"]
 [tool.setuptools.package-data]
 data_engine = [
   "ui/gui/icons/*.svg",
-  "ui/egui/*.pyd",
-  "ui/egui/*.so",
-  "ui/egui/*.dylib",
   "docs/sphinx_source/conf.py",
   "docs/sphinx_source/*.rst",
   "docs/sphinx_source/guides/*.md",

{py_data_engine-0.3.10 → py_data_engine-0.3.12}/setup.py

@@ -5,7 +5,6 @@ import sys
 from pathlib import Path
 
 from setuptools import setup
-from setuptools.command.bdist_wheel import bdist_wheel as _bdist_wheel
 from setuptools.command.build_py import build_py as _build_py
 from setuptools.command.sdist import sdist as _sdist
 
@@ -14,14 +13,6 @@ ROOT = Path(__file__).resolve().parent
 DOCS_SOURCE_DIR = ROOT / "src" / "data_engine" / "docs" / "sphinx_source"
 DOCS_OUTPUT_DIR = ROOT / "src" / "data_engine" / "docs" / "html"
 PROJECT_AST_MAP_SCRIPT = ROOT / "src" / "data_engine" / "devtools" / "project_ast_map.py"
-EGUI_PACKAGE_DIR = ROOT / "src" / "data_engine" / "ui" / "egui"
-
-
-def _vendored_native_modules() -> list[Path]:
-    native_modules: list[Path] = []
-    for suffix in (".pyd", ".so", ".dylib"):
-        native_modules.extend(EGUI_PACKAGE_DIR.glob(f"_data_engine_egui*{suffix}"))
-    return sorted(native_modules)
 
 
 def _build_packaged_docs() -> None:
@@ -63,26 +54,4 @@ class sdist(_sdist):
         super().run()
 
 
-class bdist_wheel(_bdist_wheel):
-    def finalize_options(self) -> None:
-        super().finalize_options()
-        if _vendored_native_modules():
-            self.root_is_pure = False
-
-    def get_tag(self) -> tuple[str, str, str]:
-        python_tag, abi_tag, platform_tag = super().get_tag()
-        if _vendored_native_modules():
-            python_tag = f"cp{sys.version_info.major}{sys.version_info.minor}"
-            abi_tag = "abi3"
-        return python_tag, abi_tag, platform_tag
-
-    def run(self) -> None:
-        if sys.platform == "win32" and not _vendored_native_modules():
-            raise RuntimeError(
-                "Windows wheel build requires the vendored egui native module. "
-                "Run scripts/build_egui_native.py first."
-            )
-        super().run()
-
-
-setup(cmdclass={"bdist_wheel": bdist_wheel, "build_py": build_py, "sdist": sdist})
+setup(cmdclass={"build_py": build_py, "sdist": sdist})

{py_data_engine-0.3.10 → py_data_engine-0.3.12}/src/data_engine/application/control.py

@@ -55,6 +55,7 @@ class OperatorControlApplication:
         selected_flow_valid: bool,
         blocked_status_text: str,
         timeout: float = 2.0,
+        inputs: dict[str, object] | None = None,
     ) -> OperatorActionResult:
         """Validate and request one manual run for the selected flow."""
         if not authored_workspace_is_available(paths):
@@ -76,12 +77,14 @@ class OperatorControlApplication:
             return OperatorActionResult(requested=False)
         if not action_context.control_available:
             return OperatorActionResult(requested=False, status_text=blocked_status_text)
-        result = self.runtime_application.run_flow(
-            paths,
-            name=selected_flow_name,
-            wait=False,
-            timeout=timeout,
-        )
+        run_kwargs: dict[str, object] = {
+            "name": selected_flow_name,
+            "wait": False,
+            "timeout": timeout,
+        }
+        if inputs is not None:
+            run_kwargs["inputs"] = inputs
+        result = self.runtime_application.run_flow(paths, **run_kwargs)
         if not result.ok:
             return OperatorActionResult(
                 requested=False,

{py_data_engine-0.3.10 → py_data_engine-0.3.12}/src/data_engine/application/runtime.py

@@ -242,13 +242,21 @@ class RuntimeApplication:
             signature=signature,
         )
 
-    def run_flow(self, paths: WorkspacePaths, *, name: str, wait: bool = False, timeout: float = 2.0) -> DaemonCommandResult:
+    def run_flow(
+        self,
+        paths: WorkspacePaths,
+        *,
+        name: str,
+        wait: bool = False,
+        timeout: float = 2.0,
+        inputs: dict[str, object] | None = None,
+    ) -> DaemonCommandResult:
         """Request one manual flow run through the daemon."""
         if not authored_workspace_is_available(paths):
             return DaemonCommandResult(ok=False, error="Workspace root is no longer available.")
         return self._spawn_and_request(
             paths,
-            {"command": "run_flow", "name": name, "wait": wait},
+            {"command": "run_flow", "name": name, "wait": wait, "inputs": dict(inputs or {})},
             timeout=timeout,
         )
 

{py_data_engine-0.3.10 → py_data_engine-0.3.12}/src/data_engine/authoring/flow.py

@@ -86,6 +86,7 @@ class Flow(_CoreFlow):
     def run_once(
         self,
         *,
+        inputs: dict[str, object] | None = None,
         authoring_services: AuthoringServices | None = None,
         runtime_execution_service: RuntimeExecutionService | None = None,
     ) -> list[FlowContext]:
@@ -93,6 +94,8 @@ class Flow(_CoreFlow):
 
         Parameters
         ----------
+        inputs : dict[str, object] | None
+            Submitted manual-run input values for flows that declare them.
         authoring_services : AuthoringServices | None
             Optional service bundle used by tests or embedded hosts.
         runtime_execution_service : RuntimeExecutionService | None
@@ -108,7 +111,9 @@ class Flow(_CoreFlow):
             authoring_services=authoring_services,
             runtime_execution_service=runtime_execution_service,
         ).runtime_execution_service
-        return service.run_once(flow)
+        if inputs is None:
+            return service.run_once(flow)
+        return service.run_once(flow, inputs=inputs)
 
     def preview(
         self,