py-data-engine 0.2.1__tar.gz → 0.2.2__tar.gz

{py_data_engine-0.2.1/src/py_data_engine.egg-info → py_data_engine-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: py-data-engine
-Version: 0.2.1
+Version: 0.2.2
 Summary: Workbook-driven workflow jobs with strict filename parsing and DuckDB-backed processing.
 Author: Data Engine contributors
 License-Expression: MIT

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/core/flow.py

@@ -92,6 +92,7 @@ class Flow:
         *,
         mode: str,
         run_as: str = "individual",
+        max_parallel: int = 1,
         source: str | Path | None = None,
         interval: str | None = None,
         time: str | tuple[str, ...] | list[str] | set[str] | None = None,
@@ -126,6 +127,9 @@ class Flow:
         run_as : str
             ``"individual"`` to run once per source file, or ``"batch"`` to run
             once for the full source set.
+        max_parallel : int
+            Maximum number of concurrent source-file runs for one flow when
+            ``run_as="individual"``. Defaults to ``1``.
         source : str | Path | None
             File or directory watched by poll/schedule triggers.
         interval : str | None
@@ -157,6 +161,9 @@ class Flow:
         if normalized_run_as not in {"individual", "batch"}:
             raise FlowValidationError("watch() run_as must be either 'individual' or 'batch'.")
 
+        if not isinstance(max_parallel, int) or max_parallel <= 0:
+            raise FlowValidationError("watch() max_parallel must be an integer greater than or equal to one.")
+
         if not isinstance(settle, int) or settle < 0:
             raise FlowValidationError("watch() settle must be an integer greater than or equal to zero.")
 
@@ -172,6 +179,7 @@ class Flow:
                 trigger=WatchSpec(
                     mode="manual",
                     run_as=normalized_run_as,
+                    max_parallel=max_parallel,
                     source=resolved_source,
                     extensions=normalized_extensions,
                 )
@@ -188,6 +196,7 @@ class Flow:
                 trigger=WatchSpec(
                     mode="poll",
                     run_as=normalized_run_as,
+                    max_parallel=max_parallel,
                     source=resolved_source,
                     interval=interval,
                     interval_seconds=_parse_duration(interval),
@@ -205,6 +214,7 @@ class Flow:
                 trigger=WatchSpec(
                     mode="schedule",
                     run_as=normalized_run_as,
+                    max_parallel=max_parallel,
                     source=resolved_source,
                     interval=interval,
                     interval_seconds=_parse_duration(interval),
@@ -217,6 +227,7 @@ class Flow:
             trigger=WatchSpec(
                 mode="schedule",
                 run_as=normalized_run_as,
+                max_parallel=max_parallel,
                 source=resolved_source,
                 time=time_values[0] if len(time_values) == 1 else time_values,
                 times=time_values,

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/core/primitives.py

@@ -21,6 +21,7 @@ class WatchSpec:
 
     mode: str
     run_as: str
+    max_parallel: int = 1
     source: Path | None = None
     interval: str | None = None
     interval_seconds: float | None = None

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/devtools/smoke_data.py

@@ -38,6 +38,7 @@ def build_smoke_environment(
     secondary_data_dir_name: str = "data2",
     create_app_root: bool = False,
     rows_per_workbook: int = 2,
+    parallel_rows_per_workbook: int | None = None,
     column_count: int = len(_BASE_CLAIMS_COLUMNS),
 ) -> None:
     """Generate starter data roots plus authored workspaces under one root."""
@@ -59,6 +60,12 @@ def build_smoke_environment(
         rows_per_workbook=rows_per_workbook,
         column_count=column_count,
     )
+    create_parallel_claims_data_root(
+        secondary_data_root,
+        file_count=12,
+        rows_per_workbook=parallel_rows_per_workbook or rows_per_workbook,
+        column_count=column_count,
+    )
 
     for workspace_id in workspace_ids:
         target_workspace = workspace_collection_root / workspace_id
@@ -87,7 +94,6 @@ def create_python_flow_modules(
     schedule_interval: str = "30s",
 ) -> None:
     """Write starter Python-authored flow modules into one workspace."""
-    del workspace_id
     flow_dir = target_workspace / "flow_modules"
     write_text_file(flow_dir / "example_mirror.py", _python_poll_source(data_folder_name=data_folder_name))
     write_text_file(
@@ -99,6 +105,19 @@ def create_python_flow_modules(
         flow_dir / "example_database_dimensions.py",
         _python_database_dimensions_source(data_folder_name=data_folder_name),
     )
+    if workspace_id == "claims2":
+        write_text_file(
+            flow_dir / "claims2_parallel_poll.py",
+            _python_parallel_poll_source(data_folder_name=data_folder_name),
+        )
+        write_text_file(
+            flow_dir / "claims2_parallel_schedule.py",
+            _python_parallel_schedule_source(data_folder_name=data_folder_name),
+        )
+        write_text_file(
+            flow_dir / "claims2_parallel_manual.py",
+            _python_parallel_manual_source(data_folder_name=data_folder_name),
+        )
 
 
 def create_notebook_flow_modules(
@@ -168,6 +187,25 @@ def create_smoke_data_root(
     )
 
 
+def create_parallel_claims_data_root(
+    data_root: Path,
+    *,
+    file_count: int = 12,
+    rows_per_workbook: int = 2,
+    column_count: int = len(_BASE_CLAIMS_COLUMNS),
+) -> None:
+    """Generate one fixed-size parallel smoke dataset for file-scoped runtime tests."""
+    parallel_dir = data_root / "Input" / "claims_parallel_12"
+    parallel_dir.mkdir(parents=True, exist_ok=True)
+    for index in range(1, file_count + 1):
+        _write_claims_workbook(
+            parallel_dir / f"claims_parallel_{index:02d}.xlsx",
+            workbook_index=100 + index,
+            rows_per_workbook=rows_per_workbook,
+            column_count=column_count,
+        )
+
+
 def _claims_headers(*, column_count: int) -> tuple[str, ...]:
     extra_count = column_count - len(_BASE_CLAIMS_COLUMNS)
     extra_columns = tuple(f"Attribute {index:02d}" for index in range(1, extra_count + 1))
@@ -488,6 +526,120 @@ def build():
 """
 
 
+def _parallel_runtime_helpers(*, data_folder_name: str, output_folder_name: str) -> str:
+    return f"""
+
+import polars as pl
+
+from data_engine.helpers import write_parquet_atomic
+
+
+def read_claims(context):
+    return pl.read_excel(context.source.path)
+
+
+def write_target(context):
+    output = context.mirror.with_suffix(".parquet")
+    write_parquet_atomic(context.current, output)
+    return output
+
+
+SOURCE_ROOT = "../../../{data_folder_name}/Input/claims_parallel_12"
+TARGET_ROOT = "../../../{data_folder_name}/Output/{output_folder_name}"
+"""
+
+
+def _python_parallel_poll_source(*, data_folder_name: str) -> str:
+    return (
+        """from __future__ import annotations
+
+from data_engine import Flow
+"""
+        + _parallel_runtime_helpers(data_folder_name=data_folder_name, output_folder_name="claims2_parallel_poll")
+        + """
+
+DESCRIPTION = "Claims2 parallel poll demo with 12 source workbooks and bounded file concurrency."
+
+
+def build():
+    return (
+        Flow(name="claims2_parallel_poll", group="Parallel")
+        .watch(
+            mode="poll",
+            source=SOURCE_ROOT,
+            interval="5s",
+            extensions=[".xlsx", ".xls", ".xlsm"],
+            settle=1,
+            max_parallel=4,
+        )
+        .mirror(root=TARGET_ROOT)
+        .step(read_claims, label="Read Excel")
+        .step(write_target, label="Write Parquet")
+    )
+"""
+    )
+
+
+def _python_parallel_schedule_source(*, data_folder_name: str) -> str:
+    return (
+        """from __future__ import annotations
+
+from data_engine import Flow
+"""
+        + _parallel_runtime_helpers(data_folder_name=data_folder_name, output_folder_name="claims2_parallel_schedule")
+        + """
+
+DESCRIPTION = "Claims2 parallel schedule demo with 12 source workbooks and bounded file concurrency."
+
+
+def build():
+    return (
+        Flow(name="claims2_parallel_schedule", group="Parallel")
+        .watch(
+            mode="schedule",
+            run_as="individual",
+            source=SOURCE_ROOT,
+            interval="5s",
+            extensions=[".xlsx", ".xls", ".xlsm"],
+            max_parallel=4,
+        )
+        .mirror(root=TARGET_ROOT)
+        .step(read_claims, label="Read Excel")
+        .step(write_target, label="Write Parquet")
+    )
+"""
+    )
+
+
+def _python_parallel_manual_source(*, data_folder_name: str) -> str:
+    return (
+        """from __future__ import annotations
+
+from data_engine import Flow
+"""
+        + _parallel_runtime_helpers(data_folder_name=data_folder_name, output_folder_name="claims2_parallel_manual")
+        + """
+
+DESCRIPTION = "Claims2 parallel manual demo with 12 source workbooks and bounded file concurrency."
+
+
+def build():
+    return (
+        Flow(name="claims2_parallel_manual", group="Parallel")
+        .watch(
+            mode="manual",
+            source=SOURCE_ROOT,
+            extensions=[".xlsx", ".xls", ".xlsm"],
+            max_parallel=4,
+        )
+        .mirror(root=TARGET_ROOT)
+        .step(read_claims, label="Read Excel")
+        .step(write_target, label="Write Parquet")
+    )
+"""
+    )
+
+
 def _poll_notebook_source(*, workspace_id: str, data_folder_name: str) -> str:
     return f"""from __future__ import annotations
 

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/docs/html/_modules/data_engine/authoring/flow.html

@@ -13,7 +13,7 @@
   
       <script src="../../../_static/jquery.js?v=5d32c60e"></script>
       <script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="../../../_static/documentation_options.js?v=37f418d5"></script>
+      <script src="../../../_static/documentation_options.js?v=000c92bf"></script>
       <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
       <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../../../_static/js/theme.js"></script>

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/docs/html/_modules/data_engine/core/flow.html

@@ -13,7 +13,7 @@
   
       <script src="../../../_static/jquery.js?v=5d32c60e"></script>
       <script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="../../../_static/documentation_options.js?v=37f418d5"></script>
+      <script src="../../../_static/documentation_options.js?v=000c92bf"></script>
       <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
       <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../../../_static/js/theme.js"></script>
@@ -180,6 +180,7 @@
         <span class="o">*</span><span class="p">,</span>
         <span class="n">mode</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
         <span class="n">run_as</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;individual&quot;</span><span class="p">,</span>
+        <span class="n">max_parallel</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
         <span class="n">source</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="n">Path</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">interval</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
         <span class="n">time</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="nb">tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="o">...</span><span class="p">]</span> <span class="o">|</span> <span class="nb">list</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="nb">set</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -214,6 +215,9 @@
 <span class="sd">        run_as : str</span>
 <span class="sd">            ``&quot;individual&quot;`` to run once per source file, or ``&quot;batch&quot;`` to run</span>
 <span class="sd">            once for the full source set.</span>
+<span class="sd">        max_parallel : int</span>
+<span class="sd">            Maximum number of concurrent source-file runs for one flow when</span>
+<span class="sd">            ``run_as=&quot;individual&quot;``. Defaults to ``1``.</span>
 <span class="sd">        source : str | Path | None</span>
 <span class="sd">            File or directory watched by poll/schedule triggers.</span>
 <span class="sd">        interval : str | None</span>
@@ -245,6 +249,9 @@
         <span class="k">if</span> <span class="n">normalized_run_as</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">{</span><span class="s2">&quot;individual&quot;</span><span class="p">,</span> <span class="s2">&quot;batch&quot;</span><span class="p">}:</span>
             <span class="k">raise</span> <span class="n">FlowValidationError</span><span class="p">(</span><span class="s2">&quot;watch() run_as must be either &#39;individual&#39; or &#39;batch&#39;.&quot;</span><span class="p">)</span>
 
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">max_parallel</span><span class="p">,</span> <span class="nb">int</span><span class="p">)</span> <span class="ow">or</span> <span class="n">max_parallel</span> <span class="o">&lt;=</span> <span class="mi">0</span><span class="p">:</span>
+            <span class="k">raise</span> <span class="n">FlowValidationError</span><span class="p">(</span><span class="s2">&quot;watch() max_parallel must be an integer greater than or equal to one.&quot;</span><span class="p">)</span>
+
         <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">settle</span><span class="p">,</span> <span class="nb">int</span><span class="p">)</span> <span class="ow">or</span> <span class="n">settle</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
             <span class="k">raise</span> <span class="n">FlowValidationError</span><span class="p">(</span><span class="s2">&quot;watch() settle must be an integer greater than or equal to zero.&quot;</span><span class="p">)</span>
 
@@ -260,6 +267,7 @@
                 <span class="n">trigger</span><span class="o">=</span><span class="n">WatchSpec</span><span class="p">(</span>
                     <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;manual&quot;</span><span class="p">,</span>
                     <span class="n">run_as</span><span class="o">=</span><span class="n">normalized_run_as</span><span class="p">,</span>
+                    <span class="n">max_parallel</span><span class="o">=</span><span class="n">max_parallel</span><span class="p">,</span>
                     <span class="n">source</span><span class="o">=</span><span class="n">resolved_source</span><span class="p">,</span>
                     <span class="n">extensions</span><span class="o">=</span><span class="n">normalized_extensions</span><span class="p">,</span>
                 <span class="p">)</span>
@@ -276,6 +284,7 @@
                 <span class="n">trigger</span><span class="o">=</span><span class="n">WatchSpec</span><span class="p">(</span>
                     <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;poll&quot;</span><span class="p">,</span>
                     <span class="n">run_as</span><span class="o">=</span><span class="n">normalized_run_as</span><span class="p">,</span>
+                    <span class="n">max_parallel</span><span class="o">=</span><span class="n">max_parallel</span><span class="p">,</span>
                     <span class="n">source</span><span class="o">=</span><span class="n">resolved_source</span><span class="p">,</span>
                     <span class="n">interval</span><span class="o">=</span><span class="n">interval</span><span class="p">,</span>
                     <span class="n">interval_seconds</span><span class="o">=</span><span class="n">_parse_duration</span><span class="p">(</span><span class="n">interval</span><span class="p">),</span>
@@ -293,6 +302,7 @@
                 <span class="n">trigger</span><span class="o">=</span><span class="n">WatchSpec</span><span class="p">(</span>
                     <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;schedule&quot;</span><span class="p">,</span>
                     <span class="n">run_as</span><span class="o">=</span><span class="n">normalized_run_as</span><span class="p">,</span>
+                    <span class="n">max_parallel</span><span class="o">=</span><span class="n">max_parallel</span><span class="p">,</span>
                     <span class="n">source</span><span class="o">=</span><span class="n">resolved_source</span><span class="p">,</span>
                     <span class="n">interval</span><span class="o">=</span><span class="n">interval</span><span class="p">,</span>
                     <span class="n">interval_seconds</span><span class="o">=</span><span class="n">_parse_duration</span><span class="p">(</span><span class="n">interval</span><span class="p">),</span>
@@ -305,6 +315,7 @@
             <span class="n">trigger</span><span class="o">=</span><span class="n">WatchSpec</span><span class="p">(</span>
                 <span class="n">mode</span><span class="o">=</span><span class="s2">&quot;schedule&quot;</span><span class="p">,</span>
                 <span class="n">run_as</span><span class="o">=</span><span class="n">normalized_run_as</span><span class="p">,</span>
+                <span class="n">max_parallel</span><span class="o">=</span><span class="n">max_parallel</span><span class="p">,</span>
                 <span class="n">source</span><span class="o">=</span><span class="n">resolved_source</span><span class="p">,</span>
                 <span class="n">time</span><span class="o">=</span><span class="n">time_values</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">time_values</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span> <span class="k">else</span> <span class="n">time_values</span><span class="p">,</span>
                 <span class="n">times</span><span class="o">=</span><span class="n">time_values</span><span class="p">,</span>

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/docs/html/_modules/data_engine/core/model.html

@@ -13,7 +13,7 @@
   
       <script src="../../../_static/jquery.js?v=5d32c60e"></script>
       <script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="../../../_static/documentation_options.js?v=37f418d5"></script>
+      <script src="../../../_static/documentation_options.js?v=000c92bf"></script>
       <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
       <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../../../_static/js/theme.js"></script>

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/docs/html/_modules/data_engine/core/primitives.html

@@ -13,7 +13,7 @@
   
       <script src="../../../_static/jquery.js?v=5d32c60e"></script>
       <script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="../../../_static/documentation_options.js?v=37f418d5"></script>
+      <script src="../../../_static/documentation_options.js?v=000c92bf"></script>
       <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
       <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../../../_static/js/theme.js"></script>
@@ -107,6 +107,7 @@
 
     <span class="n">mode</span><span class="p">:</span> <span class="nb">str</span>
     <span class="n">run_as</span><span class="p">:</span> <span class="nb">str</span>
+    <span class="n">max_parallel</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span>
     <span class="n">source</span><span class="p">:</span> <span class="n">Path</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
     <span class="n">interval</span><span class="p">:</span> <span class="nb">str</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>
     <span class="n">interval_seconds</span><span class="p">:</span> <span class="nb">float</span> <span class="o">|</span> <span class="kc">None</span> <span class="o">=</span> <span class="kc">None</span>

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/docs/html/_modules/data_engine/helpers/duckdb.html

@@ -13,7 +13,7 @@
   
       <script src="../../../_static/jquery.js?v=5d32c60e"></script>
       <script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="../../../_static/documentation_options.js?v=37f418d5"></script>
+      <script src="../../../_static/documentation_options.js?v=000c92bf"></script>
       <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
       <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../../../_static/js/theme.js"></script>

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/docs/html/_modules/data_engine/helpers/polars.html

@@ -13,7 +13,7 @@
   
       <script src="../../../_static/jquery.js?v=5d32c60e"></script>
       <script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="../../../_static/documentation_options.js?v=37f418d5"></script>
+      <script src="../../../_static/documentation_options.js?v=000c92bf"></script>
       <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
       <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../../../_static/js/theme.js"></script>

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/docs/html/_modules/data_engine/helpers/schema.html

@@ -13,7 +13,7 @@
   
       <script src="../../../_static/jquery.js?v=5d32c60e"></script>
       <script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="../../../_static/documentation_options.js?v=37f418d5"></script>
+      <script src="../../../_static/documentation_options.js?v=000c92bf"></script>
       <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
       <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../../../_static/js/theme.js"></script>
@@ -88,6 +88,7 @@
 
 <span class="kn">from</span><span class="w"> </span><span class="nn">collections.abc</span><span class="w"> </span><span class="kn">import</span> <span class="n">Iterable</span><span class="p">,</span> <span class="n">Mapping</span>
 <span class="kn">from</span><span class="w"> </span><span class="nn">dataclasses</span><span class="w"> </span><span class="kn">import</span> <span class="n">dataclass</span>
+<span class="kn">import</span><span class="w"> </span><span class="nn">re</span>
 <span class="kn">from</span><span class="w"> </span><span class="nn">typing</span><span class="w"> </span><span class="kn">import</span> <span class="n">TypeAlias</span>
 
 <span class="kn">import</span><span class="w"> </span><span class="nn">polars</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">pl</span>
@@ -188,7 +189,11 @@
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="p">:</span>
             <span class="k">return</span> <span class="n">df</span>
-        <span class="k">return</span> <span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="nb">tuple</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span></div>
+        <span class="n">available</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="n">pl</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">collect_schema</span><span class="p">()</span><span class="o">.</span><span class="n">names</span><span class="p">()</span>
+        <span class="n">present</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">column</span> <span class="k">for</span> <span class="n">column</span> <span class="ow">in</span> <span class="bp">self</span> <span class="k">if</span> <span class="n">column</span> <span class="ow">in</span> <span class="n">available</span><span class="p">)</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">present</span><span class="p">:</span>
+            <span class="k">return</span> <span class="n">df</span>
+        <span class="k">return</span> <span class="n">df</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">present</span><span class="p">)</span></div>
 </div>
 
 
@@ -338,10 +343,14 @@
 <span class="sd">    Returns</span>
 <span class="sd">    -------</span>
 <span class="sd">    str</span>
-<span class="sd">        Lowercase column name with leading/trailing whitespace removed, internal</span>
+<span class="sd">        Lowercase column name with separator-adjacent spaces removed, remaining</span>
 <span class="sd">        whitespace collapsed, and spaces replaced with underscores.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">return</span> <span class="s2">&quot;_&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">())</span></div>
+    <span class="n">text</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">name</span><span class="p">)</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
+    <span class="n">text</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">sub</span><span class="p">(</span><span class="sa">r</span><span class="s2">&quot;\s*([#_\-/</span><span class="se">\\\\</span><span class="s2">])\s*&quot;</span><span class="p">,</span> <span class="sa">r</span><span class="s2">&quot;\1&quot;</span><span class="p">,</span> <span class="n">text</span><span class="p">)</span>
+    <span class="n">text</span> <span class="o">=</span> <span class="s2">&quot; &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">text</span><span class="o">.</span><span class="n">split</span><span class="p">())</span>
+    <span class="n">text</span> <span class="o">=</span> <span class="n">text</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot; &quot;</span><span class="p">,</span> <span class="s2">&quot;_&quot;</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">text</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span></div>
 
 
 

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/docs/html/_modules/data_engine/hosts/scheduler.html

@@ -13,7 +13,7 @@
   
       <script src="../../../_static/jquery.js?v=5d32c60e"></script>
       <script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="../../../_static/documentation_options.js?v=37f418d5"></script>
+      <script src="../../../_static/documentation_options.js?v=000c92bf"></script>
       <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
       <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../../../_static/js/theme.js"></script>

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/docs/html/_modules/data_engine/runtime/engine.html

@@ -13,7 +13,7 @@
   
       <script src="../../../_static/jquery.js?v=5d32c60e"></script>
       <script src="../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="../../../_static/documentation_options.js?v=37f418d5"></script>
+      <script src="../../../_static/documentation_options.js?v=000c92bf"></script>
       <script src="../../../_static/doctools.js?v=fd6eb6e6"></script>
       <script src="../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../../../_static/js/theme.js"></script>

{py_data_engine-0.2.1 → py_data_engine-0.2.2}/src/data_engine/docs/html/_modules/data_engine/runtime/execution/grouped.html

@@ -13,7 +13,7 @@
   
       <script src="../../../../_static/jquery.js?v=5d32c60e"></script>
       <script src="../../../../_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
-      <script src="../../../../_static/documentation_options.js?v=37f418d5"></script>
+      <script src="../../../../_static/documentation_options.js?v=000c92bf"></script>
       <script src="../../../../_static/doctools.js?v=fd6eb6e6"></script>
       <script src="../../../../_static/sphinx_highlight.js?v=6ffebe34"></script>
     <script src="../../../../_static/js/theme.js"></script>