tesorotools-python 0.0.27__tar.gz → 0.0.28__tar.gz

{tesorotools_python-0.0.27 → tesorotools_python-0.0.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tesorotools-python
-Version: 0.0.27
+Version: 0.0.28
 Requires-Python: >=3.13
 Requires-Dist: babel>=2.17
 Requires-Dist: eikon>=1.1

{tesorotools_python-0.0.27 → tesorotools_python-0.0.28}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "tesorotools-python"
 requires-python = ">=3.13"
-version = "0.0.27"
+version = "0.0.28"
 dependencies = [
     # database and ORM
     "psycopg[binary]>=3.1",

{tesorotools_python-0.0.27 → tesorotools_python-0.0.28}/src/tesorotools/pipeline/engine.py

@@ -51,6 +51,12 @@ def apply_transformations(
 ) -> pd.DataFrame:
     """Apply rules sequentially to a DataFrame.
 
+    New columns are accumulated in a dict and merged into
+    *result* only when a later rule reads from them (or at
+    the end).  This avoids per-rule fragmentation of the
+    DataFrame and pandas' ``PerformanceWarning`` when many
+    rules (100+) are applied.
+
     Parameters
     ----------
     df
@@ -64,8 +70,10 @@ def apply_transformations(
         Copy of *df* with new columns added by the rules.
     """
     result = df.copy()
+    pending: dict[str, pd.Series[float]] = {}
     for rule in rules:
-        missing = [d for d in rule.dependencies if d not in result.columns]
+        available = set(result.columns) | set(pending)
+        missing = [d for d in rule.dependencies if d not in available]
         if missing:
             logger.warning(
                 "Skipping %s: missing dependencies %s",
@@ -73,5 +81,21 @@ def apply_transformations(
                 missing,
             )
             continue
-        result[rule.output_name] = rule.compute(result)
+        # rule.compute() reads from result; flush pending
+        # into result if this rule needs any pending column.
+        if any(d in pending for d in rule.dependencies):
+            result = pd.concat(
+                [
+                    result,
+                    pd.DataFrame(pending, index=result.index),
+                ],
+                axis=1,
+            )
+            pending = {}
+        pending[rule.output_name] = rule.compute(result)
+    if pending:
+        result = pd.concat(
+            [result, pd.DataFrame(pending, index=result.index)],
+            axis=1,
+        )
     return result