transformplan 0.1.3__tar.gz → 0.2.0__tar.gz

{transformplan-0.1.3/transformplan.egg-info → transformplan-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: transformplan
-Version: 0.1.3
+Version: 0.2.0
 Summary: Safe, reproducible data transformations with built-in auditing and validation
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/limebit/transformplan
@@ -52,7 +52,7 @@ Dynamic: license-file
 ```python
 from transformplan import TransformPlan, Col
 
-# Build readable pipelines with 88 chainable operations
+# Build readable pipelines with 89 chainable operations
 plan = (
     TransformPlan()
     # Standardize column names

{transformplan-0.1.3 → transformplan-0.2.0}/README.md

@@ -22,7 +22,7 @@
 ```python
 from transformplan import TransformPlan, Col
 
-# Build readable pipelines with 88 chainable operations
+# Build readable pipelines with 89 chainable operations
 plan = (
     TransformPlan()
     # Standardize column names

{transformplan-0.1.3 → transformplan-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "transformplan"
-version = "0.1.3"
+version = "0.2.0"
 description = "Safe, reproducible data transformations with built-in auditing and validation"
 readme = "README.md"
 requires-python = ">=3.10"

{transformplan-0.1.3 → transformplan-0.2.0}/tests/test_duckdb.py

@@ -1617,6 +1617,131 @@ class TestMathDiffFromAgg:
             )
 
 
+class TestMathDiffLag:
+    """Tests for math_diff_lag with DuckDB backend."""
+
+    def test_numeric_basic(
+        self, backend: DuckDBBackend, con: duckdb.DuckDBPyConnection
+    ) -> None:
+        rel = con.sql(
+            "SELECT * FROM (VALUES (1, 10), (2, 30), (3, 35), (4, 50)) AS t(id, val)"
+        )
+        result, _ = (
+            TransformPlan()
+            .math_diff_lag("val", order_by="id", new_column="diff")
+            .process(rel, backend=backend)
+        )
+        vals = _col_values(result, "diff")
+        assert vals[0] is None
+        assert vals[1:] == [20, 5, 15]
+
+    def test_numeric_lag2(
+        self, backend: DuckDBBackend, con: duckdb.DuckDBPyConnection
+    ) -> None:
+        rel = con.sql(
+            "SELECT * FROM (VALUES (1, 10), (2, 30), (3, 35), (4, 50)) AS t(id, val)"
+        )
+        result, _ = (
+            TransformPlan()
+            .math_diff_lag("val", order_by="id", new_column="diff", lag=2)
+            .process(rel, backend=backend)
+        )
+        vals = _col_values(result, "diff")
+        assert vals[0] is None
+        assert vals[1] is None
+        assert vals[2:] == [25, 20]
+
+    def test_grouped_numeric(
+        self, backend: DuckDBBackend, con: duckdb.DuckDBPyConnection
+    ) -> None:
+        rel = con.sql(
+            "SELECT * FROM (VALUES "
+            "('A', 1, 10), ('A', 2, 30), ('A', 3, 35), "
+            "('B', 1, 100), ('B', 2, 150), ('B', 3, 160)"
+            ") AS t(grp, seq, val)"
+        )
+        result, _ = (
+            TransformPlan()
+            .math_diff_lag("val", order_by="seq", new_column="diff", group_by="grp")
+            .rows_sort(["grp", "seq"])
+            .process(rel, backend=backend)
+        )
+        vals = _col_values(result, "diff")
+        assert vals == [None, 20, 5, None, 50, 10]
+
+    def test_datetime_column(
+        self, backend: DuckDBBackend, con: duckdb.DuckDBPyConnection
+    ) -> None:
+        rel = con.sql(
+            "SELECT * FROM (VALUES "
+            "(1, TIMESTAMP '2024-01-01 00:00:00'), "
+            "(2, TIMESTAMP '2024-01-01 01:00:00'), "
+            "(3, TIMESTAMP '2024-01-01 03:00:00')"
+            ") AS t(id, ts)"
+        )
+        result, _ = (
+            TransformPlan()
+            .math_diff_lag("ts", order_by="id", new_column="gap")
+            .process(rel, backend=backend)
+        )
+        vals = _col_values(result, "gap")
+        assert vals[0] is None
+        assert vals[1].total_seconds() == 3600
+        assert vals[2].total_seconds() == 7200
+
+    def test_datetime_grouped(
+        self, backend: DuckDBBackend, con: duckdb.DuckDBPyConnection
+    ) -> None:
+        rel = con.sql(
+            "SELECT * FROM (VALUES "
+            "('A', TIMESTAMP '2024-01-01 00:00:00'), "
+            "('A', TIMESTAMP '2024-01-01 02:00:00'), "
+            "('B', TIMESTAMP '2024-01-01 10:00:00'), "
+            "('B', TIMESTAMP '2024-01-01 13:00:00')"
+            ") AS t(patient, ts)"
+        )
+        result, _ = (
+            TransformPlan()
+            .math_diff_lag("ts", order_by="ts", new_column="gap", group_by="patient")
+            .rows_sort(["patient", "ts"])
+            .process(rel, backend=backend)
+        )
+        vals = _col_values(result, "gap")
+        assert vals[0] is None
+        assert vals[1].total_seconds() / 3600 == 2.0
+        assert vals[2] is None
+        assert vals[3].total_seconds() / 3600 == 3.0
+
+    def test_order_by_list(
+        self, backend: DuckDBBackend, con: duckdb.DuckDBPyConnection
+    ) -> None:
+        rel = con.sql(
+            "SELECT * FROM (VALUES "
+            "(1, 1, 10), (1, 2, 20), (2, 1, 30), (2, 2, 40)"
+            ") AS t(a, b, val)"
+        )
+        result, _ = (
+            TransformPlan()
+            .math_diff_lag("val", order_by=["a", "b"], new_column="diff")
+            .process(rel, backend=backend)
+        )
+        vals = _col_values(result, "diff")
+        assert vals == [None, 10, 10, 10]
+
+    def test_global_no_group(
+        self, backend: DuckDBBackend, con: duckdb.DuckDBPyConnection
+    ) -> None:
+        rel = con.sql("SELECT * FROM (VALUES (3, 30), (1, 10), (2, 20)) AS t(seq, val)")
+        result, _ = (
+            TransformPlan()
+            .math_diff_lag("val", order_by="seq", new_column="diff")
+            .process(rel, backend=backend)
+        )
+        vals = _col_values(result, "diff")
+        assert vals[0] is None
+        assert vals[1:] == [10, 10]
+
+
 class TestColExpr:
     """Tests for col_expr on DuckDB backend."""
 

{transformplan-0.1.3 → transformplan-0.2.0}/tests/test_math.py

@@ -465,3 +465,167 @@ class TestMathDiffFromAgg:
         result1, _ = plan.process(numeric_df)
         result2, _ = restored.process(numeric_df)
         assert result1["diff"].to_list() == result2["diff"].to_list()
+
+
+class TestMathDiffLag:
+    """Tests for math_diff_lag operation."""
+
+    def test_numeric_basic(self) -> None:
+        """Test lag=1 on integers ordered by id; first row null."""
+        df = pl.DataFrame({"id": [1, 2, 3, 4], "val": [10, 30, 35, 50]})
+        plan = TransformPlan().math_diff_lag("val", order_by="id", new_column="diff")
+        result, _ = plan.process(df)
+        assert result["diff"].to_list() == [None, 20.0, 5.0, 15.0]
+
+    def test_numeric_lag2(self) -> None:
+        """Test lag=2; first two rows null."""
+        df = pl.DataFrame({"id": [1, 2, 3, 4], "val": [10, 30, 35, 50]})
+        plan = TransformPlan().math_diff_lag(
+            "val", order_by="id", new_column="diff", lag=2
+        )
+        result, _ = plan.process(df)
+        assert result["diff"].to_list() == [None, None, 25.0, 20.0]
+
+    def test_grouped_numeric(self) -> None:
+        """Test partition by group; nulls restart per group."""
+        df = pl.DataFrame(
+            {
+                "grp": ["A", "A", "A", "B", "B", "B"],
+                "seq": [1, 2, 3, 1, 2, 3],
+                "val": [10, 30, 35, 100, 150, 160],
+            }
+        )
+        plan = TransformPlan().math_diff_lag(
+            "val", order_by="seq", new_column="diff", group_by="grp"
+        )
+        result, _ = plan.process(df)
+        expected = [None, 20.0, 5.0, None, 50.0, 10.0]
+        assert result["diff"].to_list() == expected
+
+    def test_datetime_column(self) -> None:
+        """Test datetime input produces duration output."""
+        df = pl.DataFrame(
+            {
+                "id": [1, 2, 3],
+                "ts": [
+                    datetime(2024, 1, 1, 0, 0),
+                    datetime(2024, 1, 1, 1, 0),
+                    datetime(2024, 1, 1, 3, 0),
+                ],
+            }
+        )
+        plan = TransformPlan().math_diff_lag("ts", order_by="id", new_column="gap")
+        result, _ = plan.process(df)
+        assert result["gap"].dtype == pl.Duration
+        vals = result["gap"].to_list()
+        assert vals[0] is None
+        assert vals[1].total_seconds() == 3600
+        assert vals[2].total_seconds() == 7200
+
+    def test_datetime_grouped(self) -> None:
+        """Test primary use case: time between events per patient."""
+        df = pl.DataFrame(
+            {
+                "patient": ["A", "A", "B", "B"],
+                "ts": [
+                    datetime(2024, 1, 1, 0, 0),
+                    datetime(2024, 1, 1, 2, 0),
+                    datetime(2024, 1, 1, 10, 0),
+                    datetime(2024, 1, 1, 13, 0),
+                ],
+            }
+        )
+        plan = TransformPlan().math_diff_lag(
+            "ts", order_by="ts", new_column="gap", group_by="patient"
+        )
+        result, _ = plan.process(df)
+        assert result["gap"].dtype == pl.Duration
+        vals = result["gap"].to_list()
+        assert vals[0] is None
+        assert vals[1].total_seconds() / 3600 == 2.0
+        assert vals[2] is None
+        assert vals[3].total_seconds() / 3600 == 3.0
+
+    def test_order_by_different_column(self) -> None:
+        """Test diffing 'value' ordered by 'timestamp'."""
+        df = pl.DataFrame(
+            {
+                "ts": [
+                    datetime(2024, 1, 1),
+                    datetime(2024, 1, 2),
+                    datetime(2024, 1, 3),
+                ],
+                "val": [100, 130, 125],
+            }
+        )
+        plan = TransformPlan().math_diff_lag("val", order_by="ts", new_column="change")
+        result, _ = plan.process(df)
+        assert result["change"].to_list() == [None, 30.0, -5.0]
+
+    def test_order_by_list(self) -> None:
+        """Test multi-column order_by."""
+        df = pl.DataFrame(
+            {
+                "a": [1, 1, 2, 2],
+                "b": [1, 2, 1, 2],
+                "val": [10, 20, 30, 40],
+            }
+        )
+        plan = TransformPlan().math_diff_lag(
+            "val", order_by=["a", "b"], new_column="diff"
+        )
+        result, _ = plan.process(df)
+        assert result["diff"].to_list() == [None, 10.0, 10.0, 10.0]
+
+    def test_global_no_group(self) -> None:
+        """Test no group_by, global ordering."""
+        df = pl.DataFrame({"seq": [3, 1, 2], "val": [30, 10, 20]})
+        plan = TransformPlan().math_diff_lag("val", order_by="seq", new_column="diff")
+        result, _ = plan.process(df)
+        # After sorting by seq: [10, 20, 30], diffs: [None, 10, 10]
+        assert result["diff"].to_list() == [None, 10.0, 10.0]
+
+    def test_validation_nonexistent_column(self, numeric_df: pl.DataFrame) -> None:
+        """Test validation catches non-existent column."""
+        plan = TransformPlan().math_diff_lag(
+            "nonexistent", order_by="a", new_column="diff"
+        )
+        result = plan.validate(numeric_df)
+        assert not result.is_valid
+        assert "does not exist" in str(result.errors[0])
+
+    def test_validation_wrong_type(self, basic_df: pl.DataFrame) -> None:
+        """Test validation catches string column."""
+        plan = TransformPlan().math_diff_lag("name", order_by="id", new_column="diff")
+        result = plan.validate(basic_df)
+        assert not result.is_valid
+        assert "numeric or datetime" in str(result.errors[0])
+
+    def test_validation_missing_order_by(self, numeric_df: pl.DataFrame) -> None:
+        """Test validation catches missing order_by column."""
+        plan = TransformPlan().math_diff_lag(
+            "a", order_by="nonexistent", new_column="diff"
+        )
+        result = plan.validate(numeric_df)
+        assert not result.is_valid
+        assert "Order-by" in str(result.errors[0])
+
+    def test_validation_missing_group_by(self, numeric_df: pl.DataFrame) -> None:
+        """Test validation catches missing group_by column."""
+        plan = TransformPlan().math_diff_lag(
+            "a", order_by="a", new_column="diff", group_by="nonexistent"
+        )
+        result = plan.validate(numeric_df)
+        assert not result.is_valid
+        assert "Group-by" in str(result.errors[0])
+
+    def test_serialization_roundtrip(self, numeric_df: pl.DataFrame) -> None:
+        """Test JSON serialization round-trip."""
+        plan = TransformPlan().math_diff_lag(
+            "a", order_by="a", new_column="diff", group_by="b", lag=2
+        )
+        json_str = plan.to_json()
+        restored = TransformPlan.from_json(json_str)
+        result1, _ = plan.process(numeric_df)
+        result2, _ = restored.process(numeric_df)
+        assert result1["diff"].to_list() == result2["diff"].to_list()

{transformplan-0.1.3 → transformplan-0.2.0}/transformplan/backends/base.py

@@ -309,6 +309,17 @@ class Backend(ABC):
         group_by: list[str] | None,
     ) -> Any: ...
 
+    @abstractmethod
+    def math_diff_lag(
+        self,
+        data: Any,
+        column: str,
+        order_by: list[str],
+        new_column: str,
+        group_by: list[str] | None,
+        lag: int,
+    ) -> Any: ...
+
     @abstractmethod
     def math_standardize(
         self,

{transformplan-0.1.3 → transformplan-0.2.0}/transformplan/backends/duckdb.py

@@ -582,6 +582,26 @@ class DuckDBBackend(Backend):
         )
         return self._con.sql(f"SELECT *, {expr} FROM {_sub(data)}")
 
+    def math_diff_lag(
+        self,
+        data: duckdb.DuckDBPyRelation,
+        column: str,
+        order_by: list[str],
+        new_column: str,
+        group_by: list[str] | None,
+        lag: int,
+    ) -> duckdb.DuckDBPyRelation:
+        partition = ""
+        if group_by:
+            partition = "PARTITION BY " + ", ".join(_q(g) for g in group_by)
+        order = "ORDER BY " + ", ".join(_q(o) for o in order_by)
+        window = f"{partition} {order}".strip()
+        expr = (
+            f"({_q(column)} - LAG({_q(column)}, {lag}) OVER ({window})) "
+            f"AS {_q(new_column)}"
+        )
+        return self._con.sql(f"SELECT *, {expr} FROM {_sub(data)}")
+
     def math_standardize(
         self,
         data: duckdb.DuckDBPyRelation,

{transformplan-0.1.3 → transformplan-0.2.0}/transformplan/backends/polars.py

@@ -366,6 +366,25 @@ class PolarsBackend(Backend):
             agg_expr = agg_expr.over(group_by)
         return data.with_columns((pl.col(column) - agg_expr).alias(new_column))
 
+    def math_diff_lag(
+        self,
+        data: pl.DataFrame,
+        column: str,
+        order_by: list[str],
+        new_column: str,
+        group_by: list[str] | None,
+        lag: int,
+    ) -> pl.DataFrame:
+        if group_by:
+            expr = pl.col(column) - pl.col(column).shift(lag).over(
+                partition_by=group_by, order_by=order_by
+            )
+            return data.with_columns(expr.alias(new_column))
+        data = data.sort(order_by)
+        return data.with_columns(
+            (pl.col(column) - pl.col(column).shift(lag)).alias(new_column)
+        )
+
     def math_standardize(
         self,
         data: pl.DataFrame,

{transformplan-0.1.3 → transformplan-0.2.0}/transformplan/chunking.py

@@ -97,6 +97,7 @@ OPERATION_CHUNK_REGISTRY: dict[str, OperationMeta] = {
     "math_diff_from_agg": OperationMeta(
         ChunkMode.GROUP_DEPENDENT, group_param="group_by"
     ),
+    "math_diff_lag": OperationMeta(ChunkMode.GROUP_DEPENDENT, group_param="group_by"),
     # String operations - all chunkable
     "str_replace": OperationMeta(ChunkMode.CHUNKABLE),
     "str_slice": OperationMeta(ChunkMode.CHUNKABLE),

{transformplan-0.1.3 → transformplan-0.2.0}/transformplan/ops/math.py

@@ -327,6 +327,46 @@ class MathOps:
             },
         )
 
+    def math_diff_lag(
+        self,
+        column: str,
+        *,
+        order_by: str | list[str],
+        new_column: str,
+        group_by: str | list[str] | None = None,
+        lag: int = 1,
+    ) -> Self:
+        """Compute row-to-row difference using lag.
+
+        Calculates column - LAG(column, lag) ordered by order_by and optionally
+        partitioned by group_by. Works on numeric columns (result is float) and
+        datetime columns (result is duration).
+
+        Args:
+            column: Source column (numeric or datetime).
+            order_by: Column(s) defining row order.
+            new_column: Name for result column.
+            group_by: Column(s) to partition by. None for global ordering.
+            lag: Number of rows to look back (must be >= 1).
+
+        Returns:
+            Self for method chaining.
+        """
+        if isinstance(order_by, str):
+            order_by = [order_by]
+        if isinstance(group_by, str):
+            group_by = [group_by]
+        return self._register(
+            "math_diff_lag",
+            {
+                "column": column,
+                "order_by": order_by,
+                "new_column": new_column,
+                "group_by": group_by,
+                "lag": lag,
+            },
+        )
+
     # =========================================================================
     # Scaling Operations
     # =========================================================================

{transformplan-0.1.3 → transformplan-0.2.0}/transformplan/validation.py

@@ -961,6 +961,51 @@ def _validate_math_diff_from_agg(
         tracker.add_column(new_column, tracker.float_type)
 
 
+def _validate_math_diff_lag(
+    tracker: SchemaTracker, params: dict[str, Any], result: ValidationResult, step: int
+) -> None:
+    column = params["column"]
+    order_by = params["order_by"]
+    new_column = params["new_column"]
+    group_by = params.get("group_by")
+
+    if _check_column_exists(tracker, column, result, step, "math_diff_lag"):
+        dtype = tracker.get_dtype(column)
+        if not (tracker.is_numeric(dtype) or tracker.is_datetime(dtype)):
+            result.add_error(
+                step,
+                "math_diff_lag",
+                f"Column '{column}' must be numeric or datetime, "
+                f"got {tracker.type_name(dtype)}",
+            )
+
+    missing_order = [c for c in order_by if not tracker.has_column(c)]
+    if missing_order:
+        result.add_error(
+            step,
+            "math_diff_lag",
+            f"Order-by columns do not exist: {missing_order}",
+        )
+
+    if group_by:
+        missing_group = [c for c in group_by if not tracker.has_column(c)]
+        if missing_group:
+            result.add_error(
+                step,
+                "math_diff_lag",
+                f"Group-by columns do not exist: {missing_group}",
+            )
+
+    if tracker.has_column(column):
+        dtype = tracker.get_dtype(column)
+        out_type = (
+            tracker.duration_type if tracker.is_datetime(dtype) else tracker.float_type
+        )
+        tracker.add_column(new_column, out_type)
+    else:
+        tracker.add_column(new_column, tracker.float_type)
+
+
 def _validate_math_percent_of(
     tracker: SchemaTracker, params: dict[str, Any], result: ValidationResult, step: int
 ) -> None:
@@ -1590,6 +1635,7 @@ _VALIDATORS: dict[str, ValidatorFunc] = {
     "math_cumsum": _validate_math_cumsum,
     "math_rank": _validate_math_rank,
     "math_diff_from_agg": _validate_math_diff_from_agg,
+    "math_diff_lag": _validate_math_diff_lag,
     "math_percent_of": _validate_math_percent_of,
     # Scaling ops
     "math_standardize": partial(_validate_math_scaling, op_name="math_standardize"),

{transformplan-0.1.3 → transformplan-0.2.0/transformplan.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: transformplan
-Version: 0.1.3
+Version: 0.2.0
 Summary: Safe, reproducible data transformations with built-in auditing and validation
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/limebit/transformplan
@@ -52,7 +52,7 @@ Dynamic: license-file
 ```python
 from transformplan import TransformPlan, Col
 
-# Build readable pipelines with 88 chainable operations
+# Build readable pipelines with 89 chainable operations
 plan = (
     TransformPlan()
     # Standardize column names