jerry-thomas 1.0.3__py3-none-any.whl → 2.0.1__py3-none-any.whl

datapipeline/transforms/filter.py

@@ -1,5 +1,5 @@
 from collections.abc import Iterator
-from typing import Any
+from typing import Any, Callable
 
 from datapipeline.filters import filters as _filters
 from datapipeline.plugins import FILTERS_EP
@@ -23,31 +23,24 @@ _ALIAS = {
 }
 
 
-def _normalize_op(op: str) -> str:
+def normalize_operator(op: str) -> str:
     op = (op or "").strip()
     return _ALIAS.get(op, op)
 
 
-def filter(
-    stream: Iterator[Any],
-    *,
+def resolve_filter(
     operator: str,
-    field: str,
+    *,
     comparand: Any,
-) -> Iterator[Any]:
-    """Generic filter transform.
+) -> tuple[str, Any | None]:
+    """Resolve a normalized operator and callable filter function.
 
-    Parameters
-    - operator: one of eq, ne, lt, le, gt, ge, in, nin (case-sensitive), or a common alias
-    - field: record attribute/key to compare
-    - comparand: scalar for unary operators; list/tuple/set for membership (in/nin)
+    Returns (op, fn) where fn may be None if comparand is missing.
     """
-
     if is_missing(comparand):
-        # Skip filter when comparand is an unresolved placeholder.
-        return stream
+        return "", None
 
-    op = _normalize_op(operator)
+    op = normalize_operator(operator)
     fn = None
     try:
         fn = load_ep(FILTERS_EP, op)
@@ -57,4 +50,67 @@ def filter(
         raise ValueError(
             f"Unsupported filter operator: {operator!r} (normalized: {op!r})"
         )
-    return fn(stream, field, comparand)
+    return op, fn
+
+
+def apply_filter(
+    stream: Iterator[Any],
+    *,
+    field_getter: Callable[[Any, str], Any],
+    operator: str,
+    field: str,
+    comparand: Any,
+) -> Iterator[Any]:
+    op, fn = resolve_filter(operator, comparand=comparand)
+    if fn is None:
+        return stream
+    if getattr(fn, "__module__", None) != _filters.__name__:
+        return fn(stream, field, comparand)
+
+    if op in {"in_", "nin"}:
+        bag = _filters._as_set(comparand)
+
+        def apply_in() -> Iterator[Any]:
+            for record in stream:
+                left = field_getter(record, field)
+                if (left in bag) == (op == "in_"):
+                    yield record
+
+        return apply_in()
+
+    cmp = getattr(_filters._op, op, None)
+    if cmp is None:
+        raise ValueError(
+            f"Unsupported filter operator: {operator!r} (normalized: {op!r})"
+        )
+
+    def apply_cmp() -> Iterator[Any]:
+        for record in stream:
+            left = field_getter(record, field)
+            if _filters.compare_values(left, comparand, cmp):
+                yield record
+
+    return apply_cmp()
+
+
+def filter(
+    stream: Iterator[Any],
+    *,
+    operator: str,
+    field: str,
+    comparand: Any,
+) -> Iterator[Any]:
+    """Generic filter transform.
+
+    Parameters
+    - operator: one of eq, ne, lt, le, gt, ge, in, nin (case-sensitive), or a common alias
+    - field: record attribute/key to compare
+    - comparand: scalar for unary operators; list/tuple/set for membership (in/nin)
+    """
+    return apply_filter(
+        stream,
+        field_getter=_filters.get_field,
+        operator=operator,
+        field=field,
+        comparand=comparand,
+    )

datapipeline/transforms/interfaces.py

@@ -0,0 +1,58 @@
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from typing import Any, TypeVar
+
+from datapipeline.domain.record import TemporalRecord
+
+
+class StreamTransformBase(ABC):
+    """Base interface for stream transforms over TemporalRecord."""
+
+    def __call__(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        return self.apply(stream)
+
+    @abstractmethod
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        ...
+
+
+class FieldStreamTransformBase(StreamTransformBase):
+    """Base for stream transforms that read/write a record field."""
+
+    def __init__(
+        self,
+        field: str,
+        to: str | None = None,
+        partition_by: str | list[str] | None = None,
+    ) -> None:
+        if not field:
+            raise ValueError("field is required")
+        self.field = field
+        self.to = to or field
+        self.partition_by = partition_by
+
+    def _ensure_output_field(
+        self,
+        record: TemporalRecord,
+        value: Any = None,
+    ) -> TemporalRecord:
+        if self.to is None:
+            return record
+        if hasattr(record, self.to):
+            return record
+        setattr(record, self.to, value)
+        return record
+
+
+TRecord = TypeVar("TRecord", bound=TemporalRecord)
+
+
+class RecordTransformBase(ABC):
+    """Base interface for record transforms over TemporalRecord."""
+
+    def __call__(self, stream: Iterator[TRecord]) -> Iterator[TRecord]:
+        return self.apply(stream)
+
+    @abstractmethod
+    def apply(self, stream: Iterator[TRecord]) -> Iterator[TRecord]:
+        ...

datapipeline/transforms/record/floor_time.py

@@ -1,17 +1,20 @@
-from __future__ import annotations
-
 from typing import Iterator
 
 from datapipeline.domain.record import TemporalRecord
-from datapipeline.config.dataset.normalize import floor_time_to_bucket
+from datapipeline.transforms.interfaces import RecordTransformBase
+from datapipeline.transforms.utils import floor_record_time
 
 
-def floor_time(stream: Iterator[TemporalRecord], cadence: str) -> Iterator[TemporalRecord]:
+class FloorTimeRecordTransform(RecordTransformBase):
     """Floor record timestamps to the given cadence bucket (e.g., '1h', '10min').
 
     Useful before granularity aggregation to downsample within bins by making
     all intra-bin records share the same timestamp.
     """
-    for record in stream:
-        record.time = floor_time_to_bucket(record.time, cadence)
-        yield record
+
+    def __init__(self, cadence: str) -> None:
+        self.cadence = cadence
+
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        for record in stream:
+            yield floor_record_time(record, self.cadence)

datapipeline/transforms/record/lag.py

@@ -1,18 +1,16 @@
-from __future__ import annotations
-
 from datetime import timedelta
 from typing import Iterator
 
 from datapipeline.domain.record import TemporalRecord
 from datapipeline.utils.time import parse_timecode
+from datapipeline.transforms.interfaces import RecordTransformBase
 
 
-def _shift_record_time(record: TemporalRecord, lag: timedelta) -> TemporalRecord:
-    record.time = record.time - lag
-    return record
-
+class LagRecordTransform(RecordTransformBase):
+    def __init__(self, lag: str) -> None:
+        self.lag = parse_timecode(lag)
 
-def apply_lag(stream: Iterator[TemporalRecord], lag: str) -> Iterator[TemporalRecord]:
-    lag_td = parse_timecode(lag)
-    for record in stream:
-        yield _shift_record_time(record, lag_td)
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        for record in stream:
+            record.time = record.time - self.lag
+            yield record

datapipeline/transforms/sequence.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from collections import deque
 from itertools import groupby
 from typing import Iterator
@@ -33,7 +31,7 @@ class WindowTransformer:
         """Assumes input is pre-sorted by (feature_id, record.time).
 
         Produces sliding windows per feature_id. Each output carries a
-        list[Record] in ``records``.
+        list[Record] in ``records`` and the selected values in ``values``.
         """
 
         grouped = groupby(stream, key=lambda fr: fr.id)
@@ -46,6 +44,7 @@ class WindowTransformer:
                 if len(window) == self.size and step % self.stride == 0:
                     yield FeatureRecordSequence(
                         records=[r.record for r in window],
+                        values=[r.value for r in window],
                         id=fid,
                     )
                 step += 1

datapipeline/transforms/stream/dedupe.py

@@ -1,22 +1,20 @@
-from __future__ import annotations
-
 from collections.abc import Iterator
 
-from datapipeline.domain.feature import FeatureRecord
+from datapipeline.domain.record import TemporalRecord
 
 
 class FeatureDeduplicateTransform:
-    """Drop consecutive identical feature records (id + timestamp + payload)."""
+    """Drop consecutive identical records (timestamp + payload)."""
 
     def __init__(self, **_: object) -> None:
         # Accept arbitrary config mapping for consistency with other transforms.
         pass
 
-    def __call__(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
+    def __call__(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
         return self.apply(stream)
 
-    def apply(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
-        last: FeatureRecord | None = None
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        last: TemporalRecord | None = None
         for record in stream:
             if last is not None and record == last:
                 continue

datapipeline/transforms/stream/ensure_ticks.py

@@ -1,34 +1,49 @@
 from typing import Iterator
 
-from dataclasses import replace
-
-from datapipeline.domain.feature import FeatureRecord
 from datapipeline.domain.record import TemporalRecord
+from datapipeline.transforms.interfaces import FieldStreamTransformBase
+from datapipeline.transforms.utils import clone_record, get_field, partition_key
 from datapipeline.utils.time import parse_timecode
 
 
-def ensure_cadence(stream: Iterator[FeatureRecord], cadence: str) -> Iterator[FeatureRecord]:
-    """Insert placeholder FeatureRecords so timestamps are exactly one cadence apart per feature id.
+class EnsureCadenceTransform(FieldStreamTransformBase):
+    """Insert placeholder records so timestamps are exactly one cadence apart per partition.
 
     - cadence: duration string (e.g., "10m", "1h", "30s").
-    - Placeholders carry value=None and inherit the feature id; group bucketing
-      is applied later at vector assembly from record.time.
-    - Assumes input sorted by (feature_id, record.time).
+    - Placeholders carry field=None and inherit partition metadata.
+    - Assumes input sorted by (partition_key, record.time).
     """
-    step = parse_timecode(cadence)
-    last: FeatureRecord | None = None
-    for fr in stream:
-        if (last is None) or (last.id != fr.id):
-            yield fr
-            last = fr
-            continue
 
-        expect = last.record.time + step
-        while expect < fr.record.time:
-            yield FeatureRecord(
-                record=replace(last.record, time=expect, value=None),
-                id=fr.id,
-            )
-            expect = expect + step
-        yield fr
-        last = fr
+    def __init__(
+        self,
+        *,
+        cadence: str,
+        field: str,
+        to: str | None = None,
+        partition_by: str | list[str] | None = None,
+    ) -> None:
+        super().__init__(field=field, to=to, partition_by=partition_by)
+        self.cadence = cadence
+
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        step = parse_timecode(self.cadence)
+        last: TemporalRecord | None = None
+        last_key: tuple | None = None
+        for record in stream:
+            if self.to != self.field:
+                record = self._ensure_output_field(
+                    record, get_field(record, self.field)
+                )
+            key = partition_key(record, self.partition_by)
+            if last is None or last_key != key:
+                yield record
+                last = record
+                last_key = key
+                continue
+
+            expect = last.time + step
+            while expect < record.time:
+                yield clone_record(last, time=expect, **{self.to: None})
+                expect = expect + step
+            yield record
+            last = record

datapipeline/transforms/stream/fill.py

@@ -1,17 +1,19 @@
+from collections import deque
 from itertools import groupby
 from statistics import mean, median
-from typing import Any, Iterator
-from collections import deque
-
-from datapipeline.domain.feature import FeatureRecord, FeatureRecordSequence
-from datapipeline.transforms.utils import is_missing, clone_record_with_value
+from typing import Iterator
 
+from datapipeline.domain.record import TemporalRecord
+from datapipeline.transforms.interfaces import FieldStreamTransformBase
+from datapipeline.transforms.utils import (
+    get_field,
+    is_missing,
+    clone_record_with_field,
+    partition_key,
+)
 
-def _extract_value(record: Any) -> Any:
-    return getattr(record, "value", None)
 
-
-class FillTransformer:
+class FillTransformer(FieldStreamTransformBase):
     """Time-aware imputer using a strict rolling tick window.
 
     - window: number of recent ticks to consider (including missing ticks). A
@@ -23,7 +25,17 @@ class FillTransformer:
       window.
     """
 
-    def __init__(self, statistic: str = "median", window: int | None = None, min_samples: int = 1) -> None:
+    def __init__(
+        self,
+        *,
+        field: str,
+        to: str | None = None,
+        statistic: str = "median",
+        window: int | None = None,
+        min_samples: int = 1,
+        partition_by: str | list[str] | None = None,
+    ) -> None:
+        super().__init__(field=field, to=to, partition_by=partition_by)
         if window is None or window <= 0:
             raise ValueError("window must be a positive integer")
         if min_samples <= 0:
@@ -43,21 +55,19 @@ class FillTransformer:
             return None
         return float(self.statistic(values))
 
-    def __call__(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecordSequence]:
-        return self.apply(stream)
-
-    def apply(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecordSequence]:
-        grouped = groupby(stream, key=lambda fr: fr.id)
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        grouped = groupby(stream, key=lambda rec: partition_key(rec, self.partition_by))
 
-        for id, feature_records in grouped:
+        for _, records in grouped:
             # Store the last `window` ticks with a flag marking whether the tick
             # had an original (non-filled) valid value, and its numeric value.
             tick_window: deque[tuple[bool, float | None]] = deque(maxlen=self.window)
 
-            for fr in feature_records:
-                if isinstance(fr.record, FeatureRecordSequence):
-                    raise TypeError("Fills should run before windowing transforms")
-                value = _extract_value(fr.record)
+            for record in records:
+                value = get_field(record, self.field)
+                record = self._ensure_output_field(
+                    record, None if is_missing(value) else value
+                )
 
                 if is_missing(value):
                     # Count valid values in the current window
@@ -67,15 +77,14 @@ class FillTransformer:
                         if fill is not None:
                             # Do NOT treat filled value as original valid; append a missing marker
                             tick_window.append((False, None))
-                            yield FeatureRecord(
-                                record=clone_record_with_value(fr.record, fill),
-                                id=id,
+                            yield clone_record_with_field(
+                                record, self.to, fill
                             )
                             continue
                     # Not enough valid samples in window: pass through missing
                     tick_window.append((False, None))
-                    yield fr
+                    yield record
                 else:
                     as_float = float(value)
                     tick_window.append((True, as_float))
-                    yield fr
+                    yield record

datapipeline/transforms/stream/filter.py

@@ -0,0 +1,25 @@
+from collections.abc import Iterator
+from typing import Any
+
+from datapipeline.domain.record import TemporalRecord
+from datapipeline.filters import filters as _filters
+from datapipeline.transforms.filter import apply_filter
+from datapipeline.transforms.interfaces import StreamTransformBase
+
+
+class FilterTransform(StreamTransformBase):
+    """Filter records by comparing a field on record payloads."""
+
+    def __init__(self, operator: str, field: str, comparand: Any) -> None:
+        self.operator = operator
+        self.field = field
+        self.comparand = comparand
+
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        return apply_filter(
+            stream,
+            field_getter=_filters.get_field,
+            operator=self.operator,
+            field=self.field,
+            comparand=self.comparand,
+        )

datapipeline/transforms/stream/floor_time.py

@@ -0,0 +1,16 @@
+from typing import Iterator
+
+from datapipeline.domain.record import TemporalRecord
+from datapipeline.transforms.interfaces import StreamTransformBase
+from datapipeline.transforms.utils import floor_record_time
+
+
+class FloorTimeTransform(StreamTransformBase):
+    """Floor record timestamps to the given cadence bucket."""
+
+    def __init__(self, cadence: str) -> None:
+        self.cadence = cadence
+
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        for record in stream:
+            yield floor_record_time(record, self.cadence)

datapipeline/transforms/stream/granularity.py

@@ -1,79 +1,101 @@
-from __future__ import annotations
-
 from statistics import mean, median
 from typing import Iterator
 
-from datapipeline.domain.feature import FeatureRecord
+from datapipeline.domain.record import TemporalRecord
+from datapipeline.transforms.interfaces import FieldStreamTransformBase
+from datapipeline.transforms.utils import (
+    get_field,
+    clone_record_with_field,
+    partition_key,
+)
 
 
-class FeatureGranularityTransform:
-    """Normalize same-timestamp duplicates for non-sequence features.
+class FeatureGranularityTransform(FieldStreamTransformBase):
+    """Normalize same-timestamp duplicates for non-sequence streams.
 
     Single-argument API (preferred for concise YAML):
       - "first" | "last" | "mean" | "median" => aggregate duplicates within a timestamp.
     """
 
-    def __init__(self, mode: str = "first") -> None:
+    def __init__(
+        self,
+        *,
+        field: str,
+        to: str | None = None,
+        mode: str = "first",
+        partition_by: str | list[str] | None = None,
+    ) -> None:
+        super().__init__(field=field, to=to, partition_by=partition_by)
         if mode not in {"first", "last", "mean", "median"}:
             raise ValueError(f"Unsupported granularity mode: {mode!r}")
         self.mode = mode
 
-    def _aggregate(self, items: list[FeatureRecord]) -> FeatureRecord:
+    def _aggregate(self, items: list[TemporalRecord]) -> TemporalRecord:
         vals: list[float] = []
-        for fr in items:
-            vals.append(float(fr.record.value))
+        for rec in items:
+            vals.append(float(get_field(rec, self.field)))
         if self.mode == "mean":
             agg_val = mean(vals)
         elif self.mode == "median":
             agg_val = median(vals)
         new = items[-1]
-        new.record.value = agg_val
-        return new
-
-    def __call__(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
-        return self.apply(stream)
+        return clone_record_with_field(new, self.to, agg_val)
 
-    def apply(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
         """Aggregate duplicates per timestamp while preserving order.
 
-        Precondition: input is sorted by (feature_id, record.time).
+        Precondition: input is sorted by (partition_key, record.time).
 
-        We process one base feature stream at a time (feature_id),
+        We process one base stream at a time (partition_key),
         bucket its records by timestamp, then aggregate each bucket according to
         the selected mode (first/last/mean/median), emitting in increasing timestamp
         order.
         """
 
-        # State for the current base stream: id
-        current_key: str | None = None
+        # State for the current base stream: partition key
+        current_key: tuple | None = None
         # Buckets of same-timestamp duplicates for the current base stream
         # Maintain insertion order of timestamps as encountered
-        time_buckets: dict[object, list[FeatureRecord]] = {}
+        time_buckets: dict[object, list[TemporalRecord]] = {}
 
-        def flush_current() -> Iterator[FeatureRecord]:
+        def flush_current() -> Iterator[TemporalRecord]:
             if current_key is None or not time_buckets:
                 return iter(())
 
             # Ordered list of timestamps as they appeared in the input
             ordered_times = list(time_buckets.keys())
 
-            out: list[FeatureRecord] = []
+            out: list[TemporalRecord] = []
             for t in ordered_times:
                 bucket = time_buckets.get(t, [])
                 if not bucket:
                     continue
                 if self.mode == "last":
-                    out.append(bucket[-1])
+                    last = bucket[-1]
+                    out.append(
+                        clone_record_with_field(
+                            last,
+                            self.to,
+                            get_field(last, self.field),
+                        )
+                    )
                 elif self.mode == "first":
-                    out.append(bucket[0])
+                    first = bucket[0]
+                    out.append(
+                        clone_record_with_field(
+                            first,
+                            self.to,
+                            get_field(first, self.field),
+                        )
+                    )
                 else:
                     out.append(self._aggregate(bucket))
             return iter(out)
 
-        for fr in stream:
-            base_key = fr.id
-            t = getattr(fr.record, "time", None)
-            # Start new base stream when feature_id changes
+        for record in stream:
+            base_key = partition_key(record, self.partition_by)
+            t = getattr(record, "time", None)
+            # Start new base stream when partition key changes
             if current_key is not None and base_key != current_key:
                 for out in flush_current():
                     yield out
@@ -82,9 +104,9 @@ class FeatureGranularityTransform:
             # Append to the bucket for this timestamp
             bucket = time_buckets.get(t)
             if bucket is None:
-                time_buckets[t] = [fr]
+                time_buckets[t] = [record]
             else:
-                bucket.append(fr)
+                bucket.append(record)
 
         # Flush any remaining base stream
         if current_key is not None:

datapipeline/transforms/stream/lag.py

@@ -0,0 +1,17 @@
+from typing import Iterator
+
+from datapipeline.domain.record import TemporalRecord
+from datapipeline.transforms.interfaces import StreamTransformBase
+from datapipeline.utils.time import parse_timecode
+
+
+class LagTransform(StreamTransformBase):
+    """Shift record timestamps backwards by the given lag."""
+
+    def __init__(self, lag: str) -> None:
+        self.lag = parse_timecode(lag)
+
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        for record in stream:
+            record.time = record.time - self.lag
+            yield record