jerry-thomas 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

datapipeline/templates/stubs/parser.py.j2

@@ -1,5 +1,7 @@
 from typing import Any
+
 from datapipeline.sources.models.parser import DataParser
+
 from .dto import {{DTO_CLASS}}
 
 

datapipeline/templates/stubs/record.py.j2

@@ -1,6 +1,8 @@
 from dataclasses import dataclass
+
 from datapipeline.domain.record import {{PARENT_CLASS}}
 
+
 @dataclass
 class {{CLASS_NAME}}({{PARENT_CLASS}}):
     """

datapipeline/templates/stubs/source.yaml.j2

@@ -1,6 +1,5 @@
-# Required identifier for this raw source. This value is referenced by
-# contracts under `source_id:`.
-source_id: "{{ source_id }}"
+# Required identifier for this raw source. Contracts reference it under `source:`.
+id: "{{ id }}"  # format: provider.dataset
 
 # parser.entrypoint: registered parser name (not a file path)
 parser:

datapipeline/transforms/debug/lint.py

@@ -1,24 +1,21 @@
 import logging
-import math
+from datetime import timedelta
 from itertools import groupby
 from typing import Iterator
 
 from datapipeline.domain.feature import FeatureRecord
-from datapipeline.transforms.utils import is_missing
+from datapipeline.utils.time import parse_timecode
+
+
 logger = logging.getLogger(__name__)
 
 
 class StreamLint:
-    """Validate a feature stream and emit actionable hints.
+    """Validate structural properties of a feature stream (order, cadence, duplicates).
 
     Parameters
     - mode: 'warn' (default) logs warnings; 'error' raises on first violation
     - tick: optional cadence (e.g. '1h', '10m'); when set, check regularity
-    - check_missing: flag missing values (value is None/NaN)
-    - check_regular: flag gaps vs. expected tick
-    - check_duplicates: flag multiple records with same timestamp
-    - check_order: flag out-of-order timestamps within a feature stream
-    - check_finite: flag non-finite values (NaN/Inf)
     """
 
     def __init__(
@@ -26,19 +23,20 @@ class StreamLint:
         *,
         mode: str = "warn",
         tick: str | None = None,
-        check_missing: bool = True,
-        check_regular: bool = True,
-        check_duplicates: bool = True,
-        check_order: bool = True,
-        check_finite: bool = True,
     ) -> None:
         self.mode = mode
         self.tick = tick
-        self.check_missing = check_missing
-        self.check_regular = check_regular
-        self.check_duplicates = check_duplicates
-        self.check_order = check_order
-        self.check_finite = check_finite
+
+        # Pre-compute tick step in seconds when provided to avoid repeated parsing.
+        self._tick_seconds: int | None = None
+        if self.tick:
+            try:
+                self._tick_seconds = int(parse_timecode(self.tick).total_seconds())
+            except Exception:
+                logger.warning(
+                    "StreamLint: invalid tick %r (cadence checks disabled)", self.tick
+                )
+                self._tick_seconds = None
 
     def __call__(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
         return self.apply(stream)
@@ -55,46 +53,33 @@ class StreamLint:
             seen_times: set = set()
             for fr in records:
                 t = getattr(fr.record, "time", None)
-                v = getattr(fr.record, "value", None)
 
                 # Check ordering
-                if self.check_order and last_time is not None and t is not None and t < last_time:
+                if last_time is not None and t is not None and t < last_time:
                     self._violation(
                         f"out-of-order timestamp for feature '{fid}': {t} < {last_time}. "
                         f"Consider sorting upstream or fixing loader."
                     )
 
                 # Check duplicates
-                if self.check_duplicates and t in seen_times:
+                if t in seen_times:
                     self._violation(
                         f"duplicate timestamp for feature '{fid}' at {t}. "
                         f"Consider a granularity transform (first/last/mean/median)."
                     )
                 seen_times.add(t)
 
-                # Check missing / non-finite
-                if self.check_missing and is_missing(v):
-                    self._violation(
-                        f"missing value for feature '{fid}' at {t}. "
-                        f"Consider using a fill transform."
-                    )
-                if self.check_finite and isinstance(v, float) and not math.isfinite(v):
-                    self._violation(
-                        f"non-finite value for feature '{fid}' at {t}: {v}. "
-                        f"Consider filtering or scaling."
-                    )
-
-                # Regularity check requires explicit tick; done at stream layer via ensure_ticks normally
-                if self.check_regular and self.tick and last_time is not None and t is not None:
-                    # Lazy import to avoid cycle
-                    from datapipeline.utils.time import parse_timecode
-
-                    step = parse_timecode(self.tick)
-                    expect = last_time + step
+                # Regularity check requires explicit tick; done at stream layer via ensure_cadence normally
+                if (
+                    self._tick_seconds
+                    and last_time is not None
+                    and t is not None
+                ):
+                    expect = last_time + timedelta(seconds=self._tick_seconds)
                     if t != expect and t > expect:
                         self._violation(
                             f"skipped tick(s) for feature '{fid}': expected {expect}, got {t}. "
-                            f"Consider using ensure_ticks."
+                            f"Consider using ensure_cadence."
                         )
 
                 last_time = t

datapipeline/transforms/feature/scaler.py

@@ -3,13 +3,14 @@ from collections import defaultdict
 from itertools import groupby
 from numbers import Real
 from pathlib import Path
-from typing import Any, Iterator
+from typing import Any, Callable, Iterator, Literal, Mapping
 
 from datapipeline.domain.feature import FeatureRecord
-from datapipeline.domain.record import TemporalRecord
+from datapipeline.domain.sample import Sample
 from datapipeline.transforms.feature.model import FeatureTransform
 from datapipeline.transforms.utils import clone_record_with_value
 from datapipeline.utils.pickle_model import PicklePersistanceMixin
+from datapipeline.pipeline.observability import TransformEvent
 
 
 def _iter_numeric_values(value: Any) -> Iterator[float]:
@@ -38,12 +39,14 @@ class StandardScaler(PicklePersistanceMixin):
         self.with_std = with_std
         self.epsilon = epsilon
         self.statistics: dict[str, dict[str, float | int]] = {}
+        self.missing_counts: dict[str, int] = {}
 
-    def fit(self, vectors: Iterator[tuple[Any, Any]]) -> int:
+    def fit(self, vectors: Iterator[Sample]) -> int:
         trackers: dict[str, StandardScaler._RunningStats] = defaultdict(
             self._RunningStats)
         total = 0
-        for _, vector in vectors:
+        for sample in vectors:
+            vector = sample.features
             values = getattr(vector, "values", {})
             for fid, raw in values.items():
                 for value in _iter_numeric_values(raw):
@@ -61,11 +64,19 @@ class StandardScaler(PicklePersistanceMixin):
         }
         return total
 
-    def transform(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
+    def transform(
+        self,
+        stream: Iterator[FeatureRecord],
+        *,
+        on_none: Literal["error", "skip"] = "skip",
+        observer: Callable[[TransformEvent], None] | None = None,
+    ) -> Iterator[FeatureRecord]:
         if not self.statistics:
             raise RuntimeError(
                 "StandardScaler must be fitted before calling transform().")
 
+        self.missing_counts = {}
+
         grouped = groupby(stream, key=lambda fr: fr.id)
         for feature_id, records in grouped:
             stats = self.statistics.get(feature_id)
@@ -75,7 +86,29 @@ class StandardScaler(PicklePersistanceMixin):
             mean = float(stats.get("mean", 0.0))
             std = float(stats.get("std", 1.0))
             for fr in records:
-                raw = self._extract_value(fr.record)
+                value = fr.record.value
+                if not isinstance(value, Real):
+                    if value is None and on_none == "skip":
+                        self.missing_counts[feature_id] = (
+                            self.missing_counts.get(feature_id, 0) + 1
+                        )
+                        if observer is not None:
+                            observer(
+                                TransformEvent(
+                                    type="scaler_none",
+                                    payload={
+                                        "feature_id": feature_id,
+                                        "record": fr.record,
+                                        "count": self.missing_counts[feature_id],
+                                    },
+                                )
+                            )
+                        yield fr
+                        continue
+                    raise TypeError(
+                        f"Record value must be numeric, got {value!r}")
+
+                raw = float(value)
                 normalized = raw
                 if self.with_mean:
                     normalized -= mean
@@ -86,12 +119,36 @@ class StandardScaler(PicklePersistanceMixin):
                     id=fr.id,
                 )
 
-    @staticmethod
-    def _extract_value(record: TemporalRecord) -> float:
-        value = record.value
-        if isinstance(value, Real):
-            return float(value)
-        raise TypeError(f"Record value must be numeric, got {value!r}")
+    def inverse_transform(
+        self,
+        stream: Iterator[FeatureRecord],
+    ) -> Iterator[FeatureRecord]:
+        if not self.statistics:
+            raise RuntimeError(
+                "StandardScaler must be fitted before calling inverse_transform().")
+
+        grouped = groupby(stream, key=lambda fr: fr.id)
+        for feature_id, records in grouped:
+            stats = self.statistics.get(feature_id)
+            if not stats:
+                raise KeyError(
+                    f"Missing scaler statistics for feature '{feature_id}'.")
+            mean = float(stats.get("mean", 0.0))
+            std = float(stats.get("std", 1.0))
+            for fr in records:
+                value = fr.record.value
+                if not isinstance(value, Real):
+                    raise TypeError(
+                        f"Record value must be numeric, got {value!r}")
+                restored = float(value)
+                if self.with_std:
+                    restored *= std
+                if self.with_mean:
+                    restored += mean
+                yield FeatureRecord(
+                    record=clone_record_with_value(fr.record, restored),
+                    id=fr.id,
+                )
 
     class _RunningStats:
         __slots__ = ("count", "mean", "m2")
@@ -132,6 +189,8 @@ class StandardScalerTransform(FeatureTransform):
         with_mean: bool = True,
         with_std: bool = True,
         epsilon: float = 1e-12,
+        on_none: Literal["error", "skip"] = "skip",
+        observer: Callable[[TransformEvent], None] | None = None,
     ) -> None:
         base: StandardScaler
         if scaler is not None:
@@ -152,6 +211,23 @@ class StandardScalerTransform(FeatureTransform):
             epsilon=epsilon,
         )
         self._scaler.statistics = dict(base.statistics)
+        self._on_none = on_none
+        self._observer = observer
+
+    @property
+    def missing_counts(self) -> dict[str, int]:
+        return dict(self._scaler.missing_counts)
+
+    def set_observer(self, observer: Callable[[TransformEvent], None] | None) -> None:
+        self._observer = observer
 
     def apply(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
-        yield from self._scaler.transform(stream)
+        yield from self._scaler.transform(
+            stream,
+            on_none=self._on_none,
+            observer=self._observer,
+        )
+
+    def inverse(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
+        """Undo scaling using the fitted statistics."""
+        yield from self._scaler.inverse_transform(stream)

datapipeline/transforms/record/floor_time.py

@@ -3,15 +3,15 @@ from __future__ import annotations
 from typing import Iterator
 
 from datapipeline.domain.record import TemporalRecord
-from datapipeline.config.dataset.normalize import floor_time_to_resolution
+from datapipeline.config.dataset.normalize import floor_time_to_bucket
 
 
-def floor_time(stream: Iterator[TemporalRecord], resolution: str) -> Iterator[TemporalRecord]:
-    """Floor record timestamps to the given resolution (e.g., '1h', '10min').
+def floor_time(stream: Iterator[TemporalRecord], cadence: str) -> Iterator[TemporalRecord]:
+    """Floor record timestamps to the given cadence bucket (e.g., '1h', '10min').
 
     Useful before granularity aggregation to downsample within bins by making
     all intra-bin records share the same timestamp.
     """
     for record in stream:
-        record.time = floor_time_to_resolution(record.time, resolution)
+        record.time = floor_time_to_bucket(record.time, cadence)
         yield record

datapipeline/transforms/sequence.py

@@ -1,14 +1,10 @@
 from __future__ import annotations
 
 from collections import deque
-import logging
 from itertools import groupby
 from typing import Iterator
 
 from datapipeline.domain.feature import FeatureRecord, FeatureRecordSequence
-from datapipeline.utils.time import parse_timecode
-
-logger = logging.getLogger(__name__)
 
 
 class WindowTransformer:
@@ -16,25 +12,15 @@ class WindowTransformer:
         self,
         size: int,
         stride: int = 1,
-        *,
-        tick: str | None = None,
     ) -> None:
         """Sliding windows over time-ordered feature streams.
 
         Parameters
         - size: window length in steps (int).
         - stride: step between windows (int number of steps).
-        - tick: duration string denoting the expected cadence of the stream.
-                Supports 's', 'm', 'h', 'd'. When provided, enforce completeness: only emit windows if
-                consecutive records are exactly one tick apart; gaps reset the
-                window. Examples: "1h", "10m". Optional.
         """
 
         self.size = int(size)
-        self._tick_seconds: int | None = (
-            int(parse_timecode(tick).total_seconds()) if tick else None
-        )
-
         self.stride = int(stride)
 
         if self.size <= 0 or self.stride <= 0:
@@ -52,33 +38,14 @@ class WindowTransformer:
 
         grouped = groupby(stream, key=lambda fr: fr.id)
 
-        for id, records in grouped:
+        for fid, records in grouped:
             window = deque(maxlen=self.size)
             step = 0
-            last_time = None
             for fr in records:
-                # Enforce completeness when configured and tick is known
-                if self._tick_seconds is not None:
-                    t = getattr(fr.record, "time", None)
-                    if t is not None and last_time is not None:
-                        delta = int((t - last_time).total_seconds())
-                        if delta != self._tick_seconds:
-                            logger.debug(
-                                "sequence gap: feature_id=%s expected=%ss delta=%ss last=%s now=%s",
-                                id,
-                                self._tick_seconds,
-                                delta,
-                                last_time,
-                                t,
-                            )
-                            window.clear()
-                            step = 0
-                    last_time = t
-
                 window.append(fr)
                 if len(window) == self.size and step % self.stride == 0:
                     yield FeatureRecordSequence(
                         records=[r.record for r in window],
-                        id=id,
+                        id=fid,
                     )
                 step += 1

datapipeline/transforms/stream/dedupe.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from datapipeline.domain.feature import FeatureRecord
+
+
+class FeatureDeduplicateTransform:
+    """Drop consecutive identical feature records (id + timestamp + payload)."""
+
+    def __init__(self, **_: object) -> None:
+        # Accept arbitrary config mapping for consistency with other transforms.
+        pass
+
+    def __call__(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
+        return self.apply(stream)
+
+    def apply(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
+        last: FeatureRecord | None = None
+        for record in stream:
+            if last is not None and record == last:
+                continue
+            last = record
+            yield record

datapipeline/transforms/stream/ensure_ticks.py

@@ -1,20 +1,21 @@
 from typing import Iterator
 
-from datapipeline.domain.record import TemporalRecord
+from dataclasses import replace
+
 from datapipeline.domain.feature import FeatureRecord
+from datapipeline.domain.record import TemporalRecord
 from datapipeline.utils.time import parse_timecode
-from dataclasses import replace
 
 
-def ensure_ticks(stream: Iterator[FeatureRecord], tick: str) -> Iterator[FeatureRecord]:
-    """Insert placeholder FeatureRecords so timestamps are exactly one tick apart per feature id.
+def ensure_cadence(stream: Iterator[FeatureRecord], cadence: str) -> Iterator[FeatureRecord]:
+    """Insert placeholder FeatureRecords so timestamps are exactly one cadence apart per feature id.
 
-    - tick: duration string (e.g., "10m", "1h", "30s").
+    - cadence: duration string (e.g., "10m", "1h", "30s").
     - Placeholders carry value=None and inherit the feature id; group bucketing
       is applied later at vector assembly from record.time.
     - Assumes input sorted by (feature_id, record.time).
     """
-    step = parse_timecode(tick)
+    step = parse_timecode(cadence)
     last: FeatureRecord | None = None
     for fr in stream:
         if (last is None) or (last.id != fr.id):

datapipeline/transforms/vector/__init__.py

@@ -0,0 +1,5 @@
+from .common import VectorContextMixin, replace_vector, select_vector
+from .drop import VectorDropTransform
+from .ensure_schema import VectorEnsureSchemaTransform
+from .fill import VectorFillTransform
+from .replace import VectorReplaceTransform

datapipeline/transforms/vector/common.py

@@ -0,0 +1,98 @@
+from typing import Literal
+
+from datapipeline.domain.sample import Sample
+from datapipeline.domain.vector import Vector
+from datapipeline.pipeline.context import (
+    PipelineContext,
+    try_get_current_context,
+)
+
+
+def select_vector(sample: Sample, payload: Literal["features", "targets"]) -> Vector | None:
+    if payload == "targets":
+        return sample.targets
+    return sample.features
+
+
+def replace_vector(sample: Sample, payload: Literal["features", "targets"], vector: Vector) -> Sample:
+    if payload == "targets":
+        return sample.with_targets(vector)
+    return sample.with_features(vector)
+
+
+class VectorContextMixin:
+    def __init__(self, payload: Literal["features", "targets"] = "features") -> None:
+        if payload not in {"features", "targets"}:
+            raise ValueError("payload must be 'features' or 'targets'")
+        self._context: PipelineContext | None = None
+        self._payload = payload
+
+    def bind_context(self, context: PipelineContext) -> None:
+        self._context = context
+
+    def _expected_ids(self, payload: str | None = None) -> list[str]:
+        """Return expected feature/target ids for the given payload.
+
+        When `payload` is omitted, the instance default is used.
+        """
+        ctx = self._context or try_get_current_context()
+        if not ctx:
+            return []
+        kind = payload or self._payload
+        if kind not in {"features", "targets"}:
+            return []
+        schema = ctx.load_schema(payload=kind) or []
+        ids = [
+            entry.get("id")
+            for entry in schema
+            if isinstance(entry, dict) and isinstance(entry.get("id"), str)
+        ]
+        return ids or []
+
+
+class VectorPostprocessBase(VectorContextMixin):
+    """Shared envelope for vector postprocess transforms.
+
+    Provides a consistent contract for payload selection and id filtering:
+    - payload: features | targets | both
+    - only: optional allow-list of ids
+    - exclude: optional deny-list of ids
+    """
+
+    def __init__(
+        self,
+        *,
+        payload: Literal["features", "targets", "both"] = "features",
+        only: list[str] | None = None,
+        exclude: list[str] | None = None,
+    ) -> None:
+        if payload not in {"features", "targets", "both"}:
+            raise ValueError(
+                "payload must be 'features', 'targets', or 'both'")
+        base_payload = "features" if payload == "both" else payload
+        super().__init__(payload=base_payload)
+        self._payload_mode: Literal["features", "targets", "both"] = payload
+        self._only = {str(fid) for fid in (only or [])} or None
+        self._exclude = {str(fid) for fid in (exclude or [])} or None
+        self._baseline_cache: dict[str, list[str]] = {}
+
+    def _payload_kinds(self) -> list[Literal["features", "targets"]]:
+        mode = self._payload_mode
+        kinds: list[Literal["features", "targets"]] = []
+        if mode in {"features", "both"}:
+            kinds.append("features")
+        if mode in {"targets", "both"}:
+            kinds.append("targets")
+        return kinds
+
+    def _ids_for(self, payload: Literal["features", "targets"]) -> list[str]:
+        cached = self._baseline_cache.get(payload)
+        if cached is not None:
+            return list(cached)
+        ids = self._expected_ids(payload=payload)
+        if self._only is not None:
+            ids = [fid for fid in ids if fid in self._only]
+        if self._exclude is not None:
+            ids = [fid for fid in ids if fid not in self._exclude]
+        self._baseline_cache[payload] = list(ids)
+        return list(ids)

datapipeline/transforms/vector/drop/__init__.py

@@ -0,0 +1,4 @@
+from .horizontal import VectorDropHorizontalTransform
+from .vertical import VectorDropVerticalTransform
+from .orchestrator import VectorDropTransform
+

datapipeline/transforms/vector/drop/horizontal.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Literal
+
+from datapipeline.domain.sample import Sample
+from datapipeline.domain.vector import Vector
+from datapipeline.transforms.vector_utils import is_missing
+
+from ..common import VectorPostprocessBase, select_vector
+
+
+def cell_coverage(value) -> float:
+    """Return coverage for a single feature value.
+
+    Scalars: 1.0 when not missing, 0.0 when missing.
+    Lists: fraction of non-missing elements (0.0 for empty lists).
+    """
+    if isinstance(value, list):
+        if not value:
+            return 0.0
+        total = len(value)
+        ok = sum(1 for item in value if not is_missing(item))
+        return ok / total if total > 0 else 0.0
+    if is_missing(value):
+        return 0.0
+    return 1.0
+
+
+class VectorDropHorizontalTransform(VectorPostprocessBase):
+    """Horizontal (row-wise) drop based on coverage thresholds."""
+
+    def __init__(
+        self,
+        *,
+        threshold: float,
+        payload: Literal["features", "targets", "both"] = "features",
+        only: list[str] | None = None,
+        exclude: list[str] | None = None,
+    ) -> None:
+        if not 0.0 <= threshold <= 1.0:
+            raise ValueError("threshold must be between 0 and 1.")
+        super().__init__(payload=payload, only=only, exclude=exclude)
+        self._threshold = threshold
+
+    def __call__(self, stream: Iterator[Sample]) -> Iterator[Sample]:
+        return self.apply(stream)
+
+    def apply(self, stream: Iterator[Sample]) -> Iterator[Sample]:
+        for sample in stream:
+            total = 0.0
+            count = 0
+            for kind in self._payload_kinds():
+                baseline = self._ids_for(kind)
+                if not baseline:
+                    continue
+                vector = select_vector(sample, kind)
+                if vector is None:
+                    continue
+                total += self._horizontal_coverage(vector, baseline) * len(baseline)
+                count += len(baseline)
+            if count == 0:
+                yield sample
+                continue
+            coverage = total / float(count)
+            if coverage < self._threshold:
+                continue
+            yield sample
+
+    @staticmethod
+    def _horizontal_coverage(vector: Vector, baseline: list[str]) -> float:
+        if not baseline:
+            return 1.0
+        total = 0.0
+        for fid in baseline:
+            value = vector.values.get(fid)
+            total += cell_coverage(value)
+        return total / float(len(baseline))
+