jerry-thomas 0.0.2__tar.gz → 0.0.5__tar.gz

{jerry_thomas-0.0.2 → jerry_thomas-0.0.5}/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: jerry-thomas
-Version: 0.0.2
+Version: 0.0.5
 Summary: Jerry-Thomas: a stream-first, plugin-friendly data pipeline (mixology-themed CLI)
-Author: Your Name
+Author: Anders Skott Lind
 License: MIT
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -12,6 +12,7 @@ Requires-Dist: pydantic>=1.8
 Requires-Dist: PyYAML>=5.4
 Requires-Dist: tqdm>=4.0
 Requires-Dist: jinja2>=3.0
+Requires-Dist: setuptools>=70
 Dynamic: license-file
 
 # Jerry Thomas

{jerry_thomas-0.0.2 → jerry_thomas-0.0.5}/pyproject.toml

@@ -4,18 +4,19 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "jerry-thomas"
-version = "0.0.2"
+version = "0.0.5"
 description = "Jerry-Thomas: a stream-first, plugin-friendly data pipeline (mixology-themed CLI)"
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.9"
 license = { text = "MIT" }
-authors = [{ name = "Your Name" }]
+authors = [{ name = "Anders Skott Lind" }]
 dependencies = [
   "numpy>=1.24,<3.0",
   "pydantic>=1.8",
   "PyYAML>=5.4",
   "tqdm>=4.0",
   "jinja2>=3.0",
+  "setuptools>=70",
 ]
 
 [project.scripts]
@@ -67,6 +68,10 @@ greater_than_or_equal    = "datapipeline.filters.filters:ge"
 
 [project.entry-points."datapipeline.transforms"]
 time_lag = "datapipeline.transforms.transforms:time_lag"
+drop_missing = "datapipeline.transforms.transforms:drop_missing_values"
+
+[project.entry-points."datapipeline.transforms.feature"]
+standard_scale = "datapipeline.transforms.transforms:StandardScalerTransform"
 
 [project.entry-points."datapipeline.mappers"]
 "time.synthetic" = "datapipeline.mappers.noop:identity"
@@ -80,5 +85,8 @@ time_window = "datapipeline.transforms.sequence:TimeWindowTransformer"
 "composed.loader" = "datapipeline.sources.factory:build_loader"
 
 [project.entry-points."datapipeline.parsers"]
-"synthetic.time" = "datapipeline.sources.synthetic.time.parser:TimeRowParser" 
+"synthetic.time" = "datapipeline.sources.synthetic.time.parser:TimeRowParser"
 identity = "datapipeline.parsers.identity:IdentityParser"
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]

{jerry_thomas-0.0.2 → jerry_thomas-0.0.5}/src/datapipeline/mappers/synthetic/time.py

@@ -12,8 +12,7 @@ def encode(stream: Iterator[TimeFeatureRecord], mode: str) -> Iterator[TimeFeatu
         elif mode == "weekday_sin":
             val = sin(2 * pi * t.weekday() / 7)
         elif mode == "linear":
-            start = t.replace(hour=0, minute=0, second=0, microsecond=0)
-            val = (t - start).total_seconds()
+            val = t.timestamp()
         else:
             raise ValueError(f"Unsupported encode_time mode: {mode}")
         yield TimeFeatureRecord(time=rec.time, value=val)

jerry_thomas-0.0.5/src/datapipeline/transforms/transforms.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+from dataclasses import is_dataclass, replace
+from datetime import timedelta
+from itertools import groupby
+from math import sqrt
+from numbers import Real
+from typing import Any, Iterator, Mapping, MutableMapping
+
+from datapipeline.domain.feature import FeatureRecord
+from datapipeline.domain.record import Record, TimeFeatureRecord
+from datapipeline.utils.time import parse_timecode
+
+
+def _get_field(record: Any, field: str, default: Any = None) -> Any:
+    """Retrieve attribute *field* from *record* supporting dicts and objects."""
+
+    if isinstance(record, Mapping):
+        return record.get(field, default)
+    return getattr(record, field, default)
+
+
+def _is_missing(value: Any) -> bool:
+    """Return True when *value* should be treated as a missing observation."""
+
+    if value is None:
+        return True
+    if isinstance(value, float):  # covers NaN/inf cases
+        return value != value  # NaN check without importing numpy
+    try:
+        if isinstance(value, Real):
+            return value != value
+    except TypeError:
+        pass
+    return False
+
+
+def _clone_with_value(record: Any, value: float) -> Any:
+    """Return a shallow copy of *record* with its ``value`` field replaced."""
+
+    if isinstance(record, list):
+        raise TypeError(
+            "StandardScalerTransform does not support sequence FeatureRecord payloads."
+        )
+
+    if isinstance(record, Mapping):
+        cloned: MutableMapping[str, Any] = type(record)(record)
+        cloned["value"] = value
+        return cloned
+
+    if hasattr(record, "value"):
+        if is_dataclass(record):
+            return replace(record, value=value)
+        cloned = type(record)(**record.__dict__)
+        cloned.value = value
+        return cloned
+
+    raise TypeError(f"Cannot replace value on record type: {type(record)!r}")
+
+
+def shift_record_time(record: TimeFeatureRecord, lag: timedelta) -> TimeFeatureRecord:
+    record.time = record.time - lag
+    return record
+
+
+def time_lag(stream: Iterator[TimeFeatureRecord], lag: str) -> Iterator[TimeFeatureRecord]:
+    lag_td = parse_timecode(lag)
+    for record in stream:
+        yield shift_record_time(record, lag_td)
+
+
+def drop_missing_values(
+    stream: Iterator[Any],
+    field: str = "value",
+) -> Iterator[Any]:
+    """Filter out records whose *field* contains a missing/null value."""
+
+    for record in stream:
+        value = _get_field(record, field)
+        if _is_missing(value):
+            continue
+        yield record
+
+
+class StandardScalerTransform:
+    """Standardize feature values to zero mean and unit variance per feature id."""
+
+    def __init__(
+        self,
+        *,
+        with_mean: bool = True,
+        with_std: bool = True,
+        epsilon: float = 1e-12,
+        statistics: Mapping[str, Mapping[str, float]] | None = None,
+    ) -> None:
+        self.with_mean = with_mean
+        self.with_std = with_std
+        self.epsilon = epsilon
+        self.statistics = dict(statistics or {})
+        self.stats_: dict[str, dict[str, float]] = {}
+
+    def _resolve_stats(
+        self, feature_id: str, values: list[float]
+    ) -> tuple[float, float]:
+        if feature_id in self.statistics:
+            stats = self.statistics[feature_id]
+            mean = float(stats.get("mean", 0.0))
+            std = float(stats.get("std", 1.0))
+        else:
+            mean = sum(values) / len(values) if self.with_mean else 0.0
+            if self.with_std:
+                variance = sum((v - mean) ** 2 for v in values) / len(values)
+                std = sqrt(variance)
+            else:
+                std = 1.0
+            self.stats_[feature_id] = {
+                "mean": mean if self.with_mean else 0.0,
+                "std": std if self.with_std else 1.0,
+            }
+        if self.with_std:
+            std = max(std, self.epsilon)
+        else:
+            std = 1.0
+        return (mean if self.with_mean else 0.0, std)
+
+    def _extract_value(self, record: Record) -> float:
+        value = _get_field(record, "value")
+        if isinstance(value, Real):
+            return float(value)
+        raise TypeError(f"Record value must be numeric, got {value!r}")
+
+    def apply(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
+        grouped = groupby(stream, key=lambda fr: fr.feature_id)
+        for feature_id, records in grouped:
+            bucket = list(records)
+            if not bucket:
+                continue
+            values = [self._extract_value(fr.record) for fr in bucket]
+            mean, std = self._resolve_stats(feature_id, values)
+            for fr, raw in zip(bucket, values):
+                normalized = raw
+                if self.with_mean:
+                    normalized -= mean
+                if self.with_std:
+                    normalized /= std
+                yield FeatureRecord(
+                    record=_clone_with_value(fr.record, normalized),
+                    feature_id=fr.feature_id,
+                    group_key=fr.group_key,
+                )

{jerry_thomas-0.0.2 → jerry_thomas-0.0.5}/src/jerry_thomas.egg-info/PKG-INFO

@@ -1,8 +1,8 @@
 Metadata-Version: 2.4
 Name: jerry-thomas
-Version: 0.0.2
+Version: 0.0.5
 Summary: Jerry-Thomas: a stream-first, plugin-friendly data pipeline (mixology-themed CLI)
-Author: Your Name
+Author: Anders Skott Lind
 License: MIT
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -12,6 +12,7 @@ Requires-Dist: pydantic>=1.8
 Requires-Dist: PyYAML>=5.4
 Requires-Dist: tqdm>=4.0
 Requires-Dist: jinja2>=3.0
+Requires-Dist: setuptools>=70
 Dynamic: license-file
 
 # Jerry Thomas

{jerry_thomas-0.0.2 → jerry_thomas-0.0.5}/src/jerry_thomas.egg-info/SOURCES.txt

@@ -99,4 +99,5 @@ src/jerry_thomas.egg-info/SOURCES.txt
 src/jerry_thomas.egg-info/dependency_links.txt
 src/jerry_thomas.egg-info/entry_points.txt
 src/jerry_thomas.egg-info/requires.txt
-src/jerry_thomas.egg-info/top_level.txt
+src/jerry_thomas.egg-info/top_level.txt
+tests/test_transforms.py

{jerry_thomas-0.0.2 → jerry_thomas-0.0.5}/src/jerry_thomas.egg-info/entry_points.txt

@@ -34,7 +34,11 @@ identity = datapipeline.parsers.identity:IdentityParser
 synthetic.time = datapipeline.sources.synthetic.time.parser:TimeRowParser
 
 [datapipeline.transforms]
+drop_missing = datapipeline.transforms.transforms:drop_missing_values
 time_lag = datapipeline.transforms.transforms:time_lag
 
+[datapipeline.transforms.feature]
+standard_scale = datapipeline.transforms.transforms:StandardScalerTransform
+
 [datapipeline.transforms.sequence]
 time_window = datapipeline.transforms.sequence:TimeWindowTransformer

{jerry_thomas-0.0.2 → jerry_thomas-0.0.5}/src/jerry_thomas.egg-info/requires.txt

@@ -3,3 +3,4 @@ pydantic>=1.8
 PyYAML>=5.4
 tqdm>=4.0
 jinja2>=3.0
+setuptools>=70

jerry_thomas-0.0.5/tests/test_transforms.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from math import isclose
+
+from datapipeline.domain.feature import FeatureRecord
+from datapipeline.domain.record import TimeFeatureRecord
+from datapipeline.transforms.transforms import (
+    StandardScalerTransform,
+    drop_missing_values,
+)
+
+
+def _make_time_record(value: float, hour: int) -> TimeFeatureRecord:
+    return TimeFeatureRecord(
+        time=datetime(2024, 1, 1, hour=hour, tzinfo=timezone.utc),
+        value=value,
+    )
+
+
+def _make_feature_record(value: float, hour: int, feature_id: str) -> FeatureRecord:
+    return FeatureRecord(
+        record=_make_time_record(value, hour),
+        feature_id=feature_id,
+        group_key=(hour,),
+    )
+
+
+def test_drop_missing_values_filters_none_and_nan():
+    stream = iter(
+        [
+            _make_time_record(1.0, 1),
+            _make_time_record(float("nan"), 2),
+            _make_time_record(3.0, 3),
+            _make_time_record(0.0, 4),
+        ]
+    )
+
+    cleaned = list(drop_missing_values(stream))
+
+    assert [rec.value for rec in cleaned] == [1.0, 3.0, 0.0]
+
+
+def test_standard_scaler_normalizes_feature_stream():
+    stream = iter(
+        [
+            _make_feature_record(1.0, 0, "radiation"),
+            _make_feature_record(2.0, 1, "radiation"),
+            _make_feature_record(3.0, 2, "radiation"),
+        ]
+    )
+    scaler = StandardScalerTransform()
+
+    transformed = list(scaler.apply(stream))
+
+    values = [fr.record.value for fr in transformed]
+    expected = [-1.22474487, 0.0, 1.22474487]
+    for observed, target in zip(values, expected):
+        assert isclose(observed, target, rel_tol=1e-6)
+    assert isclose(scaler.stats_["radiation"]["mean"], 2.0, rel_tol=1e-6)
+
+
+def test_standard_scaler_uses_provided_statistics():
+    stream = iter(
+        [
+            _make_feature_record(10.0, 0, "temperature"),
+            _make_feature_record(11.0, 1, "temperature"),
+        ]
+    )
+    scaler = StandardScalerTransform(
+        statistics={"temperature": {"mean": 5.0, "std": 5.0}}
+    )
+
+    transformed = list(scaler.apply(stream))
+
+    assert [fr.record.value for fr in transformed] == [1.0, 1.2]

jerry_thomas-0.0.2/src/datapipeline/transforms/transforms.py

@@ -1,15 +0,0 @@
-from datapipeline.domain.record import TimeFeatureRecord
-from datetime import timedelta
-from typing import Iterator
-from datapipeline.utils.time import parse_timecode
-
-
-def shift_record_time(record: TimeFeatureRecord, lag: timedelta) -> TimeFeatureRecord:
-    record.time = record.time - lag
-    return record
-
-
-def time_lag(stream: Iterator[TimeFeatureRecord], lag: str) -> Iterator[TimeFeatureRecord]:
-    lag_td = parse_timecode(lag)
-    for record in stream:
-        yield shift_record_time(record, lag_td)