jerry-thomas 1.0.3__py3-none-any.whl → 2.0.1__py3-none-any.whl

datapipeline/templates/plugin_skeleton/reference/reference/tasks/schema.reference.yaml

@@ -0,0 +1,10 @@
+# Schema task reference (all options).
+# This file is documentation only; uncomment the keys you want to use.
+#
+# version: 1                # optional
+# kind: schema
+# name: schema              # optional (defaults to filename stem)
+# enabled: true             # optional
+#
+# output: schema.json        # optional; relative to project.paths.artifacts
+# cadence_strategy: max      # optional; currently only "max"

datapipeline/templates/plugin_skeleton/reference/reference/tasks/serve.reference.yaml

@@ -0,0 +1,28 @@
+# Serve task reference (all options).
+# This file is documentation only; uncomment the keys you want to use.
+#
+# version: 1                  # optional
+# kind: serve
+# name: train                 # optional (defaults to filename stem)
+# enabled: true               # optional
+#
+# keep: train                  # optional; split label from globals.split (null disables filtering)
+#
+# output:                     # optional; omit to use CLI defaults
+#   transport: stdout          # stdout | fs
+#   format: json-lines         # stdout: print | json-lines | json
+#   payload: sample            # sample | vector
+#   # fs transport only:
+#   # transport: fs
+#   # format: csv        # csv | json | json-lines | pickle
+#   # payload: vector
+#   # directory: artifacts/serve
+#   # filename: vectors.train  # no extension, no path separators
+#
+# limit: 100                  # optional; null = unlimited
+# stage: 8                    # optional; 0-8; null lets CLI decide
+# throttle_ms: 0              # optional; milliseconds; null disables
+#
+# log_level: INFO             # optional; CRITICAL | ERROR | WARNING | INFO | DEBUG
+# visuals: AUTO               # optional; AUTO | TQDM | RICH | OFF (false -> OFF)
+# progress: AUTO              # optional; AUTO | SPINNER | BARS | OFF

datapipeline/templates/plugin_skeleton/src/{{PACKAGE_NAME}}/domains/__init__.py

@@ -0,0 +1,2 @@
+"""Domain record models live under this package."""
+

datapipeline/templates/plugin_skeleton/src/{{PACKAGE_NAME}}/mappers/__init__.py

@@ -0,0 +1 @@
+"""Stream mappers (DTO -> domain records)."""

datapipeline/templates/plugin_skeleton/your-dataset/dataset.yaml

@@ -1,18 +1,19 @@
+# See ../reference/reference/dataset.yaml for full options.
+
 group_by: ${group_by}
 
 features:
-  - id: time_linear
-    record_stream: time.ticks.linear
-    scale: true # optionally add with_mean/with_std overrides
-    # Sliding window over the regularized stream; cadence is enforced in the contract.
+  - id: first_feature
+    record_stream: your.stream.one
+    field: some_field
+    scale: true
     sequence: { size: 6, stride: 1 }
 
-  - id: time_hour_sin
-    record_stream: time.ticks.hour_sin
+  - id: second_feature
+    record_stream: your.stream.two
+    field: some_field
 
-  # - id: third_feature
-  #   record_stream: anotherstream
 # targets:
-#   - id: some_target
-#     record_stream: time.ticks.linear
-
+#   - id: target_feature
+#     record_stream: your.target.stream
+#     field: some_field

datapipeline/templates/plugin_skeleton/your-dataset/postprocess.yaml

@@ -1,16 +1,15 @@
-#### example combination of postprocessing steps ######
-#### making sure data is complete after these combinations ######
-- drop: # example of dropping sparse partitions/vertical-axis for targets
+# See ../reference/reference/postprocess.yaml for full options.
+- drop:
     axis: vertical
     payload: targets
     threshold: 0.9
 
-- drop: # example of dropping sparse partitions for features
+- drop:
     axis: vertical
     payload: features
     threshold: 0.9
 
-- drop: # dropping vectors/horizontal-axis that has features which none
+- drop:
     axis: horizontal
     payload: features
     threshold: 1
@@ -19,11 +18,3 @@
     axis: horizontal
     payload: targets
     threshold: 1
-######
-# - fill:
-#     statistic: median
-#     window: 48
-#     min_samples: 6
-# - replace:
-#     payload: targets
-#     value: 0.0

datapipeline/templates/plugin_skeleton/your-dataset/project.yaml

@@ -1,3 +1,4 @@
+# See ../reference/reference/project.yaml for full options.
 version: 1
 name: <your-dataset>
 paths:
@@ -7,16 +8,13 @@ paths:
   postprocess: postprocess.yaml
   artifacts: ../artifacts/${project_name}/v${version}
   tasks: ./tasks
-globals: # Globals to use in your .yaml files via ${var_name}.
-  # Primary dataset cadence; referenced from dataset.yaml (group_by)
-  # and contracts via ${group_by}.
-  group_by: <your-bucket-cadence>
-  start_time: null #2021-01-01T00:00:00Z
-  end_time: null #2021-01-02T00:00:00Z
-  # Configure deterministic dataset split here (applied at serve time, after postprocess).
-  # Adjust `ratios` as needed; the active split is selected via serve tasks or CLI.
+globals:
+  # TODO: Set your grouping cadence (must match ^\d+(m|min|h|d)$).
+  group_by: 1h
+  start_time: null
+  end_time: null
   split:
-    mode: hash # hash | time (time uses boundaries/labels)
-    key: group # group | feature:<id> (entity-stable split)
-    seed: 42 # deterministic hash seed
+    mode: hash
+    key: group
+    seed: 42
     ratios: { train: 0.8, val: 0.1, test: 0.1 }

datapipeline/templates/plugin_skeleton/your-dataset/tasks/metadata.yaml

@@ -1,3 +1,2 @@
+# See ../../reference/reference/tasks/metadata.reference.yaml for full options.
 kind: metadata
-# window_mode: intersection # union|intersection|strict|relaxed (default: intersection)
-

datapipeline/templates/plugin_skeleton/your-dataset/tasks/scaler.yaml

@@ -1,9 +1,3 @@
+# See ../../reference/reference/tasks/scaler.reference.yaml for full options.
 kind: scaler
-
-# Output path is relative to project.paths.artifacts; defaults to "scaler.pkl".
-# output: scaler.pkl
-
-# Split label to use when fitting scaler statistics.
-# Must match a label from globals.split.ratios.
 split_label: train
-

datapipeline/templates/plugin_skeleton/your-dataset/tasks/schema.yaml

@@ -1,2 +1,2 @@
+# See ../../reference/reference/tasks/schema.reference.yaml for full options.
 kind: schema
-

datapipeline/templates/plugin_skeleton/your-dataset/tasks/serve.test.yaml

@@ -1,4 +1,4 @@
+# See ../../reference/reference/tasks/serve.reference.yaml for full options.
 kind: serve
 name: test
 keep: test
-

datapipeline/templates/plugin_skeleton/your-dataset/tasks/serve.train.yaml

@@ -1,28 +1,4 @@
+# See ../../reference/reference/tasks/serve.reference.yaml for full options.
 kind: serve
-
-# Optional identifier for this serve task; defaults to filename stem.
 name: train
-
-# Active split label to serve; must match a label from globals.split.ratios.
-# Set to null to disable split filtering.
 keep: train
-#output:
-# transport: stdout | fs
-# format: print | json-lines | json | csv | pickle
-# When using fs transport, set a directory (and optionally filename) for outputs:
-# directory: artifacts/serve
-# filename: vectors.train
-
-# Default max number of vectors to emit (null = unlimited).
-# limit: 5
-# Optional pipeline stage preview (0-7); null lets the CLI decide.
-# stage: 7
-
-# Optional pacing between emitted vectors (milliseconds).
-# throttle_ms: null
-
-# Visuals/logging knobs (inherit CLI or jerry.yaml defaults when omitted):
-# visuals: AUTO   # AUTO | TQDM | RICH | OFF
-# progress: AUTO  # AUTO | SPINNER | BARS | OFF
-# log_level: INFO # CRITICAL | ERROR | WARNING | INFO | DEBUG
-

datapipeline/templates/plugin_skeleton/your-dataset/tasks/serve.val.yaml

@@ -1,4 +1,4 @@
+# See ../../reference/reference/tasks/serve.reference.yaml for full options.
 kind: serve
 name: val
 keep: val
-

datapipeline/templates/plugin_skeleton/your-interim-data-builder/dataset.yaml

@@ -0,0 +1,9 @@
+# Early-stage dataset definition (record/feature prep only).
+# Populate record_stream entries with your canonical stream ids.
+
+group_by: ${group_by}
+
+features:
+  - id: first_feature
+    record_stream: your.stream.one
+    field: some_field

datapipeline/templates/plugin_skeleton/your-interim-data-builder/postprocess.yaml

@@ -0,0 +1 @@
+[]

datapipeline/templates/plugin_skeleton/your-interim-data-builder/project.yaml

@@ -0,0 +1,15 @@
+# See ../reference/reference/project.yaml for full options.
+version: 1
+name: <your-interim-data-builder>
+paths:
+  streams: ./contracts
+  sources: ./sources
+  dataset: dataset.yaml
+  postprocess: postprocess.yaml
+  artifacts: ../artifacts/${project_name}/v${version}
+  tasks: ./tasks
+globals:
+  # TODO: Set your grouping cadence (must match ^\d+(m|min|h|d)$).
+  group_by: 1h
+  start_time: null
+  end_time: null

datapipeline/templates/plugin_skeleton/your-interim-data-builder/tasks/serve.all.yaml

@@ -0,0 +1,8 @@
+# See ../../reference/reference/tasks/serve.reference.yaml for full options.
+kind: serve
+name: all
+keep: null
+output:
+  transport: fs
+  format: json-lines
+  directory: data/interim/jerry

datapipeline/templates/stubs/contracts/composed.yaml.j2

@@ -0,0 +1,10 @@
+kind: composed
+id: {{ stream_id }}  # format: domain.dataset.(variant)
+# cadence: ${group_by}          # optional per-contract cadence
+# partition_by: <field or [fields]>
+inputs:
+  - {{ inputs_list }}
+
+mapper:
+  entrypoint: {{ mapper_entrypoint }}
+  args: { driver: {{ driver_key }} }

datapipeline/templates/stubs/contracts/ingest.yaml.j2

@@ -0,0 +1,25 @@
+kind: ingest
+source: {{ source }}
+id: {{ stream_id }}  # format: domain.dataset.(variant)
+
+mapper:
+  entrypoint: {{ mapper_entrypoint }}
+  args: {}
+
+cadence: ${group_by}                # optional per-contract cadence
+# partition_by: <field or [fields]>
+# sort_batch_size: 100000              # in-memory sort chunk size
+
+record:                              # record-level transforms
+  - filter: { field: time, operator: ge, comparand: "${start_time}" }
+  - filter: { field: time, operator: le, comparand: "${end_time}" }
+  - floor_time: { cadence: "${cadence}" }
+#   - lag: { lag: 10m }
+
+stream:                              # per-stream transforms (input sorted by partition,time)
+  - ensure_cadence: { field: some_field, to: some_field, cadence: "${cadence}" }
+  - granularity: { field: some_field, to: some_field, mode: first }
+#   - fill: { field: some_field, to: some_field, statistic: median, window: 6, min_samples: 1 }
+
+debug:                               # optional validation-only checks
+  - lint: { mode: warn, tick: "${cadence}" }

datapipeline/templates/stubs/dto.py.j2

@@ -5,7 +5,7 @@ from datetime import datetime
 @dataclass
 class {{CLASS_NAME}}:
     """
-    Data Transfer Object (DTO) for the '{{DOMAIN}}' source.
+    Data Transfer Object (DTO) for the '{{DOMAIN}}' records.
 
     Purpose
     - Represents the raw, source-shaped data emitted by the loader + parser.
@@ -24,4 +24,4 @@ class {{CLASS_NAME}}:
     # currency: str
     """
     # TODO: define fields matching the '{{DOMAIN}}' source schema
-    raise NotImplementedError(f"Define fields for the DTO '{{CLASS_NAME}}'")
+    pass

datapipeline/templates/stubs/filter.py.j2

@@ -11,6 +11,6 @@ def {{ FUNCTION_NAME }}(stream: Iterator[Any], field: str, target: Any) -> Itera
     Replace the condition below with your own logic.
     """
     for record in stream:
+        # TODO: implement filter logic
         # Example: pass through everything
         yield record
-

datapipeline/templates/stubs/loaders/basic.py.j2

@@ -0,0 +1,11 @@
+from typing import Iterator, Any
+
+from datapipeline.sources.models.loader import DataLoader
+
+
+class {{CLASS_NAME}}(DataLoader):
+    """Custom loader stub. Yield raw items to be parsed by a parser."""
+
+    def __iter__(self) -> Iterator[Any]:
+        # TODO: implement data loading
+        yield from ()

datapipeline/templates/stubs/mappers/composed.py.j2

@@ -0,0 +1,13 @@
+from typing import Iterator, Mapping
+
+from datapipeline.domain.record import TemporalRecord
+
+
+def mapper(
+    inputs: Mapping[str, Iterator[TemporalRecord]],
+    *, driver: str | None = None, aux: Mapping[str, Iterator[TemporalRecord]] | None = None, context=None, **params
+) -> Iterator[TemporalRecord]:
+    # TODO: implement domain math; inputs are ordered/regularized; aux is raw
+    key = driver or next(iter(inputs.keys()))
+    for rec in inputs[key]:
+        yield rec  # replace with your dataclass and computation

datapipeline/templates/stubs/mappers/ingest.py.j2

@@ -0,0 +1,20 @@
+from typing import Any, Iterator
+
+from {{DOMAIN_MODULE}} import {{DOMAIN_RECORD}}
+{% if INPUT_IMPORT != "typing" %}from {{INPUT_IMPORT}} import {{INPUT_CLASS}}
+{% endif %}
+
+
+def {{FUNCTION_NAME}}(
+    stream: Iterator[{{INPUT_CLASS}}],
+    **params: Any,
+) -> Iterator[{{DOMAIN_RECORD}}]:
+    """Map {{INPUT_CLASS}} records to domain-level {{DOMAIN_RECORD}} records."""
+    raise NotImplementedError(
+        "Implement mapper logic for {{INPUT_CLASS}} -> {{DOMAIN_RECORD}}"
+    )
+    for record in stream:
+        # TODO: construct {{DOMAIN_RECORD}} from record fields
+        yield {{DOMAIN_RECORD}}(
+            time=record.time,  # required
+        )

datapipeline/templates/stubs/parser.py.j2

@@ -2,7 +2,11 @@ from typing import Any
 
 from datapipeline.sources.models.parser import DataParser
 
+{% if DTO_IMPORT -%}
+from {{DTO_IMPORT}} import {{DTO_CLASS}}
+{% else -%}
 from .dto import {{DTO_CLASS}}
+{% endif %}
 
 
 class {{CLASS_NAME}}(DataParser[{{DTO_CLASS}}]):
@@ -18,4 +22,4 @@ class {{CLASS_NAME}}(DataParser[{{DTO_CLASS}}]):
         # return {{DTO_CLASS}}(
         #     ... map fields from `raw` ...
         # )
-        raise NotImplementedError
+        raise NotImplementedError("Implement parser logic for {{DTO_CLASS}}")

datapipeline/templates/stubs/record.py.j2

@@ -9,11 +9,11 @@ class {{CLASS_NAME}}({{PARENT_CLASS}}):
     Domain record for '{{DOMAIN}}'.
 
     Required fields inherited from the base:
-    - value: main payload used to model records value by pipelines (numeric or categorical)
     - time: tz-aware datetime (always required)
 
     Add any additional fields you need for filtering/partitioning/grouping.
     """
+    # TODO: Add domain fields for filtering/partitioning/grouping.
     # Example extra fields (uncomment and adapt):
     # region: str # e.g. 'us-west', 'eu-central', etc.
     # exchange: str #   e.g. 'NASDAQ', 'NYSE', etc.

datapipeline/templates/stubs/source.yaml.j2

@@ -1,5 +1,5 @@
 # Required identifier for this raw source. Contracts reference it under `source:`.
-id: "{{ id }}"  # format: provider.dataset
+id: "{{ id }}"  # suggested format: provider.dataset
 
 # parser.entrypoint: registered parser name (not a file path)
 parser:

datapipeline/transforms/debug/identity.py

@@ -1,10 +1,9 @@
-from __future__ import annotations
-
 import logging
 from dataclasses import asdict, is_dataclass
 from typing import Iterator, Any
 
-from datapipeline.domain.feature import FeatureRecord
+from datapipeline.domain.record import TemporalRecord
+from datapipeline.transforms.utils import partition_key
 
 logger = logging.getLogger(__name__)
 
@@ -16,14 +15,21 @@ class IdentityGuardTransform:
     - mode: 'warn' (default) logs warnings; 'error' raises on first violation
     - fields: optional explicit list of attribute names to compare. When omitted,
       the transform attempts to derive identity from dataclass fields on the
-      underlying record, excluding 'time' and 'value'.
+      underlying record, excluding 'time'.
     """
 
-    def __init__(self, *, mode: str = "warn", fields: list[str] | None = None) -> None:
+    def __init__(
+        self,
+        *,
+        mode: str = "warn",
+        fields: list[str] | None = None,
+        partition_by: str | list[str] | None = None,
+    ) -> None:
         self.mode = mode
         self.fields = fields
+        self.partition_by = partition_by
 
-    def __call__(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
+    def __call__(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
         return self.apply(stream)
 
     def _violation(self, msg: str) -> None:
@@ -41,26 +47,38 @@ class IdentityGuardTransform:
                 except Exception:
                     out[f] = None
             return out
+        # Fall back to partition_by when available
+        if self.partition_by:
+            fields = (
+                [self.partition_by]
+                if isinstance(self.partition_by, str)
+                else list(self.partition_by)
+            )
+            out = {}
+            for f in fields:
+                try:
+                    out[f] = getattr(rec, f)
+                except Exception:
+                    out[f] = None
+            return out
         # Try domain-provided hook first
         if hasattr(rec, "identity_fields") and callable(getattr(rec, "identity_fields")):
             try:
                 return rec.identity_fields()  # type: ignore[attr-defined]
             except Exception:
                 pass
-        # Fallback: dataclass fields minus time/value
+        # Fallback: dataclass fields minus time
         if is_dataclass(rec):
             data = asdict(rec)
             data.pop("time", None)
-            data.pop("value", None)
             return data
         return {}
 
-    def apply(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
-        current_key = None
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        current_key: tuple | None = None
         baseline: dict | None = None
-        for fr in stream:
-            key = fr.id
-            rec = fr.record
+        for rec in stream:
+            key = partition_key(rec, self.partition_by)
             ident = self._identity_map(rec)
             if key != current_key:
                 current_key = key
@@ -68,7 +86,7 @@ class IdentityGuardTransform:
             else:
                 if ident != baseline:
                     self._violation(
-                        "identity drift in feature stream id=%s: expected=%s observed=%s"
-                        % (fr.id, baseline, ident)
+                        "identity drift in record stream key=%s: expected=%s observed=%s"
+                        % (key, baseline, ident)
                     )
-            yield fr
+            yield rec

datapipeline/transforms/debug/lint.py

@@ -3,7 +3,8 @@ from datetime import timedelta
 from itertools import groupby
 from typing import Iterator
 
-from datapipeline.domain.feature import FeatureRecord
+from datapipeline.domain.record import TemporalRecord
+from datapipeline.transforms.utils import partition_key
 from datapipeline.utils.time import parse_timecode
 
 
@@ -23,9 +24,11 @@ class StreamLint:
         *,
         mode: str = "warn",
         tick: str | None = None,
+        partition_by: str | list[str] | None = None,
     ) -> None:
         self.mode = mode
         self.tick = tick
+        self.partition_by = partition_by
 
         # Pre-compute tick step in seconds when provided to avoid repeated parsing.
         self._tick_seconds: int | None = None
@@ -38,7 +41,7 @@ class StreamLint:
                 )
                 self._tick_seconds = None
 
-    def __call__(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
+    def __call__(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
         return self.apply(stream)
 
     def _violation(self, msg: str) -> None:
@@ -46,25 +49,25 @@ class StreamLint:
             raise ValueError(msg)
         logger.warning(msg)
 
-    def apply(self, stream: Iterator[FeatureRecord]) -> Iterator[FeatureRecord]:
-        # Group by base feature id to keep state local
-        for fid, records in groupby(stream, key=lambda fr: fr.id):
+    def apply(self, stream: Iterator[TemporalRecord]) -> Iterator[TemporalRecord]:
+        # Group by partition key to keep state local
+        for key, records in groupby(stream, key=lambda rec: partition_key(rec, self.partition_by)):
             last_time = None
             seen_times: set = set()
-            for fr in records:
-                t = getattr(fr.record, "time", None)
+            for record in records:
+                t = getattr(record, "time", None)
 
                 # Check ordering
                 if last_time is not None and t is not None and t < last_time:
                     self._violation(
-                        f"out-of-order timestamp for feature '{fid}': {t} < {last_time}. "
+                        f"out-of-order timestamp for partition '{key}': {t} < {last_time}. "
                         f"Consider sorting upstream or fixing loader."
                     )
 
                 # Check duplicates
                 if t in seen_times:
                     self._violation(
-                        f"duplicate timestamp for feature '{fid}' at {t}. "
+                        f"duplicate timestamp for partition '{key}' at {t}. "
                         f"Consider a granularity transform (first/last/mean/median)."
                     )
                 seen_times.add(t)
@@ -78,9 +81,9 @@ class StreamLint:
                     expect = last_time + timedelta(seconds=self._tick_seconds)
                     if t != expect and t > expect:
                         self._violation(
-                            f"skipped tick(s) for feature '{fid}': expected {expect}, got {t}. "
+                            f"skipped tick(s) for partition '{key}': expected {expect}, got {t}. "
                             f"Consider using ensure_cadence."
                         )
 
                 last_time = t
-                yield fr
+                yield record

datapipeline/transforms/feature/scaler.py

@@ -3,12 +3,11 @@ from collections import defaultdict
 from itertools import groupby
 from numbers import Real
 from pathlib import Path
-from typing import Any, Callable, Iterator, Literal, Mapping
+from typing import Any, Callable, Iterator, Literal
 
 from datapipeline.domain.feature import FeatureRecord
 from datapipeline.domain.sample import Sample
 from datapipeline.transforms.feature.model import FeatureTransform
-from datapipeline.transforms.utils import clone_record_with_value
 from datapipeline.utils.pickle_model import PicklePersistanceMixin
 from datapipeline.pipeline.observability import TransformEvent
 
@@ -86,7 +85,7 @@ class StandardScaler(PicklePersistanceMixin):
             mean = float(stats.get("mean", 0.0))
             std = float(stats.get("std", 1.0))
             for fr in records:
-                value = fr.record.value
+                value = fr.value
                 if not isinstance(value, Real):
                     if value is None and on_none == "skip":
                         self.missing_counts[feature_id] = (
@@ -114,10 +113,7 @@ class StandardScaler(PicklePersistanceMixin):
                     normalized -= mean
                 if self.with_std:
                     normalized /= std
-                yield FeatureRecord(
-                    record=clone_record_with_value(fr.record, normalized),
-                    id=fr.id,
-                )
+                yield FeatureRecord(record=fr.record, id=fr.id, value=normalized)
 
     def inverse_transform(
         self,
@@ -136,7 +132,7 @@ class StandardScaler(PicklePersistanceMixin):
             mean = float(stats.get("mean", 0.0))
             std = float(stats.get("std", 1.0))
             for fr in records:
-                value = fr.record.value
+                value = fr.value
                 if not isinstance(value, Real):
                     raise TypeError(
                         f"Record value must be numeric, got {value!r}")
@@ -145,10 +141,7 @@ class StandardScaler(PicklePersistanceMixin):
                     restored *= std
                 if self.with_mean:
                     restored += mean
-                yield FeatureRecord(
-                    record=clone_record_with_value(fr.record, restored),
-                    id=fr.id,
-                )
+                yield FeatureRecord(record=fr.record, id=fr.id, value=restored)
 
     class _RunningStats:
         __slots__ = ("count", "mean", "m2")