jerry-thomas 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

datapipeline/cli/commands/writers.py

@@ -1,138 +0,0 @@
-from typing import Protocol, Callable, Optional
-from pathlib import Path
-import sys
-import json
-import pickle
-import tempfile
-import os
-import csv
-import gzip
-
-
-class Writer(Protocol):
-    def write(self, rec: dict) -> None: ...
-    def close(self) -> None: ...
-
-
-class TextLineWriter:
-    def __init__(self, formatter: Callable[[dict], str], stream=None):
-        self.formatter = formatter
-        self.stream = stream or sys.stdout
-
-    def write(self, rec: dict) -> None:
-        print(self.formatter(rec), file=self.stream)
-
-    def close(self) -> None:
-        self.stream.flush()
-
-
-def JsonLinesWriter():
-    return TextLineWriter(lambda rec: json.dumps(rec, default=str))
-
-
-def PrintWriter():
-    return TextLineWriter(lambda rec: f"group={rec['key']}: {rec['values']}")
-
-
-class PickleWriter:
-    def __init__(self, destination: Path, protocol: int = pickle.HIGHEST_PROTOCOL):
-        self.dest = destination
-        self.protocol = protocol
-        self.tmp_path: Optional[Path] = None
-        self._fh = None
-        self._pickler = None
-        self._open_tmp()
-
-    def _open_tmp(self):
-        self.dest.parent.mkdir(parents=True, exist_ok=True)
-        tmp = tempfile.NamedTemporaryFile(
-            dir=str(self.dest.parent), delete=False)
-        self.tmp_path = Path(tmp.name)
-        self._fh = tmp
-        self._pickler = pickle.Pickler(self._fh, protocol=self.protocol)
-
-    def write(self, rec: dict) -> None:
-        self._pickler.dump((rec["key"], rec["values"]))
-
-    def close(self) -> None:
-        self._fh.close()
-        os.replace(self.tmp_path, self.dest)
-
-
-class CSVWriter:
-    def __init__(self, destination: Path):
-        self.dest = destination
-        self.tmp_path: Optional[Path] = None
-        self._fh = None
-        self._writer = None
-        self._open_tmp()
-
-    def _open_tmp(self):
-        self.dest.parent.mkdir(parents=True, exist_ok=True)
-        tmp = tempfile.NamedTemporaryFile(
-            dir=str(self.dest.parent), delete=False, mode="w", newline="")
-        self.tmp_path = Path(tmp.name)
-        self._fh = tmp
-        self._writer = csv.writer(self._fh)
-        self._writer.writerow(["key", "values"])  # header
-
-    def _format_field(self, value):
-        if value is None:
-            return ""
-        if isinstance(value, (int, float, bool)):
-            return value
-        if isinstance(value, (bytes, bytearray)):
-            return value.decode("utf-8", errors="replace")
-        if isinstance(value, str):
-            return value
-        return str(value)
-
-    def write(self, rec: dict) -> None:
-        key = rec["key"]
-        values = rec["values"]
-        self._writer.writerow(
-            [self._format_field(key), self._format_field(values)])
-
-    def close(self) -> None:
-        self._fh.close()
-        os.replace(self.tmp_path, self.dest)
-
-
-class GzipJsonLinesWriter:
-    def __init__(self, destination: Path):
-        self.dest = destination
-        self.tmp_path: Optional[Path] = None
-        self._fh = None
-        self._open_tmp()
-
-    def _open_tmp(self):
-        self.dest.parent.mkdir(parents=True, exist_ok=True)
-        # binary write, text wrapper for newline handling
-        tmp = tempfile.NamedTemporaryFile(
-            dir=str(self.dest.parent), delete=False)
-        self.tmp_path = Path(tmp.name)
-        self._fh = gzip.GzipFile(filename="", mode="wb", fileobj=tmp)
-
-    def write(self, rec: dict) -> None:
-        line = json.dumps(rec, default=str).encode("utf-8") + b"\n"
-        self._fh.write(line)
-
-    def close(self) -> None:
-        self._fh.close()
-        os.replace(self.tmp_path, self.dest)
-
-
-def writer_factory(output: Optional[str]) -> Writer:
-    if output and output.lower().endswith(".pt"):
-        return PickleWriter(Path(output))
-    if output and output.lower().endswith(".csv"):
-        return CSVWriter(Path(output))
-    if output and (output.lower().endswith(".jsonl.gz") or output.lower().endswith(".gz")):
-        return GzipJsonLinesWriter(Path(output))
-    mode = (output or "print").lower()
-    if mode == "print":
-        return PrintWriter()
-    if mode == "stream":
-        return JsonLinesWriter()
-    print("Error: unsupported output format. Use 'print', 'stream', '.csv', '.jsonl.gz', or a .pt file path.", file=sys.stderr)
-    raise SystemExit(2)

datapipeline/config/build.py

@@ -1,64 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-
-from pydantic import BaseModel, Field
-
-from datapipeline.services.bootstrap import _load_by_key
-
-
-class PartitionedIdsConfig(BaseModel):
-    """Configuration for writing the expected partitioned-id list."""
-
-    output: str = Field(
-        default="expected.txt",
-        description="Artifact path relative to project.paths.artifacts.",
-    )
-    include_targets: bool = Field(
-        default=False,
-        description="When true, include dataset.targets in the discovery stream.",
-    )
-
-
-class ScalerArtifactConfig(BaseModel):
-    """Configuration for computing standard-scaler statistics."""
-
-    enabled: bool = Field(
-        default=True,
-        description="Disable to skip generating the scaler statistics artifact.",
-    )
-    output: str = Field(
-        default="scaler.pkl",
-        description="Artifact path relative to project.paths.artifacts.",
-    )
-    include_targets: bool = Field(
-        default=False,
-        description="Include dataset.targets when fitting scaler statistics.",
-    )
-    split_label: str = Field(
-        default="train",
-        description="Split label to use when fitting scaler statistics.",
-    )
-
-
-class BuildConfig(BaseModel):
-    """Top-level build configuration describing materialized artifacts."""
-
-    version: int = 1
-    partitioned_ids: PartitionedIdsConfig = Field(
-        default_factory=PartitionedIdsConfig,
-        description="Partitioned-id task settings.",
-    )
-    scaler: ScalerArtifactConfig = Field(
-        default_factory=ScalerArtifactConfig,
-        description="Standard-scaler statistics artifact settings.",
-    )
-
-
-def load_build_config(project_yaml: Path) -> BuildConfig:
-    """Load build.yaml referenced by project.paths.build and validate it."""
-
-    doc = _load_by_key(project_yaml, "build")
-    if not isinstance(doc, dict):
-        raise TypeError("build.yaml must define a mapping at the top level.")
-    return BuildConfig.model_validate(doc)

datapipeline/config/run.py

@@ -1,116 +0,0 @@
-from __future__ import annotations
-
-import logging
-from pathlib import Path
-from typing import List, Sequence, Tuple
-
-from pydantic import BaseModel, Field, field_validator
-
-from datapipeline.config.project import ProjectConfig
-from datapipeline.utils.load import load_yaml
-
-VALID_LOG_LEVELS = ("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG")
-
-
-class RunConfig(BaseModel):
-    """Runtime overrides applied when serving vectors."""
-
-    version: int = Field(default=1)
-    keep: str | None = Field(
-        default=None,
-        description="Active split label to serve. Null disables filtering.",
-        min_length=1,
-    )
-    output: str | None = Field(
-        default=None,
-        description="Default output destination for jerry serve (print|stream|<path>).",
-        min_length=1,
-    )
-    limit: int | None = Field(
-        default=None,
-        description="Default max number of vectors to emit during serve runs.",
-        ge=1,
-    )
-    include_targets: bool = Field(
-        default=False,
-        description="Serve dataset.targets alongside features by default.",
-    )
-    throttle_ms: float | None = Field(
-        default=None,
-        description="Milliseconds to sleep between emitted vectors (throttle).",
-        ge=0.0,
-    )
-    log_level: str | None = Field(
-        default="INFO",
-        description="Default logging level for serve runs (DEBUG, INFO, WARNING, ERROR, CRITICAL). Use null to inherit CLI.",
-    )
-
-    @field_validator("log_level")
-    @classmethod
-    def _validate_log_level(cls, value: str | None) -> str | None:
-        if value is None:
-            return None
-        name = str(value).upper()
-        if name not in VALID_LOG_LEVELS:
-            raise ValueError(
-                f"log_level must be one of {', '.join(VALID_LOG_LEVELS)}, got {value!r}"
-            )
-        return name
-
-
-def _resolve_run_path(project_yaml: Path, run_path: str | Path) -> Path:
-    path = Path(run_path)
-    if not path.is_absolute():
-        path = project_yaml.parent / path
-    return path.resolve()
-
-
-def _list_run_paths(project_yaml: Path) -> Sequence[Path]:
-    project_data = load_yaml(project_yaml)
-    project = ProjectConfig.model_validate(project_data)
-    run_path_ref = getattr(project.paths, "run", None)
-    if not run_path_ref:
-        return []
-    run_path = _resolve_run_path(project_yaml, run_path_ref)
-    if not run_path.exists():
-        raise FileNotFoundError(f"run config not found: {run_path}")
-    if run_path.is_dir():
-        entries = sorted(
-            [
-                p
-                for p in run_path.iterdir()
-                if p.is_file() and p.suffix in {".yaml", ".yml"}
-            ],
-            key=lambda p: p.name,
-        )
-        if not entries:
-            raise FileNotFoundError(f"no run configs found under {run_path}")
-        return entries
-    return [run_path]
-
-
-def _load_run_from_path(path: Path) -> RunConfig:
-    doc = load_yaml(path)
-    if not isinstance(doc, dict):
-        raise TypeError(f"{path} must define a mapping at the top level.")
-    return RunConfig.model_validate(doc)
-
-
-def load_named_run_configs(project_yaml: Path) -> List[Tuple[str, RunConfig]]:
-    """Return (name, config) pairs for every run file (directory-aware)."""
-
-    paths = _list_run_paths(project_yaml)
-    entries: List[Tuple[str, RunConfig]] = []
-    for path in paths:
-        cfg = _load_run_from_path(path)
-        entries.append((path.stem, cfg))
-    return entries
-
-
-def load_run_config(project_yaml: Path) -> RunConfig | None:
-    """Load the first run config referenced by project.paths.run, if configured."""
-
-    paths = _list_run_paths(project_yaml)
-    if not paths:
-        return None
-    return _load_run_from_path(paths[0])

datapipeline/templates/plugin_skeleton/config/contracts/time_hour_sin.synthetic.yaml

@@ -1,24 +0,0 @@
-source_id: time_ticks
-stream_id: time_hour_sin
-
-mapper:
-  entrypoint: encode_time
-  args: { mode: hour_sin }
-
-# partition_by: field you want to partition
-
-record:
-  - filter: { operator: ge, field: time, comparand: "${start_time}" }
-  - filter: { operator: le, field: time, comparand: "${end_time}" }
-#   - floor_time: { resolution: 1h }
-#   - lag: { lag: 1h }
-
-# stream:
-#   - ensure_ticks: { tick: 1h }
-#   - granularity: { mode: last }
-#   - fill: { statistic: median, window: 24, min_samples: 4 }
-
-# debug:
-#   - lint: { mode: warn, tick: 1h }
-
-# sort_batch_size: 100000

datapipeline/templates/plugin_skeleton/config/contracts/time_linear.synthetic.yaml

@@ -1,23 +0,0 @@
-source_id: time_ticks # raw source alias (see config/sources)
-stream_id: time_linear # this stream id used by recipes
-
-mapper: # normalize/reshape DTO -> TemporalRecord if not implemented will give you idenitymapper
-  entrypoint: encode_time
-  args: { mode: linear }
-# partition_by: station_id      # optional: add partition suffixes to feature ids
-
-record: # record-level transforms
-  - filter: { operator: ge, field: time, comparand: "${start_time}" }
-  - filter: { operator: le, field: time, comparand: "${end_time}" }
-#   - floor_time: { resolution: 10m }          # snap timestamps to resolution
-#   - lag: { lag: 10m }                         # shift timestamps backwards
-
-# stream:                       # per-feature stream transforms (input sorted by id,time)
-#   - ensure_ticks: { tick: 10m }               # insert missing ticks (value=None)
-#   - granularity: { mode: first }              # aggregate duplicates within a tick
-#   - fill: { statistic: median, window: 6, min_samples: 1 }  # impute gaps
-
-# debug:                        # optional validation-only transforms
-#   - lint: { mode: warn, tick: 10m }           # flag gaps/duplicates/order issues
-
-# sort_batch_size: 100000       # in-memory chunk size used by internal sorting

datapipeline/templates/plugin_skeleton/config/datasets/default/build.yaml

@@ -1,9 +0,0 @@
-version: 1
-partitioned_ids:
-  output: expected.txt     # relative to project.paths.artifacts
-  include_targets: false   # set true to include dataset.targets
-scaler:
-  enabled: true            # disable to skip scaler statistics
-  output: scaler.pkl       # relative to project.paths.artifacts
-  include_targets: false   # include targets when fitting scaler
-  split_label: train       # label from project.globals.split to fit on

datapipeline/templates/plugin_skeleton/config/datasets/default/dataset.yaml

@@ -1,14 +0,0 @@
-group_by: 1h
-
-features:
-  - id: time
-    record_stream: time_hour_sin
-    # scale: { with_mean: true, with_std: true }
-    # sequence: { size: 6, stride: 1, tick: 10m }
-
-  # - id: second_feature
-  #   record_stream: anotherstream
-# targets:
-#   - id: some_target
-#     record_stream: time_linear
-

datapipeline/templates/plugin_skeleton/config/datasets/default/postprocess.yaml

@@ -1,13 +0,0 @@
-# - drop_missing:
-#     # require these features present OR set min_coverage below
-#     required: [time]
-#     min_coverage: 1.0
-# - fill_constant:
-#     value: 0.0
-# - fill_history:
-#     statistic: median
-#     window: 48
-#     min_samples: 6
-# - fill_horizontal:
-#     statistic: mean
-#     min_samples: 2

datapipeline/templates/plugin_skeleton/config/datasets/default/runs/run_test.yaml

@@ -1,10 +0,0 @@
-version: 1
-# Active split label to serve; must match a label defined in globals.split.
-# Set to null to disable filtering or override per run via CLI.
-keep: test
-# Optional defaults for jerry serve (override via CLI when needed).
-output: print # print | stream | /path/to/test_file.pt
-limit: 100 # max vectors per serve run (null = unlimited)
-include_targets: false # serve dataset.targets alongside features
-throttle_ms: null # milliseconds to sleep between emitted vectors
-log_level: INFO # DEBUG for progress bars, INFO for spinner, WARNING to keep quiet (null to inherit CLI)

datapipeline/templates/plugin_skeleton/config/datasets/default/runs/run_train.yaml

@@ -1,10 +0,0 @@
-version: 1
-# Active split label to serve; must match a label defined in globals.split.
-# Set to null to disable filtering or override per run via CLI.
-keep: train
-# Optional defaults for jerry serve (override via CLI when needed).
-output: print # print | stream | /path/to/train_file.pt
-limit: 100 # max vectors per serve run (null = unlimited)
-include_targets: false # serve dataset.targets alongside features
-throttle_ms: null # milliseconds to sleep between emitted vectors
-log_level: INFO # DEBUG for progress bars, INFO for spinner, WARNING to keep quiet (null to inherit CLI)

datapipeline/templates/plugin_skeleton/config/datasets/default/runs/run_val.yaml

@@ -1,10 +0,0 @@
-version: 1
-# Active split label to serve; must match a label defined in globals.split.
-# Set to null to disable filtering or override per run via CLI.
-keep: val
-# Optional defaults for jerry serve (override via CLI when needed).
-output: print # print | stream | /path/to/val_file.pt
-limit: 100 # max vectors per serve run (null = unlimited)
-include_targets: false # serve dataset.targets alongside features
-throttle_ms: null # milliseconds to sleep between emitted vectors
-log_level: INFO # DEBUG for progress bars, INFO for spinner, WARNING to keep quiet (null to inherit CLI)

datapipeline/templates/plugin_skeleton/config/sources/time_ticks.yaml

@@ -1,11 +0,0 @@
-source_id: time_ticks
-
-parser:
-  entrypoint: "synthetic.time"
-  args: {}
-loader:
-  entrypoint: "synthetic.time"
-  args:
-    start: "${start_time}"
-    end: "${end_time}"
-    frequency: "1h"

datapipeline/transforms/vector.py

@@ -1,210 +0,0 @@
-from collections import deque
-from collections.abc import Iterator
-from statistics import mean, median
-from typing import Any, Literal, Tuple
-
-from datapipeline.domain.vector import Vector
-from datapipeline.transforms.vector_utils import base_id, is_missing, clone
-from datapipeline.pipeline.context import PipelineContext, try_get_current_context
-
-
-class _ContextExpectedMixin:
-    def __init__(self) -> None:
-        self._context: PipelineContext | None = None
-
-    def bind_context(self, context: PipelineContext) -> None:
-        self._context = context
-
-    def _expected_ids(self) -> list[str]:
-        ctx = self._context or try_get_current_context()
-        if not ctx:
-            return []
-        return ctx.load_expected_ids()
-
-
-class VectorDropMissingTransform(_ContextExpectedMixin):
-    """Drop vectors that do not satisfy coverage requirements."""
-
-    def __init__(
-        self,
-        *,
-        required: list[str] | None = None,
-        min_coverage: float = 1.0,
-    ) -> None:
-        super().__init__()
-        if not 0.0 <= min_coverage <= 1.0:
-            raise ValueError("min_coverage must be between 0 and 1")
-        self.required = {str(item) for item in (required or [])}
-        self.min_coverage = min_coverage
-        # Always operate on full (partition) ids
-
-    def __call__(self, stream: Iterator[Tuple[Any, Vector]]) -> Iterator[Tuple[Any, Vector]]:
-        return self.apply(stream)
-
-    def apply(self, stream: Iterator[Tuple[Any, Vector]]) -> Iterator[Tuple[Any, Vector]]:
-        for group_key, vector in stream:
-            present = {fid for fid, value in vector.values.items()
-                       if not is_missing(value)}
-            # Enforce hard requirements first (normalize required keys for fair comparison)
-            if self.required:
-                if not set(self.required).issubset(present):
-                    continue
-
-            # Coverage baseline uses explicit expected if provided; otherwise dynamic set
-            baseline = set(self._expected_ids())
-            if baseline:
-                coverage = len(present & baseline) / len(baseline)
-                if coverage < self.min_coverage:
-                    continue
-            yield group_key, vector
-
-
-class VectorFillConstantTransform(_ContextExpectedMixin):
-    """Fill missing entries with a constant value."""
-
-    def __init__(
-        self,
-        *,
-        value: Any,
-    ) -> None:
-        super().__init__()
-        self.value = value
-
-    def __call__(self, stream: Iterator[Tuple[Any, Vector]]) -> Iterator[Tuple[Any, Vector]]:
-        return self.apply(stream)
-
-    def apply(self, stream: Iterator[Tuple[Any, Vector]]) -> Iterator[Tuple[Any, Vector]]:
-        for group_key, vector in stream:
-            targets = self._expected_ids()
-            if not targets:
-                yield group_key, vector
-                continue
-            data = clone(vector.values)
-            updated = False
-            for feature in targets:
-                if feature not in data or is_missing(data[feature]):
-                    data[feature] = self.value
-                    updated = True
-            if updated:
-                yield group_key, Vector(values=data)
-            else:
-                yield group_key, vector
-
-
-class VectorFillHistoryTransform(_ContextExpectedMixin):
-    """Fill missing entries using running statistics from prior buckets."""
-
-    def __init__(
-        self,
-        *,
-        statistic: Literal["mean", "median"] = "median",
-        window: int | None = None,
-        min_samples: int = 1,
-    ) -> None:
-        super().__init__()
-        if window is not None and window <= 0:
-            raise ValueError("window must be positive when provided")
-        if min_samples <= 0:
-            raise ValueError("min_samples must be positive")
-        self.statistic = statistic
-        self.window = window
-        self.min_samples = min_samples
-        self.history: dict[str, deque[float]] = {}
-
-    def _compute(self, feature_id: str) -> float | None:
-        values = self.history.get(feature_id)
-        if not values or len(values) < self.min_samples:
-            return None
-        if self.statistic == "mean":
-            return float(mean(values))
-        return float(median(values))
-
-    def _push(self, feature_id: str, value: Any) -> None:
-        if is_missing(value):
-            return
-        try:
-            num = float(value)
-        except (TypeError, ValueError):
-            # Ignore non-scalar/non-numeric entries
-            return
-        bucket = self.history.setdefault(
-            str(feature_id), deque(maxlen=self.window))
-        bucket.append(num)
-
-    def __call__(self, stream: Iterator[Tuple[Any, Vector]]) -> Iterator[Tuple[Any, Vector]]:
-        return self.apply(stream)
-
-    def apply(self, stream: Iterator[Tuple[Any, Vector]]) -> Iterator[Tuple[Any, Vector]]:
-        for group_key, vector in stream:
-            targets = self._expected_ids()
-            data = clone(vector.values)
-            updated = False
-            for feature in targets:
-                if feature in data and not is_missing(data[feature]):
-                    continue
-                fill = self._compute(feature)
-                if fill is not None:
-                    data[feature] = fill
-                    updated = True
-            # Push history after possibly filling
-            for fid, value in data.items():
-                self._push(fid, value)
-            if updated:
-                yield group_key, Vector(values=data)
-            else:
-                yield group_key, vector
-
-
-class VectorFillAcrossPartitionsTransform(_ContextExpectedMixin):
-    """Fill missing entries by aggregating sibling partitions at the same timestamp."""
-
-    def __init__(
-        self,
-        *,
-        statistic: Literal["mean", "median"] = "median",
-        min_samples: int = 1,
-    ) -> None:
-        super().__init__()
-        if min_samples <= 0:
-            raise ValueError("min_samples must be positive")
-        self.statistic = statistic
-        self.min_samples = min_samples
-        # Always operate on full (partition) ids
-
-    def __call__(self, stream: Iterator[Tuple[Any, Vector]]) -> Iterator[Tuple[Any, Vector]]:
-        return self.apply(stream)
-
-    def apply(self, stream: Iterator[Tuple[Any, Vector]]) -> Iterator[Tuple[Any, Vector]]:
-        for group_key, vector in stream:
-            targets = self._expected_ids()
-            if not targets:
-                yield group_key, vector
-                continue
-
-            data = clone(vector.values)
-            base_groups: dict[str, list[float]] = {}
-            for fid, value in data.items():
-                if is_missing(value):
-                    continue
-                try:
-                    num = float(value)
-                except (TypeError, ValueError):
-                    continue
-                base_groups.setdefault(base_id(fid), []).append(num)
-
-            updated = False
-            for feature in targets:
-                if feature in data and not is_missing(data[feature]):
-                    continue
-                base = base_id(feature)
-                candidates = base_groups.get(base, [])
-                if len(candidates) < self.min_samples:
-                    continue
-                fill = mean(candidates) if self.statistic == "mean" else median(
-                    candidates)
-                data[feature] = float(fill)
-                updated = True
-            if updated:
-                yield group_key, Vector(values=data)
-            else:
-                yield group_key, vector