jerry-thomas 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

datapipeline/config/workspace.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+from datapipeline.config.tasks import (
+    VALID_PROGRESS_STYLES,
+    VALID_VISUAL_PROVIDERS,
+)
+from datapipeline.utils.load import load_yaml
+
+
+class SharedDefaults(BaseModel):
+    visuals: Optional[str] = Field(
+        default=None, description="AUTO | TQDM | RICH | OFF"
+    )
+    progress: Optional[str] = Field(
+        default=None, description="AUTO | SPINNER | BARS | OFF"
+    )
+    log_level: Optional[str] = Field(default=None, description="DEFAULT LOG LEVEL")
+
+    @field_validator("visuals", "progress", "log_level", mode="before")
+    @classmethod
+    def _normalize(cls, value: object):
+        if value is None:
+            return None
+        if isinstance(value, str):
+            text = value.strip()
+            return text if text else None
+        return value
+
+    @field_validator("visuals", mode="before")
+    @classmethod
+    def _normalize_visuals(cls, value):
+        if value is None:
+            return None
+        if isinstance(value, bool):
+            return "OFF" if value is False else "AUTO"
+        name = str(value).upper()
+        if name not in VALID_VISUAL_PROVIDERS:
+            raise ValueError(
+                f"visuals must be one of {', '.join(VALID_VISUAL_PROVIDERS)}, got {value!r}"
+            )
+        return name
+
+    @field_validator("progress", mode="before")
+    @classmethod
+    def _normalize_progress(cls, value):
+        if value is None:
+            return None
+        if isinstance(value, bool):
+            return "OFF" if value is False else "AUTO"
+        name = str(value).upper()
+        if name not in VALID_PROGRESS_STYLES:
+            raise ValueError(
+                f"progress must be one of {', '.join(VALID_PROGRESS_STYLES)}, got {value!r}"
+            )
+        return name
+
+
+class ServeDefaults(BaseModel):
+    log_level: Optional[str] = None
+    limit: Optional[int] = None
+    stage: Optional[int] = None
+    throttle_ms: Optional[float] = None
+
+    class OutputDefaults(BaseModel):
+        transport: str
+        format: str
+        payload: str = Field(default="sample")
+        directory: Optional[str] = Field(
+            default=None,
+            description="Base directory for fs outputs (relative paths are resolved from jerry.yaml).",
+        )
+
+    output: Optional[OutputDefaults] = None
+
+
+class BuildDefaults(BaseModel):
+    log_level: Optional[str] = None
+    mode: Optional[str] = None
+
+    @field_validator("mode", mode="before")
+    @classmethod
+    def _normalize_mode(cls, value: object):
+        if value is None:
+            return None
+        if isinstance(value, bool):
+            return "OFF" if value is False else "AUTO"
+        text = str(value).strip()
+        if not text:
+            return None
+        name = text.upper()
+        valid_modes = {"AUTO", "FORCE", "OFF"}
+        if name not in valid_modes:
+            options = ", ".join(sorted(valid_modes))
+            raise ValueError(f"build.mode must be one of {options}, got {value!r}")
+        return name
+
+
+class WorkspaceConfig(BaseModel):
+    plugin_root: Optional[str] = None
+    datasets: dict[str, str] = Field(
+        default_factory=dict,
+        description="Named dataset aliases mapping to project.yaml paths (relative to jerry.yaml).",
+    )
+    default_dataset: Optional[str] = Field(
+        default=None,
+        description="Optional default dataset alias when --dataset/--project are omitted.",
+    )
+    shared: SharedDefaults = Field(default_factory=SharedDefaults)
+    serve: ServeDefaults = Field(default_factory=ServeDefaults)
+    build: BuildDefaults = Field(default_factory=BuildDefaults)
+
+
+@dataclass
+class WorkspaceContext:
+    file_path: Path
+    config: WorkspaceConfig
+
+    @property
+    def root(self) -> Path:
+        return self.file_path.parent
+
+    def resolve_plugin_root(self) -> Optional[Path]:
+        raw = self.config.plugin_root
+        if not raw:
+            return None
+        candidate = Path(raw)
+        return (
+            candidate.resolve()
+            if candidate.is_absolute()
+            else (self.root / candidate).resolve()
+        )
+
+
+def load_workspace_context(start_dir: Optional[Path] = None) -> Optional[WorkspaceContext]:
+    """Search from start_dir upward for jerry.yaml and return parsed config."""
+    directory = (start_dir or Path.cwd()).resolve()
+    for path in [directory, *directory.parents]:
+        candidate = path / "jerry.yaml"
+        if candidate.is_file():
+            data = load_yaml(candidate)
+            if not isinstance(data, dict):
+                raise TypeError("jerry.yaml must define a mapping at the top level")
+            # Allow users to set serve/build/shared to null to fall back to defaults
+            for key in ("shared", "serve", "build"):
+                if key in data and data[key] is None:
+                    data.pop(key)
+            cfg = WorkspaceConfig.model_validate(data)
+            return WorkspaceContext(file_path=candidate, config=cfg)
+    return None

datapipeline/domain/__init__.py

@@ -0,0 +1,12 @@
+from .sample import Sample
+from .vector import Vector
+from .feature import FeatureRecord, FeatureRecordSequence
+from .record import TemporalRecord
+
+__all__ = [
+    "Sample",
+    "Vector",
+    "FeatureRecord",
+    "FeatureRecordSequence",
+    "TemporalRecord",
+]

datapipeline/domain/record.py

@@ -26,3 +26,14 @@ class TemporalRecord(Record):
         data.pop("time", None)
         data.pop("value", None)
         return data
+
+    def __eq__(self, other: object) -> bool:
+        if self is other:
+            return True
+        if not isinstance(other, TemporalRecord):
+            return NotImplemented
+        return (
+            self.time == other.time
+            and self.value == other.value
+            and self._identity_fields() == other._identity_fields()
+        )

datapipeline/domain/sample.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, asdict
+from typing import Any, Iterator, Optional, Literal
+
+from .vector import Vector
+
+PayloadMode = Literal["sample", "vector"]
+
+
+@dataclass
+class Sample:
+    """
+    Represents a single grouped vector sample emitted by the pipeline.
+
+    Attributes:
+        key: Group identifier (tuple when group_by cadence > 1).
+        features: Feature vector payload.
+        targets: Optional target vector when requested.
+    """
+
+    key: Any
+    features: Vector
+    targets: Optional[Vector] = None
+
+    def __iter__(self) -> Iterator[Any]:
+        """Retain tuple-like unpacking compatibility."""
+        yield self.key
+        yield self.features
+
+    def __len__(self) -> int:
+        return 2
+
+    def __getitem__(self, idx: int) -> Any:
+        if idx == 0:
+            return self.key
+        if idx == 1:
+            return self.features
+        raise IndexError(idx)
+
+    def with_targets(self, targets: Optional[Vector]) -> "Sample":
+        return Sample(key=self.key, features=self.features, targets=targets)
+
+    def with_features(self, features: Vector) -> "Sample":
+        return Sample(key=self.key, features=features, targets=self.targets)
+
+    def as_full_payload(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def as_vector_payload(self) -> dict[str, Any]:
+        data: dict[str, Any] = {"features": list(self.features.values.values())}
+        if self.targets is not None:
+            data["targets"] = list(self.targets.values.values())
+        return data

datapipeline/integrations/ml/adapter.py

@@ -8,6 +8,7 @@ from typing import Any, Literal
 
 from datapipeline.config.dataset.dataset import FeatureDatasetConfig
 from datapipeline.config.dataset.loader import load_dataset
+from datapipeline.domain.sample import Sample
 from datapipeline.domain.vector import Vector
 from datapipeline.pipeline.context import PipelineContext
 from datapipeline.pipeline.pipelines import build_vector_pipeline
@@ -69,22 +70,21 @@ class VectorAdapter:
         self,
         *,
         limit: int | None = None,
-        include_targets: bool = False,
     ) -> Iterator[tuple[Sequence[Any], Vector]]:
         features = list(_ensure_features(self.dataset))
-        if include_targets:
-            try:
-                features += list(getattr(self.dataset, "targets", []) or [])
-            except Exception:
-                pass
+        target_cfgs = list(getattr(self.dataset, "targets", []) or [])
         context = PipelineContext(self.runtime)
         vectors = build_vector_pipeline(
-            context, features, self.dataset.group_by, stage=None
+            context,
+            features,
+            self.dataset.group_by,
+            target_configs=target_cfgs,
         )
-        stream = post_process(context, vectors)
+        base_stream = post_process(context, vectors)
+        sample_iter = base_stream
         if limit is not None:
-            stream = islice(stream, limit)
-        return stream
+            sample_iter = islice(sample_iter, limit)
+        return ((sample.key, sample.features) for sample in sample_iter)
 
     def iter_rows(
         self,
@@ -94,24 +94,38 @@ class VectorAdapter:
         group_format: GroupFormat = "mapping",
         group_column: str = "group",
         flatten_sequences: bool = False,
-        include_targets: bool = False,
     ) -> Iterator[dict[str, Any]]:
-        stream = self.stream(limit=limit, include_targets=include_targets)
+        features = list(_ensure_features(self.dataset))
+        target_cfgs = list(getattr(self.dataset, "targets", []) or [])
+        context = PipelineContext(self.runtime)
+        vectors = build_vector_pipeline(
+            context,
+            features,
+            self.dataset.group_by,
+            target_configs=target_cfgs,
+        )
+        base_stream = post_process(context, vectors)
+        if limit is not None:
+            base_stream = islice(base_stream, limit)
         group_by = self.dataset.group_by
 
         def _rows() -> Iterator[dict[str, Any]]:
-            for group_key, vector in stream:
+            for sample in base_stream:
                 row: dict[str, Any] = {}
                 if include_group:
                     row[group_column] = _normalize_group(
-                        group_key, group_by, group_format
+                        sample.key, group_by, group_format
                     )
-                for feature_id, value in vector.values.items():
-                    if flatten_sequences and isinstance(value, list):
-                        for idx, item in enumerate(value):
-                            row[f"{feature_id}[{idx}]"] = item
-                    else:
-                        row[feature_id] = value
+                vectors = [sample.features]
+                if sample.targets:
+                    vectors.append(sample.targets)
+                for vector in vectors:
+                    for feature_id, value in vector.values.items():
+                        if flatten_sequences and isinstance(value, list):
+                            for idx, item in enumerate(value):
+                                row[f"{feature_id}[{idx}]"] = item
+                        else:
+                            row[feature_id] = value
                 yield row
 
         return _rows()

datapipeline/integrations/ml/pandas_support.py

@@ -15,7 +15,6 @@ def dataframe_from_vectors(
     group_format: GroupFormat = "mapping",
     group_column: str = "group",
     flatten_sequences: bool = False,
-    include_targets: bool = False,
     open_stream: Callable[[str], Iterable[Any]] | None = None,
 ):
     """Return a Pandas DataFrame built from project vectors.
@@ -37,7 +36,6 @@ def dataframe_from_vectors(
         group_format=group_format,
         group_column=group_column,
         flatten_sequences=flatten_sequences,
-        include_targets=include_targets,
         open_stream=open_stream,
     )
     return pd.DataFrame(rows)

datapipeline/integrations/ml/rows.py

@@ -13,13 +13,12 @@ def stream_vectors(
     project_yaml: str | Path,
     *,
     limit: int | None = None,
-    include_targets: bool = False,
 ) -> Iterator[tuple[Sequence[Any], Vector]]:
     """Yield ``(group_key, Vector)`` pairs for the configured project."""
 
     adapter = VectorAdapter.from_project(project_yaml)
     try:
-        return adapter.stream(limit=limit, include_targets=include_targets)
+        return adapter.stream(limit=limit)
     except ValueError:
         return iter(())
 
@@ -32,7 +31,6 @@ def iter_vector_rows(
     group_format: GroupFormat = "mapping",
     group_column: str = "group",
     flatten_sequences: bool = False,
-    include_targets: bool = False,
 ) -> Iterator[dict[str, Any]]:
     """Return an iterator of row dictionaries derived from vectors."""
 
@@ -44,7 +42,6 @@ def iter_vector_rows(
             group_format=group_format,
             group_column=group_column,
             flatten_sequences=flatten_sequences,
-            include_targets=include_targets,
         )
     except ValueError:
         return iter(())
@@ -58,7 +55,6 @@ def collect_vector_rows(
     group_format: GroupFormat = "mapping",
     group_column: str = "group",
     flatten_sequences: bool = False,
-    include_targets: bool = False,
     open_stream=None,
 ) -> list[dict[str, Any]]:
     """Materialize :func:`iter_vector_rows` into a list for eager workflows."""
@@ -70,7 +66,6 @@ def collect_vector_rows(
         group_format=group_format,
         group_column=group_column,
         flatten_sequences=flatten_sequences,
-        include_targets=include_targets,
     )
     return list(iterator)
 

datapipeline/integrations/ml/torch_support.py

@@ -35,7 +35,6 @@ def torch_dataset(
     dtype: Any | None = None,
     device: Any | None = None,
     flatten_sequences: bool = False,
-    include_targets: bool = False,
 ):
     """Build a torch.utils.data.Dataset that yields tensors from vectors."""
 
@@ -52,10 +51,9 @@ def torch_dataset(
         limit=limit,
         include_group=False,
         flatten_sequences=flatten_sequences,
-        include_targets=include_targets,
     )
 
-    if include_targets and target_columns is None:
+    if target_columns is None:
         try:
             ds = load_dataset(Path(project_yaml), "vectors")
             target_columns = [cfg.id for cfg in getattr(ds, "targets", []) or []]

datapipeline/io/factory.py

@@ -0,0 +1,112 @@
+from typing import Optional
+
+from datapipeline.io.writers import (
+    JsonLinesFileWriter,
+    JsonLinesStdoutWriter,
+    GzipJsonLinesWriter,
+    CsvFileWriter,
+    PickleFileWriter,
+    LineWriter,
+)
+from datapipeline.io.protocols import Writer
+from datapipeline.io.serializers import (
+    json_line_serializer,
+    print_serializer,
+    csv_row_serializer,
+    pickle_serializer,
+    record_json_line_serializer,
+    record_print_serializer,
+    record_csv_row_serializer,
+    record_pickle_serializer,
+)
+from datapipeline.io.sinks import StdoutTextSink, RichStdoutSink, ReprRichFormatter, JsonRichFormatter, PlainRichFormatter
+from datapipeline.io.output import OutputTarget
+
+
+def stdout_sink_for(format_: str, visuals: Optional[str]) -> StdoutTextSink:
+    """Select an appropriate stdout sink given format and visuals preference.
+
+    Behavior:
+    - visuals == "rich" or "auto" -> attempt Rich formatting; fallback to plain on error.
+    - anything else               -> plain stdout (no Rich formatting).
+    """
+    fmt = (format_ or "print").lower()
+    provider = (visuals or "auto").lower()
+
+    use_rich = provider == "rich" or provider == "auto"
+    if not use_rich:
+        return StdoutTextSink()
+
+    # Prefer Rich when possible; gracefully degrade to plain stdout on any failure.
+    try:
+        if fmt in {"json", "json-lines", "jsonl"}:
+            return RichStdoutSink(JsonRichFormatter())
+        if fmt == "print":
+            return RichStdoutSink(ReprRichFormatter())
+        return RichStdoutSink(PlainRichFormatter())
+    except Exception:
+        return StdoutTextSink()
+
+
+def writer_factory(
+    target: OutputTarget,
+    *,
+    visuals: Optional[str] = None,
+    item_type: str = "sample",
+) -> Writer:
+    transport = target.transport.lower()
+    format_ = target.format.lower()
+    payload = target.payload
+
+    if item_type not in {"sample", "record"}:
+        raise ValueError(f"Unsupported writer item_type '{item_type}'")
+
+    if transport == "stdout":
+        sink = stdout_sink_for(format_, visuals)
+        if format_ in {"json-lines", "json", "jsonl"}:
+            serializer = (
+                record_json_line_serializer()
+                if item_type == "record"
+                else json_line_serializer(payload)
+            )
+            return LineWriter(sink, serializer)
+        if format_ == "print":
+            serializer = (
+                record_print_serializer()
+                if item_type == "record"
+                else print_serializer(payload)
+            )
+            return LineWriter(sink, serializer)
+        raise ValueError(f"Unsupported stdout format '{target.format}'")
+
+    destination = target.destination
+    if destination is None:
+        raise ValueError("fs output requires a destination path")
+    destination.parent.mkdir(parents=True, exist_ok=True)
+
+    suffix = "".join(destination.suffixes).lower()
+    if format_ in {"json-lines", "json", "jsonl"}:
+        serializer = (
+            record_json_line_serializer()
+            if item_type == "record"
+            else json_line_serializer(payload)
+        )
+        if suffix.endswith(".jsonl.gz") or suffix.endswith(".json.gz") or suffix.endswith(".gz"):
+            return GzipJsonLinesWriter(destination, serializer=serializer)
+        return JsonLinesFileWriter(destination, serializer=serializer)
+    if format_ == "csv":
+        serializer = (
+            record_csv_row_serializer()
+            if item_type == "record"
+            else csv_row_serializer(payload)
+        )
+        return CsvFileWriter(destination, serializer=serializer)
+    if format_ == "pickle":
+        serializer = (
+            record_pickle_serializer()
+            if item_type == "record"
+            else pickle_serializer(payload)
+        )
+        return PickleFileWriter(destination, serializer=serializer)
+
+    raise ValueError(f"Unsupported fs format '{target.format}'")

datapipeline/io/output.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from datapipeline.config.tasks import ServeOutputConfig
+from datapipeline.services.runs import RunPaths, start_run_for_directory
+
+
+def _format_suffix(fmt: str) -> str:
+    suffix_map = {
+        "json-lines": ".jsonl",
+        "json": ".json",
+        "csv": ".csv",
+        "pickle": ".pkl",
+    }
+    return suffix_map.get(fmt, ".out")
+
+
+def _default_filename_for_format(fmt: str) -> str:
+    suffix = _format_suffix(fmt)
+    return f"vectors{suffix}"
+
+
+def _sanitize_segment(value: str) -> str:
+    cleaned = "".join(
+        ch if ch.isalnum() or ch in ("_", "-", ".") else "_"
+        for ch in value.strip()
+    )
+    return cleaned or "run"
+
+
+@dataclass(frozen=True)
+class OutputTarget:
+    """Resolved writer target describing how and where to emit records."""
+
+    transport: str  # stdout | fs
+    format: str     # print | json-lines | json | csv | pickle
+    destination: Optional[Path]
+    payload: str = "sample"
+    run: RunPaths | None = None
+
+    def for_feature(self, feature_id: str) -> "OutputTarget":
+        if self.transport != "fs" or self.destination is None:
+            return self
+        safe_feature = "".join(
+            ch if ch.isalnum() or ch in ("_", "-", ".") else "_"
+            for ch in str(feature_id)
+        )
+        dest = self.destination
+        suffix = "".join(dest.suffixes)
+        stem = dest.name[: -len(suffix)] if suffix else dest.name
+        new_name = f"{stem}.{safe_feature}{suffix}"
+        new_path = dest.with_name(new_name)
+        return OutputTarget(
+            transport=self.transport,
+            format=self.format,
+            destination=new_path,
+            payload=self.payload,
+            run=self.run,
+        )
+
+
+class OutputResolutionError(ValueError):
+    """Raised when CLI/config output options cannot be resolved."""
+
+
+def resolve_output_target(
+    *,
+    cli_output: ServeOutputConfig | None,
+    config_output: ServeOutputConfig | None,
+    default: ServeOutputConfig | None = None,
+    base_path: Path | None = None,
+    run_name: str | None = None,
+    payload_override: str | None = None,
+    stage: int | None = None,
+    create_run: bool = False,
+) -> OutputTarget:
+    """
+    Resolve the effective output target using CLI override, run config, or default.
+    """
+
+    base_path = base_path or Path.cwd()
+
+    config = cli_output or config_output or default
+    if config is None:
+        config = ServeOutputConfig(transport="stdout", format="print")
+
+    payload = payload_override or config.payload or "sample"
+
+    if config.transport == "stdout":
+        return OutputTarget(
+            transport="stdout",
+            format=config.format,
+            destination=None,
+            payload=payload,
+            run=None,
+        )
+
+    if config.directory is None:
+        raise OutputResolutionError("fs output requires a directory")
+    directory = (
+        config.directory
+        if config.directory.is_absolute()
+        else (base_path / config.directory).resolve()
+    )
+    if create_run:
+        run_paths, _ = start_run_for_directory(directory, stage=stage)
+        base_dest_dir = run_paths.dataset_dir
+    else:
+        run_paths = None
+        # When not creating a managed run, nest outputs under an optional
+        # run_name subdirectory to keep layouts consistent with tests/CLI.
+        base_dest_dir = directory
+        if run_name:
+            base_dest_dir = base_dest_dir / _sanitize_segment(run_name)
+    suffix = _format_suffix(config.format)
+    filename_stem = config.filename or run_name
+    if filename_stem:
+        filename = f"{filename_stem}{suffix}"
+    else:
+        filename = _default_filename_for_format(config.format)
+    dest_path = (base_dest_dir / filename).resolve()
+
+    return OutputTarget(
+        transport="fs",
+        format=config.format,
+        destination=dest_path,
+        payload=payload,
+        run=run_paths,
+    )

datapipeline/io/protocols.py

@@ -0,0 +1,21 @@
+from typing import Protocol, Optional, runtime_checkable
+from pathlib import Path
+
+
+@runtime_checkable
+class Writer(Protocol):
+    def write(self, rec: dict) -> None: ...
+    def close(self) -> None: ...
+
+
+@runtime_checkable
+class HeaderCapable(Protocol):
+    """Writers that can accept an injected logical 'header record' as the first write."""
+
+    def write_header(self, header: dict) -> None: ...
+
+
+@runtime_checkable
+class HasFilePath(Protocol):
+    @property
+    def file_path(self) -> Optional[Path]: ...