jerry-thomas 1.0.3__py3-none-any.whl → 2.0.0__py3-none-any.whl

datapipeline/cli/commands/stream.py

@@ -0,0 +1,286 @@
+from pathlib import Path
+
+from datapipeline.config.workspace import WorkspaceContext
+from datapipeline.cli.workspace_utils import resolve_default_project_yaml
+from datapipeline.services.paths import pkg_root
+from datapipeline.services.project_paths import resolve_project_yaml_path
+from datapipeline.services.scaffold.discovery import (
+    list_domains,
+    list_mappers,
+    list_parsers,
+    list_sources,
+    list_dtos,
+)
+from datapipeline.services.scaffold.source_yaml import default_loader_config
+from datapipeline.services.scaffold.layout import (
+    default_stream_id,
+    dto_class_name,
+    default_parser_name,
+    default_mapper_name,
+    dto_module_path,
+    LABEL_DTO_FOR_PARSER,
+    LABEL_DTO_FOR_MAPPER,
+    LABEL_DOMAIN_TO_MAP,
+    LABEL_MAPPER_INPUT,
+    default_mapper_name_for_identity,
+)
+from datapipeline.services.scaffold.stream_plan import StreamPlan, ParserPlan, MapperPlan, execute_stream_plan
+from datapipeline.services.scaffold.utils import (
+    choose_existing_or_create,
+    choose_name,
+    choose_existing_or_create_name,
+    error_exit,
+    info,
+    pick_from_list,
+    pick_from_menu,
+    prompt_required,
+)
+
+
+def handle(*, plugin_root: Path | None = None, workspace: WorkspaceContext | None = None) -> None:
+    root_dir, pkg_name, _ = pkg_root(plugin_root)
+    project_yaml = resolve_default_project_yaml(workspace) or resolve_project_yaml_path(root_dir)
+
+    # Shared context
+    provider = prompt_required("Provider")
+    dataset = prompt_required("Dataset")
+    source_id = f"{provider}.{dataset}"
+
+    # Collected actions (execute at end)
+    create_source = False
+    create_domain_flag = False
+    create_parser_flag = False
+    create_mapper_flag = False
+    parser_create_dto = False
+    mapper_create_dto = False
+
+    dto_class = None
+    dto_module = None
+    mapper_input_class = None
+    mapper_input_module = None
+    loader_ep = None
+    loader_args = None
+    parser_ep = None
+    mapper_ep = None
+    parser_name = None
+    mapper_name = None
+    pchoice = "identity"
+
+    # Source selection (may override shared context if existing is chosen)
+    source_choice = pick_from_menu(
+        "Source:",
+        [
+            ("create", "Create new source (default)"),
+            ("existing", "Select existing source"),
+        ],
+    )
+
+    if source_choice == "existing":
+        sources = list_sources(project_yaml)
+        if not sources:
+            error_exit("No sources found. Create one first.")
+        source_id = pick_from_list("Select source:", sources)
+        parts = source_id.split(".", 1)
+        provider = parts[0] if len(parts) == 2 else provider
+        dataset = parts[1] if len(parts) == 2 else dataset
+    else:
+        source_id_default = f"{provider}.{dataset}"
+        source_id = choose_name("Source id", default=source_id_default)
+        create_source = True
+
+        # Loader selection
+        loader_ep = None
+        loader_args = {}
+        choice = pick_from_menu(
+            "Loader:",
+            [
+                ("fs", "Built-in fs"),
+                ("http", "Built-in http"),
+                ("synthetic", "Built-in synthetic"),
+                ("custom", "Custom loader"),
+            ],
+            allow_default=False,
+        )
+        if choice in {"fs", "http", "synthetic"}:
+            if choice in {"fs", "http"}:
+                fmt_options = [
+                    ("csv", "csv"),
+                    ("json", "json"),
+                    ("json-lines", "json-lines"),
+                ]
+                if choice == "fs":
+                    fmt_options.append(("pickle", "pickle"))
+                fmt = pick_from_menu("Format:", fmt_options, allow_default=False)
+            else:
+                fmt = None
+            loader_ep, loader_args = default_loader_config(choice, fmt)
+        else:
+            loader_ep = prompt_required("Loader entrypoint")
+
+        # Parser selection
+        parsers = list_parsers(root=plugin_root)
+        if parsers:
+            pchoice = pick_from_menu(
+                "Parser:",
+                [
+                    ("create", "Create new parser (default)"),
+                    ("existing", "Select existing parser"),
+                    ("identity", "Identity parser"),
+                ],
+            )
+        else:
+            pchoice = pick_from_menu(
+                "Parser:",
+                [
+                    ("create", "Create new parser (default)"),
+                    ("identity", "Identity parser"),
+                ],
+            )
+        if pchoice == "existing":
+            parser_ep = pick_from_menu(
+                "Select parser entrypoint:",
+                [(k, k) for k in sorted(parsers.keys())],
+            )
+        elif pchoice == "create":
+            dto_default = dto_class_name(f"{provider}_{dataset}") if provider and dataset else None
+            dto_class, parser_create_dto = choose_existing_or_create_name(
+                label=LABEL_DTO_FOR_PARSER,
+                existing=sorted(list_dtos(root=plugin_root).keys()),
+                create_label="Create new DTO",
+                prompt_new="DTO class name",
+                default_new=dto_default,
+            )
+            parser_name = choose_name(
+                "Parser class name",
+                default=default_parser_name(dto_class),
+            )
+            dto_module = dto_module_path(pkg_name, dto_class)
+            create_parser_flag = True
+        elif pchoice == "identity":
+            parser_ep = "identity"
+        else:
+            parser_ep = "identity"
+
+    # Domain selection
+    domain, create_domain_flag = choose_existing_or_create_name(
+        label=LABEL_DOMAIN_TO_MAP,
+        existing=list_domains(root=plugin_root),
+        create_label="Create new domain",
+        prompt_new="Domain name",
+        default_new=dataset,
+    )
+
+    # Mapper selection
+    mappers = list_mappers(root=plugin_root)
+    if mappers:
+        mchoice = pick_from_menu(
+            "Mapper:",
+            [
+                ("create", "Create new mapper (default)"),
+                ("existing", "Select existing mapper"),
+                ("identity", "Identity mapper"),
+            ],
+        )
+    else:
+        mchoice = pick_from_menu(
+            "Mapper:",
+            [
+                ("create", "Create new mapper (default)"),
+                ("identity", "Identity mapper"),
+            ],
+        )
+    if mchoice == "existing":
+        mapper_ep = pick_from_menu(
+            "Select mapper entrypoint:",
+            [(k, k) for k in sorted(mappers.keys())],
+        )
+    elif mchoice == "create":
+        create_mapper_flag = True
+        input_choice = pick_from_menu(
+            f"{LABEL_MAPPER_INPUT}:",
+            [
+                ("dto", "DTO (default)"),
+                ("identity", "Any"),
+            ],
+        )
+        info("Domain output: Domain record")
+        if input_choice == "dto":
+            if not dto_class:
+                dto_class, mapper_create_dto = choose_existing_or_create_name(
+                    label=LABEL_DTO_FOR_MAPPER,
+                    existing=sorted(list_dtos(root=plugin_root).keys()),
+                    create_label="Create new DTO",
+                    prompt_new="DTO class name",
+                    default_new=dto_class_name(f"{provider}_{dataset}"),
+                )
+            else:
+                mapper_create_dto = False
+            dto_module = dto_module_path(pkg_name, dto_class)
+            mapper_input_class = dto_class
+            mapper_input_module = dto_module
+        else:
+            mapper_input_module = "typing"
+            mapper_input_class = "Any"
+            mapper_create_dto = False
+        if input_choice == "identity":
+            mapper_name = choose_name(
+                "Mapper name",
+                default=default_mapper_name_for_identity(domain),
+            )
+        else:
+            mapper_name = choose_name("Mapper name", default=default_mapper_name(mapper_input_module, domain))
+    elif mchoice == "identity":
+        mapper_ep = "identity"
+    else:
+        mapper_ep = "identity"
+
+    # Stream id and contract
+    default_id = default_stream_id(domain, dataset or "dataset", None)
+    stream_id = choose_name("Stream id", default=default_id)
+
+    # Build plan and execute (no side effects during selection)
+    parser_plan = None
+    if pchoice == "create":
+        parser_plan = ParserPlan(
+            create=True,
+            create_dto=parser_create_dto,
+            dto_class=dto_class,
+            dto_module=dto_module,
+            parser_name=parser_name,
+        )
+    elif pchoice == "existing":
+        parser_plan = ParserPlan(create=False, parser_ep=parser_ep)
+    else:
+        parser_plan = ParserPlan(create=False, parser_ep="identity")
+
+    mapper_plan = None
+    if mchoice == "create":
+        mapper_plan = MapperPlan(
+            create=True,
+            create_dto=mapper_create_dto,
+            input_class=mapper_input_class,
+            input_module=mapper_input_module,
+            mapper_name=mapper_name,
+            domain=domain,
+        )
+    elif mchoice == "existing":
+        mapper_plan = MapperPlan(create=False, mapper_ep=mapper_ep, domain=domain)
+    else:
+        mapper_plan = MapperPlan(create=False, mapper_ep="identity", domain=domain)
+
+    plan = StreamPlan(
+        provider=provider,
+        dataset=dataset,
+        source_id=source_id,
+        project_yaml=project_yaml,
+        stream_id=stream_id,
+        root=plugin_root,
+        create_source=create_source,
+        loader_ep=loader_ep,
+        loader_args=loader_args,
+        parser=parser_plan,
+        mapper=mapper_plan,
+        domain=domain,
+        create_domain=create_domain_flag,
+    )
+    execute_stream_plan(plan)

datapipeline/cli/visuals/common.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from pathlib import Path
 import logging
 import os

datapipeline/cli/visuals/sections.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from pathlib import Path
 from typing import Optional, Tuple
 

datapipeline/cli/workspace_utils.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from pathlib import Path
 
 from datapipeline.config.workspace import WorkspaceContext
@@ -22,4 +20,3 @@ def resolve_default_project_yaml(workspace: WorkspaceContext | None) -> Path | N
             f"Unknown default_dataset '{alias}'. Define it under datasets: in jerry.yaml."
         )
     return resolved
-

datapipeline/config/context.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional, Sequence

datapipeline/config/dataset/feature.py

@@ -9,5 +9,6 @@ class BaseRecordConfig(BaseModel):
 
 class FeatureRecordConfig(BaseRecordConfig):
     id: str
+    field: str
     scale: Optional[Union[bool, Mapping[str, Any]]] = Field(default=False)
     sequence: Optional[Mapping[str, Any]] = Field(default=None)

datapipeline/config/metadata.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from datetime import datetime
 from typing import Any, Dict, List, Optional
 

datapipeline/config/project.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from datetime import datetime
 from typing import Optional
 from pydantic import BaseModel, Field, ConfigDict

datapipeline/config/resolution.py

@@ -1,11 +1,10 @@
-from __future__ import annotations
-
 import logging
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Optional
 
 from datapipeline.config.tasks import ServeOutputConfig
+from datapipeline.io.output import OutputResolutionError
 from datapipeline.config.workspace import WorkspaceContext
 
 
@@ -113,6 +112,15 @@ def workspace_output_defaults(
     if not serve_defaults or not serve_defaults.output:
         return None
     od = serve_defaults.output
+    transport = str(od.transport).lower() if od.transport is not None else None
+    if transport == "fs" and not od.directory:
+        raise OutputResolutionError(
+            "fs output requires a directory. Example:\n"
+            "  output:\n"
+            "    transport: fs\n"
+            "    format: json-lines\n"
+            "    directory: ./data/processed/jerry"
+        )
     output_dir = None
     if od.directory:
         candidate = Path(od.directory)

datapipeline/config/tasks.py

@@ -1,7 +1,5 @@
-from __future__ import annotations
-
 from pathlib import Path
-from typing import Annotated, Iterable, List, Literal, Sequence
+from typing import Annotated, Literal, Sequence
 
 from pydantic import BaseModel, Field, field_validator, model_validator
 from pydantic.type_adapter import TypeAdapter
@@ -21,9 +19,10 @@ PayloadMode = Literal["sample", "vector"]
 class TaskBase(BaseModel):
     version: int = Field(default=1)
     kind: str
-    name: str | None = Field(default=None, description="Optional task identifier.")
-    enabled: bool = Field(default=True, description="Disable to skip execution.")
-    depends_on: list[str] = Field(default_factory=list)
+    name: str | None = Field(
+        default=None, description="Optional task identifier.")
+    enabled: bool = Field(
+        default=True, description="Disable to skip execution.")
     source_path: Path | None = Field(default=None, exclude=True)
 
     def effective_name(self) -> str:
@@ -78,7 +77,8 @@ class RuntimeTask(TaskBase):
 
 class ServeOutputConfig(BaseModel):
     transport: Transport = Field(..., description="fs | stdout")
-    format: Format = Field(..., description="csv | json | json-lines | print | pickle")
+    format: Format = Field(...,
+                           description="csv | json | json-lines | print | pickle")
     payload: PayloadMode = Field(
         default="sample",
         description="sample (key + metadata) or vector payload (features [+targets]).",
@@ -151,9 +151,9 @@ class ServeTask(RuntimeTask):
     )
     stage: int | None = Field(
         default=None,
-        description="Default pipeline stage preview (0-7).",
+        description="Default pipeline stage preview (0-8).",
         ge=0,
-        le=7,
+        le=8,
     )
     throttle_ms: float | None = Field(
         default=None,

datapipeline/domain/feature.py

@@ -1,5 +1,6 @@
 from datapipeline.domain.record import TemporalRecord
 from dataclasses import dataclass
+from typing import Any
 
 
 @dataclass
@@ -10,8 +11,10 @@ class BaseFeature:
 @dataclass
 class FeatureRecord(BaseFeature):
     record: TemporalRecord
+    value: Any
 
 
 @dataclass
 class FeatureRecordSequence(BaseFeature):
     records: list[TemporalRecord]
+    values: list[Any]

datapipeline/domain/record.py

@@ -1,6 +1,5 @@
-from dataclasses import dataclass, asdict
+from dataclasses import dataclass
 from datetime import datetime, timezone
-from typing import Any
 
 
 @dataclass
@@ -13,7 +12,6 @@ class TemporalRecord(Record):
     """Canonical time-series payload used throughout the pipeline."""
 
     time: datetime
-    value: Any
 
     def __post_init__(self) -> None:
         if self.time.tzinfo is None:
@@ -21,10 +19,13 @@ class TemporalRecord(Record):
         self.time = self.time.astimezone(timezone.utc)
 
     def _identity_fields(self) -> dict:
-        """Return a mapping of domain fields excluding 'time' and 'value'."""
-        data = asdict(self)
+        """Return a mapping of domain fields excluding 'time'."""
+        data = {
+            key: value
+            for key, value in self.__dict__.items()
+            if not key.startswith("_")
+        }
         data.pop("time", None)
-        data.pop("value", None)
         return data
 
     def __eq__(self, other: object) -> bool:
@@ -34,6 +35,5 @@ class TemporalRecord(Record):
             return NotImplemented
         return (
             self.time == other.time
-            and self.value == other.value
             and self._identity_fields() == other._identity_fields()
         )

datapipeline/domain/sample.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from dataclasses import dataclass, asdict
 from typing import Any, Iterator, Optional, Literal
 

datapipeline/domain/vector.py

@@ -1,6 +1,4 @@
-from datapipeline.domain.record import TemporalRecord
-from typing import Dict
-from typing import Union
+from typing import Dict, Union, Any
 
 from dataclasses import dataclass
 
@@ -25,13 +23,13 @@ class Vector:
         return self.values[key]
 
 
-def vectorize_record_group(values: Dict[str, list[TemporalRecord]]) -> Vector:
+def vectorize_record_group(values: Dict[str, list[Any]]) -> Vector:
     structured: Dict[str, Union[float, list[float]]] = {}
 
-    for key, records in values.items():
-        if len(records) == 1:
-            structured[key] = records[0].value
+    for key, items in values.items():
+        if len(items) == 1:
+            structured[key] = items[0]
         else:
-            structured[key] = [r.value for r in records]
+            structured[key] = list(items)
 
     return Vector(values=structured)

datapipeline/integrations/ml/adapter.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from collections.abc import Iterator, Sequence
 from dataclasses import dataclass
 from itertools import islice

datapipeline/integrations/ml/pandas_support.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from collections.abc import Callable, Iterable
 from typing import Any
 

datapipeline/integrations/ml/rows.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from collections.abc import Iterator, Sequence
 from pathlib import Path
 from typing import Any

datapipeline/integrations/ml/torch_support.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Mapping

datapipeline/io/output.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional

datapipeline/io/serializers.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import json
 from dataclasses import asdict, is_dataclass
 from typing import Any, Dict, Type
@@ -107,20 +105,32 @@ class VectorPickleSerializer(BasePickleSerializer):
 
 
 def _record_payload(value: Any) -> Any:
-    if value is None:
-        return None
-    if is_dataclass(value):
-        return asdict(value)
-    if isinstance(value, dict):
-        return value
-    attrs = getattr(value, "__dict__", None)
-    if attrs:
-        return {
-            k: v
-            for k, v in attrs.items()
-            if not k.startswith("_")
-        }
-    return value
+    def _convert(obj: Any) -> Any:
+        if obj is None:
+            return None
+        if is_dataclass(obj):
+            attrs = getattr(obj, "__dict__", None)
+            if attrs is not None:
+                return {
+                    k: _convert(v)
+                    for k, v in attrs.items()
+                    if not k.startswith("_")
+                }
+            return asdict(obj)
+        if isinstance(obj, dict):
+            return {k: _convert(v) for k, v in obj.items()}
+        if isinstance(obj, (list, tuple)):
+            return [_convert(v) for v in obj]
+        attrs = getattr(obj, "__dict__", None)
+        if attrs:
+            return {
+                k: _convert(v)
+                for k, v in attrs.items()
+                if not k.startswith("_")
+            }
+        return obj
+
+    return _convert(value)
 
 
 def _record_key(value: Any) -> Any:

datapipeline/mappers/synthetic/time.py

@@ -1,9 +1,16 @@
-from typing import Iterator
+from dataclasses import dataclass
 from datetime import datetime
 from math import sin, pi
+from typing import Iterator
+
 from datapipeline.domain.record import TemporalRecord
 
 
+@dataclass
+class TimeEncodedRecord(TemporalRecord):
+    value: float
+
+
 def encode(stream: Iterator[TemporalRecord], mode: str) -> Iterator[TemporalRecord]:
     for rec in stream:
         t: datetime = rec.time
@@ -15,4 +22,4 @@ def encode(stream: Iterator[TemporalRecord], mode: str) -> Iterator[TemporalReco
             val = t.timestamp()
         else:
             raise ValueError(f"Unsupported encode_time mode: {mode}")
-        yield TemporalRecord(time=rec.time, value=val)
+        yield TimeEncodedRecord(time=rec.time, value=val)

datapipeline/pipeline/artifacts.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from dataclasses import dataclass
 from typing import Iterable
 
@@ -43,12 +41,12 @@ def required_artifacts_for(
     needs_metadata = False
     for demand in demands:
         stage = demand.stage
-        effective_stage = 7 if stage is None else stage
+        effective_stage = 8 if stage is None else stage
 
-        if effective_stage >= 5 and _requires_scaler(dataset):
+        if effective_stage >= 6 and _requires_scaler(dataset):
             required.add(SCALER_STATISTICS)
 
-        if effective_stage >= 6:
+        if effective_stage >= 7:
             required.add(VECTOR_SCHEMA)
             needs_metadata = True
 

datapipeline/pipeline/observability.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import logging
 from dataclasses import dataclass
 from typing import Callable, Mapping, Optional, Protocol, runtime_checkable