jerry-thomas 1.0.2__py3-none-any.whl → 1.0.3__py3-none-any.whl

datapipeline/cli/app.py

@@ -31,15 +31,9 @@ def _dataset_to_project_path(
     """Resolve a dataset selector (alias, folder, or file) into a project.yaml path."""
     # 1) Alias via jerry.yaml datasets (wins over local folders with same name)
     if workspace is not None:
-        datasets = getattr(workspace.config, "datasets", {}) or {}
-        raw = datasets.get(dataset)
-        if raw:
-            base = workspace.root
-            candidate = Path(raw)
-            candidate = candidate if candidate.is_absolute() else (base / candidate)
-            if candidate.is_dir():
-                candidate = candidate / "project.yaml"
-            return str(candidate.resolve())
+        resolved = workspace.resolve_dataset_alias(dataset)
+        if resolved is not None:
+            return str(resolved)
 
     # 2) Direct file path
     path = Path(dataset)
@@ -640,6 +634,7 @@ def main() -> None:
                 alias=getattr(args, "alias", None),
                 identity=getattr(args, "identity", False),
                 plugin_root=plugin_root,
+                workspace=workspace_context,
             )
         return
 
@@ -658,6 +653,7 @@ def main() -> None:
         handle_contract(
             plugin_root=plugin_root,
             use_identity=args.identity,
+            workspace=workspace_context,
         )
         return
 

datapipeline/cli/commands/contract.py

@@ -1,6 +1,8 @@
 import sys
 from pathlib import Path
 
+from datapipeline.config.workspace import WorkspaceContext
+from datapipeline.cli.workspace_utils import resolve_default_project_yaml
 from datapipeline.services.paths import pkg_root, resolve_base_pkg_dir
 from datapipeline.services.entrypoints import read_group_entries, inject_ep
 from datapipeline.services.constants import FILTERS_GROUP, MAPPERS_GROUP
@@ -31,8 +33,10 @@ def handle(
     *,
     plugin_root: Path | None = None,
     use_identity: bool = False,
+    workspace: WorkspaceContext | None = None,
 ) -> None:
     root_dir, name, pyproject = pkg_root(plugin_root)
+    default_project = resolve_default_project_yaml(workspace)
     # Select contract type: Ingest (source->stream) or Composed (streams->stream)
     print("Select contract type:", file=sys.stderr)
     print("  [1] Ingest (source → stream)", file=sys.stderr)
@@ -49,12 +53,13 @@ def handle(
             mapper_path=None,
             with_mapper_stub=True,
             plugin_root=plugin_root,
+            project_yaml=default_project,
         )
         return
 
     # Discover sources by scanning sources_dir YAMLs
     # Default to dataset-scoped project config
-    proj_path = resolve_project_yaml_path(root_dir)
+    proj_path = default_project or resolve_project_yaml_path(root_dir)
     # Ensure a minimal project scaffold so we can resolve dirs interactively
     ensure_project_scaffold(proj_path)
     sources_dir = resolve_sources_dir(proj_path)
@@ -187,6 +192,7 @@ def scaffold_conflux(
     mapper_path: str | None,
     with_mapper_stub: bool,
     plugin_root: Path | None,
+    project_yaml: Path | None,
 ) -> None:
     """Scaffold a composed (multi-input) contract and optional mapper stub.
 
@@ -195,7 +201,7 @@ def scaffold_conflux(
     """
     root_dir, name, _ = pkg_root(plugin_root)
     # Resolve default project path early for interactive selections
-    proj_path = resolve_project_yaml_path(root_dir)
+    proj_path = project_yaml or resolve_project_yaml_path(root_dir)
     ensure_project_scaffold(proj_path)
     # Defer target domain selection until after choosing inputs
 

datapipeline/cli/commands/source.py

@@ -1,5 +1,7 @@
 from pathlib import Path
 
+from datapipeline.config.workspace import WorkspaceContext
+from datapipeline.cli.workspace_utils import resolve_default_project_yaml
 from datapipeline.services.scaffold.source import create_source
 
 
@@ -13,6 +15,7 @@ def handle(
     identity: bool = False,
     alias: str | None = None,
     plugin_root: Path | None = None,
+    workspace: WorkspaceContext | None = None,
 ) -> None:
     if subcmd in {"create", "add"}:
         # Allow: positional provider dataset, --provider/--dataset, --alias, or provider as 'prov.ds'
@@ -43,6 +46,7 @@ def handle(
         if transport in {"fs", "http"} and not format:
             print("[error] --format is required for fs/http transports (fs: csv|json|json-lines|pickle, http: csv|json|json-lines)")
             raise SystemExit(2)
+        project_yaml = resolve_default_project_yaml(workspace)
         create_source(
             provider=provider,
             dataset=dataset,
@@ -50,4 +54,5 @@ def handle(
             format=format,
             root=plugin_root,
             identity=identity,
+            **({"project_yaml": project_yaml} if project_yaml is not None else {}),
         )

datapipeline/cli/visuals/common.py

@@ -7,6 +7,7 @@ from typing import Optional, Sequence
 
 from urllib.parse import urlparse
 from datapipeline.sources.transports import FsGlobTransport, FsFileTransport, HttpTransport
+from datapipeline.sources.foreach import ForeachLoader
 
 logger = logging.getLogger(__name__)
 
@@ -217,23 +218,74 @@ def current_transport_label(transport, *, glob_root: Optional[Path] = None) -> O
         current = getattr(transport, "current_path", None)
         if not current:
             return None
-        return relative_label(current, glob_root)
+        return f"\"{relative_label(current, glob_root)}\""
     if isinstance(transport, FsFileTransport):
         path = getattr(transport, "path", None)
         if not path:
             return None
         try:
-            return Path(path).name or str(path)
+            name = Path(path).name or str(path)
+            return f"\"{name}\""
         except Exception:
-            return str(path)
+            return f"\"{path}\""
     if isinstance(transport, HttpTransport):
         url = getattr(transport, "url", None)
         if not url:
             return None
         try:
             parts = urlparse(url)
-            name = Path(parts.path or "").name
-            return name or (parts.netloc or "http")
+            host = parts.netloc or "http"
+            return f"@{host}"
         except Exception:
             return None
     return None
+
+
+def current_loader_label(loader, transport, *, glob_root: Optional[Path] = None) -> Optional[str]:
+    """Return a human-friendly label for the loader's current unit of work."""
+    if isinstance(loader, ForeachLoader):
+        value = getattr(loader, "_current_value", None)
+        if value is None:
+            return None
+        idx = getattr(loader, "_current_index", None)
+        values = getattr(loader, "_values", None)
+        total = len(values) if isinstance(values, list) else None
+
+        item_label = f"\"{value}\""
+        status = None
+        if isinstance(idx, int) and isinstance(total, int) and total > 0:
+            status = f"({idx}/{total})"
+
+        def _with_item(action: str | None) -> str:
+            parts = []
+            if action:
+                parts.append(action)
+            parts.append(item_label)
+            if status:
+                parts.append(status)
+            return " ".join(parts)
+
+        spec = getattr(loader, "_loader_spec", None) or {}
+        entrypoint = spec.get("entrypoint", "") if isinstance(spec, dict) else ""
+        args = getattr(loader, "_current_args", None)
+        inner_transport = getattr(loader, "_current_transport", None)
+
+        if entrypoint == "core.io" and isinstance(args, dict):
+            t = args.get("transport")
+            if t == "http":
+                parts = urlparse(str(args.get("url", "")))
+                host = parts.netloc or "http"
+                return _with_item(f"Downloading @{host}")
+            if t == "fs":
+                inner_root = None
+                if isinstance(inner_transport, FsGlobTransport):
+                    inner_root = compute_glob_root(getattr(inner_transport, "files", []))
+                label = current_transport_label(inner_transport, glob_root=inner_root)
+                action = f"Loading {label}" if label else "Loading fs"
+                return _with_item(action)
+
+        if entrypoint:
+            return _with_item(f"via {entrypoint}")
+        return _with_item(None)
+
+    return current_transport_label(transport, glob_root=glob_root)

datapipeline/cli/visuals/labels.py

@@ -1,41 +1,9 @@
-from pathlib import Path
-from urllib.parse import urlparse
-
 from datapipeline.sources.models.loader import SyntheticLoader, BaseDataLoader
 from datapipeline.sources.data_loader import DataLoader
+from datapipeline.sources.foreach import ForeachLoader
 from datapipeline.sources.transports import FsFileTransport, FsGlobTransport, HttpTransport
 from datapipeline.sources.decoders import CsvDecoder, JsonDecoder, JsonLinesDecoder, PickleDecoder
 
-MAX_LABEL_LEN = 48
-GLOB_SEGMENTS = 3
-
-
-def _truncate_middle(text: str, max_len: int) -> str:
-    if len(text) <= max_len:
-        return text
-    if max_len <= 3:
-        return text[:max_len]
-    keep = max_len - 3
-    head = (keep + 1) // 2
-    tail = keep - head
-    suffix = text[-tail:] if tail > 0 else ""
-    return f"{text[:head]}...{suffix}"
-
-
-def _compact_path_label(name: str) -> str:
-    if not name:
-        return "fs"
-    normalized = name.replace("\\", "/").strip()
-    if not normalized:
-        return "fs"
-    parts = [part for part in normalized.split("/") if part]
-    if not parts:
-        return normalized
-    if len(parts) > GLOB_SEGMENTS:
-        parts = parts[-GLOB_SEGMENTS:]
-    label = "/".join(parts)
-    return _truncate_middle(label, MAX_LABEL_LEN)
-
 
 def unit_for_loader(loader) -> str:
     if isinstance(loader, SyntheticLoader):
@@ -56,18 +24,17 @@ def build_source_label(loader: BaseDataLoader) -> str:
         except Exception:
             gen_name = loader.__class__.__name__
         return "Generating data with " + gen_name
+    if isinstance(loader, ForeachLoader):
+        key = str(getattr(loader, "_key", "item"))
+        values = getattr(loader, "_values", None)
+        n = len(values) if isinstance(values, list) else "?"
+        return f"Fan-out {key}×{n}:"
     if isinstance(loader, DataLoader):
         transport = getattr(loader, "transport", None)
         if isinstance(transport, (FsFileTransport, FsGlobTransport)):
-            name = str(getattr(transport, "pattern", getattr(transport, "path", "")))
-            if isinstance(transport, FsFileTransport) and name and "*" not in name:
-                label = Path(name).name or "fs"
-            else:
-                label = _compact_path_label(name)
-            return f"Loading data from: {label}"
+            return "Loading"
         if isinstance(transport, HttpTransport):
-            host = urlparse(transport.url).netloc or "http"
-            return f"Downloading data from: @{host}"
+            return "Downloading"
     return loader.__class__.__name__
 
 

datapipeline/cli/visuals/sources_rich.py

@@ -24,13 +24,14 @@ from rich.text import Text
 from .labels import progress_meta_for_loader
 from .common import (
     compute_glob_root,
-    current_transport_label,
+    current_loader_label,
     log_combined_stream,
     transport_debug_lines,
     transport_info_lines,
 )
 from datapipeline.runtime import Runtime
 from datapipeline.sources.models.source import Source
+from datapipeline.sources.foreach import ForeachLoader
 from datapipeline.sources.transports import FsGlobTransport, FsFileTransport, HttpTransport
 logger = logging.getLogger(__name__)
 
@@ -144,8 +145,12 @@ class _RichSourceProxy(Source):
             glob_root = compute_glob_root(
                 getattr(transport, "files", []))
 
+        is_foreach_loader = isinstance(loader, ForeachLoader)
+
         def compose_text(name: Optional[str]) -> str:
             if name:
+                if is_foreach_loader:
+                    return str(name)
                 base = header if sep else desc
                 return f"{base} {name}".rstrip()
             if tail:
@@ -173,8 +178,8 @@ class _RichSourceProxy(Source):
 
         try:
             for item in self._inner.stream():
-                current_label = current_transport_label(
-                    transport, glob_root=glob_root
+                current_label = current_loader_label(
+                    loader, transport, glob_root=glob_root
                 )
                 # On first item: emit Start + transport details
                 if not started_logged:

datapipeline/cli/workspace_utils.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from datapipeline.config.workspace import WorkspaceContext
+
+
+def resolve_default_project_yaml(workspace: WorkspaceContext | None) -> Path | None:
+    """Resolve default_dataset from jerry.yaml into a project.yaml path.
+
+    Returns None when no workspace context or no default_dataset is configured.
+    Raises SystemExit when default_dataset is set but missing from datasets:.
+    """
+    if workspace is None:
+        return None
+    alias = workspace.config.default_dataset
+    if not alias:
+        return None
+    resolved = workspace.resolve_dataset_alias(alias)
+    if resolved is None:
+        raise SystemExit(
+            f"Unknown default_dataset '{alias}'. Define it under datasets: in jerry.yaml."
+        )
+    return resolved
+

datapipeline/config/dataset/dataset.py

@@ -10,6 +10,6 @@ class RecordDatasetConfig(BaseModel):
 
 
 class FeatureDatasetConfig(BaseModel):
-    group_by: str = Field(..., pattern=r"^\d+(m|min|h)$")
+    group_by: str = Field(..., pattern=r"^\d+(m|min|h|d)$")
     features: List[FeatureRecordConfig] = Field(default_factory=list)
     targets:  List[FeatureRecordConfig] = Field(default_factory=list)

datapipeline/config/dataset/normalize.py

@@ -1,14 +1,14 @@
-from datetime import datetime
+from datetime import datetime, timedelta
 import re
 
 
 def floor_time_to_bucket(ts: datetime, bucket: str) -> datetime:
     """Floor a timestamp to the nearest bucket boundary.
 
-    Supports patterns like '10m', '10min', '1h', '2h'.
+    Supports patterns like '10m', '10min', '1h', '2h', '1d'.
     Minutes may be specified as 'm' or 'min'.
     """
-    m = re.fullmatch(r"^(\d+)(m|min|h)$", bucket)
+    m = re.fullmatch(r"^(\d+)(m|min|h|d)$", bucket)
     if not m:
         raise ValueError(f"Unsupported cadence: {bucket}")
     n = int(m.group(1))
@@ -19,6 +19,11 @@ def floor_time_to_bucket(ts: datetime, bucket: str) -> datetime:
     if unit in ("m", "min"):
         floored_minute = (ts.minute // n) * n
         return ts.replace(minute=floored_minute, second=0, microsecond=0)
-    else:  # 'h'
+    if unit == "h":
         floored_hour = (ts.hour // n) * n
         return ts.replace(hour=floored_hour, minute=0, second=0, microsecond=0)
+    base = ts.replace(hour=0, minute=0, second=0, microsecond=0)
+    if n == 1:
+        return base
+    remainder = (base.toordinal() - 1) % n
+    return base - timedelta(days=remainder)

datapipeline/config/workspace.py

@@ -125,6 +125,21 @@ class WorkspaceContext:
     def root(self) -> Path:
         return self.file_path.parent
 
+    def resolve_dataset_alias(self, alias: str) -> Optional[Path]:
+        """Resolve a dataset alias from jerry.yaml into an absolute project.yaml path."""
+        raw = (self.config.datasets or {}).get(alias)
+        if not raw:
+            return None
+        candidate = Path(raw)
+        candidate = (
+            candidate.resolve()
+            if candidate.is_absolute()
+            else (self.root / candidate).resolve()
+        )
+        if candidate.is_dir():
+            candidate = candidate / "project.yaml"
+        return candidate.resolve()
+
     def resolve_plugin_root(self) -> Optional[Path]:
         raw = self.config.plugin_root
         if not raw:

datapipeline/services/scaffold/source.py

@@ -98,6 +98,7 @@ def create_source(
     format: Optional[str],
     root: Optional[Path],
     identity: bool = False,
+    project_yaml: Optional[Path] = None,
 ) -> None:
     root_dir, name, _ = pkg_root(root)
     base = resolve_base_pkg_dir(root_dir, name)
@@ -169,7 +170,7 @@ def create_source(
     # Resolve sources directory from a single dataset-scoped project config.
     # If not present or invalid, let the exception bubble up to prompt the user
     # to provide a valid project path.
-    proj_yaml = resolve_project_yaml_path(root_dir)
+    proj_yaml = project_yaml.resolve() if project_yaml is not None else resolve_project_yaml_path(root_dir)
     # Best-effort: create a minimal project scaffold if missing
     ensure_project_scaffold(proj_yaml)
     sources_dir = resolve_sources_dir(proj_yaml).resolve()

datapipeline/sources/foreach.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import re
+from typing import Any, Iterator, Mapping
+
+from datapipeline.plugins import LOADERS_EP
+from datapipeline.sources.models.loader import BaseDataLoader
+from datapipeline.utils.load import load_ep
+from datapipeline.utils.placeholders import normalize_args, MissingInterpolation, is_missing
+
+
+_VAR_RE = re.compile(r"\$\{([^}]+)\}")
+
+
+def _interpolate(obj: Any, vars_: Mapping[str, Any]) -> Any:
+    if isinstance(obj, dict):
+        return {k: _interpolate(v, vars_) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_interpolate(v, vars_) for v in obj]
+    if isinstance(obj, str):
+        match = _VAR_RE.fullmatch(obj)
+        if match:
+            key = match.group(1)
+            if key in vars_:
+                value = vars_[key]
+                if value is None or is_missing(value):
+                    return MissingInterpolation(key)
+                return value
+            return obj
+
+        def repl(m):
+            key = m.group(1)
+            value = vars_.get(key, m.group(0))
+            if value is None or is_missing(value):
+                return m.group(0)
+            return str(value)
+
+        return _VAR_RE.sub(repl, obj)
+    return obj
+
+
+class ForeachLoader(BaseDataLoader):
+    """Expand a loader spec across a foreach map and concatenate results."""
+
+    def __init__(
+        self,
+        *,
+        foreach: Mapping[str, list[Any]],
+        loader: Mapping[str, Any],
+        inject_field: str | None = None,
+        inject: Mapping[str, Any] | None = None,
+    ):
+        self._key, self._values = self._normalize_foreach(foreach)
+        self._loader_spec = self._normalize_loader_spec(loader)
+        self._inject_field = inject_field
+        self._inject = inject
+        self._current_index: int | None = None
+        self._current_value: Any | None = None
+        self._current_args: dict[str, Any] | None = None
+        self._current_transport: Any | None = None
+
+        if inject_field and inject:
+            raise ValueError("core.foreach supports only one of inject_field or inject")
+        if inject_field and self._key is None:
+            raise ValueError("inject_field requires a non-empty foreach map")
+        if inject is not None and not isinstance(inject, Mapping):
+            raise TypeError("inject must be a mapping when provided")
+
+    def load(self) -> Iterator[Any]:
+        for i, value in enumerate(self._values, 1):
+            vars_ = {self._key: value}
+            loader_args = self._make_loader_args(vars_)
+            loader = self._build_loader(loader_args)
+            self._current_index = i
+            self._current_value = value
+            self._current_args = loader_args
+            self._current_transport = getattr(loader, "transport", None)
+            inject_map = self._build_inject(vars_)
+            for row in loader.load():
+                if inject_map:
+                    yield self._apply_inject(row, inject_map)
+                else:
+                    yield row
+
+    def count(self):
+        total = 0
+        for value in self._values:
+            vars_ = {self._key: value}
+            loader_args = self._make_loader_args(vars_)
+            loader = self._build_loader(loader_args)
+            c = loader.count()
+            if c is None:
+                return None
+            total += int(c)
+        return total
+
+    @staticmethod
+    def _normalize_foreach(foreach: Mapping[str, list[Any]]):
+        if not isinstance(foreach, Mapping) or not foreach:
+            raise ValueError("core.foreach requires a non-empty foreach mapping")
+        keys = list(foreach.keys())
+        if len(keys) != 1:
+            raise ValueError("core.foreach currently supports exactly one foreach key")
+        key = keys[0]
+        values = foreach[key]
+        if not isinstance(values, list):
+            raise TypeError("core.foreach foreach values must be a list")
+        return str(key), list(values)
+
+    @staticmethod
+    def _normalize_loader_spec(loader: Mapping[str, Any]) -> Mapping[str, Any]:
+        if not isinstance(loader, Mapping):
+            raise TypeError("core.foreach loader must be a mapping with entrypoint/args")
+        entrypoint = loader.get("entrypoint")
+        if not entrypoint or not isinstance(entrypoint, str):
+            raise ValueError("core.foreach loader.entrypoint must be a non-empty string")
+        args = loader.get("args")
+        if args is not None and not isinstance(args, Mapping):
+            raise TypeError("core.foreach loader.args must be a mapping when provided")
+        return dict(loader)
+
+    def _make_loader_args(self, vars_: Mapping[str, Any]) -> dict[str, Any]:
+        args = self._loader_spec.get("args") or {}
+        interpolated = _interpolate(args, vars_)
+        return normalize_args(interpolated)
+
+    def _build_loader(self, loader_args: dict[str, Any]) -> BaseDataLoader:
+        entrypoint = self._loader_spec["entrypoint"]
+        L = load_ep(LOADERS_EP, entrypoint)
+        return L(**loader_args)
+
+    def _build_inject(self, vars_: Mapping[str, Any]) -> Mapping[str, Any] | None:
+        if self._inject_field:
+            return {self._inject_field: vars_.get(self._key)}
+        if self._inject is None:
+            return None
+        interpolated = _interpolate(self._inject, vars_)
+        if not isinstance(interpolated, Mapping):
+            raise TypeError("core.foreach inject must resolve to a mapping")
+        return normalize_args(interpolated)
+
+    @staticmethod
+    def _apply_inject(row: Any, inject_map: Mapping[str, Any]) -> Any:
+        if isinstance(row, dict):
+            row.update(inject_map)
+            return row
+        if isinstance(row, Mapping):
+            out = dict(row)
+            out.update(inject_map)
+            return out
+        raise TypeError("core.foreach inject requires mapping rows")

datapipeline/templates/plugin_skeleton/your-dataset/project.yaml

@@ -7,7 +7,7 @@ paths:
   postprocess: postprocess.yaml
   artifacts: ../artifacts/${project_name}/v${version}
   tasks: ./tasks
-globals:
+globals: # Globals to use in your .yaml files via ${var_name}.
   # Primary dataset cadence; referenced from dataset.yaml (group_by)
   # and contracts via ${group_by}.
   group_by: <your-bucket-cadence>