jerry-thomas 0.3.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

datapipeline/config/dataset/loader.py

@@ -1,12 +1,30 @@
 from typing import Literal
-from datapipeline.config.dataset.dataset import RecordDatasetConfig, FeatureDatasetConfig
-from datapipeline.services.bootstrap import _load_by_key
+
+from datapipeline.config.dataset.dataset import (
+    RecordDatasetConfig,
+    FeatureDatasetConfig,
+)
+from datapipeline.services.bootstrap import _load_by_key, _globals, _interpolate
 
 Stage = Literal["records", "features", "vectors"]
 
 
+def _normalize_dataset_doc(doc):
+    if not isinstance(doc, dict):
+        return doc
+    normalized = dict(doc)
+    for key in ("features", "targets"):
+        if normalized.get(key) is None:
+            normalized[key] = []
+    return normalized
+
+
 def load_dataset(project_yaml, stage: Stage):
-    ds_doc = _load_by_key(project_yaml, "dataset")
+    raw = _load_by_key(project_yaml, "dataset")
+    vars_ = _globals(project_yaml)
+    if vars_:
+        raw = _interpolate(raw, vars_)
+    ds_doc = _normalize_dataset_doc(raw)
 
     if stage == "records":
         return RecordDatasetConfig.model_validate(ds_doc)
@@ -16,4 +34,3 @@ def load_dataset(project_yaml, stage: Stage):
         return FeatureDatasetConfig.model_validate(ds_doc)
     else:
         raise ValueError(f"Unknown stage: {stage}")
-

datapipeline/config/dataset/normalize.py

@@ -2,15 +2,15 @@ from datetime import datetime
 import re
 
 
-def floor_time_to_resolution(ts: datetime, resolution: str) -> datetime:
-    """Floor a timestamp to the nearest resolution bucket.
+def floor_time_to_bucket(ts: datetime, bucket: str) -> datetime:
+    """Floor a timestamp to the nearest bucket boundary.
 
     Supports patterns like '10m', '10min', '1h', '2h'.
     Minutes may be specified as 'm' or 'min'.
     """
-    m = re.fullmatch(r"^(\d+)(m|min|h)$", resolution)
+    m = re.fullmatch(r"^(\d+)(m|min|h)$", bucket)
     if not m:
-        raise ValueError(f"Unsupported granularity: {resolution}")
+        raise ValueError(f"Unsupported cadence: {bucket}")
     n = int(m.group(1))
     unit = m.group(2)
     if n <= 0:

datapipeline/config/metadata.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+# Shared keys for vector metadata counts
+FEATURE_VECTORS_COUNT_KEY = "feature_vectors"
+TARGET_VECTORS_COUNT_KEY = "target_vectors"
+
+
+class Window(BaseModel):
+    """Typed representation of dataset window bounds."""
+
+    start: Optional[datetime] = None
+    end: Optional[datetime] = None
+    mode: Optional[str] = None
+    size: Optional[int] = Field(
+        default=None,
+        description="Count of cadence buckets from start to end (inclusive) when known.",
+    )
+
+
+class VectorMetadata(BaseModel):
+    """Lightweight typed model for metadata.json.
+
+    Only window/counts/entries are modeled explicitly; all other fields are
+    accepted via extra='allow' for forwards-compatibility.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    schema_version: int = 1
+    generated_at: Optional[datetime] = None
+    window: Optional[Window] = None
+    meta: Dict[str, Any] | None = None
+    features: List[Dict[str, Any]] = Field(default_factory=list)
+    targets: List[Dict[str, Any]] = Field(default_factory=list)
+    counts: Dict[str, int] = Field(default_factory=dict)
+
+    # Window is the single source of truth; no legacy fallbacks.

datapipeline/config/postprocess.py

@@ -1,14 +1,14 @@
 from typing import Any, List
+
 from pydantic import RootModel, model_validator
 
 
 class PostprocessConfig(RootModel[List[Any]]):
-    """Schema for optional postprocess.yaml."""
+    """Schema for postprocess.yaml (list of transforms)."""
 
     @model_validator(mode="before")
     @classmethod
     def allow_empty(cls, value: Any) -> Any:
-        """Coerce missing or empty mappings into an empty list."""
         if value in (None, {}):
             return []
         return value

datapipeline/config/project.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import datetime
 from typing import Optional
 from pydantic import BaseModel, Field, ConfigDict
@@ -11,8 +13,7 @@ class ProjectPaths(BaseModel):
     dataset: str
     postprocess: str
     artifacts: str
-    build: str | None = None
-    run: str | None = None
+    tasks: str | None = None
 
 
 class ProjectGlobals(BaseModel):

datapipeline/config/resolution.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from datapipeline.config.tasks import ServeOutputConfig
+from datapipeline.config.workspace import WorkspaceContext
+
+
+def cascade(*values, fallback=None):
+    """Return the first non-None value from a list, or fallback."""
+    for value in values:
+        if value is not None:
+            return value
+    return fallback
+
+
+def _normalize_lower(value: Any) -> Optional[str]:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text.lower() if text else None
+
+
+def _normalize_upper(value: Any) -> Optional[str]:
+    if value is None:
+        return None
+    if isinstance(value, int):
+        return logging.getLevelName(value).upper()
+    text = str(value).strip()
+    return text.upper() if text else None
+
+
+def _level_value(value: Any) -> Optional[int]:
+    name = _normalize_upper(value)
+    return logging._nameToLevel.get(name) if name else None
+
+
+@dataclass(frozen=True)
+class VisualSettings:
+    visuals: str
+    progress: str
+
+
+def resolve_visuals(
+    *,
+    cli_visuals: str | None,
+    config_visuals: str | None,
+    workspace_visuals: str | None,
+    cli_progress: str | None,
+    config_progress: str | None,
+    workspace_progress: str | None,
+    default_visuals: str = "auto",
+    default_progress: str = "auto",
+) -> VisualSettings:
+    visuals = cascade(
+        _normalize_lower(cli_visuals),
+        _normalize_lower(config_visuals),
+        _normalize_lower(workspace_visuals),
+        default_visuals,
+    ) or default_visuals
+    progress = cascade(
+        _normalize_lower(cli_progress),
+        _normalize_lower(config_progress),
+        _normalize_lower(workspace_progress),
+        default_progress,
+    ) or default_progress
+    return VisualSettings(visuals=visuals, progress=progress)
+
+
+@dataclass(frozen=True)
+class LogLevelDecision:
+    name: str
+    value: int
+
+
+def resolve_log_level(
+    *levels: Any,
+    fallback: str = "INFO",
+) -> LogLevelDecision:
+    name = None
+    for level in levels:
+        normalized = _normalize_upper(level)
+        if normalized:
+            name = normalized
+            break
+    if not name:
+        name = _normalize_upper(fallback) or "INFO"
+    value = logging._nameToLevel.get(name, logging.INFO)
+    return LogLevelDecision(name=name, value=value)
+
+
+def minimum_level(*levels: Any, start: int | None = None) -> int | None:
+    """Return the lowest numeric logging level among the provided values."""
+    current = start
+    for level in levels:
+        value = _level_value(level)
+        if value is None:
+            continue
+        if current is None or value < current:
+            current = value
+    return current
+
+
+def workspace_output_defaults(
+    workspace: WorkspaceContext | None,
+) -> ServeOutputConfig | None:
+    if workspace is None:
+        return None
+    serve_defaults = getattr(workspace.config, "serve", None)
+    if not serve_defaults or not serve_defaults.output:
+        return None
+    od = serve_defaults.output
+    output_dir = None
+    if od.directory:
+        candidate = Path(od.directory)
+        output_dir = (
+            candidate
+            if candidate.is_absolute()
+            else (workspace.root / candidate).resolve()
+        )
+    return ServeOutputConfig(
+        transport=od.transport,
+        format=od.format,
+        payload=od.payload,
+        directory=output_dir,
+    )

datapipeline/config/tasks.py

@@ -0,0 +1,309 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Iterable, List, Literal, Sequence
+
+from pydantic import BaseModel, Field, field_validator, model_validator
+from pydantic.type_adapter import TypeAdapter
+
+from datapipeline.services.project_paths import tasks_dir
+from datapipeline.utils.load import load_yaml
+
+VALID_LOG_LEVELS = ("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG")
+VALID_VISUAL_PROVIDERS = ("AUTO", "TQDM", "RICH", "OFF")
+VALID_PROGRESS_STYLES = ("AUTO", "SPINNER", "BARS", "OFF")
+
+Transport = Literal["fs", "stdout"]
+Format = Literal["csv", "json", "json-lines", "print", "pickle"]
+PayloadMode = Literal["sample", "vector"]
+
+
+class TaskBase(BaseModel):
+    version: int = Field(default=1)
+    kind: str
+    name: str | None = Field(default=None, description="Optional task identifier.")
+    enabled: bool = Field(default=True, description="Disable to skip execution.")
+    depends_on: list[str] = Field(default_factory=list)
+    source_path: Path | None = Field(default=None, exclude=True)
+
+    def effective_name(self) -> str:
+        return self.name or (self.source_path.stem if self.source_path else self.kind)
+
+
+class ArtifactTask(TaskBase):
+    output: str = Field(
+        ...,
+        description="Artifact path relative to project.paths.artifacts.",
+    )
+
+
+class ScalerTask(ArtifactTask):
+    kind: Literal["scaler"]
+    output: str = Field(default="scaler.pkl")
+    split_label: str = Field(
+        default="train",
+        description="Split label to use when fitting scaler statistics.",
+    )
+
+
+class SchemaTask(ArtifactTask):
+    kind: Literal["schema"]
+    output: str = Field(default="schema.json")
+    cadence_strategy: Literal["max"] = Field(
+        default="max",
+        description="Strategy for selecting cadence targets (currently only 'max').",
+    )
+
+
+class MetadataTask(ArtifactTask):
+    kind: Literal["metadata"]
+    output: str = Field(default="metadata.json")
+    enabled: bool = Field(
+        default=True,
+        description="Disable to skip generating the vector metadata artifact.",
+    )
+    cadence_strategy: Literal["max"] = Field(
+        default="max",
+        description="Strategy for selecting cadence targets.",
+    )
+    window_mode: Literal["union", "intersection", "strict", "relaxed"] = Field(
+        default="intersection",
+        description="Window mode: union (base union), intersection (base intersection), strict (partition intersection), relaxed (partition union).",
+    )
+
+
+class RuntimeTask(TaskBase):
+    """Base class for runtime-oriented tasks (serve/evaluate/etc.)."""
+
+
+class ServeOutputConfig(BaseModel):
+    transport: Transport = Field(..., description="fs | stdout")
+    format: Format = Field(..., description="csv | json | json-lines | print | pickle")
+    payload: PayloadMode = Field(
+        default="sample",
+        description="sample (key + metadata) or vector payload (features [+targets]).",
+    )
+    directory: Path | None = Field(
+        default=None,
+        description="Directory for fs outputs.",
+    )
+    filename: str | None = Field(
+        default=None,
+        description="Filename stem (format controls extension) for fs outputs.",
+    )
+
+    @field_validator("filename", mode="before")
+    @classmethod
+    def _normalize_filename(cls, value):
+        if value is None:
+            return None
+        text = str(value).strip()
+        if not text:
+            return None
+        if any(sep in text for sep in ("/", "\\")):
+            raise ValueError("filename must not contain path separators")
+        if "." in Path(text).name:
+            raise ValueError("filename must not include an extension")
+        return text
+
+    @model_validator(mode="after")
+    def _validate(self):
+        if self.transport == "stdout":
+            if self.directory is not None:
+                raise ValueError("stdout cannot define a directory")
+            if self.filename is not None:
+                raise ValueError("stdout outputs do not support filenames")
+            if self.format not in {"print", "json-lines", "json"}:
+                raise ValueError(
+                    "stdout output supports 'print', 'json-lines', or 'json' formats"
+                )
+            return self
+
+        if self.format == "print":
+            raise ValueError("fs transport cannot use 'print' format")
+        if self.directory is None:
+            raise ValueError("fs outputs require a directory")
+        return self
+
+    @field_validator("payload", mode="before")
+    @classmethod
+    def _normalize_payload(cls, value):
+        if value is None:
+            return "sample"
+        name = str(value).lower()
+        if name not in {"sample", "vector"}:
+            raise ValueError("payload must be 'sample' or 'vector'")
+        return name
+
+
+class ServeTask(RuntimeTask):
+    kind: Literal["serve"]
+    output: ServeOutputConfig | None = None
+    keep: str | None = Field(
+        default=None,
+        description="Active split label to serve.",
+        min_length=1,
+    )
+    limit: int | None = Field(
+        default=None,
+        description="Default max number of vectors to emit.",
+        ge=1,
+    )
+    stage: int | None = Field(
+        default=None,
+        description="Default pipeline stage preview (0-7).",
+        ge=0,
+        le=7,
+    )
+    throttle_ms: float | None = Field(
+        default=None,
+        description="Milliseconds to sleep between emitted vectors.",
+        ge=0.0,
+    )
+    log_level: str | None = Field(
+        default="INFO",
+        description="Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL).",
+    )
+    visuals: str | None = Field(
+        default="AUTO",
+        description="Visuals provider: AUTO, TQDM, RICH, or OFF.",
+    )
+    progress: str | None = Field(
+        default="AUTO",
+        description="Progress style: AUTO, SPINNER, BARS, or OFF.",
+    )
+
+    @field_validator("log_level")
+    @classmethod
+    def _validate_log_level(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
+        name = str(value).upper()
+        if name not in VALID_LOG_LEVELS:
+            raise ValueError(
+                f"log_level must be one of {', '.join(VALID_LOG_LEVELS)}, got {value!r}"
+            )
+        return name
+
+    @field_validator("visuals", mode="before")
+    @classmethod
+    def _validate_visuals_run(cls, value):
+        if value is None:
+            return None
+        if isinstance(value, bool):
+            return "OFF" if value is False else "AUTO"
+        name = str(value).upper()
+        if name not in VALID_VISUAL_PROVIDERS:
+            raise ValueError(
+                f"visuals must be one of {', '.join(VALID_VISUAL_PROVIDERS)}, got {value!r}"
+            )
+        return name
+
+    @field_validator("progress", mode="before")
+    @classmethod
+    def _validate_progress_run(cls, value):
+        if value is None:
+            return None
+        name = str(value).upper()
+        if name not in VALID_PROGRESS_STYLES:
+            raise ValueError(
+                f"progress must be one of {', '.join(VALID_PROGRESS_STYLES)}, got {value!r}"
+            )
+        return name
+
+    @field_validator("name")
+    @classmethod
+    def _validate_name(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
+        text = str(value).strip()
+        if not text:
+            raise ValueError("task name cannot be empty")
+        return text
+
+
+TaskModel = Annotated[
+    ScalerTask | SchemaTask | MetadataTask | ServeTask,
+    Field(discriminator="kind"),
+]
+
+TASK_ADAPTER = TypeAdapter(TaskModel)
+
+
+def _task_files(root: Path) -> Sequence[Path]:
+    if not root.exists():
+        return []
+    if root.is_file():
+        return [root]
+    return sorted(
+        p for p in root.rglob("*.y*ml") if p.is_file()
+    )
+
+
+def _load_task_docs(path: Path) -> list[TaskBase]:
+    doc = load_yaml(path)
+    if isinstance(doc, list):
+        entries = doc
+    else:
+        entries = [doc]
+    tasks: list[TaskBase] = []
+    for entry in entries:
+        if not isinstance(entry, dict):
+            raise TypeError(f"{path} must define mapping tasks.")
+        task = TASK_ADAPTER.validate_python(entry)
+        task.source_path = path
+        if task.name is None:
+            task.name = path.stem
+        tasks.append(task)
+    return tasks
+
+
+def load_all_tasks(project_yaml: Path) -> list[TaskBase]:
+    root = tasks_dir(project_yaml)
+    tasks: list[TaskBase] = []
+    for path in _task_files(root):
+        tasks.extend(_load_task_docs(path))
+    return tasks
+
+
+def artifact_tasks(project_yaml: Path) -> list[ArtifactTask]:
+    tasks = [
+        task
+        for task in load_all_tasks(project_yaml)
+        if isinstance(task, ArtifactTask)
+    ]
+    kinds = {task.kind for task in tasks}
+    if "schema" not in kinds:
+        tasks.append(SchemaTask(kind="schema"))
+    if "scaler" not in kinds:
+        tasks.append(ScalerTask(kind="scaler"))
+    if "metadata" not in kinds:
+        tasks.append(MetadataTask(kind="metadata"))
+    return tasks
+
+
+def command_tasks(project_yaml: Path, kind: str | None = None) -> list[TaskBase]:
+    tasks = [
+        task
+        for task in load_all_tasks(project_yaml)
+        if not isinstance(task, ArtifactTask)
+    ]
+    if kind is None:
+        return tasks
+    return [task for task in tasks if task.kind == kind]
+
+
+def serve_tasks(project_yaml: Path) -> list[ServeTask]:
+    """Load all serve tasks regardless of enabled state."""
+    return [
+        task
+        for task in command_tasks(project_yaml, kind="serve")
+        if isinstance(task, ServeTask)
+    ]
+
+
+def default_serve_task(project_yaml: Path) -> ServeTask | None:
+    for task in serve_tasks(project_yaml):
+        if task.enabled:
+            return task
+    return None