jerry-thomas 0.0.2__py3-none-any.whl

datapipeline/analysis/vector_analyzer.py

@@ -0,0 +1,49 @@
+from collections import defaultdict, Counter
+from typing import Any, Hashable, Iterable
+
+import numpy as np
+
+
+class VectorStatsCollector:
+    def __init__(self, expected_feature_ids: Iterable[str]):
+        self.expected_feature_ids = set(expected_feature_ids)
+        self.missing_features = Counter()
+        self.empty_vectors = 0
+        self.total_vectors = 0
+        self.per_group_missing = defaultdict(set)
+
+    def update(self, group_key: Hashable, feature_vector: dict[str, Any]):
+        self.total_vectors += 1
+
+        present_features = set(feature_vector.keys())
+
+        if not present_features:
+            self.empty_vectors += 1
+
+        missing = self.expected_feature_ids - present_features
+        for feature_id in missing:
+            self.missing_features[feature_id] += 1
+            self.per_group_missing[group_key].add(feature_id)
+
+        # 🔍 Check for features present but with missing/invalid values
+        for fid in present_features & self.expected_feature_ids:
+            val = feature_vector[fid]
+            if val is None or (isinstance(val, float) and np.isnan(val)):
+                self.missing_features[fid] += 1
+                self.per_group_missing[group_key].add(fid)
+
+    def print_report(self):
+        print("\n=== Vector Quality Report ===")
+        print(f"Total vectors processed: {self.total_vectors}")
+        print(f"Empty vectors: {self.empty_vectors}")
+        print(
+            f"Features expected: {sorted(self.expected_feature_ids)[:10] }... (total {len(self.expected_feature_ids)})")
+
+        print("\n→ Missing feature counts:")
+        for fid, count in sorted(self.missing_features.items(), key=lambda x: -x[1]):
+            print(f"  - {fid}: missing in {count} vectors")
+
+        if self.per_group_missing:
+            print("\n→ Groups with missing features (sample):")
+            for group_key, missing in list(self.per_group_missing.items())[:5]:
+                print(f"  - Group {group_key}: {sorted(missing)}")

datapipeline/cli/app.py

@@ -0,0 +1,208 @@
+import argparse
+
+from datapipeline.cli.commands.run import handle_prep, handle_serve
+from datapipeline.cli.commands.analyze import analyze as handle_analyze
+from datapipeline.cli.commands.plugin import station as handle_station
+from datapipeline.cli.commands.source import handle as handle_source
+from datapipeline.cli.commands.domain import handle as handle_domain
+from datapipeline.cli.commands.link import handle as handle_link
+from datapipeline.cli.commands.list_ import handle as handle_list
+from datapipeline.cli.commands.filter import handle as handle_filter
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="jerry",
+        description="Mixology-themed CLI for building and serving data pipelines.",
+    )
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    # prep (debug mode with visuals)
+    p_prep = sub.add_parser(
+        "prep",
+        help="run pipeline stages with visual progress",
+    )
+    prep_sub = p_prep.add_subparsers(dest="prep_cmd", required=True)
+    prep_steps = {
+        "pour": "preview record-stage output",
+        "build": "inspect feature-stage output",
+        "stir": "examine vector-stage output",
+    }
+    for step, help_text in prep_steps.items():
+        sp = prep_sub.add_parser(step, help=help_text)
+        sp.add_argument(
+            "--project", "-p", default="config/project.yaml", help="path to project.yaml"
+        )
+        sp.add_argument("--limit", "-n", type=int, default=20)
+
+    # serve (production run, no visuals)
+    p_serve = sub.add_parser(
+        "serve",
+        help="produce vectors without progress visuals",
+    )
+    p_serve.add_argument(
+        "--project", "-p", default="config/project.yaml", help="path to project.yaml"
+    )
+    p_serve.add_argument(
+        "--limit", "-n", type=int, default=None,
+        help="optional cap on the number of vectors to emit",
+    )
+    p_serve.add_argument(
+        "--output", "-o", default="print",
+        help="output destination: 'print', 'stream', or a file ending in .pt",
+    )
+
+    # taste (analysis)
+    p_taste = sub.add_parser(
+        "taste",
+        help="analyze vector completeness and feature stats",
+    )
+    p_taste.add_argument(
+        "--project", "-p", default="config/project.yaml", help="path to project.yaml"
+    )
+    p_taste.add_argument("--limit", "-n", type=int, default=None)
+
+    # distillery (sources)
+    p_dist = sub.add_parser(
+        "distillery",
+        help="add or list raw sources",
+    )
+    dist_sub = p_dist.add_subparsers(dest="dist_cmd", required=True)
+    p_dist_add = dist_sub.add_parser(
+        "add",
+        help="create a provider+dataset source",
+        description=(
+            "Scaffold a source using transport + format.\n\n"
+            "Examples:\n"
+            "  fs CSV:        -t fs  -f csv\n"
+            "  fs NDJSON:     -t fs  -f json-lines\n"
+            "  URL JSON:      -t url -f json\n"
+            "  Synthetic:     -t synthetic\n\n"
+            "Note: set 'glob: true' in the generated YAML if your 'path' contains wildcards."
+        ),
+    )
+    p_dist_add.add_argument("--provider", "-p", required=True)
+    p_dist_add.add_argument("--dataset", "-d", required=True)
+    p_dist_add.add_argument(
+        "--transport", "-t",
+        choices=["fs", "url", "synthetic"],
+        required=True,
+        help="how data is accessed: fs/url/synthetic",
+    )
+    p_dist_add.add_argument(
+        "--format", "-f",
+        choices=["csv", "json", "json-lines"],
+        help="data format for fs/url transports (ignored otherwise)",
+    )
+    dist_sub.add_parser("list", help="list known sources")
+
+    # spirit (domains)
+    p_spirit = sub.add_parser(
+        "spirit",
+        help="add or list domains",
+    )
+    spirit_sub = p_spirit.add_subparsers(dest="spirit_cmd", required=True)
+    p_spirit_add = spirit_sub.add_parser(
+        "add",
+        help="create a domain",
+        description=(
+            "Create a domain package. Defaults to Record base. "
+            "Use --time-aware to base on TimeFeatureRecord (adds 'time' and 'value' fields)."
+        ),
+    )
+    p_spirit_add.add_argument("--domain", "-d", required=True)
+    p_spirit_add.add_argument(
+        "--time-aware",
+        "-t",
+        action="store_true",
+        help="use TimeFeatureRecord base (UTC-aware 'time' + 'value' fields) instead of Record",
+    )
+    spirit_sub.add_parser("list", help="list known domains")
+
+    # contract (link source ↔ domain)
+    p_contract = sub.add_parser(
+        "contract",
+        help="link a distillery source to a spirit domain",
+    )
+    p_contract.add_argument("--time-aware", "-t", action="store_true")
+
+    # station (plugin scaffolding)
+    p_station = sub.add_parser(
+        "station",
+        help="scaffold plugin workspaces",
+    )
+    station_sub = p_station.add_subparsers(dest="station_cmd", required=True)
+    p_station_init = station_sub.add_parser(
+        "init", help="create a plugin skeleton")
+    p_station_init.add_argument("--name", "-n", required=True)
+    p_station_init.add_argument("--out", "-o", default=".")
+
+    # filter (unchanged helper)
+    p_filt = sub.add_parser("filter", help="manage filters")
+    filt_sub = p_filt.add_subparsers(dest="filter_cmd", required=True)
+    p_filt_create = filt_sub.add_parser(
+        "create", help="create a filter function")
+    p_filt_create.add_argument(
+        "--name", "-n", required=True,
+        help="filter entrypoint name and function/module name",
+    )
+
+    args = parser.parse_args()
+
+    if args.cmd == "prep":
+        handle_prep(action=args.prep_cmd,
+                    project=args.project, limit=args.limit)
+        return
+
+    if args.cmd == "serve":
+        handle_serve(
+            project=args.project,
+            limit=getattr(args, "limit", None),
+            output=args.output,
+        )
+        return
+
+    if args.cmd == "taste":
+        handle_analyze(project=args.project,
+                       limit=getattr(args, "limit", None))
+        return
+
+    if args.cmd == "distillery":
+        if args.dist_cmd == "list":
+            handle_list(subcmd="sources")
+        else:
+            handle_source(
+                subcmd="add",
+                provider=getattr(args, "provider", None),
+                dataset=getattr(args, "dataset", None),
+                transport=getattr(args, "transport", None),
+                format=getattr(args, "format", None),
+            )
+        return
+
+    if args.cmd == "spirit":
+        if args.spirit_cmd == "list":
+            handle_list(subcmd="domains")
+        else:
+            handle_domain(
+                subcmd="add",
+                domain=getattr(args, "domain", None),
+                time_aware=getattr(args, "time_aware", False),
+            )
+        return
+
+    if args.cmd == "contract":
+        handle_link(time_aware=getattr(args, "time_aware", False))
+        return
+
+    if args.cmd == "station":
+        handle_station(
+            subcmd=args.station_cmd,
+            name=getattr(args, "name", None),
+            out=getattr(args, "out", "."),
+        )
+        return
+
+    if args.cmd == "filter":
+        handle_filter(subcmd=args.filter_cmd, name=getattr(args, "name", None))
+        return

datapipeline/cli/commands/analyze.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+from typing import Optional
+
+from datapipeline.services.bootstrap import bootstrap
+from datapipeline.config.dataset.loader import load_dataset
+from datapipeline.pipeline.pipelines import build_vector_pipeline
+from datapipeline.cli.openers import open_canonical_stream_visual
+from datapipeline.analysis.vector_analyzer import VectorStatsCollector
+
+
+def analyze(project: str, limit: Optional[int] = None) -> None:
+    project_path = Path(project)
+    dataset = load_dataset(project_path, "vectors")
+    bootstrap(project_path)
+
+    expected_feature_ids = [cfg.feature_id for cfg in (dataset.features or [])]
+    if not expected_feature_ids:
+        print("(no features configured; nothing to analyze)")
+        return
+
+    collector = VectorStatsCollector(expected_feature_ids)
+
+    count = 0
+    for group_key, vector in build_vector_pipeline(
+        dataset.features, dataset.group_by, open_canonical_stream_visual
+    ):
+        collector.update(group_key, vector.values)
+        count += 1
+        if limit and count >= limit:
+            break
+
+    collector.print_report()

datapipeline/cli/commands/domain.py

@@ -0,0 +1,9 @@
+from datapipeline.services.scaffold.domain import create_domain
+
+
+def handle(subcmd: str, domain: str | None, time_aware: bool) -> None:
+    if subcmd in {"create", "add"}:
+        if not domain:
+            print("❗ --domain is required")
+            raise SystemExit(2)
+        create_domain(domain=domain, time_aware=time_aware, root=None)

datapipeline/cli/commands/filter.py

@@ -0,0 +1,10 @@
+from datapipeline.services.scaffold.filter import create_filter
+
+
+def handle(subcmd: str, name: str | None) -> None:
+    if subcmd == "create":
+        if not name:
+            print("❗ --name is required for filter create")
+            raise SystemExit(2)
+        create_filter(name=name, root=None)
+

datapipeline/cli/commands/link.py

@@ -0,0 +1,95 @@
+import sys
+from datapipeline.services.paths import pkg_root, resolve_base_pkg_dir
+from datapipeline.services.entrypoints import read_group_entries
+import yaml
+from datapipeline.services.constants import FILTERS_GROUP, MAPPER_KEY, ENTRYPOINT_KEY, ARGS_KEY, SOURCE_KEY
+from datapipeline.services.project_paths import sources_dir as resolve_sources_dir, streams_dir as resolve_streams_dir
+from datapipeline.services.scaffold.mappers import attach_source_to_domain
+import re
+
+
+def _pick_from_list(prompt: str, options: list[str]) -> str:
+    print(prompt, file=sys.stderr)
+    for i, opt in enumerate(options, 1):
+        print(f"  [{i}] {opt}", file=sys.stderr)
+    while True:
+        sel = input("> ").strip()
+        if sel.isdigit():
+            idx = int(sel)
+            if 1 <= idx <= len(options):
+                return options[idx - 1]
+        print("Please enter a number from the list.", file=sys.stderr)
+
+
+def handle(time_aware: bool) -> None:
+    root_dir, name, pyproject = pkg_root(None)
+
+    # Discover sources by scanning sources_dir YAMLs
+    proj_path = root_dir / "config" / "project.yaml"
+    sources_dir = resolve_sources_dir(proj_path)
+    source_options = []
+    if sources_dir.exists():
+        source_options = sorted(p.stem for p in sources_dir.glob("*.y*ml"))
+    if not source_options:
+        print("❗ No sources found. Create one first (jerry distillery add ...)")
+        raise SystemExit(2)
+
+    src_key = _pick_from_list("Select a source to link:", source_options)
+    # Expect aliases from sources_dir filenames: provider_dataset.yaml
+    parts = src_key.split("_", 1)
+    if len(parts) != 2:
+        print("❗ Source alias must be 'provider_dataset' (from sources/<alias>.yaml)", file=sys.stderr)
+        raise SystemExit(2)
+    provider, dataset = parts[0], parts[1]
+
+    # Discover domains by scanning the package, fallback to EPs if needed
+    base = resolve_base_pkg_dir(root_dir, name)
+    domain_options = []
+    for dirname in ("domains",):
+        dom_dir = base / dirname
+        if dom_dir.exists():
+            domain_options.extend(
+                [p.name for p in dom_dir.iterdir() if p.is_dir()
+                 and (p / "model.py").exists()]
+            )
+    domain_options = sorted(set(domain_options))
+    if not domain_options:
+        domain_options = sorted(
+            read_group_entries(pyproject, FILTERS_GROUP).keys())
+    if not domain_options:
+        print("❗ No domains found. Create one first (jerry spirit add ...)")
+        raise SystemExit(2)
+
+    dom_name = _pick_from_list("Select a domain to link to:", domain_options)
+
+    # create mapper + EP (domain.origin)
+    attach_source_to_domain(domain=dom_name, provider=provider,
+                            dataset=dataset, time_aware=time_aware, root=None)
+
+    def _slug(s: str) -> str:
+        s = s.strip().lower()
+        s = re.sub(r"[^a-z0-9]+", "_", s)
+        return s.strip("_")
+    ep_key = f"{_slug(dom_name)}.{_slug(provider)}"
+    print(f"✅ Registered mapper entry point as '{ep_key}'.")
+
+    # Inject per-file canonical stream into streams directory
+    streams_path = resolve_streams_dir(proj_path)
+
+    canonical_alias = src_key  # default canonical stream alias
+    mapper_ep = ep_key
+    # Write a single-file canonical spec into streams directory
+    try:
+        # Ensure streams_path is a directory path
+        streams_dir = streams_path if streams_path.is_dir() else streams_path.parent
+        streams_dir.mkdir(parents=True, exist_ok=True)
+        cfile = streams_dir / f"{canonical_alias}.yaml"
+        data = {
+            SOURCE_KEY: src_key,
+            MAPPER_KEY: {ENTRYPOINT_KEY: mapper_ep, ARGS_KEY: {}},
+        }
+        with cfile.open("w", encoding="utf-8") as f:
+            yaml.safe_dump(data, f, sort_keys=False, default_flow_style=False)
+        print(f"✨ Created canonical spec: {cfile}")
+    except Exception as e:
+        print(f"❗ Failed to write canonical spec: {e}", file=sys.stderr)

datapipeline/cli/commands/list_.py

@@ -0,0 +1,22 @@
+from datapipeline.services.paths import pkg_root, resolve_base_pkg_dir
+from datapipeline.services.project_paths import sources_dir as sources_dir_from_project
+
+
+def handle(subcmd: str) -> None:
+    root_dir, name, pyproject = pkg_root(None)
+    if subcmd == "sources":
+        # Discover sources by scanning sources_dir for YAML files
+        proj_path = root_dir / "config" / "project.yaml"
+        sources_dir = sources_dir_from_project(proj_path)
+        if sources_dir.exists():
+            aliases = sorted(p.stem for p in sources_dir.glob("*.y*ml"))
+            for a in aliases:
+                print(a)
+    elif subcmd == "domains":
+        base = resolve_base_pkg_dir(root_dir, name)
+        dom_dir = base / "domains"
+        if dom_dir.exists():
+            names = sorted(p.name for p in dom_dir.iterdir()
+                           if p.is_dir() and (p / "model.py").exists())
+            for k in names:
+                print(k)

datapipeline/cli/commands/plugin.py

@@ -0,0 +1,10 @@
+from pathlib import Path
+from datapipeline.services.scaffold.plugin import scaffold_plugin
+
+
+def station(subcmd: str, name: str | None, out: str) -> None:
+    if subcmd == "init":
+        if not name:
+            print("❗ --name is required for station init")
+            raise SystemExit(2)
+        scaffold_plugin(name, Path(out))

datapipeline/cli/commands/run.py

@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import json
+import pickle
+import sys
+from itertools import islice
+from pathlib import Path
+from typing import Iterator, Optional, Tuple
+
+from tqdm import tqdm
+
+from datapipeline.cli.openers import open_canonical_stream_visual
+from datapipeline.config.dataset.dataset import FeatureDatasetConfig, RecordDatasetConfig
+from datapipeline.config.dataset.loader import load_dataset
+from datapipeline.pipeline.pipelines import (
+    build_feature_pipeline,
+    build_record_pipeline,
+    build_vector_pipeline,
+)
+from datapipeline.services.bootstrap import bootstrap
+from datapipeline.streams.canonical import open_canonical_stream
+from datapipeline.domain.vector import Vector
+
+
+def _print_head(iterable: Iterator[object], limit: int) -> int:
+    count = 0
+    try:
+        for item in iterable:
+            tqdm.write(str(item))
+            count += 1
+            if count >= limit:
+                break
+    except KeyboardInterrupt:
+        pass
+    return count
+
+
+def _run_records(dataset: RecordDatasetConfig, limit: int) -> None:
+    for cfg in dataset.features:
+        print(f"\n🍷 pouring records for {cfg.feature_id}")
+        records = build_record_pipeline(cfg, open_canonical_stream_visual)
+        printed = _print_head(records, limit)
+        print(f"(poured {printed} records)")
+
+
+def _run_features(dataset: FeatureDatasetConfig, limit: int) -> None:
+    group_by = dataset.group_by
+    for cfg in dataset.features:
+        feature_id = getattr(cfg, "feature_id", "?")
+        print(f"\n🛠️ building features for {feature_id}")
+        features = build_feature_pipeline(
+            cfg, group_by, open_canonical_stream_visual)
+        printed = _print_head(features, limit)
+        tqdm.write(f"(built {printed} feature records)")
+
+
+def _run_vectors(dataset: FeatureDatasetConfig, limit: int) -> None:
+    print("\n🥄 stirring vectors")
+    vectors = build_vector_pipeline(
+        dataset.features, dataset.group_by, open_canonical_stream_visual)
+    printed = _print_head(vectors, limit)
+    print(f"(stirred {printed} vectors)")
+
+
+def handle_prep(action: str, project: str, limit: int = 20) -> None:
+    stage_lookup = {"pour": "records", "build": "features", "stir": "vectors"}
+    if action not in stage_lookup:
+        raise ValueError(f"Unknown prep action: {action}")
+
+    project_path = Path(project)
+    dataset = load_dataset(project_path, stage_lookup[action])
+    bootstrap(project_path)
+
+    features = list(dataset.features or [])
+    if not features:
+        print("(no features configured; nothing to prep)")
+        return
+
+    if action == "pour":
+        _run_records(dataset, limit)
+    elif action == "build":
+        _run_features(dataset, limit)
+    else:
+        _run_vectors(dataset, limit)
+
+
+def _limit_vectors(vectors: Iterator[Tuple[object, Vector]], limit: Optional[int]) -> Iterator[Tuple[object, Vector]]:
+    if limit is None:
+        yield from vectors
+    else:
+        yield from islice(vectors, limit)
+
+
+def _serve_print(vectors: Iterator[Tuple[object, Vector]], limit: Optional[int]) -> None:
+    count = 0
+    try:
+        for group_key, vector in _limit_vectors(vectors, limit):
+            print(f"group={group_key}: {vector.values}")
+            count += 1
+    except KeyboardInterrupt:
+        pass
+    print(f"(served {count} vectors to stdout)")
+
+
+def _serve_stream(vectors: Iterator[Tuple[object, Vector]], limit: Optional[int]) -> None:
+    count = 0
+    try:
+        for group_key, vector in _limit_vectors(vectors, limit):
+            payload = {"group": list(group_key) if isinstance(group_key, tuple) else group_key,
+                       "values": vector.values}
+            print(json.dumps(payload, default=str))
+            count += 1
+    except KeyboardInterrupt:
+        pass
+    print(f"(streamed {count} vectors)", file=sys.stderr)
+
+
+def _serve_pt(vectors: Iterator[Tuple[object, Vector]], limit: Optional[int], destination: Path) -> None:
+    data = []
+    for group_key, vector in _limit_vectors(vectors, limit):
+        normalized_key = list(group_key) if isinstance(
+            group_key, tuple) else group_key
+        data.append((normalized_key, vector.values))
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    with destination.open("wb") as fh:
+        pickle.dump(data, fh)
+    print(f"💾 Saved {len(data)} vectors to {destination}")
+
+
+def handle_serve(project: str, limit: Optional[int], output: str) -> None:
+    project_path = Path(project)
+    dataset = load_dataset(project_path, "vectors")
+    bootstrap(project_path)
+
+    features = list(dataset.features or [])
+    if not features:
+        print("(no features configured; nothing to serve)")
+        return
+
+    vectors = build_vector_pipeline(
+        dataset.features, dataset.group_by, open_canonical_stream)
+
+    if output == "print":
+        _serve_print(vectors, limit)
+    elif output == "stream":
+        _serve_stream(vectors, limit)
+    elif output.endswith(".pt"):
+        _serve_pt(vectors, limit, Path(output))
+    else:
+        print("❗ Unsupported output format. Use 'print', 'stream', or a .pt file path.")
+        raise SystemExit(2)

datapipeline/cli/commands/source.py

@@ -0,0 +1,17 @@
+from datapipeline.services.scaffold.source import create_source
+
+
+def handle(subcmd: str, provider: str | None, dataset: str | None,
+           transport: str | None = None, format: str | None = None) -> None:
+    if subcmd in {"create", "add"}:
+        if not provider or not dataset:
+            print("❗ --provider and --dataset are required")
+            raise SystemExit(2)
+        if not transport:
+            print("❗ --transport is required (fs|url|synthetic)")
+            raise SystemExit(2)
+        if transport in {"fs", "url"} and not format:
+            print("❗ --format is required for fs/url transports (csv|json|json-lines)")
+            raise SystemExit(2)
+        create_source(provider=provider, dataset=dataset,
+                      transport=transport, format=format, root=None)

datapipeline/cli/openers.py

@@ -0,0 +1,11 @@
+from typing import Iterator, Any
+from datapipeline.streams.canonical import canonical_entry
+from datapipeline.streams.raw import open_raw_stream
+from datapipeline.cli.visuals import wrap_with_tqdm
+
+
+def open_canonical_stream_visual(alias: str, show: bool = True) -> Iterator[Any]:
+    entry = canonical_entry(alias)
+    raw = open_raw_stream(entry.source_alias)
+    wrapped = wrap_with_tqdm(raw, stream_alias=alias, show=show)
+    return entry.mapper(wrapped, **entry.mapper_args)