jerry-thomas 1.0.3__py3-none-any.whl → 2.0.0__py3-none-any.whl

datapipeline/cli/commands/list_.py

@@ -1,7 +1,17 @@
 from pathlib import Path
 
+from datapipeline.config.workspace import WorkspaceContext
+from datapipeline.cli.workspace_utils import resolve_default_project_yaml
 from datapipeline.services.paths import pkg_root, resolve_base_pkg_dir
 from datapipeline.services.bootstrap.core import load_streams
+from datapipeline.services.scaffold.discovery import (
+    list_domains,
+    list_dtos,
+    list_loaders,
+    list_mappers,
+    list_parsers,
+)
+from datapipeline.services.scaffold.utils import error_exit
 
 
 def _default_project_path(root_dir: Path) -> Path | None:
@@ -19,27 +29,34 @@ def _default_project_path(root_dir: Path) -> Path | None:
     return None
 
 
-def handle(subcmd: str) -> None:
+def handle(subcmd: str, *, workspace: WorkspaceContext | None = None) -> None:
     root_dir, name, pyproject = pkg_root(None)
     if subcmd == "sources":
         # Discover sources by scanning sources_dir for YAML files
-        proj_path = _default_project_path(root_dir)
+        proj_path = resolve_default_project_yaml(workspace) if workspace is not None else None
         if proj_path is None:
-            print("[error] No project.yaml found under config/.")
-            return
+            proj_path = _default_project_path(root_dir)
+        if proj_path is None:
+            error_exit("No project.yaml found under config/.")
         try:
             streams = load_streams(proj_path)
         except FileNotFoundError as exc:
-            print(f"[error] {exc}")
-            return
+            error_exit(str(exc))
         aliases = sorted(streams.raw.keys())
         for alias in aliases:
             print(alias)
     elif subcmd == "domains":
-        base = resolve_base_pkg_dir(root_dir, name)
-        dom_dir = base / "domains"
-        if dom_dir.exists():
-            names = sorted(p.name for p in dom_dir.iterdir()
-                           if p.is_dir() and (p / "model.py").exists())
-            for k in names:
-                print(k)
+        for k in list_domains():
+            print(k)
+    elif subcmd == "parsers":
+        for k in sorted(list_parsers().keys()):
+            print(k)
+    elif subcmd == "mappers":
+        for k in sorted(list_mappers().keys()):
+            print(k)
+    elif subcmd == "loaders":
+        for k in sorted(list_loaders().keys()):
+            print(k)
+    elif subcmd == "dtos":
+        for k in sorted(list_dtos().keys()):
+            print(k)

datapipeline/cli/commands/loader.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+
+from datapipeline.services.scaffold.loader import create_loader
+from datapipeline.services.scaffold.utils import choose_name, status
+
+
+def handle(name: str | None, *, plugin_root: Path | None = None) -> None:
+    if not name:
+        name = choose_name("Loader name", default="custom_loader")
+    ep = create_loader(name=name, root=plugin_root)
+    status("ok", f"Registered loader entry point '{ep}'.")

datapipeline/cli/commands/mapper.py

@@ -0,0 +1,82 @@
+from pathlib import Path
+
+from datapipeline.services.scaffold.discovery import list_domains, list_dtos
+from datapipeline.services.scaffold.dto import create_dto
+from datapipeline.services.scaffold.domain import create_domain
+from datapipeline.services.scaffold.mapper import create_mapper
+from datapipeline.services.scaffold.utils import (
+    choose_existing_or_create,
+    choose_name,
+    error_exit,
+    info,
+    status,
+    pick_from_menu,
+    pick_from_list,
+)
+from datapipeline.services.scaffold.layout import (
+    default_mapper_name,
+    LABEL_DTO_FOR_MAPPER,
+    LABEL_DOMAIN_TO_MAP,
+    LABEL_MAPPER_INPUT,
+    default_mapper_name_for_identity,
+)
+
+
+def handle(name: str | None, *, plugin_root: Path | None = None) -> str:
+    input_class = None
+    input_module = None
+
+    input_choice = pick_from_menu(
+        f"{LABEL_MAPPER_INPUT}:",
+        [
+            ("dto", "DTO (default)"),
+            ("identity", "Any"),
+        ],
+    )
+    info("Mapper output (select domain):")
+
+    dto_map = list_dtos(root=plugin_root)
+    if input_choice == "dto":
+        dto_class = choose_existing_or_create(
+            label=LABEL_DTO_FOR_MAPPER,
+            existing=sorted(dto_map.keys()),
+            create_label="Create new DTO",
+            create_fn=create_dto,
+            prompt_new="DTO class name",
+            root=plugin_root,
+        )
+        dto_module = list_dtos(root=plugin_root).get(dto_class)
+        if not dto_module:
+            error_exit("Failed to resolve DTO module.")
+        input_class = dto_class
+        input_module = dto_module
+    else:
+        input_module = "typing"
+        input_class = "Any"
+
+    domains = list_domains(root=plugin_root)
+    domain = choose_existing_or_create(
+        label=LABEL_DOMAIN_TO_MAP,
+        existing=domains,
+        create_label="Create new domain",
+        create_fn=lambda name, root: create_domain(domain=name, root=root),
+        prompt_new="Domain name",
+        root=plugin_root,
+    )
+
+    if not name:
+        if input_choice == "identity":
+            name = choose_name(
+                "Mapper name", default=default_mapper_name_for_identity(domain))
+        else:
+            name = choose_name(
+                "Mapper name", default=default_mapper_name(input_module, domain))
+
+    ep = create_mapper(
+        name=name,
+        input_class=input_class,
+        input_module=input_module,
+        domain=domain,
+        root=plugin_root,
+    )
+    return ep

datapipeline/cli/commands/parser.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+
+from datapipeline.services.scaffold.dto import create_dto
+from datapipeline.services.scaffold.discovery import list_dtos
+from datapipeline.services.scaffold.parser import create_parser
+from datapipeline.services.scaffold.utils import (
+    choose_existing_or_create,
+    choose_name,
+    error_exit,
+    status,
+)
+from datapipeline.services.scaffold.layout import default_parser_name, LABEL_DTO_FOR_PARSER
+
+
+def handle(
+    name: str | None,
+    *,
+    plugin_root: Path | None = None,
+    default_dto: str | None = None,
+) -> str:
+    dto_map = list_dtos(root=plugin_root)
+    dto_class = choose_existing_or_create(
+        label=LABEL_DTO_FOR_PARSER,
+        existing=sorted(dto_map.keys()),
+        create_label="Create new DTO",
+        create_fn=create_dto,
+        prompt_new="DTO class name",
+        root=plugin_root,
+        default_new=default_dto or (f"{name}DTO" if name else None),
+    )
+    dto_module = list_dtos(root=plugin_root).get(dto_class)
+    if not dto_module:
+        error_exit("Failed to resolve DTO module.")
+
+    if not name:
+        name = choose_name("Parser class name", default=default_parser_name(dto_class))
+
+    ep = create_parser(
+        name=name,
+        dto_class=dto_class,
+        dto_module=dto_module,
+        root=plugin_root,
+    )
+    status("ok", f"Registered parser entry point '{ep}'.")
+    return ep

datapipeline/cli/commands/run_config.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import logging
 from pathlib import Path
 from typing import Iterator, List, NamedTuple, Optional, Sequence
@@ -93,7 +91,7 @@ def determine_preview_stage(
             return None, None
         stages.append(cfg_stage)
 
-    if not stages or any(stage > 5 for stage in stages):
+    if not stages or any(stage > 6 for stage in stages):
         return None, None
 
     if len(set(stages)) == 1:

datapipeline/cli/commands/serve_pipeline.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import logging
 import time
 from itertools import islice
@@ -70,7 +68,7 @@ def report_serve(target: OutputTarget, count: int) -> None:
 
 
 def _is_full_pipeline_stage(stage: int | None) -> bool:
-    return stage is None or stage >= 6
+    return stage is None or stage >= 7
 
 
 def serve_with_runtime(
@@ -99,12 +97,12 @@ def serve_with_runtime(
             run_status = "success"
             return
 
-        rectangular = stage is None or stage > 5
+        rectangular = stage is None or stage > 6
 
-        if stage is not None and stage <= 5:
+        if stage is not None and stage <= 6:
             if target.payload != "sample":
                 logger.warning(
-                    "Ignoring payload '%s' for stage %s preview; preview outputs stream raw records.",
+                    "Ignoring payload '%s' for stage %s preview; preview outputs record/feature streams.",
                     target.payload,
                     stage,
                 )
@@ -129,7 +127,7 @@ def serve_with_runtime(
             rectangular=rectangular,
         )
 
-        if stage in (None, 7):
+        if stage in (None, 8):
             vectors = post_process(context, vectors)
         if stage is None:
             vectors = apply_split_stage(runtime, vectors)

datapipeline/cli/commands/source.py

@@ -2,7 +2,19 @@ from pathlib import Path
 
 from datapipeline.config.workspace import WorkspaceContext
 from datapipeline.cli.workspace_utils import resolve_default_project_yaml
-from datapipeline.services.scaffold.source import create_source
+from datapipeline.services.scaffold.source_yaml import (
+    create_source_yaml,
+    default_loader_config,
+)
+from datapipeline.services.scaffold.discovery import list_loaders, list_parsers
+from datapipeline.services.scaffold.utils import (
+    error_exit,
+    info,
+    choose_name,
+    pick_from_menu,
+    prompt_required,
+)
+import sys
 
 
 def handle(
@@ -13,11 +25,13 @@ def handle(
     format: str | None = None,
     *,
     identity: bool = False,
+    loader: str | None = None,
+    parser: str | None = None,
     alias: str | None = None,
     plugin_root: Path | None = None,
     workspace: WorkspaceContext | None = None,
 ) -> None:
-    if subcmd in {"create", "add"}:
+    if subcmd == "create":
         # Allow: positional provider dataset, --provider/--dataset, --alias, or provider as 'prov.ds'
         if (not provider or not dataset):
             # Try alias flag first
@@ -26,33 +40,107 @@ def handle(
                 if len(parts) == 2 and all(parts):
                     provider, dataset = parts[0], parts[1]
                 else:
-                    print("[error] Alias must be 'provider.dataset'")
-                    raise SystemExit(2)
+                    error_exit("Alias must be 'provider.dataset'")
             # Try provider passed as 'prov.ds' positional/flag
             elif provider and ("." in provider) and not dataset:
                 parts = provider.split(".", 1)
                 if len(parts) == 2 and all(parts):
                     provider, dataset = parts[0], parts[1]
                 else:
-                    print("[error] Source must be specified as '<provider> <dataset>' or '<provider>.<dataset>'")
-                    raise SystemExit(2)
+                    error_exit("Source must be specified as '<provider> <dataset>' or '<provider>.<dataset>'")
 
         if not provider or not dataset:
-            print("[error] Source requires '<provider> <dataset>' (or -a/--alias provider.dataset)")
-            raise SystemExit(2)
-        if not transport:
-            print("[error] --transport is required (fs|http|synthetic)")
-            raise SystemExit(2)
-        if transport in {"fs", "http"} and not format:
-            print("[error] --format is required for fs/http transports (fs: csv|json|json-lines|pickle, http: csv|json|json-lines)")
-            raise SystemExit(2)
+            source_id = prompt_required("Source id (provider.dataset)")
+            parts = source_id.split(".", 1)
+            if len(parts) == 2 and all(parts):
+                provider, dataset = parts[0], parts[1]
+            else:
+                error_exit("Source id must be in the form 'provider.dataset'")
+
+        # Loader selection: either explicit loader EP or built-in transport defaults
+        loader_ep: str | None = loader
+        loader_args: dict = {}
+        if not loader_ep:
+            if not transport:
+                known_loaders = list_loaders(root=plugin_root)
+                options = [
+                    ("fs", "Built-in fs"),
+                    ("http", "Built-in http"),
+                    ("synthetic", "Built-in synthetic"),
+                ]
+                if known_loaders:
+                    options.append(("existing", "Select existing loader"))
+                options.append(("custom", "Custom loader"))
+                choice = pick_from_menu("Loader:", options)
+                if choice in {"fs", "http", "synthetic"}:
+                    transport = choice
+                elif choice == "existing":
+                    loader_ep = pick_from_menu(
+                        "Select loader entrypoint:",
+                        [(k, k) for k in sorted(known_loaders.keys())],
+                    )
+                elif choice == "custom":
+                    loader_ep = prompt_required("Loader entrypoint")
+            if not loader_ep:
+                if transport in {"fs", "http"} and not format:
+                    format_options = [
+                        ("csv", "csv"),
+                        ("json", "json"),
+                        ("json-lines", "json-lines"),
+                    ]
+                    if transport == "fs":
+                        format_options.append(("pickle", "pickle"))
+                    format = pick_from_menu("Format:", format_options)
+                if not transport:
+                    error_exit("--transport is required when no --loader is provided")
+                loader_ep, loader_args = default_loader_config(transport, format)
+
+        # Parser selection (no code generation)
+        if identity:
+            parser_ep = "identity"
+        elif parser:
+            parser_ep = parser
+        else:
+            interactive = sys.stdin.isatty()
+            if not interactive:
+                parser_ep = "identity"
+            else:
+                parsers = list_parsers(root=plugin_root)
+                if parsers:
+                    choice = pick_from_menu(
+                        "Parser:",
+                        [
+                            ("existing", "Select existing parser (default)"),
+                            ("identity", "Identity parser"),
+                            ("custom", "Custom parser"),
+                        ],
+                    )
+                    if choice == "existing":
+                        parser_ep = pick_from_menu(
+                            "Select parser entrypoint:",
+                            [(k, k) for k in sorted(parsers.keys())],
+                        )
+                    elif choice == "identity":
+                        parser_ep = "identity"
+                    else:
+                        parser_ep = prompt_required("Parser entrypoint")
+                else:
+                    choice = pick_from_menu(
+                        "Parser:",
+                        [
+                            ("identity", "Identity parser (default)"),
+                            ("custom", "Custom parser"),
+                        ],
+                    )
+                    parser_ep = "identity" if choice == "identity" else prompt_required("Parser entrypoint")
+
         project_yaml = resolve_default_project_yaml(workspace)
-        create_source(
+        create_source_yaml(
             provider=provider,
             dataset=dataset,
-            transport=transport,
-            format=format,
+            loader_ep=loader_ep,
+            loader_args=loader_args,
+            parser_ep=parser_ep,
             root=plugin_root,
-            identity=identity,
             **({"project_yaml": project_yaml} if project_yaml is not None else {}),
         )