jerry-thomas 1.0.3__py3-none-any.whl → 2.0.1__py3-none-any.whl

datapipeline/services/scaffold/mapper.py

@@ -0,0 +1,116 @@
+from pathlib import Path
+from typing import Optional
+
+from datapipeline.services.paths import pkg_root, resolve_base_pkg_dir
+from datapipeline.services.scaffold.templates import render
+from datapipeline.services.scaffold.utils import (
+    ensure_pkg_dir,
+    ep_key_from_name,
+    info,
+    status,
+    to_snake,
+    validate_identifier,
+    write_if_missing,
+)
+from datapipeline.services.scaffold.layout import (
+    DIR_MAPPERS,
+    TPL_MAPPER_COMPOSED,
+    TPL_MAPPER_INGEST,
+    domain_record_class,
+    entrypoint_target,
+    pyproject_path,
+)
+from datapipeline.services.entrypoints import inject_ep
+from datapipeline.services.constants import MAPPERS_GROUP
+
+
+def create_mapper(
+    *,
+    name: str,
+    dto_class: str | None = None,
+    dto_module: str | None = None,
+    input_class: str | None = None,
+    input_module: str | None = None,
+    domain: str,
+    root: Optional[Path],
+) -> str:
+    validate_identifier(name, "Mapper name")
+
+    root_dir, pkg_name, _ = pkg_root(root)
+    base = resolve_base_pkg_dir(root_dir, pkg_name)
+    package_name = base.name
+
+    mappers_dir = ensure_pkg_dir(base, DIR_MAPPERS)
+    module_name = to_snake(name)
+    path = mappers_dir / f"{module_name}.py"
+
+    domain_module = f"{package_name}.domains.{domain}.model"
+    domain_record = domain_record_class(domain)
+
+    resolved_class = input_class or dto_class
+    resolved_module = input_module or dto_module
+    if not resolved_class or not resolved_module:
+        raise ValueError("Mapper input class/module is required")
+
+    write_if_missing(
+        path,
+        render(
+            TPL_MAPPER_INGEST,
+            FUNCTION_NAME=module_name,
+            INPUT_CLASS=resolved_class,
+            INPUT_IMPORT=resolved_module,
+            DOMAIN_MODULE=domain_module,
+            DOMAIN_RECORD=domain_record,
+        ),
+        label="Mapper",
+    )
+
+    ep_key = ep_key_from_name(name)
+    pyproject = pyproject_path(root_dir)
+    try:
+        toml_text = pyproject.read_text()
+        updated = inject_ep(
+            toml_text,
+            MAPPERS_GROUP,
+            ep_key,
+            entrypoint_target(package_name, "mappers", module_name, module_name),
+        )
+        if updated != toml_text:
+            pyproject.write_text(updated)
+            status("ok", f"Registered mapper entry point '{ep_key}'.")
+        else:
+            status("skip", f"Mapper entry point already registered: '{ep_key}'.")
+    except FileNotFoundError:
+        info("pyproject.toml not found; skipping entry point registration")
+    return ep_key
+
+
+def create_composed_mapper(
+    *,
+    domain: str,
+    stream_id: str,
+    root: Optional[Path],
+    mapper_path: str | None = None,
+) -> str:
+    root_dir, name, _ = pkg_root(root)
+    base = resolve_base_pkg_dir(root_dir, name)
+    map_pkg_dir = ensure_pkg_dir(base, DIR_MAPPERS)
+    mapper_file = map_pkg_dir / f"{domain}.py"
+    if not mapper_file.exists():
+        mapper_file.write_text(render(TPL_MAPPER_COMPOSED))
+        status("new", str(mapper_file))
+
+    ep_key = stream_id
+    package_name = base.name
+    default_target = f"{package_name}.mappers.{domain}:mapper"
+    ep_target = mapper_path if (mapper_path and ":" in mapper_path) else default_target
+    pyproj_path = pyproject_path(root_dir)
+    try:
+        toml_text = pyproj_path.read_text()
+        updated = inject_ep(toml_text, MAPPERS_GROUP, ep_key, ep_target)
+        if updated != toml_text:
+            pyproj_path.write_text(updated)
+            status("ok", f"Registered mapper entry point '{ep_key}' -> {ep_target}")
+    except FileNotFoundError:
+        info("pyproject.toml not found; skipping entry point registration")
+    return ep_key

datapipeline/services/scaffold/parser.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+from typing import Optional
+
+from datapipeline.services.scaffold.templates import render
+from datapipeline.services.paths import pkg_root, resolve_base_pkg_dir
+from datapipeline.services.scaffold.utils import (
+    ensure_pkg_dir,
+    ep_key_from_name,
+    to_snake,
+    validate_identifier,
+    write_if_missing,
+)
+from datapipeline.services.scaffold.layout import DIR_PARSERS, TPL_PARSER
+from datapipeline.services.scaffold.layout import entrypoint_target, pyproject_path
+from datapipeline.services.entrypoints import inject_ep
+from datapipeline.services.constants import PARSERS_GROUP
+
+
+def create_parser(
+    *,
+    name: str,
+    dto_class: str,
+    dto_module: str,
+    root: Optional[Path],
+) -> str:
+    validate_identifier(name, "Parser name")
+
+    root_dir, pkg_name, _ = pkg_root(root)
+    base = resolve_base_pkg_dir(root_dir, pkg_name)
+    package_name = base.name
+
+    parsers_dir = ensure_pkg_dir(base, DIR_PARSERS)
+    module_name = to_snake(name)
+    path = parsers_dir / f"{module_name}.py"
+
+    write_if_missing(
+        path,
+        render(
+            TPL_PARSER,
+            CLASS_NAME=name,
+            DTO_CLASS=dto_class,
+            DTO_IMPORT=dto_module,
+        ),
+        label="Parser",
+    )
+
+    ep_key = ep_key_from_name(name)
+    pyproject = pyproject_path(root_dir)
+    toml = inject_ep(
+        pyproject.read_text(),
+        PARSERS_GROUP,
+        ep_key,
+        entrypoint_target(package_name, "parsers", module_name, name),
+    )
+    pyproject.write_text(toml)
+    return ep_key

datapipeline/services/scaffold/plugin.py

@@ -2,6 +2,7 @@ from importlib.resources import as_file, files
 from pathlib import Path
 import logging
 import os
+import sys
 
 import yaml
 
@@ -11,7 +12,8 @@ from ..constants import DEFAULT_IO_LOADER_EP
 
 logger = logging.getLogger(__name__)
 
-_RESERVED_PACKAGE_NAMES = {"datapipeline"}
+_RESERVED_PACKAGE_NAMES = {"datapipeline", "test", "tests"}
+_STDLIB_MODULE_NAMES = getattr(sys, "stdlib_module_names", set())
 
 
 def _normalized_package_name(dist_name: str) -> str:
@@ -21,6 +23,12 @@ def _normalized_package_name(dist_name: str) -> str:
             "`datapipeline` is reserved for the core package. Choose a different plugin name."
         )
         raise SystemExit(1)
+    if package_name in _STDLIB_MODULE_NAMES:
+        logger.error(
+            "Plugin name '%s' conflicts with a Python standard library module. Choose a different name.",
+            package_name,
+        )
+        raise SystemExit(1)
     if not package_name.isidentifier():
         logger.error(
             "Plugin names must be valid Python identifiers once hyphens are replaced with underscores."
@@ -47,7 +55,11 @@ def scaffold_plugin(name: str, outdir: Path) -> None:
         "{{DIST_NAME}}": name,
         "{{DEFAULT_IO_LOADER_EP}}": DEFAULT_IO_LOADER_EP,
     }
-    for p in (target / "pyproject.toml", target / "README.md"):
+    for p in target.rglob("*"):
+        if not p.is_file():
+            continue
+        if p.suffix not in {".py", ".toml", ".md", ".yaml", ".yml"}:
+            continue
         text = p.read_text()
         for placeholder, value in replacements.items():
             text = text.replace(placeholder, value)

datapipeline/services/scaffold/source_yaml.py

@@ -0,0 +1,91 @@
+from pathlib import Path
+from typing import Optional
+
+from datapipeline.services.paths import pkg_root
+from datapipeline.services.project_paths import (
+    sources_dir as resolve_sources_dir,
+    ensure_project_scaffold,
+    resolve_project_yaml_path,
+)
+from datapipeline.services.scaffold.templates import render
+from datapipeline.services.constants import (
+    DEFAULT_IO_LOADER_EP,
+    DEFAULT_SYNTHETIC_LOADER_EP,
+)
+from datapipeline.services.scaffold.utils import status
+
+
+def _loader_args(transport: str, fmt: Optional[str]) -> dict:
+    if transport == "fs":
+        args = {
+            "transport": "fs",
+            "format": fmt or "<FORMAT (csv|json|json-lines|pickle)>",
+            "path": "<PATH OR GLOB>",
+            "glob": False,
+            "encoding": "utf-8",
+        }
+        if fmt == "csv":
+            args["delimiter"] = ","
+        return args
+    if transport == "http":
+        args = {
+            "transport": "http",
+            "format": fmt or "<FORMAT (json|json-lines|csv)>",
+            "url": "<https://api.example.com/data.json>",
+            "headers": {},
+            "params": {},
+            "encoding": "utf-8",
+        }
+        if fmt == "csv":
+            args["delimiter"] = ","
+        return args
+    if transport == "synthetic":
+        return {"start": "<ISO8601>", "end": "<ISO8601>", "frequency": "1h"}
+    return {}
+
+
+def create_source_yaml(
+    *,
+    provider: str,
+    dataset: str,
+    loader_ep: str,
+    loader_args: dict,
+    parser_ep: str,
+    parser_args: dict | None = None,
+    root: Optional[Path],
+    project_yaml: Optional[Path] = None,
+) -> None:
+    root_dir, _, _ = pkg_root(root)
+    alias = f"{provider}.{dataset}"
+    parser_args = parser_args or {}
+
+    proj_yaml = project_yaml.resolve() if project_yaml is not None else resolve_project_yaml_path(root_dir)
+    ensure_project_scaffold(proj_yaml)
+    sources_dir = resolve_sources_dir(proj_yaml).resolve()
+    sources_dir.mkdir(parents=True, exist_ok=True)
+
+    src_cfg_path = sources_dir / f"{alias}.yaml"
+    if src_cfg_path.exists():
+        status("skip", f"Source YAML already exists: {src_cfg_path.resolve()}")
+        return
+
+    src_cfg_path.write_text(
+        render(
+            "source.yaml.j2",
+            id=alias,
+            parser_ep=parser_ep,
+            parser_args=parser_args,
+            loader_ep=loader_ep,
+            loader_args=loader_args,
+            default_io_loader_ep=DEFAULT_IO_LOADER_EP,
+        )
+    )
+    status("new", str(src_cfg_path.resolve()))
+
+
+def default_loader_config(transport: str, fmt: Optional[str]) -> tuple[str, dict]:
+    if transport in {"fs", "http"}:
+        return DEFAULT_IO_LOADER_EP, _loader_args(transport, fmt)
+    if transport == "synthetic":
+        return DEFAULT_SYNTHETIC_LOADER_EP, _loader_args(transport, fmt)
+    return DEFAULT_IO_LOADER_EP, {}

datapipeline/services/scaffold/stream_plan.py

@@ -0,0 +1,129 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from datapipeline.services.scaffold.domain import create_domain
+from datapipeline.services.scaffold.dto import create_dto
+from datapipeline.services.scaffold.parser import create_parser
+from datapipeline.services.scaffold.mapper import create_mapper
+from datapipeline.services.scaffold.source_yaml import create_source_yaml
+from datapipeline.services.scaffold.contract_yaml import write_ingest_contract
+from datapipeline.services.scaffold.discovery import list_dtos
+from datapipeline.services.paths import pkg_root
+from datapipeline.services.scaffold.utils import error_exit, status
+
+
+@dataclass
+class ParserPlan:
+    create: bool
+    create_dto: bool = False
+    dto_class: str | None = None
+    dto_module: str | None = None
+    parser_name: str | None = None
+    parser_ep: str | None = None
+
+
+@dataclass
+class MapperPlan:
+    create: bool
+    create_dto: bool = False
+    input_class: str | None = None
+    input_module: str | None = None
+    mapper_name: str | None = None
+    mapper_ep: str | None = None
+    domain: str | None = None
+
+
+@dataclass
+class StreamPlan:
+    provider: str
+    dataset: str
+    source_id: str
+    project_yaml: Path
+    stream_id: str
+    root: Path | None
+    create_source: bool
+    loader_ep: str | None = None
+    loader_args: dict | None = None
+    parser: ParserPlan | None = None
+    mapper: MapperPlan | None = None
+    domain: str | None = None
+    create_domain: bool = False
+
+
+def execute_stream_plan(plan: StreamPlan) -> None:
+    pyproject_path = None
+    before_pyproject = None
+    try:
+        root_dir, _, pyproject = pkg_root(plan.root)
+        pyproject_path = pyproject
+        if pyproject_path.exists():
+            before_pyproject = pyproject_path.read_text()
+    except SystemExit:
+        pyproject_path = None
+        before_pyproject = None
+
+    if plan.create_domain and plan.domain:
+        create_domain(domain=plan.domain, root=plan.root)
+
+    parser_ep = None
+    if plan.parser:
+        if plan.parser.create:
+            if plan.parser.dto_class and plan.parser.create_dto:
+                create_dto(name=plan.parser.dto_class, root=plan.root)
+            dto_module = plan.parser.dto_module or list_dtos(root=plan.root).get(plan.parser.dto_class or "")
+            if not dto_module:
+                error_exit("Failed to resolve DTO module.")
+            parser_ep = create_parser(
+                name=plan.parser.parser_name or "parser",
+                dto_class=plan.parser.dto_class or "DTO",
+                dto_module=dto_module,
+                root=plan.root,
+            )
+        else:
+            parser_ep = plan.parser.parser_ep
+
+    mapper_ep = None
+    if plan.mapper:
+        if plan.mapper.create:
+            if plan.mapper.input_class and plan.mapper.create_dto:
+                create_dto(name=plan.mapper.input_class, root=plan.root)
+            input_module = plan.mapper.input_module
+            if not input_module and plan.mapper.input_class:
+                input_module = list_dtos(root=plan.root).get(plan.mapper.input_class)
+            if not input_module:
+                error_exit("Failed to resolve mapper input module.")
+            mapper_ep = create_mapper(
+                name=plan.mapper.mapper_name or "mapper",
+                input_class=plan.mapper.input_class or "Record",
+                input_module=input_module,
+                domain=plan.mapper.domain or plan.domain or "domain",
+                root=plan.root,
+            )
+        else:
+            mapper_ep = plan.mapper.mapper_ep
+
+    if plan.create_source and plan.loader_ep and plan.loader_args is not None:
+        create_source_yaml(
+            provider=plan.provider,
+            dataset=plan.dataset,
+            loader_ep=plan.loader_ep,
+            loader_args=plan.loader_args,
+            parser_ep=parser_ep or "identity",
+            root=plan.root,
+            project_yaml=plan.project_yaml,
+        )
+
+    write_ingest_contract(
+        project_yaml=plan.project_yaml,
+        stream_id=plan.stream_id,
+        source=plan.source_id,
+        mapper_entrypoint=mapper_ep or "identity",
+    )
+    status("ok", "Stream created.")
+    if pyproject_path and before_pyproject is not None:
+        after_pyproject = pyproject_path.read_text()
+        if after_pyproject != before_pyproject:
+            status(
+                "note",
+                f"Entry points updated; reinstall plugin: pip install -e {pyproject_path.parent}",
+            )

datapipeline/services/scaffold/utils.py

@@ -0,0 +1,187 @@
+from pathlib import Path
+import sys
+import logging
+
+from datapipeline.services.scaffold.layout import to_snake, slugify, ep_key_from_name
+
+_LOGGER = logging.getLogger("datapipeline.cli")
+
+
+def ensure_pkg_dir(base: Path, name: str) -> Path:
+    path = base / name
+    path.mkdir(parents=True, exist_ok=True)
+    (path / "__init__.py").touch(exist_ok=True)
+    return path
+
+
+__all__ = [
+    "ensure_pkg_dir",
+    "to_snake",
+    "slugify",
+    "ep_key_from_name",
+    "validate_identifier",
+    "write_if_missing",
+    "prompt_required",
+    "prompt_optional",
+    "choose_name",
+    "status",
+    "info",
+    "error_exit",
+    "pick_from_list",
+    "pick_from_menu",
+    "choose_existing_or_create",
+]
+
+
+def validate_identifier(name: str, label: str) -> None:
+    if not name or not name.isidentifier():
+        error_exit(f"{label} must be a valid Python identifier")
+
+
+def write_if_missing(path: Path, text: str, *, label: str | None = None) -> bool:
+    if path.exists():
+        status("skip", f"{label or 'File'} already exists: {path}")
+        return False
+    path.write_text(text)
+    status("new", str(path))
+    return True
+
+
+def prompt_required(prompt: str) -> str:
+    value = input(f"{prompt}: ").strip()
+    if not value:
+        error_exit(f"{prompt} is required")
+    return value
+
+
+def prompt_optional(prompt: str) -> str | None:
+    value = input(f"{prompt}: ").strip()
+    return value or None
+
+
+def choose_name(label: str, *, default: str | None = None) -> str:
+    if not default:
+        return prompt_required(label)
+    info(f"{label}:")
+    info(f"  [1] {default} (default)")
+    info("  [2] Custom name")
+    while True:
+        sel = input("> ").strip()
+        if sel == "":
+            return default
+        if sel == "1":
+            return default
+        if sel == "2":
+            return prompt_required(label)
+        info("Please enter a number from the list.")
+
+
+def info(message: str) -> None:
+    _LOGGER.info(message)
+
+
+def status(tag: str, message: str) -> None:
+    _LOGGER.info("[%s] %s", tag, message)
+
+
+def error_exit(message: str, code: int = 2) -> None:
+    _LOGGER.error(message)
+    raise SystemExit(code)
+
+
+def pick_from_list(prompt: str, options: list[str]) -> str:
+    info(prompt)
+    for i, opt in enumerate(options, 1):
+        info(f"  [{i}] {opt}")
+    while True:
+        sel = input("> ").strip()
+        if sel.isdigit():
+            idx = int(sel)
+            if 1 <= idx <= len(options):
+                return options[idx - 1]
+        info("Please enter a number from the list.")
+
+
+def pick_from_menu(prompt: str, options: list[tuple[str, str]], *, allow_default: bool = True) -> str:
+    info(prompt)
+    for i, (_, label) in enumerate(options, 1):
+        info(f"  [{i}] {label}")
+    while True:
+        sel = input("> ").strip()
+        if sel == "" and allow_default:
+            return options[0][0]
+        if sel.isdigit():
+            idx = int(sel)
+            if 1 <= idx <= len(options):
+                return options[idx - 1][0]
+        info("Please enter a number from the list.")
+
+
+def pick_multiple_from_list(prompt: str, options: list[str]) -> list[str]:
+    info(prompt)
+    for i, opt in enumerate(options, 1):
+        info(f"  [{i}] {opt}")
+    sel = input("> ").strip()
+    try:
+        idxs = [int(x) for x in sel.split(",") if x.strip()]
+    except ValueError:
+        error_exit("Invalid selection.")
+    picked = [options[i - 1] for i in idxs if 1 <= i <= len(options)]
+    if not picked:
+        error_exit("No inputs selected.")
+    return picked
+
+
+def choose_existing_or_create(
+    *,
+    label: str,
+    existing: list[str],
+    create_label: str,
+    create_fn,
+    prompt_new: str,
+    root: Path | None,
+    default_new: str | None = None,
+) -> str:
+    info(f"{label}:")
+    info(f"  [1] {create_label} (default)")
+    info(f"  [2] Select existing {label}")
+    while True:
+        sel = input("> ").strip()
+        if sel == "":
+            sel = "1"
+        if sel == "1":
+            name = choose_name(prompt_new, default=default_new)
+            create_fn(name=name, root=root)
+            return name
+        if sel == "2":
+            if not existing:
+                error_exit(f"No existing {label} found.")
+            return pick_from_list(f"Select {label}:", existing)
+        info("Please enter a number from the list.")
+
+
+def choose_existing_or_create_name(
+    *,
+    label: str,
+    existing: list[str],
+    create_label: str,
+    prompt_new: str,
+    default_new: str | None = None,
+) -> tuple[str, bool]:
+    """Return (name, created) without side effects."""
+    info(f"{label}:")
+    info(f"  [1] {create_label} (default)")
+    info(f"  [2] Select existing {label}")
+    while True:
+        sel = input("> ").strip()
+        if sel == "":
+            sel = "1"
+        if sel == "1":
+            name = choose_name(prompt_new, default=default_new)
+            return name, True
+        if sel == "2":
+            if not existing:
+                error_exit(f"No existing {label} found.")
+            name = pick_from_list(f"Select {label}:", existing)
+            return name, False
+        info("Please enter a number from the list.")

datapipeline/sources/data_loader.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from typing import Iterator, Any, Optional
 from .models.loader import BaseDataLoader
 from .transports import Transport, HttpTransport