jerry-thomas 1.0.3__py3-none-any.whl → 2.0.0__py3-none-any.whl

datapipeline/services/scaffold/contract_yaml.py

@@ -0,0 +1,76 @@
+from pathlib import Path
+
+from datapipeline.services.project_paths import streams_dir as resolve_streams_dir, ensure_project_scaffold
+from datapipeline.services.scaffold.templates import render
+from datapipeline.services.scaffold.utils import status
+
+
+def write_ingest_contract(
+    *,
+    project_yaml: Path,
+    stream_id: str,
+    source: str,
+    mapper_entrypoint: str,
+) -> Path:
+    ensure_project_scaffold(project_yaml)
+    streams_path = resolve_streams_dir(project_yaml)
+    streams_dir = streams_path if streams_path.is_dir() else streams_path.parent
+    streams_dir.mkdir(parents=True, exist_ok=True)
+    cfile = streams_dir / f"{stream_id}.yaml"
+    cfile.write_text(
+        render(
+            "contracts/ingest.yaml.j2",
+            source=source,
+            stream_id=stream_id,
+            mapper_entrypoint=mapper_entrypoint,
+        ),
+        encoding="utf-8",
+    )
+    status("new", f"canonical spec: {cfile}")
+    return cfile
+
+
+def write_composed_contract(
+    *,
+    project_yaml: Path,
+    stream_id: str,
+    inputs_list: str,
+    mapper_entrypoint: str,
+    driver_key: str,
+) -> Path:
+    ensure_project_scaffold(project_yaml)
+    streams_path = resolve_streams_dir(project_yaml)
+    streams_dir = streams_path if streams_path.is_dir() else streams_path.parent
+    streams_dir.mkdir(parents=True, exist_ok=True)
+    cfile = streams_dir / f"{stream_id}.yaml"
+    cfile.write_text(
+        render(
+            "contracts/composed.yaml.j2",
+            stream_id=stream_id,
+            inputs_list=inputs_list,
+            mapper_entrypoint=mapper_entrypoint,
+            driver_key=driver_key,
+        ).strip() + "\n",
+        encoding="utf-8",
+    )
+    status("new", f"composed contract: {cfile}")
+    return cfile
+
+
+def compose_inputs(picked: list[str]) -> tuple[str, str]:
+    """Build composed inputs list text and driver key."""
+    built: list[str] = []
+    for ref in picked:
+        parts = ref.split(".")
+        if len(parts) >= 3:
+            domain, variant = parts[0], parts[-1]
+            alias = f"{domain}_{variant}"
+        elif len(parts) == 2:
+            alias = parts[0]
+        else:
+            alias = ref
+        built.append(f"{alias}={ref}")
+    inputs_csv = ",".join(built)
+    inputs_list = "\n  - ".join(s.strip() for s in inputs_csv.split(",") if s.strip())
+    driver_key = inputs_csv.split(",")[0].split("=")[0].strip()
+    return inputs_list, driver_key

datapipeline/services/scaffold/demo.py

@@ -0,0 +1,141 @@
+from importlib.resources import as_file, files
+from pathlib import Path
+import logging
+import os
+import shutil
+
+import yaml
+
+from datapipeline.services.entrypoints import inject_ep
+from datapipeline.services.paths import pkg_root, resolve_base_pkg_dir
+from datapipeline.utils.load import load_yaml
+
+logger = logging.getLogger(__name__)
+
+_DEMO_DATASET_ALIAS = "demo"
+_DEMO_PARSER_EP = "sandbox_ohlcv_dto_parser"
+_DEMO_MAPPER_EP = "map_sandbox_ohlcv_dto_to_equity"
+
+
+def _replace_placeholders(path: Path, replacements: dict[str, str]) -> None:
+    if not path.is_file():
+        return
+    if path.suffix not in {".py", ".toml", ".md", ".yaml", ".yml"}:
+        return
+    text = path.read_text()
+    for placeholder, value in replacements.items():
+        text = text.replace(placeholder, value)
+    path.write_text(text)
+
+
+def _inject_demo_entrypoints(pyproject: Path, pkg_name: str) -> None:
+    toml = pyproject.read_text()
+    toml = inject_ep(
+        toml,
+        "parsers",
+        _DEMO_PARSER_EP,
+        f"{pkg_name}.parsers.sandbox_ohlcv_dto_parser:SandboxOhlcvDTOParser",
+    )
+    toml = inject_ep(
+        toml,
+        "mappers",
+        _DEMO_MAPPER_EP,
+        f"{pkg_name}.mappers.map_sandbox_ohlcv_dto_to_equity:map_sandbox_ohlcv_dto_to_equity",
+    )
+    pyproject.write_text(toml)
+
+
+def _update_workspace_jerry(
+    workspace_root: Path,
+    plugin_root_rel: Path,
+    dataset_path: Path,
+) -> None:
+    workspace_jerry = workspace_root / "jerry.yaml"
+    if not workspace_jerry.exists():
+        return
+    data = load_yaml(workspace_jerry)
+    datasets = data.get("datasets") or {}
+    demo_path = (plugin_root_rel / dataset_path).as_posix()
+    datasets[_DEMO_DATASET_ALIAS] = demo_path
+    # Drop skeleton placeholders that point into this demo plugin.
+    for key in ("your-dataset", "interim-builder"):
+        path = datasets.get(key)
+        if isinstance(path, str) and path.startswith(plugin_root_rel.as_posix()):
+            datasets.pop(key, None)
+    data["datasets"] = datasets
+    data["default_dataset"] = _DEMO_DATASET_ALIAS
+    workspace_jerry.write_text(
+        yaml.safe_dump(data, sort_keys=False), encoding="utf-8"
+    )
+
+
+def _copy_tree(src: Path, dest: Path) -> None:
+    if src.is_dir():
+        shutil.copytree(src, dest, dirs_exist_ok=True)
+    else:
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src, dest)
+
+
+def scaffold_demo(root: Path | None = None) -> None:
+    root_dir, pkg_name, pyproject = pkg_root(root)
+    demo_ref = files("datapipeline") / "templates" / "demo_skeleton"
+    with as_file(demo_ref) as demo_dir:
+        demo_data = demo_dir / "demo"
+        demo_pkg = demo_dir / "src" / "{{PACKAGE_NAME}}"
+
+        target_demo = root_dir / "demo"
+        if target_demo.exists():
+            logger.error("`%s` already exists", target_demo)
+            raise SystemExit(1)
+
+        _copy_tree(demo_data, target_demo)
+
+        dest_pkg = resolve_base_pkg_dir(root_dir, pkg_name)
+        for item in demo_pkg.iterdir():
+            _copy_tree(item, dest_pkg / item.name)
+
+    for cleanup in ("your-dataset", "your-interim-data-builder"):
+        extra = root_dir / cleanup
+        if extra.exists():
+            shutil.rmtree(extra)
+
+    replacements = {
+        "{{PACKAGE_NAME}}": pkg_name,
+    }
+    for p in target_demo.rglob("*"):
+        _replace_placeholders(p, replacements)
+    for p in dest_pkg.rglob("*"):
+        _replace_placeholders(p, replacements)
+
+    _inject_demo_entrypoints(pyproject, pkg_name)
+
+    workspace_root = Path.cwd().resolve()
+    try:
+        plugin_root_rel = root_dir.relative_to(workspace_root)
+    except ValueError:
+        plugin_root_rel = Path(os.path.relpath(root_dir, workspace_root))
+
+    _update_workspace_jerry(
+        workspace_root,
+        plugin_root_rel,
+        Path("demo/project.yaml"),
+    )
+
+    # Rewrite demo source YAMLs to use plugin-root-relative paths so
+    # workspace-level runs resolve sample data correctly.
+    sources_dir = target_demo / "sources"
+    for src_file in sources_dir.glob("*.yaml"):
+        src_data = load_yaml(src_file)
+        loader = (src_data or {}).get("loader") or {}
+        args = loader.get("args") or {}
+        path = args.get("path")
+        if isinstance(path, str) and path and not Path(path).is_absolute():
+            args["path"] = (plugin_root_rel / path).as_posix()
+            loader["args"] = args
+            src_data["loader"] = loader
+            src_file.write_text(
+                yaml.safe_dump(src_data, sort_keys=False), encoding="utf-8"
+            )
+
+    logger.info("demo dataset created at %s", target_demo)

datapipeline/services/scaffold/discovery.py

@@ -0,0 +1,115 @@
+from pathlib import Path
+import ast
+from typing import Optional
+
+from datapipeline.services.paths import pkg_root, resolve_base_pkg_dir
+from datapipeline.services.entrypoints import read_group_entries
+from datapipeline.services.constants import PARSERS_GROUP, LOADERS_GROUP, MAPPERS_GROUP
+from datapipeline.services.project_paths import sources_dir as resolve_sources_dir, streams_dir as resolve_streams_dir
+
+
+def list_dtos(*, root: Optional[Path] = None) -> dict[str, str]:
+    """Return mapping of DTO class name -> module path."""
+    root_dir, pkg_name, _ = pkg_root(root)
+    base = resolve_base_pkg_dir(root_dir, pkg_name)
+    dtos_dir = base / "dtos"
+    if not dtos_dir.exists():
+        return {}
+
+    package_name = base.name
+    found: dict[str, str] = {}
+    for path in sorted(dtos_dir.glob("*.py")):
+        if path.name == "__init__.py":
+            continue
+        try:
+            tree = ast.parse(path.read_text())
+        except Exception:
+            continue
+        module = f"{package_name}.dtos.{path.stem}"
+        for node in tree.body:
+            if isinstance(node, ast.ClassDef) and _is_dataclass(node):
+                found[node.name] = module
+    return found
+
+
+def _is_dataclass(node: ast.ClassDef) -> bool:
+    for deco in node.decorator_list:
+        if isinstance(deco, ast.Name) and deco.id == "dataclass":
+            return True
+        if isinstance(deco, ast.Attribute) and deco.attr == "dataclass":
+            return True
+    return False
+
+
+def list_parsers(*, root: Optional[Path] = None) -> dict[str, str]:
+    root_dir, _, pyproject = pkg_root(root)
+    if not pyproject.exists():
+        return {}
+    return read_group_entries(pyproject, PARSERS_GROUP)
+
+
+def list_loaders(*, root: Optional[Path] = None) -> dict[str, str]:
+    root_dir, _, pyproject = pkg_root(root)
+    if not pyproject.exists():
+        return {}
+    return read_group_entries(pyproject, LOADERS_GROUP)
+
+
+def list_mappers(*, root: Optional[Path] = None) -> dict[str, str]:
+    root_dir, _, pyproject = pkg_root(root)
+    if not pyproject.exists():
+        return {}
+    return read_group_entries(pyproject, MAPPERS_GROUP)
+
+
+def list_domains(*, root: Optional[Path] = None) -> list[str]:
+    root_dir, pkg_name, _ = pkg_root(root)
+    base = resolve_base_pkg_dir(root_dir, pkg_name)
+    dom_dir = base / "domains"
+    if not dom_dir.exists():
+        return []
+    return sorted(
+        p.name
+        for p in dom_dir.iterdir()
+        if p.is_dir() and (p / "model.py").exists()
+    )
+
+
+def list_sources(project_yaml: Path) -> list[str]:
+    from datapipeline.utils.load import load_yaml
+    from datapipeline.services.constants import PARSER_KEY, LOADER_KEY, SOURCE_ID_KEY
+
+    sources_dir = resolve_sources_dir(project_yaml)
+    if not sources_dir.exists():
+        return []
+    out: list[str] = []
+    for p in sorted(sources_dir.rglob("*.y*ml")):
+        try:
+            data = load_yaml(p)
+        except Exception:
+            continue
+        if isinstance(data, dict) and isinstance(data.get(PARSER_KEY), dict) and isinstance(data.get(LOADER_KEY), dict):
+            alias = data.get(SOURCE_ID_KEY)
+            if isinstance(alias, str):
+                out.append(alias)
+    return sorted(set(out))
+
+
+def list_streams(project_yaml: Path) -> list[str]:
+    from datapipeline.utils.load import load_yaml
+    from datapipeline.services.constants import STREAM_ID_KEY
+
+    streams_dir = resolve_streams_dir(project_yaml)
+    if not streams_dir.exists():
+        return []
+    out: list[str] = []
+    for p in sorted(streams_dir.rglob("*.y*ml")):
+        try:
+            data = load_yaml(p)
+        except Exception:
+            continue
+        if isinstance(data, dict) and data.get("kind") in {"ingest", "composed"}:
+            sid = data.get(STREAM_ID_KEY)
+            if isinstance(sid, str) and sid:
+                out.append(sid)
+    return sorted(set(out))

datapipeline/services/scaffold/domain.py

@@ -2,25 +2,33 @@ from pathlib import Path
 from typing import Optional
 
 from datapipeline.services.scaffold.templates import render
+from datapipeline.services.scaffold.utils import (
+    ensure_pkg_dir,
+    to_snake,
+    validate_identifier,
+    write_if_missing,
+)
+from datapipeline.services.scaffold.layout import DIR_DOMAINS, TPL_DOMAIN_RECORD, domain_record_class
 
 from ..paths import pkg_root, resolve_base_pkg_dir
 
 
 def create_domain(*, domain: str, root: Optional[Path]) -> None:
+    validate_identifier(domain, "Domain name")
     root_dir, name, _ = pkg_root(root)
     base = resolve_base_pkg_dir(root_dir, name)
     package_name = base.name
-    pkg_dir = base / "domains" / domain
-    pkg_dir.mkdir(parents=True, exist_ok=True)
-    (pkg_dir / "__init__.py").touch(exist_ok=True)
-
-    def write_missing(path: Path, tpl: str, **ctx):
-        if not path.exists():
-            path.write_text(render(tpl, **ctx))
-            print(f"[new] {path}")
-
-    cls_ = "".join(w.capitalize() for w in domain.split("_"))
+    pkg_dir = ensure_pkg_dir(base / DIR_DOMAINS, domain)
     parent = "TemporalRecord"
-    write_missing(pkg_dir / "model.py", "record.py.j2",
-                  PACKAGE_NAME=package_name, DOMAIN=domain, CLASS_NAME=f"{cls_}Record",
-                  PARENT_CLASS=parent, time_aware=True)
+    write_if_missing(
+        pkg_dir / "model.py",
+        render(
+            TPL_DOMAIN_RECORD,
+            PACKAGE_NAME=package_name,
+            DOMAIN=domain,
+            CLASS_NAME=domain_record_class(domain),
+            PARENT_CLASS=parent,
+            time_aware=True,
+        ),
+        label="Domain",
+    )

datapipeline/services/scaffold/dto.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+from typing import Optional
+
+from datapipeline.services.scaffold.templates import render
+from datapipeline.services.paths import pkg_root, resolve_base_pkg_dir
+from datapipeline.services.scaffold.utils import (
+    ensure_pkg_dir,
+    to_snake,
+    validate_identifier,
+    write_if_missing,
+)
+from datapipeline.services.scaffold.layout import DIR_DTOS, TPL_DTO
+
+
+def create_dto(*, name: str, root: Optional[Path]) -> None:
+    validate_identifier(name, "DTO name")
+
+    root_dir, pkg_name, _ = pkg_root(root)
+    base = resolve_base_pkg_dir(root_dir, pkg_name)
+    dtos_dir = ensure_pkg_dir(base, DIR_DTOS)
+    module_name = to_snake(name)
+    path = dtos_dir / f"{module_name}.py"
+    write_if_missing(
+        path,
+        render(
+            TPL_DTO,
+            CLASS_NAME=name,
+            DOMAIN=name,
+        ),
+        label="DTO",
+    )

datapipeline/services/scaffold/filter.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 from typing import Optional
 from datapipeline.services.scaffold.templates import render
+from datapipeline.services.scaffold.utils import status
 from ..constants import FILTERS_GROUP
 from ..entrypoints import inject_ep
 from ..paths import pkg_root, resolve_base_pkg_dir
@@ -19,7 +20,7 @@ def create_filter(*, name: str, root: Optional[Path]) -> None:
     path = filters_dir / f"{module_name}.py"
     if not path.exists():
         path.write_text(render("filter.py.j2", FUNCTION_NAME=name))
-        print(f"[new] {path}")
+        status("new", str(path))
 
     # Register entry point under datapipeline.filters
     toml_path = root_dir / "pyproject.toml"

datapipeline/services/scaffold/layout.py

@@ -0,0 +1,96 @@
+from pathlib import Path
+import re
+
+from datapipeline.services.scaffold.templates import camel
+
+
+def to_snake(name: str) -> str:
+    s1 = re.sub(r"(.)([A-Z][a-z]+)", r"\1_\2", name)
+    return re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", s1).lower()
+
+
+def slugify(text: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "_", text.strip().lower()).strip("_")
+
+
+def ep_key_from_name(name: str) -> str:
+    return to_snake(name)
+
+
+# Directory names
+DIR_DTOS = "dtos"
+DIR_PARSERS = "parsers"
+DIR_LOADERS = "loaders"
+DIR_MAPPERS = "mappers"
+DIR_DOMAINS = "domains"
+
+# Template paths
+TPL_DTO = "dto.py.j2"
+TPL_PARSER = "parser.py.j2"
+TPL_LOADER_BASIC = "loaders/basic.py.j2"
+TPL_LOADER_SYNTHETIC = "loader_synthetic.py.j2"
+TPL_MAPPER_INGEST = "mappers/ingest.py.j2"
+TPL_MAPPER_COMPOSED = "mappers/composed.py.j2"
+TPL_DOMAIN_RECORD = "record.py.j2"
+
+
+def class_name_with_suffix(name: str, suffix: str) -> str:
+    return f"{camel(name)}{suffix}"
+
+
+def loader_class_name(name: str) -> str:
+    return class_name_with_suffix(name, "Loader")
+
+
+def domain_record_class(domain: str) -> str:
+    return class_name_with_suffix(domain, "Record")
+
+
+def loader_template_name(template: str) -> str:
+    if template == "synthetic":
+        return TPL_LOADER_SYNTHETIC
+    return TPL_LOADER_BASIC
+
+
+def dto_class_name(base: str) -> str:
+    return class_name_with_suffix(base, "DTO")
+
+
+def dto_module_path(package: str, dto_class: str) -> str:
+    return f"{package}.{DIR_DTOS}.{to_snake(dto_class)}"
+
+
+def default_parser_name(dto_class: str) -> str:
+    return f"{dto_class}Parser"
+
+
+def default_mapper_name(input_module: str, domain: str) -> str:
+    input_mod = input_module.rsplit(".", 1)[-1]
+    return f"map_{input_mod}_to_{domain}"
+
+
+def default_stream_id(domain: str, dataset: str, variant: str | None = None) -> str:
+    base = f"{slugify(domain)}.{slugify(dataset)}"
+    return f"{base}.{slugify(variant)}" if variant else base
+
+
+# Prompt labels (keep CLI wording consistent)
+LABEL_DTO_FOR_PARSER = "DTO for parser"
+LABEL_DTO_FOR_MAPPER = "DTO for mapper"
+LABEL_DOMAIN_TO_MAP = "Domain"
+LABEL_MAPPER_INPUT = "Mapper input"
+
+
+def default_mapper_name_for_identity(domain: str) -> str:
+    return f"map_identity_to_{slugify(domain)}"
+
+def pyproject_path(root_dir: Path) -> Path:
+    return root_dir / "pyproject.toml"
+
+
+def module_path(package: str, group: str, module: str) -> str:
+    return f"{package}.{group}.{module}"
+
+
+def entrypoint_target(package: str, group: str, module: str, attr: str) -> str:
+    return f"{module_path(package, group, module)}:{attr}"

datapipeline/services/scaffold/loader.py

@@ -0,0 +1,61 @@
+from pathlib import Path
+from typing import Optional
+
+from datapipeline.services.paths import pkg_root, resolve_base_pkg_dir
+from datapipeline.services.scaffold.templates import render
+from datapipeline.services.scaffold.utils import (
+    ensure_pkg_dir,
+    ep_key_from_name,
+    to_snake,
+    validate_identifier,
+    write_if_missing,
+)
+from datapipeline.services.scaffold.layout import (
+    DIR_LOADERS,
+    entrypoint_target,
+    loader_class_name,
+    loader_template_name,
+    pyproject_path,
+)
+from datapipeline.services.entrypoints import inject_ep
+from datapipeline.services.constants import LOADERS_GROUP
+
+
+def create_loader(
+    *,
+    name: str,
+    root: Optional[Path],
+    template: str = "basic",
+) -> str:
+    validate_identifier(name, "Loader name")
+
+    root_dir, pkg_name, _ = pkg_root(root)
+    base = resolve_base_pkg_dir(root_dir, pkg_name)
+    package_name = base.name
+
+    loaders_dir = ensure_pkg_dir(base, DIR_LOADERS)
+    module_name = to_snake(name)
+    path = loaders_dir / f"{module_name}.py"
+
+    class_name = loader_class_name(name)
+    template_name = loader_template_name(template)
+
+    write_if_missing(
+        path,
+        render(
+            template_name,
+            CLASS_NAME=class_name,
+        ),
+        label="Loader",
+    )
+
+    ep_key = ep_key_from_name(name)
+    pyproject = pyproject_path(root_dir)
+    toml = inject_ep(
+        pyproject.read_text(),
+        LOADERS_GROUP,
+        ep_key,
+        entrypoint_target(package_name, "loaders", module_name, class_name),
+    )
+    pyproject.write_text(toml)
+    return ep_key