dvcgen 0.2.0__py3-none-any.whl

dvcgen/__init__.py

@@ -0,0 +1,21 @@
+"""Generate DVC pipeline files from Python declarations."""
+
+__version__ = "0.1.0"
+
+
+def dep(path):
+    """Declare a pipeline dependency and return its runtime value."""
+    return path
+
+
+def out(path):
+    """Declare a pipeline output and return its runtime value."""
+    return path
+
+
+def param(name, default):
+    """Declare a pipeline parameter and return its default runtime value."""
+    return default
+
+
+__all__ = ["__version__", "dep", "out", "param"]

dvcgen/cli.py

@@ -0,0 +1,119 @@
+"""Command-line interface for dvcgen."""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Sequence
+from pathlib import Path
+import sys
+from typing import Optional, TextIO
+
+from dvcgen import __version__
+from dvcgen.generate import write_files
+from dvcgen.inspect import inspect_files
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="dvcgen",
+        description="Generate dvc.yaml and params.yaml from Python pipeline scripts.",
+    )
+    parser.add_argument(
+        "scripts",
+        nargs="*",
+        help="Python pipeline scripts to inspect.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        default=".",
+        help="Directory where dvc.yaml and params.yaml are written. Defaults to the current directory.",
+    )
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="Overwrite existing dvc.yaml and params.yaml files.",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+    return parser
+
+
+def main(
+    argv: Optional[Sequence[str]] = None,
+    stdout: Optional[TextIO] = None,
+    stderr: Optional[TextIO] = None,
+) -> int:
+    stdout = sys.stdout if stdout is None else stdout
+    stderr = sys.stderr if stderr is None else stderr
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if not args.scripts:
+        print("dvcgen: error: provide at least one Python pipeline script", file=stderr)
+        print("Try 'dvcgen --help' for usage.", file=stderr)
+        return 2
+
+    script_paths = tuple(Path(script) for script in args.scripts)
+    output_dir = Path(args.output_dir)
+    dvc_path = output_dir / "dvc.yaml"
+    params_path = output_dir / "params.yaml"
+
+    validation_message = _validation_error(
+        script_paths,
+        output_dir,
+        (dvc_path, params_path),
+        args.force,
+    )
+    if validation_message is not None:
+        print(f"dvcgen: error: {validation_message}", file=stderr)
+        return 2
+
+    try:
+        declarations = inspect_files(script_paths)
+        write_files(declarations, dvc_path=dvc_path, params_path=params_path)
+    except SyntaxError as syntax_error:
+        print(
+            f"dvcgen: error: failed to parse {syntax_error.filename}: {syntax_error.msg}",
+            file=stderr,
+        )
+        return 2
+    except OSError as os_error:
+        print(f"dvcgen: error: {os_error}", file=stderr)
+        return 2
+    except ValueError as value_error:
+        print(f"dvcgen: error: {value_error}", file=stderr)
+        return 2
+
+    print(f"Wrote {dvc_path} and {params_path}", file=stdout)
+    return 0
+
+
+def _validation_error(
+    script_paths: Sequence[Path],
+    output_dir: Path,
+    output_paths: Sequence[Path],
+    force: bool,
+) -> Optional[str]:
+    for script_path in script_paths:
+        if not script_path.exists():
+            return f"input script not found: {script_path}"
+        if not script_path.is_file():
+            return f"input script is not a file: {script_path}"
+        if script_path.suffix != ".py":
+            return f"input script must be a .py file: {script_path}"
+
+    if output_dir.exists() and not output_dir.is_dir():
+        return f"output directory is not a directory: {output_dir}"
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if not force:
+        existing_paths = [path for path in output_paths if path.exists()]
+        if existing_paths:
+            joined_paths = ", ".join(str(path) for path in existing_paths)
+            return f"refusing to overwrite existing file(s): {joined_paths}; use --force to replace them"
+
+    return None

dvcgen/generate.py

@@ -0,0 +1,145 @@
+"""Generate DVC configuration files from extracted declarations."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Mapping, Sequence
+from pathlib import Path
+from typing import Any
+
+from dvcgen.inspect import SourceDeclarations
+
+
+def dvc_document(declarations: Iterable[SourceDeclarations]) -> dict[str, Any]:
+    """Build a dvc.yaml document from source declarations."""
+    stages: dict[str, dict[str, Any]] = {}
+
+    for source_declarations in sorted(declarations, key=_stage_name):
+        stage_name = _stage_name(source_declarations)
+        if stage_name in stages:
+            raise ValueError(f"duplicate stage name: {stage_name}")
+
+        stage: dict[str, Any] = {
+            "cmd": f"python {source_declarations.source}",
+            "deps": [
+                source_declarations.source,
+                *(dep.path for dep in source_declarations.deps),
+            ],
+        }
+
+        if source_declarations.outs:
+            stage["outs"] = [out.path for out in source_declarations.outs]
+        if source_declarations.params:
+            stage["params"] = sorted(param.name for param in source_declarations.params)
+
+        stages[stage_name] = stage
+
+    return {"stages": stages}
+
+
+def params_document(declarations: Iterable[SourceDeclarations]) -> dict[str, Any]:
+    """Build a params.yaml document from source declarations."""
+    params: dict[str, Any] = {}
+
+    for source_declarations in sorted(declarations, key=_stage_name):
+        for param in sorted(source_declarations.params, key=lambda item: item.name):
+            _assign_dotted(params, param.name, param.default)
+
+    return params
+
+
+def write_files(
+    declarations: Sequence[SourceDeclarations],
+    dvc_path: str | Path = "dvc.yaml",
+    params_path: str | Path = "params.yaml",
+) -> None:
+    """Write dvc.yaml and params.yaml for the supplied declarations."""
+    Path(dvc_path).write_text(
+        dump_yaml(dvc_document(declarations)),
+        encoding="utf-8",
+    )
+    Path(params_path).write_text(
+        dump_yaml(params_document(declarations)),
+        encoding="utf-8",
+    )
+
+
+def dump_yaml(value: Any) -> str:
+    """Serialize a small, deterministic YAML subset."""
+    return "\n".join(_yaml_lines(value, indent=0)) + "\n"
+
+
+def _stage_name(declarations: SourceDeclarations) -> str:
+    return Path(declarations.source).stem
+
+
+def _assign_dotted(document: dict[str, Any], name: str, value: Any) -> None:
+    parts = name.split(".")
+    if not all(parts):
+        raise ValueError(f"invalid parameter name: {name}")
+
+    cursor = document
+    for part in parts[:-1]:
+        existing = cursor.setdefault(part, {})
+        if not isinstance(existing, dict):
+            raise ValueError(f"conflicting parameter name: {name}")
+        cursor = existing
+
+    leaf = parts[-1]
+    if leaf in cursor and isinstance(cursor[leaf], dict):
+        raise ValueError(f"conflicting parameter name: {name}")
+    cursor[leaf] = value
+
+
+def _yaml_lines(value: Any, indent: int) -> list[str]:
+    if isinstance(value, Mapping):
+        return _mapping_lines(value, indent)
+    if isinstance(value, list):
+        return _list_lines(value, indent)
+    return [" " * indent + _scalar(value)]
+
+
+def _mapping_lines(value: Mapping[str, Any], indent: int) -> list[str]:
+    lines: list[str] = []
+    prefix = " " * indent
+
+    for key in sorted(value):
+        item = value[key]
+        yaml_key = _string(key)
+        if isinstance(item, (Mapping, list)):
+            lines.append(f"{prefix}{yaml_key}:")
+            lines.extend(_yaml_lines(item, indent + 2))
+        else:
+            lines.append(f"{prefix}{yaml_key}: {_scalar(item)}")
+
+    return lines
+
+
+def _list_lines(value: list[Any], indent: int) -> list[str]:
+    lines: list[str] = []
+    prefix = " " * indent
+
+    for item in value:
+        if isinstance(item, (Mapping, list)):
+            lines.append(f"{prefix}-")
+            lines.extend(_yaml_lines(item, indent + 2))
+        else:
+            lines.append(f"{prefix}- {_scalar(item)}")
+
+    return lines
+
+
+def _scalar(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if value is None:
+        return "null"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, str):
+        return _string(value)
+    raise TypeError(f"unsupported YAML value: {value!r}")
+
+
+def _string(value: str) -> str:
+    return json.dumps(value)

dvcgen/inspect.py

@@ -0,0 +1,160 @@
+"""Inspect Python pipeline scripts for dvcgen declarations."""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, Optional, Union
+
+
+@dataclass(frozen=True)
+class PathDeclaration:
+    """A dependency or output declaration extracted from source code."""
+
+    target: str
+    path: str
+    lineno: int
+
+
+@dataclass(frozen=True)
+class ParamDeclaration:
+    """A parameter declaration extracted from source code."""
+
+    target: str
+    name: str
+    default: Any
+    lineno: int
+
+
+@dataclass(frozen=True)
+class SourceDeclarations:
+    """Declarations extracted from a single Python source file."""
+
+    source: str
+    deps: tuple[PathDeclaration, ...]
+    outs: tuple[PathDeclaration, ...]
+    params: tuple[ParamDeclaration, ...]
+
+
+PathLike = Union[str, Path]
+
+
+def inspect_file(path: PathLike) -> SourceDeclarations:
+    """Parse a Python file and return supported dvcgen declarations."""
+    source_path = Path(path)
+    return inspect_source(
+        source_path.read_text(encoding="utf-8"),
+        source=str(source_path),
+    )
+
+
+def inspect_files(paths: Iterable[PathLike]) -> tuple[SourceDeclarations, ...]:
+    """Parse multiple Python files and return declarations per file."""
+    return tuple(inspect_file(path) for path in paths)
+
+
+def inspect_source(source_code: str, source: str = "<string>") -> SourceDeclarations:
+    """Parse Python source and return supported top-level declarations."""
+    tree = ast.parse(source_code, filename=source)
+    deps: list[PathDeclaration] = []
+    outs: list[PathDeclaration] = []
+    params: list[ParamDeclaration] = []
+
+    for statement in tree.body:
+        target = _assignment_target(statement)
+        if target is None:
+            continue
+
+        value = _assignment_value(statement)
+        if not isinstance(value, ast.Call):
+            continue
+
+        call_name = _simple_call_name(value)
+        if call_name == "dep":
+            path_declaration = _path_declaration(target, value)
+            if path_declaration is not None:
+                deps.append(path_declaration)
+        elif call_name == "out":
+            path_declaration = _path_declaration(target, value)
+            if path_declaration is not None:
+                outs.append(path_declaration)
+        elif call_name == "param":
+            param_declaration = _param_declaration(target, value)
+            if param_declaration is not None:
+                params.append(param_declaration)
+
+    return SourceDeclarations(
+        source=source,
+        deps=tuple(deps),
+        outs=tuple(outs),
+        params=tuple(params),
+    )
+
+
+def _assignment_target(statement: ast.stmt) -> Optional[str]:
+    if isinstance(statement, ast.Assign) and len(statement.targets) == 1:
+        target = statement.targets[0]
+    elif isinstance(statement, ast.AnnAssign) and statement.simple:
+        target = statement.target
+    else:
+        return None
+
+    if isinstance(target, ast.Name):
+        return target.id
+    return None
+
+
+def _assignment_value(statement: ast.stmt) -> Optional[ast.expr]:
+    if isinstance(statement, ast.Assign):
+        return statement.value
+    if isinstance(statement, ast.AnnAssign):
+        return statement.value
+    return None
+
+
+def _simple_call_name(call: ast.Call) -> Optional[str]:
+    if isinstance(call.func, ast.Name):
+        return call.func.id
+    return None
+
+
+def _path_declaration(target: str, call: ast.Call) -> Optional[PathDeclaration]:
+    if len(call.args) != 1 or call.keywords:
+        return None
+
+    path = _literal(call.args[0])
+    if not isinstance(path, str):
+        return None
+
+    return PathDeclaration(target=target, path=path, lineno=call.lineno)
+
+
+def _param_declaration(target: str, call: ast.Call) -> Optional[ParamDeclaration]:
+    if len(call.args) != 2 or call.keywords:
+        return None
+
+    name = _literal(call.args[0])
+    if not isinstance(name, str):
+        return None
+
+    default = _literal(call.args[1])
+    if default is _UNSUPPORTED:
+        return None
+
+    return ParamDeclaration(
+        target=target,
+        name=name,
+        default=default,
+        lineno=call.lineno,
+    )
+
+
+_UNSUPPORTED = object()
+
+
+def _literal(node: ast.AST) -> Any:
+    try:
+        return ast.literal_eval(node)
+    except (ValueError, TypeError):
+        return _UNSUPPORTED

dvcgen-0.2.0.dist-info/METADATA

@@ -0,0 +1,175 @@
+Metadata-Version: 2.4
+Name: dvcgen
+Version: 0.2.0
+Summary: Generate DVC pipeline files from Python declarations.
+Author: pillyshi
+License: MIT
+Keywords: dvc,params,pipeline,yaml
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Build Tools
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# dvcgen
+
+Write your DVC pipeline once, in Python.
+
+`dvcgen` is an early-stage command-line tool for generating DVC pipeline files
+from lightweight declarations embedded in Python pipeline scripts.
+
+## Current Status
+
+Implemented:
+
+- A Python package named `dvcgen`
+- A `dvcgen` console command
+- CLI argument parsing for pipeline script paths
+- CLI input validation and overwrite protection
+- Public declaration helpers: `dep()`, `out()`, and `param()`
+- Python script inspection for top-level literal declarations
+- `dvc.yaml` generation
+- `params.yaml` generation
+
+## Installation
+
+```bash
+uv tool install dvcgen
+```
+
+Or run without installing:
+
+```bash
+uvx dvcgen --help
+```
+
+## Usage
+
+Show CLI help:
+
+```bash
+dvcgen --help
+```
+
+Generate DVC files from one or more Python pipeline scripts:
+
+```bash
+dvcgen pipeline/*.py
+```
+
+The command writes `dvc.yaml` and `params.yaml` in the current directory.
+Stage names are derived from input Python filenames. For example,
+`pipeline/train.py` becomes the `train` stage.
+
+By default, `dvcgen` refuses to overwrite existing `dvc.yaml` or `params.yaml`
+files. Use `--force` when you intentionally want to replace them:
+
+```bash
+dvcgen --force pipeline/*.py
+```
+
+Write files to another directory with `--output-dir`:
+
+```bash
+dvcgen --output-dir generated pipeline/*.py
+```
+
+Bad inputs fail with an error message and a non-zero exit code. Successful runs
+print the files that were written.
+
+Inspect declarations from Python without executing the pipeline script:
+
+```python
+from dvcgen.inspect import inspect_file
+
+declarations = inspect_file("pipeline/train.py")
+print(declarations.deps)
+print(declarations.outs)
+print(declarations.params)
+```
+
+## Release
+
+Publishing is intentionally manual while the project is early stage. Build and
+validate artifacts before uploading anything:
+
+```bash
+uv run python -m build
+uv run twine check dist/*
+```
+
+Use TestPyPI first when rehearsing a release. Create a TestPyPI API token, then
+upload with the token as the password:
+
+```bash
+uv run twine upload --repository testpypi dist/*
+```
+
+Use the production PyPI repository only when the version, changelog, and package
+name decision are ready:
+
+```bash
+uv run twine upload dist/*
+```
+
+For both repositories, use `__token__` as the username and the repository API
+token as the password. Avoid committing tokens or storing them in project files.
+
+Before the first production upload, decide whether to publish the current
+minimal release to reserve the `dvcgen` package name on PyPI. Once a version is
+uploaded to PyPI or TestPyPI, that exact version cannot be uploaded again; bump
+the version before retrying with changed artifacts.
+
+## Planned MVP
+
+The intended MVP is:
+
+1. Pipeline scripts declare dependencies, outputs, and parameters in Python.
+2. `dvcgen` inspects those declarations without executing the scripts.
+3. `dvcgen` writes `dvc.yaml` and `params.yaml`.
+
+Example API:
+
+```python
+from dvcgen import dep, out, param
+
+TRAIN_DATA = dep("data/processed.csv")
+MODEL = out("models/model.pkl")
+
+LR = param("train.lr", 0.001)
+```
+
+Running:
+
+```bash
+dvcgen pipeline/train.py
+```
+
+Generates `dvc.yaml`:
+
+```yaml
+"stages":
+  "train":
+    "cmd": "python pipeline/train.py"
+    "deps":
+      - "pipeline/train.py"
+      - "data/processed.csv"
+    "outs":
+      - "models/model.pkl"
+    "params":
+      - "train.lr"
+```
+
+And `params.yaml`:
+
+```yaml
+"train":
+  "lr": 0.001
+```

dvcgen-0.2.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+dvcgen/__init__.py,sha256=Mlag_PpW7bf5LYn21xI__c3VgGEhT4sUXYLzOmxRK8w,460
+dvcgen/cli.py,sha256=9kTsfZB0do6IydRHDo6UhSWQItiXUA2H_UbA-Q2o_Kw,3718
+dvcgen/generate.py,sha256=jB0Wcm3RVvClXOpTIUiIzYoC7F_x6YGo1tt5b1lYDmc,4450
+dvcgen/inspect.py,sha256=hU5DH_i3AYJIYuGw9Izmbj8wQGNYjuQdA-r6254KC20,4365
+dvcgen-0.2.0.dist-info/METADATA,sha256=vkLfDAi2peIR8h1Bi5QVjWRYfvHG3HdE3N0gS0-xoXI,4141
+dvcgen-0.2.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+dvcgen-0.2.0.dist-info/entry_points.txt,sha256=94FKuP8KZnWqZT3IOLj1TIrd4O_w8ycrt7dR_sJSUOk,43
+dvcgen-0.2.0.dist-info/RECORD,,

dvcgen-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

dvcgen-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dvcgen = dvcgen.cli:main