dvcgen 0.2.0__tar.gz

dvcgen-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,60 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    name: Test Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --locked --dev
+
+      - name: Run tests
+        run: uv run pytest
+
+  quality:
+    name: Quality and package
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --locked --dev
+
+      - name: Run ruff
+        run: uv run ruff check .
+
+      - name: Run mypy
+        run: uv run mypy src tests
+
+      - name: Build package
+        run: uv build

dvcgen-0.2.0/.gitignore

@@ -0,0 +1,8 @@
+.venv/
+.uv-cache/
+__pycache__/
+*.py[cod]
+.pytest_cache/
+dist/
+build/
+*.egg-info/

dvcgen-0.2.0/PKG-INFO

@@ -0,0 +1,175 @@
+Metadata-Version: 2.4
+Name: dvcgen
+Version: 0.2.0
+Summary: Generate DVC pipeline files from Python declarations.
+Author: pillyshi
+License: MIT
+Keywords: dvc,params,pipeline,yaml
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Build Tools
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# dvcgen
+
+Write your DVC pipeline once, in Python.
+
+`dvcgen` is an early-stage command-line tool for generating DVC pipeline files
+from lightweight declarations embedded in Python pipeline scripts.
+
+## Current Status
+
+Implemented:
+
+- A Python package named `dvcgen`
+- A `dvcgen` console command
+- CLI argument parsing for pipeline script paths
+- CLI input validation and overwrite protection
+- Public declaration helpers: `dep()`, `out()`, and `param()`
+- Python script inspection for top-level literal declarations
+- `dvc.yaml` generation
+- `params.yaml` generation
+
+## Installation
+
+```bash
+uv tool install dvcgen
+```
+
+Or run without installing:
+
+```bash
+uvx dvcgen --help
+```
+
+## Usage
+
+Show CLI help:
+
+```bash
+dvcgen --help
+```
+
+Generate DVC files from one or more Python pipeline scripts:
+
+```bash
+dvcgen pipeline/*.py
+```
+
+The command writes `dvc.yaml` and `params.yaml` in the current directory.
+Stage names are derived from input Python filenames. For example,
+`pipeline/train.py` becomes the `train` stage.
+
+By default, `dvcgen` refuses to overwrite existing `dvc.yaml` or `params.yaml`
+files. Use `--force` when you intentionally want to replace them:
+
+```bash
+dvcgen --force pipeline/*.py
+```
+
+Write files to another directory with `--output-dir`:
+
+```bash
+dvcgen --output-dir generated pipeline/*.py
+```
+
+Bad inputs fail with an error message and a non-zero exit code. Successful runs
+print the files that were written.
+
+Inspect declarations from Python without executing the pipeline script:
+
+```python
+from dvcgen.inspect import inspect_file
+
+declarations = inspect_file("pipeline/train.py")
+print(declarations.deps)
+print(declarations.outs)
+print(declarations.params)
+```
+
+## Release
+
+Publishing is intentionally manual while the project is early stage. Build and
+validate artifacts before uploading anything:
+
+```bash
+uv run python -m build
+uv run twine check dist/*
+```
+
+Use TestPyPI first when rehearsing a release. Create a TestPyPI API token, then
+upload with the token as the password:
+
+```bash
+uv run twine upload --repository testpypi dist/*
+```
+
+Use the production PyPI repository only when the version, changelog, and package
+name decision are ready:
+
+```bash
+uv run twine upload dist/*
+```
+
+For both repositories, use `__token__` as the username and the repository API
+token as the password. Avoid committing tokens or storing them in project files.
+
+Before the first production upload, decide whether to publish the current
+minimal release to reserve the `dvcgen` package name on PyPI. Once a version is
+uploaded to PyPI or TestPyPI, that exact version cannot be uploaded again; bump
+the version before retrying with changed artifacts.
+
+## Planned MVP
+
+The intended MVP is:
+
+1. Pipeline scripts declare dependencies, outputs, and parameters in Python.
+2. `dvcgen` inspects those declarations without executing the scripts.
+3. `dvcgen` writes `dvc.yaml` and `params.yaml`.
+
+Example API:
+
+```python
+from dvcgen import dep, out, param
+
+TRAIN_DATA = dep("data/processed.csv")
+MODEL = out("models/model.pkl")
+
+LR = param("train.lr", 0.001)
+```
+
+Running:
+
+```bash
+dvcgen pipeline/train.py
+```
+
+Generates `dvc.yaml`:
+
+```yaml
+"stages":
+  "train":
+    "cmd": "python pipeline/train.py"
+    "deps":
+      - "pipeline/train.py"
+      - "data/processed.csv"
+    "outs":
+      - "models/model.pkl"
+    "params":
+      - "train.lr"
+```
+
+And `params.yaml`:
+
+```yaml
+"train":
+  "lr": 0.001
+```

dvcgen-0.2.0/README.md

@@ -0,0 +1,155 @@
+# dvcgen
+
+Write your DVC pipeline once, in Python.
+
+`dvcgen` is an early-stage command-line tool for generating DVC pipeline files
+from lightweight declarations embedded in Python pipeline scripts.
+
+## Current Status
+
+Implemented:
+
+- A Python package named `dvcgen`
+- A `dvcgen` console command
+- CLI argument parsing for pipeline script paths
+- CLI input validation and overwrite protection
+- Public declaration helpers: `dep()`, `out()`, and `param()`
+- Python script inspection for top-level literal declarations
+- `dvc.yaml` generation
+- `params.yaml` generation
+
+## Installation
+
+```bash
+uv tool install dvcgen
+```
+
+Or run without installing:
+
+```bash
+uvx dvcgen --help
+```
+
+## Usage
+
+Show CLI help:
+
+```bash
+dvcgen --help
+```
+
+Generate DVC files from one or more Python pipeline scripts:
+
+```bash
+dvcgen pipeline/*.py
+```
+
+The command writes `dvc.yaml` and `params.yaml` in the current directory.
+Stage names are derived from input Python filenames. For example,
+`pipeline/train.py` becomes the `train` stage.
+
+By default, `dvcgen` refuses to overwrite existing `dvc.yaml` or `params.yaml`
+files. Use `--force` when you intentionally want to replace them:
+
+```bash
+dvcgen --force pipeline/*.py
+```
+
+Write files to another directory with `--output-dir`:
+
+```bash
+dvcgen --output-dir generated pipeline/*.py
+```
+
+Bad inputs fail with an error message and a non-zero exit code. Successful runs
+print the files that were written.
+
+Inspect declarations from Python without executing the pipeline script:
+
+```python
+from dvcgen.inspect import inspect_file
+
+declarations = inspect_file("pipeline/train.py")
+print(declarations.deps)
+print(declarations.outs)
+print(declarations.params)
+```
+
+## Release
+
+Publishing is intentionally manual while the project is early stage. Build and
+validate artifacts before uploading anything:
+
+```bash
+uv run python -m build
+uv run twine check dist/*
+```
+
+Use TestPyPI first when rehearsing a release. Create a TestPyPI API token, then
+upload with the token as the password:
+
+```bash
+uv run twine upload --repository testpypi dist/*
+```
+
+Use the production PyPI repository only when the version, changelog, and package
+name decision are ready:
+
+```bash
+uv run twine upload dist/*
+```
+
+For both repositories, use `__token__` as the username and the repository API
+token as the password. Avoid committing tokens or storing them in project files.
+
+Before the first production upload, decide whether to publish the current
+minimal release to reserve the `dvcgen` package name on PyPI. Once a version is
+uploaded to PyPI or TestPyPI, that exact version cannot be uploaded again; bump
+the version before retrying with changed artifacts.
+
+## Planned MVP
+
+The intended MVP is:
+
+1. Pipeline scripts declare dependencies, outputs, and parameters in Python.
+2. `dvcgen` inspects those declarations without executing the scripts.
+3. `dvcgen` writes `dvc.yaml` and `params.yaml`.
+
+Example API:
+
+```python
+from dvcgen import dep, out, param
+
+TRAIN_DATA = dep("data/processed.csv")
+MODEL = out("models/model.pkl")
+
+LR = param("train.lr", 0.001)
+```
+
+Running:
+
+```bash
+dvcgen pipeline/train.py
+```
+
+Generates `dvc.yaml`:
+
+```yaml
+"stages":
+  "train":
+    "cmd": "python pipeline/train.py"
+    "deps":
+      - "pipeline/train.py"
+      - "data/processed.csv"
+    "outs":
+      - "models/model.pkl"
+    "params":
+      - "train.lr"
+```
+
+And `params.yaml`:
+
+```yaml
+"train":
+  "lr": 0.001
+```

dvcgen-0.2.0/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dvcgen"
+version = "0.2.0"
+description = "Generate DVC pipeline files from Python declarations."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [
+  { name = "pillyshi" },
+]
+keywords = ["dvc", "pipeline", "params", "yaml"]
+classifiers = [
+  "Development Status :: 2 - Pre-Alpha",
+  "Environment :: Console",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Software Development :: Build Tools",
+]
+
+[project.scripts]
+dvcgen = "dvcgen.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dvcgen"]
+
+[dependency-groups]
+dev = [
+    "build>=1.5.0",
+    "mypy>=1.19.1",
+    "pytest>=8.4.2",
+    "ruff>=0.15.15",
+    "twine>=6.2.0",
+]

dvcgen-0.2.0/src/dvcgen/__init__.py

@@ -0,0 +1,21 @@
+"""Generate DVC pipeline files from Python declarations."""
+
+__version__ = "0.1.0"
+
+
+def dep(path):
+    """Declare a pipeline dependency and return its runtime value."""
+    return path
+
+
+def out(path):
+    """Declare a pipeline output and return its runtime value."""
+    return path
+
+
+def param(name, default):
+    """Declare a pipeline parameter and return its default runtime value."""
+    return default
+
+
+__all__ = ["__version__", "dep", "out", "param"]

dvcgen-0.2.0/src/dvcgen/cli.py

@@ -0,0 +1,119 @@
+"""Command-line interface for dvcgen."""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Sequence
+from pathlib import Path
+import sys
+from typing import Optional, TextIO
+
+from dvcgen import __version__
+from dvcgen.generate import write_files
+from dvcgen.inspect import inspect_files
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="dvcgen",
+        description="Generate dvc.yaml and params.yaml from Python pipeline scripts.",
+    )
+    parser.add_argument(
+        "scripts",
+        nargs="*",
+        help="Python pipeline scripts to inspect.",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        default=".",
+        help="Directory where dvc.yaml and params.yaml are written. Defaults to the current directory.",
+    )
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="Overwrite existing dvc.yaml and params.yaml files.",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+    return parser
+
+
+def main(
+    argv: Optional[Sequence[str]] = None,
+    stdout: Optional[TextIO] = None,
+    stderr: Optional[TextIO] = None,
+) -> int:
+    stdout = sys.stdout if stdout is None else stdout
+    stderr = sys.stderr if stderr is None else stderr
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if not args.scripts:
+        print("dvcgen: error: provide at least one Python pipeline script", file=stderr)
+        print("Try 'dvcgen --help' for usage.", file=stderr)
+        return 2
+
+    script_paths = tuple(Path(script) for script in args.scripts)
+    output_dir = Path(args.output_dir)
+    dvc_path = output_dir / "dvc.yaml"
+    params_path = output_dir / "params.yaml"
+
+    validation_message = _validation_error(
+        script_paths,
+        output_dir,
+        (dvc_path, params_path),
+        args.force,
+    )
+    if validation_message is not None:
+        print(f"dvcgen: error: {validation_message}", file=stderr)
+        return 2
+
+    try:
+        declarations = inspect_files(script_paths)
+        write_files(declarations, dvc_path=dvc_path, params_path=params_path)
+    except SyntaxError as syntax_error:
+        print(
+            f"dvcgen: error: failed to parse {syntax_error.filename}: {syntax_error.msg}",
+            file=stderr,
+        )
+        return 2
+    except OSError as os_error:
+        print(f"dvcgen: error: {os_error}", file=stderr)
+        return 2
+    except ValueError as value_error:
+        print(f"dvcgen: error: {value_error}", file=stderr)
+        return 2
+
+    print(f"Wrote {dvc_path} and {params_path}", file=stdout)
+    return 0
+
+
+def _validation_error(
+    script_paths: Sequence[Path],
+    output_dir: Path,
+    output_paths: Sequence[Path],
+    force: bool,
+) -> Optional[str]:
+    for script_path in script_paths:
+        if not script_path.exists():
+            return f"input script not found: {script_path}"
+        if not script_path.is_file():
+            return f"input script is not a file: {script_path}"
+        if script_path.suffix != ".py":
+            return f"input script must be a .py file: {script_path}"
+
+    if output_dir.exists() and not output_dir.is_dir():
+        return f"output directory is not a directory: {output_dir}"
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    if not force:
+        existing_paths = [path for path in output_paths if path.exists()]
+        if existing_paths:
+            joined_paths = ", ".join(str(path) for path in existing_paths)
+            return f"refusing to overwrite existing file(s): {joined_paths}; use --force to replace them"
+
+    return None