pypdown 0.1.0__tar.gz

pypdown-0.1.0/PKG-INFO

@@ -0,0 +1,89 @@
+Metadata-Version: 2.1
+Name: pypdown
+Version: 0.1.0
+Summary: A Pydantic model-based approach to data pipelining with file I/O linting.
+Author-Email: Louis Maddox <louismmx@gmail.com>
+License: MIT
+Requires-Python: >=3.12
+Requires-Dist: pydantic>=2.8.2
+Description-Content-Type: text/markdown
+
+# pypdown
+
+A Pydantic model-based approach to data pipelining with file I/O linting.
+
+[![PyPI version](https://badge.fury.io/py/pypdown.svg)](https://badge.fury.io/py/pypdown)
+[![Python Versions](https://img.shields.io/pypi/pyversions/pypdown.svg)](https://pypi.org/project/pypdown/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Documentation](https://img.shields.io/badge/docs-pypdown.vercel.app-blue)](https://pypdown.vercel.app/)
+[![CI Status](https://github.com/lmmx/pypdown/actions/workflows/ci.yml/badge.svg)](https://github.com/lmmx/pypdown/actions/workflows/ci.yml)
+[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/lmmx/pypdown/master.svg)](https://results.pre-commit.ci/latest/github/lmmx/pypdown/master)
+
+## Features
+
+- Pydantic model-based approach to data pipelining
+- File I/O linting for robust pipeline execution
+- Easy-to-use API for defining and running pipeline steps
+- Support for callback functions and keyword argument-based file paths
+
+## Installation
+
+```bash
+pip install pypdown
+```
+
+## Quick Start
+
+```python
+from pypdown import run_step
+from pypdown.models import Step
+from pydantic import BaseModel
+from pathlib import Path
+
+class StepParams(BaseModel):
+    input_file: Path = "input.txt"
+    output_file: Path = "output.txt"
+    final_file: Path = "final.txt"
+
+def process_input(input_file: Path, output_file: Path, config: StepParams):
+    """Process input file and create output file."""
+    output_file.write_text(input_file.read_text().upper())
+
+def finalize_output(output_file: Path, final_file: Path, config: StepParams):
+    """Process output file and create final file."""
+    final_file.write_text(f"Processed: {output_file.read_text()}")
+
+config = StepParams()
+
+# Define your pipeline tasks
+tasks = [
+    {
+        "src": {"input_file": config.input_file},
+        "dst": {"output_file": config.output_file},
+        "fn": process_input,
+    },
+    {
+        "src": {"output_file": config.output_file},
+        "dst": {"final_file": config.final_file},
+        "fn": finalize_output,
+    },
+]
+
+# Create a Step
+step = Step(name="Example Pipeline Step", tasks=tasks, config=config)
+
+# Run the step
+run_step(step)
+```
+
+## Documentation
+
+For full documentation, please visit [pypdown.vercel.app](https://pypdown.vercel.app/).
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

pypdown-0.1.0/README.md

@@ -0,0 +1,79 @@
+# pypdown
+
+A Pydantic model-based approach to data pipelining with file I/O linting.
+
+[![PyPI version](https://badge.fury.io/py/pypdown.svg)](https://badge.fury.io/py/pypdown)
+[![Python Versions](https://img.shields.io/pypi/pyversions/pypdown.svg)](https://pypi.org/project/pypdown/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Documentation](https://img.shields.io/badge/docs-pypdown.vercel.app-blue)](https://pypdown.vercel.app/)
+[![CI Status](https://github.com/lmmx/pypdown/actions/workflows/ci.yml/badge.svg)](https://github.com/lmmx/pypdown/actions/workflows/ci.yml)
+[![pre-commit.ci status](https://results.pre-commit.ci/badge/github/lmmx/pypdown/master.svg)](https://results.pre-commit.ci/latest/github/lmmx/pypdown/master)
+
+## Features
+
+- Pydantic model-based approach to data pipelining
+- File I/O linting for robust pipeline execution
+- Easy-to-use API for defining and running pipeline steps
+- Support for callback functions and keyword argument-based file paths
+
+## Installation
+
+```bash
+pip install pypdown
+```
+
+## Quick Start
+
+```python
+from pypdown import run_step
+from pypdown.models import Step
+from pydantic import BaseModel
+from pathlib import Path
+
+class StepParams(BaseModel):
+    input_file: Path = "input.txt"
+    output_file: Path = "output.txt"
+    final_file: Path = "final.txt"
+
+def process_input(input_file: Path, output_file: Path, config: StepParams):
+    """Process input file and create output file."""
+    output_file.write_text(input_file.read_text().upper())
+
+def finalize_output(output_file: Path, final_file: Path, config: StepParams):
+    """Process output file and create final file."""
+    final_file.write_text(f"Processed: {output_file.read_text()}")
+
+config = StepParams()
+
+# Define your pipeline tasks
+tasks = [
+    {
+        "src": {"input_file": config.input_file},
+        "dst": {"output_file": config.output_file},
+        "fn": process_input,
+    },
+    {
+        "src": {"output_file": config.output_file},
+        "dst": {"final_file": config.final_file},
+        "fn": finalize_output,
+    },
+]
+
+# Create a Step
+step = Step(name="Example Pipeline Step", tasks=tasks, config=config)
+
+# Run the step
+run_step(step)
+```
+
+## Documentation
+
+For full documentation, please visit [pypdown.vercel.app](https://pypdown.vercel.app/).
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

pypdown-0.1.0/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = [
+    "pdm-backend",
+]
+build-backend = "pdm.backend"
+
+[project]
+name = "pypdown"
+version = "0.1.0"
+description = "A Pydantic model-based approach to data pipelining with file I/O linting."
+authors = [
+    { name = "Louis Maddox", email = "louismmx@gmail.com" },
+]
+dependencies = [
+    "pydantic>=2.8.2",
+]
+requires-python = ">=3.12"
+readme = "README.md"
+
+[project.license]
+text = "MIT"
+
+[tool.pdm]
+distribution = true
+
+[tool.pdm.dev-dependencies]
+docs = [
+    "mkdocs-material[recommended,imaging]>=9.5.2",
+    "mkdocs-section-index>=0.3.8",
+    "mkdocs>=1.5.3",
+    "mkdocstrings[python]>=0.24.0",
+]
+test = [
+    "pytest>=8.2.2",
+]
+vercel = [
+    "urllib3<2",
+]

pypdown-0.1.0/src/pypdown/__init__.py

@@ -0,0 +1,3 @@
+from .run import run_step
+
+__all__ = ["run_step"]

pypdown-0.1.0/src/pypdown/models.py

@@ -0,0 +1,66 @@
+"""Pydantic models to represent the tasks within a step in a data pipeline."""
+
+from pathlib import Path
+from typing import TypeVar
+from collections.abc import Callable
+
+from pydantic import BaseModel, FilePath, NewPath, OnErrorOmit, TypeAdapter
+
+__all__ = [
+    "AvailableTask",
+    "CompletedTask",
+    "Task",
+    "Step",
+    "AvailableTA",
+    "CompletedTA",
+    "RunContext",
+]
+
+
+class Executable(BaseModel):
+    """All tasks must have an associated function to make them executable."""
+
+    fn: Callable
+
+
+class AvailableTask(Executable):
+    """A task is available when its input files exist and its outputs don't."""
+
+    src: dict[str, FilePath]
+    dst: dict[str, NewPath]
+
+
+class CompletedTask(Executable):
+    """A task is completed when its output files exist, whether inputs exist or not."""
+
+    src: dict[str, Path]
+    dst: dict[str, FilePath]
+
+
+class Task(Executable):
+    """A task has zero or more input files and zero or more output files."""
+
+    src: dict[str, Path]
+    dst: dict[str, Path]
+
+
+C = TypeVar("C", bound=BaseModel)
+
+
+class Step(BaseModel):
+    """A named step in a data pipeline, split up into tasks with specified file I/O."""
+
+    name: str
+    tasks: list[Task]
+    config: C
+
+
+AvailableTA = TypeAdapter(list[OnErrorOmit[AvailableTask]])
+CompletedTA = TypeAdapter(list[OnErrorOmit[CompletedTask]])
+
+
+class RunContext(BaseModel):
+    """The context available to a task runner."""
+
+    step: Step
+    idx: int

pypdown-0.1.0/src/pypdown/run.py

@@ -0,0 +1,55 @@
+"""Control flow using the Pydantic runtime file I/O checks."""
+
+from .models import AvailableTA, AvailableTask, CompletedTA, Step, RunContext
+
+__all__ = ["run_step"]
+
+
+def task_runner(task: AvailableTask, context: RunContext) -> None:
+    print(f"Hello world {task.model_dump(mode='json', exclude='fn')}")
+    task.fn.__call__(**task.src, **task.dst, config=context.step.config)
+
+
+def run_step(step: Step):
+    """Run a pipeline step's tasks based on the availability of task files.
+
+    Tasks are iterated through, and the relevant in/output files' existence existence
+    is checked when the task is reached in the loop (rather than at the start). This
+    means that intermediate files can be created by tasks, and their existence will be
+    checked when those output files become inputs to subsequent tasks.
+
+    If any task's required input files are missing, the step bails out: no further tasks
+    will run.
+    """
+    if step.tasks:
+        print(f"Running step {step.name!r} with {len(step.tasks)} tasks")
+    else:
+        raise ValueError("No tasks were assigned")
+
+    bail = False
+    for idx, task in enumerate(step.tasks):
+        if idx > 0 and not bail:
+            prev_task = step.tasks[idx - 1]
+            prev_completed = CompletedTA.validate_python([prev_task.model_dump()])
+            if not prev_completed:
+                bail = True
+                print("(!) Incomplete previous task detected, bailing")
+        task_repr = " --> ".join(
+            map(str, (task.model_dump(include=["src", "dst"], mode="json").values())),
+        )
+        print(f"\n--- Task {idx + 1} --- {task_repr}")
+        if bail:
+            print("(-) Bailing out of step, skipping task")
+            continue
+
+        available = AvailableTA.validate_python([task.model_dump()])
+        completed = CompletedTA.validate_python([task.model_dump()])
+
+        if available:
+            print("\033[92;1m>>>\033[0m Running available task")
+            task_runner(task=task, context=RunContext(step=step, idx=idx))
+        elif completed:
+            print("(x) Task already completed, skipping")
+        else:
+            print("(!) Task requisite missing, bailing")
+            bail = True

pypdown-0.1.0/tests/long_test.py

@@ -0,0 +1,76 @@
+from pypdown import run_step
+from pypdown.models import Step
+from pydantic import BaseModel
+from pathlib import Path
+
+
+def test_long_example():
+    class StepParams(BaseModel):
+        n1_o: Path = "nil1.out"
+        n2_o: Path = "nil2.out"
+        a_i: Path = "a.in"
+        a_o: Path = "a.out"
+        b_o: Path = "b.out"
+        c_o: Path = "c.out"
+        d_i: Path = "d.in"
+        d_o: Path = "d.out"
+        e_i: Path = "e.in"
+        e_o: Path = "e.out"
+
+    config = StepParams()
+
+    def cb_n1(n1_o: Path, config: StepParams):
+        n1_o.touch()
+        print(f"Touched {n1_o=}")
+
+    def cb_a(a_i: Path, a_o: Path, config: StepParams):
+        assert a_i.exists()
+        a_o.touch()
+        print(f"Touched {a_o=}")
+
+    def cb_b(a_o: Path, b_o: Path, config: StepParams):
+        assert a_o.exists()
+        b_o.touch()
+        print(f"Touched {b_o=}")
+
+    def cb_c(a_o: Path, b_o: Path, c_o: Path, config: StepParams):
+        assert a_o.exists() and b_o.exists()
+        c_o.touch()
+        print(f"Touched {c_o=}")
+
+    def cb_d(d_i: Path, d_o: Path, config: StepParams):
+        assert d_i.exists()
+        d_o.touch()
+        print(f"Touched {d_o=}")
+
+    def cb_e(e_i: Path, e_o: Path, config: StepParams):
+        assert e_i.exists()
+        e_o.touch()
+        print(f"Touched {e_o=}")
+
+    def cb_n2(n2_o: Path, config: StepParams):
+        n2_o.touch()
+        print(f"Touched {n2_o=}")
+
+    task_fields = [
+        ([], ["n1_o"], cb_n1),
+        (["a_i"], ["a_o"], cb_a),
+        (["a_o"], ["b_o"], cb_b),
+        (["a_o", "b_o"], ["c_o"], cb_c),
+        (["d_i"], ["d_o"], cb_d),
+        (["e_i"], ["e_o"], cb_e),
+        ([], ["n2_o"], cb_n2),
+    ]
+
+    # Turn the in/output lists into dicts keyed by config field name with filename values
+    tasks = [
+        {
+            "src": {field: getattr(config, field) for field in inputs},
+            "dst": {field: getattr(config, field) for field in outputs},
+            "fn": func,
+        }
+        for inputs, outputs, func in task_fields
+    ]
+
+    step = Step(name="Demo Step", tasks=tasks, config=config)
+    run_step(step)

pypdown-0.1.0/tests/package_test.py

@@ -0,0 +1,2 @@
+def test_package():
+    pass

pypdown-0.1.0/tests/requirements.txt

@@ -0,0 +1,2 @@
+mypy
+pytest

pypdown-0.1.0/tests/simple_test.py

@@ -0,0 +1,43 @@
+from pypdown import run_step
+from pypdown.models import Step
+from pydantic import BaseModel
+from pathlib import Path
+
+
+def test_simple_example():
+    class StepParams(BaseModel):
+        a1_i: Path = "a1.in"
+        a2_i: Path = "a2.in"
+        a_o: Path = "a.out"
+        b_i: Path = "b.in"
+        b_o: Path = "b.out"
+
+    def cb_a(a1_i: Path, a2_i: Path, a_o: Path, config: StepParams):
+        assert a1_i.exists() and a2_i.exists()
+        a_o.touch()
+        print(f"Touched {a_o=}")
+
+    def cb_b(a_o: Path, b_i: Path, b_o: Path, config: StepParams):
+        assert a_o.exists() and b_i.exists()
+        b_o.touch()
+        print(f"Touched {b_o=}")
+
+    task_fields = [
+        (["a1_i", "a2_i"], ["a_o"], cb_a),
+        (["a_o", "b_i"], ["b_o"], cb_b),
+    ]
+
+    config = StepParams()
+
+    # Turn the in/output lists into dicts keyed by config field name with filename values
+    tasks = [
+        {
+            "src": {field: getattr(config, field) for field in inputs},
+            "dst": {field: getattr(config, field) for field in outputs},
+            "fn": func,
+        }
+        for inputs, outputs, func in task_fields
+    ]
+
+    step = Step(name="Demo Step", tasks=tasks, config=config)
+    run_step(step)