pycairn 0.0.1__tar.gz

pycairn-0.0.1/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.3
+Name: pycairn
+Version: 0.0.1
+Summary: A simple and lightweight pipeline manifest writer tool for Python.
+Author: Shekhrozx
+Author-email: Shekhrozx <shekhrozx@gmail.com>
+Requires-Dist: pydantic>=2.13.4
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# PyCairn (A simple and lightweight pipeline manifest writer tool for Python)
+
+This is a simple and lightweight pipeline manifest writer tool for Python. It allows you to easily track the steps of your data processing pipeline, including inputs, outputs, parameters, and metrics.
+
+## Installation
+You can install PyCairn using pip:
+
+```bash
+pip install pycairn
+```
+
+## Usage
+
+```python
+from pycairn import Artifact, Cairn
+
+cairn = Cairn(pipeline="etl", run_id="2026-06-23T01", path="2026-06-23T01.json")
+
+# Step 1: extract
+with cairn.step("extract", params={"source": "api/v2"}) as s:
+    out = run_extract()  # writes data/raw.parquet
+    art = Artifact.from_path("/path/to/file", type="file-type", rows=len(out))
+    s.outputs.append(art)
+    s.metrics["rows"] = len(out)
+
+# Step 2: transform
+prev = cairn.output_of("extract")[0].path
+with cairn.step("transform", inputs=[prev]) as s:
+    df = run_transform(prev)  # writes data/clean.parquet
+    s.outputs.append(Artifact.from_path("data/clean.parquet", type="parquet", rows=len(df)))
+    s.metrics["null_rate"] = 0.02
+
+...
+```
+
+## License
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details
+
+## Contributing
+Contributions are welcome! Please feel free to submit issues or pull requests.
+
+## Acknowledgements
+- Inspired by the need for a simple and effective way to track data processing pipelines in Python.

pycairn-0.0.1/README.md

@@ -0,0 +1,43 @@
+# PyCairn (A simple and lightweight pipeline manifest writer tool for Python)
+
+This is a simple and lightweight pipeline manifest writer tool for Python. It allows you to easily track the steps of your data processing pipeline, including inputs, outputs, parameters, and metrics.
+
+## Installation
+You can install PyCairn using pip:
+
+```bash
+pip install pycairn
+```
+
+## Usage
+
+```python
+from pycairn import Artifact, Cairn
+
+cairn = Cairn(pipeline="etl", run_id="2026-06-23T01", path="2026-06-23T01.json")
+
+# Step 1: extract
+with cairn.step("extract", params={"source": "api/v2"}) as s:
+    out = run_extract()  # writes data/raw.parquet
+    art = Artifact.from_path("/path/to/file", type="file-type", rows=len(out))
+    s.outputs.append(art)
+    s.metrics["rows"] = len(out)
+
+# Step 2: transform
+prev = cairn.output_of("extract")[0].path
+with cairn.step("transform", inputs=[prev]) as s:
+    df = run_transform(prev)  # writes data/clean.parquet
+    s.outputs.append(Artifact.from_path("data/clean.parquet", type="parquet", rows=len(df)))
+    s.metrics["null_rate"] = 0.02
+
+...
+```
+
+## License
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details
+
+## Contributing
+Contributions are welcome! Please feel free to submit issues or pull requests.
+
+## Acknowledgements
+- Inspired by the need for a simple and effective way to track data processing pipelines in Python.

pycairn-0.0.1/pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "pycairn"
+version = "0.0.1"
+description = "A simple and lightweight pipeline manifest writer tool for Python."
+readme = "README.md"
+authors = [
+    { name = "Shekhrozx", email = "shekhrozx@gmail.com" }
+]
+requires-python = ">=3.10"
+dependencies = [
+    "pydantic>=2.13.4",
+]
+
+[[tool.mypy.overrides]]
+module = "pycairn"
+ignore_missing_imports = true
+
+[build-system]
+requires = ["uv_build>=0.11.6,<0.12.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+test = [
+    "pytest>=9.1.1",
+]
+

pycairn-0.0.1/src/pycairn/__init__.py

@@ -0,0 +1,10 @@
+from ._models import Manifest, Step, Artifact
+from .cairn import Cairn
+
+
+__all__ = [
+    "Artifact",
+    "Manifest",
+    "Step",
+    "Cairn"
+]

pycairn-0.0.1/src/pycairn/_models.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+from pathlib import Path
+from pydantic import BaseModel, Field
+
+from pycairn import utils
+
+
+class Artifact(BaseModel):
+    path: str
+    type: str | None = None          # "parquet", "model", "csv"...
+    bytes: int | None = None
+    sha256: str | None = None
+    meta: dict[str, Any] = Field(default_factory=dict)  # rows, schema, etc.
+
+    @classmethod
+    def from_path(cls, path: str | Path, type: str | None = None, **meta: Any) -> "Artifact":
+        """
+        Create an Artifact instance from a file path.
+
+        Args:
+            path (str | Path): The file path.
+            type (str | None): The type of the artifact (e.g., "parquet", "model", "csv").
+            **meta (Any): Additional metadata.
+
+        Returns:
+            Artifact: The created Artifact instance.
+        """
+
+        p = Path(path)
+        info: dict[str, Any] = {"path": str(p)}
+        if p.exists() and p.is_file():
+            info |= {"bytes": p.stat().st_size, "sha256": utils.sha256(p)}
+        return cls(**info, type=type, meta=meta)
+
+
+class Step(BaseModel):
+    name: str
+    status: utils.Status = utils.Status.pending          # pending|running|success|failed|skipped
+    started_at: str | None = None
+    ended_at: str | None = None
+    duration_s: float | None = None
+    inputs: list[str] = Field(default_factory=list)
+    outputs: list[Artifact] = Field(default_factory=list)
+    metrics: dict[str, Any] = Field(default_factory=dict)
+    params: dict[str, Any] = Field(default_factory=dict)
+    error: str | None = None
+
+    def add_artifact(self, artifact: Artifact) -> None:
+        """
+        Add an artifact to the step's outputs.
+
+        Args:
+            artifact (Artifact): The artifact to add.
+        """
+        if not isinstance(artifact, Artifact):
+            raise TypeError(f"Expected Artifact, got {type(artifact).__name__}")
+
+        if artifact in self.inputs:
+            raise ValueError(f"Artifact with path '{artifact.path}' is already listed as an input.")
+
+        self.outputs.append(artifact)
+        return None
+
+    def running(self, inputs: list[str] | None = None, params: dict | None = None) -> None:
+        """
+        Mark the step as running and record the start time.
+
+        Args:
+            inputs (list[str] | None): The input paths for the step.
+            params (dict | None): The parameters for the step.
+        """
+        self.status = utils.Status.running
+        self.started_at = utils.now_iso()
+        self.inputs = inputs or []
+        self.params = params or {}
+        return None
+
+    def failed(self, error: str) -> None:
+        """
+        Mark the step as failed and record the error message.
+
+        Args:
+            error (str): The error message.
+        """
+        self.status = utils.Status.failed
+        self.error = error
+        return None
+
+    def success(self) -> None:
+        """Mark the step as successful."""
+        self.status = utils.Status.success
+        return None
+
+    def skipped(self) -> None:
+        """Mark the step as skipped."""
+        self.status = utils.Status.skipped
+        return None
+
+    def end(self, start: datetime) -> None:
+        """
+        Record the end time and calculate the duration of the step.
+
+        Args:
+            start (datetime): The start time of the step.
+        """
+        self.ended_at = utils.now_iso()
+        self.duration_s = (utils.now() - start).total_seconds()
+        return None
+
+
+class Manifest(BaseModel):
+    run_id: str
+    pipeline: str
+    created_at: str = Field(default_factory=utils.now_iso)
+    status: utils.Status = utils.Status.running          # running|success|failed
+    steps: list[Step] = Field(default_factory=list)
+
+    def find_step(self, name: str) -> Step | None:
+        """
+        Find a step by name.
+
+        Args:
+            name (str): The name of the step.
+
+        Returns:
+            Step | None: The found step or None if not found.
+        """
+        return next((s for s in self.steps if s.name == name), None)
+
+    def get_or_create_step(self, name: str) -> Step:
+        """
+        Get an existing step by name or create a new one if it doesn't exist.
+
+        Args:
+            name (str): The name of the step.
+
+        Returns:
+            Step: The found or newly created step.
+        """
+        step = self.find_step(name)
+        if step is None:
+            step = Step(name=name)
+            self.steps.append(step)
+        return step
+
+    def output_of(self, step_name: str) -> list[Artifact]:
+        """
+        Get the outputs of a specific step by name.
+
+        Args:
+            step_name (str): The name of the step.
+
+        Returns:
+            list[Artifact]: A list of artifacts produced by the step.
+        """
+        step = self.find_step(step_name)
+        return step.outputs if step else []
+
+    def failed(self) -> None:
+        """Mark the manifest status as failed."""
+        self.status = utils.Status.failed
+        return None
+
+    def success(self) -> None:
+        """Mark the manifest status as success."""
+        self.status = utils.Status.success
+        return None
+
+    def is_all_done(self) -> bool:
+        """Check if all steps are done (either success or skipped)."""
+        return all(s.status in utils.SUCCESS_STATUSES for s in self.steps)

pycairn-0.0.1/src/pycairn/cairn.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import os
+import traceback
+from pathlib import Path
+from contextlib import contextmanager
+
+from pycairn import Artifact, Manifest, utils
+
+
+class Cairn:
+    def __init__(self, pipeline: str, run_id: str, path: str | Path):
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        if self.path.exists():
+            self.manifest = Manifest.model_validate_json(self.path.read_text())
+        else:
+            self.manifest = Manifest(run_id=run_id, pipeline=pipeline)
+        self._save()
+
+    def _save(self) -> None:
+        # atomic write: tmp -> fsync -> rename
+        tmp = self.path.with_suffix(self.path.suffix + ".tmp")
+        tmp.write_text(self.manifest.model_dump_json(indent=2))
+        os.replace(tmp, self.path)
+
+    @contextmanager
+    def step(self, name: str, inputs: list[str] | None = None, params: dict | None = None):
+        step = self.manifest.get_or_create_step(name)
+
+        step.running(inputs=inputs, params=params)
+        self._save()
+
+        start = utils.now()
+        try:
+            yield step                       # caller fills outputs/metrics
+            step.success()
+        except Exception:
+            step.failed(traceback.format_exc())
+            self.manifest.failed()
+            self._save()
+            raise
+        finally:
+            step.end(start)
+            self._save()
+
+        # mark whole run done if last step succeeded and nothing failed
+        if self.manifest.is_all_done():
+            self.manifest.success()
+            self._save()
+
+    def output_of(self, step_name: str) -> list[Artifact]:
+        return self.manifest.output_of(step_name)

pycairn-0.0.1/src/pycairn/utils.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from enum import StrEnum
+import hashlib
+from pathlib import Path
+from datetime import datetime, timezone
+
+
+def now() -> datetime:
+    return datetime.now(timezone.utc)
+
+def now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def sha256(path: Path, chunk: int = 1 << 20) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for block in iter(lambda: f.read(chunk), b""):
+            h.update(block)
+    return h.hexdigest()
+
+
+class Status(StrEnum):
+    pending = "pending"
+    running = "running"
+    success = "success"
+    failed = "failed"
+    skipped = "skipped"
+
+SUCCESS_STATUSES = (Status.success, Status.skipped)