xplainable-preprocessing 0.1.0__tar.gz → 0.2.0__tar.gz

{xplainable_preprocessing-0.1.0 → xplainable_preprocessing-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xplainable-preprocessing
-Version: 0.1.0
+Version: 0.2.0
 Summary: Shared preprocessing pipeline package for xplainable
 Requires-Python: >=3.9
 Requires-Dist: cloudpickle>=3.0

{xplainable_preprocessing-0.1.0 → xplainable_preprocessing-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "xplainable-preprocessing"
-version = "0.1.0"
+version = "0.2.0"
 description = "Shared preprocessing pipeline package for xplainable"
 requires-python = ">=3.9"
 dependencies = [

xplainable_preprocessing-0.2.0/scripts/release.sh

@@ -0,0 +1,292 @@
+#!/bin/bash
+# Simple release script for xplainable-preprocessing
+
+set -e
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Functions
+print_header() {
+    echo -e "${BLUE}$1${NC}"
+    echo "=================================="
+}
+
+print_success() {
+    echo -e "${GREEN}✅ $1${NC}"
+}
+
+print_warning() {
+    echo -e "${YELLOW}⚠️  $1${NC}"
+}
+
+print_error() {
+    echo -e "${RED}❌ $1${NC}"
+}
+
+# Parse arguments
+VERSION=""
+INCREMENT=""
+DRY_RUN=false
+PUBLISH_PYPI=false
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -v|--version)
+            VERSION="$2"
+            shift 2
+            ;;
+        -i|--increment)
+            INCREMENT="$2"
+            shift 2
+            ;;
+        --dry-run)
+            DRY_RUN=true
+            shift
+            ;;
+        --publish)
+            PUBLISH_PYPI=true
+            shift
+            ;;
+        -h|--help)
+            echo "Usage: $0 [OPTIONS]"
+            echo ""
+            echo "Options:"
+            echo "  -v, --version VERSION    Specific version (e.g., 1.2.3)"
+            echo "  -i, --increment TYPE     Auto-increment: major, minor, or patch"
+            echo "      --dry-run           Show what would be done"
+            echo "      --publish           Publish to PyPI after building"
+            echo "  -h, --help              Show this help"
+            echo ""
+            echo "Examples:"
+            echo "  $0 --version 0.2.0"
+            echo "  $0 --increment patch"
+            echo "  $0 --increment minor --publish"
+            echo "  $0 --increment minor --dry-run"
+            exit 0
+            ;;
+        *)
+            print_error "Unknown option: $1"
+            exit 1
+            ;;
+    esac
+done
+
+# Validate arguments
+if [[ -z "$VERSION" && -z "$INCREMENT" ]]; then
+    print_error "Must specify either --version or --increment"
+    echo "Run $0 --help for usage"
+    exit 1
+fi
+
+if [[ -n "$VERSION" && -n "$INCREMENT" ]]; then
+    print_error "Cannot specify both --version and --increment"
+    exit 1
+fi
+
+print_header "🚀 xplainable-preprocessing Release"
+
+# Check if we're in the right directory
+if [[ ! -f "pyproject.toml" ]]; then
+    print_error "Not in project root (no pyproject.toml found)"
+    exit 1
+fi
+
+# Check git status
+print_header "📋 Pre-release Checks"
+
+if ! git rev-parse --git-dir > /dev/null 2>&1; then
+    print_error "Not in a git repository"
+    exit 1
+fi
+
+if [[ -n $(git status --porcelain) ]]; then
+    print_error "Uncommitted changes detected"
+    git status --short
+    exit 1
+fi
+
+CURRENT_BRANCH=$(git branch --show-current)
+if [[ "$CURRENT_BRANCH" != "main" && "$CURRENT_BRANCH" != "master" ]]; then
+    print_warning "Currently on branch '$CURRENT_BRANCH'"
+    read -p "Continue anyway? (y/N): " -n 1 -r
+    echo
+    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+        exit 1
+    fi
+fi
+
+print_success "Git repository is clean"
+
+# Run tests before release
+print_header "🧪 Running Tests"
+if python -m pytest tests/ -q; then
+    print_success "All tests passed"
+else
+    print_error "Tests failed — aborting release"
+    exit 1
+fi
+
+# Get current version from pyproject.toml
+CURRENT_VERSION=$(grep '^version =' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
+
+if [[ -z "$CURRENT_VERSION" ]]; then
+    print_error "Could not determine current version"
+    exit 1
+fi
+
+print_success "Current version: $CURRENT_VERSION"
+
+# Determine new version
+if [[ -n "$VERSION" ]]; then
+    NEW_VERSION="${VERSION#v}"  # Remove 'v' prefix if present
+else
+    # Auto-increment
+    IFS='.' read -ra VERSION_PARTS <<< "$CURRENT_VERSION"
+    MAJOR=${VERSION_PARTS[0]}
+    MINOR=${VERSION_PARTS[1]}
+    PATCH=${VERSION_PARTS[2]}
+
+    case $INCREMENT in
+        major)
+            MAJOR=$((MAJOR + 1))
+            MINOR=0
+            PATCH=0
+            ;;
+        minor)
+            MINOR=$((MINOR + 1))
+            PATCH=0
+            ;;
+        patch)
+            PATCH=$((PATCH + 1))
+            ;;
+        *)
+            print_error "Invalid increment type: $INCREMENT"
+            exit 1
+            ;;
+    esac
+
+    NEW_VERSION="$MAJOR.$MINOR.$PATCH"
+fi
+
+TAG="v$NEW_VERSION"
+
+print_header "📦 Release Plan"
+echo "Current:  $CURRENT_VERSION"
+echo "New:      $NEW_VERSION"
+echo "Tag:      $TAG"
+
+if [[ "$DRY_RUN" == true ]]; then
+    print_header "🔍 DRY RUN - Would perform:"
+    echo "1. Clean build artifacts"
+    echo "2. Update version in pyproject.toml"
+    echo "3. Commit version changes"
+    echo "4. Create git tag $TAG"
+    echo "5. Build package (wheel and sdist)"
+    echo "6. Push changes and tag"
+    if [[ "$PUBLISH_PYPI" == true ]]; then
+        echo "7. Publish package to PyPI"
+        echo "8. Create GitHub release (if gh CLI available)"
+    else
+        echo "7. Create GitHub release (if gh CLI available)"
+    fi
+    exit 0
+fi
+
+# Confirm
+read -p "Create release v$NEW_VERSION? (y/N): " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    print_error "Release cancelled"
+    exit 1
+fi
+
+print_header "🔨 Creating Release"
+
+# Clean up build artifacts
+print_success "Cleaning up build artifacts..."
+rm -rf build/ dist/ *.egg-info/ src/*.egg-info/ || true
+
+# Update version in pyproject.toml
+print_success "Updating version..."
+sed -i.bak "s/^version = .*/version = \"$NEW_VERSION\"/" pyproject.toml && rm pyproject.toml.bak
+print_success "Updated pyproject.toml"
+
+# Commit changes
+print_success "Committing version changes..."
+git add .
+git commit -m "chore: bump version to $NEW_VERSION"
+
+# Create tag
+print_success "Creating tag $TAG..."
+RELEASE_DATE=$(date +%Y-%m-%d)
+git tag -a "$TAG" -m "Release $TAG — $RELEASE_DATE"
+
+# Build package
+print_success "Building package..."
+python -m build
+
+# Push changes
+print_success "Pushing changes..."
+git push
+git push --tags
+
+# Publish to PyPI if requested
+if [[ "$PUBLISH_PYPI" == true ]]; then
+    print_success "Publishing to PyPI..."
+
+    if ! command -v twine &> /dev/null; then
+        print_error "twine not found"
+        echo "Install with: pip install twine"
+        exit 1
+    fi
+
+    if [[ -z "$TWINE_PASSWORD" && ! -f ~/.pypirc ]]; then
+        print_error "PyPI credentials not configured"
+        echo "Set up credentials with one of:"
+        echo "  1. Environment: export TWINE_USERNAME=__token__ TWINE_PASSWORD=pypi-..."
+        echo "  2. Config file: ~/.pypirc"
+        exit 1
+    fi
+
+    if twine upload dist/*; then
+        print_success "Package published to PyPI!"
+        echo "🔗 View at: https://pypi.org/project/xplainable-preprocessing/$NEW_VERSION/"
+    else
+        print_error "PyPI upload failed"
+        exit 1
+    fi
+fi
+
+# Create GitHub release if gh CLI is available
+if command -v gh &> /dev/null; then
+    if gh auth status &> /dev/null; then
+        print_success "Creating GitHub release..."
+
+        gh release create "$TAG" \
+            --title "v$NEW_VERSION" \
+            --notes "## xplainable-preprocessing v$NEW_VERSION — $RELEASE_DATE
+
+### Installation
+
+\`\`\`bash
+pip install xplainable-preprocessing==$NEW_VERSION
+\`\`\`" \
+            --latest
+
+        print_success "GitHub release created!"
+    else
+        print_warning "GitHub CLI not authenticated, skipping release creation"
+    fi
+else
+    print_warning "GitHub CLI not found, skipping release creation"
+fi
+
+print_header "🎉 RELEASE COMPLETE!"
+echo "Version:     $NEW_VERSION"
+echo "Tag:         $TAG"
+echo "PyPI:        https://pypi.org/project/xplainable-preprocessing/$NEW_VERSION/"

xplainable_preprocessing-0.2.0/src/xplainable_preprocessing/schema.py

@@ -0,0 +1,372 @@
+"""Pydantic models for pipeline specification with mutation support."""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, field_validator
+
+
+class StepSpec(BaseModel):
+    """A single preprocessing step in the pipeline.
+
+    The ``outputs``, ``modifies``, and ``drops`` fields form the step's
+    **column contract**.  They are populated after the first fit via
+    ``PipelineSpec.enrich_from_deltas()`` and enable cascade analysis
+    for step mutations (remove, reorder) without recompiling the pipeline.
+    """
+
+    id: str
+    type: str
+    columns: Optional[List[str]] = None
+    params: Dict = {}
+    description: Optional[str] = None
+
+    # Column contract — populated after first fit via enrich_from_deltas()
+    outputs: Optional[List[str]] = None   # columns this step CREATES
+    modifies: Optional[List[str]] = None  # columns this step CHANGES in place
+    drops: Optional[List[str]] = None     # columns this step REMOVES
+
+    @field_validator("id")
+    @classmethod
+    def id_must_be_non_empty(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("Step id must be non-empty")
+        return v
+
+    @field_validator("type")
+    @classmethod
+    def type_must_be_non_empty(cls, v: str) -> str:
+        if not v.strip():
+            raise ValueError("Step type must be non-empty")
+        return v
+
+
+class RemovalAnalysis(BaseModel):
+    """Result of analyzing what happens when a step is removed."""
+
+    step_id: str
+    cascade_remove: List[str] = []       # step IDs that must also be removed
+    columns_lost: List[str] = []         # columns that would disappear
+    columns_restored: List[str] = []     # columns that were dropped and would reappear
+    safe: bool = True                    # True if no downstream steps are affected
+
+
+class PipelineSpec(BaseModel):
+    """Full pipeline specification containing ordered steps.
+
+    Supports mutation methods (``remove_step``, ``insert_step``,
+    ``reorder_step``) that return new ``PipelineSpec`` instances.
+    Cascade analysis via ``analyze_removal`` works from the column
+    contract without needing to recompile the pipeline.
+    """
+
+    version: str = "2.0"
+    steps: List[StepSpec] = []
+
+    @field_validator("steps")
+    @classmethod
+    def step_ids_must_be_unique(cls, v: List[StepSpec]) -> List[StepSpec]:
+        ids = [step.id for step in v]
+        if len(ids) != len(set(ids)):
+            duplicates = [id_ for id_ in ids if ids.count(id_) > 1]
+            raise ValueError(f"Duplicate step ids: {set(duplicates)}")
+        return v
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def get_step(self, step_id: str) -> Optional[StepSpec]:
+        """Return a step by ID, or None."""
+        for step in self.steps:
+            if step.id == step_id:
+                return step
+        return None
+
+    def step_index(self, step_id: str) -> int:
+        """Return the index of a step by ID. Raises ValueError if not found."""
+        for i, step in enumerate(self.steps):
+            if step.id == step_id:
+                return i
+        raise ValueError(f"Step '{step_id}' not found in spec")
+
+    def steps_after(self, step_id: str) -> List[StepSpec]:
+        """Return all steps that come after the given step."""
+        idx = self.step_index(step_id)
+        return self.steps[idx + 1:]
+
+    @property
+    def is_enriched(self) -> bool:
+        """True if column contracts have been populated on all steps."""
+        return all(
+            step.outputs is not None
+            for step in self.steps
+        )
+
+    # ------------------------------------------------------------------
+    # Enrichment — write column contracts from deltas
+    # ------------------------------------------------------------------
+
+    def enrich_from_deltas(self, step_deltas: list[dict]) -> "PipelineSpec":
+        """Return a new spec with column contracts populated from step deltas.
+
+        Parameters
+        ----------
+        step_deltas : list[dict]
+            Output of ``preview.compute_step_deltas()``.  Each entry has
+            ``step_id`` and ``delta: {added, updated, dropped}``.
+
+        Returns
+        -------
+        PipelineSpec
+            A new spec instance with ``outputs``/``modifies``/``drops``
+            populated on each step.
+        """
+        delta_map = {d["step_id"]: d for d in step_deltas}
+        enriched_steps = []
+        for step in self.steps:
+            entry = delta_map.get(step.id, {})
+            delta = entry.get("delta", {})
+            updates = {
+                "outputs": delta.get("added", []),
+                "modifies": delta.get("updated", []),
+                "drops": delta.get("dropped", []),
+            }
+            # columns (inputs) is left as-is — only the LLM-declared value
+            # is used for dependency resolution. Outputs/modifies/drops are
+            # sufficient for cascade analysis.
+            enriched_steps.append(step.model_copy(update=updates))
+        return PipelineSpec(version=self.version, steps=enriched_steps)
+
+    # ------------------------------------------------------------------
+    # Cascade analysis
+    # ------------------------------------------------------------------
+
+    def analyze_removal(self, step_id: str) -> RemovalAnalysis:
+        """Analyze what happens if a step is removed.
+
+        Uses the column contract (``outputs``/``drops``) to determine
+        which downstream steps would lose their input columns and must
+        also be removed (cascade).
+
+        Works without recompiling the pipeline — purely from the spec
+        metadata.  Requires the spec to be enriched first.
+
+        Parameters
+        ----------
+        step_id : str
+            The ID of the step to analyze removing.
+
+        Returns
+        -------
+        RemovalAnalysis
+        """
+        step = self.get_step(step_id)
+        if step is None:
+            raise ValueError(f"Step '{step_id}' not found")
+
+        # Columns that would disappear if this step is removed
+        lost_columns = set(step.outputs or [])
+
+        # Walk downstream and find cascade
+        cascade = []
+        for other in self.steps_after(step_id):
+            inputs = set(other.columns or [])
+            if inputs & lost_columns:
+                cascade.append(other.id)
+                # This step's outputs are also lost in the cascade
+                lost_columns.update(other.outputs or [])
+
+        # Columns that this step drops would be restored
+        restored = list(step.drops or [])
+
+        return RemovalAnalysis(
+            step_id=step_id,
+            cascade_remove=cascade,
+            columns_lost=sorted(lost_columns),
+            columns_restored=sorted(restored),
+            safe=len(cascade) == 0,
+        )
+
+    # ------------------------------------------------------------------
+    # Mutations — all return NEW PipelineSpec instances (immutable pattern)
+    # ------------------------------------------------------------------
+
+    def remove_step(self, step_id: str, cascade: bool = True) -> "PipelineSpec":
+        """Return a new spec with the step (and optionally its cascade) removed.
+
+        Parameters
+        ----------
+        step_id : str
+            The step to remove.
+        cascade : bool
+            If True, also remove downstream steps that depend on columns
+            created by this step.  If False, only remove this single step
+            (caller is responsible for ensuring validity).
+
+        Returns
+        -------
+        PipelineSpec
+        """
+        analysis = self.analyze_removal(step_id)
+        remove_ids = {step_id}
+        if cascade:
+            remove_ids.update(analysis.cascade_remove)
+        return PipelineSpec(
+            version=self.version,
+            steps=[s for s in self.steps if s.id not in remove_ids],
+        )
+
+    def insert_step(
+        self,
+        step: StepSpec,
+        after: Optional[str] = None,
+    ) -> "PipelineSpec":
+        """Return a new spec with a step inserted.
+
+        Parameters
+        ----------
+        step : StepSpec
+            The step to insert.
+        after : str or None
+            Insert after this step ID.  If None, append to the end.
+
+        Returns
+        -------
+        PipelineSpec
+        """
+        new_steps = list(self.steps)
+        if after is None:
+            new_steps.append(step)
+        else:
+            idx = self.step_index(after)
+            new_steps.insert(idx + 1, step)
+        return PipelineSpec(version=self.version, steps=new_steps)
+
+    def reorder_step(self, step_id: str, new_index: int) -> "PipelineSpec":
+        """Return a new spec with a step moved to a new position.
+
+        Parameters
+        ----------
+        step_id : str
+            The step to move.
+        new_index : int
+            The target index (0-based).
+
+        Returns
+        -------
+        PipelineSpec
+        """
+        idx = self.step_index(step_id)
+        step = self.steps[idx]
+        remaining = [s for s in self.steps if s.id != step_id]
+        clamped = max(0, min(new_index, len(remaining)))
+        remaining.insert(clamped, step)
+        return PipelineSpec(version=self.version, steps=remaining)
+
+    def update_step_params(self, step_id: str, params: Dict) -> "PipelineSpec":
+        """Return a new spec with a step's params updated.
+
+        Parameters
+        ----------
+        step_id : str
+            The step to update.
+        params : dict
+            New params to merge into the step's existing params.
+
+        Returns
+        -------
+        PipelineSpec
+        """
+        new_steps = []
+        for step in self.steps:
+            if step.id == step_id:
+                merged = {**step.params, **params}
+                new_steps.append(step.model_copy(update={"params": merged}))
+            else:
+                new_steps.append(step)
+        return PipelineSpec(version=self.version, steps=new_steps)
+
+
+    def optimize(self) -> "PipelineSpec":
+        """Return a new spec with steps in optimal execution order.
+
+        Topologically sorts steps so that any step producing a column
+        comes before steps that consume it.  Independent steps retain
+        their relative order.
+
+        Requires column contracts (``outputs``) and declared ``columns``
+        (inputs) on each step.  If the spec is not enriched, returns
+        a copy with the original order.
+        """
+        if not self.is_enriched:
+            return self.model_copy()
+
+        from collections import defaultdict
+
+        step_map = {s.id: s for s in self.steps}
+        ids = [s.id for s in self.steps]
+
+        # Build producer map: column → step that creates it
+        producers: dict[str, str] = {}
+        for s in self.steps:
+            for col in s.outputs or []:
+                if col not in producers:
+                    producers[col] = s.id
+
+        # Build adjacency from declared inputs
+        adj: dict[str, set[str]] = defaultdict(set)
+        in_deg: dict[str, int] = {sid: 0 for sid in ids}
+        for s in self.steps:
+            for col in s.columns or []:
+                prod = producers.get(col)
+                if prod and prod != s.id and prod in step_map:
+                    if prod not in adj or s.id not in adj[prod]:
+                        adj[prod].add(s.id)
+                        in_deg[s.id] += 1
+
+        # Kahn's algorithm — stable (preserves original order within tiers)
+        ordered: list[str] = []
+        remaining = set(ids)
+        while remaining:
+            ready = [sid for sid in ids if sid in remaining and in_deg.get(sid, 0) == 0]
+            if not ready:
+                ordered.extend(sorted(remaining))
+                break
+            ordered.extend(ready)
+            for sid in ready:
+                remaining.discard(sid)
+                for nb in adj.get(sid, set()):
+                    in_deg[nb] = max(in_deg.get(nb, 1) - 1, 0)
+
+        return PipelineSpec(
+            version=self.version,
+            steps=[step_map[sid] for sid in ordered if sid in step_map],
+        )
+
+
+def validate_spec(spec: PipelineSpec) -> None:
+    """Validate a PipelineSpec. Raises ValueError if invalid.
+
+    Checks beyond Pydantic validation:
+    - All non-custom types must exist in the registry
+    - Custom steps must have 'code' and 'class_name' in params
+    """
+    from xplainable_preprocessing.registry import REGISTRY
+
+    for step in spec.steps:
+        if step.type == "custom":
+            if "code" not in step.params:
+                raise ValueError(
+                    f"Step '{step.id}': custom type requires 'code' in params"
+                )
+            if "class_name" not in step.params:
+                raise ValueError(
+                    f"Step '{step.id}': custom type requires 'class_name' in params"
+                )
+        elif step.type not in REGISTRY:
+            raise ValueError(
+                f"Step '{step.id}': unknown type '{step.type}'. "
+                f"Available: {sorted(REGISTRY.keys())}"
+            )

xplainable_preprocessing-0.1.0/src/xplainable_preprocessing/schema.py

@@ -1,71 +0,0 @@
-"""Pydantic models for pipeline specification."""
-
-from typing import Dict, List, Optional
-
-from pydantic import BaseModel, field_validator
-
-
-class StepSpec(BaseModel):
-    """A single preprocessing step in the pipeline."""
-
-    id: str
-    type: str
-    columns: Optional[List[str]] = None
-    params: Dict = {}
-    description: Optional[str] = None
-
-    @field_validator("id")
-    @classmethod
-    def id_must_be_non_empty(cls, v: str) -> str:
-        if not v.strip():
-            raise ValueError("Step id must be non-empty")
-        return v
-
-    @field_validator("type")
-    @classmethod
-    def type_must_be_non_empty(cls, v: str) -> str:
-        if not v.strip():
-            raise ValueError("Step type must be non-empty")
-        return v
-
-
-class PipelineSpec(BaseModel):
-    """Full pipeline specification containing ordered steps."""
-
-    version: str = "2.0"
-    steps: List[StepSpec] = []
-
-    @field_validator("steps")
-    @classmethod
-    def step_ids_must_be_unique(cls, v: List[StepSpec]) -> List[StepSpec]:
-        ids = [step.id for step in v]
-        if len(ids) != len(set(ids)):
-            duplicates = [id_ for id_ in ids if ids.count(id_) > 1]
-            raise ValueError(f"Duplicate step ids: {set(duplicates)}")
-        return v
-
-
-def validate_spec(spec: PipelineSpec) -> None:
-    """Validate a PipelineSpec. Raises ValueError if invalid.
-
-    Checks beyond Pydantic validation:
-    - All non-custom types must exist in the registry
-    - Custom steps must have 'code' and 'class_name' in params
-    """
-    from xplainable_preprocessing.registry import REGISTRY
-
-    for step in spec.steps:
-        if step.type == "custom":
-            if "code" not in step.params:
-                raise ValueError(
-                    f"Step '{step.id}': custom type requires 'code' in params"
-                )
-            if "class_name" not in step.params:
-                raise ValueError(
-                    f"Step '{step.id}': custom type requires 'class_name' in params"
-                )
-        elif step.type not in REGISTRY:
-            raise ValueError(
-                f"Step '{step.id}': unknown type '{step.type}'. "
-                f"Available: {sorted(REGISTRY.keys())}"
-            )