dataweave-lib 0.1.0__py3-none-any.whl

dataweave/__init__.py

@@ -0,0 +1,8 @@
+# MIT License
+# Copyright (c) 2024 Maharshi Soni
+
+"""DataWeave - Declarative Data Pipeline Framework."""
+
+from dataweave.__version__ import __version__
+
+__all__ = ["__version__"]

dataweave/__version__.py

@@ -0,0 +1,6 @@
+# MIT License
+# Copyright (c) 2024 Maharshi Soni
+
+"""Version information for DataWeave."""
+
+__version__: str = "0.1.0"

dataweave/cli.py

@@ -0,0 +1,185 @@
+# MIT License
+# Copyright (c) 2024 Maharshi Soni
+
+"""Click-based CLI for DataWeave."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+import pandas as pd
+
+from dataweave.__version__ import __version__
+
+
+@click.group()
+@click.version_option(version=__version__, prog_name="dataweave")
+def cli() -> None:
+    """DataWeave - Declarative Data Pipeline Framework.
+
+    Define ETL workflows in YAML. Transform, validate, profile, and load data
+    with zero configuration.
+    """
+    pass
+
+
+@cli.command()
+@click.argument("pipeline", type=click.Path(exists=True))
+@click.option(
+    "--output", "-o",
+    type=click.Path(),
+    default=None,
+    help="Override output path from the pipeline config.",
+)
+@click.option("--dry-run", is_flag=True, help="Validate config without executing.")
+@click.option("--verbose", "-v", is_flag=True, help="Show detailed step output.")
+def run(pipeline: str, output: str | None, dry_run: bool, verbose: bool) -> None:
+    """Run a YAML-defined data pipeline.
+
+    PIPELINE is the path to a .yaml pipeline configuration file.
+    """
+    from dataweave.loader import load_pipeline
+    from dataweave.engine import PipelineEngine
+
+    config = load_pipeline(pipeline)
+    click.echo(f"Pipeline: {config.name}")
+    click.echo(f"Source: {config.source.path}")
+    click.echo(f"Steps: {len(config.steps)}")
+
+    if output:
+        if config.output:
+            config.output.path = output
+        else:
+            from dataweave.models import OutputConfig
+            config.output = OutputConfig(path=output)
+
+    if dry_run:
+        click.echo("\n[DRY RUN] Configuration is valid. No data was processed.")
+        for i, step in enumerate(config.steps, 1):
+            click.echo(f"  Step {i}: {step.name} ({step.operator.value})")
+        return
+
+    base_dir = Path(pipeline).parent
+    engine = PipelineEngine(config, base_dir=base_dir)
+    result = engine.run()
+
+    if result.success:
+        click.secho("\nPipeline completed successfully!", fg="green", bold=True)
+    else:
+        click.secho(f"\nPipeline failed: {result.error}", fg="red", bold=True)
+
+    click.echo(f"Total duration: {result.total_duration_ms:.1f}ms")
+    click.echo(f"Final shape: {result.final_row_count} rows x {result.final_column_count} columns")
+
+    if result.output_path:
+        click.echo(f"Output: {result.output_path}")
+
+    if verbose:
+        click.echo("\nStep Details:")
+        for sr in result.step_results:
+            status = click.style("OK", fg="green") if sr.success else click.style("FAIL", fg="red")
+            click.echo(
+                f"  [{status}] {sr.step_name} ({sr.operator}): "
+                f"{sr.rows_in} -> {sr.rows_out} rows, {sr.duration_ms}ms"
+            )
+            if sr.error:
+                click.echo(f"       Error: {sr.error}")
+
+    if not result.success:
+        raise SystemExit(1)
+
+
+@cli.command()
+@click.argument("data_file", type=click.Path(exists=True))
+@click.option(
+    "--output", "-o",
+    type=click.Path(),
+    default=None,
+    help="Output path for the HTML report (default: <data_file>_profile.html).",
+)
+@click.option("--json", "as_json", is_flag=True, help="Output profile as JSON instead of HTML.")
+def profile(data_file: str, output: str | None, as_json: bool) -> None:
+    """Generate a profiling report for a CSV data file.
+
+    DATA_FILE is the path to a CSV file to profile.
+    """
+    import json as json_mod
+    from dataweave.profiler import profile_dataframe
+    from dataweave.report import generate_report
+
+    click.echo(f"Profiling: {data_file}")
+    df = pd.read_csv(data_file)
+    click.echo(f"Shape: {df.shape[0]} rows x {df.shape[1]} columns")
+
+    prof = profile_dataframe(df)
+
+    if as_json:
+        out_path = output or str(Path(data_file).with_suffix(".profile.json"))
+        Path(out_path).write_text(
+            json_mod.dumps(prof.to_dict(), indent=2, default=str),
+            encoding="utf-8",
+        )
+        click.echo(f"JSON profile written to: {out_path}")
+    else:
+        out_path = output or str(Path(data_file).with_suffix("")) + "_profile.html"
+        generate_report(prof, out_path)
+        click.secho(f"HTML report written to: {out_path}", fg="green")
+
+    # Summary
+    click.echo(f"\nDuplicate rows: {prof.duplicate_row_count}")
+    for cp in prof.columns:
+        if cp.null_pct > 0:
+            click.echo(f"  {cp.name}: {cp.null_pct:.1f}% nulls")
+        if cp.outlier_count > 0:
+            click.echo(f"  {cp.name}: {cp.outlier_count} outliers detected")
+
+
+@cli.command()
+@click.argument("data_file", type=click.Path(exists=True))
+@click.argument("rules_file", type=click.Path(exists=True))
+@click.option("--verbose", "-v", is_flag=True, help="Show all check results, not just failures.")
+def validate(data_file: str, rules_file: str, verbose: bool) -> None:
+    """Validate a CSV file against a YAML rules file.
+
+    DATA_FILE is the path to a CSV file.
+    RULES_FILE is a YAML file with validation rules.
+    """
+    import yaml
+    from dataweave.models import ValidationRule
+    from dataweave.operators.core import run_validations
+
+    click.echo(f"Validating: {data_file}")
+    df = pd.read_csv(data_file)
+
+    with open(rules_file, "r", encoding="utf-8") as f:
+        raw = yaml.safe_load(f)
+
+    rules = [ValidationRule(**r) for r in raw.get("rules", raw.get("validations", []))]
+    click.echo(f"Rules: {len(rules)}")
+
+    results = run_validations(df, rules)
+
+    passed = sum(1 for r in results if r["passed"])
+    failed = sum(1 for r in results if not r["passed"])
+
+    for r in results:
+        if r["passed"] and not verbose:
+            continue
+        icon = click.style("PASS", fg="green") if r["passed"] else click.style("FAIL", fg="red")
+        sev = f" [{r['severity']}]" if not r["passed"] else ""
+        click.echo(f"  [{icon}] {r['column']}.{r['check']}{sev}: {r['message']}")
+
+    click.echo(f"\n{passed} passed, {failed} failed out of {len(results)} checks")
+
+    if failed > 0:
+        errors = [r for r in results if not r["passed"] and r["severity"] == "error"]
+        if errors:
+            click.secho(f"{len(errors)} error-level failures", fg="red", bold=True)
+            raise SystemExit(1)
+        else:
+            click.secho("All failures are warnings only", fg="yellow")
+
+
+if __name__ == "__main__":
+    cli()

dataweave/engine.py

@@ -0,0 +1,236 @@
+# MIT License
+# Copyright (c) 2024 Maharshi Soni
+
+"""Pipeline execution engine for DataWeave."""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+
+from dataweave.models import PipelineConfig, StepConfig
+from dataweave.operators.base import OperatorRegistry
+
+# Ensure all operators are registered by importing the core module.
+import dataweave.operators.core  # noqa: F401
+
+
+@dataclass
+class StepResult:
+    """Result from executing a single pipeline step."""
+
+    step_name: str
+    operator: str
+    rows_in: int
+    rows_out: int
+    columns_out: int
+    duration_ms: float
+    success: bool
+    error: str | None = None
+
+
+@dataclass
+class PipelineResult:
+    """Result from executing an entire pipeline."""
+
+    pipeline_name: str
+    success: bool
+    total_duration_ms: float
+    step_results: list[StepResult] = field(default_factory=list)
+    output_path: str | None = None
+    final_row_count: int = 0
+    final_column_count: int = 0
+    error: str | None = None
+
+
+class PipelineEngine:
+    """Executes a DataWeave pipeline configuration against data."""
+
+    def __init__(self, config: PipelineConfig, base_dir: Path | None = None) -> None:
+        """Initialize the engine.
+
+        Args:
+            config: Validated pipeline configuration.
+            base_dir: Base directory for resolving relative file paths.
+                      Defaults to the current working directory.
+        """
+        self.config = config
+        self.base_dir = base_dir or Path.cwd()
+
+    def _resolve_path(self, path_str: str) -> Path:
+        """Resolve a file path relative to the base directory.
+
+        Args:
+            path_str: Potentially relative path string.
+
+        Returns:
+            Resolved absolute Path.
+        """
+        p = Path(path_str)
+        if p.is_absolute():
+            return p
+        return self.base_dir / p
+
+    def _load_source(self) -> pd.DataFrame:
+        """Load the source data from the configured path.
+
+        Returns:
+            The loaded DataFrame.
+
+        Raises:
+            FileNotFoundError: If the source file does not exist.
+            ValueError: If the format is unsupported.
+        """
+        source = self.config.source
+        path = self._resolve_path(source.path)
+
+        if not path.exists():
+            raise FileNotFoundError(f"Source file not found: {path}")
+
+        fmt = source.format.lower()
+        if fmt == "csv":
+            return pd.read_csv(path, **source.options)
+        elif fmt == "json":
+            return pd.read_json(path, **source.options)
+        elif fmt == "parquet":
+            return pd.read_parquet(path, **source.options)
+        else:
+            raise ValueError(f"Unsupported source format: '{fmt}'")
+
+    def _write_output(self, df: pd.DataFrame) -> str | None:
+        """Write the final DataFrame to the configured output.
+
+        Args:
+            df: Final DataFrame to write.
+
+        Returns:
+            Output path string, or None if no output configured.
+        """
+        if self.config.output is None:
+            return None
+
+        out = self.config.output
+        path = self._resolve_path(out.path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+        fmt = out.format.lower()
+        if fmt == "csv":
+            df.to_csv(path, index=False, **out.options)
+        elif fmt == "json":
+            df.to_json(path, orient="records", indent=2, **out.options)
+        elif fmt == "parquet":
+            df.to_parquet(path, index=False, **out.options)
+        else:
+            raise ValueError(f"Unsupported output format: '{fmt}'")
+
+        return str(path)
+
+    def run(self) -> PipelineResult:
+        """Execute the full pipeline.
+
+        Returns:
+            PipelineResult with details of each step and the final outcome.
+        """
+        pipeline_start = time.perf_counter()
+        step_results: list[StepResult] = []
+
+        try:
+            df = self._load_source()
+        except Exception as e:
+            duration = (time.perf_counter() - pipeline_start) * 1000
+            return PipelineResult(
+                pipeline_name=self.config.name,
+                success=False,
+                total_duration_ms=duration,
+                error=f"Failed to load source: {e}",
+            )
+
+        for step in self.config.steps:
+            step_start = time.perf_counter()
+            rows_in = len(df)
+
+            try:
+                operator = OperatorRegistry.get(step.operator.value)
+                df = operator.execute(df, step)
+                duration = (time.perf_counter() - step_start) * 1000
+
+                step_results.append(StepResult(
+                    step_name=step.name,
+                    operator=step.operator.value,
+                    rows_in=rows_in,
+                    rows_out=len(df),
+                    columns_out=len(df.columns),
+                    duration_ms=round(duration, 2),
+                    success=True,
+                ))
+            except Exception as e:
+                duration = (time.perf_counter() - step_start) * 1000
+                step_results.append(StepResult(
+                    step_name=step.name,
+                    operator=step.operator.value,
+                    rows_in=rows_in,
+                    rows_out=rows_in,
+                    columns_out=len(df.columns),
+                    duration_ms=round(duration, 2),
+                    success=False,
+                    error=str(e),
+                ))
+
+                total_duration = (time.perf_counter() - pipeline_start) * 1000
+                return PipelineResult(
+                    pipeline_name=self.config.name,
+                    success=False,
+                    total_duration_ms=round(total_duration, 2),
+                    step_results=step_results,
+                    final_row_count=len(df),
+                    final_column_count=len(df.columns),
+                    error=f"Step '{step.name}' failed: {e}",
+                )
+
+        # Write output
+        output_path: str | None = None
+        try:
+            output_path = self._write_output(df)
+        except Exception as e:
+            total_duration = (time.perf_counter() - pipeline_start) * 1000
+            return PipelineResult(
+                pipeline_name=self.config.name,
+                success=False,
+                total_duration_ms=round(total_duration, 2),
+                step_results=step_results,
+                final_row_count=len(df),
+                final_column_count=len(df.columns),
+                error=f"Failed to write output: {e}",
+            )
+
+        total_duration = (time.perf_counter() - pipeline_start) * 1000
+        return PipelineResult(
+            pipeline_name=self.config.name,
+            success=True,
+            total_duration_ms=round(total_duration, 2),
+            step_results=step_results,
+            output_path=output_path,
+            final_row_count=len(df),
+            final_column_count=len(df.columns),
+        )
+
+    def run_to_dataframe(self) -> pd.DataFrame:
+        """Execute the pipeline and return the final DataFrame directly.
+
+        Returns:
+            The transformed DataFrame.
+
+        Raises:
+            RuntimeError: If the pipeline fails at any step.
+        """
+        df = self._load_source()
+
+        for step in self.config.steps:
+            operator = OperatorRegistry.get(step.operator.value)
+            df = operator.execute(df, step)
+
+        return df

dataweave/loader.py

@@ -0,0 +1,52 @@
+# MIT License
+# Copyright (c) 2024 Maharshi Soni
+
+"""YAML pipeline configuration loader for DataWeave."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from dataweave.models import PipelineConfig
+
+
+def load_pipeline(path: str | Path) -> PipelineConfig:
+    """Load and validate a pipeline configuration from a YAML file.
+
+    Args:
+        path: Path to the YAML pipeline file.
+
+    Returns:
+        A validated PipelineConfig instance.
+
+    Raises:
+        FileNotFoundError: If the YAML file does not exist.
+        yaml.YAMLError: If the YAML is malformed.
+        pydantic.ValidationError: If the config does not match the schema.
+    """
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(f"Pipeline config not found: {path}")
+
+    with open(path, "r", encoding="utf-8") as f:
+        raw: dict[str, Any] = yaml.safe_load(f)
+
+    if raw is None:
+        raise ValueError(f"Empty pipeline config: {path}")
+
+    return PipelineConfig(**raw)
+
+
+def load_pipeline_from_dict(data: dict[str, Any]) -> PipelineConfig:
+    """Load a pipeline configuration from a dictionary.
+
+    Args:
+        data: Dictionary matching the PipelineConfig schema.
+
+    Returns:
+        A validated PipelineConfig instance.
+    """
+    return PipelineConfig(**data)

dataweave/models.py

@@ -0,0 +1,110 @@
+# MIT License
+# Copyright (c) 2024 Maharshi Soni
+
+"""Pydantic models for DataWeave pipeline configuration."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class OperatorType(str, Enum):
+    """Supported pipeline operator types."""
+
+    FILTER = "filter"
+    TRANSFORM = "transform"
+    AGGREGATE = "aggregate"
+    JOIN = "join"
+    VALIDATE = "validate"
+    RENAME = "rename"
+    SELECT = "select"
+    SORT = "sort"
+    DEDUPLICATE = "deduplicate"
+    FILL_NA = "fill_na"
+
+
+class DataSourceConfig(BaseModel):
+    """Configuration for a data source."""
+
+    path: str = Field(..., description="File path to the data source (CSV)")
+    format: str = Field(default="csv", description="File format")
+    options: dict[str, Any] = Field(
+        default_factory=dict, description="Extra read options"
+    )
+
+
+class JoinConfig(BaseModel):
+    """Configuration for a join operation."""
+
+    right_source: str = Field(..., description="Path to the right dataset")
+    on: str | list[str] = Field(..., description="Column(s) to join on")
+    how: str = Field(default="inner", description="Join type: inner, left, right, outer")
+
+    @field_validator("how")
+    @classmethod
+    def validate_how(cls, v: str) -> str:
+        allowed = {"inner", "left", "right", "outer"}
+        if v not in allowed:
+            raise ValueError(f"Join type must be one of {allowed}, got '{v}'")
+        return v
+
+
+class ValidationRule(BaseModel):
+    """A single data validation rule."""
+
+    column: str = Field(..., description="Column to validate")
+    check: str = Field(..., description="Check type: not_null, unique, min, max, pattern, in_set")
+    value: Any = Field(default=None, description="Expected value for the check")
+    severity: str = Field(default="error", description="error or warning")
+
+    @field_validator("check")
+    @classmethod
+    def validate_check(cls, v: str) -> str:
+        allowed = {"not_null", "unique", "min", "max", "pattern", "in_set", "dtype"}
+        if v not in allowed:
+            raise ValueError(f"Check must be one of {allowed}, got '{v}'")
+        return v
+
+    @field_validator("severity")
+    @classmethod
+    def validate_severity(cls, v: str) -> str:
+        if v not in {"error", "warning"}:
+            raise ValueError(f"Severity must be 'error' or 'warning', got '{v}'")
+        return v
+
+
+class StepConfig(BaseModel):
+    """Configuration for a single pipeline step."""
+
+    name: str = Field(..., description="Step name")
+    operator: OperatorType = Field(..., description="Operator type")
+    params: dict[str, Any] = Field(
+        default_factory=dict, description="Operator-specific parameters"
+    )
+    join: Optional[JoinConfig] = Field(default=None, description="Join config (if operator is join)")
+    validations: list[ValidationRule] = Field(
+        default_factory=list, description="Validation rules (if operator is validate)"
+    )
+
+
+class OutputConfig(BaseModel):
+    """Configuration for pipeline output."""
+
+    path: str = Field(..., description="Output file path")
+    format: str = Field(default="csv", description="Output format")
+    options: dict[str, Any] = Field(
+        default_factory=dict, description="Extra write options"
+    )
+
+
+class PipelineConfig(BaseModel):
+    """Top-level pipeline configuration parsed from YAML."""
+
+    name: str = Field(..., description="Pipeline name")
+    description: str = Field(default="", description="Pipeline description")
+    source: DataSourceConfig = Field(..., description="Input data source")
+    steps: list[StepConfig] = Field(default_factory=list, description="Pipeline steps")
+    output: Optional[OutputConfig] = Field(default=None, description="Output config")

dataweave/operators/__init__.py

@@ -0,0 +1,33 @@
+# MIT License
+# Copyright (c) 2024 Maharshi Soni
+
+"""Built-in operators for DataWeave pipelines."""
+
+from dataweave.operators.base import Operator, OperatorRegistry
+from dataweave.operators.core import (
+    AggregateOperator,
+    DeduplicateOperator,
+    FillNaOperator,
+    FilterOperator,
+    JoinOperator,
+    RenameOperator,
+    SelectOperator,
+    SortOperator,
+    TransformOperator,
+    ValidateOperator,
+)
+
+__all__ = [
+    "Operator",
+    "OperatorRegistry",
+    "FilterOperator",
+    "TransformOperator",
+    "AggregateOperator",
+    "JoinOperator",
+    "ValidateOperator",
+    "RenameOperator",
+    "SelectOperator",
+    "SortOperator",
+    "DeduplicateOperator",
+    "FillNaOperator",
+]