datasemver 0.2.0__py3-none-any.whl

datasemver/__init__.py

@@ -0,0 +1,8 @@
+"""SemVer-style versioning for datasets."""
+
+from datasemver.core.analyzer import analyze
+from datasemver.core.models import AnalysisReport, ChangeType, DiffResult, Severity
+
+__version__ = "0.2.0"
+
+__all__ = ["analyze", "AnalysisReport", "ChangeType", "DiffResult", "Severity", "__version__"]

datasemver/__main__.py

@@ -0,0 +1,4 @@
+from datasemver.cli.main import app
+
+if __name__ == "__main__":
+    app()

datasemver/cli/main.py

@@ -0,0 +1,177 @@
+"""Command line interface."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+
+from datasemver.core.analyzer import DEFAULT_VERSION, analyze
+from datasemver.core.changelog import render_entry, severity_label, write_changelog
+from datasemver.core.models import AnalysisReport, ColumnStatus, Severity
+from datasemver.rules.engine import EVALUATION_ORDER, RuleError, load_rules
+from datasemver.utils.version import InvalidVersionError
+
+app = typer.Typer(
+    name="datasemver",
+    help="Semantic versioning for datasets.",
+    add_completion=False,
+    no_args_is_help=True,
+)
+
+console = Console()
+error_console = Console(stderr=True)
+
+SEVERITY_COLORS: dict[Severity, str] = {
+    Severity.MAJOR: "bold red",
+    Severity.MINOR: "bold yellow",
+    Severity.PATCH: "bold green",
+}
+
+STATUS_COLORS: dict[ColumnStatus, str] = {
+    ColumnStatus.ADDED: "green",
+    ColumnStatus.REMOVED: "red",
+    ColumnStatus.RENAMED: "magenta",
+    ColumnStatus.MODIFIED: "yellow",
+    ColumnStatus.UNCHANGED: "dim",
+}
+
+
+@app.command()
+def diff(
+    old: Annotated[Path, typer.Argument(help="Previous version of the dataset (CSV or JSON).")],
+    new: Annotated[Path, typer.Argument(help="New version of the dataset (CSV or JSON).")],
+    rules: Annotated[
+        Path | None,
+        typer.Option("--rules", "-r", help="Custom rules file overriding the defaults."),
+    ] = None,
+    as_json: Annotated[
+        bool, typer.Option("--json", help="Print the report as JSON instead of a table.")
+    ] = False,
+    output: Annotated[
+        Path | None,
+        typer.Option("--output", "-o", help="Write the changelog entry to this file."),
+    ] = None,
+    current_version: Annotated[
+        str,
+        typer.Option("--current-version", "-c", help="Version the new dataset is bumped from."),
+    ] = DEFAULT_VERSION,
+) -> None:
+    """Compare two dataset versions and suggest a semantic version bump."""
+    try:
+        report = analyze(old, new, rules=rules, current_version=current_version)
+    except (FileNotFoundError, ValueError, RuleError, InvalidVersionError) as error:
+        error_console.print(f"[bold red]error:[/] {error}")
+        raise typer.Exit(code=2) from error
+
+    if output is not None:
+        write_changelog(report, output)
+
+    if as_json:
+        console.print_json(json.dumps(report.model_dump(mode="json")))
+        return
+
+    _render_report(report, output)
+
+
+def _render_report(report: AnalysisReport, output: Path | None) -> None:
+    bump = severity_label(report.bump)
+    style = SEVERITY_COLORS.get(report.bump, "bold blue") if report.bump else "bold blue"
+
+    console.print(
+        Panel(
+            f"[{style}]Suggested bump: {bump}[/]\n"
+            f"{report.current_version} -> {report.next_version}\n\n"
+            f"old: {report.old_source} ({report.diff.old.row_count} rows)\n"
+            f"new: {report.new_source} ({report.diff.new.row_count} rows)",
+            title="DataSemver",
+            expand=False,
+        )
+    )
+
+    console.print(_columns_table(report))
+    console.print(_changes_table(report))
+
+    if output is not None:
+        console.print(f"[dim]changelog written to {output}[/]")
+    else:
+        console.print(Panel(render_entry(report).rstrip(), title="CHANGELOG", expand=False))
+
+
+def _columns_table(report: AnalysisReport) -> Table:
+    table = Table(title="Columns", header_style="bold")
+    for header in ("column", "status", "type old", "type new", "nulls", "cardinality"):
+        table.add_column(header)
+
+    for column in report.diff.columns:
+        name = column.name if not column.renamed_from else f"{column.renamed_from} -> {column.name}"
+        table.add_row(
+            name,
+            f"[{STATUS_COLORS[column.status]}]{column.status.value}[/]",
+            column.dtype_old or "-",
+            column.dtype_new or "-",
+            f"{_percent(column.null_ratio_old)} -> {_percent(column.null_ratio_new)}",
+            f"{_number(column.cardinality_old)} -> {_number(column.cardinality_new)}",
+        )
+    return table
+
+
+def _changes_table(report: AnalysisReport) -> Table:
+    table = Table(title="Changes", header_style="bold")
+    table.add_column("severity")
+    table.add_column("rule")
+    table.add_column("description")
+
+    ordered = sorted(
+        report.classified,
+        key=lambda item: -item.severity.rank if item.severity else 1,
+    )
+    for item in ordered:
+        severity = item.severity
+        label = severity.value.upper() if severity else "unclassified"
+        style = SEVERITY_COLORS.get(severity, "dim") if severity else "dim"
+        table.add_row(f"[{style}]{label}[/]", item.rule or "-", item.change.description)
+
+    if not ordered:
+        table.add_row("[dim]none[/]", "-", "Datasets are identical")
+    return table
+
+
+def _percent(value: float | None) -> str:
+    return "-" if value is None else f"{value:.1%}"
+
+
+def _number(value: int | None) -> str:
+    return "-" if value is None else str(value)
+
+
+@app.command("rules")
+def show_rules(
+    path: Annotated[
+        Path | None, typer.Argument(help="Rules file to inspect; defaults to the bundled rules.")
+    ] = None,
+) -> None:
+    """Print the rules that will be applied, grouped by severity."""
+    try:
+        rule_set = load_rules(path)
+    except (FileNotFoundError, RuleError) as error:
+        error_console.print(f"[bold red]error:[/] {error}")
+        raise typer.Exit(code=2) from error
+
+    for severity in EVALUATION_ORDER:
+        entries = rule_set.rules.get(severity, [])
+        console.print(f"[{SEVERITY_COLORS[severity]}]{severity.value}[/]")
+        for rule in entries or []:
+            suffix = f" > {rule.threshold:g}" if rule.threshold is not None else ""
+            console.print(f"  - {rule.name}{suffix}")
+        if not entries:
+            console.print("  [dim]- none[/]")
+
+
+if __name__ == "__main__":
+    app()

datasemver/core/analyzer.py

@@ -0,0 +1,58 @@
+"""Analysis pipeline: load, diff, classify and version."""
+
+from __future__ import annotations
+
+from datetime import date
+from pathlib import Path
+
+from datasemver.core.differ import DiffConfig, diff_schemas
+from datasemver.core.models import AnalysisReport, DatasetSchema
+from datasemver.formats.loader import load_schema
+from datasemver.rules.engine import RuleSet, highest_severity, load_rules
+from datasemver.utils.version import bump_version
+
+DEFAULT_VERSION = "0.0.0"
+
+
+def analyze(
+    old_path: str | Path,
+    new_path: str | Path,
+    rules: RuleSet | str | Path | None = None,
+    current_version: str = DEFAULT_VERSION,
+    diff_config: DiffConfig | None = None,
+) -> AnalysisReport:
+    """Compare two dataset files and return the suggested version bump."""
+    old_schema = load_schema(old_path)
+    new_schema = load_schema(new_path)
+    return analyze_schemas(
+        old_schema,
+        new_schema,
+        rules=rules,
+        current_version=current_version,
+        diff_config=diff_config,
+    )
+
+
+def analyze_schemas(
+    old: DatasetSchema,
+    new: DatasetSchema,
+    rules: RuleSet | str | Path | None = None,
+    current_version: str = DEFAULT_VERSION,
+    diff_config: DiffConfig | None = None,
+) -> AnalysisReport:
+    """Compare two already loaded dataset profiles."""
+    rule_set = rules if isinstance(rules, RuleSet) else load_rules(rules)
+    diff = diff_schemas(old, new, config=diff_config)
+    classified = rule_set.evaluate(diff)
+    bump = highest_severity(classified)
+
+    return AnalysisReport(
+        generated_at=date.today(),
+        old_source=old.source,
+        new_source=new.source,
+        current_version=current_version,
+        next_version=bump_version(current_version, bump),
+        bump=bump,
+        diff=diff,
+        classified=classified,
+    )

datasemver/core/changelog.py

@@ -0,0 +1,56 @@
+"""Changelog rendering."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from datasemver.core.models import AnalysisReport, Severity
+from datasemver.rules.engine import EVALUATION_ORDER
+
+CHANGELOG_TITLE = "# Changelog"
+
+
+def render_entry(report: AnalysisReport) -> str:
+    """Render a single changelog entry for an analysis report."""
+    lines = [f"## [{report.next_version}] - {report.generated_at.isoformat()}", ""]
+
+    for severity in EVALUATION_ORDER:
+        items = report.by_severity(severity)
+        if not items:
+            continue
+        lines.append(f"### {severity.value.capitalize()}")
+        lines.extend(f"- {item.change.description}" for item in items)
+        lines.append("")
+
+    if len(lines) == 2:
+        lines.append("No classified changes detected.")
+        lines.append("")
+
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def render_changelog(report: AnalysisReport) -> str:
+    """Render a full changelog document containing a single entry."""
+    return f"{CHANGELOG_TITLE}\n\n{render_entry(report)}"
+
+
+def write_changelog(report: AnalysisReport, path: str | Path) -> Path:
+    """Write the entry to a changelog file, prepending it when the file already exists."""
+    path = Path(path)
+    entry = render_entry(report)
+
+    if not path.exists():
+        path.write_text(render_changelog(report), encoding="utf-8")
+        return path
+
+    existing = path.read_text(encoding="utf-8").lstrip()
+    if existing.startswith(CHANGELOG_TITLE):
+        body = existing[len(CHANGELOG_TITLE) :].lstrip("\n")
+        path.write_text(f"{CHANGELOG_TITLE}\n\n{entry}\n{body}", encoding="utf-8")
+    else:
+        path.write_text(f"{entry}\n{existing}", encoding="utf-8")
+    return path
+
+
+def severity_label(severity: Severity | None) -> str:
+    return severity.value.upper() if severity else "NONE"

datasemver/core/differ.py

@@ -0,0 +1,352 @@
+"""Comparison of two dataset profiles."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from datasemver.core.models import (
+    Change,
+    ChangeType,
+    ColumnComparison,
+    ColumnStats,
+    ColumnStatus,
+    DatasetSchema,
+    DiffResult,
+)
+from datasemver.utils.similarity import column_similarity
+
+COMPATIBLE_WIDENINGS: set[tuple[str, str]] = {
+    ("bool", "int64"),
+    ("bool", "float64"),
+    ("int64", "float64"),
+}
+
+
+@dataclass(frozen=True)
+class DiffConfig:
+    """Sensitivity thresholds used while comparing two datasets."""
+
+    rename_threshold: float = 0.7
+    null_ratio_tolerance: float = 0.01
+    distribution_sigma: float = 0.5
+    stat_change_tolerance: float = 0.01
+    cardinality_tolerance: float = 0.1
+
+
+def diff_schemas(
+    old: DatasetSchema,
+    new: DatasetSchema,
+    config: DiffConfig | None = None,
+) -> DiffResult:
+    """Compare two dataset profiles and return every detected change."""
+    config = config or DiffConfig()
+
+    removed = [name for name in old.column_names if name not in new.columns]
+    added = [name for name in new.column_names if name not in old.columns]
+    renames = _detect_renames(old, new, removed, added, config)
+
+    removed = [name for name in removed if name not in renames]
+    added = [name for name in added if name not in renames.values()]
+
+    changes: list[Change] = list(_row_count_changes(old, new))
+    comparisons: list[ColumnComparison] = []
+
+    for name in removed:
+        stats = old.columns[name]
+        changes.append(
+            Change(
+                type=ChangeType.COLUMN_REMOVED,
+                column=name,
+                description=f"Column '{name}' was removed",
+                details={"dtype": stats.dtype},
+            )
+        )
+        comparisons.append(
+            ColumnComparison(
+                name=name,
+                status=ColumnStatus.REMOVED,
+                dtype_old=stats.dtype,
+                null_ratio_old=stats.null_ratio,
+                cardinality_old=stats.cardinality,
+            )
+        )
+
+    for name in added:
+        stats = new.columns[name]
+        changes.append(
+            Change(
+                type=ChangeType.COLUMN_ADDED,
+                column=name,
+                description=f"Column '{name}' was added",
+                metrics={"null_ratio": stats.null_ratio},
+                details={"dtype": stats.dtype},
+            )
+        )
+        comparisons.append(
+            ColumnComparison(
+                name=name,
+                status=ColumnStatus.ADDED,
+                dtype_new=stats.dtype,
+                null_ratio_new=stats.null_ratio,
+                cardinality_new=stats.cardinality,
+            )
+        )
+
+    for old_name, new_name in renames.items():
+        changes.append(
+            Change(
+                type=ChangeType.COLUMN_RENAMED,
+                column=new_name,
+                description=f"Column '{old_name}' was renamed to '{new_name}'",
+                details={"previous_name": old_name},
+            )
+        )
+
+    paired = [(name, name) for name in old.column_names if name in new.columns]
+    paired.extend(renames.items())
+
+    for old_name, new_name in paired:
+        old_stats = old.columns[old_name]
+        new_stats = new.columns[new_name]
+        column_changes = list(_column_changes(old_stats, new_stats, config))
+        changes.extend(column_changes)
+
+        if old_name != new_name:
+            status = ColumnStatus.RENAMED
+        elif column_changes:
+            status = ColumnStatus.MODIFIED
+        else:
+            status = ColumnStatus.UNCHANGED
+
+        comparisons.append(
+            ColumnComparison(
+                name=new_name,
+                status=status,
+                renamed_from=old_name if old_name != new_name else None,
+                dtype_old=old_stats.dtype,
+                dtype_new=new_stats.dtype,
+                null_ratio_old=old_stats.null_ratio,
+                null_ratio_new=new_stats.null_ratio,
+                cardinality_old=old_stats.cardinality,
+                cardinality_new=new_stats.cardinality,
+            )
+        )
+
+    comparisons.sort(key=lambda item: (item.status.value, item.name))
+    return DiffResult(old=old, new=new, changes=changes, columns=comparisons)
+
+
+def _detect_renames(
+    old: DatasetSchema,
+    new: DatasetSchema,
+    removed: list[str],
+    added: list[str],
+    config: DiffConfig,
+) -> dict[str, str]:
+    """Pair removed and added columns that look like the same column renamed."""
+    candidates: list[tuple[float, str, str]] = []
+    for old_name in removed:
+        for new_name in added:
+            score = column_similarity(
+                old_name,
+                new_name,
+                old.columns[old_name].categories,
+                new.columns[new_name].categories,
+            )
+            if score >= config.rename_threshold:
+                candidates.append((score, old_name, new_name))
+
+    candidates.sort(reverse=True)
+    renames: dict[str, str] = {}
+    taken: set[str] = set()
+    for _, old_name, new_name in candidates:
+        if old_name in renames or new_name in taken:
+            continue
+        renames[old_name] = new_name
+        taken.add(new_name)
+    return renames
+
+
+def _row_count_changes(old: DatasetSchema, new: DatasetSchema):
+    if old.row_count == new.row_count:
+        return
+
+    delta = new.row_count - old.row_count
+    base = old.row_count or 1
+    percentage = round(abs(delta) / base * 100, 4)
+    metrics = {
+        "old_rows": float(old.row_count),
+        "new_rows": float(new.row_count),
+        "delta": float(delta),
+        "change_pct": percentage,
+    }
+
+    if delta > 0:
+        yield Change(
+            type=ChangeType.ROW_COUNT_INCREASED,
+            description=f"Row count grew from {old.row_count} to {new.row_count} (+{percentage:.2f}%)",
+            metrics=metrics | {"increase_pct": percentage},
+        )
+    else:
+        yield Change(
+            type=ChangeType.ROW_COUNT_DECREASED,
+            description=f"Row count fell from {old.row_count} to {new.row_count} (-{percentage:.2f}%)",
+            metrics=metrics | {"decrease_pct": percentage},
+        )
+
+
+def _column_changes(old: ColumnStats, new: ColumnStats, config: DiffConfig):
+    yield from _type_changes(old, new)
+    yield from _null_changes(old, new, config)
+    yield from _category_changes(old, new)
+    yield from _numeric_changes(old, new, config)
+    yield from _cardinality_changes(old, new, config)
+
+
+def _type_changes(old: ColumnStats, new: ColumnStats):
+    if old.dtype == new.dtype:
+        return
+
+    details = {"dtype_old": old.dtype, "dtype_new": new.dtype}
+    if (old.dtype, new.dtype) in COMPATIBLE_WIDENINGS:
+        yield Change(
+            type=ChangeType.TYPE_CHANGED_COMPATIBLE,
+            column=new.name,
+            description=f"Column '{new.name}' widened from {old.dtype} to {new.dtype}",
+            details=details,
+        )
+    else:
+        yield Change(
+            type=ChangeType.TYPE_CHANGED_INCOMPATIBLE,
+            column=new.name,
+            description=f"Column '{new.name}' changed type from {old.dtype} to {new.dtype}",
+            details=details,
+        )
+
+
+def _null_changes(old: ColumnStats, new: ColumnStats, config: DiffConfig):
+    delta = new.null_ratio - old.null_ratio
+    if abs(delta) < config.null_ratio_tolerance:
+        return
+
+    metrics = {
+        "null_ratio_old": round(old.null_ratio * 100, 4),
+        "null_ratio_new": round(new.null_ratio * 100, 4),
+        "delta_pct": round(abs(delta) * 100, 4),
+    }
+    if delta < 0:
+        yield Change(
+            type=ChangeType.NULLS_FIXED,
+            column=new.name,
+            description=(
+                f"Column '{new.name}' nulls dropped from {old.null_ratio:.1%} to {new.null_ratio:.1%}"
+            ),
+            metrics=metrics,
+        )
+    else:
+        yield Change(
+            type=ChangeType.NULLS_INTRODUCED,
+            column=new.name,
+            description=(
+                f"Column '{new.name}' nulls rose from {old.null_ratio:.1%} to {new.null_ratio:.1%}"
+            ),
+            metrics=metrics,
+        )
+
+
+def _category_changes(old: ColumnStats, new: ColumnStats):
+    if old.categories is None or new.categories is None:
+        return
+
+    old_set, new_set = set(old.categories), set(new.categories)
+    gained = sorted(new_set - old_set)
+    lost = sorted(old_set - new_set)
+
+    if gained:
+        yield Change(
+            type=ChangeType.NEW_CATEGORY_ADDED,
+            column=new.name,
+            description=f"Column '{new.name}' gained {len(gained)} category value(s)",
+            metrics={"added_count": float(len(gained))},
+            details={"categories": gained[:20]},
+        )
+    if lost:
+        yield Change(
+            type=ChangeType.CATEGORY_REMOVED,
+            column=new.name,
+            description=f"Column '{new.name}' lost {len(lost)} category value(s)",
+            metrics={"removed_count": float(len(lost))},
+            details={"categories": lost[:20]},
+        )
+
+
+def _numeric_changes(old: ColumnStats, new: ColumnStats, config: DiffConfig):
+    if not (old.is_numeric and new.is_numeric):
+        return
+    if old.mean is None or new.mean is None:
+        return
+    if _is_sequential_key(old) and _is_sequential_key(new):
+        return
+
+    shift = abs(new.mean - old.mean)
+    if shift == 0:
+        return
+
+    base = abs(old.mean) or 1.0
+    relative = round(shift / base * 100, 4)
+    spread = old.std or 0.0
+    sigma = round(shift / spread, 4) if spread else float("inf")
+    metrics = {"mean_old": old.mean, "mean_new": new.mean, "mean_shift_pct": relative}
+
+    if spread and sigma >= config.distribution_sigma:
+        yield Change(
+            type=ChangeType.DISTRIBUTION_SHIFT,
+            column=new.name,
+            description=(
+                f"Column '{new.name}' mean moved from {old.mean:.4g} to {new.mean:.4g} "
+                f"({sigma:.2f} sigma)"
+            ),
+            metrics=metrics | {"sigma_shift": sigma},
+        )
+    elif relative >= config.stat_change_tolerance * 100:
+        yield Change(
+            type=ChangeType.MINOR_STAT_CHANGE,
+            column=new.name,
+            description=(
+                f"Column '{new.name}' mean moved from {old.mean:.4g} to {new.mean:.4g} "
+                f"({relative:.2f}%)"
+            ),
+            metrics=metrics,
+        )
+
+
+def _is_sequential_key(stats: ColumnStats) -> bool:
+    """Detect contiguous integer keys, whose statistics carry no business meaning."""
+    if stats.dtype != "int64" or stats.minimum is None or stats.maximum is None:
+        return False
+    if stats.uniqueness < 1.0 or stats.null_ratio > 0:
+        return False
+    return stats.cardinality == int(stats.maximum - stats.minimum) + 1
+
+
+def _cardinality_changes(old: ColumnStats, new: ColumnStats, config: DiffConfig):
+    if old.categories is not None and new.categories is not None:
+        return
+    if abs(new.uniqueness - old.uniqueness) < config.cardinality_tolerance:
+        return
+
+    base = old.cardinality or 1
+    change = round(abs(new.cardinality - old.cardinality) / base * 100, 4)
+
+    yield Change(
+        type=ChangeType.CARDINALITY_CHANGED,
+        column=new.name,
+        description=(
+            f"Column '{new.name}' cardinality moved from {old.cardinality} to {new.cardinality}"
+        ),
+        metrics={
+            "cardinality_old": float(old.cardinality),
+            "cardinality_new": float(new.cardinality),
+            "change_pct": change,
+        },
+    )