datawash 0.2.0__py3-none-any.whl

datawash/detectors/similarity_detector.py

@@ -0,0 +1,294 @@
+"""Similar column detector using MinHash + LSH.
+
+Multi-stage similarity detection with O(n) average complexity:
+1. N-gram blocking for column names
+2. MinHash + LSH for column values
+3. Size filtering to prune candidates
+4. Exact verification only on candidates
+
+Based on:
+- VLDB 2016: "An Empirical Evaluation of Set Similarity Join Techniques"
+- Broder 1997: "On the resemblance and containment of documents" (MinHash)
+"""
+
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from typing import Any
+
+import pandas as pd
+
+from datawash.core.models import DatasetProfile, Finding, Severity
+from datawash.detectors.base import BaseDetector
+from datawash.detectors.registry import register_detector
+
+
+class SimilarityDetector(BaseDetector):
+    """Detect similar columns using MinHash + LSH.
+
+    Uses a multi-stage pipeline:
+    - Stage 1: N-gram blocking for name similarity
+    - Stage 2: MinHash signatures + LSH banding for value similarity
+    - Stage 3: Exact verification only on candidate pairs
+    """
+
+    # Thresholds
+    NAME_SIMILARITY_THRESHOLD = 0.7
+    VALUE_OVERLAP_THRESHOLD = 0.8
+    COMBINED_THRESHOLD = 0.6
+
+    # Algorithm parameters
+    NGRAM_SIZE = 2
+    MIN_SHARED_NGRAMS = 2
+    MINHASH_SIGNATURES = 100
+    LSH_BANDS = 20
+    MAX_UNIQUE_VALUES = 10000
+    MAX_CANDIDATES = 5000  # Cap to avoid O(n^2) blowup on wide datasets
+
+    def __init__(
+        self, name_threshold: float = 0.7, value_threshold: float = 0.8
+    ) -> None:
+        self._name_threshold = name_threshold
+        self._value_threshold = value_threshold
+
+    @property
+    def name(self) -> str:
+        return "similarity"
+
+    @property
+    def description(self) -> str:
+        return "Detects similar or potentially duplicate columns"
+
+    def detect(
+        self, df: pd.DataFrame, profile: DatasetProfile, config: Any = None
+    ) -> list[Finding]:
+        columns = list(df.columns)
+        n_cols = len(columns)
+
+        if n_cols < 2:
+            return []
+
+        # Precompute value sets (use unique() first to reduce work)
+        column_value_sets: dict[int, set[str]] = {}
+        for idx, col in enumerate(columns):
+            unique_vals = df[col].dropna().unique()
+            if len(unique_vals) > self.MAX_UNIQUE_VALUES:
+                column_value_sets[idx] = set()  # Too many unique values (likely IDs)
+            else:
+                column_value_sets[idx] = set(str(v) for v in unique_vals)
+
+        # Stage 1: Blocking
+        name_candidates = self._ngram_blocking(columns)
+        value_candidates = self._minhash_lsh_blocking(columns, column_value_sets)
+
+        # Stage 2: Merge candidates (cap to avoid blowup)
+        all_candidates = name_candidates | value_candidates
+        if len(all_candidates) > self.MAX_CANDIDATES:
+            all_candidates = set(list(all_candidates)[: self.MAX_CANDIDATES])
+
+        # Stage 3: Verify only candidates
+        findings: list[Finding] = []
+        for i, j in all_candidates:
+            finding = self._verify_pair(columns, i, j, column_value_sets)
+            if finding:
+                findings.append(finding)
+
+        return findings
+
+    # -----------------------------------------------------------------
+    # STAGE 1a: N-GRAM BLOCKING FOR NAMES
+    # -----------------------------------------------------------------
+
+    def _ngram_blocking(self, columns: list[str]) -> set[tuple[int, int]]:
+        """Find candidate pairs based on shared character n-grams.
+
+        Complexity: O(n * k) where k = average column name length
+        """
+        ngram_index: dict[str, list[int]] = defaultdict(list)
+
+        for idx, col in enumerate(columns):
+            ngrams = self._get_ngrams(col.lower())
+            for ng in ngrams:
+                ngram_index[ng].append(idx)
+
+        pair_counts: Counter[tuple[int, int]] = Counter()
+        for indices in ngram_index.values():
+            if len(indices) < 2:
+                continue
+            for i in range(len(indices)):
+                for j in range(i + 1, len(indices)):
+                    pair = (min(indices[i], indices[j]), max(indices[i], indices[j]))
+                    pair_counts[pair] += 1
+
+        return {
+            pair
+            for pair, count in pair_counts.items()
+            if count >= self.MIN_SHARED_NGRAMS
+        }
+
+    def _get_ngrams(self, s: str) -> set[str]:
+        """Extract character n-grams from string."""
+        n = self.NGRAM_SIZE
+        if len(s) < n:
+            return {s}
+        return {s[i : i + n] for i in range(len(s) - n + 1)}
+
+    # -----------------------------------------------------------------
+    # STAGE 1b + 2: MINHASH + LSH BLOCKING FOR VALUES
+    # -----------------------------------------------------------------
+
+    def _minhash_lsh_blocking(
+        self,
+        columns: list[str],
+        column_value_sets: dict[int, set[str]],
+    ) -> set[tuple[int, int]]:
+        """Find candidate pairs using MinHash signatures + LSH banding.
+
+        MinHash: O(n * v * m) where m = signature size
+        LSH:     O(n * b) where b = number of bands
+        """
+        n_cols = len(columns)
+
+        # Generate MinHash signatures for each column
+        signatures: list[list[int]] = []
+        sizes: list[int] = []
+        for idx in range(n_cols):
+            val_set = column_value_sets.get(idx, set())
+            signatures.append(self._minhash_signature(val_set))
+            sizes.append(len(val_set))
+
+        # LSH Banding: hash signature bands into buckets
+        candidates: set[tuple[int, int]] = set()
+        rows_per_band = self.MINHASH_SIGNATURES // self.LSH_BANDS
+
+        # Size filter ratio: for Jaccard >= t, ratio must be <= (2-t)/t
+        max_ratio = (2 - self._value_threshold) / self._value_threshold
+
+        for band_idx in range(self.LSH_BANDS):
+            buckets: dict[int, list[int]] = defaultdict(list)
+            start = band_idx * rows_per_band
+            end = start + rows_per_band
+
+            for col_idx, sig in enumerate(signatures):
+                band_hash = hash(tuple(sig[start:end]))
+                buckets[band_hash].append(col_idx)
+
+            for bucket_cols in buckets.values():
+                if len(bucket_cols) < 2 or len(bucket_cols) > 50:
+                    # Skip very large buckets (too many false positives)
+                    continue
+                for i in range(len(bucket_cols)):
+                    for j in range(i + 1, len(bucket_cols)):
+                        idx1, idx2 = bucket_cols[i], bucket_cols[j]
+                        s1, s2 = sizes[idx1], sizes[idx2]
+                        if s1 == 0 or s2 == 0:
+                            continue
+                        ratio = max(s1, s2) / min(s1, s2)
+                        if ratio <= max_ratio:
+                            candidates.add((min(idx1, idx2), max(idx1, idx2)))
+                    if len(candidates) > self.MAX_CANDIDATES:
+                        break
+                if len(candidates) > self.MAX_CANDIDATES:
+                    break
+
+        return candidates
+
+    def _minhash_signature(self, values: set[str]) -> list[int]:
+        """Generate MinHash signature for a set of values."""
+        if not values:
+            return [0] * self.MINHASH_SIGNATURES
+        signature = []
+        for seed in range(self.MINHASH_SIGNATURES):
+            min_hash = min(hash((seed, v)) % (2**32) for v in values)
+            signature.append(min_hash)
+        return signature
+
+    # -----------------------------------------------------------------
+    # STAGE 3: VERIFICATION
+    # -----------------------------------------------------------------
+
+    def _verify_pair(
+        self,
+        columns: list[str],
+        i: int,
+        j: int,
+        column_value_sets: dict[int, set[str]],
+    ) -> Finding | None:
+        """Verify if a candidate pair is actually similar."""
+        col1, col2 = columns[i], columns[j]
+
+        name_sim = self._normalized_levenshtein(col1.lower(), col2.lower())
+        set1 = column_value_sets.get(i, set())
+        set2 = column_value_sets.get(j, set())
+        value_sim = self._jaccard_similarity(set1, set2)
+
+        combined_score = 0.4 * name_sim + 0.6 * value_sim
+        if combined_score < self.COMBINED_THRESHOLD:
+            return None
+
+        severity = Severity.MEDIUM if combined_score > 0.8 else Severity.LOW
+
+        return Finding(
+            detector=self.name,
+            issue_type="similar_columns",
+            severity=severity,
+            columns=[col1, col2],
+            details={
+                "name_similarity": round(name_sim, 3),
+                "value_similarity": round(value_sim, 3),
+                "combined_score": round(combined_score, 3),
+            },
+            message=(
+                f"Columns '{col1}' and '{col2}' appear similar "
+                f"(name: {name_sim:.0%}, values: {value_sim:.0%})"
+            ),
+            confidence=combined_score,
+        )
+
+    # -----------------------------------------------------------------
+    # HELPERS
+    # -----------------------------------------------------------------
+
+    def _normalized_levenshtein(self, s1: str, s2: str) -> float:
+        """Calculate normalized Levenshtein similarity (0 to 1)."""
+        if s1 == s2:
+            return 1.0
+        len1, len2 = len(s1), len(s2)
+        max_len = max(len1, len2)
+        if max_len == 0:
+            return 1.0
+
+        # Early termination if length difference too large
+        if abs(len1 - len2) / max_len > (1 - self._name_threshold):
+            return 0.0
+
+        # Two-row Levenshtein
+        if len1 > len2:
+            s1, s2 = s2, s1
+            len1, len2 = len2, len1
+
+        previous_row = list(range(len2 + 1))
+        for i, c1 in enumerate(s1):
+            current_row = [i + 1]
+            for j, c2 in enumerate(s2):
+                insertions = previous_row[j + 1] + 1
+                deletions = current_row[j] + 1
+                substitutions = previous_row[j] + (c1 != c2)
+                current_row.append(min(insertions, deletions, substitutions))
+            previous_row = current_row
+
+        distance = previous_row[-1]
+        return 1 - (distance / max_len)
+
+    def _jaccard_similarity(self, set1: set[str], set2: set[str]) -> float:
+        """Calculate exact Jaccard similarity."""
+        if not set1 and not set2:
+            return 1.0
+        if not set1 or not set2:
+            return 0.0
+        intersection = len(set1 & set2)
+        union = len(set1 | set2)
+        return intersection / union if union > 0 else 0.0
+
+
+register_detector(SimilarityDetector())

datawash/detectors/type_detector.py

@@ -0,0 +1,100 @@
+"""Semantic type detection using pattern matching."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from datawash.core.models import DatasetProfile, Finding, Severity
+from datawash.detectors.base import BaseDetector
+from datawash.detectors.registry import register_detector
+
+
+class TypeDetector(BaseDetector):
+    @property
+    def name(self) -> str:
+        return "types"
+
+    @property
+    def description(self) -> str:
+        return "Detects semantic types and type mismatches"
+
+    def detect(self, df: pd.DataFrame, profile: DatasetProfile) -> list[Finding]:
+        findings: list[Finding] = []
+
+        for col_name, col_profile in profile.columns.items():
+            is_string = pd.api.types.is_string_dtype(df[col_name])
+
+            # Flag numeric columns stored as strings
+            if is_string:
+                series = df[col_name].dropna()
+                if series.empty:
+                    continue
+                # Sample for large columns to avoid expensive pd.to_numeric
+                if len(series) > 1000:
+                    sample = series.sample(1000, random_state=42)
+                else:
+                    sample = series
+                numeric_count = pd.to_numeric(sample, errors="coerce").notna().sum()
+                ratio = numeric_count / len(sample)
+                if ratio > 0.8:
+                    findings.append(
+                        Finding(
+                            detector=self.name,
+                            issue_type="numeric_as_string",
+                            severity=Severity.MEDIUM,
+                            columns=[col_name],
+                            details={"numeric_ratio": round(float(ratio), 3)},
+                            message=(
+                                f"Column '{col_name}' appears numeric "
+                                f"but stored as string "
+                                f"({ratio:.0%} parseable)"
+                            ),
+                            confidence=float(ratio),
+                        )
+                    )
+
+            # Flag boolean-like columns
+            if is_string:
+                bool_values = {
+                    "true",
+                    "false",
+                    "yes",
+                    "no",
+                    "y",
+                    "n",
+                    "1",
+                    "0",
+                    "t",
+                    "f",
+                    "on",
+                    "off",
+                }
+                lowered_unique = set(
+                    df[col_name].dropna().astype(str).str.strip().str.lower().unique()
+                )
+                if lowered_unique <= bool_values and len(lowered_unique) >= 2:
+                    findings.append(
+                        Finding(
+                            detector=self.name,
+                            issue_type="boolean_as_string",
+                            severity=Severity.LOW,
+                            columns=[col_name],
+                            details={"values": sorted(lowered_unique)},
+                            message=(
+                                f"Column '{col_name}' contains "
+                                f"boolean-like values "
+                                f"stored as strings"
+                            ),
+                            confidence=0.95,
+                        )
+                    )
+
+            # Report detected semantic types from patterns
+            if col_profile.patterns:
+                for pattern_name, pattern_info in col_profile.patterns.items():
+                    col_profile.semantic_type = pattern_name
+
+        return findings
+
+
+register_detector(TypeDetector())

datawash/profiler/__init__.py

@@ -0,0 +1 @@
+from .engine import profile_dataset as profile_dataset

datawash/profiler/engine.py

@@ -0,0 +1,88 @@
+"""Main profiling orchestrator."""
+
+from __future__ import annotations
+
+import logging
+import sys
+from typing import Any
+
+import pandas as pd
+
+from datawash.core.models import ColumnProfile, DatasetProfile
+from datawash.profiler.patterns import detect_column_patterns
+from datawash.profiler.statistics import (
+    compute_categorical_stats,
+    compute_numeric_stats,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def profile_dataset(df: pd.DataFrame) -> DatasetProfile:
+    """Generate a complete profile for a DataFrame.
+
+    Args:
+        df: The DataFrame to profile.
+
+    Returns:
+        DatasetProfile with column-level and dataset-level statistics.
+    """
+    logger.info("Profiling dataset: %d rows, %d columns", len(df), len(df.columns))
+
+    columns: dict[str, ColumnProfile] = {}
+    use_progress = len(df) > 10000 or len(df.columns) > 20
+    if use_progress and sys.stderr.isatty():
+        from rich.progress import Progress
+
+        with Progress() as progress:
+            task = progress.add_task("Profiling columns...", total=len(df.columns))
+            for col_name in df.columns:
+                columns[col_name] = _profile_column(df[col_name])
+                progress.update(task, advance=1)
+    else:
+        for col_name in df.columns:
+            columns[col_name] = _profile_column(df[col_name])
+
+    return DatasetProfile(
+        row_count=len(df),
+        column_count=len(df.columns),
+        memory_bytes=int(df.memory_usage(deep=True).sum()),
+        columns=columns,
+        duplicate_row_count=int(df.duplicated().sum()),
+    )
+
+
+def _profile_column(series: pd.Series) -> ColumnProfile:
+    """Profile a single column."""
+    name = str(series.name)
+    null_count = int(series.isna().sum())
+    total = len(series)
+    unique_count = int(series.nunique())
+
+    # Compute type-appropriate statistics
+    stats: dict[str, Any] = {}
+    # Boolean columns should use categorical stats (quantile fails on bool)
+    if pd.api.types.is_bool_dtype(series):
+        stats = compute_categorical_stats(series)
+    elif pd.api.types.is_numeric_dtype(series):
+        stats = compute_numeric_stats(series)
+    else:
+        stats = compute_categorical_stats(series)
+
+    # Detect patterns
+    patterns = detect_column_patterns(series)
+
+    # Sample values (up to 5 non-null)
+    sample_values = series.dropna().head(5).tolist()
+
+    return ColumnProfile(
+        name=name,
+        dtype=str(series.dtype),
+        null_count=null_count,
+        null_ratio=round(null_count / total, 4) if total > 0 else 0.0,
+        unique_count=unique_count,
+        unique_ratio=round(unique_count / total, 4) if total > 0 else 0.0,
+        sample_values=sample_values,
+        statistics=stats,
+        patterns=patterns,
+    )

datawash/profiler/parallel.py

@@ -0,0 +1,122 @@
+"""Parallel column profiling and detector execution."""
+
+from __future__ import annotations
+
+import logging
+import os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Optional
+
+import pandas as pd
+
+from datawash.core.cache import ComputationCache
+from datawash.core.models import ColumnProfile, DatasetProfile, Finding
+from datawash.detectors.base import BaseDetector
+from datawash.profiler.patterns import detect_column_patterns
+from datawash.profiler.statistics import (
+    compute_categorical_stats,
+    compute_numeric_stats,
+)
+
+logger = logging.getLogger(__name__)
+
+MAX_WORKERS = min(8, os.cpu_count() or 4)
+
+
+def profile_dataset_parallel(
+    df: pd.DataFrame,
+    cache: Optional[ComputationCache] = None,
+) -> DatasetProfile:
+    """Profile all columns in parallel using ThreadPoolExecutor."""
+    columns: dict[str, ColumnProfile] = {}
+
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+        future_to_col = {
+            executor.submit(_profile_column, df[col_name]): col_name
+            for col_name in df.columns
+        }
+        for future in as_completed(future_to_col):
+            col_name = future_to_col[future]
+            try:
+                columns[col_name] = future.result()
+            except Exception:
+                logger.exception("Failed to profile column %s", col_name)
+                columns[col_name] = _empty_profile(col_name)
+
+    return DatasetProfile(
+        row_count=len(df),
+        column_count=len(df.columns),
+        memory_bytes=int(df.memory_usage(deep=True).sum()),
+        columns=columns,
+        duplicate_row_count=int(df.duplicated().sum()),
+    )
+
+
+def run_detectors_parallel(
+    df: pd.DataFrame,
+    profile: DatasetProfile,
+    detectors: dict[str, BaseDetector],
+) -> list[Finding]:
+    """Run all detectors in parallel."""
+    findings: list[Finding] = []
+
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+        future_to_name = {
+            executor.submit(detector.detect, df, profile): name
+            for name, detector in detectors.items()
+        }
+        for future in as_completed(future_to_name):
+            name = future_to_name[future]
+            try:
+                results = future.result()
+                findings.extend(results)
+            except Exception:
+                logger.exception("Detector %s failed", name)
+
+    return findings
+
+
+def _profile_column(series: pd.Series) -> ColumnProfile:
+    """Profile a single column (runs inside thread)."""
+    name = str(series.name)
+    null_count = int(series.isna().sum())
+    total = len(series)
+    unique_count = int(series.nunique())
+
+    stats: dict[str, Any] = {}
+    if pd.api.types.is_bool_dtype(series):
+        stats = compute_categorical_stats(series)
+    elif pd.api.types.is_numeric_dtype(series):
+        stats = compute_numeric_stats(series)
+    else:
+        stats = compute_categorical_stats(series)
+
+    patterns = detect_column_patterns(series)
+    sample_values = series.dropna().head(5).tolist()
+
+    return ColumnProfile(
+        name=name,
+        dtype=str(series.dtype),
+        null_count=null_count,
+        null_ratio=round(null_count / total, 4) if total > 0 else 0.0,
+        unique_count=unique_count,
+        unique_ratio=round(unique_count / total, 4) if total > 0 else 0.0,
+        sample_values=sample_values,
+        statistics=stats,
+        patterns=patterns,
+    )
+
+
+def _empty_profile(col_name: str) -> ColumnProfile:
+    """Fallback profile when profiling fails."""
+    return ColumnProfile(
+        name=col_name,
+        dtype="unknown",
+        null_count=0,
+        null_ratio=0.0,
+        unique_count=0,
+        unique_ratio=0.0,
+        sample_values=[],
+        statistics={},
+        patterns={},
+    )

datawash/profiler/patterns.py

@@ -0,0 +1,80 @@
+"""Pattern detection for common data formats."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+import pandas as pd
+
+PATTERNS: dict[str, re.Pattern[str]] = {
+    "email": re.compile(r"^[\w.+-]+@[\w-]+\.[\w.-]+$"),
+    "url": re.compile(r"^https?://[\w\-._~:/?#\[\]@!$&'()*+,;=]+$"),
+    "phone": re.compile(r"^[\+]?[(]?[0-9]{1,4}[)]?[-\s./0-9]{6,15}$"),
+    "ipv4": re.compile(r"^(?:\d{1,3}\.){3}\d{1,3}$"),
+    "uuid": re.compile(
+        r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE
+    ),
+    "currency": re.compile("^[$\u20ac\u00a3\u00a5][\\s]?[\\d,]+\\.?\\d*$"),
+    "zip_us": re.compile(r"^\d{5}(-\d{4})?$"),
+}
+
+DATE_FORMATS = [
+    "%Y-%m-%d",
+    "%m/%d/%Y",
+    "%d/%m/%Y",
+    "%Y/%m/%d",
+    "%m-%d-%Y",
+    "%d-%m-%Y",
+    "%Y%m%d",
+    "%b %d, %Y",
+    "%B %d, %Y",
+    "%d %b %Y",
+    "%d %B %Y",
+    "%Y-%m-%d %H:%M:%S",
+    "%m/%d/%Y %H:%M:%S",
+]
+
+
+def detect_column_patterns(series: pd.Series) -> dict[str, Any]:
+    """Detect patterns in a column's values.
+
+    Returns dict with detected pattern name and match ratio.
+    """
+    clean = series.dropna().astype(str)
+    if clean.empty:
+        return {}
+
+    sample = clean.head(1000)
+    total = len(sample)
+    results: dict[str, Any] = {}
+
+    for name, pattern in PATTERNS.items():
+        matches = sample.str.match(pattern).sum()
+        ratio = matches / total
+        if ratio > 0.5:
+            results[name] = {
+                "match_ratio": round(float(ratio), 3),
+                "pattern": pattern.pattern,
+            }
+
+    # Date detection
+    if not results:
+        date_ratio = _detect_date_pattern(sample)
+        if date_ratio and date_ratio["match_ratio"] > 0.5:
+            results["date"] = date_ratio
+
+    return results
+
+
+def _detect_date_pattern(sample: pd.Series) -> dict[str, Any] | None:
+    """Try parsing dates with common formats."""
+    for fmt in DATE_FORMATS:
+        try:
+            parsed = pd.to_datetime(sample, format=fmt, errors="coerce")
+            ratio = parsed.notna().sum() / len(sample)
+            if ratio > 0.7:
+                return {"match_ratio": round(float(ratio), 3), "format": fmt}
+        except Exception:
+            continue
+    return None