datascope-dq 2.1.0__py3-none-any.whl

datascope/__init__.py

@@ -0,0 +1,5 @@
+"""datascope -- data quality diagnostics for tabular datasets."""
+
+from __future__ import annotations
+
+__version__ = "2.1.0"

datascope/__main__.py

@@ -0,0 +1,6 @@
+"""Entry point for ``python -m datascope``."""
+
+from datascope.cli import main
+
+if __name__ == "__main__":
+    main()

datascope/analyzers/__init__.py

@@ -0,0 +1,16 @@
+"""datascope.analyzers -- detectors that produce Finding objects."""
+
+from __future__ import annotations
+
+from datascope.analyzers.type_consistency import analyze_type_consistency
+from datascope.analyzers.sentinel import analyze_sentinels
+from datascope.analyzers.format_check import analyze_leading_zeros, analyze_mixed_dates
+from datascope.analyzers.cardinality import analyze_cardinality
+
+__all__ = [
+    "analyze_type_consistency",
+    "analyze_sentinels",
+    "analyze_leading_zeros",
+    "analyze_mixed_dates",
+    "analyze_cardinality",
+]

datascope/analyzers/base.py

@@ -0,0 +1,19 @@
+"""Analyzer contract and registry.
+
+There is no abstract base class.  An analyzer is any callable matching::
+
+    Callable[[LoaderResult], list[Finding]]
+
+It receives a :class:`~datascope.models.LoaderResult` and returns zero or
+more :class:`~datascope.models.Finding` instances.  The pipeline runner
+(U9) will call each registered analyzer and merge the results.
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+from datascope.models import Finding, LoaderResult
+
+#: Type alias documenting the analyzer contract.
+Analyzer = Callable[[LoaderResult], list[Finding]]

datascope/analyzers/cardinality.py

@@ -0,0 +1,112 @@
+"""Cardinality-anomaly detector.
+
+Flags columns that are near-constant (very low uniqueness) or that look
+like ID columns with unexpected duplicates (very high but not 100%
+uniqueness).
+
+Produces :class:`~datascope.models.Finding` instances with
+:attr:`~datascope.models.FindingType.NEAR_CONSTANT` or
+:attr:`~datascope.models.FindingType.DUPLICATE_IDS`.
+
+Severity is *not* assigned here -- that is the severity classifier's job (U7).
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+
+from datascope.models import Finding, FindingType, LoaderResult
+
+
+# ---------------------------------------------------------------------------
+# Thresholds
+# ---------------------------------------------------------------------------
+
+_MIN_ROWS = 10             # Skip columns with fewer non-null rows
+_NEAR_CONSTANT_MAX = 0.01  # uniqueness_ratio < this => near-constant
+_SUSPECTED_ID_MIN = 0.95   # uniqueness_ratio > this AND < 1.0 => suspected-ID dups
+
+
+# ---------------------------------------------------------------------------
+# Detector
+# ---------------------------------------------------------------------------
+
+def analyze_cardinality(result: LoaderResult) -> list[Finding]:
+    """Detect cardinality anomalies in each column.
+
+    Two patterns are flagged:
+
+    * **Near-constant**: fewer than 1% of values are unique.
+    * **Suspected duplicate IDs**: more than 95% unique but less than
+      100%, suggesting an ID column with unexpected duplicates.
+
+    Columns with fewer than 10 non-null rows are skipped because
+    cardinality ratios are unreliable for tiny samples.
+
+    Parameters
+    ----------
+    result:
+        A :class:`~datascope.models.LoaderResult`.
+
+    Returns
+    -------
+    list[Finding]
+        One finding per column that exhibits a cardinality anomaly.
+    """
+    findings: list[Finding] = []
+
+    for col_name in result.dataframe.columns:
+        series = result.dataframe[col_name]
+        filled = series.dropna()
+        total_count = len(filled)
+
+        if total_count < _MIN_ROWS:
+            continue
+
+        unique_count = filled.nunique()
+        uniqueness_ratio = round(unique_count / total_count, 4)
+
+        if uniqueness_ratio < _NEAR_CONSTANT_MAX:
+            # Near-constant column
+            value_counts = Counter(filled)
+            top_values = [
+                {"value": str(val), "count": cnt}
+                for val, cnt in value_counts.most_common(5)
+            ]
+
+            evidence = {
+                "unique_count": unique_count,
+                "total_count": total_count,
+                "uniqueness_ratio": uniqueness_ratio,
+                "top_values": top_values,
+            }
+
+            findings.append(Finding(
+                field_name=col_name,
+                finding_type=FindingType.NEAR_CONSTANT,
+                evidence=evidence,
+            ))
+
+        elif _SUSPECTED_ID_MIN < uniqueness_ratio < 1.0:
+            # Suspected duplicate IDs
+            value_counts = Counter(filled)
+            duplicate_values = [
+                str(val)
+                for val, cnt in value_counts.most_common()
+                if cnt > 1
+            ][:10]
+
+            evidence = {
+                "unique_count": unique_count,
+                "total_count": total_count,
+                "uniqueness_ratio": uniqueness_ratio,
+                "duplicate_values": duplicate_values,
+            }
+
+            findings.append(Finding(
+                field_name=col_name,
+                finding_type=FindingType.DUPLICATE_IDS,
+                evidence=evidence,
+            ))
+
+    return findings

datascope/analyzers/format_check.py

@@ -0,0 +1,232 @@
+"""Format-inconsistency detectors: leading zeros and mixed date formats.
+
+Produces :class:`~datascope.models.Finding` instances with
+:attr:`~datascope.models.FindingType.LEADING_ZEROS` or
+:attr:`~datascope.models.FindingType.MIXED_DATES`.
+
+Severity is *not* assigned here -- that is the severity classifier's job (U7).
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime
+
+from datascope.models import Finding, FindingType, LoaderResult
+from datascope.analyzers.type_consistency import normalize_type
+
+
+# ---------------------------------------------------------------------------
+# Leading-zero detector
+# ---------------------------------------------------------------------------
+
+# Matches strings that are purely digits (at least two chars) and start with 0.
+_LEADING_ZERO_RE = re.compile(r"^0\d+$")
+
+
+def analyze_leading_zeros(result: LoaderResult) -> list[Finding]:
+    """Detect leading-zero inconsistencies within string-typed cell values.
+
+    For each column, examines cells whose Python type is ``str``.
+    Among those that consist entirely of digits, checks whether some
+    have leading zeros and others do not.  Also checks for the
+    cross-type case: a column with mostly numeric cells but some
+    string cells that carry leading zeros (e.g. ``"00123"`` alongside
+    ``123``).
+
+    Leading-zero detection is limited to string-typed cells because
+    Excel returns numeric cells as Python numbers with no original
+    string representation (openpyxl limitation).
+
+    Parameters
+    ----------
+    result:
+        A :class:`~datascope.models.LoaderResult`.
+
+    Returns
+    -------
+    list[Finding]
+        One finding per column that exhibits leading-zero
+        inconsistency.
+    """
+    findings: list[Finding] = []
+
+    for col_name, types_list in result.cell_types.items():
+        col_values = list(result.dataframe[col_name])
+
+        # Collect string-typed values that are purely digits
+        with_leading_zero: list[str] = []
+        without_leading_zero: list[str] = []
+
+        for val, ct in zip(col_values, types_list):
+            if ct is not str:
+                continue
+            if val is None:
+                continue
+            val_str = str(val)
+            # Must be purely digits and at least one char
+            if not val_str.isdigit() or len(val_str) == 0:
+                continue
+            if _LEADING_ZERO_RE.match(val_str):
+                with_leading_zero.append(val_str)
+            else:
+                without_leading_zero.append(val_str)
+
+        leading_zero_count = len(with_leading_zero)
+        no_leading_zero_count = len(without_leading_zero)
+
+        if leading_zero_count == 0:
+            # No leading zeros found at all -- nothing to report
+            continue
+
+        # Case 1: mixed within string-typed cells (some have zeros, some don't)
+        has_string_inconsistency = (
+            leading_zero_count > 0 and no_leading_zero_count > 0
+        )
+
+        # Case 2: cross-type -- string cells with leading zeros alongside
+        # numeric cells (the numeric cells implicitly lack leading zeros)
+        non_null_types = [ct for ct in types_list if ct is not type(None)]
+        normalized = [normalize_type(ct) for ct in non_null_types]
+        numeric_count = sum(1 for n in normalized if n == "numeric")
+        has_cross_type = leading_zero_count > 0 and numeric_count > 0
+
+        if not has_string_inconsistency and not has_cross_type:
+            # All string-digit values consistently have leading zeros,
+            # and no numeric cells exist -- no inconsistency.
+            continue
+
+        evidence = {
+            "leading_zero_count": leading_zero_count,
+            "no_leading_zero_count": no_leading_zero_count,
+            "examples_with_zeros": with_leading_zero[:5],
+            "examples_without_zeros": without_leading_zero[:5],
+            "total_checked": leading_zero_count + no_leading_zero_count,
+        }
+
+        findings.append(Finding(
+            field_name=col_name,
+            finding_type=FindingType.LEADING_ZEROS,
+            evidence=evidence,
+        ))
+
+    return findings
+
+
+# ---------------------------------------------------------------------------
+# Mixed-date-format detector
+# ---------------------------------------------------------------------------
+
+# Common date format patterns to try, ordered from most specific to least.
+_DATE_FORMATS: list[str] = [
+    "%Y-%m-%d",      # 2026-01-15
+    "%Y/%m/%d",      # 2026/01/15
+    "%m/%d/%Y",      # 01/15/2026
+    "%d/%m/%Y",      # 15/01/2026
+    "%m-%d-%Y",      # 01-15-2026
+    "%d-%m-%Y",      # 15-01-2026
+    "%m/%d/%y",      # 01/15/26
+    "%d/%m/%y",      # 15/01/26
+    "%Y.%m.%d",      # 2026.01.15
+    "%d.%m.%Y",      # 15.01.2026
+    "%b %d, %Y",     # Jan 15, 2026
+    "%B %d, %Y",     # January 15, 2026
+    "%d %b %Y",      # 15 Jan 2026
+    "%d %B %Y",      # 15 January 2026
+]
+
+# Quick pre-filter: a date-like string has at least one digit and a
+# separator or is in month-name form.
+_DATE_LIKE_RE = re.compile(
+    r"^\d{1,4}[\-/\.]\d{1,2}[\-/\.]\d{1,4}$"
+    r"|^\w+ \d{1,2},? \d{2,4}$"
+    r"|^\d{1,2} \w+ \d{2,4}$"
+)
+
+
+def _try_parse_date(value: str) -> str | None:
+    """Try to parse *value* with each known format.
+
+    Returns the first matching format string, or ``None`` if no
+    format matched.
+    """
+    value = value.strip()
+    for fmt in _DATE_FORMATS:
+        try:
+            datetime.strptime(value, fmt)
+            return fmt
+        except ValueError:
+            continue
+    return None
+
+
+def analyze_mixed_dates(result: LoaderResult) -> list[Finding]:
+    """Detect columns where date-like strings use multiple date formats.
+
+    Only inspects string-typed cell values that look date-like.
+    If all parseable dates within a column match a single format,
+    no finding is produced.
+
+    Parameters
+    ----------
+    result:
+        A :class:`~datascope.models.LoaderResult`.
+
+    Returns
+    -------
+    list[Finding]
+        One finding per column that contains mixed date formats.
+    """
+    findings: list[Finding] = []
+
+    for col_name, types_list in result.cell_types.items():
+        col_values = list(result.dataframe[col_name])
+
+        # Track which format each date-like string matched
+        format_to_examples: dict[str, list[str]] = {}
+        total_date_values = 0
+
+        for val, ct in zip(col_values, types_list):
+            if ct is not str:
+                continue
+            if val is None:
+                continue
+            val_str = str(val).strip()
+            if not val_str:
+                continue
+
+            # Quick pre-filter before expensive strptime calls
+            if not _DATE_LIKE_RE.match(val_str):
+                continue
+
+            fmt = _try_parse_date(val_str)
+            if fmt is None:
+                # Unparseable -- skip gracefully
+                continue
+
+            total_date_values += 1
+            if fmt not in format_to_examples:
+                format_to_examples[fmt] = []
+            if len(format_to_examples[fmt]) < 5:
+                format_to_examples[fmt].append(val_str)
+
+        if len(format_to_examples) <= 1:
+            # Zero or one format -- no inconsistency
+            continue
+
+        evidence = {
+            "formats_found": list(format_to_examples.keys()),
+            "examples_per_format": {
+                fmt: examples
+                for fmt, examples in format_to_examples.items()
+            },
+            "total_date_values": total_date_values,
+        }
+
+        findings.append(Finding(
+            field_name=col_name,
+            finding_type=FindingType.MIXED_DATES,
+            evidence=evidence,
+        ))
+
+    return findings

datascope/analyzers/sentinel.py

@@ -0,0 +1,157 @@
+"""Sentinel-value detector.
+
+Scans each column for hidden sentinel strings -- values like "N/A", "TBD",
+or "pending" that lurk inside otherwise-typed columns.  Tools silently drop
+or coerce these, so surfacing them early prevents data loss.
+
+Produces a :class:`~datascope.models.Finding` for every non-string column
+that contains sentinel values after frequency disambiguation.
+
+Severity is *not* assigned here -- that is the severity classifier's job (U7).
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+
+from datascope.models import Finding, FindingType, LoaderResult
+from datascope.analyzers.type_consistency import normalize_type
+
+
+# ---------------------------------------------------------------------------
+# Default sentinel list
+# ---------------------------------------------------------------------------
+
+DEFAULT_SENTINELS: frozenset[str] = frozenset({
+    "n/a",
+    "na",
+    "n.a.",
+    "null",
+    "none",
+    "nil",
+    "tbd",
+    "pending",
+    "unknown",
+    "missing",
+    "—",       # em-dash
+    "-",
+    ".",
+    "..",
+    "...",
+    "#n/a",
+    "#ref!",
+    "#value!",
+    "#div/0!",
+    "#name?",
+})
+
+
+# ---------------------------------------------------------------------------
+# Detector
+# ---------------------------------------------------------------------------
+
+def analyze_sentinels(
+    result: LoaderResult,
+    sentinel_list: frozenset[str] | None = None,
+) -> list[Finding]:
+    """Detect hidden sentinel values in non-string columns.
+
+    Parameters
+    ----------
+    result:
+        A :class:`~datascope.models.LoaderResult` whose ``cell_types``
+        dict maps column names to lists of Python ``type`` objects (one
+        per row).
+    sentinel_list:
+        Optional custom set of lowercase sentinel strings to match
+        against.  Defaults to :data:`DEFAULT_SENTINELS`.
+
+    Returns
+    -------
+    list[Finding]
+        One finding per non-string column that contains sentinel values
+        (after frequency disambiguation).  String/categorical columns
+        are skipped because sentinels there are not "hidden".
+    """
+    sentinels = sentinel_list if sentinel_list is not None else DEFAULT_SENTINELS
+    findings: list[Finding] = []
+
+    for col_name, types_list in result.cell_types.items():
+        # 1. Determine majority type (ignoring NoneType)
+        non_null_types = [t for t in types_list if t is not type(None)]
+        if not non_null_types:
+            continue
+
+        normalized = [normalize_type(t) for t in non_null_types]
+        type_counts = Counter(normalized)
+        majority_type = type_counts.most_common(1)[0][0]
+
+        # 2. Skip string/categorical columns -- sentinels are not hidden there
+        if majority_type == "str":
+            continue
+
+        # 3. Scan cell values for sentinel matches
+        col_values = list(result.dataframe[col_name])
+        total_non_null = sum(
+            1 for v in col_values if v is not None and str(v).strip() != ""
+        )
+
+        if total_non_null == 0:
+            continue
+
+        # Collect sentinel hits: track original-case value and count
+        sentinel_hits: Counter[str] = Counter()  # lowercase -> count
+        sentinel_originals: dict[str, str] = {}  # lowercase -> first original-case
+        for val in col_values:
+            if val is None:
+                continue
+            val_str = str(val)
+            if not val_str.strip():
+                continue
+            val_lower = val_str.lower()
+            if val_lower in sentinels:
+                sentinel_hits[val_lower] += 1
+                if val_lower not in sentinel_originals:
+                    sentinel_originals[val_lower] = val_str
+
+        if not sentinel_hits:
+            continue
+
+        # 4. Frequency disambiguation: if a specific sentinel is >50%
+        #    of non-null values, treat it as a legitimate category
+        surviving: dict[str, int] = {}
+        for val_lower, count in sentinel_hits.items():
+            if count / total_non_null <= 0.50:
+                surviving[val_lower] = count
+        sentinel_hits = Counter(surviving)
+
+        if not sentinel_hits:
+            continue
+
+        # 5. Build evidence and produce Finding
+        total_sentinel_count = sum(sentinel_hits.values())
+        sentinel_pct = round(total_sentinel_count / total_non_null * 100, 2) if total_non_null else 0.0
+
+        sentinels_found = [
+            {
+                "value": sentinel_originals[val_lower],
+                "count": count,
+                "normalized": val_lower,
+            }
+            for val_lower, count in sentinel_hits.most_common()
+        ]
+
+        evidence = {
+            "sentinels_found": sentinels_found,
+            "column_majority_type": majority_type,
+            "total_non_null": total_non_null,
+            "sentinel_pct": sentinel_pct,
+        }
+
+        findings.append(Finding(
+            field_name=col_name,
+            finding_type=FindingType.SENTINEL_VALUE,
+            evidence=evidence,
+        ))
+
+    return findings

datascope/analyzers/type_consistency.py

@@ -0,0 +1,134 @@
+"""Type-inconsistency detector.
+
+Scans each column's ``cell_types`` for mixed Python types that pandas would
+silently coerce.  Produces a :class:`~datascope.models.Finding` for every
+column where more than one non-null type is present after normalization.
+
+Severity is *not* assigned here -- that is the severity classifier's job.
+The ``evidence`` dict carries all the raw counts and examples that
+downstream stages (severity classifier, NL composer) need.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from datascope.models import Finding, FindingType, LoaderResult
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def normalize_type(t: type) -> str:
+    """Map a Python type to a canonical bucket for type-consistency analysis.
+
+    ``int`` and ``float`` are both mapped to ``"numeric"`` so that the
+    openpyxl int-vs-float distinction (which is an artefact of the Excel
+    number format, not a real type difference) does not create false
+    positives.
+
+    ``NoneType`` passes through unchanged -- callers filter it out before
+    counting.
+    """
+    if t in (int, float):
+        return "numeric"
+    return t.__name__
+
+
+def _examples_for_type(
+    values: list[Any],
+    cell_types: list[type],
+    target_type: str,
+    limit: int = 5,
+) -> list[Any]:
+    """Return up to *limit* example values whose normalized type matches *target_type*."""
+    examples: list[Any] = []
+    for val, ct in zip(values, cell_types):
+        if normalize_type(ct) == target_type:
+            examples.append(val)
+            if len(examples) >= limit:
+                break
+    return examples
+
+
+# ---------------------------------------------------------------------------
+# Detector
+# ---------------------------------------------------------------------------
+
+def analyze_type_consistency(result: LoaderResult) -> list[Finding]:
+    """Detect columns with mixed non-null types.
+
+    Parameters
+    ----------
+    result:
+        A :class:`~datascope.models.LoaderResult` whose ``cell_types``
+        dict maps column names to lists of Python ``type`` objects (one
+        per row).
+
+    Returns
+    -------
+    list[Finding]
+        One finding per column that has more than one distinct non-null
+        type after normalization.  Columns that are homogeneous (or
+        entirely null) produce no finding.
+    """
+    findings: list[Finding] = []
+
+    for col_name, types_list in result.cell_types.items():
+        # 1. Filter out NoneType (null cells don't count as a "type")
+        non_null_types = [t for t in types_list if t is not type(None)]
+
+        if not non_null_types:
+            # All null -- nothing to report
+            continue
+
+        # 2. Normalize (collapse int/float -> "numeric")
+        normalized = [normalize_type(t) for t in non_null_types]
+
+        # 3. Count types after normalization
+        counts = Counter(normalized)
+
+        if len(counts) <= 1:
+            # Homogeneous column -- no finding
+            continue
+
+        # 4. Multiple types detected -- build evidence
+        total_non_null = len(non_null_types)
+        majority_type = counts.most_common(1)[0][0]
+        majority_count = counts[majority_type]
+        majority_pct = round(majority_count / total_non_null * 100, 2)
+
+        # Column values for gathering examples
+        col_values = list(result.dataframe[col_name])
+
+        # Build minority_types list (everything except majority)
+        minority_types = []
+        for type_name, count in counts.most_common():
+            if type_name == majority_type:
+                continue
+            examples = _examples_for_type(
+                col_values, types_list, type_name, limit=5,
+            )
+            minority_types.append({
+                "type_name": type_name,
+                "count": count,
+                "examples": examples,
+            })
+
+        evidence = {
+            "majority_type": majority_type,
+            "majority_count": majority_count,
+            "minority_types": minority_types,
+            "total_non_null": total_non_null,
+            "majority_pct": majority_pct,
+        }
+
+        findings.append(Finding(
+            field_name=col_name,
+            finding_type=FindingType.TYPE_INCONSISTENCY,
+            evidence=evidence,
+        ))
+
+    return findings