datasentry-core 0.1.0__py3-none-any.whl

datasentry_core/detectors/initial/categorical.py

@@ -0,0 +1,252 @@
+"""类别异常检测器（11.6）：placeholder / rare_category / category_explosion / inconsistent_case。"""
+
+from __future__ import annotations
+
+from datetime import date, datetime
+from typing import Any, ClassVar
+
+from datasentry_core.detectors.base import DetectionContext
+from datasentry_core.detectors.common import (
+    DetectorBase,
+    make_candidate,
+    make_evidence,
+    quote_ident,
+    quote_literal,
+    string_columns,
+)
+from datasentry_core.models.detector import DetectorCapabilities, IssueCandidate
+from datasentry_core.models.enums import EvidenceType, QualityDimension, Severity
+
+
+def _json_safe(value: Any) -> Any:
+    """evidence 数据必须 JSON 可序列化：date/datetime 转 ISO 字符串。"""
+    if isinstance(value, (date, datetime)):
+        return value.isoformat()
+    return value
+
+
+_PLACEHOLDERS = ("test", "xxx", "foo", "abc", "12345", "dummy", "example")
+_IDENTIFIER_HINTS = ("_id", "id_", " id", "key", "code", "uuid", "hash")
+
+
+class SuspiciousPlaceholderDetector(DetectorBase):
+    """占位符值（11.6）：匹配即报告。"""
+
+    detector_id = "suspicious_placeholder"
+    display_name = "Suspicious Placeholder"
+    description = "Reports values like 'test', 'xxx', 'foo' that are likely placeholders."
+    quality_dimension = QualityDimension.VALIDITY
+    capabilities: ClassVar[DetectorCapabilities] = DetectorCapabilities(supports_sql_pushdown=True)
+    default_thresholds: ClassVar[dict[str, float | int | str]] = {
+        "placeholder_count": len(_PLACEHOLDERS)
+    }
+
+    def supports(self, context: DetectionContext) -> bool:
+        return bool(string_columns(context))
+
+    def detect(self, context: DetectionContext) -> list[IssueCandidate]:
+        candidates: list[IssueCandidate] = []
+        for col in string_columns(context):
+            q = quote_ident(col)
+            tokens = ", ".join(quote_literal(t) for t in _PLACEHOLDERS)
+            table = context.handle.sql_aggregate(
+                f"SELECT count(*) AS n FROM data WHERE lower(trim({q})) IN ({tokens})"
+            ).table
+            count = int(table.column("n").to_pylist()[0])
+            if count > 0:
+                candidates.append(
+                    make_candidate(
+                        detector_id=self.detector_id,
+                        detector_version=self.detector_version,
+                        context=context,
+                        issue_type="suspicious_placeholder",
+                        columns=[col],
+                        affected_count=count,
+                        evidence=[
+                            make_evidence(
+                                detector_id=self.detector_id,
+                                detector_version=self.detector_version,
+                                evidence_type=EvidenceType.PATTERN_MATCH,
+                                description=f"{count} placeholder values",
+                                data={"count": count},
+                            )
+                        ],
+                        raw_score=count,
+                        confidence=0.95,
+                        severity=Severity.LOW,
+                    )
+                )
+        return candidates
+
+
+class RareCategoryDetector(DetectorBase):
+    """稀有类别（11.6）：频数 < 5 且占比 < 0.001；仅高频字段（distinct 2~1000）启用。"""
+
+    detector_id = "rare_category"
+    display_name = "Rare Category"
+    description = "Reports categories with very low frequency in high-cardinality fields."
+    quality_dimension = QualityDimension.VALIDITY
+    capabilities: ClassVar[DetectorCapabilities] = DetectorCapabilities(supports_sql_pushdown=True)
+    default_thresholds: ClassVar[dict[str, float | int | str]] = {
+        "min_count": 5,
+        "min_ratio": 0.001,
+    }
+
+    def detect(self, context: DetectionContext) -> list[IssueCandidate]:
+        candidates: list[IssueCandidate] = []
+        for col in context.columns:
+            q = quote_ident(col)
+            meta = context.handle.sql_aggregate(
+                f"SELECT count(*) AS n, count(DISTINCT {q}) AS d, count({q}) AS nn FROM data"
+            ).table
+            row = meta.to_pylist()[0]
+            total = int(row["n"])
+            distinct = int(row["d"])
+            non_null = int(row["nn"])
+            if total <= 0:
+                continue
+            unique_ratio = distinct / non_null if non_null else 1.0
+            if not (2 <= distinct <= 1000) or unique_ratio > 0.9:
+                continue  # 单值/爆炸列不适用（category_explosion 处理后者）
+            table = context.handle.sql_aggregate(
+                f"SELECT {q} AS v, count(*) AS n FROM data "
+                f"WHERE {q} IS NOT NULL GROUP BY {q} "
+                f"HAVING count(*) < 5 AND count(*)::DOUBLE * 1000.0 < {total} "
+                f"ORDER BY n LIMIT 20"
+            ).table
+            values = table.column("v").to_pylist()
+            counts = table.column("n").to_pylist()
+            if not values:
+                continue
+            candidates.append(
+                make_candidate(
+                    detector_id=self.detector_id,
+                    detector_version=self.detector_version,
+                    context=context,
+                    issue_type="rare_category",
+                    columns=[col],
+                    affected_count=sum(int(c) for c in counts),
+                    evidence=[
+                        make_evidence(
+                            detector_id=self.detector_id,
+                            detector_version=self.detector_version,
+                            evidence_type=EvidenceType.STATISTICAL_MEASURE,
+                            description=f"{len(values)} rare categories",
+                            data={
+                                "categories": [
+                                    (_json_safe(v), int(c))
+                                    for v, c in zip(values, counts, strict=True)
+                                ]
+                            },
+                        )
+                    ],
+                    raw_score=len(values),
+                    confidence=0.85,
+                    severity=Severity.LOW,
+                )
+            )
+        return candidates
+
+
+class CategoryExplosionDetector(DetectorBase):
+    """类别爆炸（11.6）：唯一值比例 > 0.9 且列名不像标识符。"""
+
+    detector_id = "category_explosion"
+    display_name = "Category Explosion"
+    description = "Flags near-unique columns that may be free-text masquerading as categories."
+    quality_dimension = QualityDimension.VALIDITY
+    capabilities: ClassVar[DetectorCapabilities] = DetectorCapabilities(supports_sql_pushdown=True)
+    default_thresholds: ClassVar[dict[str, float | int | str]] = {"unique_ratio_threshold": 0.9}
+
+    def detect(self, context: DetectionContext) -> list[IssueCandidate]:
+        candidates: list[IssueCandidate] = []
+        for col in context.columns:
+            lowered = col.lower()
+            if any(hint in lowered for hint in _IDENTIFIER_HINTS):
+                continue
+            q = quote_ident(col)
+            table = context.handle.sql_aggregate(
+                f"SELECT count({q}) AS nn, count(DISTINCT {q}) AS d FROM data"
+            ).table
+            row = table.to_pylist()[0]
+            non_null = int(row["nn"])
+            distinct = int(row["d"])
+            if non_null <= 0:
+                continue
+            ratio = distinct / non_null
+            if ratio > 0.9:
+                candidates.append(
+                    make_candidate(
+                        detector_id=self.detector_id,
+                        detector_version=self.detector_version,
+                        context=context,
+                        issue_type="category_explosion",
+                        columns=[col],
+                        affected_count=distinct,
+                        evidence=[
+                            make_evidence(
+                                detector_id=self.detector_id,
+                                detector_version=self.detector_version,
+                                evidence_type=EvidenceType.STATISTICAL_MEASURE,
+                                description=f"unique_ratio={ratio:.3f} > 0.9",
+                                data={"unique_ratio": round(ratio, 6), "distinct": distinct},
+                            )
+                        ],
+                        raw_score=ratio,
+                        confidence=0.8,
+                        severity=Severity.LOW,
+                    )
+                )
+        return candidates
+
+
+class InconsistentCaseDetector(DetectorBase):
+    """大小写不一致（11.6）：归一化（小写）后同一值出现 ≥2 种形态（如 California/california）。"""
+
+    detector_id = "inconsistent_case"
+    display_name = "Inconsistent Case"
+    description = "Reports normalized values that appear in multiple case forms."
+    quality_dimension = QualityDimension.VALIDITY
+    capabilities: ClassVar[DetectorCapabilities] = DetectorCapabilities(supports_sql_pushdown=True)
+    default_thresholds: ClassVar[dict[str, float | int | str]] = {"min_group_rows": 2}
+
+    def detect(self, context: DetectionContext) -> list[IssueCandidate]:
+        candidates: list[IssueCandidate] = []
+        for col in string_columns(context):
+            q = quote_ident(col)
+            table = context.handle.sql_aggregate(
+                f"SELECT lower(trim({q})) AS k, count(*) AS n, count(DISTINCT trim({q})) AS forms "
+                f"FROM data WHERE {q} IS NOT NULL "
+                f"GROUP BY lower(trim({q})) "
+                f"HAVING count(DISTINCT trim({q})) >= 2 AND count(*) >= 2 "
+                f"ORDER BY n DESC LIMIT 20"
+            ).table
+            keys = table.column("k").to_pylist()
+            counts = table.column("n").to_pylist()
+            forms = table.column("forms").to_pylist()
+            if not keys:
+                continue
+            affected = sum(int(c) for c in counts)
+            candidates.append(
+                make_candidate(
+                    detector_id=self.detector_id,
+                    detector_version=self.detector_version,
+                    context=context,
+                    issue_type="inconsistent_case",
+                    columns=[col],
+                    affected_count=affected,
+                    evidence=[
+                        make_evidence(
+                            detector_id=self.detector_id,
+                            detector_version=self.detector_version,
+                            evidence_type=EvidenceType.STATISTICAL_MEASURE,
+                            description=f"{affected} rows in {len(keys)} case-variant groups",
+                            data={"groups": list(zip(keys, counts, forms, strict=True))},
+                        )
+                    ],
+                    raw_score=len(keys),
+                    confidence=0.9,
+                    severity=Severity.LOW,
+                )
+            )
+        return candidates

datasentry_core/detectors/initial/formula.py

@@ -0,0 +1,66 @@
+"""公式注入检测器（11.7）：值以 = + - @ \\t \\r 开头（导出风险）。"""
+
+from __future__ import annotations
+
+from typing import ClassVar
+
+from datasentry_core.detectors.base import DetectionContext
+from datasentry_core.detectors.common import (
+    DetectorBase,
+    make_candidate,
+    make_evidence,
+    quote_ident,
+    quote_literal,
+    string_columns,
+)
+from datasentry_core.models.detector import DetectorCapabilities, IssueCandidate
+from datasentry_core.models.enums import EvidenceType, QualityDimension, Severity
+
+FORMULA_INJECTION_PREFIXES: tuple[str, ...] = ("=", "+", "-", "@", "\t", "\r")
+
+
+class FormulaInjectionDetector(DetectorBase):
+    """公式注入（11.7）：CSV 导出时可能被电子表格执行。"""
+
+    detector_id = "suspicious_formula_injection"
+    display_name = "Formula Injection"
+    description = "Reports values starting with spreadsheet formula prefixes."
+    quality_dimension = QualityDimension.VALIDITY
+    capabilities: ClassVar[DetectorCapabilities] = DetectorCapabilities(supports_sql_pushdown=True)
+
+    def detect(self, context: DetectionContext) -> list[IssueCandidate]:
+        candidates: list[IssueCandidate] = []
+        prefixes = ", ".join(
+            f"chr({ord(p)})" if ord(p) < 32 else quote_literal(p)
+            for p in FORMULA_INJECTION_PREFIXES
+        )
+        for col in string_columns(context):
+            q = quote_ident(col)
+            table = context.handle.sql_aggregate(
+                f"SELECT count(*) AS n FROM data WHERE left({q}, 1) IN ({prefixes})"
+            ).table
+            count = int(table.column("n").to_pylist()[0])
+            if count > 0:
+                candidates.append(
+                    make_candidate(
+                        detector_id=self.detector_id,
+                        detector_version=self.detector_version,
+                        context=context,
+                        issue_type="suspicious_formula_injection",
+                        columns=[col],
+                        affected_count=count,
+                        evidence=[
+                            make_evidence(
+                                detector_id=self.detector_id,
+                                detector_version=self.detector_version,
+                                evidence_type=EvidenceType.PATTERN_MATCH,
+                                description=f"{count} values start with formula-injection prefixes",
+                                data={"count": count},
+                            )
+                        ],
+                        raw_score=count,
+                        confidence=0.95,
+                        severity=Severity.LOW,
+                    )
+                )
+        return candidates

datasentry_core/detectors/initial/missing.py

@@ -0,0 +1,150 @@
+"""缺失类检测器（11.3）：excessive_null_rate / suspicious_missing_token。"""
+
+from __future__ import annotations
+
+from typing import ClassVar
+
+from datasentry_core.detectors.base import DetectionContext
+from datasentry_core.detectors.common import (
+    DetectorBase,
+    make_candidate,
+    make_evidence,
+    quote_ident,
+    quote_literal,
+    string_columns,
+)
+from datasentry_core.models.detector import DetectorCapabilities, IssueCandidate
+from datasentry_core.models.enums import EvidenceType, QualityDimension, Severity
+
+_MISSING_TOKENS = (
+    "na",
+    "n/a",
+    "null",
+    "none",
+    "-",
+    "?",
+    "unknown",
+    "missing",
+    "todo",
+    "tbd",
+    "n.a.",
+)
+
+
+class ExcessiveNullRateDetector(DetectorBase):
+    """缺失率过高（11.3）：null_rate > threshold（默认 0.05，critical 列收紧至 0.01）。"""
+
+    detector_id = "excessive_null_rate"
+    display_name = "Excessive Null Rate"
+    description = "Reports columns whose null ratio exceeds the configured threshold."
+    quality_dimension = QualityDimension.COMPLETENESS
+    capabilities: ClassVar[DetectorCapabilities] = DetectorCapabilities(supports_sql_pushdown=True)
+    default_thresholds: ClassVar[dict[str, float | int | str]] = {
+        "threshold": 0.05,
+        "critical_threshold": 0.01,
+    }
+
+    def detect(self, context: DetectionContext) -> list[IssueCandidate]:
+        # MVP 固定默认阈值；可配置化经 YAML 契约（Step 13）与 detector options 接入
+        threshold = 0.05
+        candidates: list[IssueCandidate] = []
+        for col in context.columns:
+            q = quote_ident(col)
+            table = context.handle.sql_aggregate(
+                f"SELECT count(*) AS n, sum({q} IS NULL) AS nulls FROM data"
+            ).table
+            total = int(table.column("n").to_pylist()[0])
+            nulls = int(table.column("nulls").to_pylist()[0])
+            if total <= 0:
+                continue
+            ratio = nulls / total
+            if ratio > threshold:
+                severity = (
+                    Severity.HIGH
+                    if ratio > 0.3
+                    else Severity.MEDIUM
+                    if ratio > 0.1
+                    else Severity.LOW
+                )
+                candidates.append(
+                    make_candidate(
+                        detector_id=self.detector_id,
+                        detector_version=self.detector_version,
+                        context=context,
+                        issue_type="excessive_null_rate",
+                        columns=[col],
+                        affected_count=nulls,
+                        evidence=[
+                            make_evidence(
+                                detector_id=self.detector_id,
+                                detector_version=self.detector_version,
+                                evidence_type=EvidenceType.STATISTICAL_MEASURE,
+                                description=f"null_ratio={ratio:.4f} exceeds threshold={threshold}",
+                                data={
+                                    "null_ratio": round(ratio, 6),
+                                    "threshold": threshold,
+                                    "nulls": nulls,
+                                    "total": total,
+                                },
+                            )
+                        ],
+                        raw_score=ratio,
+                        confidence=0.95,
+                        severity=severity,
+                    )
+                )
+        return candidates
+
+
+class SuspiciousMissingTokenDetector(DetectorBase):
+    """特殊缺失标记（11.3）：标记占比 > 0.005。"""
+
+    detector_id = "suspicious_missing_token"
+    display_name = "Suspicious Missing Token"
+    description = "Reports values that are common stand-ins for missing data."
+    quality_dimension = QualityDimension.COMPLETENESS
+    capabilities: ClassVar[DetectorCapabilities] = DetectorCapabilities(supports_sql_pushdown=True)
+    default_thresholds: ClassVar[dict[str, float | int | str]] = {"ratio_threshold": 0.005}
+
+    def supports(self, context: DetectionContext) -> bool:
+        return bool(string_columns(context))
+
+    def detect(self, context: DetectionContext) -> list[IssueCandidate]:
+        threshold = 0.005
+        candidates: list[IssueCandidate] = []
+        for col in string_columns(context):
+            q = quote_ident(col)
+            tokens = ", ".join(quote_literal(t) for t in _MISSING_TOKENS)
+            table = context.handle.sql_aggregate(
+                f"SELECT count(*) AS n FROM data WHERE lower(trim({q})) IN ({tokens})"
+            ).table
+            count = int(table.column("n").to_pylist()[0])
+            if count == 0:
+                continue
+            table_total = context.handle.sql_aggregate("SELECT count(*) AS n FROM data").table
+            total = int(table_total.column("n").to_pylist()[0])
+            ratio = count / total if total else 0.0
+            if ratio > threshold:
+                candidates.append(
+                    make_candidate(
+                        detector_id=self.detector_id,
+                        detector_version=self.detector_version,
+                        context=context,
+                        issue_type="suspicious_missing_token",
+                        columns=[col],
+                        affected_count=count,
+                        evidence=[
+                            make_evidence(
+                                detector_id=self.detector_id,
+                                detector_version=self.detector_version,
+                                evidence_type=EvidenceType.PATTERN_MATCH,
+                                description=f"{count} values are missing stand-ins ({ratio:.4f})",
+                                data={"count": count, "ratio": round(ratio, 6)},
+                            )
+                        ],
+                        raw_score=ratio,
+                        confidence=0.9,
+                        severity=Severity.LOW,
+                    )
+                )
+        return candidates