datasentry-core 0.1.0__py3-none-any.whl

datasentry_core/contracts/exporters.py

@@ -0,0 +1,330 @@
+"""契约导出器（V1：Pandera / Great Expectations）。
+
+纯函数式：Contract → 外部生态格式（零运行时依赖，不引入
+pandera / great_expectations 包）。契约是可移植质量声明的
+单一来源，导出使其落地既有数据栈。
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from datasentry_core.models.contract import ColumnCheck, ColumnContract, Contract
+from datasentry_core.models.enums import RuleType
+from datasentry_core.models.rules import Rule
+
+#: 语义类型 → Pandera dtype（映射表保持收敛，未知类型回退 object）
+_PANDERA_DTYPE = {
+    "string": "pa.String()",
+    "text": "pa.String()",
+    "integer": "pa.Int64()",
+    "bigint": "pa.Int64()",
+    "float": "pa.Float64()",
+    "double": "pa.Float64()",
+    "boolean": "pa.Bool()",
+    "datetime": "pa.DateTime()",
+    "date": "pa.DateTime()",
+    "timestamp": "pa.DateTime()",
+}
+
+_GE_DTYPE = {
+    "string": "String",
+    "text": "String",
+    "integer": "Integer",
+    "bigint": "Integer",
+    "float": "Float",
+    "double": "Float",
+    "boolean": "Boolean",
+    "datetime": "DateTime",
+    "date": "DateTime",
+    "timestamp": "DateTime",
+}
+
+#: RuleType → Pandera Check 生成器（value 经 repr 嵌入代码）
+_RULE_PANDERA: dict[str, str] = {
+    RuleType.VALUE_RANGE.value: "pa.Check.ge({min}) & pa.Check.le({max})",
+    RuleType.ALLOWED_VALUES.value: "pa.Check.isin({allowed})",
+    RuleType.REGEX.value: "pa.Check.str_matches({pattern})",
+    RuleType.NOT_NULL.value: "pa.Check.not_null()",
+}
+
+
+def _pandera_checks(column: ColumnContract, name: str) -> list[str]:
+    """列契约 + 内嵌 checks → Pandera Check 表达式列表。"""
+    checks: list[str] = []
+    if column.min is not None:
+        checks.append(f"pa.Check.ge({column.min!r}, title='{name}_min')")
+    if column.max is not None:
+        checks.append(f"pa.Check.le({column.max!r}, title='{name}_max')")
+    if column.allowed_values is not None:
+        checks.append(f"pa.Check.isin({column.allowed_values!r}, title='{name}_allowed')")
+    if column.regex is not None:
+        checks.append(f"pa.Check.str_matches({column.regex!r}, title='{name}_regex')")
+    for check in column.checks:
+        checks.extend(_pandera_column_check(check, name))
+    return checks
+
+
+def _pandera_column_check(check: ColumnCheck, name: str) -> list[str]:
+    """单条 ColumnCheck → Pandera Check 表达式。"""
+    if check.type == "regex" and check.pattern:
+        return [f"pa.Check.str_matches({check.pattern!r}, title='{name}_check_regex')"]
+    if check.type == "range":
+        parts = []
+        if check.min is not None:
+            parts.append(f"pa.Check.ge({check.min!r})")
+        if check.max is not None:
+            parts.append(f"pa.Check.le({check.max!r})")
+        return parts
+    if check.type == "allowed_values" and check.allowed_values is not None:
+        return [f"pa.Check.isin({check.allowed_values!r}, title='{name}_check_allowed')"]
+    if check.type == "not_null":
+        return ["pa.Check.not_null()"]
+    if check.type == "unique":
+        return ["pa.Check.unique()"]
+    return []  # custom：不可表达，忽略
+
+
+def _pandera_rule_check(rule: Rule) -> str | None:
+    """顶层 Rule → Pandera Check 表达式；不可表达的返回 None。"""
+    if rule.type == RuleType.NOT_NULL:
+        return "pa.Check.not_null()"
+    if rule.type == RuleType.UNIQUENESS:
+        return "pa.Check.unique()"
+    if rule.when is None:
+        return None
+    cond = rule.when
+    if rule.type == RuleType.VALUE_RANGE:
+        if cond.operator == "between" and isinstance(cond.value, list) and len(cond.value) == 2:
+            return f"pa.Check.ge({cond.value[0]!r}) & pa.Check.le({cond.value[1]!r})"
+        if cond.operator in ("gte", "lte", "gt", "lt") and isinstance(cond.value, (int, float)):
+            op = {"gte": "ge", "lte": "le", "gt": "gt", "lt": "lt"}[cond.operator]
+            return f"pa.Check.{op}({cond.value!r})"
+        return None
+    if rule.type == RuleType.ALLOWED_VALUES and cond.operator == "in":
+        if isinstance(cond.value, list):
+            return f"pa.Check.isin({cond.value!r})"
+        return None
+    if rule.type == RuleType.REGEX and cond.operator == "matches" and cond.value:
+        return f"pa.Check.str_matches({str(cond.value)!r})"
+    return None
+
+
+def to_pandera(contract: Contract) -> str:
+    """Contract → Pandera DataFrameSchema 代码（字符串，用户可复制/执行）。"""
+    lines = [
+        "import pandera as pa",
+        "import pandas as pd",
+        "",
+        f"# generated by datasentry contract export from dataset: {contract.dataset.name}",
+        "# unsupported rules are listed as comments (custom/conditional/aggregate)",
+        "schema = pa.DataFrameSchema({",
+    ]
+    for name, column in contract.columns.items():
+        dtype = _PANDERA_DTYPE.get(column.type, "pa.Object()")
+        args = [f"    {name!r}: pa.Column("]
+        args.append(f"        dtype={dtype},")
+        if not column.nullable:
+            args.append("        nullable=False,")
+        if column.unique:
+            args.append("        unique=True,")
+        checks = _pandera_checks(column, name)
+        for rule in contract.rules:
+            if name not in rule.columns:
+                continue
+            expr = _pandera_rule_check(rule)
+            if expr is not None:
+                checks.append(expr)
+        if checks:
+            args.append("        checks=[")
+            args.extend(f"            {c}," for c in checks)
+            args.append("        ],")
+        args.append("    ),")
+        lines.extend(args)
+    lines.append("})")
+    unsupported = [
+        f"# unsupported rule '{r.id}' (type={r.type.value})"
+        for r in contract.rules
+        if _pandera_rule_check(r) is None
+    ]
+    if unsupported:
+        lines.append("")
+        lines.extend(unsupported)
+    return "\n".join(lines) + "\n"
+
+
+def _ge_column_expectations(name: str, column: ColumnContract) -> list[dict[str, Any]]:
+    """列契约 → GE expect_* 序列。"""
+    exps: list[dict[str, Any]] = []
+    if column.min is not None or column.max is not None:
+        kwargs: dict[str, Any] = {"column": name}
+        if column.min is not None:
+            kwargs["min_value"] = column.min
+        if column.max is not None:
+            kwargs["max_value"] = column.max
+        exps.append({"expectation_type": "expect_column_values_to_be_between", "kwargs": kwargs})
+    if column.allowed_values is not None:
+        exps.append(
+            {
+                "expectation_type": "expect_column_values_to_be_in_set",
+                "kwargs": {"column": name, "value_set": column.allowed_values},
+            }
+        )
+    if column.regex is not None:
+        exps.append(
+            {
+                "expectation_type": "expect_column_values_to_match_regex",
+                "kwargs": {"column": name, "regex": column.regex},
+            }
+        )
+    for check in column.checks:
+        exps.extend(_ge_column_check(name, check))
+    if not column.nullable:
+        exps.append(
+            {
+                "expectation_type": "expect_column_values_to_not_be_null",
+                "kwargs": {"column": name},
+            }
+        )
+    if column.unique:
+        exps.append(
+            {
+                "expectation_type": "expect_column_values_to_be_unique",
+                "kwargs": {"column": name},
+            }
+        )
+    return exps
+
+
+def _ge_column_check(name: str, check: ColumnCheck) -> list[dict[str, Any]]:
+    """单条 ColumnCheck → GE expect_* 序列。"""
+    if check.type == "regex" and check.pattern:
+        return [
+            {
+                "expectation_type": "expect_column_values_to_match_regex",
+                "kwargs": {"column": name, "regex": check.pattern},
+            }
+        ]
+    if check.type == "range":
+        if check.min is None and check.max is None:
+            return []
+        kwargs: dict[str, Any] = {"column": name}
+        if check.min is not None:
+            kwargs["min_value"] = check.min
+        if check.max is not None:
+            kwargs["max_value"] = check.max
+        return [
+            {
+                "expectation_type": "expect_column_values_to_be_between",
+                "kwargs": kwargs,
+            }
+        ]
+    if check.type == "allowed_values" and check.allowed_values is not None:
+        return [
+            {
+                "expectation_type": "expect_column_values_to_be_in_set",
+                "kwargs": {"column": name, "value_set": check.allowed_values},
+            }
+        ]
+    if check.type == "not_null":
+        return [
+            {
+                "expectation_type": "expect_column_values_to_not_be_null",
+                "kwargs": {"column": name},
+            }
+        ]
+    if check.type == "unique":
+        return [
+            {
+                "expectation_type": "expect_column_values_to_be_unique",
+                "kwargs": {"column": name},
+            }
+        ]
+    return []
+
+
+def _ge_rule_expectations(rule: Rule) -> list[dict[str, Any]]:
+    """顶层 Rule → GE expect_* 序列。"""
+    if not rule.columns:
+        return []
+    name = rule.columns[0]
+    if rule.type == RuleType.NOT_NULL:
+        return [
+            {
+                "expectation_type": "expect_column_values_to_not_be_null",
+                "kwargs": {"column": name},
+            }
+        ]
+    if rule.type == RuleType.UNIQUENESS:
+        return [
+            {
+                "expectation_type": "expect_column_values_to_be_unique",
+                "kwargs": {"column": name},
+            }
+        ]
+    if rule.when is None:
+        return []
+    cond = rule.when
+    if rule.type == RuleType.VALUE_RANGE:
+        if cond.operator == "between" and isinstance(cond.value, list) and len(cond.value) == 2:
+            return [
+                {
+                    "expectation_type": "expect_column_values_to_be_between",
+                    "kwargs": {
+                        "column": name,
+                        "min_value": cond.value[0],
+                        "max_value": cond.value[1],
+                    },
+                }
+            ]
+        if cond.operator in ("gte", "lte", "gt", "lt") and isinstance(cond.value, (int, float)):
+            exp_type = {
+                "gte": "expect_column_values_to_be_greater_than_or_equal_to",
+                "lte": "expect_column_values_to_be_less_than_or_equal_to",
+                "gt": "expect_column_values_to_be_greater_than",
+                "lt": "expect_column_values_to_be_less_than",
+            }[cond.operator]
+            return [{"expectation_type": exp_type, "kwargs": {"column": name, "value": cond.value}}]
+        return []
+    if rule.type == RuleType.ALLOWED_VALUES and cond.operator == "in":
+        if isinstance(cond.value, list):
+            return [
+                {
+                    "expectation_type": "expect_column_values_to_be_in_set",
+                    "kwargs": {"column": name, "value_set": cond.value},
+                }
+            ]
+        return []
+    if rule.type == RuleType.REGEX and cond.operator == "matches" and cond.value:
+        return [
+            {
+                "expectation_type": "expect_column_values_to_match_regex",
+                "kwargs": {"column": name, "regex": str(cond.value)},
+            }
+        ]
+    return []
+
+
+def to_great_expectations(contract: Contract) -> dict[str, Any]:
+    """Contract → GE ExpectationSuite 字典（完整 suite 文档）。"""
+    expectations: list[dict[str, Any]] = []
+    for name, column in contract.columns.items():
+        expectations.extend(_ge_column_expectations(name, column))
+        if column.type in _GE_DTYPE:
+            expectations.append(
+                {
+                    "expectation_type": "expect_column_values_to_be_of_type",
+                    "kwargs": {"column": name, "type_": _GE_DTYPE[column.type]},
+                }
+            )
+    for rule in contract.rules:
+        expectations.extend(_ge_rule_expectations(rule))
+    return {
+        "expectation_suite_name": f"{contract.dataset.name}.datasentry",
+        "expectations": expectations,
+        "meta": {
+            "datasentry_version": contract.version,
+            "dataset": contract.dataset.name,
+            "description": contract.dataset.description,
+        },
+    }

datasentry_core/detectors/__init__.py

@@ -0,0 +1,15 @@
+"""检测器包（Step 5 注册表 + Step 6 起接入确定性检测器）。"""
+
+from datasentry_core.detectors.base import (
+    DetectionContext,
+    Detector,
+    DetectorRegistry,
+    filter_by_config,
+)
+
+__all__ = [
+    "DetectionContext",
+    "Detector",
+    "DetectorRegistry",
+    "filter_by_config",
+]

datasentry_core/detectors/base.py

@@ -0,0 +1,120 @@
+"""检测器注册表（11.1）：注册/查询/能力过滤/启停。
+
+Detector 协议与规格 11 章一致；`metadata()` 为可选扩展（UI 与文档生成用）。
+"""
+
+from __future__ import annotations
+
+import builtins
+import threading
+from collections.abc import Sequence
+from typing import ClassVar, Protocol, runtime_checkable
+
+from datasentry_core.connectors.base import DataHandle
+from datasentry_core.models.contract import TableReference
+from datasentry_core.models.detector import DetectorMeta, IssueCandidate
+from datasentry_core.models.enums import QualityDimension
+from datasentry_core.models.profile import DatasetProfile
+from datasentry_core.models.scan import ScanConfig
+
+
+class DetectionContext:
+    """一次检测的上下文：句柄 + 扫描配置 + 可选画像/采样。
+
+    懒加载字段由调度层注入（Step 7），检测器不得自行打开数据源。
+    """
+
+    def __init__(
+        self,
+        dataset_id: str,
+        table_name: str | None,
+        columns: list[str],
+        handle: DataHandle,
+        config: ScanConfig | None = None,
+        profile: DatasetProfile | None = None,
+        sample_rows: int | None = None,
+        references: list[TableReference] | None = None,
+    ) -> None:
+        self.dataset_id = dataset_id
+        self.table_name = table_name
+        self.columns = columns
+        self.handle = handle
+        self.config = config
+        self.profile = profile
+        self.sample_rows = sample_rows
+        self.references: list[TableReference] = references or []
+
+
+@runtime_checkable
+class Detector(Protocol):
+    """检测器协议（11 章）。"""
+
+    detector_id: ClassVar[str]
+    detector_version: ClassVar[str]
+    quality_dimension: ClassVar[QualityDimension]
+
+    def supports(self, context: DetectionContext) -> bool: ...
+
+    def detect(self, context: DetectionContext) -> list[IssueCandidate]: ...
+
+    def metadata(self) -> DetectorMeta: ...
+
+
+class DetectorRegistry:
+    """线程安全注册表：register/get/list/enable（11.1）。"""
+
+    def __init__(self) -> None:
+        self._detectors: dict[str, Detector] = {}
+        self._enabled: set[str] = set()
+        self._lock = threading.RLock()
+
+    def register(self, detector: Detector) -> None:
+        """注册检测器；重复 detector_id 抛 ValueError。"""
+        with self._lock:
+            if detector.detector_id in self._detectors:
+                raise ValueError(f"detector already registered: {detector.detector_id}")
+            self._detectors[detector.detector_id] = detector
+            self._enabled.add(detector.detector_id)
+
+    def get(self, detector_id: str) -> Detector:
+        with self._lock:
+            try:
+                return self._detectors[detector_id]
+            except KeyError as exc:
+                raise KeyError(f"detector not registered: {detector_id}") from exc
+
+    def list(self, capability: str | None = None) -> list[Detector]:
+        """按注册顺序返回；capability 按 DetectorCapabilities 字段名过滤（True）。"""
+        with self._lock:
+            detectors = list(self._detectors.values())
+        if capability is None:
+            return detectors
+        return [d for d in detectors if getattr(d.metadata().capabilities, capability, False)]
+
+    def enable(self, detector_id: str, enabled: bool) -> None:
+        with self._lock:
+            if detector_id not in self._detectors:
+                raise KeyError(f"detector not registered: {detector_id}")
+            if enabled:
+                self._enabled.add(detector_id)
+            else:
+                self._enabled.discard(detector_id)
+
+    def is_enabled(self, detector_id: str) -> bool:
+        with self._lock:
+            return detector_id in self._enabled
+
+    def list_active(self) -> builtins.list[Detector]:
+        """启用的检测器（按注册顺序）；ScanConfig.detectors 过滤在调度层做。"""
+        with self._lock:
+            return [d for d in self._detectors.values() if d.detector_id in self._enabled]
+
+
+def filter_by_config(
+    detectors: Sequence[Detector], config_detectors: list[str] | None
+) -> list[Detector]:
+    """ScanConfig.detectors 白名单过滤（None = 全部启用）。"""
+    if config_detectors is None:
+        return list(detectors)
+    selected = set(config_detectors)
+    return [d for d in detectors if d.detector_id in selected]

datasentry_core/detectors/common.py

@@ -0,0 +1,156 @@
+"""确定性检测器公共设施：基类、SQL 工具、证据/候选构造（Step 6）。"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any, ClassVar
+
+from datasentry_core.detectors.base import DetectionContext
+from datasentry_core.models.detector import DetectorCapabilities, DetectorMeta, IssueCandidate
+from datasentry_core.models.enums import EvidenceType, QualityDimension, Severity
+from datasentry_core.models.evidence import Evidence
+
+
+def quote_ident(column: str) -> str:
+    """SQL 标识符引号转义。"""
+    return '"' + column.replace('"', '""') + '"'
+
+
+def quote_literal(value: str) -> str:
+    """SQL 单引号字面量转义。"""
+    return "'" + value.replace("'", "''") + "'"
+
+
+def quote_re(pattern: str) -> str:
+    """正则字面量（SQL 单引号转义）。"""
+    return quote_literal(pattern)
+
+
+def make_evidence(
+    detector_id: str,
+    detector_version: str,
+    evidence_type: EvidenceType,
+    description: str,
+    data: dict[str, Any] | None = None,
+    confidence: float = 1.0,
+) -> Evidence:
+    return Evidence(
+        evidence_id=f"ev_{uuid.uuid4().hex[:12]}",
+        evidence_type=evidence_type,
+        detector_id=detector_id,
+        detector_version=detector_version,
+        description=description,
+        data=data or {},
+        confidence=confidence,
+    )
+
+
+def make_candidate(
+    detector_id: str,
+    detector_version: str,
+    context: DetectionContext,
+    issue_type: str,
+    columns: list[str],
+    affected_count: int,
+    evidence: list[Evidence],
+    raw_score: float,
+    confidence: float,
+    severity: Severity,
+    fpr: float = 0.1,
+    affected_rows: list[str] | None = None,
+) -> IssueCandidate:
+    return IssueCandidate(
+        issue_type=issue_type,
+        detector_id=detector_id,
+        detector_version=detector_version,
+        dataset_id=context.dataset_id,
+        table_name=context.table_name,
+        columns=columns,
+        affected_rows=affected_rows,
+        affected_count=affected_count,
+        evidence=evidence,
+        raw_score=raw_score,
+        confidence=confidence,
+        estimated_false_positive_risk=fpr,
+        suggested_severity=severity.value,
+    )
+
+
+_STRING_TYPES = frozenset({"VARCHAR", "CHAR", "TEXT", "BPCHAR"})
+# TIME 列不与 DATE/TIMESTAMP 互比（类型错误），MVP 不使用
+_TEMPORAL_TYPES = frozenset({"DATE", "TIMESTAMP", "TIMESTAMPTZ"})
+_NUMERIC_TYPES = frozenset(
+    {
+        "TINYINT",
+        "SMALLINT",
+        "INTEGER",
+        "BIGINT",
+        "HUGEINT",
+        "UTINYINT",
+        "USMALLINT",
+        "UINTEGER",
+        "UBIGINT",
+        "UHUGEINT",
+        "FLOAT",
+        "REAL",
+        "DOUBLE",
+        "DECIMAL",
+    }
+)
+
+
+def string_columns(context: DetectionContext) -> list[str]:
+    """字符串列名（按 schema 顺序）。"""
+    return [
+        c.name for c in context.handle.schema().columns if c.physical_type.upper() in _STRING_TYPES
+    ]
+
+
+def numeric_columns(context: DetectionContext) -> list[str]:
+    """数值列名（按 schema 顺序）。"""
+    return [
+        c.name for c in context.handle.schema().columns if c.physical_type.upper() in _NUMERIC_TYPES
+    ]
+
+
+def datetime_columns(context: DetectionContext) -> list[str]:
+    """日期时间列名（物理类型 DATE/TIMESTAMP/TIME，按 schema 顺序）。"""
+    return [
+        c.name
+        for c in context.handle.schema().columns
+        if c.physical_type.upper() in _TEMPORAL_TYPES
+    ]
+
+
+class DetectorBase:
+    """检测器基类：类属性声明元数据，metadata() 统一构建。"""
+
+    detector_id: ClassVar[str] = ""
+    detector_version: ClassVar[str] = "1.0.0"
+    display_name: ClassVar[str] = ""
+    description: ClassVar[str] = ""
+    quality_dimension: ClassVar[QualityDimension] = QualityDimension.VALIDITY
+    capabilities: ClassVar[DetectorCapabilities] = DetectorCapabilities()
+    needs_llm: ClassVar[bool] = False
+    requires_reference: ClassVar[bool] = False
+    experimental: ClassVar[bool] = False
+    default_thresholds: ClassVar[dict[str, float | int | str]] = {}
+
+    def supports(self, context: DetectionContext) -> bool:
+        return bool(context.columns)
+
+    def detect(self, context: DetectionContext) -> list[IssueCandidate]:
+        raise NotImplementedError
+
+    def metadata(self) -> DetectorMeta:
+        return DetectorMeta(
+            detector_id=self.detector_id,
+            display_name=self.display_name,
+            description=self.description,
+            quality_dimension=self.quality_dimension,
+            capabilities=self.capabilities,
+            default_thresholds=dict(self.default_thresholds),
+            needs_llm=self.needs_llm,
+            requires_reference=self.requires_reference,
+            experimental=self.experimental,
+        )