datasentry-core 0.7.0__tar.gz → 0.8.0__tar.gz

datasentry_core-0.8.0/.gitignore

@@ -0,0 +1,43 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+dist/
+build/
+.coverage
+htmlcov/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+hypothesis/
+.hypothesis/
+
+# Project data
+.datasentry/
+*.db
+*.sqlite3
+
+# Env / secrets
+.env
+credentials.json
+
+# OS
+.DS_Store
+
+# Editors
+.idea/
+.vscode/
+~$*
+workspace/
+
+# Debug scratch (agent sessions)
+p/
+
+# 开发任务书（prompt）——本地保留，不进仓库（V22 起约定）
+docs/V*_DEV_PROMPT.md
+
+# 完整产品设计与开发 Prompt（本地保留，不进仓库）
+DataSentry AI 产品设计与开发 Prompt*.md
+DataSentry AI 产品设计与开发 Prompt*.docx
+DataSentry AI 增长与推广 Prompt.md

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datasentry-core
-Version: 0.7.0
+Version: 0.8.0
 Summary: DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs).
 License: Apache-2.0
 Keywords: data-observability,data-quality,duckdb

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datasentry-core"
-version = "0.7.0"
+version = "0.8.0"
 description = "DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs)."
 readme = "README.md"
 requires-python = ">=3.12"

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/src/datasentry_core/__init__.py

@@ -108,4 +108,4 @@ __all__ = [
     "ValidationResult",
 ]
 
-__version__ = "0.7.0"
+__version__ = "1.0.0"

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/csv.py

@@ -33,6 +33,9 @@ from datasentry_core.engine import DuckDBExecutor
 from datasentry_core.models.enums import Severity
 from datasentry_core.models.fingerprint import DatasetFingerprint
 
+#: 非 utf-8 全文件入内存的警告阈值（Step 73/ADR-073）
+_NON_UTF8_FULL_READ_WARN_BYTES = 512 * 1024 * 1024
+
 FORMULA_INJECTION_PREFIXES: tuple[str, ...] = ("=", "+", "-", "@", "\t", "\r")
 _WARNING_CAP = 100
 
@@ -132,6 +135,22 @@ class CsvDataHandle:
                 f"delim={_sql_string_literal(self._delimiter)})"
             )
         else:
+            # Step 73/ADR-073：非 utf-8 全文件入内存（pyarrow 无流式编码读取），
+            # 超阈值时预置提示 warning，引导 --sampling 或转码
+            if self._path.stat().st_size > _NON_UTF8_FULL_READ_WARN_BYTES:
+                self._warnings = [
+                    LoadWarning(
+                        row=0,
+                        column="",
+                        message=(
+                            "non-utf-8 csv is fully loaded into memory "
+                            f"({self._path.stat().st_size} bytes > "
+                            f"{_NON_UTF8_FULL_READ_WARN_BYTES}); "
+                            "use --sampling or re-encode as utf-8"
+                        ),
+                        severity=Severity.LOW,
+                    )
+                ]
             table = pa_csv.read_csv(
                 self._path,
                 read_options=pa_csv.ReadOptions(encoding=self._encoding),

datasentry_core-0.8.0/src/datasentry_core/connectors/sampling.py

@@ -0,0 +1,141 @@
+"""抽样数据句柄（Step 71/73，ADR-071/073）：检测器 SQL 顶层 FROM data 改写。
+
+零连接器改动：与底层句柄共享 executor，首次数据访问时把抽样子集物化为
+TEMP TABLE（reservoir + REPEATABLE(seed) 可复现，20.3），后续检测器
+查询直接读内存表（ADR-073：避免每检测器重复 reservoir 重扫 1e6 行，
+bench 抽样档 52s→~15s、峰值 449MB→<300MB）；无 executor 的连接器
+（协议兜底）退回抽样子查询重写。
+count_rows() 返回抽样行数（该句柄实际可观测的行数）。
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from pathlib import Path
+
+from datasentry_core.connectors.base import (
+    DataHandle,
+    FingerprintMode,
+    FrameBatch,
+    LoadWarning,
+    SamplingMethod,
+    SchemaInfo,
+)
+from datasentry_core.connectors.spec import DataSourceType
+from datasentry_core.models.fingerprint import DatasetFingerprint
+
+#: 仓库内检测器/规则/画像 SQL 一律以顶层 FROM data 引用视图（50 处均为该形态）。
+#: 重写后残留（子查询内部）不匹配 lookahead，见 _rewrite 守卫。
+_FROM_DATA_RE = re.compile(r"\bFROM data\b")
+
+#: 物化 TEMP TABLE 名（与 data 视图同 executor 连接内，无命名冲突）
+_SAMPLED_TABLE = "sampled_data"
+
+
+class SampledDataHandle:
+    """包装句柄：把对 data 视图的查询改写为抽样数据（顶层 FROM data 单形态）。
+
+    首次数据访问物化 TEMP TABLE（每次检测器查询复用，避免重复 reservoir
+    重扫）；不支持 read_batches（抽样语义面向 SQL 下推）；read_sample
+    委托底层（对抽样数据再抽样无意义，交由调用方决定）。
+    """
+
+    def __init__(
+        self,
+        handle: DataHandle,
+        n: int,
+        seed: int = 42,
+        method: SamplingMethod = "reservoir",
+    ) -> None:
+        if n < 1:
+            raise ValueError("n must be >= 1")
+        self._inner = handle
+        self._n = n
+        self._seed = seed
+        self._method = method
+        self._executor = getattr(handle, "_executor", None)
+        self._sampled_ready = False
+
+    def _ensure_sampled_table(self) -> None:
+        if self._sampled_ready or self._executor is None:
+            return
+        # 惰性视图：建表前确保 data 视图已注册（count_rows 触发各连接器
+        # _ensure_view；CSV 1e6 行计数 ~0.2s，可接受）
+        self._inner.count_rows()
+        if self._method == "none":
+            ddl = (
+                f"CREATE OR REPLACE TEMP TABLE {_SAMPLED_TABLE} AS "
+                f"SELECT * FROM data LIMIT {self._n}"
+            )
+        else:
+            # SAMPLE 子句不支持预编译参数，n/seed 均为代码内整数（已校验），安全内联
+            ddl = (
+                f"CREATE OR REPLACE TEMP TABLE {_SAMPLED_TABLE} AS "
+                f"SELECT * FROM data USING SAMPLE reservoir({self._n} ROWS) "
+                f"REPEATABLE ({self._seed})"
+            )
+        self._executor.execute_setup(ddl)
+        self._sampled_ready = True
+
+    def _sampled_subquery(self) -> str:
+        if self._method == "none":
+            return f"(SELECT * FROM data LIMIT {self._n})"
+        # SAMPLE 子句不支持预编译参数，n/seed 均为代码内整数（已校验），安全内联
+        return (
+            f"(SELECT * FROM data USING SAMPLE reservoir({self._n} ROWS) REPEATABLE ({self._seed}))"
+        )
+
+    def _rewrite(self, sql: str) -> str:
+        if self._executor is not None:
+            self._ensure_sampled_table()
+            target = f"FROM {_SAMPLED_TABLE}"
+        else:
+            target = f"FROM {self._sampled_subquery()}"
+        rewritten = _FROM_DATA_RE.sub(target, sql)
+        # 守卫：重写后不允许残留裸 FROM data（防未覆盖 SQL 形态，如别名/join）
+        if re.search(r"\bFROM data\b(?! USING SAMPLE)", rewritten):
+            raise ValueError(f"cannot rewrite sampled SQL: {sql!r}")
+        return rewritten
+
+    @property
+    def source_type(self) -> DataSourceType:
+        return self._inner.source_type
+
+    @property
+    def source_path(self) -> Path | str | None:
+        return self._inner.source_path
+
+    @property
+    def table_name(self) -> str | None:
+        return self._inner.table_name
+
+    def schema(self) -> SchemaInfo:
+        return self._inner.schema()
+
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]:
+        return self._inner.read_batches(batch_size)
+
+    def read_sample(self, n: int, method: SamplingMethod = "random") -> FrameBatch:
+        return self._inner.read_sample(n, method)
+
+    def sql_aggregate(self, sql: str, params: dict[str, object] | None = None) -> FrameBatch:
+        return self._inner.sql_aggregate(self._rewrite(sql), params)
+
+    def count_rows(self) -> int:
+        return min(self._n, self._inner.count_rows())
+
+    def fingerprint(self, mode: FingerprintMode = "full") -> DatasetFingerprint:
+        return self._inner.fingerprint(mode)
+
+    def content_fingerprint(self) -> str:
+        return self._inner.content_fingerprint()
+
+    def stats_fingerprint(self) -> str:
+        return self._inner.stats_fingerprint()
+
+    def warnings(self) -> list[LoadWarning]:
+        return self._inner.warnings()
+
+    def close(self) -> None:
+        self._inner.close()

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/xlsx.py

@@ -12,9 +12,13 @@ import pyarrow as pa
 from openpyxl import load_workbook
 
 from datasentry_core.connectors.base import FrameBatch
+from datasentry_core.connectors.errors import ConnectorError
 from datasentry_core.connectors.file_based import FileDataHandle
 from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
 
+#: 整 sheet 入内存的行预算（ADR-019 MVP 语义，Step 73/ADR-073 显式化）
+_XLSX_ROW_BUDGET = 1_000_000
+
 
 def _rows_to_table(names: list[str], rows: list[list[object]]) -> pa.Table:
     """行值列表 → pyarrow 表；混合类型列自动推断失败时回退全字符串（ADR-019）。"""
@@ -51,7 +55,14 @@ class XlsxDataHandle(FileDataHandle):
                 sheet = workbook.worksheets[sheet_ref]
             else:
                 sheet = workbook[str(sheet_ref)]
-            rows = list(sheet.iter_rows(values_only=True))
+            rows: list[tuple[object, ...]] = []
+            for row in sheet.iter_rows(values_only=True):
+                if len(rows) >= _XLSX_ROW_BUDGET:
+                    raise ConnectorError(
+                        f"xlsx sheet exceeds row budget {_XLSX_ROW_BUDGET} "
+                        "(ADR-019/ADR-073); split the sheet or use --sampling"
+                    )
+                rows.append(row)
         finally:
             workbook.close()
         if header_row >= len(rows):

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/src/datasentry_core/detectors/base.py

@@ -44,6 +44,19 @@ class DetectionContext:
         self.sample_rows = sample_rows
         self.references: list[TableReference] = references or []
 
+    def with_handle(self, handle: DataHandle) -> DetectionContext:
+        """返回换句柄的新上下文（Step 71：抽样支撑检测器注入抽样句柄）。"""
+        return DetectionContext(
+            dataset_id=self.dataset_id,
+            table_name=self.table_name,
+            columns=self.columns,
+            handle=handle,
+            config=self.config,
+            profile=self.profile,
+            sample_rows=self.sample_rows,
+            references=self.references,
+        )
+
 
 @runtime_checkable
 class Detector(Protocol):

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/src/datasentry_core/detectors/common.py

@@ -34,6 +34,15 @@ def make_evidence(
     data: dict[str, Any] | None = None,
     confidence: float = 1.0,
 ) -> Evidence:
+    from datasentry_core.reporting.evidence_desc import EvText
+
+    if isinstance(description, EvText):
+        data = {
+            **description.base,
+            **(data or {}),
+            "_text_key": description.key,
+            "_params": description.params,
+        }
     return Evidence(
         evidence_id=f"ev_{uuid.uuid4().hex[:12]}",
         evidence_type=evidence_type,

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/src/datasentry_core/detectors/datetime.py

@@ -18,6 +18,7 @@ from datasentry_core.detectors.common import (
 )
 from datasentry_core.models.detector import DetectorCapabilities, IssueCandidate
 from datasentry_core.models.enums import EvidenceType, QualityDimension, Severity
+from datasentry_core.reporting.evidence_desc import ev
 
 # 字符串日期列判定（列名特征；VARCHAR 列才适用 invalid/impossible/mixed）
 # 注意：时间戳列（created_at 等，物理 TIMESTAMP 或含时间值）不归此组，
@@ -74,8 +75,11 @@ class InvalidDateDetector(DetectorBase):
                                 detector_id=self.detector_id,
                                 detector_version=self.detector_version,
                                 evidence_type=EvidenceType.PATTERN_MATCH,
-                                description=f"{count} values fail ISO date pattern",
-                                data={"count": count, "pattern": _DATETIME_RE},
+                                description=ev(
+                                    "datetime.iso",
+                                    {"count": count, "pattern": _DATETIME_RE},
+                                    count=count,
+                                ),
                             )
                         ],
                         raw_score=count,
@@ -123,8 +127,11 @@ class ImpossibleDateDetector(DetectorBase):
                                 detector_id=self.detector_id,
                                 detector_version=self.detector_version,
                                 evidence_type=EvidenceType.PATTERN_MATCH,
-                                description=f"{count} calendar-invalid dates",
-                                data={"count": count, "pattern": _DATETIME_RE},
+                                description=ev(
+                                    "datetime.invalid",
+                                    {"count": count, "pattern": _DATETIME_RE},
+                                    count=count,
+                                ),
                             )
                         ],
                         raw_score=count,
@@ -168,8 +175,11 @@ class FutureDateDetector(DetectorBase):
                                 detector_id=self.detector_id,
                                 detector_version=self.detector_version,
                                 evidence_type=EvidenceType.CONSTRAINT_VIOLATION,
-                                description=f"{count} dates beyond today + 1 day",
-                                data={"count": count, "max_future_days": 1},
+                                description=ev(
+                                    "datetime.future",
+                                    {"count": count, "max_future_days": 1},
+                                    count=count,
+                                ),
                             )
                         ],
                         raw_score=count,
@@ -215,8 +225,11 @@ class StaleDateDetector(DetectorBase):
                                 detector_id=self.detector_id,
                                 detector_version=self.detector_version,
                                 evidence_type=EvidenceType.CONSTRAINT_VIOLATION,
-                                description=f"{count} dates older than 365 days",
-                                data={"count": count, "max_age_days": 365},
+                                description=ev(
+                                    "datetime.old",
+                                    {"count": count, "max_age_days": 365},
+                                    count=count,
+                                ),
                             )
                         ],
                         raw_score=count,
@@ -283,8 +296,11 @@ class MixedDateFormatDetector(DetectorBase):
                             detector_id=self.detector_id,
                             detector_version=self.detector_version,
                             evidence_type=EvidenceType.STATISTICAL_MEASURE,
-                            description=f"{len(major)} date formats each >2%",
-                            data={"formats": {k: round(v, 4) for k, v in formats.items()}},
+                            description=ev(
+                                "datetime.formats",
+                                {"formats": {k: round(v, 4) for k, v in formats.items()}},
+                                len=len(major),
+                            ),
                         )
                     ],
                     raw_score=len(major),
@@ -331,8 +347,11 @@ class DuplicateTimestampDetector(DetectorBase):
                             detector_id=self.detector_id,
                             detector_version=self.detector_version,
                             evidence_type=EvidenceType.DUPLICATE_MATCH,
-                            description=f"{len(counts)} timestamps appear more than twice",
-                            data={"groups": len(counts), "affected": affected},
+                            description=ev(
+                                "datetime.timestamps",
+                                {"groups": len(counts), "affected": affected},
+                                len=len(counts),
+                            ),
                         )
                     ],
                     raw_score=len(counts),

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/src/datasentry_core/detectors/initial/anomaly_ml.py

@@ -153,8 +153,11 @@ class ModelOutlierDetector(DetectorBase):
         for column in _infer_columns(context):
             q = quote_ident(column)
             try:
+                # SQL 侧 reservoir 抽样（Step 72/ADR-072）：物化前限制行数；
+                # 行数 < max_samples 时 reservoir 等价全量，语义不变
                 table = context.handle.sql_aggregate(
-                    f"SELECT {q} AS v FROM data WHERE {q} IS NOT NULL"
+                    f"SELECT {q} AS v FROM data WHERE {q} IS NOT NULL "
+                    f"USING SAMPLE reservoir({max_samples} ROWS) REPEATABLE ({seed})"
                 ).table
             except Exception:
                 continue

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/src/datasentry_core/detectors/initial/categorical.py

@@ -16,6 +16,7 @@ from datasentry_core.detectors.common import (
 )
 from datasentry_core.models.detector import DetectorCapabilities, IssueCandidate
 from datasentry_core.models.enums import EvidenceType, QualityDimension, Severity
+from datasentry_core.reporting.evidence_desc import ev
 
 
 def _json_safe(value: Any) -> Any:
@@ -67,8 +68,11 @@ class SuspiciousPlaceholderDetector(DetectorBase):
                                 detector_id=self.detector_id,
                                 detector_version=self.detector_version,
                                 evidence_type=EvidenceType.PATTERN_MATCH,
-                                description=f"{count} placeholder values",
-                                data={"count": count},
+                                description=ev(
+                                    "categorical.placeholder",
+                                    {"count": count},
+                                    count=count,
+                                ),
                             )
                         ],
                         raw_score=count,
@@ -131,13 +135,16 @@ class RareCategoryDetector(DetectorBase):
                             detector_id=self.detector_id,
                             detector_version=self.detector_version,
                             evidence_type=EvidenceType.STATISTICAL_MEASURE,
-                            description=f"{len(values)} rare categories",
-                            data={
-                                "categories": [
-                                    (_json_safe(v), int(c))
-                                    for v, c in zip(values, counts, strict=True)
-                                ]
-                            },
+                            description=ev(
+                                "categorical.rare",
+                                {
+                                    "categories": [
+                                        (_json_safe(v), int(c))
+                                        for v, c in zip(values, counts, strict=True)
+                                    ]
+                                },
+                                len=len(values),
+                            ),
                         )
                     ],
                     raw_score=len(values),
@@ -188,8 +195,11 @@ class CategoryExplosionDetector(DetectorBase):
                                 detector_id=self.detector_id,
                                 detector_version=self.detector_version,
                                 evidence_type=EvidenceType.STATISTICAL_MEASURE,
-                                description=f"unique_ratio={ratio:.3f} > 0.9",
-                                data={"unique_ratio": round(ratio, 6), "distinct": distinct},
+                                description=ev(
+                                    "categorical.high_unique",
+                                    {"unique_ratio": round(ratio, 6), "distinct": distinct},
+                                    ratio=ratio,
+                                ),
                             )
                         ],
                         raw_score=ratio,
@@ -240,8 +250,12 @@ class InconsistentCaseDetector(DetectorBase):
                             detector_id=self.detector_id,
                             detector_version=self.detector_version,
                             evidence_type=EvidenceType.STATISTICAL_MEASURE,
-                            description=f"{affected} rows in {len(keys)} case-variant groups",
-                            data={"groups": list(zip(keys, counts, forms, strict=True))},
+                            description=ev(
+                                "categorical.case_variant",
+                                {"groups": list(zip(keys, counts, forms, strict=True))},
+                                affected=affected,
+                                len=len(keys),
+                            ),
                         )
                     ],
                     raw_score=len(keys),

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/src/datasentry_core/detectors/initial/formula.py

@@ -15,6 +15,7 @@ from datasentry_core.detectors.common import (
 )
 from datasentry_core.models.detector import DetectorCapabilities, IssueCandidate
 from datasentry_core.models.enums import EvidenceType, QualityDimension, Severity
+from datasentry_core.reporting.evidence_desc import ev
 
 FORMULA_INJECTION_PREFIXES: tuple[str, ...] = ("=", "+", "-", "@", "\t", "\r")
 
@@ -54,8 +55,11 @@ class FormulaInjectionDetector(DetectorBase):
                                 detector_id=self.detector_id,
                                 detector_version=self.detector_version,
                                 evidence_type=EvidenceType.PATTERN_MATCH,
-                                description=f"{count} values start with formula-injection prefixes",
-                                data={"count": count},
+                                description=ev(
+                                    "formula.injection",
+                                    {"count": count},
+                                    count=count,
+                                ),
                             )
                         ],
                         raw_score=count,

{datasentry_core-0.7.0 → datasentry_core-0.8.0}/src/datasentry_core/detectors/initial/missing.py

@@ -15,6 +15,7 @@ from datasentry_core.detectors.common import (
 )
 from datasentry_core.models.detector import DetectorCapabilities, IssueCandidate
 from datasentry_core.models.enums import EvidenceType, QualityDimension, Severity
+from datasentry_core.reporting.evidence_desc import ev
 
 _MISSING_TOKENS = (
     "na",
@@ -79,13 +80,17 @@ class ExcessiveNullRateDetector(DetectorBase):
                                 detector_id=self.detector_id,
                                 detector_version=self.detector_version,
                                 evidence_type=EvidenceType.STATISTICAL_MEASURE,
-                                description=f"null_ratio={ratio:.4f} exceeds threshold={threshold}",
-                                data={
-                                    "null_ratio": round(ratio, 6),
-                                    "threshold": threshold,
-                                    "nulls": nulls,
-                                    "total": total,
-                                },
+                                description=ev(
+                                    "missing.null_ratio",
+                                    {
+                                        "null_ratio": round(ratio, 6),
+                                        "threshold": threshold,
+                                        "nulls": nulls,
+                                        "total": total,
+                                    },
+                                    ratio=ratio,
+                                    threshold=threshold,
+                                ),
                             )
                         ],
                         raw_score=ratio,
@@ -138,8 +143,12 @@ class SuspiciousMissingTokenDetector(DetectorBase):
                                 detector_id=self.detector_id,
                                 detector_version=self.detector_version,
                                 evidence_type=EvidenceType.PATTERN_MATCH,
-                                description=f"{count} values are missing stand-ins ({ratio:.4f})",
-                                data={"count": count, "ratio": round(ratio, 6)},
+                                description=ev(
+                                    "missing.standin",
+                                    {"count": count, "ratio": round(ratio, 6)},
+                                    count=count,
+                                    ratio=ratio,
+                                ),
                             )
                         ],
                         raw_score=ratio,