datasentry-core 0.8.2__tar.gz → 0.8.4__tar.gz

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/.gitignore

@@ -41,3 +41,9 @@ docs/V*_DEV_PROMPT.md
 DataSentry AI 产品设计与开发 Prompt*.md
 DataSentry AI 产品设计与开发 Prompt*.docx
 DataSentry AI 增长与推广 Prompt.md
+
+# Prompt 与体检报告所在目录（本地保留，不进仓库）
+AI Data Quality Copilot/
+
+# 审计产物（本地保留，不进仓库；阶段 4 指定位置）
+AUDIT/

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datasentry-core
-Version: 0.8.2
+Version: 0.8.4
 Summary: DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs).
 License: Apache-2.0
 Keywords: data-observability,data-quality,duckdb

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datasentry-core"
-version = "0.8.2"
+version = "0.8.4"
 description = "DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs)."
 readme = "README.md"
 requires-python = ">=3.12"

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/src/datasentry_core/connectors/csv.py

@@ -79,6 +79,23 @@ def _sniff_delimiter(path: Path, encoding: str) -> str:
         return ","
 
 
+def sniff_profile(path: Path) -> tuple[str, str]:
+    """The (encoding, delimiter) this connector would use for `path`.
+
+    Public for exactly one reason: the repair auditor reads the artefacts, and reading them under
+    a different encoding than the bytes are in is not a smaller lie than reading them under a
+    different delimiter. A cp1250 snapshot raised `invalid UTF8 data` and the repair lost its audit
+    evidence while `apply` had already succeeded (F2).
+    """
+    encoding = _detect_encoding(path)
+    return encoding, _sniff_delimiter(path, encoding)
+
+
+def sniff_dialect(path: Path) -> str:
+    """The delimiter this connector would use for `path`."""
+    return sniff_profile(path)[1]
+
+
 def _schema_hash(column_signature: list[tuple[str, str]]) -> str:
     """schema_hash = 列签名规范 JSON 的 SHA-256（19.1）。"""
     canonical = json.dumps(column_signature, sort_keys=True, ensure_ascii=True)

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/src/datasentry_core/detectors/initial/missing.py

@@ -54,8 +54,10 @@ class ExcessiveNullRateDetector(DetectorBase):
             table = context.handle.sql_aggregate(
                 f"SELECT count(*) AS n, sum({q} IS NULL) AS nulls FROM data"
             ).table
-            total = int(table.column("n").to_pylist()[0])
-            nulls = int(table.column("nulls").to_pylist()[0])
+            raw_total = table.column("n").to_pylist()[0]
+            raw_nulls = table.column("nulls").to_pylist()[0]
+            total = int(raw_total) if raw_total is not None else 0
+            nulls = int(raw_nulls) if raw_nulls is not None else 0
             if total <= 0:
                 continue
             ratio = nulls / total

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/src/datasentry_core/detectors/initial/numeric.py

@@ -12,6 +12,7 @@ from datasentry_core.detectors.common import (
     numeric_columns,
     quote_ident,
 )
+from datasentry_core.engine.base import finite_only
 from datasentry_core.models.detector import DetectorCapabilities, IssueCandidate
 from datasentry_core.models.enums import EvidenceType, QualityDimension, Severity
 from datasentry_core.reporting.evidence_desc import ev
@@ -33,7 +34,8 @@ class IqrOutlierDetector(DetectorBase):
         for col in numeric_columns(context):
             q = quote_ident(col)
             stat = context.handle.sql_aggregate(
-                f"SELECT quantile_cont({q}, 0.25) AS q25, quantile_cont({q}, 0.75) AS q75 FROM data"
+                f"SELECT quantile_cont({finite_only(q)}, 0.25) AS q25, "
+                f"quantile_cont({finite_only(q)}, 0.75) AS q75 FROM data"
             ).table
             row = stat.to_pylist()[0]
             q25, q75 = row["q25"], row["q75"]
@@ -102,7 +104,7 @@ class ModifiedZScoreDetector(DetectorBase):
         for col in numeric_columns(context):
             q = quote_ident(col)
             stat = context.handle.sql_aggregate(
-                f"SELECT median({q}) AS m, mad({q}) AS mad FROM data"
+                f"SELECT median({finite_only(q)}) AS m, mad({finite_only(q)}) AS mad FROM data"
             ).table
             row = stat.to_pylist()[0]
             median, mad = row["m"], row["mad"]
@@ -211,8 +213,8 @@ class PercentileOutlierDetector(DetectorBase):
         for col in numeric_columns(context):
             q = quote_ident(col)
             stat = context.handle.sql_aggregate(
-                f"SELECT quantile_cont({q}, 0.001) AS p_low, quantile_cont({q}, 0.999) AS p_high "
-                f"FROM data"
+                f"SELECT quantile_cont({finite_only(q)}, 0.001) AS p_low, "
+                f"quantile_cont({finite_only(q)}, 0.999) AS p_high FROM data"
             ).table
             row = stat.to_pylist()[0]
             p_low, p_high = row["p_low"], row["p_high"]
@@ -271,7 +273,8 @@ class HistogramRarityDetector(DetectorBase):
         for col in numeric_columns(context):
             q = quote_ident(col)
             stat = context.handle.sql_aggregate(
-                f"SELECT min({q}) AS lo, max({q}) AS hi, count({q}) AS n FROM data"
+                f"SELECT min({finite_only(q)}) AS lo, max({finite_only(q)}) AS hi, "
+                f"count({q}) AS n FROM data"
             ).table
             row = stat.to_pylist()[0]
             lo, hi, total = row["lo"], row["hi"], row["n"]

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/src/datasentry_core/detectors/missingness.py

@@ -61,8 +61,10 @@ def _column_nulls(context: DetectionContext, column: str) -> tuple[int, int]:
     table = context.handle.sql_aggregate(
         f"SELECT count(*) AS n, sum({q} IS NULL) AS nulls FROM data"
     ).table
-    total = int(table.column("n").to_pylist()[0])
-    nulls = int(table.column("nulls").to_pylist()[0])
+    raw_total = table.column("n").to_pylist()[0]
+    raw_nulls = table.column("nulls").to_pylist()[0]
+    total = int(raw_total) if raw_total is not None else 0
+    nulls = int(raw_nulls) if raw_nulls is not None else 0
     return total, nulls
 
 

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/src/datasentry_core/engine/__init__.py

@@ -1,6 +1,6 @@
 """执行层：Step 3 DuckDB 只读执行器 + Step 4 Profiling + Step 7 证据融合（Polars 归 V1）。"""
 
-from datasentry_core.engine.base import SqlExecutor, SqlParams
+from datasentry_core.engine.base import SqlExecutor, SqlParams, finite_only
 from datasentry_core.engine.duckdb import DuckDBExecutor
 from datasentry_core.engine.fusion import EvidenceFusionEngine
 from datasentry_core.engine.profiler import Profiler
@@ -13,4 +13,5 @@ __all__ = [
     "SqlExecutor",
     "SqlParams",
     "assert_read_only_sql",
+    "finite_only",
 ]

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/src/datasentry_core/engine/base.py

@@ -59,3 +59,15 @@ class SetupExecutor(Protocol):
     ) -> Iterator[pa.RecordBatch]: ...
 
     def close(self) -> None: ...
+
+
+def finite_only(quoted: str) -> str:
+    """The expression naming only the finite values of a numeric column.
+
+    NaN and ±Inf are values -- they still count and they are not NULL -- but they are not numbers.
+    Unguarded, DuckDB's `stddev` aborts the whole aggregate (`OutOfRangeException`), `max([1.0,
+    nan, 3.0])` answers `nan`, a median orders NaN as a value, and a detector that splices a NaN
+    bound into a predicate fails with `Referenced column "nan" not found`. One rule for the
+    profiler and for every detector, so the same column cannot be summarised two ways (G-2).
+    """
+    return f"CASE WHEN isfinite({quoted}) THEN {quoted} END"

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/src/datasentry_core/engine/profiler.py

@@ -6,6 +6,8 @@
 约定：
 - 数值列（INTEGER/FLOAT/DOUBLE/DECIMAL…）计算 min/max/mean/std/q25/median/q75；
   字符串列仅统计 distinct/min/max；其余类型取 min/max
+- 数值列的统计量只在**有限值**上计算：NaN/±Inf 仍算一个"值"（进 count 与 distinct、
+  不算 NULL），但不进 min/max/mean/std/分位（G-2：不守卫则 stddev 直接抛，整次扫描失败）
 - 空字符串在 duckdb 视图路径已被折叠为 NULL（见 CSV 语义约定），null 统计包含空串
 - examples 字段刻意留空：脱敏设施在 Step 15（LLM Provider 抽象）之后提供，
   在此之前避免把原始样本写入画像输出
@@ -19,6 +21,7 @@ from typing import Any
 import pyarrow as pa
 
 from datasentry_core.connectors.base import DataHandle
+from datasentry_core.engine.base import finite_only
 from datasentry_core.models.profile import ColumnProfile, DatasetProfile
 
 _NUMERIC_TYPES = frozenset(
@@ -46,16 +49,28 @@ def _column_exprs(column: str, quoted: str, numeric: bool) -> list[str]:
     exprs = [
         f"count({quoted}) AS {alias}",
         f"count(DISTINCT {quoted}) AS {_quote_ident(f'{column}__distinct')}",
-        f"min({quoted}) AS {_quote_ident(f'{column}__min')}",
-        f"max({quoted}) AS {_quote_ident(f'{column}__max')}",
     ]
     if numeric:
+        # NaN and ±Inf are values (they still count) but they are not numbers. Left unguarded,
+        # `stddev` aborts the whole aggregate -- `OutOfRangeException: STDDEV_SAMP is out of
+        # range!` -- so `scan_file` fails on an ordinary CSV whose numeric column carries the token
+        # `nan`, which is what pandas-style exports produce; and the aggregates that do not abort
+        # answer wrongly (`max([1.0, nan, 3.0])` is `nan`, the median becomes `3.0`). Statistics are
+        # therefore taken over the finite values only (G-2).
+        stat = finite_only(quoted)
         exprs += [
-            f"avg({quoted}) AS {_quote_ident(f'{column}__mean')}",
-            f"stddev({quoted}) AS {_quote_ident(f'{column}__std')}",
-            f"quantile_cont({quoted}, 0.25) AS {_quote_ident(f'{column}__q25')}",
-            f"quantile_cont({quoted}, 0.5) AS {_quote_ident(f'{column}__median')}",
-            f"quantile_cont({quoted}, 0.75) AS {_quote_ident(f'{column}__q75')}",
+            f"min({stat}) AS {_quote_ident(f'{column}__min')}",
+            f"max({stat}) AS {_quote_ident(f'{column}__max')}",
+            f"avg({stat}) AS {_quote_ident(f'{column}__mean')}",
+            f"stddev({stat}) AS {_quote_ident(f'{column}__std')}",
+            f"quantile_cont({stat}, 0.25) AS {_quote_ident(f'{column}__q25')}",
+            f"quantile_cont({stat}, 0.5) AS {_quote_ident(f'{column}__median')}",
+            f"quantile_cont({stat}, 0.75) AS {_quote_ident(f'{column}__q75')}",
+        ]
+    else:
+        exprs += [
+            f"min({quoted}) AS {_quote_ident(f'{column}__min')}",
+            f"max({quoted}) AS {_quote_ident(f'{column}__max')}",
         ]
     return exprs
 

{datasentry_core-0.8.2 → datasentry_core-0.8.4}/src/datasentry_core/models/scan.py

@@ -21,7 +21,7 @@ class SamplingConfig(BaseModel):
 
     method: Literal[
         "random", "stratified", "reservoir", "time_based", "rare_oversampling", "none"
-    ] = "random"
+    ] = "reservoir"
     sample_size: int | None = Field(default=None, ge=1)
     ratio: float | None = Field(default=None, gt=0.0, le=1.0)
     seed: int = 42