datasentry-core 0.5.0__tar.gz → 0.6.0__tar.gz

{datasentry_core-0.5.0 → datasentry_core-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datasentry-core
-Version: 0.5.0
+Version: 0.6.0
 Summary: DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs).
 License: Apache-2.0
 Keywords: data-observability,data-quality,duckdb

{datasentry_core-0.5.0 → datasentry_core-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datasentry-core"
-version = "0.5.0"
+version = "0.6.0"
 description = "DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs)."
 readme = "README.md"
 requires-python = ">=3.12"

{datasentry_core-0.5.0 → datasentry_core-0.6.0}/src/datasentry_core/__init__.py

@@ -108,4 +108,4 @@ __all__ = [
     "ValidationResult",
 ]
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"

{datasentry_core-0.5.0 → datasentry_core-0.6.0}/src/datasentry_core/connectors/__init__.py

@@ -20,6 +20,7 @@ from datasentry_core.connectors.errors import (
 )
 from datasentry_core.connectors.jsonl import JsonlConnector
 from datasentry_core.connectors.parquet import ParquetConnector
+from datasentry_core.connectors.postgres import PostgresConnector, PostgresDataHandle
 from datasentry_core.connectors.registry import ConnectorRegistry, default_registry
 from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
 from datasentry_core.connectors.sqlite import SqliteConnector
@@ -41,6 +42,8 @@ __all__ = [
     "JsonlConnector",
     "LoadWarning",
     "ParquetConnector",
+    "PostgresConnector",
+    "PostgresDataHandle",
     "SamplingMethod",
     "SchemaInfo",
     "SqliteConnector",

{datasentry_core-0.5.0 → datasentry_core-0.6.0}/src/datasentry_core/connectors/base.py

@@ -93,6 +93,11 @@ class DataHandle(Protocol):
     def sql_aggregate(self, sql: str, params: dict[str, Any] | None = None) -> FrameBatch: ...
     def count_rows(self) -> int: ...
     def fingerprint(self, mode: FingerprintMode = "full") -> DatasetFingerprint: ...
+    def content_fingerprint(self) -> str:
+        """内容指纹（Step 55）：文件源=文件 SHA-256；无文件字节源（如 PostgreSQL）
+        由子类实现内容哈希——调度器变更感知（Step 53/55）的跳过判定基准。"""
+        ...
+
     def warnings(self) -> list[LoadWarning]: ...
     def close(self) -> None: ...
 

{datasentry_core-0.5.0 → datasentry_core-0.6.0}/src/datasentry_core/connectors/csv.py

@@ -286,6 +286,11 @@ class CsvDataHandle:
         self._executor.close()
         self._closed = True
 
+    def content_fingerprint(self) -> str:
+        """内容指纹（Step 55）：文件源 = 文件 SHA-256（Step 53 调度哈希语义）。"""
+        assert self._path is not None
+        return _file_sha256(self._path)
+
 
 def _file_sha256(path: Path) -> str:
     """流式文件 SHA-256（≤1GB 全量，大文件走 sampled 档）。"""

{datasentry_core-0.5.0 → datasentry_core-0.6.0}/src/datasentry_core/connectors/file_based.py

@@ -12,6 +12,7 @@ import hashlib
 import json
 from collections.abc import Iterator
 from pathlib import Path
+from typing import ClassVar
 
 import pyarrow as pa
 import pyarrow.compute as pc
@@ -24,15 +25,33 @@ from datasentry_core.connectors.base import (
     SamplingMethod,
     SchemaInfo,
 )
-from datasentry_core.connectors.errors import DataSourceNotFoundError
+from datasentry_core.connectors.errors import ConnectorError, DataSourceNotFoundError
 from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
 from datasentry_core.engine import DuckDBExecutor
+from datasentry_core.engine.base import SetupExecutor
 from datasentry_core.models.enums import Severity
 from datasentry_core.models.fingerprint import DatasetFingerprint
 
 FORMULA_INJECTION_PREFIXES: tuple[str, ...] = ("=", "+", "-", "@", "\t", "\r")
 _WARNING_CAP = 100
 
+#: DuckDB DESCRIBE 物理类型 → 规范名（Step 55：检测器/画像的类型集合按规范名精确匹配）
+_TYPE_ALIASES = {
+    "TIMESTAMP WITH TIME ZONE": "TIMESTAMPTZ",
+    "TIMESTAMP WITHOUT TIME ZONE": "TIMESTAMP",
+}
+
+
+def _normalize_physical_type(typ: str) -> str:
+    """物理类型归一化：DECIMAL(n,p) → DECIMAL；别名时间类型 → 规范名。
+
+    PG/Parquet 等源的 DECIMAL(10,2) 与 DuckDB 原生 TIMESTAMP WITH TIME ZONE
+    直接进 schema 时无法命中检测器/画像的精确类型集合（Step 55 实测发现）。
+    """
+    if typ.startswith("DECIMAL(") and typ.endswith(")"):
+        return "DECIMAL"
+    return _TYPE_ALIASES.get(typ, typ)
+
 
 def _schema_hash(column_signature: list[tuple[str, str]]) -> str:
     """schema_hash = 列签名规范 JSON 的 SHA-256（19.1）。"""
@@ -52,12 +71,15 @@ def _file_sha256(path: Path) -> str:
 class FileDataHandle:
     """文件型数据句柄基类（DataHandle 协议共享实现）。"""
 
+    #: 需要文件路径的子类默认 True；无文件源的子类（如 PostgreSQL）置 False
+    requires_path: ClassVar[bool] = True
+
     def __init__(self, spec: DataSourceSpec) -> None:
-        if spec.path is None:
-            raise DataSourceNotFoundError(f"{type(self).__name__} requires a path")
         self._spec = spec
         self._path = spec.path
-        self._executor = DuckDBExecutor()
+        if self.requires_path and spec.path is None:
+            raise DataSourceNotFoundError(f"{type(self).__name__} requires a path")
+        self._executor: SetupExecutor = DuckDBExecutor()
         self._view_ready = False
         self._schema_info: SchemaInfo | None = None
         self._warnings: list[LoadWarning] | None = None
@@ -99,7 +121,7 @@ class FileDataHandle:
             types = table.column(1).to_pylist()
             self._schema_info = SchemaInfo(
                 columns=[
-                    ColumnInfo(name=str(name), physical_type=str(typ))
+                    ColumnInfo(name=str(name), physical_type=_normalize_physical_type(str(typ)))
                     for name, typ in zip(names, types, strict=True)
                 ]
             )
@@ -152,6 +174,8 @@ class FileDataHandle:
         file_sha256: str | None = None
         content_sample_hash: str | None = None
         if mode == "full":
+            if self._path is None:
+                raise ConnectorError("file fingerprint requires a path")
             file_sha256 = _file_sha256(self._path)
         elif mode == "sampled":
             head = self._executor.execute("SELECT * FROM data LIMIT 1000")
@@ -173,6 +197,11 @@ class FileDataHandle:
             content_sample_hash=content_sample_hash,
         )
 
+    def content_fingerprint(self) -> str:
+        """内容指纹（Step 55）：文件源 = 文件 SHA-256（Step 53 调度哈希语义）。"""
+        assert self._path is not None
+        return _file_sha256(self._path)
+
     def warnings(self) -> list[LoadWarning]:
         """公式注入标记（11.7）：遍历批处理扫描字符串列，前 _WARNING_CAP 条。"""
         self._check_open()

datasentry_core-0.6.0/src/datasentry_core/connectors/postgres.py

@@ -0,0 +1,253 @@
+"""PostgreSQL 数据源连接器（Step 55，V4：多数据源第二落点，ADR-055）。
+
+经 DuckDB postgres 扩展（LOAD postgres）只读 ATTACH 远程库
+（TYPE postgres, READ_ONLY），注册只读 data 视图后复用 FileDataHandle
+共享实现（schema/read_sample/sql_aggregate/count_rows/warnings）。
+表名必填；schema 名经 options["schema"] 传入，默认 public。
+
+凭据红线（V4 硬性约束）：
+- DSN 只存在于内存态（spec.options["dsn"]）或经 connection_ref
+  指向的环境变量（如 DATASENTRY_PG_DSN），不落库、不进日志/evidence/报告
+- DuckDB 错误文本会原样包含 DSN 与密码：_RedactingExecutor 将
+  DuckDB 异常净化（DSN → postgresql://***、密码 → ***）后转
+  ConnectorError 传播，REST 400 / CLI 错误面只见净化文本
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from collections.abc import Iterator
+from typing import ClassVar, cast
+from urllib.parse import urlparse
+
+import duckdb
+import pyarrow as pa
+
+from datasentry_core.connectors.base import FingerprintMode, FrameBatch
+from datasentry_core.connectors.errors import ConnectorError, DataSourceNotFoundError
+from datasentry_core.connectors.file_based import FileDataHandle, _schema_hash
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+from datasentry_core.engine import DuckDBExecutor
+from datasentry_core.engine.base import SqlParams
+from datasentry_core.models.fingerprint import DatasetFingerprint
+
+#: ATTACH 目标别名（连接器内部名，避开用户数据命名冲突概率极低）
+_DB_ALIAS = "pg"
+
+#: 加载 postgres 扩展失败的提示（缺网络/离线/发行版未带扩展）
+_PG_EXTENSION_HINT = (
+    "failed to load duckdb postgres extension: network access is required on first use "
+    "(or the extension must be pre-provisioned for offline environments); "
+    "original error: {}"
+)
+
+#: 连接失败提示（凭据已净化）
+_PG_CONNECT_HINT = "postgres connection failed (check host/port/database/credentials): {}"
+
+
+def _sql_string_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _ident(value: str) -> str:
+    """双引号标识符转义（防表名/schema 名注入）。"""
+    return '"' + value.replace('"', '""') + '"'
+
+
+def redact_credentials(text: str, dsn: str) -> str:
+    """净化错误文本：DSN 整体与密码不对外（凭据红线，可单测）。"""
+    redacted = text.replace(dsn, "postgresql://***")
+    try:
+        parsed = urlparse(dsn)
+    except ValueError:
+        return redacted
+    if parsed.password:
+        redacted = redacted.replace(parsed.password, "***")
+    return redacted
+
+
+def _resolve_dsn(spec: DataSourceSpec) -> str:
+    """DSN 解析：spec.options["dsn"]（内存态）或 connection_ref 环境变量引用。"""
+    dsn = spec.options.get("dsn")
+    if isinstance(dsn, str) and dsn:
+        return dsn
+    if spec.connection_ref:
+        ref = spec.connection_ref
+        dsn = os.environ.get(ref)
+        if dsn:
+            return dsn
+        raise DataSourceNotFoundError(f"postgres connection_ref env not set: {ref}")
+    raise DataSourceNotFoundError(
+        "postgres connector requires a dsn (options['dsn'] or connection_ref env)"
+    )
+
+
+class _RedactingExecutor:
+    """PG 句柄专用执行器包装：DuckDB 异常净化后转 ConnectorError 传播。
+
+    只读守卫（sql_guard）在 DuckDBExecutor 内先于执行触发，其异常
+    （UnsafeSqlError 等连接器族）原样透传；查询期远程错误（断连等）
+    同样净化，避免 DSN 出现在 DetectorRun.error / 调度 summary 等
+    落库面。
+    """
+
+    def __init__(self, inner: DuckDBExecutor, dsn: str) -> None:
+        self._inner = inner
+        self._dsn = dsn
+
+    def execute_setup(self, sql: str) -> None:
+        try:
+            self._inner.execute_setup(sql)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_credentials(str(exc), self._dsn)) from exc
+
+    def register(self, name: str, obj: object) -> None:
+        self._inner.register(name, obj)
+
+    def execute(self, sql: str, params: SqlParams = None) -> pa.Table:
+        try:
+            return self._inner.execute(sql, params)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_credentials(str(exc), self._dsn)) from exc
+
+    def execute_stream(self, sql: str, batch_size: int = 65536) -> Iterator[pa.RecordBatch]:
+        try:
+            yield from self._inner.execute_stream(sql, batch_size)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_credentials(str(exc), self._dsn)) from exc
+
+    def close(self) -> None:
+        self._inner.close()
+
+
+class PostgresDataHandle(FileDataHandle):
+    """PostgreSQL 远程表数据句柄（只读，表名必填，无文件字节）。"""
+
+    requires_path: ClassVar[bool] = False
+
+    def __init__(self, spec: DataSourceSpec) -> None:
+        super().__init__(spec)
+        if spec.table_name is None:
+            raise DataSourceNotFoundError("postgres connector requires table_name")
+        self._dsn = _resolve_dsn(spec)
+        self._table_name = spec.table_name
+        self._schema_name = spec.options.get("schema")
+        assert isinstance(self._executor, DuckDBExecutor)
+        self._executor = _RedactingExecutor(self._executor, self._dsn)
+
+    @property
+    def table_name(self) -> str | None:
+        return self._table_name
+
+    def _ensure_view(self) -> None:
+        if self._view_ready:
+            return
+        try:
+            self._executor.execute_setup("LOAD postgres")
+        except ConnectorError as exc:
+            raise ConnectorError(_PG_EXTENSION_HINT.format(exc)) from exc
+        try:
+            self._executor.execute_setup(
+                f"ATTACH {_sql_string_literal(self._dsn)} AS {_DB_ALIAS} (TYPE postgres, READ_ONLY)"
+            )
+        except ConnectorError as exc:
+            raise ConnectorError(_PG_CONNECT_HINT.format(exc)) from exc
+        schema_ident = _ident(cast(str, self._schema_name) or "public")
+        table_ident = f"{_DB_ALIAS}.{schema_ident}.{_ident(self._table_name)}"
+        try:
+            self._executor.execute(f"SELECT 1 FROM {table_ident} LIMIT 0")
+        except ConnectorError as exc:
+            raise DataSourceNotFoundError(
+                f"postgres table not found: {self._schema_name or 'public'}.{self._table_name}"
+            ) from exc
+        self._executor.execute_setup(f"CREATE OR REPLACE VIEW data AS SELECT * FROM {table_ident}")
+        self._view_ready = True
+
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]:
+        self._check_open()
+        if batch_size < 1:
+            raise ValueError("batch_size must be >= 1")
+        self._ensure_view()
+        row_offset = 0
+        for idx, batch in enumerate(
+            self._executor.execute_stream("SELECT * FROM data", batch_size)
+        ):
+            table = pa.Table.from_batches([batch])
+            yield FrameBatch(
+                source=self._spec,
+                table=table,
+                batch_index=idx,
+                row_offset=row_offset,
+            )
+            row_offset += table.num_rows
+
+    def content_fingerprint(self) -> str:
+        """内容指纹（Step 55）：单查询全表哈希（行序无关、NULL 不折叠）。
+
+        逐行哈希 = md5(concat_ws(chr(31), coalesce(col::VARCHAR, chr(0))...))，
+        排序聚合后并入 schema_hash 与行数 —— 类型/数量/取值任一变化
+        指纹即变；相同内容指纹恒定（供调度变更感知作跳过基准）。
+        """
+        self._check_open()
+        self._ensure_view()
+        signature = [(c.name, c.physical_type) for c in self.schema().columns]
+        schema_hash = _schema_hash(signature)
+        exprs = ", ".join(
+            f"coalesce(cast({_ident(name)} AS VARCHAR), chr(0))" for name, _typ in signature
+        )
+        table = self._executor.execute(
+            "SELECT count(*) AS n, md5(string_agg(rh, '' ORDER BY rh)) AS h FROM "
+            f"(SELECT md5(concat_ws(chr(31), {exprs})) AS rh FROM data) t"
+        )
+        row_count = table.column(0).to_pylist()[0]
+        rows_hash = table.column(1).to_pylist()[0]
+        assert isinstance(row_count, int)
+        digest = hashlib.sha256(f"{schema_hash}|{row_count}|{rows_hash or ''}".encode()).hexdigest()
+        return digest
+
+    def fingerprint(self, mode: FingerprintMode = "full") -> DatasetFingerprint:
+        """PG 无文件字节：file_sha256=None，内容指纹放 content_sample_hash。
+
+        sampled 档 MVP 与 full 同（远程源暂不做增量采样优化，ADR-055）。
+        """
+        self._check_open()
+        signature = [(c.name, c.physical_type) for c in self.schema().columns]
+        row_count = self.count_rows()
+        content_hash = self.content_fingerprint() if mode != "metadata_only" else None
+        return DatasetFingerprint(
+            dataset_id=self._spec.options.get("dataset_id", ""),
+            fingerprint_type=mode,
+            file_sha256=None,
+            schema_hash=_schema_hash(signature),
+            row_count=row_count,
+            column_count=len(signature),
+            column_signature=signature,
+            content_sample_hash=content_hash,
+        )
+
+
+class PostgresConnector:
+    """PostgreSQL 数据源连接器（Step 55，V4，ADR-055）。"""
+
+    connector_id = "postgres"
+    display_name = "PostgreSQL"
+
+    def supports(self, source: DataSourceSpec) -> bool:
+        if source.source_type != DataSourceType.POSTGRESQL:
+            return False
+        dsn = source.options.get("dsn")
+        return (isinstance(dsn, str) and bool(dsn)) or source.connection_ref is not None
+
+    def open(self, source: DataSourceSpec) -> PostgresDataHandle:
+        if source.table_name is None:
+            raise DataSourceNotFoundError(
+                "postgres connector requires table_name (e.g. scan --table <table>)"
+            )
+        return PostgresDataHandle(source)
+
+    def close(self) -> None:
+        pass
+
+
+__all__ = ["PostgresConnector", "PostgresDataHandle", "redact_credentials"]

{datasentry_core-0.5.0 → datasentry_core-0.6.0}/src/datasentry_core/connectors/registry.py

@@ -8,6 +8,7 @@ from datasentry_core.connectors.duckdb import DuckdbConnector
 from datasentry_core.connectors.errors import UnsupportedFormatError
 from datasentry_core.connectors.jsonl import JsonlConnector
 from datasentry_core.connectors.parquet import ParquetConnector
+from datasentry_core.connectors.postgres import PostgresConnector
 from datasentry_core.connectors.spec import DataSourceSpec
 from datasentry_core.connectors.sqlite import SqliteConnector
 from datasentry_core.connectors.xlsx import XlsxConnector
@@ -46,7 +47,7 @@ class ConnectorRegistry:
 
 
 def default_registry() -> ConnectorRegistry:
-    """默认注册表：文件型连接器 + DuckDB/SQLite 文件连接器（Step 38/54）。"""
+    """默认注册表：文件型连接器 + DuckDB/SQLite 文件连接器（Step 38/54）+ PostgreSQL（Step 55）。"""
     registry = ConnectorRegistry()
     registry.register(CsvConnector())
     registry.register(ParquetConnector())
@@ -54,4 +55,5 @@ def default_registry() -> ConnectorRegistry:
     registry.register(XlsxConnector())
     registry.register(DuckdbConnector())
     registry.register(SqliteConnector())
+    registry.register(PostgresConnector())
     return registry

{datasentry_core-0.5.0 → datasentry_core-0.6.0}/src/datasentry_core/engine/base.py

@@ -5,7 +5,7 @@ Polars 引擎（ADR-005 归 V1）将实现同一协议，消费方无需感知
 
 from __future__ import annotations
 
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from typing import Protocol, runtime_checkable
 
 import pyarrow as pa
@@ -32,3 +32,30 @@ class SqlExecutor(Protocol):
     def close(self) -> None:
         """释放底层连接。关闭后 execute 抛错误。"""
         ...
+
+
+@runtime_checkable
+class SetupExecutor(Protocol):
+    """连接器内部执行器协议：只读面 + 受信 setup 面（视图/ATTACH/register）。
+
+    FileDataHandle 等连接器基类以此类型持有执行器，便于包装器
+    （如 Step 55 的凭据净化包装）在类型安全的前提下替换实现。
+    """
+
+    def execute_setup(self, sql: str) -> None: ...
+
+    def register(self, name: str, obj: object) -> None: ...
+
+    def execute(
+        self,
+        sql: str,
+        params: SqlParams = None,
+    ) -> pa.Table: ...
+
+    def execute_stream(
+        self,
+        sql: str,
+        batch_size: int = 65536,
+    ) -> Iterator[pa.RecordBatch]: ...
+
+    def close(self) -> None: ...

{datasentry_core-0.5.0 → datasentry_core-0.6.0}/src/datasentry_core/storage/store.py

@@ -9,6 +9,7 @@
 
 from __future__ import annotations
 
+import decimal
 import json
 import sqlite3
 import threading
@@ -39,6 +40,19 @@ def _iso(dt: datetime) -> str:
     return dt.isoformat()
 
 
+def _json_default(obj: Any) -> Any:
+    """evidence/参数 JSON 兜底（Step 55：PG 源聚合结果可能带 Decimal）。
+
+    检测器 evidence.data 契约是 JSON 可序列化；DuckDB postgres 扩展对
+    numeric 聚合（avg/sum 等）返回 Decimal，此处统一收敛为 float。
+    """
+    if isinstance(obj, decimal.Decimal):
+        return float(obj)
+    if isinstance(obj, (datetime,)):
+        return obj.isoformat()
+    raise TypeError(f"Object of type {obj.__class__.__name__} is not JSON serializable")
+
+
 class MetadataStore:
     """项目元数据库门面。open 即迁移；close 后不可用。"""
 
@@ -184,7 +198,7 @@ class MetadataStore:
                             evidence.detector_id,
                             evidence.detector_version,
                             evidence.description,
-                            json.dumps(evidence.data),
+                            json.dumps(evidence.data, default=_json_default),
                             evidence.confidence,
                             (
                                 evidence.provenance.model_dump_json()