datasentry-core 0.6.0__tar.gz → 0.7.0__tar.gz

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datasentry-core
-Version: 0.6.0
+Version: 0.7.0
 Summary: DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs).
 License: Apache-2.0
 Keywords: data-observability,data-quality,duckdb

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datasentry-core"
-version = "0.6.0"
+version = "0.7.0"
 description = "DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs)."
 readme = "README.md"
 requires-python = ">=3.12"

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/__init__.py

@@ -108,4 +108,4 @@ __all__ = [
     "ValidationResult",
 ]
 
-__version__ = "0.5.1"
+__version__ = "0.7.0"

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/__init__.py

@@ -19,9 +19,11 @@ from datasentry_core.connectors.errors import (
     UnsupportedFormatError,
 )
 from datasentry_core.connectors.jsonl import JsonlConnector
+from datasentry_core.connectors.mysql import MySQLConnector, MySQLDataHandle
 from datasentry_core.connectors.parquet import ParquetConnector
 from datasentry_core.connectors.postgres import PostgresConnector, PostgresDataHandle
 from datasentry_core.connectors.registry import ConnectorRegistry, default_registry
+from datasentry_core.connectors.remote_file import RemoteFileConnector, RemoteFileDataHandle
 from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
 from datasentry_core.connectors.sqlite import SqliteConnector
 from datasentry_core.connectors.xlsx import XlsxConnector
@@ -41,9 +43,13 @@ __all__ = [
     "FrameBatch",
     "JsonlConnector",
     "LoadWarning",
+    "MySQLConnector",
+    "MySQLDataHandle",
     "ParquetConnector",
     "PostgresConnector",
     "PostgresDataHandle",
+    "RemoteFileConnector",
+    "RemoteFileDataHandle",
     "SamplingMethod",
     "SchemaInfo",
     "SqliteConnector",

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/base.py

@@ -80,7 +80,7 @@ class DataHandle(Protocol):
     @property
     def source_type(self) -> DataSourceType: ...
     @property
-    def source_path(self) -> Path | None: ...
+    def source_path(self) -> Path | str | None: ...
 
     @property
     def table_name(self) -> str | None:
@@ -98,6 +98,16 @@ class DataHandle(Protocol):
         由子类实现内容哈希——调度器变更感知（Step 53/55）的跳过判定基准。"""
         ...
 
+    def stats_fingerprint(self) -> str:
+        """统计层指纹（Step 58，ADR-058）：schema_hash + row_count 组合，零内容读取。
+
+        调度两层快速失效第一层：统计层一致才计算内容指纹；统计层变化
+        立即判定变更。远程源（PG/MySQL/云文件）由 FileDataHandle 默认
+        实现（DESCRIBE 目录查询 + count），文件源不参与（沿用 Step 53
+        单层文件 SHA-256）。协议默认体：本地文件句柄不实现（调度器不会
+        对本地文件调用），实现者不覆盖即视为不适用。"""
+        raise NotImplementedError("stats_fingerprint is not supported by this handle")
+
     def warnings(self) -> list[LoadWarning]: ...
     def close(self) -> None: ...
 

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/csv.py

@@ -27,7 +27,7 @@ from datasentry_core.connectors.base import (
     SamplingMethod,
     SchemaInfo,
 )
-from datasentry_core.connectors.errors import DataSourceNotFoundError
+from datasentry_core.connectors.errors import ConnectorError, DataSourceNotFoundError
 from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
 from datasentry_core.engine import DuckDBExecutor
 from datasentry_core.models.enums import Severity
@@ -91,8 +91,8 @@ class CsvDataHandle:
     """CSV 数据句柄：pyarrow 流式读取 + duckdb 只读聚合。"""
 
     def __init__(self, spec: DataSourceSpec) -> None:
-        if spec.path is None:
-            raise DataSourceNotFoundError("csv source requires a path")
+        if spec.path is None or isinstance(spec.path, str):
+            raise DataSourceNotFoundError("csv source requires a local file path")
         self._spec = spec
         self._path = spec.path
         self._encoding = _detect_encoding(spec.path, override=spec.options.get("encoding"))
@@ -108,7 +108,7 @@ class CsvDataHandle:
         return self._spec.source_type
 
     @property
-    def source_path(self) -> Path | None:
+    def source_path(self) -> Path | str | None:
         return self._path
 
     @property
@@ -288,9 +288,17 @@ class CsvDataHandle:
 
     def content_fingerprint(self) -> str:
         """内容指纹（Step 55）：文件源 = 文件 SHA-256（Step 53 调度哈希语义）。"""
-        assert self._path is not None
+        if not isinstance(self._path, Path):
+            raise ConnectorError("csv fingerprint requires a local file path")
         return _file_sha256(self._path)
 
+    def stats_fingerprint(self) -> str:
+        """统计层指纹（Step 58）：本地文件源不参与两层指纹——调度器仅对
+        远程源（PG/MySQL/云文件）调用，本地文件沿用 Step 53 单层语义。"""
+        raise NotImplementedError(
+            "local file handles do not participate in two-layer fingerprinting"
+        )
+
 
 def _file_sha256(path: Path) -> str:
     """流式文件 SHA-256（≤1GB 全量，大文件走 sampled 档）。"""
@@ -308,7 +316,12 @@ class CsvConnector:
     display_name = "CSV"
 
     def supports(self, source: DataSourceSpec) -> bool:
-        return source.source_type == DataSourceType.CSV and source.path is not None
+        # Step 57：本地文件连接器排除云 URI（s3:// gs:// az:// 归 RemoteFileConnector）
+        return (
+            source.source_type == DataSourceType.CSV
+            and source.path is not None
+            and not isinstance(source.path, str)
+        )
 
     def open(self, source: DataSourceSpec) -> DataHandle:
         return CsvDataHandle(source)

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/duckdb.py

@@ -35,7 +35,7 @@ class DuckDBDataHandle(FileDataHandle):
 
     def __init__(self, spec: DataSourceSpec) -> None:
         super().__init__(spec)
-        if spec.path is None or not spec.path.is_file():
+        if spec.path is None or isinstance(spec.path, str) or not spec.path.is_file():
             raise DataSourceNotFoundError(f"duckdb file not found: {spec.path}")
         if spec.table_name is None:
             raise DataSourceNotFoundError("duckdb connector requires table_name")

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/file_based.py

@@ -90,7 +90,7 @@ class FileDataHandle:
         return self._spec.source_type
 
     @property
-    def source_path(self) -> Path | None:
+    def source_path(self) -> Path | str | None:
         return self._path
 
     @property
@@ -174,8 +174,8 @@ class FileDataHandle:
         file_sha256: str | None = None
         content_sample_hash: str | None = None
         if mode == "full":
-            if self._path is None:
-                raise ConnectorError("file fingerprint requires a path")
+            if not isinstance(self._path, Path):
+                raise ConnectorError("file fingerprint requires a local file path")
             file_sha256 = _file_sha256(self._path)
         elif mode == "sampled":
             head = self._executor.execute("SELECT * FROM data LIMIT 1000")
@@ -199,9 +199,22 @@ class FileDataHandle:
 
     def content_fingerprint(self) -> str:
         """内容指纹（Step 55）：文件源 = 文件 SHA-256（Step 53 调度哈希语义）。"""
-        assert self._path is not None
+        if not isinstance(self._path, Path):
+            raise ConnectorError("file fingerprint requires a local file path")
         return _file_sha256(self._path)
 
+    def stats_fingerprint(self) -> str:
+        """统计层指纹（Step 58，ADR-058）：schema_hash + row_count 组合。
+
+        零内容读取：DESCRIBE（目录查询，PG/MySQL 为扩展目录、云文件为
+        parquet footer/CSV 元数据）+ count（远程库为 count 扫描，云文件
+        为 footer 行数）。调度两层快速失效第一层：统计层一致才进内容层。
+        """
+        self._check_open()
+        signature = [(c.name, c.physical_type) for c in self.schema().columns]
+        row_count = self.count_rows()
+        return hashlib.sha256(f"{_schema_hash(signature)}|{row_count}".encode()).hexdigest()
+
     def warnings(self) -> list[LoadWarning]:
         """公式注入标记（11.7）：遍历批处理扫描字符串列，前 _WARNING_CAP 条。"""
         self._check_open()

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/jsonl.py

@@ -59,7 +59,11 @@ class JsonlConnector:
     display_name = "JSONL"
 
     def supports(self, source: DataSourceSpec) -> bool:
-        return source.source_type == DataSourceType.JSONL and source.path is not None
+        return (
+            source.source_type == DataSourceType.JSONL
+            and source.path is not None
+            and not isinstance(source.path, str)
+        )
 
     def open(self, source: DataSourceSpec) -> JsonlDataHandle:
         return JsonlDataHandle(source)

datasentry_core-0.7.0/src/datasentry_core/connectors/mysql.py

@@ -0,0 +1,266 @@
+"""MySQL 数据源连接器（Step 56，V5：多数据源第三落点，ADR-056）。
+
+经 DuckDB mysql 扩展（LOAD mysql）只读 ATTACH 远程库
+（TYPE mysql, READ_ONLY），注册只读 data 视图后复用 FileDataHandle
+共享实现（schema/read_sample/sql_aggregate/count_rows/warnings）。
+表名必填；MySQL 无独立 schema 层（database 已在 DSN 内）。
+
+凭据红线（沿用 Step 55 全套）：
+- DSN 只存在于内存态（spec.options["dsn"]）或经 connection_ref
+  指向的环境变量（如 DATASENTRY_MYSQL_DSN），不落库、不进日志/evidence/报告
+- mysql 扩展连接失败时错误文本会原样回显 DSN 参数串
+  （"Failed to connect to MySQL database with parameters ..."）：
+  _RedactingExecutor 净化（DSN 整体 → mysql://***、密码 → ***）后转
+  ConnectorError 传播，CLI 错误面 / REST 400 只见净化文本
+- DuckDB 1.5.x mysql 扩展已知 bug：视图之上的聚合（count/groupby）触发
+  聚合下推时报内部绑定错误（直连 attach 正常，PG 扩展无此问题）。连接器
+  _ensure_view 统一 SET mysql_aggregate_pushdown_enabled = false 绕过：
+  聚合改在 DuckDB 本地执行，语义不变（ADR-056）
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from collections.abc import Iterator
+from typing import ClassVar
+from urllib.parse import urlparse
+
+import duckdb
+import pyarrow as pa
+
+from datasentry_core.connectors.base import FingerprintMode, FrameBatch
+from datasentry_core.connectors.errors import ConnectorError, DataSourceNotFoundError
+from datasentry_core.connectors.file_based import FileDataHandle, _schema_hash
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+from datasentry_core.engine import DuckDBExecutor
+from datasentry_core.engine.base import SqlParams
+from datasentry_core.models.fingerprint import DatasetFingerprint
+from datasentry_core.secrets import lookup_secret
+
+#: ATTACH 目标别名（连接器内部名，避开用户数据命名冲突概率极低）
+_DB_ALIAS = "my"
+
+#: 加载 mysql 扩展失败的提示（缺网络/离线/发行版未带扩展）
+_MYSQL_EXTENSION_HINT = (
+    "failed to load duckdb mysql extension: network access is required on first use "
+    "(or the extension must be pre-provisioned for offline environments); "
+    "original error: {}"
+)
+
+#: 连接失败提示（凭据已净化）
+_MYSQL_CONNECT_HINT = "mysql connection failed (check host/port/database/credentials): {}"
+
+#: KV 形式 DSN 的密码键（mysql 扩展用 passwd 而非 password）
+_KV_PASSWORD_PATTERN = re.compile(r"(passwd=\S+)")
+
+
+def _sql_string_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _ident(value: str) -> str:
+    """双引号标识符转义（防表名注入）。"""
+    return '"' + value.replace('"', '""') + '"'
+
+
+def redact_credentials(text: str, dsn: str) -> str:
+    """净化错误文本：DSN 整体与密码不对外（凭据红线，可单测）。
+
+    mysql 扩展错误文本回显两种形式：URL 形式（mysql://user:pass@host/db）
+    与 KV 形式（host=... passwd=...）——两种都要打码。
+    """
+    redacted = text.replace(dsn, "mysql://***")
+    redacted = _KV_PASSWORD_PATTERN.sub("passwd=***", redacted)
+    try:
+        parsed = urlparse(dsn)
+    except ValueError:
+        return redacted
+    if parsed.password:
+        redacted = redacted.replace(parsed.password, "***")
+    return redacted
+
+
+def _resolve_dsn(spec: DataSourceSpec) -> str:
+    """DSN 解析（Step 59，ADR-059）：options["dsn"] 内存态 > connection_ref
+    统一解析链（进程环境变量 > secrets.env，`lookup_secret`）。"""
+    dsn = spec.options.get("dsn")
+    if isinstance(dsn, str) and dsn:
+        return dsn
+    if spec.connection_ref:
+        ref = spec.connection_ref
+        dsn = lookup_secret(ref)
+        if dsn:
+            return dsn
+        raise DataSourceNotFoundError(f"mysql connection_ref not set (env or secrets.env): {ref}")
+    raise DataSourceNotFoundError(
+        "mysql connector requires a dsn (options['dsn'] or connection_ref env/secrets)"
+    )
+
+
+class _RedactingExecutor:
+    """MySQL 句柄专用执行器包装：DuckDB 异常净化后转 ConnectorError 传播。
+
+    只读守卫（sql_guard）在 DuckDBExecutor 内先于执行触发，其异常
+    （UnsafeSqlError 等连接器族）原样透传；查询期远程错误（断连等）
+    同样净化，避免 DSN 出现在 DetectorRun.error / 调度 summary 等
+    落库面。
+    """
+
+    def __init__(self, inner: DuckDBExecutor, dsn: str) -> None:
+        self._inner = inner
+        self._dsn = dsn
+
+    def execute_setup(self, sql: str) -> None:
+        try:
+            self._inner.execute_setup(sql)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_credentials(str(exc), self._dsn)) from exc
+
+    def register(self, name: str, obj: object) -> None:
+        self._inner.register(name, obj)
+
+    def execute(self, sql: str, params: SqlParams = None) -> pa.Table:
+        try:
+            return self._inner.execute(sql, params)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_credentials(str(exc), self._dsn)) from exc
+
+    def execute_stream(self, sql: str, batch_size: int = 65536) -> Iterator[pa.RecordBatch]:
+        try:
+            yield from self._inner.execute_stream(sql, batch_size)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_credentials(str(exc), self._dsn)) from exc
+
+    def close(self) -> None:
+        self._inner.close()
+
+
+class MySQLDataHandle(FileDataHandle):
+    """MySQL 远程表数据句柄（只读，表名必填，无文件字节）。"""
+
+    requires_path: ClassVar[bool] = False
+
+    def __init__(self, spec: DataSourceSpec) -> None:
+        super().__init__(spec)
+        if spec.table_name is None:
+            raise DataSourceNotFoundError("mysql connector requires table_name")
+        self._dsn = _resolve_dsn(spec)
+        self._table_name = spec.table_name
+        assert isinstance(self._executor, DuckDBExecutor)
+        self._executor = _RedactingExecutor(self._executor, self._dsn)
+
+    @property
+    def table_name(self) -> str | None:
+        return self._table_name
+
+    def _ensure_view(self) -> None:
+        if self._view_ready:
+            return
+        try:
+            # DuckDB 1.5.x mysql 扩展绑定 bug：SQL 经「mysql-attach 之上的视图」触发
+            # 聚合下推时报 "Failed to bind column reference"（直连 attach 无此问题）。
+            # 关闭聚合下推后聚合在 DuckDB 本地计算（结果语义不变，仅少一次远端
+            # 聚合执行），视图 + count/groupby 全部正常（ADR-056 已知问题记录）。
+            self._executor.execute_setup("SET mysql_aggregate_pushdown_enabled = false")
+            self._executor.execute_setup("LOAD mysql")
+        except ConnectorError as exc:
+            raise ConnectorError(_MYSQL_EXTENSION_HINT.format(exc)) from exc
+        try:
+            self._executor.execute_setup(
+                f"ATTACH {_sql_string_literal(self._dsn)} AS {_DB_ALIAS} (TYPE mysql, READ_ONLY)"
+            )
+        except ConnectorError as exc:
+            raise ConnectorError(_MYSQL_CONNECT_HINT.format(exc)) from exc
+        table_ident = f"{_DB_ALIAS}.{_ident(self._table_name)}"
+        try:
+            self._executor.execute(f"SELECT 1 FROM {table_ident} LIMIT 0")
+        except ConnectorError as exc:
+            raise DataSourceNotFoundError(f"mysql table not found: {self._table_name}") from exc
+        self._executor.execute_setup(f"CREATE OR REPLACE VIEW data AS SELECT * FROM {table_ident}")
+        self._view_ready = True
+
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]:
+        self._check_open()
+        if batch_size < 1:
+            raise ValueError("batch_size must be >= 1")
+        self._ensure_view()
+        row_offset = 0
+        for idx, batch in enumerate(
+            self._executor.execute_stream("SELECT * FROM data", batch_size)
+        ):
+            table = pa.Table.from_batches([batch])
+            yield FrameBatch(
+                source=self._spec,
+                table=table,
+                batch_index=idx,
+                row_offset=row_offset,
+            )
+            row_offset += table.num_rows
+
+    def content_fingerprint(self) -> str:
+        """内容指纹（Step 56）：单查询全表哈希（行序无关、NULL 不折叠）。
+
+        语义与 PostgresDataHandle 一致（Step 55 模式）：逐行
+        md5(concat_ws(chr(31), coalesce(col::VARCHAR, chr(0))...)) +
+        排序聚合 string_agg；并入 schema_hash 与行数。
+        """
+        self._check_open()
+        self._ensure_view()
+        signature = [(c.name, c.physical_type) for c in self.schema().columns]
+        schema_hash = _schema_hash(signature)
+        exprs = ", ".join(
+            f"coalesce(cast({_ident(name)} AS VARCHAR), chr(0))" for name, _typ in signature
+        )
+        table = self._executor.execute(
+            "SELECT count(*) AS n, md5(string_agg(rh, '' ORDER BY rh)) AS h FROM "
+            f"(SELECT md5(concat_ws(chr(31), {exprs})) AS rh FROM data) t"
+        )
+        row_count = table.column(0).to_pylist()[0]
+        rows_hash = table.column(1).to_pylist()[0]
+        assert isinstance(row_count, int)
+        digest = hashlib.sha256(f"{schema_hash}|{row_count}|{rows_hash or ''}".encode()).hexdigest()
+        return digest
+
+    def fingerprint(self, mode: FingerprintMode = "full") -> DatasetFingerprint:
+        """MySQL 无文件字节：file_sha256=None，内容指纹放 content_sample_hash。"""
+        self._check_open()
+        signature = [(c.name, c.physical_type) for c in self.schema().columns]
+        row_count = self.count_rows()
+        content_hash = self.content_fingerprint() if mode != "metadata_only" else None
+        return DatasetFingerprint(
+            dataset_id=self._spec.options.get("dataset_id", ""),
+            fingerprint_type=mode,
+            file_sha256=None,
+            schema_hash=_schema_hash(signature),
+            row_count=row_count,
+            column_count=len(signature),
+            column_signature=signature,
+            content_sample_hash=content_hash,
+        )
+
+
+class MySQLConnector:
+    """MySQL 数据源连接器（Step 56，V5，ADR-056）。"""
+
+    connector_id = "mysql"
+    display_name = "MySQL"
+
+    def supports(self, source: DataSourceSpec) -> bool:
+        if source.source_type != DataSourceType.MYSQL:
+            return False
+        dsn = source.options.get("dsn")
+        return (isinstance(dsn, str) and bool(dsn)) or source.connection_ref is not None
+
+    def open(self, source: DataSourceSpec) -> MySQLDataHandle:
+        if source.table_name is None:
+            raise DataSourceNotFoundError(
+                "mysql connector requires table_name (e.g. scan --table <table>)"
+            )
+        return MySQLDataHandle(source)
+
+    def close(self) -> None:
+        pass
+
+
+__all__ = ["MySQLConnector", "MySQLDataHandle", "redact_credentials"]

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/parquet.py

@@ -52,7 +52,11 @@ class ParquetConnector:
     display_name = "Parquet"
 
     def supports(self, source: DataSourceSpec) -> bool:
-        return source.source_type == DataSourceType.PARQUET and source.path is not None
+        return (
+            source.source_type == DataSourceType.PARQUET
+            and source.path is not None
+            and not isinstance(source.path, str)
+        )
 
     def open(self, source: DataSourceSpec) -> ParquetDataHandle:
         return ParquetDataHandle(source)

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/postgres.py

@@ -16,7 +16,6 @@
 from __future__ import annotations
 
 import hashlib
-import os
 from collections.abc import Iterator
 from typing import ClassVar, cast
 from urllib.parse import urlparse
@@ -31,6 +30,7 @@ from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
 from datasentry_core.engine import DuckDBExecutor
 from datasentry_core.engine.base import SqlParams
 from datasentry_core.models.fingerprint import DatasetFingerprint
+from datasentry_core.secrets import lookup_secret
 
 #: ATTACH 目标别名（连接器内部名，避开用户数据命名冲突概率极低）
 _DB_ALIAS = "pg"
@@ -68,18 +68,21 @@ def redact_credentials(text: str, dsn: str) -> str:
 
 
 def _resolve_dsn(spec: DataSourceSpec) -> str:
-    """DSN 解析：spec.options["dsn"]（内存态）或 connection_ref 环境变量引用。"""
+    """DSN 解析（Step 59，ADR-059）：options["dsn"] 内存态 > connection_ref
+    统一解析链（进程环境变量 > secrets.env，`lookup_secret`）。"""
     dsn = spec.options.get("dsn")
     if isinstance(dsn, str) and dsn:
         return dsn
     if spec.connection_ref:
         ref = spec.connection_ref
-        dsn = os.environ.get(ref)
+        dsn = lookup_secret(ref)
         if dsn:
             return dsn
-        raise DataSourceNotFoundError(f"postgres connection_ref env not set: {ref}")
+        raise DataSourceNotFoundError(
+            f"postgres connection_ref not set (env or secrets.env): {ref}"
+        )
     raise DataSourceNotFoundError(
-        "postgres connector requires a dsn (options['dsn'] or connection_ref env)"
+        "postgres connector requires a dsn (options['dsn'] or connection_ref env/secrets)"
     )
 
 

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/registry.py

@@ -7,8 +7,10 @@ from datasentry_core.connectors.csv import CsvConnector
 from datasentry_core.connectors.duckdb import DuckdbConnector
 from datasentry_core.connectors.errors import UnsupportedFormatError
 from datasentry_core.connectors.jsonl import JsonlConnector
+from datasentry_core.connectors.mysql import MySQLConnector
 from datasentry_core.connectors.parquet import ParquetConnector
 from datasentry_core.connectors.postgres import PostgresConnector
+from datasentry_core.connectors.remote_file import RemoteFileConnector
 from datasentry_core.connectors.spec import DataSourceSpec
 from datasentry_core.connectors.sqlite import SqliteConnector
 from datasentry_core.connectors.xlsx import XlsxConnector
@@ -56,4 +58,6 @@ def default_registry() -> ConnectorRegistry:
     registry.register(DuckdbConnector())
     registry.register(SqliteConnector())
     registry.register(PostgresConnector())
+    registry.register(MySQLConnector())
+    registry.register(RemoteFileConnector())
     return registry

datasentry_core-0.7.0/src/datasentry_core/connectors/remote_file.py

@@ -0,0 +1,263 @@
+"""云存储文件源连接器（Step 57，V5：云存储文件源，ADR-057）。
+
+经 DuckDB httpfs 扩展只读直读对象存储：s3://、gs://、az:// 前缀的
+CSV/Parquet/JSONL 文件，注册只读 data 视图后复用 FileDataHandle
+共享实现（schema/read_sample/sql_aggregate/count_rows/warnings）。
+零新依赖（httpfs 随 DuckDB 分发，离线需预装）。
+
+凭据红线（沿用 Step 55/56 全套语义）：
+- 凭据只走进程环境变量（AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY 等
+  httpfs 原生读取）或 spec.options 内存态覆盖；不落库、不进日志/
+  evidence/报告
+- 非机密会话配置（endpoint/region/url_style/use_ssl）可经 options
+  传入（如 MinIO：options["s3_endpoint"]），有自定义 endpoint 时
+  自动 path-style + 非 SSL（MinIO 必需）；无 endpoint 走 AWS 默认
+  （vhost-style + SSL）
+- httpfs 错误文本可能回显完整 URL（az:// 可含 SAS token 查询参数）：
+  _RemoteRedactingExecutor 将 URI 整体净化后转 ConnectorError 传播
+
+变更感知（快速失效层）：
+- httpfs 不暴露对象 ETag（glob 仅 file 列）→ `content_fingerprint()`
+  用 size + last_modified 元数据组合哈希（read_blob 元数据列，
+  HEAD 级开销 ~0.004s，免下载）作为调度跳过判定基准
+- 语义：同内容 → size/mtime 不变 → 指纹相同（跳过）；变更（重新
+  上传）→ Last-Modified 必更新 → 指纹变（重扫）。已知局限（ADR-057）：
+  同秒内同 size 覆盖的极限窗口会漏判；元数据抖动会多扫一次（无害）
+- 源不可达（对象删除/断网）→ DataSourceNotFoundError，调度回退
+  正常失败路径，不误跳过
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+from collections.abc import Iterator
+from typing import ClassVar
+
+import duckdb
+import pyarrow as pa
+
+from datasentry_core.connectors.base import FingerprintMode, FrameBatch
+from datasentry_core.connectors.errors import ConnectorError, DataSourceNotFoundError
+from datasentry_core.connectors.file_based import FileDataHandle, _schema_hash
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+from datasentry_core.engine import DuckDBExecutor
+from datasentry_core.engine.base import SqlParams
+from datasentry_core.models.fingerprint import DatasetFingerprint
+
+#: 云存储 URI 前缀（httpfs 原生支持的三种对象存储）
+_CLOUD_PREFIXES: tuple[str, ...] = ("s3://", "gs://", "az://")
+
+#: 扩展加载失败提示（缺网络/离线/发行版未带扩展）
+_HTTPFS_EXTENSION_HINT = (
+    "failed to load duckdb httpfs extension: network access is required on first use "
+    "(or the extension must be pre-provisioned for offline environments); "
+    "original error: {}"
+)
+
+#: read_* 函数按格式分派
+_READ_FUNCS = {
+    DataSourceType.CSV: "read_csv_auto",
+    DataSourceType.PARQUET: "read_parquet",
+    DataSourceType.JSONL: "read_json_auto",
+}
+
+
+def _sql_string_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def redact_uri(text: str, uri: str) -> str:
+    """净化错误文本：完整 URI（含 az:// SAS token 等查询参数）不对外。"""
+    return text.replace(uri, "<remote-uri>")
+
+
+def _is_cloud_uri(path: str) -> bool:
+    return path.startswith(_CLOUD_PREFIXES)
+
+
+def _s3_session_options(spec: DataSourceSpec) -> list[str]:
+    """S3 会话配置（仅 s3://）：endpoint 经 options 或标准 env；凭据只走 env。
+
+    有自定义 endpoint（MinIO 等）→ 自动 path-style + 非 SSL（MinIO 必需）；
+    无 endpoint → AWS 默认（vhost-style + SSL），零配置。
+    """
+    options = spec.options
+    endpoint = (
+        options.get("s3_endpoint")
+        or os.environ.get("AWS_ENDPOINT_URL_S3")
+        or os.environ.get("AWS_ENDPOINT_URL")
+    )
+    if not endpoint:
+        return []
+    statements = [
+        f"SET s3_endpoint = {_sql_string_literal(str(endpoint))}",
+        "SET s3_region = "
+        f"{_sql_string_literal(str(options.get('s3_region') or os.environ.get('AWS_REGION') or 'us-east-1'))}",  # noqa: E501
+        f"SET s3_url_style = {_sql_string_literal(str(options.get('s3_url_style') or 'path'))}",
+        f"SET s3_use_ssl = {_sql_string_literal(str(options.get('s3_use_ssl', 'false')).lower())}",
+    ]
+    return statements
+
+
+class _RemoteRedactingExecutor:
+    """云文件句柄专用执行器包装：DuckDB 异常净化（URI 打码）后转 ConnectorError。
+
+    只读守卫（sql_guard）在 DuckDBExecutor 内先于执行触发，其异常
+    （UnsafeSqlError 等连接器族）原样透传；httpfs 错误文本可能回显
+    完整 URL（az:// SAS token），净化后传播避免凭据出现在落库面。
+    """
+
+    def __init__(self, inner: DuckDBExecutor, uri: str) -> None:
+        self._inner = inner
+        self._uri = uri
+
+    def execute_setup(self, sql: str) -> None:
+        try:
+            self._inner.execute_setup(sql)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_uri(str(exc), self._uri)) from exc
+
+    def register(self, name: str, obj: object) -> None:
+        self._inner.register(name, obj)
+
+    def execute(self, sql: str, params: SqlParams = None) -> pa.Table:
+        try:
+            return self._inner.execute(sql, params)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_uri(str(exc), self._uri)) from exc
+
+    def execute_stream(self, sql: str, batch_size: int = 65536) -> Iterator[pa.RecordBatch]:
+        try:
+            yield from self._inner.execute_stream(sql, batch_size)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_uri(str(exc), self._uri)) from exc
+
+    def close(self) -> None:
+        self._inner.close()
+
+
+class RemoteFileDataHandle(FileDataHandle):
+    """云存储文件数据句柄（只读，s3:// gs:// az:// 前缀）。"""
+
+    requires_path: ClassVar[bool] = True
+
+    def __init__(self, spec: DataSourceSpec) -> None:
+        super().__init__(spec)
+        if not isinstance(self._path, str) or not _is_cloud_uri(self._path):
+            raise DataSourceNotFoundError(
+                f"remote file connector requires a cloud uri (s3:// gs:// az://), got: {self._path}"
+            )
+        self._uri = self._path
+        assert isinstance(self._executor, DuckDBExecutor)
+        self._executor = _RemoteRedactingExecutor(self._executor, self._uri)
+
+    @property
+    def source_path(self) -> str | None:
+        return self._uri
+
+    def _ensure_view(self) -> None:
+        if self._view_ready:
+            return
+        try:
+            self._executor.execute_setup("LOAD httpfs")
+        except ConnectorError as exc:
+            raise ConnectorError(_HTTPFS_EXTENSION_HINT.format(exc)) from exc
+        for statement in _s3_session_options(self._spec):
+            self._executor.execute_setup(statement)
+        read_func = _READ_FUNCS[self._spec.source_type]
+        try:
+            self._executor.execute(
+                f"SELECT 1 FROM {read_func}({_sql_string_literal(self._uri)}) LIMIT 0"
+            )
+        except ConnectorError as exc:
+            raise DataSourceNotFoundError(
+                f"remote file not found or unreadable: {read_func}: {exc}"
+            ) from exc
+        self._executor.execute_setup(
+            "CREATE OR REPLACE VIEW data AS "
+            f"SELECT * FROM {read_func}({_sql_string_literal(self._uri)})"
+        )
+        self._view_ready = True
+
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]:
+        self._check_open()
+        if batch_size < 1:
+            raise ValueError("batch_size must be >= 1")
+        self._ensure_view()
+        row_offset = 0
+        for idx, batch in enumerate(
+            self._executor.execute_stream("SELECT * FROM data", batch_size)
+        ):
+            table = pa.Table.from_batches([batch])
+            yield FrameBatch(
+                source=self._spec,
+                table=table,
+                batch_index=idx,
+                row_offset=row_offset,
+            )
+            row_offset += table.num_rows
+
+    def content_fingerprint(self) -> str:
+        """内容指纹（Step 57）：size + last_modified 元数据组合（快速失效层）。
+
+        read_blob 元数据列（HEAD 级开销 ~0.004s）免下载；对象重新上传
+        Last-Modified 必更新 → 指纹变。极限窗口（同秒同 size 覆盖）与
+        元数据抖动行为记录在 ADR-057 已知局限。
+        """
+        self._check_open()
+        self._ensure_view()
+        try:
+            meta = self._executor.execute(
+                f"SELECT size, last_modified FROM read_blob({_sql_string_literal(self._uri)})"
+            )
+        except ConnectorError as exc:
+            raise DataSourceNotFoundError(f"remote file metadata unavailable: {exc}") from exc
+        rows = meta.to_pylist()
+        if not rows:
+            raise DataSourceNotFoundError(f"remote file not found: {self._uri}")
+        size = rows[0]["size"]
+        last_modified = str(rows[0]["last_modified"])
+        assert isinstance(size, int)
+        return hashlib.sha256(f"{self._uri}|{size}|{last_modified}".encode()).hexdigest()
+
+    def fingerprint(self, mode: FingerprintMode = "full") -> DatasetFingerprint:
+        """云文件无本地字节：file_sha256=None，内容指纹（快速失效层）放
+        content_sample_hash（与 PG/MySQL 语义一致）。"""
+        self._check_open()
+        signature = [(c.name, c.physical_type) for c in self.schema().columns]
+        row_count = self.count_rows()
+        content_hash = self.content_fingerprint() if mode != "metadata_only" else None
+        return DatasetFingerprint(
+            dataset_id=self._spec.options.get("dataset_id", ""),
+            fingerprint_type=mode,
+            file_sha256=None,
+            schema_hash=_schema_hash(signature),
+            row_count=row_count,
+            column_count=len(signature),
+            column_signature=signature,
+            content_sample_hash=content_hash,
+        )
+
+
+class RemoteFileConnector:
+    """云存储文件源连接器（Step 57，V5，ADR-057）：CSV/Parquet/JSONL。"""
+
+    connector_id = "remote"
+    display_name = "Cloud Storage"
+
+    def supports(self, source: DataSourceSpec) -> bool:
+        return (
+            source.source_type in _READ_FUNCS
+            and isinstance(source.path, str)
+            and _is_cloud_uri(source.path)
+        )
+
+    def open(self, source: DataSourceSpec) -> RemoteFileDataHandle:
+        return RemoteFileDataHandle(source)
+
+    def close(self) -> None:
+        pass
+
+
+__all__ = ["RemoteFileConnector", "RemoteFileDataHandle", "_is_cloud_uri", "redact_uri"]

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/spec.py

@@ -21,16 +21,34 @@ class DataSourceType(StrEnum):
     XLSX = "xlsx"
     SQLITE = "sqlite"
     POSTGRESQL = "postgresql"
+    MYSQL = "mysql"
     DUCKDB = "duckdb"
 
 
+#: 文件扩展名 → 数据源类型（Step 18/57：scan_file 与调度器按后缀推断，
+#: 云存储文件源仅限 CSV/Parquet/JSONL，见 RemoteFileConnector）
+EXT_TO_SOURCE_TYPE: dict[str, DataSourceType] = {
+    ".csv": DataSourceType.CSV,
+    ".tsv": DataSourceType.CSV,
+    ".parquet": DataSourceType.PARQUET,
+    ".pq": DataSourceType.PARQUET,
+    ".jsonl": DataSourceType.JSONL,
+    ".ndjson": DataSourceType.JSONL,
+    ".xlsx": DataSourceType.XLSX,
+    ".duckdb": DataSourceType.DUCKDB,
+    ".db": DataSourceType.SQLITE,
+    ".sqlite": DataSourceType.SQLITE,
+    ".sqlite3": DataSourceType.SQLITE,
+}
+
+
 class DataSourceSpec(BaseModel):
     """数据源规格：描述"从哪读、怎么读"，不含凭据原文。"""
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     source_type: DataSourceType
-    path: Path | None = None
+    path: Path | str | None = None
     table_name: str | None = None
     options: dict[str, Any] = Field(default_factory=dict)
     connection_ref: str | None = None

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/connectors/sqlite.py

@@ -28,7 +28,7 @@ class SQLiteDataHandle(FileDataHandle):
 
     def __init__(self, spec: DataSourceSpec) -> None:
         super().__init__(spec)
-        if spec.path is None or not spec.path.is_file():
+        if spec.path is None or isinstance(spec.path, str) or not spec.path.is_file():
             raise DataSourceNotFoundError(f"sqlite file not found: {spec.path}")
         if spec.table_name is None:
             raise DataSourceNotFoundError("sqlite connector requires table_name")

{datasentry_core-0.6.0 → datasentry_core-0.7.0}/src/datasentry_core/repair/engine.py

@@ -251,7 +251,7 @@ class RepairEngine:
         repairs_dir.mkdir(parents=True, exist_ok=True)
         run_id = f"rep_{uuid.uuid4().hex[:12]}"
         source_path = context.handle.source_path
-        if source_path is None or not source_path.exists():
+        if source_path is None or isinstance(source_path, str) or not source_path.exists():
             raise FileNotFoundError("repair requires an on-disk source file")
         suffix = self._suffix(context)
         artifact_path = repairs_dir / f"{run_id}.before{suffix}"

datasentry_core-0.7.0/src/datasentry_core/secrets.py

@@ -0,0 +1,116 @@
+"""凭据文件管理（Step 59，ADR-059）：~/.config/datasentry/secrets.env。
+
+- 行格式 `KEY=VALUE`（空行/`#` 注释忽略），键名必须匹配环境变量命名
+  形态 `[A-Z][A-Z0-9_]*`（连接器 connection_ref 直接按键名引用）；
+- 文件权限强制 600（读取与写入双向校验，过松给可操作错误）；
+- 统一解析链（CLI 参数 > 进程环境变量 > secrets.env 自动加载）：
+  `lookup_secret` 先查进程 env 再查文件，均无返回 None；
+- 值不落库/日志/报告（连接器层仅内存态流转，沿用 Step 55/56 红线）。
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import stat
+from pathlib import Path
+
+_KEY_PATTERN = re.compile(r"[A-Z][A-Z0-9_]*")
+_FILE_NAME = "secrets.env"
+
+
+class SecretsFileError(ValueError):
+    """secrets.env 解析或权限错误；消息可操作（含文件路径与行号）。"""
+
+
+def secrets_path() -> Path:
+    """secrets.env 路径：DATASENTRY_CONFIG_HOME > XDG_CONFIG_HOME > ~/.config。"""
+    base = os.environ.get("DATASENTRY_CONFIG_HOME") or os.environ.get("XDG_CONFIG_HOME")
+    if base:
+        return Path(base) / "datasentry" / _FILE_NAME
+    return Path.home() / ".config" / "datasentry" / _FILE_NAME
+
+
+def _check_permissions(path: Path) -> None:
+    """权限校验（读取侧）：组/他位可读即拒绝——凭据 600 是强制语义。"""
+    mode = stat.S_IMODE(path.stat().st_mode)
+    if mode & 0o077:
+        raise SecretsFileError(
+            f"secrets file permissions too open: {path} is {oct(mode)}; "
+            "run `chmod 600 <file>` (datasentry secrets set 会自动修正)"
+        )
+
+
+def _read_secrets(path: Path) -> dict[str, str]:
+    """解析 secrets.env（不校验权限——写入路径自会重写为 600）。"""
+    secrets: dict[str, str] = {}
+    for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        if "=" not in stripped:
+            raise SecretsFileError(f"malformed secrets line {path}:{lineno} (expected KEY=VALUE)")
+        name, value = stripped.split("=", 1)
+        if not _KEY_PATTERN.fullmatch(name):
+            raise SecretsFileError(
+                f"invalid secret key {path}:{lineno}: {name!r} (keys must match [A-Z][A-Z0-9_]*)"
+            )
+        secrets[name] = value
+    return secrets
+
+
+def load_secrets(path: Path | None = None) -> dict[str, str]:
+    """解析 secrets.env 为 dict；文件缺失返回 {}，权限过松/格式错误抛错。"""
+    secrets_file = path or secrets_path()
+    if not secrets_file.is_file():
+        return {}
+    _check_permissions(secrets_file)
+    return _read_secrets(secrets_file)
+
+
+def lookup_secret(name: str, path: Path | None = None) -> str | None:
+    """统一解析链：进程环境变量优先，回落 secrets.env；均无返回 None。"""
+    value = os.environ.get(name)
+    if value:
+        return value
+    return load_secrets(path).get(name)
+
+
+def write_secrets(secrets: dict[str, str], path: Path | None = None) -> Path:
+    """原子写回 secrets.env：父目录 700、文件 600（set/rm 共用）。"""
+    secrets_file = path or secrets_path()
+    secrets_file.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
+    tmp = secrets_file.with_name(f".{secrets_file.name}.tmp")
+    with tmp.open("w", encoding="utf-8") as fh:
+        os.fchmod(fh.fileno(), 0o600)
+        for name in sorted(secrets):
+            fh.write(f"{name}={secrets[name]}\n")
+    os.chmod(tmp, 0o600)
+    os.replace(tmp, secrets_file)
+    os.chmod(secrets_file, 0o600)
+    return secrets_file
+
+
+def set_secret(name: str, value: str, path: Path | None = None) -> Path:
+    """写入/更新单个凭据（CLI `secrets set` 入口）。
+
+    读取侧绕过权限检查：本操作随即将文件整体重写为 600——过松权限
+    在此被自动修正（而非拒绝写入）。
+    """
+    secrets_file = path or secrets_path()
+    secrets = _read_secrets(secrets_file) if secrets_file.is_file() else {}
+    secrets[name] = value
+    return write_secrets(secrets, secrets_file)
+
+
+def remove_secret(name: str, path: Path | None = None) -> bool:
+    """删除单个凭据；键不存在返回 False（CLI `secrets rm` 入口）。"""
+    secrets_file = path or secrets_path()
+    if not secrets_file.is_file():
+        return False
+    secrets = _read_secrets(secrets_file)
+    if name not in secrets:
+        return False
+    del secrets[name]
+    write_secrets(secrets, secrets_file)
+    return True