datasentry-core 0.6.0__tar.gz → 0.8.0__tar.gz

datasentry_core-0.8.0/.gitignore

@@ -0,0 +1,43 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+dist/
+build/
+.coverage
+htmlcov/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+hypothesis/
+.hypothesis/
+
+# Project data
+.datasentry/
+*.db
+*.sqlite3
+
+# Env / secrets
+.env
+credentials.json
+
+# OS
+.DS_Store
+
+# Editors
+.idea/
+.vscode/
+~$*
+workspace/
+
+# Debug scratch (agent sessions)
+p/
+
+# 开发任务书（prompt）——本地保留，不进仓库（V22 起约定）
+docs/V*_DEV_PROMPT.md
+
+# 完整产品设计与开发 Prompt（本地保留，不进仓库）
+DataSentry AI 产品设计与开发 Prompt*.md
+DataSentry AI 产品设计与开发 Prompt*.docx
+DataSentry AI 增长与推广 Prompt.md

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datasentry-core
-Version: 0.6.0
+Version: 0.8.0
 Summary: DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs).
 License: Apache-2.0
 Keywords: data-observability,data-quality,duckdb

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datasentry-core"
-version = "0.6.0"
+version = "0.8.0"
 description = "DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs)."
 readme = "README.md"
 requires-python = ">=3.12"

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/src/datasentry_core/__init__.py

@@ -108,4 +108,4 @@ __all__ = [
     "ValidationResult",
 ]
 
-__version__ = "0.5.1"
+__version__ = "1.0.0"

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/__init__.py

@@ -19,9 +19,11 @@ from datasentry_core.connectors.errors import (
     UnsupportedFormatError,
 )
 from datasentry_core.connectors.jsonl import JsonlConnector
+from datasentry_core.connectors.mysql import MySQLConnector, MySQLDataHandle
 from datasentry_core.connectors.parquet import ParquetConnector
 from datasentry_core.connectors.postgres import PostgresConnector, PostgresDataHandle
 from datasentry_core.connectors.registry import ConnectorRegistry, default_registry
+from datasentry_core.connectors.remote_file import RemoteFileConnector, RemoteFileDataHandle
 from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
 from datasentry_core.connectors.sqlite import SqliteConnector
 from datasentry_core.connectors.xlsx import XlsxConnector
@@ -41,9 +43,13 @@ __all__ = [
     "FrameBatch",
     "JsonlConnector",
     "LoadWarning",
+    "MySQLConnector",
+    "MySQLDataHandle",
     "ParquetConnector",
     "PostgresConnector",
     "PostgresDataHandle",
+    "RemoteFileConnector",
+    "RemoteFileDataHandle",
     "SamplingMethod",
     "SchemaInfo",
     "SqliteConnector",

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/base.py

@@ -80,7 +80,7 @@ class DataHandle(Protocol):
     @property
     def source_type(self) -> DataSourceType: ...
     @property
-    def source_path(self) -> Path | None: ...
+    def source_path(self) -> Path | str | None: ...
 
     @property
     def table_name(self) -> str | None:
@@ -98,6 +98,16 @@ class DataHandle(Protocol):
         由子类实现内容哈希——调度器变更感知（Step 53/55）的跳过判定基准。"""
         ...
 
+    def stats_fingerprint(self) -> str:
+        """统计层指纹（Step 58，ADR-058）：schema_hash + row_count 组合，零内容读取。
+
+        调度两层快速失效第一层：统计层一致才计算内容指纹；统计层变化
+        立即判定变更。远程源（PG/MySQL/云文件）由 FileDataHandle 默认
+        实现（DESCRIBE 目录查询 + count），文件源不参与（沿用 Step 53
+        单层文件 SHA-256）。协议默认体：本地文件句柄不实现（调度器不会
+        对本地文件调用），实现者不覆盖即视为不适用。"""
+        raise NotImplementedError("stats_fingerprint is not supported by this handle")
+
     def warnings(self) -> list[LoadWarning]: ...
     def close(self) -> None: ...
 

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/csv.py

@@ -27,12 +27,15 @@ from datasentry_core.connectors.base import (
     SamplingMethod,
     SchemaInfo,
 )
-from datasentry_core.connectors.errors import DataSourceNotFoundError
+from datasentry_core.connectors.errors import ConnectorError, DataSourceNotFoundError
 from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
 from datasentry_core.engine import DuckDBExecutor
 from datasentry_core.models.enums import Severity
 from datasentry_core.models.fingerprint import DatasetFingerprint
 
+#: 非 utf-8 全文件入内存的警告阈值（Step 73/ADR-073）
+_NON_UTF8_FULL_READ_WARN_BYTES = 512 * 1024 * 1024
+
 FORMULA_INJECTION_PREFIXES: tuple[str, ...] = ("=", "+", "-", "@", "\t", "\r")
 _WARNING_CAP = 100
 
@@ -91,8 +94,8 @@ class CsvDataHandle:
     """CSV 数据句柄：pyarrow 流式读取 + duckdb 只读聚合。"""
 
     def __init__(self, spec: DataSourceSpec) -> None:
-        if spec.path is None:
-            raise DataSourceNotFoundError("csv source requires a path")
+        if spec.path is None or isinstance(spec.path, str):
+            raise DataSourceNotFoundError("csv source requires a local file path")
         self._spec = spec
         self._path = spec.path
         self._encoding = _detect_encoding(spec.path, override=spec.options.get("encoding"))
@@ -108,7 +111,7 @@ class CsvDataHandle:
         return self._spec.source_type
 
     @property
-    def source_path(self) -> Path | None:
+    def source_path(self) -> Path | str | None:
         return self._path
 
     @property
@@ -132,6 +135,22 @@ class CsvDataHandle:
                 f"delim={_sql_string_literal(self._delimiter)})"
             )
         else:
+            # Step 73/ADR-073：非 utf-8 全文件入内存（pyarrow 无流式编码读取），
+            # 超阈值时预置提示 warning，引导 --sampling 或转码
+            if self._path.stat().st_size > _NON_UTF8_FULL_READ_WARN_BYTES:
+                self._warnings = [
+                    LoadWarning(
+                        row=0,
+                        column="",
+                        message=(
+                            "non-utf-8 csv is fully loaded into memory "
+                            f"({self._path.stat().st_size} bytes > "
+                            f"{_NON_UTF8_FULL_READ_WARN_BYTES}); "
+                            "use --sampling or re-encode as utf-8"
+                        ),
+                        severity=Severity.LOW,
+                    )
+                ]
             table = pa_csv.read_csv(
                 self._path,
                 read_options=pa_csv.ReadOptions(encoding=self._encoding),
@@ -288,9 +307,17 @@ class CsvDataHandle:
 
     def content_fingerprint(self) -> str:
         """内容指纹（Step 55）：文件源 = 文件 SHA-256（Step 53 调度哈希语义）。"""
-        assert self._path is not None
+        if not isinstance(self._path, Path):
+            raise ConnectorError("csv fingerprint requires a local file path")
         return _file_sha256(self._path)
 
+    def stats_fingerprint(self) -> str:
+        """统计层指纹（Step 58）：本地文件源不参与两层指纹——调度器仅对
+        远程源（PG/MySQL/云文件）调用，本地文件沿用 Step 53 单层语义。"""
+        raise NotImplementedError(
+            "local file handles do not participate in two-layer fingerprinting"
+        )
+
 
 def _file_sha256(path: Path) -> str:
     """流式文件 SHA-256（≤1GB 全量，大文件走 sampled 档）。"""
@@ -308,7 +335,12 @@ class CsvConnector:
     display_name = "CSV"
 
     def supports(self, source: DataSourceSpec) -> bool:
-        return source.source_type == DataSourceType.CSV and source.path is not None
+        # Step 57：本地文件连接器排除云 URI（s3:// gs:// az:// 归 RemoteFileConnector）
+        return (
+            source.source_type == DataSourceType.CSV
+            and source.path is not None
+            and not isinstance(source.path, str)
+        )
 
     def open(self, source: DataSourceSpec) -> DataHandle:
         return CsvDataHandle(source)

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/duckdb.py

@@ -35,7 +35,7 @@ class DuckDBDataHandle(FileDataHandle):
 
     def __init__(self, spec: DataSourceSpec) -> None:
         super().__init__(spec)
-        if spec.path is None or not spec.path.is_file():
+        if spec.path is None or isinstance(spec.path, str) or not spec.path.is_file():
             raise DataSourceNotFoundError(f"duckdb file not found: {spec.path}")
         if spec.table_name is None:
             raise DataSourceNotFoundError("duckdb connector requires table_name")

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/file_based.py

@@ -90,7 +90,7 @@ class FileDataHandle:
         return self._spec.source_type
 
     @property
-    def source_path(self) -> Path | None:
+    def source_path(self) -> Path | str | None:
         return self._path
 
     @property
@@ -174,8 +174,8 @@ class FileDataHandle:
         file_sha256: str | None = None
         content_sample_hash: str | None = None
         if mode == "full":
-            if self._path is None:
-                raise ConnectorError("file fingerprint requires a path")
+            if not isinstance(self._path, Path):
+                raise ConnectorError("file fingerprint requires a local file path")
             file_sha256 = _file_sha256(self._path)
         elif mode == "sampled":
             head = self._executor.execute("SELECT * FROM data LIMIT 1000")
@@ -199,9 +199,22 @@ class FileDataHandle:
 
     def content_fingerprint(self) -> str:
         """内容指纹（Step 55）：文件源 = 文件 SHA-256（Step 53 调度哈希语义）。"""
-        assert self._path is not None
+        if not isinstance(self._path, Path):
+            raise ConnectorError("file fingerprint requires a local file path")
         return _file_sha256(self._path)
 
+    def stats_fingerprint(self) -> str:
+        """统计层指纹（Step 58，ADR-058）：schema_hash + row_count 组合。
+
+        零内容读取：DESCRIBE（目录查询，PG/MySQL 为扩展目录、云文件为
+        parquet footer/CSV 元数据）+ count（远程库为 count 扫描，云文件
+        为 footer 行数）。调度两层快速失效第一层：统计层一致才进内容层。
+        """
+        self._check_open()
+        signature = [(c.name, c.physical_type) for c in self.schema().columns]
+        row_count = self.count_rows()
+        return hashlib.sha256(f"{_schema_hash(signature)}|{row_count}".encode()).hexdigest()
+
     def warnings(self) -> list[LoadWarning]:
         """公式注入标记（11.7）：遍历批处理扫描字符串列，前 _WARNING_CAP 条。"""
         self._check_open()

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/jsonl.py

@@ -59,7 +59,11 @@ class JsonlConnector:
     display_name = "JSONL"
 
     def supports(self, source: DataSourceSpec) -> bool:
-        return source.source_type == DataSourceType.JSONL and source.path is not None
+        return (
+            source.source_type == DataSourceType.JSONL
+            and source.path is not None
+            and not isinstance(source.path, str)
+        )
 
     def open(self, source: DataSourceSpec) -> JsonlDataHandle:
         return JsonlDataHandle(source)

datasentry_core-0.8.0/src/datasentry_core/connectors/mysql.py

@@ -0,0 +1,266 @@
+"""MySQL 数据源连接器（Step 56，V5：多数据源第三落点，ADR-056）。
+
+经 DuckDB mysql 扩展（LOAD mysql）只读 ATTACH 远程库
+（TYPE mysql, READ_ONLY），注册只读 data 视图后复用 FileDataHandle
+共享实现（schema/read_sample/sql_aggregate/count_rows/warnings）。
+表名必填；MySQL 无独立 schema 层（database 已在 DSN 内）。
+
+凭据红线（沿用 Step 55 全套）：
+- DSN 只存在于内存态（spec.options["dsn"]）或经 connection_ref
+  指向的环境变量（如 DATASENTRY_MYSQL_DSN），不落库、不进日志/evidence/报告
+- mysql 扩展连接失败时错误文本会原样回显 DSN 参数串
+  （"Failed to connect to MySQL database with parameters ..."）：
+  _RedactingExecutor 净化（DSN 整体 → mysql://***、密码 → ***）后转
+  ConnectorError 传播，CLI 错误面 / REST 400 只见净化文本
+- DuckDB 1.5.x mysql 扩展已知 bug：视图之上的聚合（count/groupby）触发
+  聚合下推时报内部绑定错误（直连 attach 正常，PG 扩展无此问题）。连接器
+  _ensure_view 统一 SET mysql_aggregate_pushdown_enabled = false 绕过：
+  聚合改在 DuckDB 本地执行，语义不变（ADR-056）
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from collections.abc import Iterator
+from typing import ClassVar
+from urllib.parse import urlparse
+
+import duckdb
+import pyarrow as pa
+
+from datasentry_core.connectors.base import FingerprintMode, FrameBatch
+from datasentry_core.connectors.errors import ConnectorError, DataSourceNotFoundError
+from datasentry_core.connectors.file_based import FileDataHandle, _schema_hash
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+from datasentry_core.engine import DuckDBExecutor
+from datasentry_core.engine.base import SqlParams
+from datasentry_core.models.fingerprint import DatasetFingerprint
+from datasentry_core.secrets import lookup_secret
+
+#: ATTACH 目标别名（连接器内部名，避开用户数据命名冲突概率极低）
+_DB_ALIAS = "my"
+
+#: 加载 mysql 扩展失败的提示（缺网络/离线/发行版未带扩展）
+_MYSQL_EXTENSION_HINT = (
+    "failed to load duckdb mysql extension: network access is required on first use "
+    "(or the extension must be pre-provisioned for offline environments); "
+    "original error: {}"
+)
+
+#: 连接失败提示（凭据已净化）
+_MYSQL_CONNECT_HINT = "mysql connection failed (check host/port/database/credentials): {}"
+
+#: KV 形式 DSN 的密码键（mysql 扩展用 passwd 而非 password）
+_KV_PASSWORD_PATTERN = re.compile(r"(passwd=\S+)")
+
+
+def _sql_string_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _ident(value: str) -> str:
+    """双引号标识符转义（防表名注入）。"""
+    return '"' + value.replace('"', '""') + '"'
+
+
+def redact_credentials(text: str, dsn: str) -> str:
+    """净化错误文本：DSN 整体与密码不对外（凭据红线，可单测）。
+
+    mysql 扩展错误文本回显两种形式：URL 形式（mysql://user:pass@host/db）
+    与 KV 形式（host=... passwd=...）——两种都要打码。
+    """
+    redacted = text.replace(dsn, "mysql://***")
+    redacted = _KV_PASSWORD_PATTERN.sub("passwd=***", redacted)
+    try:
+        parsed = urlparse(dsn)
+    except ValueError:
+        return redacted
+    if parsed.password:
+        redacted = redacted.replace(parsed.password, "***")
+    return redacted
+
+
+def _resolve_dsn(spec: DataSourceSpec) -> str:
+    """DSN 解析（Step 59，ADR-059）：options["dsn"] 内存态 > connection_ref
+    统一解析链（进程环境变量 > secrets.env，`lookup_secret`）。"""
+    dsn = spec.options.get("dsn")
+    if isinstance(dsn, str) and dsn:
+        return dsn
+    if spec.connection_ref:
+        ref = spec.connection_ref
+        dsn = lookup_secret(ref)
+        if dsn:
+            return dsn
+        raise DataSourceNotFoundError(f"mysql connection_ref not set (env or secrets.env): {ref}")
+    raise DataSourceNotFoundError(
+        "mysql connector requires a dsn (options['dsn'] or connection_ref env/secrets)"
+    )
+
+
+class _RedactingExecutor:
+    """MySQL 句柄专用执行器包装：DuckDB 异常净化后转 ConnectorError 传播。
+
+    只读守卫（sql_guard）在 DuckDBExecutor 内先于执行触发，其异常
+    （UnsafeSqlError 等连接器族）原样透传；查询期远程错误（断连等）
+    同样净化，避免 DSN 出现在 DetectorRun.error / 调度 summary 等
+    落库面。
+    """
+
+    def __init__(self, inner: DuckDBExecutor, dsn: str) -> None:
+        self._inner = inner
+        self._dsn = dsn
+
+    def execute_setup(self, sql: str) -> None:
+        try:
+            self._inner.execute_setup(sql)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_credentials(str(exc), self._dsn)) from exc
+
+    def register(self, name: str, obj: object) -> None:
+        self._inner.register(name, obj)
+
+    def execute(self, sql: str, params: SqlParams = None) -> pa.Table:
+        try:
+            return self._inner.execute(sql, params)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_credentials(str(exc), self._dsn)) from exc
+
+    def execute_stream(self, sql: str, batch_size: int = 65536) -> Iterator[pa.RecordBatch]:
+        try:
+            yield from self._inner.execute_stream(sql, batch_size)
+        except duckdb.Error as exc:
+            raise ConnectorError(redact_credentials(str(exc), self._dsn)) from exc
+
+    def close(self) -> None:
+        self._inner.close()
+
+
+class MySQLDataHandle(FileDataHandle):
+    """MySQL 远程表数据句柄（只读，表名必填，无文件字节）。"""
+
+    requires_path: ClassVar[bool] = False
+
+    def __init__(self, spec: DataSourceSpec) -> None:
+        super().__init__(spec)
+        if spec.table_name is None:
+            raise DataSourceNotFoundError("mysql connector requires table_name")
+        self._dsn = _resolve_dsn(spec)
+        self._table_name = spec.table_name
+        assert isinstance(self._executor, DuckDBExecutor)
+        self._executor = _RedactingExecutor(self._executor, self._dsn)
+
+    @property
+    def table_name(self) -> str | None:
+        return self._table_name
+
+    def _ensure_view(self) -> None:
+        if self._view_ready:
+            return
+        try:
+            # DuckDB 1.5.x mysql 扩展绑定 bug：SQL 经「mysql-attach 之上的视图」触发
+            # 聚合下推时报 "Failed to bind column reference"（直连 attach 无此问题）。
+            # 关闭聚合下推后聚合在 DuckDB 本地计算（结果语义不变，仅少一次远端
+            # 聚合执行），视图 + count/groupby 全部正常（ADR-056 已知问题记录）。
+            self._executor.execute_setup("SET mysql_aggregate_pushdown_enabled = false")
+            self._executor.execute_setup("LOAD mysql")
+        except ConnectorError as exc:
+            raise ConnectorError(_MYSQL_EXTENSION_HINT.format(exc)) from exc
+        try:
+            self._executor.execute_setup(
+                f"ATTACH {_sql_string_literal(self._dsn)} AS {_DB_ALIAS} (TYPE mysql, READ_ONLY)"
+            )
+        except ConnectorError as exc:
+            raise ConnectorError(_MYSQL_CONNECT_HINT.format(exc)) from exc
+        table_ident = f"{_DB_ALIAS}.{_ident(self._table_name)}"
+        try:
+            self._executor.execute(f"SELECT 1 FROM {table_ident} LIMIT 0")
+        except ConnectorError as exc:
+            raise DataSourceNotFoundError(f"mysql table not found: {self._table_name}") from exc
+        self._executor.execute_setup(f"CREATE OR REPLACE VIEW data AS SELECT * FROM {table_ident}")
+        self._view_ready = True
+
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]:
+        self._check_open()
+        if batch_size < 1:
+            raise ValueError("batch_size must be >= 1")
+        self._ensure_view()
+        row_offset = 0
+        for idx, batch in enumerate(
+            self._executor.execute_stream("SELECT * FROM data", batch_size)
+        ):
+            table = pa.Table.from_batches([batch])
+            yield FrameBatch(
+                source=self._spec,
+                table=table,
+                batch_index=idx,
+                row_offset=row_offset,
+            )
+            row_offset += table.num_rows
+
+    def content_fingerprint(self) -> str:
+        """内容指纹（Step 56）：单查询全表哈希（行序无关、NULL 不折叠）。
+
+        语义与 PostgresDataHandle 一致（Step 55 模式）：逐行
+        md5(concat_ws(chr(31), coalesce(col::VARCHAR, chr(0))...)) +
+        排序聚合 string_agg；并入 schema_hash 与行数。
+        """
+        self._check_open()
+        self._ensure_view()
+        signature = [(c.name, c.physical_type) for c in self.schema().columns]
+        schema_hash = _schema_hash(signature)
+        exprs = ", ".join(
+            f"coalesce(cast({_ident(name)} AS VARCHAR), chr(0))" for name, _typ in signature
+        )
+        table = self._executor.execute(
+            "SELECT count(*) AS n, md5(string_agg(rh, '' ORDER BY rh)) AS h FROM "
+            f"(SELECT md5(concat_ws(chr(31), {exprs})) AS rh FROM data) t"
+        )
+        row_count = table.column(0).to_pylist()[0]
+        rows_hash = table.column(1).to_pylist()[0]
+        assert isinstance(row_count, int)
+        digest = hashlib.sha256(f"{schema_hash}|{row_count}|{rows_hash or ''}".encode()).hexdigest()
+        return digest
+
+    def fingerprint(self, mode: FingerprintMode = "full") -> DatasetFingerprint:
+        """MySQL 无文件字节：file_sha256=None，内容指纹放 content_sample_hash。"""
+        self._check_open()
+        signature = [(c.name, c.physical_type) for c in self.schema().columns]
+        row_count = self.count_rows()
+        content_hash = self.content_fingerprint() if mode != "metadata_only" else None
+        return DatasetFingerprint(
+            dataset_id=self._spec.options.get("dataset_id", ""),
+            fingerprint_type=mode,
+            file_sha256=None,
+            schema_hash=_schema_hash(signature),
+            row_count=row_count,
+            column_count=len(signature),
+            column_signature=signature,
+            content_sample_hash=content_hash,
+        )
+
+
+class MySQLConnector:
+    """MySQL 数据源连接器（Step 56，V5，ADR-056）。"""
+
+    connector_id = "mysql"
+    display_name = "MySQL"
+
+    def supports(self, source: DataSourceSpec) -> bool:
+        if source.source_type != DataSourceType.MYSQL:
+            return False
+        dsn = source.options.get("dsn")
+        return (isinstance(dsn, str) and bool(dsn)) or source.connection_ref is not None
+
+    def open(self, source: DataSourceSpec) -> MySQLDataHandle:
+        if source.table_name is None:
+            raise DataSourceNotFoundError(
+                "mysql connector requires table_name (e.g. scan --table <table>)"
+            )
+        return MySQLDataHandle(source)
+
+    def close(self) -> None:
+        pass
+
+
+__all__ = ["MySQLConnector", "MySQLDataHandle", "redact_credentials"]

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/parquet.py

@@ -52,7 +52,11 @@ class ParquetConnector:
     display_name = "Parquet"
 
     def supports(self, source: DataSourceSpec) -> bool:
-        return source.source_type == DataSourceType.PARQUET and source.path is not None
+        return (
+            source.source_type == DataSourceType.PARQUET
+            and source.path is not None
+            and not isinstance(source.path, str)
+        )
 
     def open(self, source: DataSourceSpec) -> ParquetDataHandle:
         return ParquetDataHandle(source)

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/postgres.py

@@ -16,7 +16,6 @@
 from __future__ import annotations
 
 import hashlib
-import os
 from collections.abc import Iterator
 from typing import ClassVar, cast
 from urllib.parse import urlparse
@@ -31,6 +30,7 @@ from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
 from datasentry_core.engine import DuckDBExecutor
 from datasentry_core.engine.base import SqlParams
 from datasentry_core.models.fingerprint import DatasetFingerprint
+from datasentry_core.secrets import lookup_secret
 
 #: ATTACH 目标别名（连接器内部名，避开用户数据命名冲突概率极低）
 _DB_ALIAS = "pg"
@@ -68,18 +68,21 @@ def redact_credentials(text: str, dsn: str) -> str:
 
 
 def _resolve_dsn(spec: DataSourceSpec) -> str:
-    """DSN 解析：spec.options["dsn"]（内存态）或 connection_ref 环境变量引用。"""
+    """DSN 解析（Step 59，ADR-059）：options["dsn"] 内存态 > connection_ref
+    统一解析链（进程环境变量 > secrets.env，`lookup_secret`）。"""
     dsn = spec.options.get("dsn")
     if isinstance(dsn, str) and dsn:
         return dsn
     if spec.connection_ref:
         ref = spec.connection_ref
-        dsn = os.environ.get(ref)
+        dsn = lookup_secret(ref)
         if dsn:
             return dsn
-        raise DataSourceNotFoundError(f"postgres connection_ref env not set: {ref}")
+        raise DataSourceNotFoundError(
+            f"postgres connection_ref not set (env or secrets.env): {ref}"
+        )
     raise DataSourceNotFoundError(
-        "postgres connector requires a dsn (options['dsn'] or connection_ref env)"
+        "postgres connector requires a dsn (options['dsn'] or connection_ref env/secrets)"
     )
 
 

{datasentry_core-0.6.0 → datasentry_core-0.8.0}/src/datasentry_core/connectors/registry.py

@@ -7,8 +7,10 @@ from datasentry_core.connectors.csv import CsvConnector
 from datasentry_core.connectors.duckdb import DuckdbConnector
 from datasentry_core.connectors.errors import UnsupportedFormatError
 from datasentry_core.connectors.jsonl import JsonlConnector
+from datasentry_core.connectors.mysql import MySQLConnector
 from datasentry_core.connectors.parquet import ParquetConnector
 from datasentry_core.connectors.postgres import PostgresConnector
+from datasentry_core.connectors.remote_file import RemoteFileConnector
 from datasentry_core.connectors.spec import DataSourceSpec
 from datasentry_core.connectors.sqlite import SqliteConnector
 from datasentry_core.connectors.xlsx import XlsxConnector
@@ -56,4 +58,6 @@ def default_registry() -> ConnectorRegistry:
     registry.register(DuckdbConnector())
     registry.register(SqliteConnector())
     registry.register(PostgresConnector())
+    registry.register(MySQLConnector())
+    registry.register(RemoteFileConnector())
     return registry