datasentry-core 0.1.0__py3-none-any.whl

datasentry_core/connectors/errors.py

@@ -0,0 +1,19 @@
+"""连接器错误类型。SDK 层的 DataSentryError 体系在 SDK 包建立时映射（47.2）。"""
+
+from __future__ import annotations
+
+
+class ConnectorError(Exception):
+    """连接器基类错误。"""
+
+
+class UnsupportedFormatError(ConnectorError):
+    """数据源格式不被任何已注册连接器支持。"""
+
+
+class DataSourceNotFoundError(ConnectorError):
+    """数据源路径/资源不存在或不可读。"""
+
+
+class UnsafeSqlError(ConnectorError):
+    """聚合查询未通过只读白名单校验（11.10/49.2）。"""

datasentry_core/connectors/file_based.py

@@ -0,0 +1,209 @@
+"""文件型数据句柄基类（Step 18）：Parquet/JSONL/XLSX 连接器的共享实现。
+
+与 CSV 连接器保持独立（CSV 有编码探测/分隔符嗅探等专属逻辑，不重构，ADR-019）。
+共享部分：schema（DESCRIBE 视图）/read_sample/sql_aggregate/count_rows/
+fingerprint/warnings（公式注入扫描）/close。
+子类需实现：_ensure_view()（注册只读 data 视图）与 read_batches()（流式）。
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Iterator
+from pathlib import Path
+
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from datasentry_core.connectors.base import (
+    ColumnInfo,
+    FingerprintMode,
+    FrameBatch,
+    LoadWarning,
+    SamplingMethod,
+    SchemaInfo,
+)
+from datasentry_core.connectors.errors import DataSourceNotFoundError
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+from datasentry_core.engine import DuckDBExecutor
+from datasentry_core.models.enums import Severity
+from datasentry_core.models.fingerprint import DatasetFingerprint
+
+FORMULA_INJECTION_PREFIXES: tuple[str, ...] = ("=", "+", "-", "@", "\t", "\r")
+_WARNING_CAP = 100
+
+
+def _schema_hash(column_signature: list[tuple[str, str]]) -> str:
+    """schema_hash = 列签名规范 JSON 的 SHA-256（19.1）。"""
+    canonical = json.dumps(column_signature, sort_keys=True, ensure_ascii=True)
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _file_sha256(path: Path) -> str:
+    """流式文件 SHA-256（≤1GB 全量，大文件走 sampled 档）。"""
+    digest = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+class FileDataHandle:
+    """文件型数据句柄基类（DataHandle 协议共享实现）。"""
+
+    def __init__(self, spec: DataSourceSpec) -> None:
+        if spec.path is None:
+            raise DataSourceNotFoundError(f"{type(self).__name__} requires a path")
+        self._spec = spec
+        self._path = spec.path
+        self._executor = DuckDBExecutor()
+        self._view_ready = False
+        self._schema_info: SchemaInfo | None = None
+        self._warnings: list[LoadWarning] | None = None
+        self._closed = False
+
+    @property
+    def source_type(self) -> DataSourceType:
+        return self._spec.source_type
+
+    @property
+    def source_path(self) -> Path | None:
+        return self._path
+
+    @property
+    def table_name(self) -> str | None:
+        """DuckDB 等需要表名的数据源返回表名，其他返回 None（Step 40）。"""
+        return self._spec.table_name
+
+    # ---- 子类契约 ------------------------------------------------------
+
+    def _ensure_view(self) -> None:
+        raise NotImplementedError
+
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]:
+        raise NotImplementedError
+
+    # ---- 共享实现 ------------------------------------------------------
+
+    def _check_open(self) -> None:
+        if self._closed:
+            raise ValueError("handle is closed")
+
+    def schema(self) -> SchemaInfo:
+        self._check_open()
+        if self._schema_info is None:
+            self._ensure_view()
+            table = self._executor.execute("DESCRIBE SELECT * FROM data")
+            names = table.column(0).to_pylist()
+            types = table.column(1).to_pylist()
+            self._schema_info = SchemaInfo(
+                columns=[
+                    ColumnInfo(name=str(name), physical_type=str(typ))
+                    for name, typ in zip(names, types, strict=True)
+                ]
+            )
+        return self._schema_info
+
+    def read_sample(self, n: int, method: SamplingMethod = "random") -> FrameBatch:
+        """抽样读取：reservoir(REPEATABLE seed) 保证可复现（20.3）。"""
+        self._check_open()
+        if n < 1:
+            raise ValueError("n must be >= 1")
+        self._ensure_view()
+        seed = int(self._spec.options.get("seed", 42))
+        if method == "none":
+            sql = "SELECT * FROM data LIMIT ?"
+            params: list[object] = [n]
+        elif method == "time_based":
+            column = self._spec.options.get("time_column")
+            if not isinstance(column, str) or not column:
+                method = "reservoir"
+                sql = f"SELECT * FROM data USING SAMPLE reservoir({n} ROWS) REPEATABLE ({seed})"
+                params = []
+            else:
+                sql = f'SELECT * FROM data ORDER BY "{column}" LIMIT ?'
+                params = [n]
+        else:
+            # SAMPLE 子句不支持预编译参数，n/seed 均为代码内整数（已校验），安全内联
+            sql = f"SELECT * FROM data USING SAMPLE reservoir({n} ROWS) REPEATABLE ({seed})"
+            params = []
+        table = self._executor.execute(sql, params)
+        return FrameBatch(source=self._spec, table=table)
+
+    def sql_aggregate(self, sql: str, params: dict[str, object] | None = None) -> FrameBatch:
+        """只读聚合（DuckDB pushdown）。SQL 必须通过 engine.sql_guard 只读白名单。"""
+        self._check_open()
+        self._ensure_view()
+        table = self._executor.execute(sql, params)
+        return FrameBatch(source=self._spec, table=table)
+
+    def count_rows(self) -> int:
+        self._check_open()
+        self._ensure_view()
+        value = self._executor.execute("SELECT count(*) FROM data").column(0).to_pylist()[0]
+        assert isinstance(value, int)
+        return value
+
+    def fingerprint(self, mode: FingerprintMode = "full") -> DatasetFingerprint:
+        self._check_open()
+        signature = [(col.name, col.physical_type) for col in self.schema().columns]
+        row_count = self.count_rows()
+        file_sha256: str | None = None
+        content_sample_hash: str | None = None
+        if mode == "full":
+            file_sha256 = _file_sha256(self._path)
+        elif mode == "sampled":
+            head = self._executor.execute("SELECT * FROM data LIMIT 1000")
+            rand = self._executor.execute(
+                "SELECT * FROM data USING SAMPLE reservoir(100000) REPEATABLE (42)"
+            )
+            digest = hashlib.sha256()
+            digest.update(str(head.to_pylist()).encode("utf-8"))
+            digest.update(str(rand.to_pylist()).encode("utf-8"))
+            content_sample_hash = digest.hexdigest()
+        return DatasetFingerprint(
+            dataset_id=self._spec.options.get("dataset_id", ""),
+            fingerprint_type=mode,
+            file_sha256=file_sha256,
+            schema_hash=_schema_hash(signature),
+            row_count=row_count,
+            column_count=len(signature),
+            column_signature=signature,
+            content_sample_hash=content_sample_hash,
+        )
+
+    def warnings(self) -> list[LoadWarning]:
+        """公式注入标记（11.7）：遍历批处理扫描字符串列，前 _WARNING_CAP 条。"""
+        self._check_open()
+        if self._warnings is not None:
+            return list(self._warnings)
+        warnings: list[LoadWarning] = []
+        for batch in self.read_batches():
+            for col_name in batch.table.column_names:
+                col = batch.table[col_name]
+                if not pa.types.is_string(col.type) or pa.types.is_null(col.type):
+                    continue
+                for prefix in FORMULA_INJECTION_PREFIXES:
+                    mask = pc.starts_with(col, pattern=prefix)
+                    indices = pc.indices_nonzero(mask)
+                    for row in indices.to_pylist():
+                        value = col[row].as_py() or ""
+                        warnings.append(
+                            LoadWarning(
+                                row=batch.row_offset + int(row),
+                                column=col_name,
+                                message=f"value starts with formula-injection prefix {prefix!r}",
+                                severity=Severity.LOW,
+                                value_preview=value[:20],
+                            )
+                        )
+                        if len(warnings) >= _WARNING_CAP:
+                            self._warnings = warnings
+                            return warnings
+        self._warnings = warnings
+        return list(warnings)
+
+    def close(self) -> None:
+        self._executor.close()
+        self._closed = True

datasentry_core/connectors/jsonl.py

@@ -0,0 +1,71 @@
+"""JSONL 连接器（Step 18）：duckdb read_json_auto(newline_delimited) 视图。
+
+read_batches 用 SQL LIMIT/OFFSET 分页（duckdb 无 JSONL 流式读接口；
+1e6 行预算内可接受，ADR-019）。
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from datasentry_core.connectors.base import FrameBatch
+from datasentry_core.connectors.file_based import FileDataHandle
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+
+
+def _sql_string_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+class JsonlDataHandle(FileDataHandle):
+    """JSONL 数据句柄。"""
+
+    def _ensure_view(self) -> None:
+        if self._view_ready:
+            return
+        self._executor.execute_setup(
+            "CREATE OR REPLACE VIEW data AS "
+            "SELECT * FROM read_json_auto("
+            f"{_sql_string_literal(str(self._path))}, "
+            "format='newline_delimited')"
+        )
+        self._view_ready = True
+
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]:
+        self._check_open()
+        if batch_size < 1:
+            raise ValueError("batch_size must be >= 1")
+        self._ensure_view()
+        offset = 0
+        idx = 0
+        while True:
+            table = self._executor.execute(f"SELECT * FROM data LIMIT {batch_size} OFFSET {offset}")
+            if table.num_rows == 0:
+                return
+            yield FrameBatch(
+                source=self._spec,
+                table=table,
+                batch_index=idx,
+                row_offset=offset,
+            )
+            offset += table.num_rows
+            idx += 1
+
+
+class JsonlConnector:
+    """JSONL 连接器（7.1）。"""
+
+    connector_id = "jsonl"
+    display_name = "JSONL"
+
+    def supports(self, source: DataSourceSpec) -> bool:
+        return source.source_type == DataSourceType.JSONL and source.path is not None
+
+    def open(self, source: DataSourceSpec) -> JsonlDataHandle:
+        return JsonlDataHandle(source)
+
+    def close(self) -> None:
+        pass
+
+
+__all__ = ["JsonlConnector", "JsonlDataHandle"]

datasentry_core/connectors/parquet.py

@@ -0,0 +1,64 @@
+"""Parquet 连接器（Step 18）：duckdb read_parquet 视图 + pyarrow 流式批读。"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+from datasentry_core.connectors.base import FrameBatch
+from datasentry_core.connectors.file_based import FileDataHandle
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+
+
+def _sql_string_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+class ParquetDataHandle(FileDataHandle):
+    """Parquet 数据句柄。"""
+
+    def _ensure_view(self) -> None:
+        if self._view_ready:
+            return
+        self._executor.execute_setup(
+            "CREATE OR REPLACE VIEW data AS "
+            f"SELECT * FROM read_parquet({_sql_string_literal(str(self._path))})"
+        )
+        self._view_ready = True
+
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]:
+        self._check_open()
+        if batch_size < 1:
+            raise ValueError("batch_size must be >= 1")
+        row_offset = 0
+        with pq.ParquetFile(self._path) as reader:
+            for idx, batch in enumerate(reader.iter_batches(batch_size=batch_size)):
+                table = pa.Table.from_batches([batch])
+                yield FrameBatch(
+                    source=self._spec,
+                    table=table,
+                    batch_index=idx,
+                    row_offset=row_offset,
+                )
+                row_offset += table.num_rows
+
+
+class ParquetConnector:
+    """Parquet 连接器（7.1）。"""
+
+    connector_id = "parquet"
+    display_name = "Parquet"
+
+    def supports(self, source: DataSourceSpec) -> bool:
+        return source.source_type == DataSourceType.PARQUET and source.path is not None
+
+    def open(self, source: DataSourceSpec) -> ParquetDataHandle:
+        return ParquetDataHandle(source)
+
+    def close(self) -> None:
+        pass
+
+
+__all__ = ["ParquetConnector", "ParquetDataHandle"]

datasentry_core/connectors/registry.py

@@ -0,0 +1,55 @@
+"""连接器注册表（49.1）：按 source_type 分发。"""
+
+from __future__ import annotations
+
+from datasentry_core.connectors.base import DataConnector, DataHandle
+from datasentry_core.connectors.csv import CsvConnector
+from datasentry_core.connectors.duckdb import DuckdbConnector
+from datasentry_core.connectors.errors import UnsupportedFormatError
+from datasentry_core.connectors.jsonl import JsonlConnector
+from datasentry_core.connectors.parquet import ParquetConnector
+from datasentry_core.connectors.spec import DataSourceSpec
+from datasentry_core.connectors.xlsx import XlsxConnector
+
+
+class ConnectorRegistry:
+    """连接器注册表：register/get/list/get_for/open。"""
+
+    def __init__(self) -> None:
+        self._connectors: dict[str, DataConnector] = {}
+
+    def register(self, connector: DataConnector) -> None:
+        """注册连接器，重复注册抛 ValueError。"""
+        if connector.connector_id in self._connectors:
+            raise ValueError(f"connector already registered: {connector.connector_id}")
+        self._connectors[connector.connector_id] = connector
+
+    def get(self, connector_id: str) -> DataConnector | None:
+        """按 id 获取连接器。"""
+        return self._connectors.get(connector_id)
+
+    def list(self) -> list[DataConnector]:
+        """全部已注册连接器。"""
+        return list(self._connectors.values())
+
+    def get_for(self, source: DataSourceSpec) -> DataConnector:
+        """找到支持该数据源的连接器，找不到抛 UnsupportedFormatError。"""
+        for connector in self._connectors.values():
+            if connector.supports(source):
+                return connector
+        raise UnsupportedFormatError(f"no connector supports source type={source.source_type}")
+
+    def open(self, source: DataSourceSpec) -> DataHandle:
+        """打开数据源，返回 DataHandle。"""
+        return self.get_for(source).open(source)
+
+
+def default_registry() -> ConnectorRegistry:
+    """默认注册表：文件型连接器 + DuckDB 文件连接器（Step 38，V1 数据库型落地）。"""
+    registry = ConnectorRegistry()
+    registry.register(CsvConnector())
+    registry.register(ParquetConnector())
+    registry.register(JsonlConnector())
+    registry.register(XlsxConnector())
+    registry.register(DuckdbConnector())
+    return registry

datasentry_core/connectors/spec.py

@@ -0,0 +1,37 @@
+"""数据源规格（49.1/四十三第 10 项：凭据不入库，只存引用）。"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from enum import StrEnum
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from datasentry_core.models.evidence import utcnow
+
+
+class DataSourceType(StrEnum):
+    """数据源类型（7.1 MVP 六个 + V1 扩展）。"""
+
+    CSV = "csv"
+    PARQUET = "parquet"
+    JSONL = "jsonl"
+    XLSX = "xlsx"
+    SQLITE = "sqlite"
+    POSTGRESQL = "postgresql"
+    DUCKDB = "duckdb"
+
+
+class DataSourceSpec(BaseModel):
+    """数据源规格：描述"从哪读、怎么读"，不含凭据原文。"""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    source_type: DataSourceType
+    path: Path | None = None
+    table_name: str | None = None
+    options: dict[str, Any] = Field(default_factory=dict)
+    connection_ref: str | None = None
+    created_at: datetime = Field(default_factory=utcnow)

datasentry_core/connectors/xlsx.py

@@ -0,0 +1,109 @@
+"""XLSX 连接器（Step 18）：openpyxl 读取 → pyarrow 表注册。
+
+公式注入扫描价值最高（电子表格场景）；混合类型列自动推断失败时回退全字符串
+（ADR-019）。read_batches 内存切片（MVP 预算 1e6 行内，ADR-019）。
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import pyarrow as pa
+from openpyxl import load_workbook
+
+from datasentry_core.connectors.base import FrameBatch
+from datasentry_core.connectors.file_based import FileDataHandle
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+
+
+def _rows_to_table(names: list[str], rows: list[list[object]]) -> pa.Table:
+    """行值列表 → pyarrow 表；混合类型列自动推断失败时回退全字符串（ADR-019）。"""
+    pylist = [dict(zip(names, r, strict=False)) for r in rows]
+    try:
+        return pa.Table.from_pylist(pylist)
+    except (pa.ArrowInvalid, pa.ArrowTypeError):
+        stringified = [
+            {name: (str(value) if value is not None else None) for name, value in r.items()}
+            for r in pylist
+        ]
+        return pa.Table.from_pylist(
+            stringified,
+            schema=pa.schema([(name, pa.string()) for name in names]),
+        )
+
+
+class XlsxDataHandle(FileDataHandle):
+    """XLSX 数据句柄。"""
+
+    def _load_rows(self) -> tuple[list[str], list[list[object]]]:
+        """读取指定 sheet 为 (列名, 行值)（header_row 可配，默认第 0 行）。
+
+        data_only=False：公式单元格返回公式文本（公式注入可检测，ADR-019）；
+        计算结果缓存读取归 V1。
+        """
+        sheet_ref = self._spec.options.get("sheet")
+        header_row = int(self._spec.options.get("header_row", 0))
+        workbook = load_workbook(self._path, read_only=True, data_only=False)
+        try:
+            if sheet_ref is None:
+                sheet = workbook.worksheets[0]
+            elif isinstance(sheet_ref, int):
+                sheet = workbook.worksheets[sheet_ref]
+            else:
+                sheet = workbook[str(sheet_ref)]
+            rows = list(sheet.iter_rows(values_only=True))
+        finally:
+            workbook.close()
+        if header_row >= len(rows):
+            return [], []
+        names = [str(c) if c is not None else f"col_{i}" for i, c in enumerate(rows[header_row])]
+        data = [list(r) for r in rows[header_row + 1 :]]
+        return names, data
+
+    def _ensure_view(self) -> None:
+        if self._view_ready:
+            return
+        names, rows = self._load_rows()
+        if not names:
+            raise ValueError("xlsx sheet is empty")
+        if not rows:
+            table = pa.table({name: pa.array([], type=pa.null()) for name in names})
+        else:
+            table = _rows_to_table(names, rows)
+        self._executor.register("data", table)
+        self._view_ready = True
+
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]:
+        self._check_open()
+        if batch_size < 1:
+            raise ValueError("batch_size must be >= 1")
+        names, rows = self._load_rows()
+        if not names:
+            return
+        for idx in range(0, len(rows), batch_size):
+            table = _rows_to_table(names, rows[idx : idx + batch_size])
+            yield FrameBatch(
+                source=self._spec,
+                table=table,
+                batch_index=idx // batch_size,
+                row_offset=idx,
+            )
+
+
+class XlsxConnector:
+    """XLSX 连接器（7.1）。"""
+
+    connector_id = "xlsx"
+    display_name = "XLSX"
+
+    def supports(self, source: DataSourceSpec) -> bool:
+        return source.source_type == DataSourceType.XLSX and source.path is not None
+
+    def open(self, source: DataSourceSpec) -> XlsxDataHandle:
+        return XlsxDataHandle(source)
+
+    def close(self) -> None:
+        pass
+
+
+__all__ = ["XlsxConnector", "XlsxDataHandle"]

datasentry_core/contracts/__init__.py

@@ -0,0 +1,24 @@
+"""契约导出（V1：Pandera / Great Expectations）。"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from datasentry_core.models.contract import Contract
+
+
+def to_pandera(contract: Contract) -> str:
+    """契约 → Pandera DataFrameSchema 代码字符串。"""
+    from datasentry_core.contracts.exporters import to_pandera as _to
+
+    return _to(contract)
+
+
+def to_great_expectations(contract: Contract) -> dict[str, Any]:
+    """契约 → Great Expectations ExpectationSuite 字典。"""
+    from datasentry_core.contracts.exporters import to_great_expectations as _to
+
+    return _to(contract)
+
+
+__all__ = ["to_great_expectations", "to_pandera"]