datasentry-core 0.3.0__tar.gz → 0.5.0__tar.gz

{datasentry_core-0.3.0 → datasentry_core-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: datasentry-core
-Version: 0.3.0
+Version: 0.5.0
 Summary: DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs).
 License: Apache-2.0
 Keywords: data-observability,data-quality,duckdb

{datasentry_core-0.3.0 → datasentry_core-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datasentry-core"
-version = "0.3.0"
+version = "0.5.0"
 description = "DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs)."
 readme = "README.md"
 requires-python = ">=3.12"

{datasentry_core-0.3.0 → datasentry_core-0.5.0}/src/datasentry_core/__init__.py

@@ -108,4 +108,4 @@ __all__ = [
     "ValidationResult",
 ]
 
-__version__ = "0.3.0"
+__version__ = "0.5.0"

{datasentry_core-0.3.0 → datasentry_core-0.5.0}/src/datasentry_core/connectors/__init__.py

@@ -22,6 +22,7 @@ from datasentry_core.connectors.jsonl import JsonlConnector
 from datasentry_core.connectors.parquet import ParquetConnector
 from datasentry_core.connectors.registry import ConnectorRegistry, default_registry
 from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+from datasentry_core.connectors.sqlite import SqliteConnector
 from datasentry_core.connectors.xlsx import XlsxConnector
 
 __all__ = [
@@ -42,6 +43,7 @@ __all__ = [
     "ParquetConnector",
     "SamplingMethod",
     "SchemaInfo",
+    "SqliteConnector",
     "UnsafeSqlError",
     "UnsupportedFormatError",
     "XlsxConnector",

{datasentry_core-0.3.0 → datasentry_core-0.5.0}/src/datasentry_core/connectors/registry.py

@@ -9,6 +9,7 @@ from datasentry_core.connectors.errors import UnsupportedFormatError
 from datasentry_core.connectors.jsonl import JsonlConnector
 from datasentry_core.connectors.parquet import ParquetConnector
 from datasentry_core.connectors.spec import DataSourceSpec
+from datasentry_core.connectors.sqlite import SqliteConnector
 from datasentry_core.connectors.xlsx import XlsxConnector
 
 
@@ -45,11 +46,12 @@ class ConnectorRegistry:
 
 
 def default_registry() -> ConnectorRegistry:
-    """默认注册表：文件型连接器 + DuckDB 文件连接器（Step 38，V1 数据库型落地）。"""
+    """默认注册表：文件型连接器 + DuckDB/SQLite 文件连接器（Step 38/54）。"""
     registry = ConnectorRegistry()
     registry.register(CsvConnector())
     registry.register(ParquetConnector())
     registry.register(JsonlConnector())
     registry.register(XlsxConnector())
     registry.register(DuckdbConnector())
+    registry.register(SqliteConnector())
     return registry

datasentry_core-0.5.0/src/datasentry_core/connectors/sqlite.py

@@ -0,0 +1,87 @@
+"""SQLite 数据源连接器（Step 54：V3 多数据源第一落点）。
+
+对 `.db` / `.sqlite` / `.sqlite3` 文件经 DuckDB 的 sqlite 扩展
+（sqlite_scan 表函数）暴露只读 data 视图，复用 FileDataHandle
+共享实现（schema/read_sample/sql_aggregate/count_rows/fingerprint/
+warnings）。表名必填（spec.table_name）；路径/表名一律字符串
+字面量转义（防注入）。
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import pyarrow as pa
+
+from datasentry_core.connectors.base import FrameBatch
+from datasentry_core.connectors.errors import DataSourceNotFoundError
+from datasentry_core.connectors.file_based import FileDataHandle
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+
+
+def _sql_string_literal(value: str) -> str:
+    return "'" + value.replace("'", "''") + "'"
+
+
+class SQLiteDataHandle(FileDataHandle):
+    """SQLite 文件表数据句柄（只读，表名必填）。"""
+
+    def __init__(self, spec: DataSourceSpec) -> None:
+        super().__init__(spec)
+        if spec.path is None or not spec.path.is_file():
+            raise DataSourceNotFoundError(f"sqlite file not found: {spec.path}")
+        if spec.table_name is None:
+            raise DataSourceNotFoundError("sqlite connector requires table_name")
+        self._table_name = spec.table_name
+
+    @property
+    def table_name(self) -> str | None:
+        return self._table_name
+
+    def _ensure_view(self) -> None:
+        if self._view_ready:
+            return
+        assert self._path is not None
+        self._executor.execute_setup("LOAD sqlite")
+        self._executor.execute_setup(
+            "CREATE OR REPLACE VIEW data AS "
+            f"SELECT * FROM sqlite_scan({_sql_string_literal(str(self._path))}, "
+            f"{_sql_string_literal(self._table_name)})"
+        )
+        self._view_ready = True
+
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]:
+        self._check_open()
+        if batch_size < 1:
+            raise ValueError("batch_size must be >= 1")
+        self._ensure_view()
+        row_offset = 0
+        for idx, batch in enumerate(
+            self._executor.execute_stream("SELECT * FROM data", batch_size)
+        ):
+            table = pa.Table.from_batches([batch])
+            yield FrameBatch(
+                source=self._spec,
+                table=table,
+                batch_index=idx,
+                row_offset=row_offset,
+            )
+            row_offset += table.num_rows
+
+
+class SqliteConnector:
+    """SQLite 文件连接器（Step 54，V3 多数据源第一落点）。"""
+
+    connector_id = "sqlite"
+    display_name = "SQLite"
+
+    def supports(self, source: DataSourceSpec) -> bool:
+        return source.source_type == DataSourceType.SQLITE and source.path is not None
+
+    def open(self, source: DataSourceSpec) -> SQLiteDataHandle:
+        if source.table_name is None:
+            raise DataSourceNotFoundError("sqlite connector requires table_name")
+        return SQLiteDataHandle(source)
+
+    def close(self) -> None:
+        pass

{datasentry_core-0.3.0 → datasentry_core-0.5.0}/src/datasentry_core/storage/schema.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 import sqlite3
 
 #: 当前 schema 版本（PRAGMA user_version）
-SCHEMA_VERSION = 5
+SCHEMA_VERSION = 6
 
 _SCHEMA_DDL = """
 PRAGMA journal_mode = WAL;
@@ -329,7 +329,9 @@ CREATE TABLE IF NOT EXISTS job_runs (
     scan_run_id TEXT,
     summary     TEXT,
     error       TEXT,
-    webhook_at  TEXT
+    webhook_at  TEXT,
+    file_hash   TEXT,
+    skipped     INTEGER NOT NULL DEFAULT 0
 );
 
 CREATE INDEX IF NOT EXISTS idx_scan_runs_dataset   ON scan_runs(dataset_id);
@@ -387,5 +389,9 @@ def migrate(conn: sqlite3.Connection) -> None:
     # v4 → v5：调度质量门禁（Step 52）：scheduled_jobs.gate_quality_min
     if version < 5:
         _ensure_column(conn, "scheduled_jobs", "gate_quality_min", "gate_quality_min REAL")
+    # v5 → v6：变更感知增量调度（Step 53）：job_runs.file_hash + skipped
+    if version < 6:
+        _ensure_column(conn, "job_runs", "file_hash", "file_hash TEXT")
+        _ensure_column(conn, "job_runs", "skipped", "skipped INTEGER NOT NULL DEFAULT 0")
     conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
     conn.commit()