datasentry-core 0.1.0__tar.gz

datasentry_core-0.1.0/.gitignore

@@ -0,0 +1,32 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.venv/
+dist/
+build/
+.coverage
+htmlcov/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+hypothesis/
+.hypothesis/
+
+# Project data
+.datasentry/
+*.db
+*.sqlite3
+
+# Env / secrets
+.env
+credentials.json
+
+# OS
+.DS_Store
+
+# Editors
+.idea/
+.vscode/
+~$*
+workspace/

datasentry_core-0.1.0/PKG-INFO

@@ -0,0 +1,59 @@
+Metadata-Version: 2.4
+Name: datasentry-core
+Version: 0.1.0
+Summary: DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs).
+License: Apache-2.0
+Keywords: data-observability,data-quality,duckdb
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.12
+Requires-Dist: charset-normalizer>=3.3
+Requires-Dist: duckdb>=1.0
+Requires-Dist: openpyxl>=3.1
+Requires-Dist: pyarrow>=16.0
+Requires-Dist: pydantic>=2.7
+Requires-Dist: scikit-learn>=1.5
+Description-Content-Type: text/markdown
+
+# datasentry-core
+
+DataSentry AI 核心领域模型与引擎：连接器、检测器、融合、评分、
+修复、规则引擎与隐私脱敏。纯离线、零网络依赖，可独立测试。
+
+```
+pip install datasentry-core
+```
+
+## 模块
+
+- `connectors/` — 多格式数据源（CSV / Parquet / JSONL / XLSX）
+- `detectors/` — 检测器注册表与 36+ 内置检测器（`Detector` 协议）
+- `engine/` — 画像、融合、评分
+- `repair/` — 修复引擎（propose → preview → apply → rollback）
+- `rules/` — 规则模型与预运行引擎（期望语义取反）
+- `privacy/` — 确定性 PII 脱敏（mask_profile / mask_rows）
+- `llm/` — LLM Provider 协议与数据结构（零网络）
+- `plugins.py` — 检测器插件加载（插件 API v1，ADR-031）
+
+## 使用
+
+```python
+from datasentry_core.connectors import DataSourceSpec, default_registry
+from datasentry_core.detectors.initial import register_default_detectors
+from datasentry_core.detectors import DetectorRegistry
+from datasentry_core.detectors.runner import ScanRunner
+
+registry = DetectorRegistry()
+register_default_detectors(registry)
+runner = ScanRunner(registry)
+
+handle = default_registry().open(DataSourceSpec(source_type="csv", path="data.csv"))
+runs = runner.run_all(handle, config)
+```
+
+## 许可
+
+Apache-2.0。完整产品（CLI / API / UI）见 `datasentry` 包。

datasentry_core-0.1.0/README.md

@@ -0,0 +1,39 @@
+# datasentry-core
+
+DataSentry AI 核心领域模型与引擎：连接器、检测器、融合、评分、
+修复、规则引擎与隐私脱敏。纯离线、零网络依赖，可独立测试。
+
+```
+pip install datasentry-core
+```
+
+## 模块
+
+- `connectors/` — 多格式数据源（CSV / Parquet / JSONL / XLSX）
+- `detectors/` — 检测器注册表与 36+ 内置检测器（`Detector` 协议）
+- `engine/` — 画像、融合、评分
+- `repair/` — 修复引擎（propose → preview → apply → rollback）
+- `rules/` — 规则模型与预运行引擎（期望语义取反）
+- `privacy/` — 确定性 PII 脱敏（mask_profile / mask_rows）
+- `llm/` — LLM Provider 协议与数据结构（零网络）
+- `plugins.py` — 检测器插件加载（插件 API v1，ADR-031）
+
+## 使用
+
+```python
+from datasentry_core.connectors import DataSourceSpec, default_registry
+from datasentry_core.detectors.initial import register_default_detectors
+from datasentry_core.detectors import DetectorRegistry
+from datasentry_core.detectors.runner import ScanRunner
+
+registry = DetectorRegistry()
+register_default_detectors(registry)
+runner = ScanRunner(registry)
+
+handle = default_registry().open(DataSourceSpec(source_type="csv", path="data.csv"))
+runs = runner.run_all(handle, config)
+```
+
+## 许可
+
+Apache-2.0。完整产品（CLI / API / UI）见 `datasentry` 包。

datasentry_core-0.1.0/pyproject.toml

@@ -0,0 +1,30 @@
+[project]
+name = "datasentry-core"
+version = "0.1.0"
+description = "DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs)."
+readme = "README.md"
+requires-python = ">=3.12"
+license = { text = "Apache-2.0" }
+keywords = ["data-quality", "data-observability", "duckdb"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+dependencies = [
+    "pydantic>=2.7",
+    "duckdb>=1.0",
+    "pyarrow>=16.0",
+    "charset-normalizer>=3.3",
+    "openpyxl>=3.1",
+    "scikit-learn>=1.5",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/datasentry_core"]

datasentry_core-0.1.0/src/datasentry_core/__init__.py

@@ -0,0 +1,111 @@
+"""DataSentry AI core package."""
+
+from datasentry_core.models import (
+    AIExplanation,
+    AuditEvent,
+    BusinessCriticality,
+    CauseHypothesis,
+    ColumnCheck,
+    ColumnContract,
+    ColumnDrift,
+    ColumnProfile,
+    Condition,
+    Contract,
+    DatasetContract,
+    DatasetFingerprint,
+    DatasetProfile,
+    DetectorRun,
+    DriftReport,
+    Evidence,
+    EvidenceProvenance,
+    EvidenceType,
+    FeedbackEntry,
+    Issue,
+    IssueStatus,
+    LLMInvocation,
+    LLMUsageSummary,
+    MaskConfig,
+    QualityDimension,
+    QualityGate,
+    QualityScore,
+    RecommendedAction,
+    RepairOperation,
+    RepairOperationRecord,
+    RepairPreview,
+    RepairProposal,
+    RepairProposalStatus,
+    RepairRun,
+    RepairRunStatus,
+    ReproducibilityInfo,
+    RiskLevel,
+    RowBeforeAfter,
+    Rule,
+    RuleCandidate,
+    RulePreflightReport,
+    RuleType,
+    SamplingConfig,
+    SamplingInfo,
+    ScanConfig,
+    ScanRun,
+    SchemaChange,
+    SemanticEvidence,
+    SemanticProfile,
+    Severity,
+    ValidationResult,
+)
+
+__all__ = [
+    "AIExplanation",
+    "AuditEvent",
+    "BusinessCriticality",
+    "CauseHypothesis",
+    "ColumnCheck",
+    "ColumnContract",
+    "ColumnDrift",
+    "ColumnProfile",
+    "Condition",
+    "Contract",
+    "DatasetContract",
+    "DatasetFingerprint",
+    "DatasetProfile",
+    "DetectorRun",
+    "DriftReport",
+    "Evidence",
+    "EvidenceProvenance",
+    "EvidenceType",
+    "FeedbackEntry",
+    "Issue",
+    "IssueStatus",
+    "LLMInvocation",
+    "LLMUsageSummary",
+    "MaskConfig",
+    "QualityDimension",
+    "QualityGate",
+    "QualityScore",
+    "RecommendedAction",
+    "RepairOperation",
+    "RepairOperationRecord",
+    "RepairPreview",
+    "RepairProposal",
+    "RepairProposalStatus",
+    "RepairRun",
+    "RepairRunStatus",
+    "ReproducibilityInfo",
+    "RiskLevel",
+    "RowBeforeAfter",
+    "Rule",
+    "RuleCandidate",
+    "RulePreflightReport",
+    "RuleType",
+    "SamplingConfig",
+    "SamplingInfo",
+    "ScanConfig",
+    "ScanRun",
+    "SchemaChange",
+    "SemanticEvidence",
+    "SemanticProfile",
+    "Severity",
+    "ValidationResult",
+]
+
+__version__ = "0.1.0"

datasentry_core-0.1.0/src/datasentry_core/connectors/__init__.py

@@ -0,0 +1,49 @@
+"""连接器包：数据源读取抽象（49.1/四十一 Step 2）。"""
+
+from datasentry_core.connectors.base import (
+    ColumnInfo,
+    DataConnector,
+    DataHandle,
+    FingerprintMode,
+    FrameBatch,
+    LoadWarning,
+    SamplingMethod,
+    SchemaInfo,
+)
+from datasentry_core.connectors.csv import CsvConnector
+from datasentry_core.connectors.duckdb import DuckdbConnector
+from datasentry_core.connectors.errors import (
+    ConnectorError,
+    DataSourceNotFoundError,
+    UnsafeSqlError,
+    UnsupportedFormatError,
+)
+from datasentry_core.connectors.jsonl import JsonlConnector
+from datasentry_core.connectors.parquet import ParquetConnector
+from datasentry_core.connectors.registry import ConnectorRegistry, default_registry
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+from datasentry_core.connectors.xlsx import XlsxConnector
+
+__all__ = [
+    "ColumnInfo",
+    "ConnectorError",
+    "ConnectorRegistry",
+    "CsvConnector",
+    "DataConnector",
+    "DataHandle",
+    "DataSourceNotFoundError",
+    "DataSourceSpec",
+    "DataSourceType",
+    "DuckdbConnector",
+    "FingerprintMode",
+    "FrameBatch",
+    "JsonlConnector",
+    "LoadWarning",
+    "ParquetConnector",
+    "SamplingMethod",
+    "SchemaInfo",
+    "UnsafeSqlError",
+    "UnsupportedFormatError",
+    "XlsxConnector",
+    "default_registry",
+]

datasentry_core-0.1.0/src/datasentry_core/connectors/base.py

@@ -0,0 +1,108 @@
+"""连接器协议与共享类型（49.1）。
+
+统一返回 PyArrow 帧（FrameBatch），与 DuckDB 零拷贝互操作（ADR-005）。
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any, Literal, Protocol
+
+import pyarrow as pa
+from pydantic import BaseModel, ConfigDict, Field
+
+from datasentry_core.connectors.spec import DataSourceSpec, DataSourceType
+from datasentry_core.models.enums import Severity
+from datasentry_core.models.fingerprint import DatasetFingerprint
+
+SamplingMethod = Literal[
+    "random", "stratified", "reservoir", "time_based", "rare_oversampling", "none"
+]
+FingerprintMode = Literal["full", "sampled", "metadata_only"]
+
+
+class FrameBatch(BaseModel):
+    """一批数据（49.1）：PyArrow Table + 来源与位置元信息。"""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    source: DataSourceSpec
+    table: pa.Table
+    batch_index: int = 0
+    row_offset: int = 0
+
+    @property
+    def row_count(self) -> int:
+        """本批行数。"""
+        rows = self.table.num_rows
+        assert isinstance(rows, int)
+        return rows
+
+    @property
+    def column_names(self) -> list[str]:
+        """本批列名。"""
+        return list(self.table.column_names)
+
+
+class ColumnInfo(BaseModel):
+    """单列信息（49.1 describe 输出）。"""
+
+    name: str
+    physical_type: str
+    nullable: bool = True
+
+
+class SchemaInfo(BaseModel):
+    """数据源 Schema 描述。"""
+
+    columns: list[ColumnInfo] = Field(default_factory=list)
+
+    @property
+    def column_names(self) -> list[str]:
+        """列名列表。"""
+        return [c.name for c in self.columns]
+
+
+class LoadWarning(BaseModel):
+    """加载期警告（如 CSV 公式注入标记），value_preview 必须截断脱敏。"""
+
+    row: int = Field(ge=0)
+    column: str
+    message: str
+    severity: Severity = Severity.MEDIUM
+    value_preview: str = ""
+
+
+class DataHandle(Protocol):
+    """打开后的数据句柄（49.1）。连接器不持有分析逻辑。"""
+
+    @property
+    def source_type(self) -> DataSourceType: ...
+    @property
+    def source_path(self) -> Path | None: ...
+
+    @property
+    def table_name(self) -> str | None:
+        """DuckDB 等需要表名的数据源返回表名，其他返回 None（Step 40）。"""
+        return None
+
+    def schema(self) -> SchemaInfo: ...
+    def read_batches(self, batch_size: int = 65536) -> Iterator[FrameBatch]: ...
+    def read_sample(self, n: int, method: SamplingMethod = "random") -> FrameBatch: ...
+    def sql_aggregate(self, sql: str, params: dict[str, Any] | None = None) -> FrameBatch: ...
+    def count_rows(self) -> int: ...
+    def fingerprint(self, mode: FingerprintMode = "full") -> DatasetFingerprint: ...
+    def warnings(self) -> list[LoadWarning]: ...
+    def close(self) -> None: ...
+
+
+class DataConnector(Protocol):
+    """数据源连接器（49.1）。"""
+
+    connector_id: str
+    display_name: str
+
+    def supports(self, source: DataSourceSpec) -> bool: ...
+    def open(self, source: DataSourceSpec) -> DataHandle: ...
+    def close(self) -> None: ...