datasentry-core 0.1.0__tar.gz → 0.3.0__tar.gz

{datasentry_core-0.1.0 → datasentry_core-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: datasentry-core
-Version: 0.1.0
+Version: 0.3.0
 Summary: DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs).
 License: Apache-2.0
 Keywords: data-observability,data-quality,duckdb

{datasentry_core-0.1.0 → datasentry_core-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "datasentry-core"
-version = "0.1.0"
+version = "0.3.0"
 description = "DataSentry AI core domain models and engine (detectors, fusion, scoring, repairs)."
 readme = "README.md"
 requires-python = ">=3.12"

{datasentry_core-0.1.0 → datasentry_core-0.3.0}/src/datasentry_core/__init__.py

@@ -108,4 +108,4 @@ __all__ = [
     "ValidationResult",
 ]
 
-__version__ = "0.1.0"
+__version__ = "0.3.0"

{datasentry_core-0.1.0 → datasentry_core-0.3.0}/src/datasentry_core/models/llm.py

@@ -66,4 +66,5 @@ class LLMInvocation(BaseModel):
     masked_sample_count: int = Field(default=0, ge=0)
     injection_flagged: bool = False
     error_message: str | None = None
+    pii_session_id: str | None = None
     created_at: datetime = Field(default_factory=utcnow)

datasentry_core-0.3.0/src/datasentry_core/plugins.py

@@ -0,0 +1,177 @@
+"""检测器插件加载（Step 31 目录版 + Step 50 entry points 版，ADR-031 / ADR-050）。
+
+- `load_plugin_detectors(registry, dirs)`：从目录加载 Python 模块（Step 31，
+  ADR-031，fail-fast 语义保持不破坏）。
+- `discover_entrypoint_detectors(registry)`：`importlib.metadata.entry_points`
+  发现已安装插件包（Step 50，V2-C）：entry point group `datasentry.detectors`。
+  与目录版不同，入口点版**优雅降级**：单个入口加载/实例化/注册失败只记录
+  `PluginError`，不影响其他入口与内置检测器（缺依赖给明确报错不崩）。
+- 入口值形态（`ep.load()` 返回）：Detector 实例 / Detector 子类（无参实例化）/
+  返回 Detector 实例的工厂函数；其余形态记为错误。
+- 加载语义（目录版沿用 Step 31）：只加载 `*.py`，跳过 `_`/`.` 前缀文件；
+  模块级非 Detector 属性忽略；冲突抛 `PluginLoadError`。
+- 安全边界：插件是**用户本机可信代码**（与内置检测器同权），不做沙箱——
+  11.10/ADR-015 的安全表达式求值只约束规则表达式；仅加载已安装包
+  （`importlib.metadata` 只枚举已安装发行版），不执行任意文件路径。
+
+稳定性承诺（ADR-031 扩展，ADR-050）：`Detector` 协议、`DetectionContext`、
+`DetectorRegistry`、`load_plugin_detectors` 签名保持 v1 稳定；新增
+`discover_entrypoint_detectors` / `PluginLoadReport` / `PluginError` 作为
+V2-C 插件生态 v1 接口。
+"""
+
+from __future__ import annotations
+
+import importlib.metadata
+import importlib.util
+import traceback
+from collections.abc import Iterator, Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, cast
+
+from datasentry_core.detectors.base import Detector, DetectorRegistry
+
+#: 检测器插件的 entry point group（V2-C，ADR-050）。
+DETECTOR_ENTRY_POINT_GROUP = "datasentry.detectors"
+
+
+class PluginLoadError(Exception):
+    """插件加载失败：import/实例化/注册冲突，message 含文件与原因。"""
+
+
+@dataclass(frozen=True)
+class PluginError:
+    """单个入口点插件的加载失败记录（优雅降级：不中断其他插件）。"""
+
+    name: str
+    message: str
+
+
+@dataclass
+class PluginLoadReport:
+    """entry points 发现结果：成功注册的 detector_id + 失败项列表。"""
+
+    loaded: list[str] = field(default_factory=list)
+    errors: list[PluginError] = field(default_factory=list)
+
+    def __bool__(self) -> bool:
+        return bool(self.loaded or self.errors)
+
+
+def _module_paths(directory: Path) -> list[Path]:
+    if not directory.is_dir():
+        return []
+    return sorted(
+        p
+        for p in directory.iterdir()
+        if p.is_file() and p.suffix == ".py" and not p.name.startswith(("_", "."))
+    )
+
+
+def _load_module(path: Path, module_name: str) -> Any:
+    try:
+        spec = importlib.util.spec_from_file_location(module_name, path)
+        if spec is None or spec.loader is None:
+            raise PluginLoadError(f"cannot create module spec for {path.name}")
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        return module
+    except PluginLoadError:
+        raise
+    except Exception as exc:
+        raise PluginLoadError(
+            f"failed to import plugin {path.name}: {exc}\n{traceback.format_exc(limit=3)}"
+        ) from exc
+
+
+def _iter_detector_classes(module: Any) -> Iterator[type[Detector]]:
+    for obj in vars(module).values():
+        if isinstance(obj, type) and isinstance(obj, Detector):
+            yield cast(type[Detector], obj)
+
+
+def load_plugin_detectors(
+    registry: DetectorRegistry,
+    plugin_dirs: Sequence[str | Path],
+) -> list[str]:
+    """从目录加载插件检测器并注册，返回新注册的 detector_id 列表。
+
+    加载顺序：目录按给定顺序，文件按名称排序（确定性）。
+    """
+    loaded: list[str] = []
+    for directory in plugin_dirs:
+        path = Path(directory)
+        for file_path in _module_paths(path):
+            module_name = f"datasentry_plugin_{path.name}_{file_path.stem}"
+            module = _load_module(file_path, module_name)
+            for cls in _iter_detector_classes(module):
+                try:
+                    detector = cls()
+                    registry.register(detector)
+                except Exception as exc:
+                    raise PluginLoadError(
+                        f"failed to register detector from {file_path.name}: {exc}"
+                    ) from exc
+                loaded.append(detector.detector_id)
+    return loaded
+
+
+def _entry_points_for(group: str) -> Sequence[Any]:
+    """importlib.metadata.entry_points 按组筛选（3.10+ 均支持 select）。"""
+    return importlib.metadata.entry_points().select(group=group)
+
+
+def _coerce_entry_value(value: Any, name: str) -> Detector:
+    """entry 值 → Detector 实例：类无参实例化 / 实例直通 / 工厂调用。"""
+    if isinstance(value, type):
+        return cast(Detector, value())
+    if isinstance(value, Detector):
+        return value
+    if callable(value):
+        detector = cast(Detector, value())
+        if not isinstance(detector, Detector):
+            raise TypeError(f"factory returned {type(detector).__name__}, expected Detector")
+        return detector
+    raise TypeError(f"entry value is {type(value).__name__}, expected Detector/class/factory")
+
+
+def discover_entrypoint_detectors(
+    registry: DetectorRegistry,
+    group: str = DETECTOR_ENTRY_POINT_GROUP,
+) -> PluginLoadReport:
+    """发现并注册已安装包的检测器（entry points，V2-C / ADR-050）。
+
+    优雅降级：单个入口加载/实例化/注册失败记录到 `errors`（含入口名与原因），
+    其余入口与内置检测器不受影响；不会中断扫描。
+    """
+    report = PluginLoadReport()
+    for entry_point in _entry_points_for(group):
+        name = entry_point.name
+        try:
+            detector = _coerce_entry_value(entry_point.load(), name)
+            try:
+                registry.register(detector)
+            except ValueError as exc:
+                report.errors.append(PluginError(name=name, message=str(exc)))
+                continue
+        except Exception as exc:
+            report.errors.append(
+                PluginError(
+                    name=name,
+                    message=f"{type(exc).__name__}: {exc}\n{traceback.format_exc(limit=3)}",
+                )
+            )
+            continue
+        report.loaded.append(detector.detector_id)
+    return report
+
+
+__all__ = [
+    "DETECTOR_ENTRY_POINT_GROUP",
+    "PluginError",
+    "PluginLoadError",
+    "PluginLoadReport",
+    "discover_entrypoint_detectors",
+    "load_plugin_detectors",
+]

{datasentry_core-0.1.0 → datasentry_core-0.3.0}/src/datasentry_core/reporting/__init__.py

@@ -12,6 +12,7 @@ HTML/Markdown 为同一 dict 的纯函数渲染（无外部依赖，HTML 单文
 
 from __future__ import annotations
 
+import re
 from datetime import UTC, datetime
 from typing import Any
 
@@ -19,6 +20,7 @@ from datasentry_core import __version__
 from datasentry_core.models.issue import Issue
 from datasentry_core.models.quality import QualityScore
 from datasentry_core.models.scan import DetectorRun, ScanRun
+from datasentry_core.privacy.redactor import redact
 
 #: 26 章规范报告结构（build_report 返回，JSON 机器契约）
 type Report = dict[str, Any]
@@ -66,6 +68,20 @@ def _severity_rank(severity: str) -> int:
     return {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4}.get(severity, 9)
 
 
+_PII_HOLDER_RE = re.compile(r"\{\{REDACTED:\w+:\d+\}\}")
+
+
+def mask_text_pii(text: str) -> str:
+    """人类可读输出前的 PII 防御性掩码（Step 48，ADR-048）。
+
+    命中的 PII（email/手机号/身份证/IP/URL）替换为 [REDACTED]；
+    无 PII 文本原样返回。JSON 机器契约不调用（保留完整证据链）。
+    """
+    if not text:
+        return text
+    return _PII_HOLDER_RE.sub("[REDACTED]", redact(text).masked)
+
+
 def critical_findings(report: Report) -> list[dict[str, Any]]:
     """Critical Findings：按严重度与 priority_score 排序取前 N。"""
     return sorted(

{datasentry_core-0.1.0 → datasentry_core-0.3.0}/src/datasentry_core/reporting/html.py

@@ -4,6 +4,9 @@ MVP 节（26 章清单可填充部分）：Executive Summary / Dataset Overview
 Quality Score（含 27.3 维度条形分解与「哪些 Issue 扣分」悬停）/ Issue Breakdown /
 Critical Findings / Column Profiles / Methodology / Reproducibility。
 Drift / Suggested Rules / Repair History 归 V1。
+Step 49（V2-B，ADR-049）：Issue Breakdown 升级为交互表格（severity/维度筛选、
+列排序、详情折叠、分页，原生 JS 零依赖内联）；可选 Quality Trends 迷你 SVG
+（消费 trends.py 序列化数据）；server_base_url 非空时 issue 行联动修复工作台。
 """
 
 from __future__ import annotations
@@ -11,7 +14,8 @@ from __future__ import annotations
 from html import escape
 from typing import Any
 
-from datasentry_core.reporting import HTML_SECTIONS, Report, critical_findings
+from datasentry_core.reporting import HTML_SECTIONS, Report, critical_findings, mask_text_pii
+from datasentry_core.reporting.interactive import render_interactive_issue_table, render_trend_svg
 
 _CSS = """
 :root { color-scheme: light; }
@@ -32,6 +36,24 @@ th { background: #f6f8fa; }
 footer { margin-top: 3rem; font-size: .8rem; color: #57606a; border-top: 1px solid #d0d7de; }
 .notes { font-size: .85rem; color: #57606a; background: #f6f8fa; padding: .6rem;
          border-radius: .4rem; }
+.issue-controls { display: flex; gap: .5rem; align-items: center; flex-wrap: wrap;
+                  margin: .6rem 0; }
+.issue-controls select, .issue-controls input[type=search] { padding: .25rem .45rem;
+                  font-size: .85rem; }
+.issue-row td { cursor: pointer; }
+.issue-detail td { background: #f6f8fa; font-size: .85rem; }
+.issue-detail.collapsed { display: none; }
+th[data-key] { cursor: pointer; user-select: none; white-space: nowrap; }
+th[data-key]::after { content: " \\21C5"; font-size: .7rem; color: #8c959f; }
+th[data-key].sorted-asc::after { content: " \\2191"; }
+th[data-key].sorted-desc::after { content: " \\2193"; }
+.trend-block { margin: 1rem 0; }
+.trend-svg { display: block; background: #fff; border: 1px solid #d0d7de;
+             border-radius: .4rem; padding: .3rem; }
+.issue-pager { display: flex; gap: .6rem; align-items: center; margin: .6rem 0; }
+.issue-pager button { background: #f6f8fa; color: #0969da; border: 1px solid #d0d7de;
+                      border-radius: .3rem; padding: .2rem .7rem; cursor: pointer; }
+.workbench-link { margin-left: .6rem; font-size: .8rem; }
 """
 
 _BAR_COLORS = [
@@ -44,8 +66,18 @@ _BAR_COLORS = [
 ]
 
 
-def render_html(report: Report) -> str:
-    """26 章报告 → 自包含 HTML（内嵌 CSS，无外部资源）。"""
+def render_html(
+    report: Report,
+    *,
+    trends: list[dict[str, Any]] | None = None,
+    page_size: int = 25,
+    server_base_url: str | None = None,
+) -> str:
+    """26 章报告 → 自包含 HTML（内嵌 CSS/JS，无外部资源）。
+
+    trends：trends.py `DatasetTrend.to_report_dict()` 列表 → Quality Trends 迷你 SVG；
+    server_base_url：非空时 issue 行附带修复工作台链接（server 模式联动 REST API）。
+    """
     parts = [
         "<!DOCTYPE html>",
         '<html lang="en"><head><meta charset="utf-8">',
@@ -55,14 +87,20 @@ def render_html(report: Report) -> str:
         _meta(report),
         _executive_summary(report),
         _quality_score(report),
-        _issue_breakdown(report),
-        _critical_findings(report),
-        _dataset_overview(report),
-        _methodology(report),
-        _reproducibility(report),
-        _footer(report),
-        "</body></html>",
     ]
+    if trends:
+        parts.append(_trends_section(trends))
+    parts.extend(
+        [
+            _issue_breakdown(report, page_size=page_size, server_base_url=server_base_url),
+            _critical_findings(report),
+            _dataset_overview(report),
+            _methodology(report),
+            _reproducibility(report),
+            _footer(report),
+            "</body></html>",
+        ]
+    )
     return "\n".join(parts)
 
 
@@ -141,24 +179,22 @@ def _contributions_tooltip(quality: dict[str, Any], dim: str) -> str:
     return f"{dim}: " + "; ".join(f"{i}: {v:.4f}" for i, v in items.items())
 
 
-def _issue_breakdown(report: Report) -> str:
-    rows = [
-        "<tr><th>Severity</th><th>Priority</th><th>Title</th>"
-        "<th>Columns</th><th>Detectors</th><th>Affected</th></tr>"
-    ]
-    for issue in report["issues"]:
-        rows.append(
-            "<tr>"
-            f'<td class="badge-{escape(issue["severity"])}">{escape(issue["severity"])}</td>'
-            f"<td>{issue['priority_score']:.1f}</td>"
-            f"<td>{escape(issue['title'])}</td>"
-            f"<td>{escape(', '.join(issue['columns']))}</td>"
-            f"<td>{escape(', '.join(issue['detector_ids']))}</td>"
-            f"<td>{issue['affected_count']} ({issue['affected_ratio']:.4f})</td>"
-            "</tr>"
-        )
-    body = "".join(rows) if report["issues"] else '<tr><td colspan="6">no issues</td></tr>'
-    return f'<h2 id="issue_breakdown">Issue Breakdown</h2><table>{body}</table>'
+def _trends_section(trends: list[dict[str, Any]]) -> str:
+    svgs = [svg for svg in (render_trend_svg(t) for t in trends) if svg]
+    if not svgs:
+        return ""
+    return '<h2 id="quality_trends">Quality Trends</h2>' + "".join(svgs)
+
+
+def _issue_breakdown(
+    report: Report,
+    *,
+    page_size: int = 25,
+    server_base_url: str | None = None,
+) -> str:
+    return render_interactive_issue_table(
+        report, page_size=page_size, server_base_url=server_base_url
+    )
 
 
 def _critical_findings(report: Report) -> str:
@@ -170,7 +206,8 @@ def _critical_findings(report: Report) -> str:
         items.append(
             "<li>"
             f'<span class="badge-{escape(issue["severity"])}">[{escape(issue["severity"])}]</span> '
-            f"{escape(issue['title'])} &mdash; priority {issue['priority_score']:.1f}, "
+            f"{escape(mask_text_pii(issue['title']))} &mdash; "
+            f"priority {issue['priority_score']:.1f}, "
             f"{issue['affected_count']} rows affected"
             "</li>"
         )