precis-cli 0.1.3__py3-none-any.whl

app/shared/services/validation/extractors.py

@@ -0,0 +1,266 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 派生列提取模块
+
+功能概述:
+- 在格式解析后、约束校验前提取派生列（Extracted 类型列）
+- 通过正则表达式的命名捕获组从源列提取数据
+- 支持向量化提取和类型转换（Integer、Float、Boolean、String）
+
+架构设计:
+- 两阶段处理：格式解析 → 提取派生列 → 约束校验
+- 正则驱动：从 schema 中读取正则模式，使用 pandas str.extract 向量化提取
+- 错误上报：提取失败作为验证错误返回，避免静默通过
+
+输入示例:
+    _extract_derived_columns(
+        parsed_datasets={"users": df},
+        schema=DataSetSchema(...),
+        raw_datasets={"users": raw_df},
+        all_errors=[]
+    )
+
+输出示例:
+    parsed_datasets 被就地修改，新增提取列
+    all_errors 可能追加提取失败错误
+"""
+
+import logging
+from typing import cast
+
+import pandas as pd
+
+from app.shared.domain.dataset_schema import DataSetSchema
+
+logger = logging.getLogger(__name__)
+
+
+# Extracted 类型列需要的属性（使用 Protocol 避免直接依赖 domain 子类）
+class _ExtractedTypeProtocol:
+    """仅用于类型检查的 Extracted 数据类型协议。"""
+
+    name: str
+    source_column: str
+    extract_key: str
+    result_type: str | None
+
+
+def _extract_derived_columns(
+    parsed_datasets: dict[str, pd.DataFrame],
+    schema: DataSetSchema,
+    raw_datasets: dict[str, pd.DataFrame],
+    all_errors: list[dict],
+) -> None:
+    """
+    @methoddesc 提取派生列（Extracted 类型列）
+
+    在格式解析后、执行约束校验前，从 Extracted 类型的列中通过正则提取派生列。
+    Extracted 类型标识该列是从另一个列的正则匹配结果中提取的。
+
+    参数:
+        parsed_datasets: 解析后的 DataFrame 字典（会被就地修改）
+        schema: 完整的数据集 Schema
+        raw_datasets: 原始 DataFrame 字典（用于保留原始字符串格式）
+        all_errors: 错误列表（提取失败时会追加错误）
+    """
+    import re
+
+    from app.shared.domain.regex_flags import parse_regex_flags
+
+    for table_id, parsed_df in parsed_datasets.items():
+        logger.debug(f"Processing extracted columns for table: {table_id}")
+        if table_id not in schema.tables:
+            continue
+
+        table_schema = schema.tables[table_id]
+
+        # 查找所有 Extracted 类型的列
+        extracted_columns = []
+        for col in table_schema.columns.values():
+            data_type = col.data_type
+            logger.debug(f"Column {col.name} has data_type: {data_type}")
+            if getattr(data_type, "name", None) == "Extracted":
+                extracted_type = cast(_ExtractedTypeProtocol, data_type)
+                extracted_columns.append(
+                    {
+                        "column_name": col.name,
+                        "source_column": extracted_type.source_column,
+                        "extract_key": extracted_type.extract_key,
+                        "result_type": getattr(extracted_type, "result_type", None),
+                    }
+                )
+
+        if not extracted_columns:
+            continue
+
+        logger.debug(f"表 '{table_id}' 发现 {len(extracted_columns)} 个提取列")
+
+        for ext_info in extracted_columns:
+            col_name = ext_info["column_name"]
+            source_column = ext_info["source_column"]
+            if not isinstance(source_column, str):
+                continue
+            extract_key = ext_info["extract_key"]
+            result_type = ext_info.get("result_type")
+
+            if source_column not in parsed_df.columns:
+                all_errors.append(
+                    {
+                        "stage": "format",
+                        "table": table_id,
+                        "column": col_name,
+                        "check_type": "ExtractedColumnValidation",
+                        "error_type": "ExtractedColumnValidationError",
+                        "message": (
+                            f"提取列 '{col_name}' 失败：源列 '{source_column}' 不存在于表 '{table_id}'。请检查 Schema 配置。"
+                        ),
+                    }
+                )
+                continue
+
+            # 获取源列绑定的正则表达式模式
+            source_col = table_schema.columns.get(source_column)
+            if not source_col:
+                all_errors.append(
+                    {
+                        "stage": "format",
+                        "table": table_id,
+                        "column": col_name,
+                        "check_type": "ExtractedColumnValidation",
+                        "error_type": "ExtractedColumnValidationError",
+                        "message": (
+                            f"提取列 '{col_name}' 失败：源列 '{source_column}' 未在 schema 中定义。请检查 Schema 配置。"
+                        ),
+                    }
+                )
+                continue
+
+            regex_pattern = None
+            regex_flags = "g"
+            case_sensitive = True
+
+            source_data_type = source_col.data_type
+            if hasattr(source_data_type, "name") and source_data_type.name == "Expr":
+                # 显式 Expr：有 pattern 属性
+                if hasattr(source_data_type, "pattern"):
+                    regex_pattern = source_data_type.pattern
+                    if hasattr(source_data_type, "flags"):
+                        regex_flags = source_data_type.flags
+                    if hasattr(source_data_type, "case_sensitive"):
+                        case_sensitive = source_data_type.case_sensitive
+                # 隐式 Expr：无 pattern 属性，但有 registry
+                elif hasattr(source_data_type, "registry") and source_data_type.registry:
+                    registry = source_data_type.registry
+                    # 遍历 registry 中的所有 pattern，查找包含 extract_key 的
+                    if hasattr(registry, "_patterns"):
+                        for pattern_obj in registry._patterns:
+                            if hasattr(pattern_obj, "regex") and pattern_obj.regex:
+                                regex_obj = pattern_obj.regex
+                                group_index = regex_obj.groupindex
+                                if extract_key in group_index:
+                                    regex_pattern = regex_obj.pattern
+                                    break
+
+            if not regex_pattern:
+                all_errors.append(
+                    {
+                        "stage": "format",
+                        "table": table_id,
+                        "column": col_name,
+                        "check_type": "ExtractedColumnValidation",
+                        "error_type": "ExtractedColumnValidationError",
+                        "message": (
+                            f"提取列 '{col_name}' 失败：源列 '{source_column}' 没有绑定正则表达式。请检查 Schema 配置。"
+                        ),
+                    }
+                )
+                continue
+
+            # 编译正则表达式（flags 整词匹配解析，防 "multiline" 子串误开 IGNORECASE）
+            flags = parse_regex_flags(regex_flags)
+            if not case_sensitive:
+                flags |= re.IGNORECASE
+
+            # 执行提取
+            try:
+                compiled = re.compile(regex_pattern, flags)
+                group_names = list(compiled.groupindex.keys())
+
+                # 获取源列的原始数据作为 Series（优先使用 raw_df 保留原始字符串格式）
+                raw_df = raw_datasets.get(table_id)
+                if raw_df is not None and source_column in raw_df.columns:
+                    source_series = raw_df[source_column].fillna("").astype(str)
+                else:
+                    source_series = parsed_df[source_column].fillna("").astype(str)
+
+                # 使用 pandas str.extract 进行向量化提取
+                extracted_df = source_series.str.extract(compiled)
+                match_count = extracted_df.notna().any(axis=1).sum()
+
+                if extract_key in extracted_df.columns:
+                    extracted_series = extracted_df[extract_key].replace("", pd.NA)
+
+                    # 根据 result_type 进行向量化类型转换
+                    if result_type == "Integer":
+                        parsed_df[col_name] = pd.to_numeric(extracted_series, errors="coerce").astype("Int64")
+                    elif result_type == "Float":
+                        parsed_df[col_name] = pd.to_numeric(extracted_series, errors="coerce")
+                    elif result_type == "Boolean":
+                        is_null = extracted_series.isna() | (extracted_series.str.strip() == "")
+                        bool_series = extracted_series.str.lower().isin(["true", "1", "yes"]).astype("boolean")
+                        bool_series[is_null] = pd.NA
+                        parsed_df[col_name] = bool_series
+                    else:
+                        # String 或其他类型保持原样
+                        parsed_df[col_name] = extracted_series
+
+                    logger.debug(
+                        f"提取列 '{col_name}' <- '{source_column}'[{extract_key}], 匹配 {match_count} 行, 类型: {result_type or 'String'}"
+                    )
+                else:
+                    error_msg = f"提取键 '{extract_key}' 不在正则表达式的命名捕获组中，可用组: {group_names}"
+                    logger.warning(error_msg)
+                    # 将配置错误上报为验证错误，避免用户看到"通过"（M4）
+                    all_errors.append(
+                        {
+                            "stage": "format",
+                            "table": table_id,
+                            "column": col_name,
+                            "check_type": "ExtractedColumnValidation",
+                            "message": error_msg,
+                        }
+                    )
+
+            except Exception as e:
+                error_msg = (
+                    f"正则提取列 '{col_name}' 执行失败 (来源列: '{source_column}', 提取键: '{extract_key}'): {e}"
+                )
+                logger.warning(error_msg, exc_info=True)
+                # 提取失败也作为验证错误上报：派生列整列缺失时下游约束会在空数据上
+                # 运行，报告"通过"属于静默跳过校验（与模块契约"避免静默通过"一致）
+                all_errors.append(
+                    {
+                        "stage": "format",
+                        "table": table_id,
+                        "column": col_name,
+                        "check_type": "ExtractedColumnValidation",
+                        "message": error_msg,
+                    }
+                )
+
+        # 更新 parsed_datasets 中的 DataFrame
+        parsed_datasets[table_id] = parsed_df

app/shared/services/validation/history.py

@@ -0,0 +1,209 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview 校验历史存储服务
+
+功能概述:
+- 将每次校验结果持久化到本地 JSON 文件
+- 支持历史列表查询、单次详情查询、删除
+- 支持聚合统计（趋势数据）
+
+存储位置: {projectDir}/.precis/validation_history.json
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+HISTORY_FILENAME = "validation_history.json"
+MAX_HISTORY_ENTRIES = 200
+
+# 回归 C7: 进程级写锁。ValidationHistoryStore 每请求新建实例(load 全量→改→全量覆写),
+# 并发请求会产生 lost-update。用模块级锁串行化同进程内的 add_run/delete_run 写操作;
+# 但锁只保证互斥不保证新鲜——add_run/delete_run 还须在锁内重新 _load 磁盘最新内容
+# 再合并保存,否则先写实例的记录会被后写实例的旧内存快照覆写(跨进程仍依赖文件锁,
+# 但本地单进程场景已足够)。
+_history_write_lock = threading.Lock()
+
+
+@dataclass
+class ValidationRunRecord:
+    """单次校验运行记录"""
+
+    id: str
+    timestamp: str
+    duration_ms: int
+    scope: str
+    summary: dict[str, Any]
+    by_type: dict[str, dict[str, int]]
+    by_table: dict[str, dict[str, int]]
+    errors: list[dict[str, Any]]
+    warnings: list[str] = field(default_factory=list)
+
+
+class ValidationHistoryStore:
+    """校验历史持久化存储"""
+
+    def __init__(self, project_dir: str) -> None:
+        self._file_path = Path(project_dir) / ".precis" / HISTORY_FILENAME
+        self._runs: list[dict[str, Any]] = []
+        self._load()
+
+    def _load(self) -> None:
+        """从磁盘加载历史记录。
+
+        回归 C7: 文件损坏时不应静默清空后用空列表覆写(否则下次 add_run 永久丢失全部
+        历史)。这里把损坏文件重命名为 .corrupted 备份保留现场,内存置空但不破坏磁盘,
+        便于用户手工恢复/排查。
+        """
+        if not self._file_path.exists():
+            self._runs = []
+            return
+        try:
+            with open(self._file_path, encoding="utf-8-sig") as f:
+                data = json.load(f)
+            self._runs = data.get("runs", [])
+        except Exception as e:
+            logger.warning(f"[ValidationHistory] 加载历史文件失败: {e}")
+            self._runs = []
+            # 把损坏文件备份为 .corrupted,避免下次保存静默清空现场(回归 C7)
+            try:
+                backup = self._file_path.with_suffix(self._file_path.suffix + ".corrupted")
+                self._file_path.rename(backup)
+                logger.warning(f"[ValidationHistory] 损坏的历史文件已备份至: {backup}")
+            except OSError:
+                # 重命名失败也不影响:构造阶段不写盘,损坏文件原样保留
+                pass
+
+    def _save(self) -> None:
+        """将历史记录写入磁盘(原子写:tmp + os.replace)。
+
+        回归 C7: 原实现直接以 "w" 打开目标文件,写一半崩溃/断电会留下截断损坏的 JSON,
+        下次 _load 失败 → 清空 → 永久丢失全部历史。原子写保证目标文件要么是完整旧内容、
+        要么是完整新内容,绝不出现部分写入。
+        """
+        self._file_path.parent.mkdir(parents=True, exist_ok=True)
+        # 临时文件与目标同目录,保证 os.replace 是原子的(同文件系统)。
+        tmp_path = self._file_path.with_suffix(self._file_path.suffix + ".tmp")
+        try:
+            with open(tmp_path, "w", encoding="utf-8") as f:
+                json.dump({"runs": self._runs}, f, ensure_ascii=False, indent=2)
+            os.replace(tmp_path, self._file_path)
+        except Exception as e:
+            logger.error(f"[ValidationHistory] 保存历史文件失败: {e}")
+            # 清理可能残留的临时文件,避免堆积
+            try:
+                tmp_path.unlink()
+            except OSError:
+                pass
+
+    def add_run(self, record: ValidationRunRecord) -> str:
+        """添加一次校验记录，返回记录 ID。
+
+        回归 C7: 加进程级写锁,串行化同进程内并发请求的 load→insert→save。
+        回归 lost-update: 锁只保证互斥,不保证新鲜——每请求新建实例时各实例的
+        _runs 都是构造时的快照,先写实例落盘的记录会被后写实例的旧快照覆写。
+        因此在锁内先重新 _load 磁盘最新内容再合并插入,两个实例交错 add 后
+        双方记录都在。
+        """
+        entry = asdict(record)
+        with _history_write_lock:
+            self._load()  # 锁内重读,合并其他实例已落盘的记录
+            self._runs.insert(0, entry)
+            if len(self._runs) > MAX_HISTORY_ENTRIES:
+                self._runs = self._runs[:MAX_HISTORY_ENTRIES]
+            self._save()
+        return record.id
+
+    def get_runs(self, limit: int = 20, offset: int = 0) -> dict[str, Any]:
+        """获取历史列表（分页）"""
+        total = len(self._runs)
+        items = self._runs[offset : offset + limit]
+        return {"total": total, "limit": limit, "offset": offset, "items": items}
+
+    def get_run(self, run_id: str) -> dict[str, Any] | None:
+        """获取单次校验详情"""
+        for run in self._runs:
+            if run.get("id") == run_id:
+                return run
+        return None
+
+    def delete_run(self, run_id: str) -> bool:
+        """删除单次记录(回归 C7: 加写锁避免并发覆盖;锁内重读避免基于旧快照删除)。"""
+        with _history_write_lock:
+            self._load()  # 锁内重读,基于磁盘最新内容删除
+            original_len = len(self._runs)
+            self._runs = [r for r in self._runs if r.get("id") != run_id]
+            if len(self._runs) < original_len:
+                self._save()
+                return True
+            return False
+
+    def get_stats(self, last_n: int = 10) -> dict[str, Any]:
+        """获取聚合统计（最近 N 次的趋势数据）"""
+        recent = self._runs[:last_n]
+        trend = []
+        for run in reversed(recent):
+            summary = run.get("summary", {})
+            trend.append(
+                {
+                    "id": run.get("id"),
+                    "timestamp": run.get("timestamp"),
+                    "pass_rate": summary.get("pass_rate", 0),
+                    "total_checks": summary.get("total_checks", 0),
+                    "failed_count": summary.get("failed_count", 0),
+                }
+            )
+        latest = self._runs[0] if self._runs else None
+        return {
+            "total_runs": len(self._runs),
+            "trend": trend,
+            "latest": {
+                "pass_rate": latest.get("summary", {}).get("pass_rate", 0) if latest else 0,
+                "total_checks": latest.get("summary", {}).get("total_checks", 0) if latest else 0,
+                "passed_count": latest.get("summary", {}).get("passed_count", 0) if latest else 0,
+                "failed_count": latest.get("summary", {}).get("failed_count", 0) if latest else 0,
+                "timestamp": latest.get("timestamp") if latest else None,
+            },
+        }
+
+
+_run_id_lock = threading.Lock()
+_run_id_last_us: int = -1
+_run_id_counter: int = 0
+
+
+def generate_run_id() -> str:
+    """生成运行 ID（时间戳 + 单调计数器保证唯一性）"""
+    global _run_id_last_us, _run_id_counter
+    t = time.strftime("%Y%m%d_%H%M%S")
+    us = int(time.time() * 1000000) % 1000000
+    with _run_id_lock:
+        if us == _run_id_last_us:
+            _run_id_counter += 1
+        else:
+            _run_id_last_us = us
+            _run_id_counter = 0
+        suffix = f"{us:06d}_{_run_id_counter:03d}"
+    return f"run_{t}_{suffix}"

app/shared/services/validation/json_payload.py

@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview 校验结果 → JSON 契约 payload（CLI 与 MCP 共用单一事实源）
+
+契约文档：docs/contracts/validate-json-v1.md（v1 已冻结，只增不减）。
+本模块是 `_build_json_payload` 结构的唯一实现：
+- CLI `--format json` 的 stdout 输出
+- MCP tool `validate_data` 的返回值
+- `--report` 报告导出的数据源
+三处消费同一结构，禁止分叉出第二套输出格式。
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Sized
+from typing import Any
+
+# JSON 输出契约的版本号（未来契约演进时递增）
+JSON_SCHEMA_VERSION = 1
+
+
+def json_safe_value(value: Any) -> Any:
+    """将校验错误中的单元格值转换为 JSON 可序列化的原生类型。
+
+    pandas 校验链路可能透出 numpy 标量（int64/float64 等）与非有限浮点
+    （NaN/Inf），两者均非合法 JSON。numpy 标量经 .item() 归一为 Python
+    原生类型；非有限浮点转为字符串保留诊断信息。
+
+    Args:
+        value: 错误条目中的任意取值
+
+    Returns:
+        JSON 可序列化的值（None/bool/int/有限 float/str）
+    """
+    if value is None or isinstance(value, (str, bool, int)):
+        return value
+    if isinstance(value, float):
+        # np.float64 是 float 子类可直接命中；NaN/Inf 不是合法 JSON，转字符串
+        return value if math.isfinite(value) else str(value)
+    if hasattr(value, "item"):
+        # numpy 标量统一经 .item() 归一后再递归判定
+        try:
+            return json_safe_value(value.item())
+        except (AttributeError, ValueError):
+            pass
+    # 兜底：其余类型（如 dict/list 以外的对象）转字符串，保证不破坏序列化
+    return str(value)
+
+
+def build_json_payload(result: dict) -> dict:
+    """将 ValidationExecutor.execute() 的结果转换为 JSON 输出契约结构。
+
+    契约字段必须齐全，无法取到的值填 null：
+    - tables 来自 validation_details.format_checks（表显示名），
+      行数优先取 raw_datasets 中的 DataFrame 长度，分块模式取 row_count
+    - summary 来自 validation_details.constraint_checks 的 passed 统计
+    - errors 中 table/column/constraint_type(check_type)/constraint_file
+      由校验链路提供（constraint_file 由 executor 挂载，见 P0-3）
+    - loading_errors 原样透传为 loading_warnings
+
+    Args:
+        result: ValidationExecutor.execute() 返回的结果字典
+
+    Returns:
+        符合契约 v1 的字典（可直接 json.dumps）
+    """
+    errors = result.get("errors", [])
+    validation_details = result.get("validation_details") or {}
+    constraint_checks = validation_details.get("constraint_checks") or []
+    format_checks = validation_details.get("format_checks") or []
+    raw_datasets = result.get("raw_datasets") or {}
+
+    tables: list[dict[str, Any]] = []
+    for check in format_checks:
+        # postprocess 已把 table 换成显示名并保留原 ID 到 table_id；
+        # raw_datasets 以表 ID 为键，优先按 ID 查（显示名与 ID 不同时也能命中行数）
+        dataset = raw_datasets.get(check.get("table_id") or check.get("table"))
+        rows: int | None = None
+        if isinstance(dataset, dict):
+            # 分块模式下 raw_datasets 值为 {"chunk_count": ..., "row_count": ...}
+            rows = dataset.get("row_count")
+        elif isinstance(dataset, Sized):
+            # 标准模式下值为 DataFrame，行数即长度
+            rows = len(dataset)
+        tables.append({"name": check.get("table"), "rows": rows})
+
+    constraints_total = len(constraint_checks)
+    constraints_failed = sum(1 for check in constraint_checks if not check.get("passed", True))
+
+    error_entries: list[dict[str, Any]] = []
+    for error in errors:
+        # 格式错误携带 cell_value/error_message；域约束错误携带 value/message，做双字段兼容
+        cell_value = error.get("cell_value") if "cell_value" in error else error.get("value")
+        error_message = error.get("error_message") if error.get("error_message") is not None else error.get("message")
+        error_entries.append(
+            {
+                "table": error.get("table"),
+                "column": error.get("column"),
+                "constraint_type": error.get("check_type"),
+                "constraint_file": error.get("constraint_file"),
+                "row_index": error.get("row_index"),
+                "cell_value": json_safe_value(cell_value),
+                "error_message": error_message,
+                # 可选修复建议（AllowedValues 相近值 / 日期布局提示等），生成器未给则为 null
+                "suggestion": error.get("suggestion"),
+            }
+        )
+
+    return {
+        "schema_version": JSON_SCHEMA_VERSION,
+        "is_valid": not errors,
+        "interrupted": bool(result.get("interrupted", False)),
+        "duration_ms": result.get("duration_ms", 0),
+        "tables": tables,
+        "summary": {
+            "constraints_total": constraints_total,
+            "constraints_passed": constraints_total - constraints_failed,
+            "constraints_failed": constraints_failed,
+        },
+        "errors": error_entries,
+        "loading_warnings": result.get("loading_errors", []),
+    }