precis-cli 0.1.3__py3-none-any.whl

app/shared/services/validation/validators/adapter.py

@@ -0,0 +1,182 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 约束适配器模块
+
+功能概述:
+- 提供 ConstraintAdapter，将任意 Domain Constraint 包装为 Service Validator
+- 统一委托模式，消除简单 Validator 文件中的重复模板代码
+- 支持预检、参数映射、自定义 datasets 构建、错误格式化
+
+架构设计:
+- 继承 BaseValidator，实现 validate() 接口
+- 内部创建 Domain Constraint 实例并通过 _delegate_validation 委托
+- kwargs 透传机制：从 Validator kwargs 映射到 Constraint 构造参数
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from typing import Any
+
+import pandas as pd
+
+from app.shared.domain.constraints.base import Constraint
+
+from ..types import ValidationResult
+from .base import BaseValidator
+
+PreCheckFn = Callable[[pd.DataFrame, str, dict], str | None]
+KwargsBuilderFn = Callable[[str, dict], dict[str, Any]]
+DatasetsBuilderFn = Callable[[pd.DataFrame, str, dict], dict[str, pd.DataFrame]]
+
+
+class ConstraintAdapter(BaseValidator):
+    """
+    @classdesc 通用约束适配器
+
+    将任意 Domain Constraint 包装为 Service Validator，
+    自动处理 datasets 映射、预检、参数构建和错误格式转换。
+
+    使用示例:
+        # 简单约束（无额外参数）
+        adapter = ConstraintAdapter(
+            constraint_cls=UniqueConstraint,
+            column_param="column",
+        )
+
+        # 带参数映射和预检
+        adapter = ConstraintAdapter(
+            constraint_cls=RegexConstraint,
+            column_param="column",
+            kwargs_mapping={"regex_pattern": "pattern", ...},
+            pre_checks=[PreCheck.column_exists(), PreCheck.param_required("regex_pattern")],
+        )
+
+        # 带自定义 datasets 构建（如外键双表）
+        adapter = ConstraintAdapter(
+            constraint_cls=ForeignKeyConstraints,
+            column_param="from_column",
+            datasets_builder=PreCheck.build_fk_datasets,
+            ...
+        )
+    """
+
+    def __init__(
+        self,
+        constraint_cls: type[Constraint],
+        column_param: str = "column",
+        extra_params: dict[str, Any] | None = None,
+        kwargs_mapping: dict[str, str] | None = None,
+        error_formatter: Callable[[dict], dict] | None = None,
+        pre_checks: list[PreCheckFn] | None = None,
+        kwargs_builder: KwargsBuilderFn | None = None,
+        datasets_builder: DatasetsBuilderFn | None = None,
+        constraint_kwargs_keys: list[str] | None = None,
+    ):
+        self.constraint_cls = constraint_cls
+        self.column_param = column_param
+        self.extra_params = extra_params or {}
+        self.kwargs_mapping = kwargs_mapping or {}
+        self.error_formatter = error_formatter
+        self.pre_checks = pre_checks or []
+        self.kwargs_builder = kwargs_builder
+        self.datasets_builder = datasets_builder
+        self.constraint_kwargs_keys = constraint_kwargs_keys or []
+
+    def validate(self, df: pd.DataFrame, column: str, **kwargs: Any) -> ValidationResult:
+        start_time = time.time()
+
+        for check in self.pre_checks:
+            error_msg = check(df, column, kwargs)
+            if error_msg:
+                return ValidationResult(
+                    is_valid=False,
+                    error_count=1,
+                    total_rows=len(df),
+                    error_rows=[{"row_index": 0, "cell_value": None, "error_message": error_msg}],
+                    validation_time=f"{time.time() - start_time:.3f}s",
+                )
+
+        params: dict[str, Any]
+        if self.kwargs_builder:
+            params = self.kwargs_builder(column, kwargs)
+        else:
+            params = {"table": "temp", self.column_param: column}
+            params.update(self.extra_params)
+            for validator_kwarg, constraint_param in self.kwargs_mapping.items():
+                if validator_kwarg in kwargs:
+                    params[constraint_param] = kwargs[validator_kwarg]
+
+        constraint = self.constraint_cls(**params)
+
+        constraint_kwargs = {}
+        for key in self.constraint_kwargs_keys:
+            if key in kwargs:
+                constraint_kwargs[key] = kwargs[key]
+
+        if self.datasets_builder:
+            datasets = self.datasets_builder(df, column, kwargs)
+            return self._delegate_validation(
+                df,
+                column,
+                constraint,
+                error_formatter=self.error_formatter,
+                datasets=datasets,
+                constraint_kwargs=constraint_kwargs or None,
+            )
+
+        return self._delegate_validation(
+            df,
+            column,
+            constraint,
+            error_formatter=self.error_formatter,
+            constraint_kwargs=constraint_kwargs or None,
+        )
+
+
+class PreCheck:
+    """内置预检工厂方法"""
+
+    @staticmethod
+    def column_exists() -> PreCheckFn:
+        def check(df: pd.DataFrame, column: str, kwargs: dict) -> str | None:
+            if column not in df.columns:
+                return f"列 '{column}' 不存在"
+            return None
+
+        return check
+
+    @staticmethod
+    def param_required(*keys: str) -> PreCheckFn:
+        def check(df: pd.DataFrame, column: str, kwargs: dict) -> str | None:
+            for key in keys:
+                if not kwargs.get(key):
+                    return f"参数 '{key}' 不能为空"
+            return None
+
+        return check
+
+    @staticmethod
+    def param_required_any(*key_groups: tuple[str, ...]) -> PreCheckFn:
+        def check(df: pd.DataFrame, column: str, kwargs: dict) -> str | None:
+            for group in key_groups:
+                if any(kwargs.get(k) for k in group):
+                    return None
+            return "校验配置不完整"
+
+        return check

app/shared/services/validation/validators/base.py

@@ -0,0 +1,332 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 基础校验器抽象类 - Base Validator
+
+功能概述:
+- 定义所有具体校验器的通用接口和基础功能
+- 提供校验结果的标准格式化和错误处理
+- 采用抽象基类模式（Abstract Base Class），定义校验器的标准行为
+
+架构设计:
+- 抽象基类模式: 所有具体校验器继承 BaseValidator
+- 模板方法模式: 定义 validate 方法的标准校验流程
+- 静态方法封装: 错误格式化逻辑复用
+
+继承体系:
+    BaseValidator (抽象基类)
+        ├── RegexValidator
+        ├── UniqueValidator
+        ├── NotNullValidator
+        ├── AllowedValuesValidator
+        ├── RangeValidator
+        ├── ForeignKeyValidator
+        ├── ConditionalValidator
+        ├── ScriptedValidator
+        ├── CharsetValidator
+        └── DateLogicValidator
+
+数据流:
+    UnifiedValidationService.validate()
+        ↓ [调用]
+    具体校验器.validate(df, column, **kwargs)
+        ↓ [执行]
+    返回错误列表 errors
+        ↓ [格式化]
+    BaseValidator._format_errors(errors, total_rows, time)
+        ↓ [返回]
+    ValidationResult 对象
+
+输入示例:
+    class CustomValidator(BaseValidator):
+        def validate(self, df, column, **kwargs):
+            errors = []
+            for idx, value in df[column].items():
+                if not self._custom_check(value):
+                    errors.append({"row_index": idx, "value": value, "message": "..."})
+            return self._format_errors(errors, len(df), time.time() - start_time)
+
+输出示例:
+    ValidationResult(
+        is_valid=False,
+        error_count=1,
+        error_rows=[{"row_index": 0, "value": "xxx", "message": "..."}]
+    )
+"""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+import pandas as pd
+
+from ..types import ValidationResult
+
+if TYPE_CHECKING:
+    from app.shared.domain.constraints.base import Constraint
+
+logger = logging.getLogger(__name__)
+
+
+class BaseValidator(ABC):
+    """
+    @classdesc 基础校验器抽象基类
+
+    所有具体校验器（如 RangeValidator、RegexValidator 等）必须继承此类，
+    并实现 validate 方法定义具体的校验逻辑。
+
+    设计原则：
+    - 统一接口：所有校验器提供相同的 validate 方法签名
+    - 结果标准化：校验结果统一格式化为 ValidationResult
+    - 错误格式化：提供通用的错误格式化逻辑
+    - 异常处理：统一的异常捕获和日志记录
+
+    示例：
+        class RangeValidator(BaseValidator):
+            def validate(self, df, column, **kwargs):
+                # 具体校验逻辑
+                return ValidationResult(...)
+    """
+
+    def validate_with_error_handling(self, df: pd.DataFrame, column: str, **kwargs: Any) -> ValidationResult:
+        """
+        带异常处理的校验方法包装器
+
+        捕获并记录校验过程中的异常，返回标准化的错误结果。
+        子类应重写 validate 方法，而不是此方法。
+
+        Args:
+            df: 待校验的 DataFrame
+            column: 待校验的列名
+            **kwargs: 校验参数
+
+        Returns:
+            ValidationResult: 校验结果（成功或失败）
+        """
+        try:
+            return self.validate(df, column, **kwargs)
+        except Exception as e:
+            logger.error(f"Validation error in {self.__class__.__name__}: {e}", exc_info=True)
+            return ValidationResult(
+                is_valid=False,
+                error_count=1,
+                total_rows=len(df),
+                error_rows=[
+                    {
+                        "row_index": 0,
+                        "cell_value": None,
+                        "error_message": f"\u6821\u9a8c\u6267\u884c\u5931\u8d25: {str(e)}",
+                    }
+                ],
+                validation_time="0.000s",
+            )
+
+    @abstractmethod
+    def validate(self, df: pd.DataFrame, column: str, **kwargs: Any) -> ValidationResult:
+        """
+        @methoddesc 执行数据校验
+
+        校验 DataFrame 中指定列的数据,返回标准化的校验结果。
+        子类必须实现此方法定义具体的校验逻辑。
+
+        @param df: 待校验的 DataFrame 对象
+        @param column: 待校验的列名
+        @param kwargs: 子类特定的校验参数(如 min、max、pattern 等)
+
+        @return ValidationResult: 标准化校验结果,包含:
+            - is_valid: 校验是否通过
+            - error_count: 错误行数
+            - total_rows: 总行数
+            - match_count: 匹配行数
+            - error_rows: 错误详情列表
+            - validation_time: 校验耗时
+
+        @sideeffect:
+            - 可能读取 DataFrame 数据
+            - 可能抛出 ValueError(列不存在等)
+
+        @raises ValueError: 列名不存在或数据类型错误
+        """
+        pass
+
+    def _delegate_validation(
+        self,
+        df: pd.DataFrame,
+        column: str,
+        constraint: Constraint,
+        error_formatter: Callable[..., Any] | None = None,
+        datasets: dict[str, pd.DataFrame] | None = None,
+        constraint_kwargs: dict[str, Any] | None = None,
+    ) -> ValidationResult:
+        """
+        委托校验通用模板
+
+        将单列 DataFrame 校验委托给 Domain Constraint 的通用流程。
+        适用于大多数通过 table="temp" 技巧委托给 Constraint 的 Validator。
+
+        Args:
+            df: 待校验的 DataFrame
+            column: 待校验的列名
+            constraint: Domain Constraint 实例
+            error_formatter: 可选的错误格式化回调函数(err -> dict)
+            datasets: 可选的自定义数据集字典，默认 {"temp": df}
+            constraint_kwargs: 可选的额外参数传递给 constraint.validate()
+
+        Returns:
+            ValidationResult: 标准化校验结果
+        """
+        import time
+
+        start_time = time.time()
+        if datasets is None:
+            datasets = {"temp": df}
+        if constraint_kwargs is None:
+            constraint_kwargs = {}
+        result = constraint.validate(datasets, **constraint_kwargs)
+        errors = result.get("errors", [])
+
+        formatted_errors = []
+        for err in errors:
+            if error_formatter:
+                formatted_errors.append(error_formatter(err))
+            else:
+                formatted_err = {
+                    "row_index": err.get("row_index"),
+                    "cell_value": err.get("value"),
+                    "error_message": err.get("message"),
+                }
+                # 透传 table/column/error_type 上下文
+                for extra_key in ("table", "column", "error_type"):
+                    if extra_key in err:
+                        formatted_err[extra_key] = err[extra_key]
+                formatted_errors.append(formatted_err)
+
+        return self._format_errors(formatted_errors, len(df), time.time() - start_time)
+
+    @staticmethod
+    def _format_errors(errors: list[dict], total_rows: int, validation_time: float) -> ValidationResult:
+        """
+        @methoddesc 格式化校验错误为标准结果
+
+        将校验过程中收集的错误信息转换为标准化的 ValidationResult 对象。
+        处理数据类型转换(浮点数、None 值等)并生成可读的错误详情。
+
+        @param errors: 原始错误列表,每个元素包含:
+            - value 或 cell_value: 单元格原始值
+            - message 或 error_message: 错误描述
+            - row_index: 行索引(可选)
+        @param total_rows: 校验的总行数
+        @param validation_time: 校验耗时(秒)
+
+        @return ValidationResult: 标准化校验结果
+
+        @sideeffect:
+            - 行索引类型转换(支持字符串和数值)
+            - 单元格值保留原始值(不做取整/round 改写，展示格式化由前端负责)
+
+        处理逻辑:
+            Step 1: 计算错误数量和匹配数量
+            Step 2: 遍历错误列表,格式化每个错误:
+                - 单元格值: 原样保留(数据质量工具需展示文件原始值以便对账)
+                - 行索引处理: 转换为 int,无效值默认为 0
+                - 错误消息: 兼容多种字段名
+            Step 3: 构建 ValidationResult 对象并返回
+        """
+        # Step 1: 计算校验统计
+        # 计算错误行数
+        error_count = len(errors)
+        # 判断校验是否通过(无错误即为通过)
+        is_valid = error_count == 0
+        # 计算匹配行数(总行数 - 错误行数)
+        match_count = total_rows - error_count
+
+        # Step 2: 格式化错误列表
+        formatted_errors = []
+        for err in errors:
+            # 获取单元格值,兼容多种字段名
+            # 【关键数据流】err 可能是 {"value": ...} 或 {"cell_value": ...}
+            # 原样保留(含浮点数的原始精度)：错误详情用于和源文件对账，
+            # 取整/round 改写会掩盖真实数据问题；可读格式化由前端展示层负责。
+            cell_value = err.get("value") if "value" in err else err.get("cell_value")
+
+            # 处理行索引:支持字符串和数值类型
+            # 【副作用】统一行索引为整数类型
+            row_index = err.get("row_index")
+            # None 值默认为 0
+            if row_index is None:
+                row_index = 0
+            try:
+                # 尝试转换为整数
+                row_index = int(row_index)
+            except (TypeError, ValueError):
+                # 转换失败时默认为 0
+                row_index = 0
+
+            # 格式化单个错误记录
+            # 【关键数据流】统一错误格式，便于前端展示
+            formatted_err = {
+                "row_index": row_index,
+                "cell_value": cell_value,
+                "error_message": err.get("message", err.get("error_message")),
+            }
+            # 透传 table/column/error_type 上下文（对跨表约束如 ForeignKey 至关重要）
+            for extra_key in ("table", "column", "error_type"):
+                if extra_key in err:
+                    formatted_err[extra_key] = err[extra_key]
+            formatted_errors.append(formatted_err)
+
+        # Step 3: 构建并返回校验结果
+        # 【副作用】创建 ValidationResult 对象
+        return ValidationResult(
+            is_valid=is_valid,
+            error_count=error_count,
+            total_rows=total_rows,
+            match_count=match_count,
+            error_rows=formatted_errors,
+            validation_time=f"{validation_time:.3f}s",
+        )
+
+
+# ==============================================================================
+# _format_errors 使用示例
+# ==============================================================================
+# # 原始错误列表
+# errors = [
+#     {"row_index": 0, "value": "invalid@email", "message": "邮箱格式错误"},
+#     {"row_index": 3, "cell_value": "bad@format", "error_message": "格式不正确"},
+#     {"row_index": 5, "value": 1.5, "message": "数值超范围"}  # 浮点数会被处理
+# ]
+#
+# # 调用格式化方法
+# result = BaseValidator._format_errors(errors, total_rows=100, validation_time=0.025)
+#
+# # 返回结果
+# # ValidationResult(
+# #     is_valid=False,
+# #     error_count=3,
+# #     total_rows=100,
+# #     match_count=97,
+# #     error_rows=[
+# #         {"row_index": 0, "cell_value": "invalid@email", "error_message": "邮箱格式错误"},
+# #         {"row_index": 3, "cell_value": "bad@format", "error_message": "格式不正确"},
+# #         {"row_index": 5, "cell_value": 1.5, "error_message": "数值超范围"}
+# #     ],
+# #     validation_time="0.025s"
+# # )
+# ==============================================================================