precis-cli 0.1.3__py3-none-any.whl

app/shared/core/reporter/reporters/wecom_app_reporter.py

@@ -0,0 +1,268 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview 企业微信自建应用消息推送报告器
+
+功能概述:
+- 通过企业微信自建应用发送数据验证错误报告
+- 支持 Markdown 格式消息和自动 access_token 刷新
+- 实现 access_token 的缓存和自动刷新机制（提前 5 分钟刷新）
+
+架构设计:
+- 继承 Reporter 抽象基类，实现 configure() 和 report() 方法
+- Token 管理: _get_access_token() 实现缓存 + 自动刷新策略
+- 消息截断: 企业微信消息限制约 2KB，超长内容自动截断
+- 统一请求: _send_request() 封装 HTTP POST 请求和错误处理
+
+输入示例:
+    配置参数:
+    {
+        "corp_id": "wwxxxxxxxxxxxxxxxx",
+        "corp_secret": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+        "agent_id": 1000002,
+        "touser": "@all"
+    }
+
+    错误数据:
+    [{"error_type": "ValidationError", "row_index": 2, "column": "age", "value": "abc"}]
+
+输出示例:
+    企业微信消息 (Markdown 格式):
+    ## <font color="warning">【数据验证警告】发现 1 个错误!</font>
+    **请相关人员尽快处理。**
+    **错误详情:**
+    ```json
+    [{"error_type": "ValidationError", ...}]
+    ```
+    > 报告时间: 2024-01-15 14:30:52
+
+    控制台输出:
+    [WeComAppReporter] ✓ 配置成功。
+    [WeComAppReporter] ✓ 成功获取 Access Token。
+    [WeComAppReporter] ✓ 成功发送消息。
+"""
+
+import json
+import logging
+import time
+import urllib.request
+from datetime import datetime
+from typing import Any
+
+from .base import TEMPLATE_HEADROOM_BYTES, Reporter, sanitize_url_for_log, truncate_to_byte_limit
+
+logger = logging.getLogger(__name__)
+
+
+class WeComAppReporter(Reporter):
+    """
+    @classdesc 企业微信自建应用消息推送报告器
+
+    通过企业微信自建应用发送消息，将数据验证错误报告推送给指定用户。
+
+    Attributes:
+        config: 存储配置参数的字典，包含 corp_id, corp_secret, agent_id, touser
+        is_configured: 配置是否有效的标志
+        access_token: 企业微信 API 访问令牌
+        token_expires_at: 令牌过期时间戳
+    """
+
+    def __init__(self) -> None:
+        """
+        @methoddesc 初始化企业微信报告器
+
+        设置默认属性，包括配置字典、配置状态标志、
+        访问令牌及其过期时间戳。
+        """
+        super().__init__("WeComAppReporter")
+        self.config: dict[str, Any] = {}  # 存储配置参数
+        self.is_configured = False  # 配置是否有效
+        self.access_token = None  # 企业微信 API 访问令牌
+        self.token_expires_at = 0  # 令牌过期时间戳
+
+    def configure(self, **kwargs: Any) -> bool:
+        """
+        @methoddesc 配置企业微信报告器参数
+
+        验证所有必需参数是否提供，并保存配置信息。
+
+        Args:
+            corp_id: 企业 ID（企业在企业微信中的唯一标识）
+            corp_secret: 应用 Secret（用于获取 access_token）
+            agent_id: 应用 AgentID（企业微信应用编号）
+            touser: 接收者用户账号（@all 表示发送给全部成员）
+            **kwargs: 额外参数（保留用于扩展）
+
+        Returns:
+            bool: 配置是否成功
+        """
+        # 参数验证：确保所有必需参数都提供
+        corp_id = kwargs.get("corp_id")
+        corp_secret = kwargs.get("corp_secret")
+        agent_id = kwargs.get("agent_id")
+        touser = kwargs.get("touser")
+        if not all([corp_id, corp_secret, agent_id, touser]):
+            self.is_configured = False
+            return False
+
+        # 显式处理 None，避免 mypy 将 Any | None 传给 int() 时报错
+        if agent_id is None:
+            self.is_configured = False
+            return False
+
+        # 保存配置参数
+        self.config = {
+            "corp_id": str(corp_id),
+            "corp_secret": str(corp_secret),
+            "agent_id": int(agent_id),
+            "touser": str(touser),
+        }
+        self.is_configured = True
+        print(f"[{self.name}] ✓ 配置成功。")
+        return True
+
+    def _get_access_token(self) -> str | None:
+        """
+        @methoddesc 获取或刷新企业微信 API 访问令牌（Access Token）
+
+        访问令牌用于调用企业微信 API，有效期默认为 7200 秒（2 小时）。
+        实现自动刷新机制：在过期前 5 分钟自动刷新，避免令牌失效。
+
+        Token 刷新逻辑：
+        - 检查当前时间是否早于过期时间
+        - 如果是，直接返回缓存的令牌
+        - 否则调用企业微信 API 获取新令牌并更新缓存
+
+        Returns:
+            str: 有效的访问令牌，如果获取失败返回 None
+
+        API 调用：
+        - URL: https://qyapi.weixin.qq.com/cgi-bin/gettoken
+        - 参数: corpid, corpsecret
+        - 响应: access_token, expires_in (有效期秒数)
+        """
+        # 检查令牌是否仍然有效（提前 5 分钟刷新）
+        if time.time() < self.token_expires_at:
+            return self.access_token
+
+        # 调用企业微信 API 获取令牌
+        url = f"https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={self.config['corp_id']}&corpsecret={self.config['corp_secret']}"
+        try:
+            # 发送请求
+            with urllib.request.urlopen(url, timeout=10) as response:
+                data = json.loads(response.read().decode())
+                if data.get("errcode") == 0:
+                    # 成功获取令牌
+                    self.access_token = data["access_token"]
+                    # 设置过期时间：当前时间 + 有效期 - 缓冲时间（5分钟）
+                    self.token_expires_at = time.time() + data["expires_in"] - 300
+                    print(f"[{self.name}] ✓ 成功获取 Access Token。")
+                    return self.access_token
+                else:
+                    print(f"[{self.name}] !! 错误: 获取 Access Token 失败: {data.get('errmsg')}")
+                    return None
+        except Exception as e:
+            # gettoken URL 的 query 含 corpsecret，日志只打脱敏后的 host+path
+            logger.warning(
+                "[%s] !! 错误: 请求 Access Token 失败 (%s): %s",
+                self.name,
+                sanitize_url_for_log(url),
+                e,
+            )
+            return None
+
+    def report(self, errors: list[dict]) -> None:
+        """
+        @methoddesc 发送错误报告的主入口方法
+
+        流程：
+        1. 检查配置是否有效
+        2. 获取有效的访问令牌
+        3. 格式化错误信息为 Markdown
+        4. 调用企业微信发送应用消息 API
+
+        Args:
+            errors: 错误信息列表，每个元素是一个包含错误详情的字典
+        """
+        # 前置检查：确保已正确配置
+        if not self.is_configured:
+            return
+
+        # 步骤1: 获取有效的访问令牌
+        token = self._get_access_token()
+        if not token:
+            return
+
+        # 步骤2: 统计错误数量并格式化详情
+        error_count = len(errors)
+        error_details = json.dumps(errors, indent=2, ensure_ascii=False)
+
+        # 安全截断：企业微信消息内容限制约 2KB
+        # 必须按 UTF-8 字节截断：平台限额按字节计，中文按字符截断后实际字节数
+        # 仍可能超限被拒收（见 truncate_to_byte_limit）
+        error_details = truncate_to_byte_limit(error_details, 2048 - TEMPLATE_HEADROOM_BYTES)
+
+        # 步骤3: 构建 Markdown 格式消息内容
+        # 企业微信支持部分 HTML 标签和字体颜色
+        content = f"""## <font color=\"warning\">【数据验证警告】发现 {error_count} 个错误!</font>
+**请相关人员尽快处理。**
+**错误详情:**
+```json
+{error_details}
+```
+> 报告时间: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
+"""
+
+        # 步骤4: 构建发送应用消息的请求负载
+        payload = {
+            "touser": self.config["touser"],  # 接收者：用户账号
+            "msgtype": "markdown",  # 消息类型：Markdown
+            "agentid": self.config["agent_id"],  # 应用 AgentID
+            "markdown": {
+                "content": content  # 消息正文内容
+            },
+        }
+
+        # 步骤5: 发送消息
+        url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}"
+        self._send_request(url, payload)
+
+    def _send_request(self, url: str, payload: dict[str, Any]) -> None:
+        """
+        @methoddesc 发送 HTTP POST 请求到企业微信 API
+
+        处理所有企业微信 API 的 HTTP 通信，包括错误处理和日志记录。
+
+        Args:
+            url: 请求目标 URL
+            payload: 请求体数据字典
+        """
+        try:
+            # 序列化请求负载为 JSON 字节
+            data = json.dumps(payload).encode("utf-8")
+            headers = {"Content-Type": "application/json"}
+
+            # 构建 HTTP 请求
+            req = urllib.request.Request(url, data=data, headers=headers)
+
+            # 发送请求并获取响应
+            with urllib.request.urlopen(req, timeout=10) as response:
+                result = json.loads(response.read().decode("utf-8"))
+                if result.get("errcode") == 0:
+                    print(f"[{self.name}] ✓ 成功发送消息。")
+                else:
+                    print(f"[{self.name}] !! 错误: 发送消息失败: {result.get('errmsg')}")
+        except Exception as e:
+            logger.warning("[%s] !! 错误: 发送请求失败: %s", self.name, e)

app/shared/core/utils/__init__.py

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 核心工具函数模块入口
+"""

app/shared/core/utils/path_utils.py

@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""路径工具函数：POSIX 风格规范化与跨平台路径等价比较。"""
+
+from __future__ import annotations
+
+import os
+import re
+
+
+def normalize_to_posix(path: str) -> str:
+    """将路径统一为 POSIX 格式（正斜杠、无连续斜杠、无末尾斜杠）。
+
+    不转换大小写（文件系统操作可能需要保留原始大小写）。
+    """
+    if not path:
+        return ""
+    result = path.replace("\\", "/")
+    result = re.sub(r"/+", "/", result)
+    result = result.rstrip("/")
+    return result
+
+
+def paths_equal(a: str, b: str, *, ignore_case: bool = True) -> bool:
+    """标准化后比较两个路径是否等价。
+
+    默认忽略大小写（适用于 Windows 等不区分大小写的文件系统）。
+    """
+    na = normalize_to_posix(a)
+    nb = normalize_to_posix(b)
+    if ignore_case:
+        return na.lower() == nb.lower()
+    return na == nb
+
+
+def make_relative(base: str, target: str) -> str:
+    """计算 target 相对于 base 的相对路径，返回 POSIX 格式。
+
+    如果无法计算相对路径（例如 Windows 下跨驱动器），返回 target 的 POSIX 格式。
+    """
+    try:
+        rel = os.path.relpath(target, base)
+    except ValueError:
+        # os.path.relpath 在 Windows 跨盘符（如 base=C:\，target=D:\data）时抛 ValueError，
+        # 按 docstring 承诺降级：返回 target 的 POSIX 格式而非向上抛异常
+        return normalize_to_posix(target)
+    return normalize_to_posix(rel)

app/shared/core/utils/regex_extract.py

@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview 正则提取工具模块
+
+功能概述:
+- 从数据列值中按命名捕获组提取派生列数据
+- 支持将提取结果应用到 pandas DataFrame
+"""
+
+import re
+
+import pandas as pd
+
+from app.shared.domain.regex_flags import parse_regex_flags
+
+__all__ = ["extract_columns_from_values", "parse_regex_flags"]
+
+
+def extract_columns_from_values(
+    regex_pattern: str, regex_flags: str, case_sensitive: bool, values: list[str], match_mode: str = "extract"
+) -> tuple[dict[str, list[str]], list[str], int, int]:
+    """
+    @methoddesc 从值列表中提取命名捕获组数据。
+
+    【功能说明】
+    这是后端 /utils/regex/validate-extract 接口的核心逻辑封装，
+    可以在数据校验流程中直接调用，无需通过 HTTP 请求。
+
+    【参数说明】
+    :param regex_pattern: 正则表达式模式字符串
+    :param regex_flags: 正则标志字符串（如 "gi"）
+    :param case_sensitive: 是否区分大小写
+    :param values: 目标列的值列表
+    :param match_mode: 匹配模式，"full" 或 "partial"（extract 本质也是 partial）
+
+    【返回值】
+    :return: (extracted_columns, group_names, match_count, error_count)
+             - extracted_columns: 提取的列数据字典
+             - group_names: 命名捕获组名称列表
+             - match_count: 匹配成功数量
+             - error_count: 匹配失败数量
+    """
+    # flags 解析统一走 parse_regex_flags（整词匹配，防 "multiline" 误开 IGNORECASE）
+    flags = parse_regex_flags(regex_flags)
+    if not case_sensitive:
+        flags |= re.IGNORECASE
+
+    compiled = re.compile(regex_pattern, flags)
+    group_names = list(compiled.groupindex.keys())
+    extracted_columns: dict[str, list[str]] = {name: [] for name in group_names}
+
+    match_count = 0
+    error_count = 0
+
+    for value in values:
+        # §1.32: NaN 按空串处理（对齐 extractors.py 的 fillna("") 口径）——
+        # 原实现 str(nan)="nan" 参与 match/search，缺失行被计入匹配统计
+        value_str = "" if (value is None or pd.isna(value)) else str(value)
+        if match_mode == "full":
+            match = compiled.match(value_str)
+        else:
+            match = compiled.search(value_str)
+
+        if match:
+            match_count += 1
+            groups = match.groupdict()
+            for name in group_names:
+                extracted_columns[name].append("" if groups.get(name) is None else str(groups.get(name)))
+        else:
+            error_count += 1
+            for name in group_names:
+                extracted_columns[name].append("")
+
+    return extracted_columns, group_names, match_count, error_count
+
+
+def apply_extracted_columns_to_dataframe(
+    df: pd.DataFrame, extracted_columns: dict[str, list[str]], column_names: list[str]
+) -> pd.DataFrame:
+    """
+    @methoddesc 将提取的列数据应用到 DataFrame。
+
+    【功能说明】
+    将 extracted_columns 中的数据作为新列添加到 DataFrame。
+
+    【参数说明】
+    :param df: 原始 DataFrame
+    :param extracted_columns: 提取的列数据字典
+    :param column_names: 新列的名称列表（与 extracted_columns 的 key 对应）
+
+    【返回值】
+    :return: 添加了新列的 DataFrame
+    """
+    result_df = df.copy()
+
+    for col_name in column_names:
+        if col_name in extracted_columns:
+            result_df[col_name] = extracted_columns[col_name]
+
+    return result_df

app/shared/domain/__init__.py

@@ -0,0 +1,114 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 领域模型包入口
+
+功能概述:
+- 统一导出数据校验相关的领域业务模型
+- 提供数据集结构定义、数据类型系统、验证约束系统、表达式系统、数据处理引擎的集中入口
+- 降低外部调用方的导入成本
+
+架构设计:
+- 聚合导出模式: 通过 __init__.py 将各子模块的核心符号集中暴露
+- 模块间关系: 各子模块独立实现，通过本入口统一对外
+
+输入示例:
+    from app.shared.domain import DataSetSchema, StringType, NotNullConstraint
+
+输出示例:
+    # 可直接使用 ColumnSchema、process_dataframe、ExpressionRegistry 等核心类型和函数
+"""
+
+# 1. 项目内部导入
+from app.shared.domain.data_engine import (
+    process_dataframe,
+)
+from app.shared.domain.data_types import (
+    CompositeConditionType,
+    DataType,
+    DecimalType,
+    ExpressionType,
+    FloatType,
+    IntegerType,
+    SequenceType,
+    SpecificCompositeConditionType,
+    SpecificExpressionType,
+    StringType,
+)
+from app.shared.domain.dataset_schema import (
+    TYPE_REGISTRY,
+    ColumnSchema,
+    DataSetSchema,
+    TableSchema,
+    build_type_from_config,
+)
+from app.shared.domain.expression_system import (
+    TYPE_CASTERS,
+    ExpressionPattern,
+    ExpressionRegistry,
+    create_templated_parser,
+)
+from app.shared.domain.validation_constraints import (
+    CONDITION_REGISTRY,
+    AllowedValuesConstraint,
+    CharsetConstraint,
+    CompositeConstraint,
+    ConditionalConstraint,
+    Constraint,
+    DateLogicConstraint,
+    ForeignKeyConstraints,
+    NotNullConstraint,
+    RangeConstraint,
+    ScriptedConstraint,
+    UniqueConstraint,
+    register_condition,
+)
+
+__all__ = [
+    "DataSetSchema",
+    "TableSchema",
+    "ColumnSchema",
+    "build_type_from_config",
+    "TYPE_REGISTRY",
+    "DataType",
+    "IntegerType",
+    "StringType",
+    "FloatType",
+    "DecimalType",
+    "SequenceType",
+    "ExpressionType",
+    "SpecificExpressionType",
+    "CompositeConditionType",
+    "SpecificCompositeConditionType",
+    "CharsetConstraint",
+    "CompositeConstraint",
+    "Constraint",
+    "NotNullConstraint",
+    "UniqueConstraint",
+    "ForeignKeyConstraints",
+    "AllowedValuesConstraint",
+    "ConditionalConstraint",
+    "RangeConstraint",
+    "ScriptedConstraint",
+    "DateLogicConstraint",
+    "CONDITION_REGISTRY",
+    "register_condition",
+    "ExpressionRegistry",
+    "ExpressionPattern",
+    "create_templated_parser",
+    "TYPE_CASTERS",
+    "process_dataframe",
+]

app/shared/domain/constraints/__init__.py

@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 约束系统包入口
+
+功能概述:
+- 作为约束实现拆分后的包入口，统一导出所有约束类
+- 保持向后兼容，旧模块仍可通过 re-export 访问同名符号
+- 集中管理 Unique、NotNull、Range、ForeignKey、Conditional 等约束实现
+
+架构设计:
+- 各约束子模块独立实现，继承自 Constraint 基类
+- 通过 __init__.py 聚合导出，降低调用方导入成本
+
+输入示例:
+    from app.shared.domain.constraints import NotNullConstraint, UniqueConstraint, CONDITION_REGISTRY
+
+输出示例:
+    # 可直接使用所有约束类及 register_condition 注册装饰器
+"""
+
+# ============================================================================
+# 1. 项目内部导入: 从各子模块导入具体的约束实现类
+# ============================================================================
+# 每个约束类型独立一个模块，继承自 Constraint 基类
+# 通过包入口统一聚合导出，降低调用方的导入成本
+
+from app.shared.domain.constraints.allowed_values import AllowedValuesConstraint
+from app.shared.domain.constraints.base import Constraint
+from app.shared.domain.constraints.charset import CharsetConstraint
+from app.shared.domain.constraints.composite import CompositeConstraint
+from app.shared.domain.constraints.condition_registry import CONDITION_REGISTRY, register_condition
+from app.shared.domain.constraints.conditional import ConditionalConstraint
+from app.shared.domain.constraints.date_logic import DateLogicConstraint
+from app.shared.domain.constraints.foreign_key import ForeignKeyConstraints
+from app.shared.domain.constraints.not_null import NotNullConstraint
+from app.shared.domain.constraints.range import RangeConstraint
+from app.shared.domain.constraints.regex import RegexConstraint
+from app.shared.domain.constraints.scripted import ScriptedConstraint
+from app.shared.domain.constraints.unique import UniqueConstraint
+
+# ============================================================================
+# __all__ 控制 from module import * 时导出的符号列表
+# ============================================================================
+# 显式声明对外暴露的公共 API，避免导入内部实现细节
+# 新增约束类型时需同步更新此列表，否则无法通过通配符导入
+
+__all__ = [
+    "AllowedValuesConstraint",
+    "CharsetConstraint",
+    "CompositeConstraint",
+    "CONDITION_REGISTRY",
+    "ConditionalConstraint",
+    "Constraint",
+    "DateLogicConstraint",
+    "ForeignKeyConstraints",
+    "NotNullConstraint",
+    "RangeConstraint",
+    "RegexConstraint",
+    "ScriptedConstraint",
+    "UniqueConstraint",
+    "register_condition",
+]