precis-cli 0.1.3__py3-none-any.whl

app/shared/core/project/regex/writer.py

@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 正则节点写入模块
+
+功能概述:
+- 写入 *.regex.yaml 文件
+- 优化字段顺序，提升 YAML 可读性
+
+架构设计:
+- 唯一入口: 封装 Pydantic 模型到 YAML 文件的转换逻辑
+- 字段排序: 按重要程度分组（核心 / 可选 / 内部）
+- 一致性: 与 schema/writer.py 保持一致的排序策略
+
+输入示例:
+    node = RegexNodeFile(
+        version=2,
+        id="email",
+        name="邮箱校验",
+        pattern="^[\\w\\.-]+@[\\w\\.-]+\\.\\w+$",
+        match_mode="full",
+        enabled=True,
+    )
+
+输出示例:
+    save_regex_node(node, "regex/email.regex.yaml")
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from app.shared.core.io.yaml import write_yaml_atomic
+
+from .types import RegexNodeFile
+
+if TYPE_CHECKING:
+    from .types import RegexNodeFile
+
+
+def save_regex_node(regex_node: RegexNodeFile, regex_path: str | Path) -> None:
+    """
+    @methoddesc 保存正则节点到 YAML 文件。
+
+    字段顺序优化（按重要程度分组）：
+    L1（核心）：name, description, pattern/uses_pattern
+    L2（可选）：enabled, match_mode
+    L3（内部）：version, id, _internal
+
+    处理流程：
+    1. 将 RegexNodeFile 对象序列化为字典（model_dump）
+    2. 按重要程度重新组织字段顺序（其余字段追加在末尾，确保不丢失）
+    3. 写入 YAML 文件
+
+    设计说明：
+    历史版本曾用手动白名单挑字段，导致新增字段（如 source_ref、input_from_node、
+    rules 等）被静默丢弃。现改为「全量保留 + 顺序优化」：先 dump 全部字段，
+    再按指定顺序重排，未在顺序表中的字段自动追加到末尾，从根本上避免丢字段。
+
+    :param regex_node: RegexNodeFile 对象，包含正则节点的完整配置
+    :param regex_path: 保存路径，可以是相对路径或绝对路径
+    :raises IOError: 文件写入失败时抛出
+    """
+    # 步骤1：将 Pydantic 模型序列化为字典（包含全部字段）
+    data = regex_node.model_dump()
+
+    # 步骤2：按重要程度重排字段顺序
+    # 核心字段在前，其余字段追加其后，保证不丢失任何字段
+    ordered_keys = [
+        # L1 - 核心字段（用户最常编辑）
+        "name",
+        "description",
+        "pattern",
+        "uses_pattern",
+        "pattern_overrides",
+        # L2 - 可选配置（功能开关和模式）
+        "enabled",
+        "match_mode",
+        "case_sensitive",
+        "flags",
+        # 数据流与上游绑定（重建画布连线必需）
+        "source_ref",
+        "source_column_name",
+        "input_from_node",
+        "input_column",
+        # Extract 模式专用
+        "capture_groups",
+        "output_columns",
+        # 前端扩展结构
+        "parameters",
+        "rules",
+        # L3 - 内部字段（开发者/系统使用）
+        "version",
+        "id",
+        "_internal",
+    ]
+
+    ordered: dict[str, object] = {}
+    # 先按指定顺序写入已知字段
+    for key in ordered_keys:
+        if key in data:
+            ordered[key] = data[key]
+    # 再追加未在顺序表中的字段（未来新增字段自动保留，防止再次丢字段）
+    for key, value in data.items():
+        if key not in ordered:
+            ordered[key] = value
+
+    # 步骤3：调用底层 YAML 写入工具
+    write_yaml_atomic(Path(regex_path), ordered)

app/shared/core/project/schema/reader.py

@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 表结构读取模块
+
+功能概述:
+- 从 *.schema.yaml 文件读取表结构配置
+- 返回 core 层的 TableSchemaFile 配置对象
+- 运行时 TableSchema 对象的构建已迁移至 services.schema_runtime_builder，
+  以避免 core 层反向依赖 domain 层。
+
+架构设计:
+- 单一职责: 仅负责 Schema 文件读取和基础配置解析
+- 依赖注入: 使用 YAML 读取工具进行文件解析
+- 类型安全: 使用 Pydantic 模型验证确保配置有效
+
+输入示例:
+    # users.schema.yaml
+    version: 2
+    id: users
+    name: users
+    source:
+      mode: relative_file
+      path: data/users.xlsx
+    columns:
+      - id: user_id
+        name: user_id
+        type: string
+
+输出示例:
+    schema_file = load_schema("schemas/users.schema.yaml")
+    # schema_file 是 TableSchemaFile 对象，可交由 services 层转换为运行时对象
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import ValidationError
+
+from app.shared.core.io.yaml import read_yaml
+
+from .types import TableSchemaFile
+
+
+def load_schema(schema_path: str | Path) -> TableSchemaFile:
+    """
+    @methoddesc 从 YAML 文件加载表结构配置。
+
+    ============================================================================
+    配置文件示例 (本函数处理的配置文件长这样)
+    ============================================================================
+    本函数处理以下格式的 *.schema.yaml 文件：
+
+    ```yaml
+    # ============================================================
+    # 表结构配置文件 (*.schema.yaml)
+    # ============================================================
+
+    version: 2
+
+    id: users
+
+    name: users
+
+    source:
+      mode: relative_file
+      path: data/users.xlsx
+      sheet: Sheet1
+      header_row: 0
+
+    columns:
+      - id: user_id
+        name: user_id
+        type: string
+        primary_key: true
+        expand: false
+      - id: email
+        name: email
+        type: string
+        primary_key: false
+
+    constraints:
+      - id: email_notnull
+        type: NotNull
+        column: email
+        enabled: true
+
+    script_checks: []
+    ```
+
+    ============================================================================
+    业务场景 (什么情况下会调用这个函数)
+    ============================================================================
+    - 场景1: 加载表结构进行数据校验
+      系统需要读取表结构配置来获取列定义和数据源信息。
+
+    - 场景2: 构建运行时数据模型
+      需要将配置文件转换为运行时对象供校验引擎使用。
+      运行时转换请使用 app.shared.services.schema_runtime_builder。
+
+    ============================================================================
+    数据流 (输入如何变成输出)
+    ============================================================================
+    输入参数:
+      - schema_path: str | Path，schema 文件路径
+        示例值: "schemas/users.schema.yaml"
+
+    处理步骤:
+      路径标准化 -> YAML 解析 -> Pydantic 验证 -> TableSchemaFile 对象
+
+    最终输出: TableSchemaFile 对象
+      示例:
+        schema = load_schema("schemas/users.schema.yaml")
+        print(schema.name)  # 输出: users
+        print(len(schema.columns))  # 输出: 2
+
+    ============================================================================
+    异常处理
+    ============================================================================
+    | 异常类型 | 触发条件 | 处理方式 |
+    |---------|---------|---------|
+    | ValueError | 文件不存在 | 检查文件路径 |
+    | ValueError | YAML 格式错误 | 验证 YAML 文件 |
+    | ValueError | 验证失败 | 检查配置格式 |
+
+    :param schema_path: schema 文件路径
+    :return: TableSchemaFile 对象
+    :raises ValueError: 文件不存在或格式错误
+    """
+    path = Path(schema_path)
+    raw = read_yaml(path)
+
+    try:
+        return TableSchemaFile.model_validate(raw)
+    except ValidationError as e:
+        raise ValueError(f"schema 校验失败: {path}\n{e}") from e
+
+
+def schema_column_name_by_id(schema: TableSchemaFile) -> dict[str, str]:
+    """
+    @methoddesc 构建列 ID 到列名的映射。
+
+    :param schema: TableSchemaFile 对象
+    :return: Dict[str, str]，列 ID -> 列名 映射
+    """
+    return {c.id: c.name for c in schema.columns if c.id is not None}
+
+
+def schema_column_id_by_name(schema: TableSchemaFile) -> dict[str, str]:
+    """
+    @methoddesc 构建列名到列 ID 的映射。
+
+    :param schema: TableSchemaFile 对象
+    :return: Dict[str, str]，列名 -> 列 ID 映射
+    """
+    return {c.name: c.id for c in schema.columns if c.id is not None}

app/shared/core/project/schema/types.py

@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview Schema 类型统一导出模块
+
+功能概述:
+- 聚合导出 Schema 相关核心类型（列定义、表结构、数据源等）
+- 提供 V2 版本别名及 source 标准化工具函数
+"""
+
+from __future__ import annotations
+
+from app.shared.core.project.schema.types_parts.column import ColumnSpec, ExtractedSpec
+from app.shared.core.project.schema.types_parts.constraint import ConstraintItem
+from app.shared.core.project.schema.types_parts.schema_id import normalize_source_key
+from app.shared.core.project.schema.types_parts.source import SourceSpec
+from app.shared.core.project.schema.types_parts.table import TableSchemaFile
+
+SourceSpecV2 = SourceSpec
+ColumnSpecV2 = ColumnSpec
+TableSchemaFileV2 = TableSchemaFile
+ConstraintItemV2 = ConstraintItem
+
+__all__ = [
+    "SourceSpec",
+    "ExtractedSpec",
+    "ColumnSpec",
+    "ConstraintItem",
+    "TableSchemaFile",
+    "SourceSpecV2",
+    "ColumnSpecV2",
+    "TableSchemaFileV2",
+    "ConstraintItemV2",
+    "normalize_source_key",
+]

app/shared/core/project/schema/types_parts/__init__.py

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview Schema 子类型聚合导出模块
+
+功能概述:
+- 集中导出 Schema 各组成部分的类型定义与工具函数
+- 提供列定义、表结构、数据源、约束项及 source 标准化工具
+"""
+
+from app.shared.core.project.schema.types_parts.column import ColumnSpec, ExtractedSpec
+from app.shared.core.project.schema.types_parts.constraint import ConstraintItem
+from app.shared.core.project.schema.types_parts.schema_id import normalize_source_key
+from app.shared.core.project.schema.types_parts.source import SourceSpec
+from app.shared.core.project.schema.types_parts.table import TableSchemaFile
+
+__all__ = [
+    "ColumnSpec",
+    "ExtractedSpec",
+    "ConstraintItem",
+    "SourceSpec",
+    "TableSchemaFile",
+    "normalize_source_key",
+]

app/shared/core/project/schema/types_parts/column.py

@@ -0,0 +1,174 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 列类型定义模块
+
+功能概述:
+- 定义 Schema 中的列结构 (ColumnSpec)
+- 支持提取列类型 (ExtractedSpec) 用于正则提取
+
+架构设计:
+- 列 ID 自动生成: 如果未指定 id，默认使用 name
+- 类型灵活: type 字段可以是字符串或字典
+
+输入示例 (schema.yaml):
+    columns:
+      - id: col_1
+        name: id
+        type: integer
+        primary_key: true
+      - id: col_2
+        name: email
+        type: string
+      - id: col_3
+        name: username
+        type:
+          name: Extracted
+          source_column: email
+          extract_key: username
+          result_type: string
+
+输出示例:
+    ColumnSpec(
+        id="col_1",
+        name="id",
+        type="integer",
+        primary_key=True,
+        expand=False
+    )
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class ExtractedSpec(BaseModel):
+    """@classdesc 提取列规范
+
+    用于定义从源列中提取数据的规则。
+    常用于正则提取创建派生列。
+
+    字段说明:
+        - name: 固定为 "Extracted"
+        - source_column: 源列名
+        - extract_key: 正则表达式中的命名捕获组
+        - result_type: 提取结果的数据类型
+
+    示例:
+        # 从 email 列中提取用户名
+        name: Extracted
+        source_column: email
+        extract_key: username
+        result_type: string
+    """
+
+    name: Literal["Extracted"] = "Extracted"
+    source_column: str = Field(..., description="源列名（从中提取数据）")
+    extract_key: str = Field(..., description="提取键（正则命名捕获组）")
+    result_type: str | None = Field(None, description="提取结果的数据类型")
+
+
+class ColumnSpec(BaseModel):
+    """
+    @classdesc 列规范
+
+    定义表中的一列。
+
+    字段说明:
+        - id: 列的唯一标识符 (可选，默认使用 name)
+        - name: 列名 (用于显示和校验)
+        - type: 数据类型
+            - 简单类型: "integer", "string", "boolean" 等
+            - 复杂类型: ExtractedSpec 字典
+        - primary_key: 是否为主键
+        - expand: 是否展开 (用于数组/对象类型)
+
+    输入示例 (schema.yaml):
+        columns:
+          - id: col_1
+            name: id
+            type: integer
+            primary_key: true
+          - id: col_2
+            name: email
+            type: string
+          - id: col_3
+            name: username
+            type:
+              name: Extracted
+              source_column: email
+              extract_key: username
+              result_type: string
+
+    输出示例:
+        ColumnSpec(
+            id="col_1",
+            name="id",
+            type="integer",
+            primary_key=True,
+            expand=False
+        )
+    """
+
+    id: str | None = Field(None, description="列 ID（稳定标识，缺失时自动使用 name）")
+    name: str = Field(..., description="列名（展示/校验使用）")
+    type: Any = Field(..., description="数据类型配置：支持字符串或字典（与现有 TYPE_REGISTRY 兼容）")
+    primary_key: bool = Field(False, description="是否主键")
+    nullable: bool = Field(True, description="是否允许为空")
+    expand: bool = Field(False, description="是否展开（与现有 schema 模型保持一致）")
+    json_path: str | None = Field(None, description="JSON 特有路径映射")
+    children: list[ColumnSpec] | None = Field(None, description="JSON 嵌套子列（树形结构）")
+
+    @model_validator(mode="before")
+    @classmethod
+    def _generate_id_from_name(cls, data: Any) -> Any:
+        """@methoddesc 当未提供列 ID 时，自动使用列名作为 ID
+
+        处理逻辑：
+        1. 检查输入数据是否为字典类型（Pydantic 验证器接收原始数据）
+        2. 如果是字典，检查是否同时满足以下两个条件：
+           - 包含 "name" 字段（有列名可用）
+           - 不包含 "id" 字段（用户未手动指定 ID）
+        3. 满足条件时，将 name 的值复制给 id，实现 ID 自动补全
+        4. 返回处理后的数据（无论是否修改），供后续验证流程继续处理
+
+        为什么需要这个验证器？
+        - 简化配置：用户只需写 name，不必重复写 id
+        - 向后兼容：旧配置中 id 和 name 相同的场景无需修改
+        - 一致性保证：确保每列都有稳定的 ID 用于内部引用
+
+        示例:
+            输入: {"name": "email", "type": "string"}
+            处理: 自动补充 id -> {"name": "email", "type": "string", "id": "email"}
+            输出: {"name": "email", "type": "string", "id": "email"}
+
+            输入: {"id": "col_1", "name": "email", "type": "string"}
+            处理: 已有 id，不修改 -> {"id": "col_1", "name": "email", "type": "string"}
+            输出: {"id": "col_1", "name": "email", "type": "string"}
+        """
+        # 只有字典类型的原始数据才需要处理
+        # Pydantic 验证器可能接收到 dict、已实例化的模型对象或其他类型
+        if isinstance(data, dict):
+            # 条件判断：有 name 但没有 id 时才自动补全
+            # 使用 "in" 操作符检查键是否存在（即使值为 None 也算存在）
+            if "name" in data and "id" not in data:
+                # 将 name 的值赋给 id，实现自动 ID 生成
+                data["id"] = data["name"]
+        # 返回处理后的数据对象，供 Pydantic 继续后续验证流程
+        return data

app/shared/core/project/schema/types_parts/column_utils.py

@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 列递归遍历工具
+
+提供递归处理 ColumnSpec(含嵌套 children)的工具函数。
+平面 schema(无 children)行为等价于原顶层遍历,向后兼容。
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+from app.shared.core.project.schema.types_parts.column import ColumnSpec
+
+
+def iter_all_columns(columns: list[ColumnSpec] | None) -> Iterator[ColumnSpec]:
+    """递归遍历列(含嵌套 children),深度优先遍历父节点本身。
+
+    对无 children 的平面列,等价于直接遍历顶层(行为不变)。
+
+    :param columns: 顶层列列表,可为 None
+    :yield: 每个列(父 + 所有子孙),深度优先
+    """
+    for col in columns or []:
+        yield col
+        if col.children:
+            yield from iter_all_columns(col.children)
+
+
+def build_column_id_to_name_map(columns: list[ColumnSpec] | None) -> dict[str, str]:
+    """递归构建 column_id -> column_name 映射。
+
+    嵌套子列用「父.子」全限定名(与 json_normalize 展平后的点分列一致),
+    使约束能直接 df['profile.name'] 取值。
+    平面列(无父)映射为自身名。
+
+    :param columns: 顶层列列表
+    :return: {column_id: 全限定列名},跳过 id 为 None 的列
+    """
+    result: dict[str, str] = {}
+    _collect_qualified_names(columns or [], "", result)
+    return result
+
+
+def _collect_qualified_names(columns: list[ColumnSpec], parent_path: str, result: dict[str, str]) -> None:
+    """递归收集 column_id -> 全限定列名。
+
+    :param columns: 当前层级的列列表
+    :param parent_path: 父级全限定路径(空串表示顶层)
+    :param result: 累积结果的字典
+    """
+    prefix = f"{parent_path}." if parent_path else ""
+    for col in columns:
+        qualified_name = f"{prefix}{col.name}"
+        if col.id is not None:
+            result[col.id] = qualified_name
+        if col.children:
+            _collect_qualified_names(col.children, qualified_name, result)