precis-cli 0.1.3__py3-none-any.whl

app/shared/core/project/loader/__init__.py

@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 项目加载器统一入口
+
+功能概述:
+- 聚合导出项目加载的核心类型与入口函数
+- 对外提供统一的 load_project 加载接口
+- 作为 app.shared.core.project.loader 包的门面（Facade），封装底层实现
+
+架构设计:
+- 门面模式: 将 loader.py（入口函数实现）和 types.py（数据模型）等
+  子模块的细节隐藏在本模块之后，调用方无需关心内部文件结构。
+- 类型重导出: LoadedProject、LoadedRegexNode、LoadingError 等核心类型
+  从底层子模块重新导出，确保调用方只需导入本模块即可获得完整类型支持。
+
+导入路径示例:
+    >>> from app.shared.core.project.loader import load_project, LoadedProject
+    >>> project = load_project(Path("project.precis.yaml"))
+    >>> print(project.schemas)
+
+注意事项:
+- 若需新增对外暴露的类型，应在本模块执行重导出，
+  并在 __all__ 中补充名称以维持显式控制。
+- 禁止在本模块编写具体业务逻辑，所有实现应下沉到 loader.py 或 types.py。
+"""
+
+# 从 loader 子模块导入项目加载入口函数
+# load_project 是项目加载流程的顶层编排函数，负责协调解析、校验和组装
+from .loader import load_project
+
+# 从 types 子模块导入核心数据类型
+# 这些类型定义了加载结果的数据结构，供上层服务和 API 路由使用
+from .types import LoadedProject, LoadedRegexNode, LoadingError
+
+# 控制 `from module import *` 时导出的公开名称
+# 显式列出公开 API，防止内部辅助函数或子模块被意外暴露
+__all__ = [
+    "LoadedProject",
+    "LoadedRegexNode",
+    "LoadingError",
+    "load_project",
+]

app/shared/core/project/loader/loader.py

@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 项目加载器代理模块
+
+功能概述:
+- 从 loader_parts.main 代理导出 load_project 入口函数
+- 保持向后兼容的导入路径
+"""
+
+from __future__ import annotations
+
+# 从主加载逻辑模块导入唯一的入口函数
+from app.shared.core.project.loader.loader_parts.main import load_project
+
+# 限定模块的公开接口，仅暴露 load_project
+__all__ = ["load_project"]

app/shared/core/project/loader/loader_parts/config_inspector.py

@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""配置文件格式自检模块 — 编排入口。
+
+本文件已拆分为 4 个职责清晰的子模块，此处保留 inspect_config 编排入口
++ re-export 全部公开符号，保持 3 个调用方（loader/main.py、full_config.py、CLI）
+的 import 路径完全兼容。
+
+拆分后的模块结构（按 DAG 依赖层级）：
+- inspector_helpers.py          — 公共工具（display/actions/columns，Layer 0）
+- inspector_id_checks.py        — ID 一致性检查（Layer 1）
+- inspector_reference_checks.py — 引用完整性检查（Layer 1）
+- inspector_uniqueness_checks.py — Schema 唯一性检查（Layer 1，不依赖 helpers）
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    from app.shared.core.project.constraint.types import ConstraintFile
+    from app.shared.core.project.manifest.types import ProjectManifest
+    from app.shared.core.project.manual_data.types import ManualDataFile
+    from app.shared.core.project.regex.types import RegexNodeFile
+    from app.shared.core.project.schema.types import TableSchemaFile
+    from app.shared.core.project.transform.types import TransformFile
+
+# re-export 全部公开符号，保持调用方零改动
+# 向后兼容别名：测试直接 import 了原文件的 _xxx 私有函数，
+# 拆分后这些函数移到了 inspector_helpers 并去掉了 _ 前缀。
+# 此处提供带 _ 前缀的别名保持测试零改动。
+from app.shared.core.project.loader.loader_parts.inspector_helpers import (  # noqa: F401
+    default_actions_for_file as _default_actions_for_file,
+)
+from app.shared.core.project.loader.loader_parts.inspector_id_checks import (  # noqa: F401
+    inspect_id_consistency,
+)
+from app.shared.core.project.loader.loader_parts.inspector_reference_checks import (  # noqa: F401
+    inspect_reference_integrity,
+    inspect_regex_reference_integrity,
+)
+from app.shared.core.project.loader.loader_parts.inspector_uniqueness_checks import (  # noqa: F401
+    inspect_schema_id_orphan_conflict,
+    inspect_source_uniqueness,
+)
+
+
+def inspect_config(
+    manifest_path: Path,
+    manifest: ProjectManifest,
+    schema_files: dict[str, TableSchemaFile],
+    constraint_files: dict[str, ConstraintFile],
+    regex_node_files: dict[str, RegexNodeFile],
+    transform_files: dict[str, TransformFile],
+    manual_data_files: dict[str, ManualDataFile],
+    warnings: list[str],
+    loading_errors: list,  # list[LoadingError]，用 list 避免循环 import
+) -> None:
+    """配置文件格式自检主入口。
+
+    按 5 个检查维度依次执行，结果就地追加到 warnings / loading_errors：
+    1. ID 跨文件一致性（manifest ID vs 文件内部 ID）
+    2. Schema ID 全局唯一性（基于磁盘扫描，检测重复 ID）
+    3. Schema 数据源唯一性（检测多个 schema 指向同一数据源）
+    4. 约束引用完整性（表/列是否存在）
+    5. 正则引用完整性（表/列是否存在）
+    """
+    logger.info("[配置自检] 开始检查项目配置: %s", manifest_path.parent.name)
+    logger.info(
+        "[配置自检] 检查范围: %d schemas, %d constraints, %d regex, %d transforms, %d manual_data",
+        len(schema_files),
+        len(constraint_files),
+        len(regex_node_files),
+        len(transform_files),
+        len(manual_data_files),
+    )
+
+    errors_before = len(loading_errors)
+    warnings_before = len(warnings)
+
+    inspect_id_consistency(
+        manifest,
+        schema_files,
+        constraint_files,
+        regex_node_files,
+        transform_files,
+        manual_data_files,
+        warnings,
+        loading_errors,
+    )
+
+    # id → 文件相对路径索引（来自 manifest 引用），供引用/唯一性检查
+    # 把问题归属到具体配置文件（前端"按文件"分组依赖 file_path 字段）
+    schema_paths = {ref.id: ref.path for ref in manifest.schemas}
+    constraint_paths = {ref.id: ref.path for ref in manifest.constraints}
+    regex_paths = {ref.id: ref.path for ref in manifest.regex_nodes}
+
+    # Schema ID 全局唯一性检测（基于磁盘扫描）
+    # 同时覆盖 manifest 内文件冲突与孤儿文件冲突，避免重复上报
+    inspect_schema_id_orphan_conflict(manifest_path.parent, manifest, schema_files, loading_errors)
+
+    inspect_source_uniqueness(schema_files, loading_errors, schema_paths)
+
+    inspect_reference_integrity(
+        schema_files, constraint_files, warnings, loading_errors, constraint_paths, schema_paths
+    )
+
+    inspect_regex_reference_integrity(
+        regex_node_files, schema_files, warnings, loading_errors, regex_paths, schema_paths
+    )
+
+    errors_found = len(loading_errors) - errors_before
+    warnings_found = len(warnings) - warnings_before
+
+    if errors_found == 0 and warnings_found == 0:
+        logger.info("[配置自检] 检查通过，未发现问题")
+    else:
+        logger.warning("[配置自检] 发现 %d 个问题", errors_found)
+        for err in loading_errors[errors_before:]:
+            logger.warning("[配置自检] [%s] %s", err.error_type, err.message)

app/shared/core/project/loader/loader_parts/embedded_constraints.py

@@ -0,0 +1,224 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 内嵌约束收集模块
+
+功能概述:
+- 从 Schema 文件中收集内嵌的约束定义
+- 将 Schema 文件中的 inline 约束转换为独立的 ConstraintFile
+
+架构设计:
+- 扫描所有 schema 的 constraints 字段
+- 转换约束引用: column name -> column id (规范化)
+
+输入示例:
+    schema_files = {
+        "users": TableSchemaFile(
+            id="users",
+            name="用户表",
+            columns=[
+                Column(id="col_1", name="id", type="integer"),
+                Column(id="col_2", name="username", type="string"),
+                Column(id="col_3", name="email", type="string")
+            ],
+            constraints=[
+                ConstraintItem(
+                    id="not_null_username",
+                    type="NotNull",
+                    column="username",  # 列名 (需要转换为 id)
+                    enabled=True
+                ),
+                ConstraintItem(
+                    id="fk_orders",
+                    type="ForeignKey",
+                    from_column="user_id",
+                    to_table="users",
+                    to_column="id",
+                    enabled=True
+                )
+            ]
+        )
+    }
+
+输出示例:
+    constraint_files = {
+        "users_not_null_username": ConstraintFile(
+            version=2,
+            id="users_not_null_username",
+            type="NotNull",
+            enabled=True,
+            refs={"table_id": "users", "column_id": "col_2"},
+            params={}
+        ),
+        "users_fk_orders": ConstraintFile(
+            version=2,
+            id="users_fk_orders",
+            type="ForeignKey",
+            enabled=True,
+            refs={
+                "from_table_id": "users",
+                "from_column_id": "user_id",  # 保持原名（未找到对应 column）
+                "to_table_id": "users",
+                "to_column_id": "id"
+            },
+            params={}
+        )
+    }
+"""
+
+from __future__ import annotations
+
+from typing import Any, cast
+
+from app.shared.core.project.constraint.registry import normalize_constraint_type
+from app.shared.core.project.constraint.types import ConstraintFile
+from app.shared.core.project.schema.types import TableSchemaFile
+from app.shared.core.project.schema.types_parts.column_utils import iter_all_columns
+
+
+def collect_constraints_from_schemas(
+    schema_files: dict[str, TableSchemaFile],
+) -> dict[str, ConstraintFile]:
+    """@methoddesc 从 Schema 文件中收集内嵌约束
+
+    遍历所有 Schema，提取其中的 constraints 字段，
+    转换为独立的 ConstraintFile 对象。
+
+    核心转换逻辑:
+    1. 约束 ID: "{table_id}_{constraint_id}" 格式，确保全局唯一
+    2. 列引用: column name -> column id (通过列名查找对应 id)
+    3. 外键特殊处理: from_column/to_column 也需要转换
+
+    输入示例:
+        schema_files = {
+            "users": TableSchemaFile(
+                id="users",
+                constraints=[
+                    ConstraintItem(
+                        id="not_null_email",
+                        type="NotNull",
+                        column="email",  # 列名
+                        enabled=True,
+                        params={}
+                    )
+                ]
+            )
+        }
+
+    输出示例:
+        constraint_files = {
+            "users_not_null_email": ConstraintFile(
+                version=2,
+                id="users_not_null_email",
+                type="NotNull",
+                enabled=True,
+                refs={"table_id": "users", "column_id": "col_3"},  # email 的 column id
+                params={}
+            )
+        }
+
+    原理说明:
+        - normalize_constraint_type() 确保约束类型名称标准化 (如 "not null" -> "NotNull")
+        - 列名到列 ID 的转换: 遍历 columns 列表，找到 name 匹配的列，取其 id
+        - 如果找不到对应的列，则保持原名（向后兼容）
+        - ConstraintFile.version 固定为 2 (当前版本)
+    """
+    # 用于存储转换后的约束文件对象，键为约束全局唯一 ID
+    constraint_files: dict[str, ConstraintFile] = {}
+    # 遍历所有已加载的 Schema 文件
+    for schema in schema_files.values():
+        # 如果当前 Schema 没有定义任何内嵌约束，则直接跳过
+        if not schema.constraints:
+            continue
+
+        # 遍历该 Schema 中定义的每一条内嵌约束
+        for constraint_item in schema.constraints:
+            # 将约束类型名称标准化（如 "not null" 转换为 "NotNull"）
+            constraint_type = normalize_constraint_type(constraint_item.type)
+
+            # 初始化 refs 字典，至少包含当前表 ID
+            refs: dict[str, Any] = {"table_id": schema.id}
+            # 如果约束指定了单列名称，则查找对应的列 ID
+            if constraint_item.column:
+                # 递归遍历列(含嵌套 children)搜索 name 匹配的列,返回其 id
+                column_id = next(
+                    (c.id for c in iter_all_columns(schema.columns) if c.name == constraint_item.column),
+                    constraint_item.column,
+                )
+                refs["column_id"] = column_id
+            # 如果约束指定了多列名称，则逐个查找对应的列 ID
+            elif constraint_item.columns:
+                column_ids = []
+                for col_name in constraint_item.columns:
+                    col_id = next((c.id for c in iter_all_columns(schema.columns) if c.name == col_name), col_name)
+                    column_ids.append(col_id)
+                refs["column_ids"] = column_ids
+
+            # 外键约束需要特殊处理：涉及源列和目标列的映射
+            if constraint_type == "ForeignKey":
+                # 查找源列名称对应的列 ID（若找不到则保留原名称）
+                from_col_id = (
+                    next(
+                        (c.id for c in iter_all_columns(schema.columns) if c.name == constraint_item.from_column),
+                        constraint_item.from_column,
+                    )
+                    if constraint_item.from_column
+                    else None
+                )
+
+                # 对目标列也执行列名 → 列 ID 的转换（在目标 schema 中查找）
+                to_col_id = constraint_item.to_column
+                if constraint_item.to_table and constraint_item.to_column:
+                    to_schema = schema_files.get(constraint_item.to_table)
+                    if to_schema:
+                        to_col_id = next(
+                            (c.id for c in iter_all_columns(to_schema.columns) if c.name == constraint_item.to_column),
+                            constraint_item.to_column,
+                        )
+
+                # 外键的 refs 结构包含源表/列和目标表/列
+                refs = {
+                    "from_table_id": schema.id,
+                    "from_column_id": from_col_id,
+                    "to_table_id": constraint_item.to_table,
+                    "to_column_id": to_col_id,
+                }
+
+            # Conditional 约束：将 params 中的字段提取到 refs（前端 embedded 约束无 refs 字段）
+            # 提取后从 params 中移除，避免下游同时看到 refs 和 params 中的重复字段
+            if constraint_type == "Conditional":
+                params = dict(constraint_item.params or {})
+                refs["if_logic"] = params.pop("if_logic", "and")
+                refs["if_conditions"] = params.pop("if_conditions", []) or []
+                refs["then_column_id"] = params.pop("then_column_id", None)
+                params.pop("table_id", None)
+            else:
+                params = constraint_item.params or {}
+
+            # 构建 ConstraintFile 对象，将内嵌约束转换为独立约束文件格式
+            cf = ConstraintFile.model_construct(
+                version=2,
+                id=f"{schema.id}_{constraint_item.id}",  # 使用 "{table_id}_{constraint_id}" 确保全局唯一
+                type=cast(Any, constraint_type),
+                enabled=constraint_item.enabled,
+                refs=refs,
+                params=params,
+            )
+            # 以全局唯一 ID 为键存入字典
+            constraint_files[cf.id] = cf
+
+    # 返回所有收集到的约束文件对象
+    return constraint_files

app/shared/core/project/loader/loader_parts/file_loaders.py

@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 配置文件加载器适配模块
+
+功能概述:
+- 为项目加载器提供 ManualData 文件的加载接口
+- 其余配置文件（Schema/Constraint/Regex/Transform/Template）由各子模块的 reader 直接加载
+
+架构设计:
+- load_manual_data_file: 加载 ManualData 配置文件（YAML 读取 + 模型校验）
+- 其他类型由调用方直接 import 对应 reader 模块的 load_xxx 函数
+
+异常处理:
+    - FileNotFoundError: 文件不存在
+    - yaml.YAMLError: YAML 格式错误
+    - ValidationError: 数据验证失败
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from app.shared.core.project.manual_data.types import ManualDataFile
+
+
+def load_manual_data_file(manual_data_path: Path) -> ManualDataFile:
+    """@methoddesc 加载 ManualData 配置文件
+
+    输入示例:
+        manual_data_path = Path("/path/to/project/manual_data/ti1__md1.yaml")
+
+    输出示例:
+        ManualDataFile(
+            id="ti1__md1",
+            column_name="age",
+            column_data_type="integer",
+            rows=[["18"], ["25"], ["65"]],
+        )
+    """
+    import yaml
+
+    with open(manual_data_path, encoding="utf-8") as f:
+        raw = yaml.safe_load(f) or {}
+    return ManualDataFile.model_validate(raw)

app/shared/core/project/loader/loader_parts/inspection_ids.py

@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 配置自检问题 ID 集中管理
+
+功能概述:
+- 集中定义所有自检问题（LoadingError.id）的拼接规则
+- 作为前后端约定的单一事实来源，避免 id 格式散落各处导致不一致
+- id 的稳定性直接影响前端"忽略"列表（localStorage）的持久化有效性
+
+设计原则:
+- 纯函数，无副作用
+- 保持现有 id 格式不变（向后兼容已持久化的 ignore 列表）
+- 每个工厂函数对应一种问题类型，命名见文知意
+
+id 格式总览（前后端契约，请勿随意变更）:
+    ID 不一致:        id_mismatch_{resource_type}:{manifest_id}:{file_id}
+    重复登记约束:     id_mismatch_constraint:{manifest_id}:{file_id}
+    引用表缺失:       {error_prefix}:{constraint_id}:{table_id}
+    引用列缺失:       {error_prefix}:{constraint_id}:{table_id}:{column_id}
+    正则表缺失:       regex_table_missing:{regex_id}:{table_id}
+    正则列缺失:       regex_col_missing:{regex_id}:{table_id}:{column_id}
+    Schema ID 重复:   schema_id_duplicate:{schema_id}
+    数据源重复:        schema_source_duplicate:{path}:{sheet}
+"""
+
+from __future__ import annotations
+
+
+def id_mismatch(resource_type: str, manifest_id: str, file_id: str) -> str:
+    """ID 不一致问题 id（schema/constraint/regex/transform 通用）。"""
+    return f"id_mismatch_{resource_type}:{manifest_id}:{file_id}"
+
+
+def constraint_dup_ref(manifest_id: str, file_id: str) -> str:
+    """同一条 constraint 被重复登记问题 id。"""
+    return f"id_mismatch_constraint:{manifest_id}:{file_id}"
+
+
+def ref_table_missing(error_prefix: str, constraint_id: str, table_id: str) -> str:
+    """约束引用的表缺失问题 id。
+
+    Args:
+        error_prefix: 检查前缀（如 fk_src_table_missing / ref_table_missing），
+            同时作为 fix_api.body.field 使用
+    """
+    return f"{error_prefix}:{constraint_id}:{table_id}"
+
+
+def ref_column_missing(error_prefix: str, constraint_id: str, table_id: str, column_id: str) -> str:
+    """约束引用的列缺失问题 id。"""
+    return f"{error_prefix}:{constraint_id}:{table_id}:{column_id}"
+
+
+def regex_table_missing(regex_id: str, table_id: str) -> str:
+    """正则节点引用的表缺失问题 id。"""
+    return f"regex_table_missing:{regex_id}:{table_id}"
+
+
+def regex_column_missing(regex_id: str, table_id: str, column_id: str) -> str:
+    """正则节点引用的列缺失问题 id。"""
+    return f"regex_col_missing:{regex_id}:{table_id}:{column_id}"
+
+
+def schema_id_duplicate(schema_id: str) -> str:
+    """Schema ID 全局重复问题 id。"""
+    return f"schema_id_duplicate:{schema_id}"
+
+
+def schema_source_duplicate(path: str, sheet: str | None) -> str:
+    """数据源重复问题 id。"""
+    return f"schema_source_duplicate:{path}:{sheet}"