precis-cli 0.1.3__py3-none-any.whl

app/shared/services/llm/generation/config_builder.py

@@ -0,0 +1,386 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview 配置构建器
+
+功能概述:
+- 将 LLM 输出、画像数据和现有配置整合为最终的项目配置
+- 生成标准 V2 格式的 manifest、schemas、constraints、regex_nodes
+
+输入示例:
+    result = build_config(
+        project_id="ecommerce",
+        project_name="电商数据校验",
+        config_path="/path/to/project",
+        profiling_data=[...],
+        llm_result={"schemas": [...], "constraints": [...]},
+        options=GenerationOptions(),
+        existing_config=None,
+    )
+
+输出示例:
+    {"manifest": {...}, "schemas": {...}, "constraints": {...}, "yaml_preview": "..."}
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any
+
+import yaml
+
+from app.shared.core.utils.path_utils import make_relative, normalize_to_posix
+
+
+def build_config(
+    project_id: str,
+    project_name: str,
+    config_path: str | None,
+    profiling_data: list[dict],
+    llm_result: dict,
+    options: Any,
+    existing_config: dict[str, Any] | None,
+) -> dict[str, Any]:
+    """
+    @methoddesc 将 LLM 输出、画像数据和现有配置整合为最终的项目配置
+
+    处理流程：
+    1. 如需保留现有配置，先加载已有内容
+    2. 根据选项生成/合并 schema、constraint、regex_node
+    3. 构建标准 V2 清单文件
+    4. 生成 YAML 预览
+
+    参数:
+        project_id: 项目唯一标识
+        project_name: 项目显示名称
+        config_path: 配置根目录（用于计算相对路径）
+        profiling_data: 数据画像结果
+        llm_result: LLM 返回的解析结果
+        options: 生成选项
+        existing_config: 现有配置（keep_existing 时使用）
+
+    返回:
+        包含 manifest、schemas、constraints、regex_nodes 等的字典
+    """
+    # 初始化结果容器
+    schemas: dict[str, Any] = {}
+    constraints: dict[str, Any] = {}
+    regex_nodes: dict[str, Any] = {}
+    warnings: list[str] = []
+
+    # 如果有现有配置且 keep_existing，先加载现有内容
+    if options.keep_existing and existing_config:
+        schemas.update(existing_config.get("schemas", {}))
+        constraints.update(existing_config.get("constraints", {}))
+        regex_nodes.update(existing_config.get("regex_nodes", {}))
+
+    # 构建 profiling_data 的 path -> table_name 映射，用于 source 配置
+    path_to_profile = {item["path"]: item for item in profiling_data}
+
+    def _find_existing_schema_id_by_source(src_path: str, sheet_name: str | None) -> str | None:
+        """根据 source.path + source.sheet 查找已有 schema，复用其 ID。
+
+        避免同一文件因 ID 计算方式不同而被识别为新增 schema。
+        """
+        for sid, sdoc in schemas.items():
+            existing_src = sdoc.get("source", {})
+            existing_path = existing_src.get("path", "")
+            existing_sheet = existing_src.get("sheet")
+            if existing_path == src_path and existing_sheet == sheet_name:
+                return sid
+        return None
+
+    def _make_relative_path(abs_path: str) -> str:
+        """将绝对路径转为相对于项目根目录的路径。
+
+        统一使用正斜杠，确保跨平台兼容性。
+        """
+        if config_path:
+            try:
+                return make_relative(config_path, abs_path)
+            except ValueError:
+                pass
+        return normalize_to_posix(os.path.basename(abs_path))
+
+    def _sanitize_id(name: str) -> str:
+        """将名称转为合法的 id（小写，替换特殊字符为下划线）。"""
+        return re.sub(r"[^a-z0-9_]", "_", name.lower()).strip("_")
+
+    def _generate_semantic_constraint_id(table_id: str, column_ref: str, ctype: str, existing_ids: set) -> str:
+        """生成语义化约束 ID，如遇冲突则自动添加序号后缀。
+
+        格式: {table_id}_{column_ref}_{ctype} 或 {table_id}_{column_ref}_{ctype}_2
+        """
+        base = f"{_sanitize_id(table_id)}_{_sanitize_id(column_ref)}_{_sanitize_id(ctype)}"
+        if base not in existing_ids:
+            return base
+        for i in range(2, 1000):
+            candidate = f"{base}_{i}"
+            if candidate not in existing_ids:
+                return candidate
+        return base
+
+    def _normalize_constraint(cdef: dict, schema_id: str, existing_ids: set) -> dict[str, Any] | None:
+        """将 LLM 输出的约束（简化或 V2 格式）统一转换为 V2 标准格式。
+
+        支持两种输入：
+        - 已有 refs 字段的 V2 格式：做基本校验和补充
+        - 简化格式：自动构建 refs 和 params
+        """
+        if not isinstance(cdef, dict):
+            return None
+
+        # 检测是否已经是 V2 格式（有 refs 字段）
+        if "refs" in cdef and isinstance(cdef.get("refs"), dict):
+            # 已经是 V2 格式，做基本校验和补充
+            cid = cdef.get("id") or _generate_semantic_constraint_id(
+                cdef["refs"].get("table_id", schema_id),
+                # 显式空 column_ids 列表时 .get 的默认值不生效，需 or 兜底，否则 [0] 越界
+                cdef["refs"].get("column_id") or (cdef["refs"].get("column_ids") or ["unknown"])[0],
+                cdef.get("type", "unknown"),
+                existing_ids,
+            )
+            existing_ids.add(cid)
+            return {
+                "version": 2,
+                "id": cid,
+                "type": cdef.get("type", "NotNull"),
+                "enabled": cdef.get("enabled", True),
+                "description": cdef.get("description", ""),
+                "refs": cdef["refs"],
+                "params": cdef.get("params", {}),
+            }
+
+        # 简化格式转换
+        ctype = cdef.get("type", "")
+        if not ctype:
+            return None
+
+        # 统一类型名首字母大写
+        ctype_normalized = ctype[0].upper() + ctype[1:].lower() if ctype else "NotNull"
+
+        table_id = cdef.get("table_id", schema_id)
+        column_id = cdef.get("column_id", "")
+        column_ids = cdef.get("column_ids", [column_id] if column_id else [])
+
+        # 构建 refs
+        refs: dict[str, Any] = {"table_id": table_id}
+        if ctype_normalized == "Unique":
+            refs["column_ids"] = column_ids if column_ids else [column_id]
+        elif ctype_normalized == "ForeignKey":
+            refs["from_table_id"] = cdef.get("from_table_id", table_id)
+            refs["from_column_id"] = cdef.get("from_column_id", column_id)
+            refs["to_table_id"] = cdef.get("to_table_id", "")
+            refs["to_column_id"] = cdef.get("to_column_id", "")
+        elif ctype_normalized == "Conditional":
+            refs["table_id"] = table_id
+            refs["then_column_id"] = cdef.get("then_column_id", column_id)
+            refs["if_conditions"] = cdef.get("if_conditions", [])
+            refs["if_logic"] = cdef.get("if_logic", "and")
+        else:
+            refs["column_id"] = column_id
+
+        # 构建 params
+        params: dict[str, Any] = {}
+        if ctype_normalized == "AllowedValues":
+            params["allowed_values"] = cdef.get("allowed_values", [])
+        elif ctype_normalized == "Range":
+            params["min"] = cdef.get("min")
+            params["max"] = cdef.get("max")
+        elif ctype_normalized == "Conditional":
+            params["then_value"] = cdef.get("then_value")
+        elif ctype_normalized == "Scripted":
+            params["expression"] = cdef.get("expression", "")
+
+        # 表级约束（无 column_id/column_ids）时以 "unknown" 兜底参与 ID 生成，
+        # 避免空列表 [0] 索引 IndexError 崩掉整个配置生成
+        cid = _generate_semantic_constraint_id(
+            table_id, column_id or (column_ids[0] if column_ids else "unknown"), ctype_normalized, existing_ids
+        )
+        existing_ids.add(cid)
+
+        return {
+            "version": 2,
+            "id": cid,
+            "type": ctype_normalized,
+            "enabled": True,
+            "description": cdef.get("description", ""),
+            "refs": refs,
+            "params": params,
+        }
+
+    def _normalize_regex_node(rdef: dict, existing_ids: set) -> dict[str, Any] | None:
+        """将 LLM 输出的正则节点标准化为 V2 格式。
+
+        如果缺少 ID，则根据名称自动生成；如遇 ID 冲突则添加序号后缀。
+        """
+        if not isinstance(rdef, dict):
+            return None
+        rid = rdef.get("id")
+        if not rid:
+            rid = f"regex_{_sanitize_id(rdef.get('name', 'unnamed'))}"
+        if rid in existing_ids:
+            for i in range(2, 1000):
+                candidate = f"{rid}_{i}"
+                if candidate not in existing_ids:
+                    rid = candidate
+                    break
+        existing_ids.add(rid)
+        return {
+            "version": 2,
+            "id": rid,
+            "name": rdef.get("name", rid),
+            "description": rdef.get("description", ""),
+            "pattern": rdef.get("pattern", ""),
+            "match_mode": rdef.get("match_mode", "full"),
+            "case_sensitive": rdef.get("case_sensitive", False),
+            "flags": rdef.get("flags", ""),
+            "enabled": rdef.get("enabled", True),
+            "parameters": [],
+            "rules": [],
+            "source_ref": rdef.get("source_ref", {}),
+        }
+
+    # 处理 Schema —— 使用 LLM 提供的语义化 ID
+    llm_id_to_schema_id: dict[str, str] = {}  # LLM 返回的原始 ID -> 最终 ID 映射
+    used_schema_ids: set[str] = set(schemas.keys())  # 已占用的 ID（含 keep_existing）
+    if options.generate_schemas:
+        for schema_def in llm_result.get("schemas", []):
+            if not isinstance(schema_def, dict):
+                continue
+            llm_schema_id = schema_def.get("id") or schema_def.get("name", "")
+            if not llm_schema_id:
+                continue
+
+            # 补充 source 配置
+            profile = path_to_profile.get(schema_def.get("_source_path", ""))
+            if not profile:
+                for p in profiling_data:
+                    if p["table_name"] == schema_def.get("name", llm_schema_id):
+                        profile = p
+                        break
+
+            source: dict[str, Any] = {"mode": "relative_file", "path": "", "header_row": 0}
+            if profile:
+                source["path"] = _make_relative_path(profile["path"])
+                if profile.get("sheet_name"):
+                    source["sheet"] = profile["sheet_name"]
+                ext = os.path.splitext(profile["path"])[1].lower()
+                if ext == ".csv":
+                    source["options"] = {"delimiter": ",", "encoding": "utf-8"}
+                elif ext in [".xlsx", ".xls"]:
+                    source["options"] = {"engine": "openpyxl"}
+                elif ext in [".json", ".jsonl"]:
+                    source["options"] = {"format": "auto"}
+
+            # 优先复用已有 schema 的 ID（按 source.path + sheet 匹配）
+            src_path = source.get("path", "")
+            sheet_name = source.get("sheet")
+            existing_id = _find_existing_schema_id_by_source(src_path, sheet_name) if src_path else None
+            if existing_id:
+                proper_schema_id = existing_id
+            else:
+                # 使用 LLM 提供的语义化 ID，清洗并保证唯一
+                proper_schema_id = _sanitize_id(llm_schema_id)
+                base = proper_schema_id
+                i = 2
+                while proper_schema_id in used_schema_ids:
+                    proper_schema_id = f"{base}_{i}"
+                    i += 1
+                used_schema_ids.add(proper_schema_id)
+
+            llm_id_to_schema_id[llm_schema_id] = proper_schema_id
+
+            # 构建标准 schema 格式
+            schema_doc = {
+                "version": 2,
+                "id": proper_schema_id,
+                "name": schema_def.get("name", llm_schema_id),
+                "source": schema_def.get("source", source),
+                "columns": schema_def.get("columns", []),
+                "constraints": [],
+            }
+
+            # 处理内联约束（schema 级别的 constraints）
+            for cdef in schema_def.get("constraints", []):
+                if isinstance(cdef, dict):
+                    schema_doc["constraints"].append(cdef)
+
+            schemas[proper_schema_id] = schema_doc
+
+    def _remap_schema_refs(obj: Any) -> Any:
+        """递归替换约束中的 table_id 为规范 Schema ID。"""
+        if isinstance(obj, dict):
+            remapped: dict[str, Any] = {}
+            for k, v in obj.items():
+                if (
+                    k in ("table_id", "from_table_id", "to_table_id")
+                    and isinstance(v, str)
+                    and v in llm_id_to_schema_id
+                ):
+                    remapped[k] = llm_id_to_schema_id[v]
+                else:
+                    remapped[k] = _remap_schema_refs(v)
+            return remapped
+        if isinstance(obj, list):
+            return [_remap_schema_refs(item) for item in obj]
+        return obj
+
+    # 先处理独立 Constraints，再统一做 ID 重映射
+    if options.generate_constraints:
+        constraint_ids = set(constraints.keys())
+        for cdef in llm_result.get("constraints", []):
+            # 将约束中的 table_id 从 LLM ID 替换为规范 ID
+            cdef = _remap_schema_refs(cdef)
+            normalized = _normalize_constraint(cdef, "", constraint_ids)
+            if normalized:
+                constraints[normalized["id"]] = normalized
+            else:
+                warnings.append(f"无法解析约束: {cdef}")
+
+    # 对 schema 内联约束也做 ID 重映射
+    for schema_doc in schemas.values():
+        schema_doc["constraints"] = [
+            _remap_schema_refs(c) for c in schema_doc.get("constraints", []) if isinstance(c, dict)
+        ]
+
+    # 处理 Regex Nodes
+    if options.generate_regex_nodes:
+        regex_ids = set(regex_nodes.keys())
+        for rdef in llm_result.get("regex_nodes", []):
+            normalized = _normalize_regex_node(rdef, regex_ids)
+            if normalized:
+                regex_nodes[normalized["id"]] = normalized
+
+    # 构建 Manifest
+    manifest = {"version": 2, "project": {"id": project_id, "name": project_name}}
+    manifest["schemas"] = [{"id": sid, "path": f"schemas/{sid}.schema.yaml"} for sid in schemas]
+    manifest["constraints"] = [{"id": cid, "path": f"constraints/{cid}.constraint.yaml"} for cid in constraints]
+    manifest["regex_nodes"] = [{"id": rid, "path": f"regex/{rid}.regex.yaml"} for rid in regex_nodes]
+
+    # 生成 YAML 预览
+    preview = {"manifest": manifest, "schemas": schemas, "constraints": constraints, "regex_nodes": regex_nodes}
+    yaml_preview = yaml.safe_dump(preview, sort_keys=False, allow_unicode=True)
+
+    return {
+        "success": True,
+        "yaml_preview": yaml_preview,
+        "manifest": manifest,
+        "schemas": schemas,
+        "constraints": constraints,
+        "regex_nodes": regex_nodes,
+        "warnings": warnings,
+    }

app/shared/services/llm/generation/errors.py

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview 配置生成异常定义
+
+生成链路各子模块（响应解析/数据剖析/Agent 装配）共享的异常类型，
+独立成模块以避免 service ↔ 子模块循环导入；service.py 对外重导出保持既有导入路径不变。
+"""
+
+from __future__ import annotations
+
+
+class GenerationParseError(Exception):
+    """LLM 响应解析失败"""
+
+    def __init__(self, message: str, raw_content: str = ""):
+        super().__init__(message)
+        self.raw_content = raw_content
+
+
+class CancelledError(Exception):
+    """生成被取消"""
+
+    pass

app/shared/services/llm/generation/existing_config.py

@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview 既有项目配置加载（keep_existing 合并基线）
+
+读取项目清单文件以及其引用的 schema、constraint、regex 文件，组装成
+供 build_config 合并使用的既有配置字典。读取失败仅记警告、按条跳过，
+保证生成流程对局部损坏的项目配置保持宽容。
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+from app.shared.core.io.yaml import read_yaml
+
+logger = logging.getLogger(__name__)
+
+
+def load_existing_config(config_path: str) -> dict[str, Any] | None:
+    """加载现有配置（用于 keep_existing 合并）。
+
+    读取项目清单文件以及引用的 schema、constraint、regex 文件。
+
+    参数:
+        config_path: 项目配置目录（含 project.precis.yaml）
+
+    返回:
+        {"manifest": ..., "schemas": {...}, "constraints": {...}, "regex_nodes": {...}}；
+        清单不存在或整体读取失败时返回 None
+    """
+    manifest_path = os.path.join(config_path, "project.precis.yaml")
+    if not os.path.isfile(manifest_path):
+        return None
+
+    try:
+        manifest_data = read_yaml(Path(manifest_path))
+        if not isinstance(manifest_data, dict):
+            return None
+
+        existing: dict[str, Any] = {
+            "manifest": manifest_data,
+            "schemas": {},
+            "constraints": {},
+            "regex_nodes": {},
+        }
+
+        # 读取现有 schema 文件
+        for ref in manifest_data.get("schemas", []):
+            schema_path = os.path.join(config_path, ref.get("path", ""))
+            if os.path.isfile(schema_path):
+                try:
+                    raw = read_yaml(Path(schema_path))
+                    if isinstance(raw, dict):
+                        existing["schemas"][ref["id"]] = raw
+                except Exception as e:
+                    logger.warning(f"读取现有 schema 文件失败 {schema_path}: {e}")
+
+        # 读取现有 constraint 文件
+        for ref in manifest_data.get("constraints", []):
+            constraint_path = os.path.join(config_path, ref.get("path", ""))
+            if os.path.isfile(constraint_path):
+                try:
+                    raw = read_yaml(Path(constraint_path))
+                    if isinstance(raw, dict):
+                        existing["constraints"][ref["id"]] = raw
+                except Exception as e:
+                    logger.warning(f"读取现有 constraint 文件失败 {constraint_path}: {e}")
+
+        # 读取现有 regex 文件
+        for ref in manifest_data.get("regex_nodes", []):
+            regex_path = os.path.join(config_path, ref.get("path", ""))
+            if os.path.isfile(regex_path):
+                try:
+                    raw = read_yaml(Path(regex_path))
+                    if isinstance(raw, dict):
+                        existing["regex_nodes"][ref["id"]] = raw
+                except Exception as e:
+                    logger.warning(f"读取现有 regex 文件失败 {regex_path}: {e}")
+
+        return existing
+    except Exception as e:
+        logger.warning(f"加载现有配置失败: {e}")
+        return None