precis-cli 0.1.3__py3-none-any.whl

app/shared/services/ai/utils.py

@@ -0,0 +1,322 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview AI 服务通用工具函数
+
+功能概述:
+- 估算文本 Token 数量（支持中英文混合）
+- 按 Token 上限截断聊天历史记录
+- 扫描项目目录生成项目概览（Schema / Constraint 列表）
+
+架构设计:
+- 纯函数设计，无副作用，便于单元测试
+- Token 估算采用字符分类加权策略：中文单字 + 英文单词 + 数字 + 标点
+- 项目概览扫描递归读取 schemas/ 和 constraints/ 目录下的 YAML 文件
+
+输入示例:
+    estimate_tokens("Hello 世界")
+    truncate_history_by_tokens(history, "系统提示", max_tokens=120000)
+    get_project_overview("/path/to/project")
+
+输出示例:
+    7  # Token 估算值
+    [{"role": "user", "content": "..."}]  # 截断后的历史
+    {"schemas": [...], "constraints": [...]}  # 项目概览
+"""
+
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+
+def estimate_tokens(text: str) -> int:
+    """
+    @methoddesc 估算文本的 Token 数量
+
+    采用简单的字符分类加权策略，适合快速估算：
+    - 中文字符：每个字算 1 个 Token
+    - 英文单词：每个单词算 1 个 Token
+    - 数字串：每组连续数字算 1 个 Token
+    - 标点符号：每个算 1 个 Token
+    最后加上 10 个 Token 的固定开销（用于角色标记等）。
+
+    参数:
+        text: 要估算的文本字符串
+
+    返回:
+        估算的 Token 数量（整数）
+    """
+    if not text:
+        return 0
+
+    # 按字符类型分别统计数量
+    chinese_chars = len(re.findall(r"[\u4e00-\u9fff]", text))  # 中文字符
+    english_words = len(re.findall(r"[a-zA-Z]+", text))  # 英文单词
+    numbers = len(re.findall(r"\d+", text))  # 数字串
+    punctuations = len(re.findall(r"[^\w\s]", text))  # 标点符号
+
+    total = chinese_chars + english_words + numbers + punctuations
+    # 加上固定开销（系统提示、消息格式等消耗的额外 Token）
+    return total + 10
+
+
+def truncate_history_by_tokens(
+    chat_history: list[dict[str, str]],
+    system_prompt: str,
+    max_tokens: int = 120000,
+) -> list[dict[str, str]]:
+    """
+    @methoddesc 按 Token 上限截断聊天历史记录
+
+    从最近的消息开始往前累加，直到接近 max_tokens 上限为止。
+    至少保留最近 2 条消息（确保对话连贯性）。
+
+    参数:
+        chat_history: 历史消息列表，每条消息格式为 {"role": "user", "content": "..."}
+        system_prompt: 系统提示词文本（用于计算初始 Token 占用）
+        max_tokens: 允许的最大 Token 数量，默认 120000
+
+    返回:
+        截断后的历史消息列表
+    """
+    if not chat_history:
+        return []
+
+    # 计算系统提示词占用的 Token 数
+    system_tokens = estimate_tokens(system_prompt)
+
+    # 为每条历史消息估算 Token 数
+    history_with_tokens = []
+    for msg in chat_history:
+        tokens = estimate_tokens(msg.get("content", ""))
+        history_with_tokens.append((msg, tokens))
+
+    total_tokens = system_tokens
+    keep_count = 0
+
+    # 从最新的消息往前累加，直到超出上限
+    for msg, tokens in reversed(history_with_tokens):
+        if total_tokens + tokens > max_tokens:
+            break
+        total_tokens += tokens
+        keep_count += 1
+
+    # 至少保留最近 2 条消息（一问一答），避免截断后完全失去上下文
+    keep_count = max(keep_count, min(2, len(chat_history)))
+
+    truncated = chat_history[-keep_count:] if keep_count < len(chat_history) else chat_history
+
+    if len(truncated) < len(chat_history):
+        removed = len(chat_history) - len(truncated)
+        logger.info(
+            f"[truncate_history] 截断 {removed} 条历史，保留 {len(truncated)} 条，"
+            f"估算 Token: {total_tokens}/{max_tokens}"
+        )
+
+    return truncated
+
+
+def get_project_overview(project_path: str) -> dict[str, Any]:
+    """
+    @methoddesc 扫描项目目录，生成项目概览信息
+
+    读取项目下的 schemas/ 和 constraints/ 目录中的 YAML 文件，
+    汇总所有表结构、字段信息以及约束规则，供 AI 上下文使用。
+
+    本函数以 manifest 为权威来源：磁盘上存在但未登记到 manifest 的孤儿文件
+    会标注 unlisted=True，避免 AI 视角与用户资源树视角产生偏差（前者纯 glob，
+    后者走 manifest）。
+
+    参数:
+        project_path: 项目根目录路径
+
+    返回:
+        项目概览字典，包含 schemas 和 constraints 两个列表
+    """
+    overview: dict[str, Any] = {"schemas": [], "constraints": [], "transforms": [], "regex_nodes": [], "settings": {}}
+
+    if not project_path:
+        return overview
+
+    project_root = Path(project_path)
+
+    # 读取 manifest 的 schemas 白名单，用于标注孤儿文件
+    listed_schema_paths: set[str] = set()
+    manifest_path = project_root / "project.precis.yaml"
+    if manifest_path.exists():
+        try:
+            with open(manifest_path, encoding="utf-8") as f:
+                manifest_data = yaml.safe_load(f) or {}
+            for ref in manifest_data.get("schemas", []) or []:
+                p = ref.get("path") if isinstance(ref, dict) else None
+                if isinstance(p, str) and p:
+                    listed_schema_paths.add(p.replace("\\", "/").lower())
+        except Exception as e:
+            logger.debug(f"读取 manifest schemas 失败 {manifest_path}: {e}")
+
+    schemas_dir = project_root / "schemas"
+    if schemas_dir.exists():
+        for schema_file in schemas_dir.glob("*.yaml"):
+            try:
+                with open(schema_file, encoding="utf-8") as f:
+                    schema_data = yaml.safe_load(f) or {}
+
+                table_name = schema_data.get("name", "")
+                table_id = schema_data.get("id", table_name)
+                columns = schema_data.get("columns", [])
+                inline_constraints = schema_data.get("constraints", [])
+
+                column_list = []
+                column_map = {}
+                for c in columns:
+                    col_name = c.get("name", "")
+                    col_id = c.get("id", col_name)
+                    col_type = c.get("type", "")
+                    column_list.append({"id": col_id, "name": col_name, "type": col_type})
+                    column_map[col_id] = col_name
+
+                if table_name or table_id:
+                    # 标注该 schema 是否登记在 manifest 中，孤儿文件 unlisted=True
+                    rel_path = f"schemas/{schema_file.name}".replace("\\", "/").lower()
+                    overview["schemas"].append(
+                        {
+                            "id": table_id,
+                            "name": table_name,
+                            "columns": column_list,
+                            "unlisted": rel_path not in listed_schema_paths,
+                            "path": f"schemas/{schema_file.name}",
+                        }
+                    )
+
+                for ic in inline_constraints:
+                    col_id = ic.get("column", "")
+                    col_name = column_map.get(col_id, col_id)
+                    constraint_type = ic.get("type", "")
+                    params = ic.get("params", {})
+
+                    overview["constraints"].append(
+                        {
+                            "id": ic.get("id", ""),
+                            "type": constraint_type,
+                            "description": f"{table_name}.{col_name} - {constraint_type} 约束 (内联)",
+                            # §2.13: 读取磁盘真实 enabled（无字段默认 true，与运行时语义一致）——
+                            # 原硬编码 True 使 AI 对已禁用约束误判"在生效"
+                            "enabled": ic.get("enabled", True),
+                            "table_id": table_id,
+                            "table_name": table_name,
+                            "column_id": col_id,
+                            "column_name": col_name,
+                            "params": params,
+                            "is_inline": True,
+                        }
+                    )
+            except Exception as e:
+                logger.debug(f"读取 schema 文件失败 {schema_file}: {e}")
+
+    constraints_dir = project_root / "constraints"
+    if constraints_dir.exists():
+        for constraint_file in constraints_dir.glob("*.constraint.yaml"):
+            try:
+                with open(constraint_file, encoding="utf-8") as f:
+                    constraint_data = yaml.safe_load(f) or {}
+
+                constraint_id = constraint_data.get("id", "")
+                constraint_type = constraint_data.get("type", "")
+                description = constraint_data.get("description", "")
+                enabled = constraint_data.get("enabled", True)
+                refs = constraint_data.get("refs", {})
+                params = constraint_data.get("params", {})
+
+                table_id = refs.get("table_id", "")
+                column_id = refs.get("column_id", "")
+                column_ids = refs.get("column_ids", [])
+
+                overview["constraints"].append(
+                    {
+                        "id": constraint_id,
+                        "type": constraint_type,
+                        "description": description,
+                        "enabled": enabled,
+                        "table_id": table_id,
+                        "column_id": column_id,
+                        "column_ids": column_ids,
+                        "params": params,
+                        "is_inline": False,
+                    }
+                )
+            except Exception as e:
+                logger.warning(f"读取 constraint 文件失败 {constraint_file}: {e}")
+
+    # 扫描 Regex 节点
+    for dirname in ("regex_nodes", "regex"):
+        regex_dir = project_root / dirname
+        if regex_dir.exists():
+            for regex_file in regex_dir.glob("*.yaml"):
+                try:
+                    with open(regex_file, encoding="utf-8") as f:
+                        regex_data = yaml.safe_load(f) or {}
+
+                    overview["regex_nodes"].append(
+                        {
+                            "id": regex_data.get("id", ""),
+                            "name": regex_data.get("name", ""),
+                            "pattern": regex_data.get("pattern", ""),
+                            "match_mode": regex_data.get("match_mode", "full"),
+                            "enabled": regex_data.get("enabled", True),
+                            "source_ref": regex_data.get("source_ref"),
+                        }
+                    )
+                except Exception as e:
+                    logger.debug(f"读取 regex 文件失败 {regex_file}: {e}")
+
+    # 扫描 Transform 节点
+    transforms_dir = project_root / "transforms"
+    if transforms_dir.exists():
+        for transform_file in transforms_dir.glob("*.yaml"):
+            try:
+                with open(transform_file, encoding="utf-8") as f:
+                    transform_data = yaml.safe_load(f) or {}
+
+                overview["transforms"].append(
+                    {
+                        "id": transform_data.get("id", ""),
+                        "type": transform_data.get("type", ""),
+                        "enabled": transform_data.get("enabled", True),
+                        "input_from_node": transform_data.get("input_from_node"),
+                        "input_column": transform_data.get("input_column"),
+                        "output_columns": transform_data.get("output_columns", []),
+                    }
+                )
+            except Exception as e:
+                logger.debug(f"读取 transform 文件失败 {transform_file}: {e}")
+
+    # 读取项目设置
+    manifest_path = project_root / "project.precis.yaml"
+    if manifest_path.exists():
+        try:
+            with open(manifest_path, encoding="utf-8") as f:
+                manifest_data = yaml.safe_load(f) or {}
+
+            overview["settings"] = manifest_data.get("settings", {})
+        except Exception as e:
+            logger.debug(f"读取 manifest 设置失败 {manifest_path}: {e}")
+
+    return overview

app/shared/services/diff/config_diff.py

@@ -0,0 +1,320 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 配置差异比对模块
+
+功能概述:
+- 比较两份项目配置（manifest/schemas/constraints/regex_nodes）的差异
+- 支持新增、修改、删除三种差异类型的识别
+- 为 AI 配置生成后的合并确认提供数据支持
+
+架构设计:
+- 服务类模式: ConfigDiffService 提供类方法 compare 作为统一入口
+- 递归比对: _diff_recursive 处理 dict/list 嵌套结构的深层差异
+- 资源级比对: _compare_resources 处理 schemas/constraints/regex_nodes 的增删改
+- 属性级比对: _build_property_diff 生成 PropertyDiff 变更列表
+
+输入示例:
+    old = {"manifest": {...}, "schemas": {"users": old_schema}}
+    new = {"manifest": {...}, "schemas": {"users": new_schema}}
+    result = ConfigDiffService.compare(old, new)
+
+输出示例:
+    ConfigDiffResult(
+        manifest=[{"path": ["project", "name"], "old_value": "A", "new_value": "B", "diff_type": "modified"}],
+        schemas=[ConfigItemDiff(id="users", type=DiffType.MODIFIED, changes=[...])],
+        constraints=[],
+        regex_nodes=[]
+    )
+"""
+
+import logging
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+
+class DiffType(str, Enum):
+    """
+    @classdesc 差异类型枚举
+
+    定义配置比对时可能出现的四种差异状态：
+    - ADDED: 新增内容
+    - MODIFIED: 修改内容
+    - DELETED: 删除内容
+    - UNCHANGED: 未变更（通常不展示）
+    """
+
+    ADDED = "added"
+    MODIFIED = "modified"
+    DELETED = "deleted"
+    UNCHANGED = "unchanged"
+
+
+class PropertyDiff(BaseModel):
+    """
+    @classdesc 属性级差异模型
+
+    记录单个配置属性的变更详情，包括属性名、旧值、新值和变更类型。
+    """
+
+    key: str
+    oldValue: Any | None = None
+    newValue: Any | None = None
+    type: DiffType
+
+
+class ConfigItemDiff(BaseModel):
+    """
+    @classdesc 配置项差异模型
+
+    记录单个配置资源（如 schema、constraint）的整体变更，
+    包含资源 ID、名称、变更类型以及具体的属性变更列表。
+    """
+
+    id: str
+    name: str
+    type: DiffType
+    original: dict[str, Any] | None = None
+    generated: dict[str, Any] | None = None
+    changes: list[PropertyDiff] = Field(default_factory=list)
+
+
+class ConfigDiffResult(BaseModel):
+    """
+    @classdesc 配置差异结果模型
+
+    聚合整个项目配置的差异信息，分别存储 manifest、schemas、constraints、regex_nodes 的变更列表。
+    """
+
+    manifest: list[Any] = Field(default_factory=list)
+    schemas: list[ConfigItemDiff] = Field(default_factory=list)
+    constraints: list[ConfigItemDiff] = Field(default_factory=list)
+    regex_nodes: list[ConfigItemDiff] = Field(default_factory=list)
+
+
+class ConfigDiffService:
+    """
+    @classdesc 配置差异比对服务
+
+    提供类方法 compare 作为统一入口，支持递归比对 dict/list 嵌套结构，
+    识别新增、修改、删除三种差异类型。
+    """
+
+    @classmethod
+    def compare(cls, old_config: dict[str, Any], new_config: dict[str, Any]) -> ConfigDiffResult:
+        """
+        @methoddesc 比较两份完整配置并返回差异结果
+
+        参数:
+            old_config: 旧配置字典
+            new_config: 新配置字典
+
+        返回:
+            ConfigDiffResult 差异结果对象
+        """
+        logger.debug("Starting config comparison")
+        result = ConfigDiffResult()
+
+        old_manifest = old_config.get("manifest", {})
+        new_manifest = new_config.get("manifest", {})
+        if hasattr(old_manifest, "model_dump"):
+            old_manifest = old_manifest.model_dump(exclude_none=True)
+        if hasattr(new_manifest, "model_dump"):
+            new_manifest = new_manifest.model_dump(exclude_none=True)
+
+        result.manifest = cls._diff_recursive(old_manifest, new_manifest, [])
+
+        result.schemas = cls._compare_resources(old_config.get("schemas", {}), new_config.get("schemas", {}))
+        result.constraints = cls._compare_resources(
+            old_config.get("constraints", {}), new_config.get("constraints", {})
+        )
+        result.regex_nodes = cls._compare_resources(
+            old_config.get("regex_nodes", {}), new_config.get("regex_nodes", {})
+        )
+
+        total_changes = len(result.manifest) + len(result.schemas) + len(result.constraints) + len(result.regex_nodes)
+        logger.debug(f"Config comparison completed: {total_changes} changes detected")
+        return result
+
+    @classmethod
+    def _compare_resources(cls, old_resources: dict[str, Any], new_resources: dict[str, Any]) -> list[ConfigItemDiff]:
+        """
+        @methoddesc 比对资源字典（schemas/constraints/regex_nodes）
+
+        识别新增、删除和修改的资源项。
+
+        参数:
+            old_resources: 旧资源字典
+            new_resources: 新资源字典
+
+        返回:
+            ConfigItemDiff 列表
+        """
+        result: list[ConfigItemDiff] = []
+
+        old_keys = set(old_resources.keys())
+        new_keys = set(new_resources.keys())
+
+        added_keys = new_keys - old_keys
+        deleted_keys = old_keys - new_keys
+        common_keys = old_keys.intersection(new_keys)
+
+        # 旧配置存在、新配置缺失的资源必须以 DELETED 呈现——
+        # 否则 AI 重新生成后的合并确认界面看不到"将被删除"的资源
+        for key in deleted_keys:
+            old_val = old_resources[key]
+            if hasattr(old_val, "model_dump"):
+                old_val = old_val.model_dump(exclude_none=True)
+            result.append(
+                ConfigItemDiff(id=key, name=key, type=DiffType.DELETED, original=old_val, generated=None, changes=[])
+            )
+
+        for key in added_keys:
+            new_val = new_resources[key]
+            if hasattr(new_val, "model_dump"):
+                new_val = new_val.model_dump(exclude_none=True)
+            result.append(
+                ConfigItemDiff(id=key, name=key, type=DiffType.ADDED, original=None, generated=new_val, changes=[])
+            )
+
+        for key in common_keys:
+            old_val = old_resources[key]
+            new_val = new_resources[key]
+
+            if hasattr(old_val, "model_dump"):
+                old_val = old_val.model_dump(exclude_none=True)
+            if hasattr(new_val, "model_dump"):
+                new_val = new_val.model_dump(exclude_none=True)
+
+            changes = cls._build_property_diff(old_val, new_val, [])
+            if changes:
+                result.append(
+                    ConfigItemDiff(
+                        id=key, name=key, type=DiffType.MODIFIED, original=old_val, generated=new_val, changes=changes
+                    )
+                )
+
+        return result
+
+    @classmethod
+    def _build_property_diff(cls, old: dict[str, Any], new: dict[str, Any], path: list[str]) -> list[PropertyDiff]:
+        """
+        @methoddesc 递归构建属性级差异列表
+
+        深度遍历字典结构，对比每个键值对的变更。
+
+        参数:
+            old: 旧字典
+            new: 新字典
+            path: 当前路径前缀
+
+        返回:
+            PropertyDiff 列表
+        """
+        changes: list[PropertyDiff] = []
+
+        if not isinstance(old, dict) or not isinstance(new, dict):
+            return changes
+
+        old_keys = set(old.keys())
+        new_keys = set(new.keys())
+
+        for key in new_keys - old_keys:
+            changes.append(
+                PropertyDiff(
+                    key=".".join(path + [key]) if path else key, oldValue=None, newValue=new[key], type=DiffType.ADDED
+                )
+            )
+
+        for key in old_keys - new_keys:
+            changes.append(
+                PropertyDiff(
+                    key=".".join(path + [key]) if path else key, oldValue=old[key], newValue=None, type=DiffType.DELETED
+                )
+            )
+
+        for key in old_keys.intersection(new_keys):
+            old_v = old[key]
+            new_v = new[key]
+
+            if isinstance(old_v, dict) and isinstance(new_v, dict):
+                changes.extend(cls._build_property_diff(old_v, new_v, path + [key]))
+            elif old_v != new_v:
+                changes.append(
+                    PropertyDiff(
+                        key=".".join(path + [key]) if path else key,
+                        oldValue=old_v,
+                        newValue=new_v,
+                        type=DiffType.MODIFIED,
+                    )
+                )
+
+        return changes
+
+    @classmethod
+    def _diff_recursive(cls, old: Any, new: Any, path: list[str | int]) -> list[Any]:
+        """
+        @methoddesc 递归比对任意数据结构
+
+        支持 dict、list 和基本类型的深层比对。
+
+        参数:
+            old: 旧值
+            new: 新值
+            path: 当前路径
+
+        返回:
+            变更字典列表
+        """
+        changes = []
+
+        if isinstance(old, dict) and isinstance(new, dict):
+            old_keys = set(old.keys())
+            new_keys = set(new.keys())
+
+            for key in new_keys - old_keys:
+                changes.append({"path": path + [key], "old_value": None, "new_value": new[key], "diff_type": "added"})
+
+            for key in old_keys - new_keys:
+                changes.append({"path": path + [key], "old_value": old[key], "new_value": None, "diff_type": "removed"})
+
+            for key in old_keys.intersection(new_keys):
+                changes.extend(cls._diff_recursive(old[key], new[key], path + [key]))
+
+        elif isinstance(old, list) and isinstance(new, list):
+            len_old = len(old)
+            len_new = len(new)
+
+            for i in range(min(len_old, len_new)):
+                changes.extend(cls._diff_recursive(old[i], new[i], path + [i]))
+
+            if len_new > len_old:
+                for i in range(len_old, len_new):
+                    changes.append({"path": path + [i], "old_value": None, "new_value": new[i], "diff_type": "added"})
+
+            if len_old > len_new:
+                for i in range(len_new, len_old):
+                    changes.append({"path": path + [i], "old_value": old[i], "new_value": None, "diff_type": "removed"})
+
+        else:
+            if old != new:
+                changes.append({"path": path, "old_value": old, "new_value": new, "diff_type": "modified"})
+
+        return changes