precis-cli 0.1.3__py3-none-any.whl

app/shared/services/ai/agent/chat_tools/read_table.py

@@ -0,0 +1,301 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview read_table 工具
+
+Chat mini-agent 可调用的工具：读取指定表的数据样本。
+
+根据表名定位 schema 文件，解析其 source 配置找到数据文件，
+加载前 N 行采样数据返回给 LLM。让 LLM 在设计约束规则前能"看到真实数据"，
+从而做出更准确的判断（如合理的 Range 范围、AllowedValues 取值等）。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from pathlib import Path
+from typing import Any
+
+import pandas as pd
+import yaml
+
+logger = logging.getLogger(__name__)
+
+# read_table 最多返回的单元格字符数，避免 observation 过长
+_MAX_CELL_CHARS = 200
+# 默认采样行数
+_DEFAULT_SAMPLE_ROWS = 10
+# 最大采样行数上限，防止 LLM 传入过大值
+_MAX_SAMPLE_ROWS = 100
+
+
+class ReadTableTool:
+    """
+    @classdesc 读取表数据样本工具
+
+    根据表名查找 schema，加载对应数据文件的前 N 行。
+    """
+
+    NAME = "read_table"
+
+    def __init__(self, project_path: str):
+        """
+        @methoddesc 初始化工具
+
+        参数:
+            project_path: 当前项目配置目录路径
+        """
+        self.project_path = project_path
+
+        # schema 索引缓存：避免每次 read_table 都 glob + 逐个解析 YAML
+        # 结构: {table_name_lower: (schema_path, source_config)}
+        # _schema_index_mtime 记录构建缓存时 schemas 目录的 mtime，用于失效检测
+        self._schema_index: dict[str, tuple[str, dict[str, Any]]] = {}
+        self._schema_index_mtime: float | None = None
+
+    def get_definition(self) -> dict[str, Any]:
+        """返回 OpenAI tool 定义。"""
+        return {
+            "type": "function",
+            "function": {
+                "name": self.NAME,
+                "description": (
+                    "读取指定表的数据样本（前 N 行），展示列结构和真实数据值。"
+                    "当需要为某列设计约束规则（如 Range/AllowedValues）时，"
+                    "先调用此工具查看真实数据分布，再决定合理的参数。"
+                    "表名可以是 schema 的 name 或 id。"
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "table_name": {
+                            "type": "string",
+                            "description": "要查看的表名（schema 的 name 或 id）",
+                        },
+                        "sample_rows": {
+                            "type": "integer",
+                            "description": f"采样行数，默认 {_DEFAULT_SAMPLE_ROWS}，最大 {_MAX_SAMPLE_ROWS}",
+                        },
+                    },
+                    "required": ["table_name"],
+                },
+            },
+        }
+
+    async def run(self, arguments: dict[str, Any]) -> dict[str, Any]:
+        """
+        @methoddesc 执行表数据采样读取
+
+        参数:
+            arguments: tool 参数，包含 table_name 和可选 sample_rows
+
+        返回:
+            {"success": bool, "table_name": str, "columns": [...], "sample_rows": [...], "error": str}
+        """
+        table_name = arguments.get("table_name", "")
+        if not table_name:
+            return {"success": False, "error": "未指定 table_name", "table_name": ""}
+
+        if not self.project_path:
+            return {"success": False, "error": "未配置项目路径", "table_name": table_name}
+
+        sample_rows = arguments.get("sample_rows", _DEFAULT_SAMPLE_ROWS)
+        sample_rows = max(1, min(int(sample_rows), _MAX_SAMPLE_ROWS))
+
+        try:
+            # 同步文件读取 + DataFrame 加载，放到线程池
+            result = await asyncio.to_thread(self._load_table_sample, table_name, sample_rows)
+            return result
+        except Exception as e:
+            logger.exception("read_table 工具执行失败")
+            return {"success": False, "error": f"读取表数据失败: {e}", "table_name": table_name}
+
+    def _load_table_sample(self, table_name: str, sample_rows: int) -> dict[str, Any]:
+        """
+        @methoddesc 同步加载表数据样本（在线程池中执行）
+
+        参数:
+            table_name: 表名
+            sample_rows: 采样行数
+
+        返回:
+            结果字典
+        """
+        # 步骤 1+2: 查 schema 文件路径 + source 配置（带缓存，合并原两步）
+        entry = self._get_schema_entry(table_name)
+        if not entry:
+            return {"success": False, "error": f"未找到表 '{table_name}' 的 schema 文件", "table_name": table_name}
+
+        _schema_path, source_config = entry
+        if not source_config.get("path"):
+            return {
+                "success": False,
+                "error": f"表 '{table_name}' 的 schema 未配置 source.path，无法读取数据",
+                "table_name": table_name,
+            }
+
+        # 步骤 3: 解析数据文件绝对路径
+        data_file = source_config["path"]
+        if not os.path.isabs(data_file):
+            # relative_file 模式：路径相对于项目根目录
+            data_file = os.path.join(self.project_path, data_file)
+
+        if not os.path.isfile(data_file):
+            return {
+                "success": False,
+                "error": f"数据文件不存在: {data_file}",
+                "table_name": table_name,
+            }
+
+        # 步骤 4: 加载数据文件
+        df = self._load_data_file(data_file, source_config)
+        if df is None:
+            return {
+                "success": False,
+                "error": f"无法加载数据文件: {data_file}",
+                "table_name": table_name,
+            }
+
+        # 步骤 5: 采样并格式化
+        df_sample = df.head(sample_rows)
+
+        # 列结构
+        columns = []
+        for col in df.columns:
+            col_str = str(col)
+            dtype = str(df[col].dtype)
+            null_count = int(df[col].isna().sum())
+            columns.append({"name": col_str, "dtype": dtype, "null_count": null_count})
+
+        # 采样行（转为可 JSON 序列化的列表，处理 NaN）
+        sample_records = self._dataframe_to_records(df_sample)
+
+        return {
+            "success": True,
+            "table_name": table_name,
+            "file_path": data_file,
+            "total_rows": int(len(df)),
+            "columns": columns,
+            "sample_rows": sample_records,
+        }
+
+    def _ensure_schema_index(self) -> None:
+        """
+        @methoddesc 构建/刷新 schema 索引缓存
+
+        首次调用或 schemas 目录 mtime 变化时，扫描所有 *.yaml 文件，
+        构建 {table_name_lower: (schema_path, source_config)} 映射。
+        之后同一 tool 实例内的 read_table 调用直接查缓存，避免重复 glob + 解析。
+
+        失效策略：比较目录 mtime。单文件内容变更（不触发目录 mtime）极少见，
+        且 agent 会话内通常不编辑 schema 文件，该策略足够可靠。
+        """
+        import os
+
+        schemas_dir = Path(self.project_path) / "schemas"
+        if not schemas_dir.exists():
+            self._schema_index = {}
+            self._schema_index_mtime = None
+            return
+
+        try:
+            current_mtime = os.path.getmtime(schemas_dir)
+        except OSError:
+            current_mtime = 0.0
+
+        # mtime 未变 → 缓存有效，直接返回
+        if self._schema_index_mtime is not None and current_mtime == self._schema_index_mtime:
+            return
+
+        # 重建索引：一次扫描收集所有 schema 的 id/name → (path, source)
+        index: dict[str, tuple[str, dict[str, Any]]] = {}
+        for schema_file in schemas_dir.glob("*.yaml"):
+            try:
+                with open(schema_file, encoding="utf-8") as f:
+                    data = yaml.safe_load(f) or {}
+                schema_id = str(data.get("id", "")).strip().lower()
+                schema_name = str(data.get("name", "")).strip().lower()
+                source_config = data.get("source") or {}
+                entry = (str(schema_file), source_config)
+                # id 和 name 都建索引（两者都可能作为 table_name 传入）
+                if schema_id:
+                    index[schema_id] = entry
+                if schema_name:
+                    index[schema_name] = entry
+            except Exception:
+                continue
+
+        self._schema_index = index
+        self._schema_index_mtime = current_mtime
+        logger.debug(f"[read_table] schema 索引已重建: {len(index)} 个条目")
+
+    def _get_schema_entry(self, table_name: str) -> tuple[str, dict[str, Any]] | None:
+        """
+        @methoddesc 查询 schema 文件路径和 source 配置（带缓存）
+
+        合并了原 _find_schema_file + _parse_schema_source 两步：
+        首次扫描 schemas 目录构建索引，之后直接查内存缓存。
+
+        参数:
+            table_name: 表名（schema 的 id 或 name）
+
+        返回:
+            (schema_path, source_config) 或 None（未找到）
+        """
+        self._ensure_schema_index()
+        target = table_name.strip().lower()
+        return self._schema_index.get(target)
+
+    def _load_data_file(self, data_file: str, source_config: dict[str, Any]) -> pd.DataFrame | None:
+        """
+        @methoddesc 根据文件扩展名加载数据文件
+
+        使用 load_file_data 统一入口，避免重复实现格式适配逻辑。
+        """
+        try:
+            from app.shared.services.validation.loader import load_file_data
+
+            sheet_name = source_config.get("sheet")
+            header_row = source_config.get("header_row", 0)
+            return load_file_data(
+                source_file_path=data_file,
+                sheet_name=sheet_name,
+                header_row=header_row,
+            )
+        except Exception as e:
+            logger.warning(f"load_file_data 加载失败 {data_file}: {e}")
+            return None
+
+    @staticmethod
+    def _dataframe_to_records(df: pd.DataFrame) -> list[dict[str, Any]]:
+        """将 DataFrame 转为可 JSON 序列化的记录列表，处理 NaN 和超长值。"""
+        records: list[dict[str, Any]] = []
+        for _, row in df.iterrows():
+            record: dict[str, Any] = {}
+            for col in df.columns:
+                value = row[col]
+                # NaN/None 转为 null
+                if pd.isna(value):
+                    record[str(col)] = None
+                    continue
+                # 截断超长字符串
+                value_str = str(value)
+                if len(value_str) > _MAX_CELL_CHARS:
+                    value_str = value_str[:_MAX_CELL_CHARS] + "..."
+                record[str(col)] = value_str
+            records.append(record)
+        return records

app/shared/services/ai/agent/chat_tools/schemas.py

@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview chat 工具入参的 Pydantic 校验模型
+
+P1-1：为每个 chat 工具定义入参模型，由 ToolRegistry 在 execute 入口做结构校验，
+统一拦截"必填字段缺失/类型错误"，把结构化错误回灌给 LLM（配合 P0-1 的错误回灌机制）。
+
+设计原则：
+- extra="allow"：LLM 偶尔会带额外字段（如调试信息、变体参数），不为此拒绝合法调用。
+- 复杂嵌套结构（如 apply_actions 的 *Spec）不在这一层校验，留给业务层的 ActionValidator，
+  避免两处校验逻辑漂移。这里只校验"工具入参骨架"。
+- MODEL_FOR_TOOL 是工具名 → 模型类的映射，ToolRegistry.register 时传入。
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class _ToolArgsBase(BaseModel):
+    """所有工具入参模型的基类：允许额外字段，避免误拒 LLM 的合法变体调用。"""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class ReadProjectArgs(_ToolArgsBase):
+    """read_project 无参数（空模型仍可拦截"传了非 dict 入参"的极端情况）。"""
+
+
+class ReadTableArgs(_ToolArgsBase):
+    """read_table 入参：table_name 必填，sample_rows 可选。"""
+
+    table_name: str = Field(..., description="要查看的表名（schema 的 name 或 id）")
+    sample_rows: int | None = Field(default=None, description="采样行数")
+
+
+class ReadCanvasArgs(_ToolArgsBase):
+    """read_canvas 无参数。"""
+
+
+class ValidateTableArgs(_ToolArgsBase):
+    """validate_table 入参：table_name 可选（不传则校验所有表）。"""
+
+    table_name: str | None = Field(default=None, description="要校验的表名；不传则校验所有表")
+
+
+class ApplyActionsArgs(_ToolArgsBase):
+    """apply_actions 入参：actions 必填（非空数组）。
+
+    actions 的元素是 dict（每个含 actionType + 对应 *Spec），元素内部结构由 ActionValidator
+    校验，这里只保证 actions 是非空 list[dict]，避免重复校验。
+    intent_scope 字段为 P2-1 意图校验改造预留（本轮可选）。
+    """
+
+    actions: list[dict[str, Any]] = Field(..., description="要执行的动作列表（非空）")
+    intent_scope: dict[str, Any] | None = Field(default=None, description="意图范围声明（预留）")
+
+    @field_validator("actions")
+    @classmethod
+    def _actions_non_empty(cls, v: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        # OpenAI schema 的 required 只保证字段存在，空数组会漏过；这里补非空校验
+        if not v:
+            raise ValueError("actions 不能为空数组")
+        return v
+
+
+class AskUserArgs(_ToolArgsBase):
+    """ask_user 入参：question_type 和 prompt 必填，其余按类型可选。"""
+
+    question_type: Literal["free_text", "choice", "value", "confirm"] = Field(..., description="提问类型")
+    prompt: str = Field(..., description="给用户看的问题文本")
+    options: list[dict[str, Any]] | None = Field(default=None, description="choice 类型必填：候选项列表")
+    multiple: bool | None = Field(default=None, description="choice 类型：是否多选")
+    value_type: Literal["string", "integer", "float", "boolean"] | None = Field(
+        default=None, description="value 类型必填：期望的值类型"
+    )
+    placeholder: str | None = Field(default=None, description="输入框占位提示")
+    optional: bool | None = Field(default=None, description="value 类型：是否允许留空")
+
+
+# 工具名 → 入参模型 的注册表。ToolRegistry.register 时按名查表。
+# 未在此注册的工具（如未来新工具）保持原透传行为，向后兼容。
+MODEL_FOR_TOOL: dict[str, type[_ToolArgsBase]] = {
+    "read_project": ReadProjectArgs,
+    "read_table": ReadTableArgs,
+    "read_canvas": ReadCanvasArgs,
+    "validate_table": ValidateTableArgs,
+    "apply_actions": ApplyActionsArgs,
+    "ask_user": AskUserArgs,
+}

app/shared/services/ai/agent/chat_tools/validate_table.py

@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""@fileoverview validate_table 工具
+
+Chat mini-agent 可调用的工具：执行项目数据校验。
+
+复用 execute_validate_project，支持校验全部表或单个表。
+让 LLM 在执行配置修改后，能自主验证改动效果，
+若发现问题可进一步调整 actions，实现"改-验-修"自我纠错循环。
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+from app.shared.services.llm.validate_executor import execute_validate_project
+
+logger = logging.getLogger(__name__)
+
+# 返回给 LLM 的错误条目上限，避免 observation 过长
+_MAX_ERRORS_IN_OBSERVATION = 15
+
+
+class ValidateTableTool:
+    """
+    @classdesc 数据校验工具
+
+    封装 execute_validate_project，支持全表或单表校验。
+    """
+
+    NAME = "validate_table"
+
+    def __init__(self, project_path: str):
+        """
+        @methoddesc 初始化工具
+
+        参数:
+            project_path: 当前项目配置目录路径
+        """
+        self.project_path = project_path
+
+    def get_definition(self) -> dict[str, Any]:
+        """返回 OpenAI tool 定义。"""
+        return {
+            "type": "function",
+            "function": {
+                "name": self.NAME,
+                "description": (
+                    "执行项目数据校验，检查当前约束规则是否通过。"
+                    "可校验全部表（不传 table_name）或指定单表（传 table_name）。"
+                    "建议在 apply_actions 修改约束后调用此工具验证效果；"
+                    "用户明确要求'校验项目'或'校验某表'时也应调用。"
+                    "返回错误数量和具体错误列表。"
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "table_name": {
+                            "type": "string",
+                            "description": "要校验的表名（schema 的 name 或 id）。不传则校验所有表。",
+                        }
+                    },
+                    "required": [],
+                },
+            },
+        }
+
+    async def run(self, arguments: dict[str, Any]) -> dict[str, Any]:
+        """
+        @methoddesc 执行数据校验
+
+        参数:
+            arguments: tool 参数，包含可选的 table_name
+
+        返回:
+            {"success": bool, "error_count": int, "errors": [...], "message": str}
+        """
+        if not self.project_path:
+            return {"success": False, "error": "未配置项目路径", "error_count": 0, "errors": []}
+
+        table_name = arguments.get("table_name")
+        # 空字符串视为不指定表
+        table_filter = table_name if table_name else None
+
+        try:
+            # execute_validate_project 是同步函数（内部跑校验引擎），放到线程池
+            result = await asyncio.to_thread(
+                execute_validate_project,
+                self.project_path,
+                table_filter,
+            )
+        except Exception as e:
+            logger.exception("validate_table 工具执行失败")
+            return {"success": False, "error": f"校验执行失败: {e}", "error_count": 0, "errors": []}
+
+        # execute_validate_project 返回 {success, message, details}
+        # success=True 表示校验流程跑通（不代表数据无错），details 含 error_count
+        details = result.get("details") or {}
+        error_count = details.get("error_count", 0)
+        raw_errors = details.get("errors", []) or []
+
+        # 截断错误列表，避免 observation 过长
+        truncated_errors = raw_errors[:_MAX_ERRORS_IN_OBSERVATION]
+        if len(raw_errors) > _MAX_ERRORS_IN_OBSERVATION:
+            truncated_count = len(raw_errors) - _MAX_ERRORS_IN_OBSERVATION
+        else:
+            truncated_count = 0
+
+        return {
+            "success": result.get("success", False),
+            "error_count": error_count,
+            "errors": truncated_errors,
+            "truncated_error_count": truncated_count,
+            "table_filter": table_filter,
+            "message": result.get("message", ""),
+        }