precis-cli 0.1.3__py3-none-any.whl

app/api/routers/project/full_config_writer.py

@@ -0,0 +1,327 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview V2 全量配置写入模块
+
+功能概述:
+- 处理 V2 全量配置的写入逻辑
+- 合并策略：保留现有 manifest 中未覆盖的引用
+- 从目录扫描补充缺失的资源引用
+
+输入示例:
+    result = write_v2_full_config(payload, config_path, manifest_path)
+
+输出示例:
+    {"message": "V2 全量配置已保存。"}
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+
+from fastapi import HTTPException
+
+from app.shared.core.io.yaml import read_yaml, write_yaml_atomic
+from app.shared.core.project.manifest.types import ManualDataRefV2, ProjectManifestV2
+
+from .base import (
+    ConstraintRefV2,
+    FullConfigV2Request,
+    RegexNodeRefV2,
+    SchemaRefV2,
+    TransformRefV2,
+    _v2_manifest_path,
+)
+from .helpers import _resolve_project_path, project_lock
+
+logger = logging.getLogger(__name__)
+
+
+def _merge_manifest_references(
+    payload: FullConfigV2Request,
+    existing_manifest: ProjectManifestV2 | None,
+    config_path: str,
+) -> ProjectManifestV2:
+    """合并 manifest 引用。
+
+    合并策略：对于客户端**未显式设置**的资源字段，保留现有 manifest 中的同类引用；
+    对于显式设置的字段（即使设为空列表 []），遵从客户端意图（允许清空）。
+    通过 model_fields_set 区分"未提供"与"显式置空"。
+
+    Args:
+        payload: 全量配置请求
+        existing_manifest: 现有 manifest（可能为 None）
+        config_path: 项目配置根目录
+
+    Returns:
+        合并后的最终 manifest
+    """
+    final_manifest = payload.manifest
+    # 客户端显式设置的字段集合（区分"未提供"与"显式置空 []"）
+    set_fields = payload.manifest.model_fields_set
+
+    def _should_merge(field: str) -> bool:
+        """字段未被客户端显式设置，且现有 manifest 有值时才合并。"""
+        return field not in set_fields and existing_manifest is not None and bool(getattr(existing_manifest, field))
+
+    # mypy 无法从 _should_merge 的返回值推断 existing_manifest 非 None，
+    # 用 existing 局部变量（仅在 existing_manifest 非 None 时赋值）帮助类型收窄。
+    existing: ProjectManifestV2 | None = existing_manifest
+
+    if _should_merge("schemas") and existing is not None:
+        logger.info(f"[put_v2_full_config] 合并现有 schemas: {len(existing.schemas)} 个")
+        final_manifest = final_manifest.model_copy(update={"schemas": existing.schemas})
+
+    if _should_merge("constraints") and existing is not None:
+        final_manifest = final_manifest.model_copy(update={"constraints": existing.constraints})
+
+    if _should_merge("regex_nodes") and existing is not None:
+        final_manifest = final_manifest.model_copy(update={"regex_nodes": existing.regex_nodes})
+
+    if _should_merge("transforms") and existing is not None:
+        final_manifest = final_manifest.model_copy(update={"transforms": existing.transforms})
+
+    if _should_merge("manual_data") and existing is not None:
+        final_manifest = final_manifest.model_copy(update={"manual_data": existing.manual_data})
+
+    if _should_merge("data_sources") and existing is not None:
+        final_manifest = final_manifest.model_copy(update={"data_sources": existing.data_sources})
+
+    # B-fix(settings 回滚防御): settings 同样纳入"未显式设置则保留磁盘值"的合并策略。
+    # 此前 payload 的 settings 直接覆盖——前端 builder 硬编码默认值时,用户在设置面板
+    # 改过的 CSV 分隔符/超时等会被静默回滚。注意:当前前端总是显式传 settings,
+    # 此防御需配合前端 builder 不传或回传真实 settings 才完整生效(协议先正确)。
+    if _should_merge("settings") and existing is not None:
+        final_manifest = final_manifest.model_copy(update={"settings": existing.settings})
+
+    # 回归(2026-09-04 扫描): project.description 保全。前端全量保存总是显式构造
+    # project: {id, name}（project 键恒在 set_fields，整字段级合并不适用），但
+    # description 是磁盘上手写的可选元数据——payload 未显式提供该嵌套字段时从磁盘
+    # 透传，防止每次全量保存静默抹除（与 settings 防御同型，按嵌套字段粒度判 fields_set；
+    # 显式传 null/新值则遵从客户端意图）。
+    if existing is not None and existing.project.description:
+        payload_project = final_manifest.project
+        if "description" not in payload_project.model_fields_set:
+            merged_project = payload_project.model_copy(update={"description": existing.project.description})
+            final_manifest = final_manifest.model_copy(update={"project": merged_project})
+
+    # 回归(2026-09): templates/template_instances 并入统一 fields_set 防线。
+    # 此前因 _validate_unique_ids 就地重赋值污染 model_fields_set（列表字段恒被标记），
+    # fields_set 判定对列表字段全部失效，这两个字段只能退化为"空列表=未携带"启发式；
+    # 现校验器已改用 object.__setattr__ 不再污染 fields_set（纯净性守卫见
+    # test_full_config_reliability），本分支与其余列表字段走同一判定，启发式退役。
+    # 语义变化：payload 显式传 templates: [] 现在遵从"显式置空=清空意图"（旧启发式
+    # 会强制保留磁盘值）；与 schemas/constraints 等字段的既定语义对齐。
+    if _should_merge("templates") and existing is not None:
+        final_manifest = final_manifest.model_copy(update={"templates": existing.templates})
+
+    if _should_merge("template_instances") and existing is not None:
+        final_manifest = final_manifest.model_copy(update={"template_instances": existing.template_instances})
+
+    # 目录扫描补充：仅对客户端"未显式设置"且当前为空的字段从磁盘发现文件，
+    # 显式置空 [] 的字段不会被目录扫描覆盖（尊重用户清空意图）
+    def _should_scan(field: str) -> bool:
+        return field not in set_fields and not bool(getattr(final_manifest, field))
+
+    if _should_scan("schemas"):
+        schemas_dir = os.path.join(config_path, "schemas")
+        if os.path.isdir(schemas_dir):
+            schema_refs = []
+            for filename in os.listdir(schemas_dir):
+                if filename.endswith(".schema.yaml"):
+                    schema_id = filename[:-12]
+                    schema_refs.append(SchemaRefV2(id=schema_id, path=f"schemas/{filename}"))
+            if schema_refs:
+                logger.info(f"[put_v2_full_config] 从 schemas/ 目录扫描到 {len(schema_refs)} 个 schema 文件")
+                final_manifest = final_manifest.model_copy(update={"schemas": schema_refs})
+
+    if _should_scan("constraints"):
+        constraints_dir = os.path.join(config_path, "constraints")
+        if os.path.isdir(constraints_dir):
+            constraint_refs = []
+            for filename in os.listdir(constraints_dir):
+                if filename.endswith(".constraint.yaml"):
+                    constraint_id = filename[:-16]
+                    constraint_refs.append(ConstraintRefV2(id=constraint_id, path=f"constraints/{filename}"))
+            if constraint_refs:
+                logger.info(
+                    f"[put_v2_full_config] 从 constraints/ 目录扫描到 {len(constraint_refs)} 个 constraint 文件"
+                )
+                final_manifest = final_manifest.model_copy(update={"constraints": constraint_refs})
+
+    if _should_scan("regex_nodes"):
+        regex_dirs = [os.path.join(config_path, "regex"), os.path.join(config_path, "regex_nodes")]
+        seen_regex_ids = set()
+        regex_refs: list[RegexNodeRefV2] = []
+        for d in regex_dirs:
+            if not os.path.isdir(d):
+                continue
+            for filename in os.listdir(d):
+                if filename.endswith(".regex.yaml"):
+                    regex_id = filename[: -len(".regex.yaml")]
+                    if regex_id in seen_regex_ids:
+                        continue
+                    rel_dir = os.path.basename(d)
+                    regex_refs.append(RegexNodeRefV2(id=regex_id, path=f"{rel_dir}/{filename}"))
+                    seen_regex_ids.add(regex_id)
+        if regex_refs:
+            logger.info(f"[put_v2_full_config] 从 regex/ 目录扫描到 {len(regex_refs)} 个 regex 文件")
+            final_manifest = final_manifest.model_copy(update={"regex_nodes": regex_refs})
+
+    if _should_scan("transforms"):
+        transforms_dir = os.path.join(config_path, "transforms")
+        if os.path.isdir(transforms_dir):
+            transform_refs = []
+            for filename in os.listdir(transforms_dir):
+                if filename.endswith(".transform.yaml"):
+                    transform_id = filename[:-15]
+                    transform_refs.append(TransformRefV2(id=transform_id, path=f"transforms/{filename}"))
+            if transform_refs:
+                logger.info(f"[put_v2_full_config] 从 transforms/ 目录扫描到 {len(transform_refs)} 个 transform 文件")
+                final_manifest = final_manifest.model_copy(update={"transforms": transform_refs})
+
+    if _should_scan("manual_data"):
+        manual_data_dir = os.path.join(config_path, "manual_data")
+        if os.path.isdir(manual_data_dir):
+            manual_data_refs = []
+            for filename in os.listdir(manual_data_dir):
+                if filename.endswith(".manual_data.yaml"):
+                    manual_data_id = filename[: -len(".manual_data.yaml")]
+                    manual_data_refs.append(ManualDataRefV2(id=manual_data_id, path=f"manual_data/{filename}"))
+            if manual_data_refs:
+                logger.info(
+                    f"[put_v2_full_config] 从 manual_data/ 目录扫描到 {len(manual_data_refs)} 个 manual_data 文件"
+                )
+                final_manifest = final_manifest.model_copy(update={"manual_data": manual_data_refs})
+
+    return final_manifest
+
+
+def _write_resource_files(
+    payload: FullConfigV2Request,
+    config_path: str,
+) -> None:
+    """写入资源文件。
+
+    将 payload 中的 schemas、constraints、regex_nodes 写入对应的文件。
+
+    Args:
+        payload: 全量配置请求
+        config_path: 项目配置根目录
+    """
+    for schema_ref in payload.manifest.schemas:
+        schema = payload.schemas.get(schema_ref.id)
+        if not schema:
+            continue
+        try:
+            abs_path = _resolve_project_path(config_path, schema_ref.path)
+        except ValueError as e:
+            logger.error(f"[put_v2_full_config] 非法 Schema 路径: {schema_ref.path}, 错误: {e}")
+            continue
+        write_yaml_atomic(Path(abs_path), schema.model_dump(exclude_none=True))
+
+    for constraint_ref in payload.manifest.constraints:
+        constraint = payload.constraints.get(constraint_ref.id)
+        if not constraint:
+            continue
+        try:
+            abs_path = _resolve_project_path(config_path, constraint_ref.path)
+        except ValueError as e:
+            logger.error(f"[put_v2_full_config] 非法 Constraint 路径: {constraint_ref.path}, 错误: {e}")
+            continue
+        write_yaml_atomic(Path(abs_path), constraint.model_dump(exclude_none=True))
+
+    for regex_ref in payload.manifest.regex_nodes:
+        regex_node = payload.regex_nodes.get(regex_ref.id)
+        if not regex_node:
+            continue
+        try:
+            abs_path = _resolve_project_path(config_path, regex_ref.path)
+        except ValueError as e:
+            logger.error(f"[put_v2_full_config] 非法 Regex 路径: {regex_ref.path}, 错误: {e}")
+            continue
+        write_yaml_atomic(Path(abs_path), regex_node.model_dump(exclude_none=True))
+
+    for transform_ref in payload.manifest.transforms or []:
+        transform = payload.transforms.get(transform_ref.id)
+        if not transform:
+            continue
+        try:
+            abs_path = _resolve_project_path(config_path, transform_ref.path)
+        except ValueError as e:
+            logger.error(f"[put_v2_full_config] 非法 Transform 路径: {transform_ref.path}, 错误: {e}")
+            continue
+        write_yaml_atomic(Path(abs_path), transform.model_dump(exclude_none=True))
+
+    for manual_data_ref in payload.manifest.manual_data or []:
+        manual_data = payload.manual_data.get(manual_data_ref.id)
+        if not manual_data:
+            continue
+        try:
+            abs_path = _resolve_project_path(config_path, manual_data_ref.path)
+        except ValueError as e:
+            logger.error(f"[put_v2_full_config] 非法 ManualData 路径: {manual_data_ref.path}, 错误: {e}")
+            continue
+        write_yaml_atomic(Path(abs_path), manual_data.model_dump(exclude_none=True))
+
+
+def write_v2_full_config(
+    payload: FullConfigV2Request,
+    config_path: str,
+) -> dict[str, str]:
+    """写入 V2 全量配置。
+
+    会覆盖写入 manifest 与其引用的文件。
+
+    Args:
+        payload: 包含 manifest 和所有资源配置的请求
+        config_path: 项目配置根目录
+
+    Returns:
+        操作结果消息字典
+    """
+    manifest_path = _v2_manifest_path(config_path)
+
+    # B-reliability: existing manifest 的读取必须在锁内(与写入同一临界区),否则:
+    # 1) 并发场景读到他人写入一半的状态; 2) 解析失败时 existing=None 继续写入,
+    # data_sources/templates/settings 等无目录扫描兜底的字段会被空基准覆盖丢失。
+    # manifest 已存在但损坏时直接拒绝写入(500),不带空基准继续。
+    with project_lock(config_path):
+        existing_manifest = None
+        if os.path.isfile(manifest_path):
+            try:
+                existing_manifest = ProjectManifestV2.model_validate(read_yaml(Path(manifest_path)))
+                logger.info("[put_v2_full_config] 读取现有 manifest 成功")
+            except Exception as e:
+                logger.error(f"[put_v2_full_config] 读取现有 manifest 失败,拒绝写入以保护现有配置: {e}")
+                raise HTTPException(
+                    status_code=500,
+                    detail="现有 project.precis.yaml 解析失败,已拒绝本次写入。请先修复或回退 manifest 再保存。",
+                )
+
+        # Step 1: 合并 manifest 引用
+        final_manifest = _merge_manifest_references(payload, existing_manifest, config_path)
+
+        # Step 2: 写入 manifest 与资源文件（在同一锁内保证一致性）
+        write_yaml_atomic(Path(manifest_path), final_manifest.model_dump(exclude_none=True))
+        logger.info(f"[put_v2_full_config] 写入 manifest 完成，schemas: {len(final_manifest.schemas)} 个")
+
+        # Step 3: 写入资源文件
+        _write_resource_files(payload, config_path)
+
+    return {"message": "V2 全量配置已保存。"}

app/api/routers/project/helpers.py

@@ -0,0 +1,167 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Copyright 2026 Precis Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+@fileoverview 项目辅助函数模块
+
+功能概述:
+- 安全加载 YAML 文件的辅助函数
+- 计算 V2 项目清单、视图、工作区配置文件的绝对路径
+- 提供安全的路径解析和项目级文件锁机制
+- 原子写入由 shared.core.io.yaml.write_yaml_atomic 提供
+
+架构设计:
+- 纯 Python 实现，不依赖 FastAPI，便于单元测试
+- 使用 filelock 实现项目级并发写保护
+- 原子写入使用临时文件+重命名策略
+
+输入示例:
+    path = _v2_manifest_path("/project/root")
+    config = _load_yaml_file(path)
+
+输出示例:
+    {"version": 2, "project": {"id": "my-project", "name": "My Project"}}
+"""
+
+import os
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+import yaml
+from filelock import FileLock
+
+V2_MANIFEST_FILENAME = "project.precis.yaml"
+V2_VIEW_FILENAME = "project.view.json"
+V2_WORKSPACES_FILENAME = ".precis/workspaces.json"
+
+
+def _load_yaml_file(path: str) -> dict:
+    """
+    安全加载 YAML 文件的辅助函数。
+
+    设计意图：
+    - 提供统一的错误处理机制，避免调用方处理各种解析异常
+    - 返回空字典而不是 None，简化调用方的空值判断
+
+    副作用：
+    - 如果文件不存在或解析失败，仅记录日志而不抛出异常
+
+    参数:
+        path: YAML 文件的绝对路径
+
+    返回:
+        解析后的字典，失败时返回空字典
+    """
+    if not os.path.isfile(path):
+        return {}
+    try:
+        with open(path, encoding="utf-8") as f:
+            return yaml.safe_load(f) or {}
+    except yaml.YAMLError:
+        return {}
+
+
+def _v2_manifest_path(config_path: str) -> str:
+    """
+    计算 V2 项目清单文件的绝对路径。
+
+    参数:
+        config_path: 项目配置根目录
+
+    返回:
+        manifest 文件的绝对路径
+    """
+    return os.path.join(config_path, V2_MANIFEST_FILENAME)
+
+
+def _v2_view_path(config_path: str) -> str:
+    """
+    计算 V2 项目视图文件的绝对路径。
+
+    参数:
+        config_path: 项目配置根目录
+
+    返回:
+        视图文件的绝对路径
+    """
+    return os.path.join(config_path, V2_VIEW_FILENAME)
+
+
+def _v2_workspaces_path(config_path: str) -> str:
+    """
+    计算 V2 工作区配置文件的绝对路径。
+
+    参数:
+        config_path: 项目配置根目录
+
+    返回:
+        工作区配置文件的绝对路径
+    """
+    return os.path.join(config_path, V2_WORKSPACES_FILENAME)
+
+
+def _resolve_project_path(base_path: str, relative_path: str) -> str:
+    """安全解析项目内的相对路径，防止路径遍历攻击。
+
+    验证规则:
+        - 解析后的绝对路径必须在 base_path 目录内
+        - 拒绝包含 .. 的相对路径组件
+        - B-arch2: 用 Path.resolve() 解析符号链接，防止 symlink 逃逸
+          （原 os.path.abspath 不解析 symlink，若项目目录含指向外部的 symlink
+          子目录，startswith 检查会通过但实际落在项目外）。
+
+    参数:
+        base_path: 项目根目录的绝对路径
+        relative_path: 用户提供的相对路径
+
+    返回:
+        安全的绝对路径（已 resolve，symlink 已展开）
+
+    抛出:
+        ValueError: 当路径试图跳出项目目录时
+    """
+    from pathlib import Path
+
+    base_resolved = Path(base_path).resolve()
+    # 拼接后 resolve：同时处理 .. 与 symlink
+    target_resolved = Path(base_path, relative_path).resolve()
+    target_str = str(target_resolved)
+    base_str = str(base_resolved)
+    # 包含关系：target == base 或 target 在 base 之下
+    if target_str != base_str and not target_str.startswith(base_str + os.sep):
+        raise ValueError(f"路径 traversal 被拒绝: {relative_path}")
+    return target_str
+
+
+@contextmanager
+def project_lock(config_path: str, timeout: float = 10.0) -> Iterator[None]:
+    """获取项目级文件锁，防止并发写操作损坏 manifest 等关键配置文件。
+
+    锁文件存放在 {config_path}/.precis/project.lock，所有写 manifest 的操作共用同一把锁。
+
+    参数:
+        config_path: 项目配置根目录
+        timeout: 获取锁的超时时间（秒），默认 10 秒
+
+    使用示例:
+        with project_lock(config_path):
+            write_yaml_atomic(Path(manifest_path), manifest.model_dump(exclude_none=True))
+    """
+    lock_dir = os.path.join(config_path, ".precis")
+    os.makedirs(lock_dir, exist_ok=True)
+    lock_path = os.path.join(lock_dir, "project.lock")
+    lock = FileLock(lock_path, timeout=timeout)
+    with lock:
+        yield