jarvis-ai-assistant 0.7.16__py3-none-any.whl → 1.0.2__py3-none-any.whl

jarvis/jarvis_c2rust/library_replacer_checkpoint.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+"""库替换器的检查点管理模块。"""
+
+import json
+import time
+from pathlib import Path
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+
+from jarvis.jarvis_c2rust.constants import DEFAULT_CHECKPOINT_INTERVAL
+from jarvis.jarvis_c2rust.constants import JSON_INDENT
+from jarvis.jarvis_c2rust.library_replacer_utils import normalize_list
+from jarvis.jarvis_c2rust.library_replacer_utils import normalize_list_lower
+
+
+def make_checkpoint_key(
+    sjsonl: Path,
+    library_name: str,
+    llm_group: Optional[str],
+    candidates: Optional[List[str]],
+    disabled_libraries: Optional[List[str]],
+    max_funcs: Optional[int],
+) -> Dict[str, Any]:
+    """构建检查点键"""
+    try:
+        abs_sym = str(Path(sjsonl).resolve())
+    except Exception:
+        abs_sym = str(sjsonl)
+    key: Dict[str, Any] = {
+        "symbols": abs_sym,
+        "library_name": str(library_name),
+        "llm_group": str(llm_group or ""),
+        "candidates": normalize_list(candidates),
+        "disabled_libraries": normalize_list_lower(disabled_libraries),
+        "max_funcs": (
+            int(max_funcs)
+            if isinstance(max_funcs, int)
+            or (isinstance(max_funcs, float) and max_funcs.is_integer())  # type: ignore
+            else None
+        ),
+    }
+    return key
+
+
+def load_checkpoint_if_match(
+    ckpt_path: Path,
+    resume: bool,
+    checkpoint_key: Dict[str, Any],
+) -> Optional[Dict[str, Any]]:
+    """加载匹配的检查点"""
+    try:
+        if not resume:
+            return None
+        if not ckpt_path.exists():
+            return None
+        obj = json.loads(ckpt_path.read_text(encoding="utf-8"))
+        if not isinstance(obj, dict):
+            return None
+        if obj.get("key") != checkpoint_key:
+            return None
+        return obj
+    except Exception:
+        return None
+
+
+def atomic_write(path: Path, content: str) -> None:
+    """原子写入文件"""
+    try:
+        tmp = path.with_suffix(path.suffix + ".tmp")
+        tmp.write_text(content, encoding="utf-8")
+        tmp.replace(path)
+    except Exception:
+        try:
+            path.write_text(content, encoding="utf-8")
+        except Exception:
+            pass
+
+
+def create_checkpoint_state(
+    checkpoint_key: Dict[str, Any],
+    eval_counter: int,
+    processed_roots: set,
+    pruned_dynamic: set,
+    selected_roots: list,
+) -> Dict[str, Any]:
+    """创建检查点状态字典"""
+    try:
+        ts = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime())
+    except Exception:
+        ts = ""
+    return {
+        "key": checkpoint_key,
+        "eval_counter": eval_counter,
+        "processed_roots": sorted(list(processed_roots)),
+        "pruned_dynamic": sorted(list(pruned_dynamic)),
+        "selected_roots": [{"fid": fid, "res": res} for fid, res in selected_roots],
+        "timestamp": ts,
+    }
+
+
+def periodic_checkpoint_save(
+    ckpt_path: Path,
+    checkpoint_state: Dict[str, Any],
+    eval_counter: int,
+    last_ckpt_saved: int,
+    checkpoint_interval: int,
+    resume: bool,
+) -> int:
+    """
+    定期保存检查点。
+
+    Returns:
+        更新后的 last_ckpt_saved 值
+    """
+    if not resume:
+        return last_ckpt_saved
+    try:
+        interval = int(checkpoint_interval)
+    except Exception:
+        interval = DEFAULT_CHECKPOINT_INTERVAL
+    need_save = (interval <= 0) or ((eval_counter - last_ckpt_saved) >= interval)
+    if not need_save:
+        return last_ckpt_saved
+    try:
+        atomic_write(
+            ckpt_path,
+            json.dumps(checkpoint_state, ensure_ascii=False, indent=JSON_INDENT),
+        )
+        return eval_counter
+    except Exception:
+        return last_ckpt_saved

jarvis/jarvis_c2rust/library_replacer_llm.py

@@ -0,0 +1,287 @@
+# -*- coding: utf-8 -*-
+"""库替换器的 LLM 模型创建和评估模块。"""
+
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Tuple
+
+from jarvis.jarvis_utils.output import PrettyOutput
+
+from jarvis.jarvis_c2rust.constants import MAX_LLM_RETRIES
+from jarvis.jarvis_c2rust.library_replacer_prompts import build_subtree_prompt
+
+
+def check_llm_availability() -> tuple[bool, Any, Any, Any]:
+    """检查LLM可用性，返回(是否可用, PlatformRegistry, get_smart_platform_name, get_smart_model_name)
+    使用smart平台，适用于代码生成等复杂场景
+    """
+    try:
+        from jarvis.jarvis_platform.registry import PlatformRegistry
+        from jarvis.jarvis_utils.config import get_smart_model_name
+        from jarvis.jarvis_utils.config import get_smart_platform_name
+
+        return True, PlatformRegistry, get_smart_platform_name, get_smart_model_name
+    except Exception:
+        return False, None, None, None
+
+
+def create_llm_model(
+    llm_group: Optional[str],
+    disabled_display: str,
+    model_available: bool,
+    PlatformRegistry: Any,
+    get_smart_platform_name: Any,
+    get_smart_model_name: Any,
+) -> Optional[Any]:
+    """创建LLM模型，使用smart平台，适用于代码生成等复杂场景"""
+    if not model_available:
+        return None
+    try:
+        registry = PlatformRegistry.get_global_platform_registry()
+        model = None
+        if llm_group:
+            try:
+                platform_name = get_smart_platform_name(llm_group)
+                if platform_name:
+                    model = registry.create_platform(platform_name)
+            except Exception:
+                model = None
+        if model is None:
+            model = registry.get_smart_platform()
+        try:
+            model.set_model_group(llm_group)
+        except Exception:
+            pass
+        if llm_group:
+            try:
+                mn = get_smart_model_name(llm_group)
+                if mn:
+                    model.set_model_name(mn)
+            except Exception:
+                pass
+        model.set_system_prompt(
+            "你是资深 C→Rust 迁移专家。任务：给定一个函数及其调用子树（依赖图摘要、函数签名、源码片段），"
+            "判断是否可以使用一个或多个成熟的 Rust 库整体替代该子树的功能（允许库内多个 API 协同，允许多个库组合；不允许使用不成熟/不常见库）。"
+            "如可替代，请给出 libraries 列表（库名），可选给出代表性 API/模块与实现备注 notes（如何用这些库协作实现）。"
+            "输出格式：仅输出一个 <SUMMARY> 块，块内直接包含 JSON 对象（不需要额外的标签），字段: replaceable(bool), libraries(list[str]), confidence(float 0..1)，可选 library(str,首选主库), api(str) 或 apis(list)，notes(str)。"
+        )
+        return model
+    except Exception as e:
+        PrettyOutput.auto_print(
+            f"⚠️ [c2rust-library] 初始化 LLM 平台失败，将回退为保守策略: {e}"
+        )
+        return None
+
+
+def parse_agent_json_summary(
+    text: str,
+) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
+    """
+    解析Agent返回的JSON摘要
+    返回(解析结果, 错误信息)
+    如果解析成功，返回(data, None)
+    如果解析失败，返回(None, 错误信息)
+    """
+    if not isinstance(text, str) or not text.strip():
+        return None, "摘要文本为空"
+    import re as _re
+
+    from jarvis.jarvis_utils.jsonnet_compat import loads as _json_loads
+
+    # 提取 <SUMMARY> 块
+    m_sum = _re.search(r"<SUMMARY>([\s\S]*?)</SUMMARY>", text, flags=_re.IGNORECASE)
+    block = (m_sum.group(1) if m_sum else text).strip()
+
+    if not block:
+        return None, "未找到 <SUMMARY> 或 </SUMMARY> 标签，或标签内容为空"
+
+    # 直接解析 <SUMMARY> 块内的内容为 JSON
+    # jsonnet_compat.loads 会自动处理 markdown 代码块标记（如 ```json5、```json、``` 等）
+    try:
+        data = _json_loads(block)
+        if isinstance(data, dict):
+            return data, None
+        return None, f"JSON 解析结果不是字典，而是 {type(data).__name__}"
+    except Exception as json_err:
+        return None, f"JSON 解析失败: {str(json_err)}"
+
+
+def llm_evaluate_subtree(
+    fid: int,
+    desc: set,
+    by_id: Dict[int, Dict[str, Any]],
+    adj_func: Dict[int, List[int]],
+    disabled_norm: List[str],
+    disabled_display: str,
+    model_available: bool,
+    new_model_func: Callable[[], Optional[Any]],
+    additional_notes: str = "",
+) -> Dict[str, Any]:
+    """使用LLM评估子树是否可替代，支持最多3次重试"""
+    if not model_available:
+        return {"replaceable": False}
+    model = new_model_func()
+    if not model:
+        return {"replaceable": False}
+
+    base_prompt = build_subtree_prompt(
+        fid, desc, by_id, adj_func, disabled_display, additional_notes
+    )
+    last_parse_error = None
+
+    for attempt in range(1, MAX_LLM_RETRIES + 1):
+        try:
+            # 构建当前尝试的提示词
+            if attempt == 1:
+                prompt = base_prompt
+            else:
+                # 重试时包含之前的错误信息
+                error_hint = ""
+                if last_parse_error:
+                    error_hint = (
+                        f"\n\n**格式错误详情（请根据以下错误修复输出格式）：**\n- {last_parse_error}\n\n"
+                        + "请确保输出的JSON格式正确，包括正确的引号、逗号、大括号等。仅输出一个 <SUMMARY> 块，块内直接包含 JSON 对象（不需要额外的标签）。"
+                    )
+                prompt = base_prompt + error_hint
+
+            # 调用LLM
+            result = model.chat_until_success(prompt)
+            parsed, parse_error = parse_agent_json_summary(result or "")
+
+            if parse_error:
+                # JSON解析失败，记录错误并准备重试
+                last_parse_error = parse_error
+                PrettyOutput.auto_print(
+                    f"⚠️ [c2rust-library] 第 {attempt}/{MAX_LLM_RETRIES} 次尝试：JSON解析失败: {parse_error}"
+                )
+                # 打印原始内容以便调试
+                result_text = str(result or "").strip()
+                if result_text:
+                    PrettyOutput.auto_print(
+                        f"📄 [c2rust-library] 原始LLM响应内容（前1000字符）:\n{result_text[:1000]}"
+                    )
+                    if len(result_text) > 1000:
+                        PrettyOutput.auto_print(
+                            f"📄 [c2rust-library] ... (还有 {len(result_text) - 1000} 个字符未显示)"
+                        )
+                if attempt < MAX_LLM_RETRIES:
+                    continue  # 继续重试
+                else:
+                    # 最后一次尝试也失败，使用默认值
+                    PrettyOutput.auto_print(
+                        f"⚠️ [c2rust-library] 重试 {MAX_LLM_RETRIES} 次后JSON解析仍然失败: {parse_error}，使用默认值"
+                    )
+                    return {"replaceable": False}
+
+            # 解析成功，检查是否为字典
+            if not isinstance(parsed, dict):
+                last_parse_error = f"解析结果不是字典，而是 {type(parsed).__name__}"
+                PrettyOutput.auto_print(
+                    f"⚠️ [c2rust-library] 第 {attempt}/{MAX_LLM_RETRIES} 次尝试：{last_parse_error}"
+                )
+                # 打印解析结果和原始内容以便调试
+                PrettyOutput.auto_print(
+                    f"📊 [c2rust-library] 解析结果类型: {type(parsed).__name__}, 值: {repr(parsed)[:500]}"
+                )
+                result_text = str(result or "").strip()
+                if result_text:
+                    PrettyOutput.auto_print(
+                        f"📄 [c2rust-library] 原始LLM响应内容（前1000字符）:\n{result_text[:1000]}"
+                    )
+                if attempt < MAX_LLM_RETRIES:
+                    continue  # 继续重试
+                else:
+                    PrettyOutput.auto_print(
+                        f"🔴 [c2rust-library] 重试 {MAX_LLM_RETRIES} 次后结果格式仍然不正确，视为不可替代。"
+                    )
+                    return {"replaceable": False}
+
+            # 成功解析为字典，处理结果
+            rep = bool(parsed.get("replaceable") is True)
+            lib = str(parsed.get("library") or "").strip()
+            api = str(parsed.get("api") or parsed.get("function") or "").strip()
+            apis = parsed.get("apis")
+            libs_raw = parsed.get("libraries")
+            notes = str(parsed.get("notes") or "").strip()
+            # 归一化 libraries
+            libraries: List[str] = []
+            if isinstance(libs_raw, list):
+                libraries = [str(x).strip() for x in libs_raw if str(x).strip()]
+            elif isinstance(libs_raw, str):
+                libraries = [s.strip() for s in libs_raw.split(",") if s.strip()]
+            conf = parsed.get("confidence")
+            try:
+                conf = float(conf) if conf is not None else 0.0
+            except Exception:
+                conf = 0.0
+            # 不强制要求具体 API 或特定库名；若缺省且存在 library 字段，则纳入 libraries
+            if not libraries and lib:
+                libraries = [lib]
+
+            # 禁用库命中时，强制视为不可替代
+            if disabled_norm:
+                libs_lower = [lib_name.lower() for lib_name in libraries]
+                lib_single_lower = lib.lower() if lib else ""
+                banned_hit = any(
+                    lower_lib in disabled_norm for lower_lib in libs_lower
+                ) or (lib_single_lower and lib_single_lower in disabled_norm)
+                if banned_hit:
+                    rep = False
+                    warn_libs = (
+                        ", ".join(sorted(set([lib] + libraries)))
+                        if (libraries or lib)
+                        else "(未提供库名)"
+                    )
+                    root_rec = by_id.get(fid, {})
+                    root_name = (
+                        root_rec.get("qualified_name")
+                        or root_rec.get("name")
+                        or f"sym_{fid}"
+                    )
+                    PrettyOutput.auto_print(
+                        f"🚫 [c2rust-library] 评估结果包含禁用库，强制判定为不可替代: {root_name} | 命中库: {warn_libs}"
+                    )
+                    if notes:
+                        notes = notes + f" | 禁用库命中: {warn_libs}"
+                    else:
+                        notes = f"禁用库命中: {warn_libs}"
+
+            result_obj: Dict[str, Any] = {
+                "replaceable": rep,
+                "library": lib,
+                "libraries": libraries,
+                "api": api,
+                "confidence": conf,
+            }
+            if isinstance(apis, list):
+                result_obj["apis"] = apis
+            if notes:
+                result_obj["notes"] = notes
+
+            # 成功获取结果，返回
+            if attempt > 1:
+                PrettyOutput.auto_print(
+                    f"✅ [c2rust-library] 第 {attempt} 次尝试成功获取评估结果"
+                )
+            return result_obj
+
+        except Exception as e:
+            # LLM调用异常，记录并准备重试
+            last_parse_error = f"LLM调用异常: {str(e)}"
+            PrettyOutput.auto_print(
+                f"⚠️ [c2rust-library] 第 {attempt}/{MAX_LLM_RETRIES} 次尝试：LLM评估失败: {e}"
+            )
+            if attempt < MAX_LLM_RETRIES:
+                continue  # 继续重试
+            else:
+                # 最后一次尝试也失败，返回默认值
+                PrettyOutput.auto_print(
+                    f"🔴 [c2rust-library] 重试 {MAX_LLM_RETRIES} 次后LLM评估仍然失败: {e}，视为不可替代"
+                )
+                return {"replaceable": False}
+
+    # 理论上不会到达这里，但作为保险
+    return {"replaceable": False}

jarvis/jarvis_c2rust/library_replacer_loader.py

@@ -0,0 +1,191 @@
+# -*- coding: utf-8 -*-
+"""库替换器的符号表加载和图构建模块。"""
+
+import json
+from pathlib import Path
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Set
+
+from jarvis.jarvis_utils.output import PrettyOutput
+from jarvis.jarvis_c2rust.scanner import find_root_function_ids
+
+
+def load_symbols(
+    sjsonl: Path,
+) -> tuple[
+    List[Dict[str, Any]],
+    Dict[int, Dict[str, Any]],
+    Dict[str, int],
+    Set[int],
+    Dict[int, List[str]],
+]:
+    """加载符号表，返回(所有记录, id到记录映射, 名称到id映射, 函数id集合, id到引用名称映射)"""
+    all_records: List[Dict[str, Any]] = []
+    by_id: Dict[int, Dict[str, Any]] = {}
+    name_to_id: Dict[str, int] = {}
+    func_ids: Set[int] = set()
+    id_refs_names: Dict[int, List[str]] = {}
+
+    with open(sjsonl, "r", encoding="utf-8") as f:
+        idx = 0
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                obj = json.loads(line)
+            except Exception:
+                continue
+            idx += 1
+            fid = int(obj.get("id") or idx)
+            obj["id"] = fid
+            nm = obj.get("name") or ""
+            qn = obj.get("qualified_name") or ""
+            cat = obj.get("category") or ""  # "function" | "type"
+            refs = obj.get("ref")
+            if not isinstance(refs, list):
+                refs = []
+            refs = [r for r in refs if isinstance(r, str) and r]
+
+            all_records.append(obj)
+            by_id[fid] = obj
+            id_refs_names[fid] = refs
+            if nm:
+                name_to_id.setdefault(nm, fid)
+            if qn:
+                name_to_id.setdefault(qn, fid)
+            if cat == "function":
+                func_ids.add(fid)
+
+    return all_records, by_id, name_to_id, func_ids, id_refs_names
+
+
+def build_function_graph(
+    func_ids: Set[int],
+    id_refs_names: Dict[int, List[str]],
+    name_to_id: Dict[str, int],
+) -> Dict[int, List[int]]:
+    """构建函数依赖图，返回id到依赖id列表的映射"""
+    adj_func: Dict[int, List[int]] = {}
+    for fid in func_ids:
+        internal: List[int] = []
+        for target in id_refs_names.get(fid, []):
+            tid = name_to_id.get(target)
+            if tid is not None and tid in func_ids and tid != fid:
+                internal.append(tid)
+        try:
+            internal = list(dict.fromkeys(internal))
+        except Exception:
+            internal = sorted(list(set(internal)))
+        adj_func[fid] = internal
+    return adj_func
+
+
+def build_evaluation_order(
+    sjsonl: Path,
+    func_ids: Set[int],
+    adj_func: Dict[int, List[int]],
+) -> List[int]:
+    """构建评估顺序（广度优先，父先子后）"""
+    # 评估队列：从所有无入边函数作为种子开始，按层次遍历整个图，使"父先于子"被评估；
+    # 若不存在无入边节点（如强连通环），则回退为全量函数集合。
+    try:
+        roots_all = find_root_function_ids(sjsonl)
+    except Exception:
+        roots_all = []
+    seeds = [rid for rid in roots_all if rid in func_ids]
+    if not seeds:
+        seeds = sorted(list(func_ids))
+
+    visited: Set[int] = set()
+    order: List[int] = []
+    q: List[int] = list(seeds)
+    qi = 0
+    while qi < len(q):
+        u = q[qi]
+        qi += 1
+        if u in visited or u not in func_ids:
+            continue
+        visited.add(u)
+        order.append(u)
+        for v in adj_func.get(u, []):
+            if v not in visited and v in func_ids:
+                q.append(v)
+    # 若存在未覆盖的孤立/循环组件，补充其节点（确保每个函数节点都将被作为"候选根"参与评估）
+    if len(visited) < len(func_ids):
+        leftovers = [fid for fid in sorted(func_ids) if fid not in visited]
+        order.extend(leftovers)
+
+    return order
+
+
+def collect_descendants(
+    start: int,
+    adj_func: Dict[int, List[int]],
+    desc_cache: Dict[int, Set[int]],
+) -> Set[int]:
+    """收集从start开始的所有后代节点（使用缓存）"""
+    if start in desc_cache:
+        return desc_cache[start]
+    visited: Set[int] = set()
+    stack: List[int] = [start]
+    visited.add(start)
+    while stack:
+        u = stack.pop()
+        for v in adj_func.get(u, []):
+            if v not in visited:
+                visited.add(v)
+                stack.append(v)
+    desc_cache[start] = visited
+    return visited
+
+
+def process_candidate_scope(
+    candidates: Optional[List[str]],
+    all_records: List[Dict[str, Any]],
+    root_funcs: List[int],
+    func_ids: Set[int],
+    adj_func: Dict[int, List[int]],
+    desc_cache: Dict[int, Set[int]],
+) -> tuple[List[int], Set[int]]:
+    """处理候选根和作用域，返回(过滤后的根函数列表, 不可达函数集合)"""
+
+    scope_unreachable_funcs: Set[int] = set()
+    if not candidates:
+        return root_funcs, scope_unreachable_funcs
+
+    cand_ids: Set[int] = set()
+    # 支持重载：同名/同限定名可能对应多个函数ID，需全部纳入候选
+    key_set = set(candidates)
+    for rec in all_records:
+        if (rec.get("category") or "") != "function":
+            continue
+        nm = rec.get("name") or ""
+        qn = rec.get("qualified_name") or ""
+        if nm in key_set or qn in key_set:
+            try:
+                rec_id = rec.get("id")
+                if rec_id is not None:
+                    cand_ids.add(int(rec_id))
+            except Exception:
+                continue
+
+    if not cand_ids:
+        return root_funcs, scope_unreachable_funcs
+
+    filtered_roots = [rid for rid in root_funcs if rid in cand_ids]
+    # 计算从候选根出发的可达函数集合（含根）
+    reachable_all: Set[int] = set()
+    for rid in filtered_roots:
+        reachable_all.update(collect_descendants(rid, adj_func, desc_cache))
+    # 不可达函数（仅限函数类别）将被直接删除
+    scope_unreachable_funcs = {fid for fid in func_ids if fid not in reachable_all}
+    if scope_unreachable_funcs:
+        PrettyOutput.auto_print(
+            f"⚠️ [c2rust-library] 根据根列表，标记不可达函数删除: {len(scope_unreachable_funcs)} 个"
+        )
+
+    return filtered_roots, scope_unreachable_funcs