self-evolve-framework 1.5.0 → 1.6.0

package/template/skills/skillopt-sleep/scripts/framework/skillopt/envs/officeqa/rollout.py

@@ -0,0 +1,799 @@
+from __future__ import annotations
+import json
+import os
+import re
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from skillopt.envs.officeqa.evaluator import evaluate
+from skillopt.envs.officeqa.tool_runtime import (
+    build_oracle_parsed_pages_context,
+    custom_search,
+    resolve_candidate_files,
+    resolve_docs_roots,
+    run_tool,
+)
+from skillopt.model import chat_target_messages, get_target_backend, is_target_exec_backend
+from skillopt.model.codex_harness import prepare_workspace, render_skill_md, run_target_exec
+from skillopt.prompts import load_prompt
+_TOOL_SCHEMAS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "glob",
+            "description": "Find candidate local document files by filename or relative-path glob pattern.",
+            "parameters": {
+                "type": "object",
+                "properties": {"pattern": {"type": "string"}},
+                "required": ["pattern"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "read",
+            "description": "Read a local text document excerpt by path and line window.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "path": {"type": "string"},
+                    "start": {"type": "integer"},
+                    "limit": {"type": "integer"},
+                },
+                "required": ["path"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "grep",
+            "description": "Search a local text document for a literal pattern and return matching lines.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "pattern": {"type": "string"},
+                    "path": {"type": "string"},
+                },
+                "required": ["pattern", "path"],
+            },
+        },
+    },
+]
+_FINAL_RE = re.compile(r"<answer>(.*?)</answer>", re.IGNORECASE | re.DOTALL)
+_SEARCH_RE = re.compile(r"<search_queries>(.*?)</search_queries>", re.IGNORECASE | re.DOTALL)
+_DEFAULT_SEARCH_MODE = "offline"
+_CUSTOM_SEARCH_MODE = "custom_search"
+_AZURE_SEARCH_MODE = "azure_search"
+def _normalize_search_mode(search_mode: str | None) -> str:
+    normalized = str(search_mode or _DEFAULT_SEARCH_MODE).strip().lower()
+    if normalized in {"custom", _CUSTOM_SEARCH_MODE}:
+        return _CUSTOM_SEARCH_MODE
+    if normalized in {"azure", _AZURE_SEARCH_MODE}:
+        return _AZURE_SEARCH_MODE
+    return _DEFAULT_SEARCH_MODE
+def _build_system(
+    skill_content: str,
+    *,
+    search_mode: str = _DEFAULT_SEARCH_MODE,
+    use_local_tools: bool = True,
+    max_tool_turns: int = 12,
+    max_queries_per_turn: int = 4,
+) -> str:
+    if skill_content.strip():
+        skill_section = f"## Skill\n{skill_content.strip()}\n\n"
+    else:
+        skill_section = ""
+    normalized_search_mode = _normalize_search_mode(search_mode)
+    if normalized_search_mode == _AZURE_SEARCH_MODE:
+        return (
+            "You are an expert OfficeQA research assistant. Solve the question using the model's built-in web "
+            "search tool when needed, keep the answer grounded in authoritative evidence, and return the final "
+            "answer inside <answer>...</answer>.\n\n"
+            + skill_section
+        ).rstrip()
+    if normalized_search_mode == _CUSTOM_SEARCH_MODE:
+        protocol = (
+            "You are an expert OfficeQA research assistant. Solve the question using the provided oracle parsed "
+            "OfficeQA page(s) and evidence returned by the controller-managed custom search loop.\n\n"
+            "Search protocol:\n"
+            f"- You have at most {max_tool_turns} model rounds total.\n"
+            f"- On any non-final round, you may either return `<search_queries>[\"query 1\", \"query 2\"]</search_queries>` "
+            f"with up to {max_queries_per_turn} queries, or return `<answer>...</answer>` if you are ready.\n"
+            "- If you request search, do not include an answer in the same response.\n"
+            "- On the final round, you must return `<answer>...</answer>` and must not request more search.\n"
+            "- Base your answer on the returned evidence, reconcile conflicting snippets carefully, and stay concise.\n\n"
+        )
+        return protocol + skill_section + "Return the final answer inside <answer>...</answer> when you are ready."
+    if not use_local_tools:
+        return (
+            "You are an expert OfficeQA research assistant. Solve the question using the provided oracle parsed "
+            "OfficeQA page(s) and source hints. Do not request or assume access to any external search or local "
+            "function tools. Return the final answer inside <answer>...</answer>.\n\n"
+            + skill_section
+        ).rstrip()
+    return load_prompt("rollout_system", env="officeqa").format(skill_section=skill_section)
+def _build_round_instruction(
+    *,
+    turn: int,
+    max_tool_turns: int,
+    max_queries_per_turn: int,
+) -> str:
+    if turn >= max_tool_turns:
+        return (
+            "## Round Policy\n"
+            f"This is the final round ({turn}/{max_tool_turns}). You must return `<answer>...</answer>` now. "
+            "Do not output `<search_queries>`."
+        )
+    remaining_rounds = max_tool_turns - turn
+    return (
+        "## Round Policy\n"
+        f"This is round {turn}/{max_tool_turns}. "
+        f"You may either return `<answer>...</answer>` now, or request up to {max_queries_per_turn} search queries "
+        f"inside `<search_queries>...</search_queries>`. "
+        f"After this response, at most {remaining_rounds} model rounds remain."
+    )
+def _message_debug_metadata(message: object) -> dict:
+    metadata = getattr(message, "metadata", None)
+    if isinstance(metadata, dict):
+        return metadata
+    return {}
+def _build_user(
+    item: dict,
+    candidate_files: list[str] | None = None,
+    *,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    corpus_note: str = "",
+    search_mode: str = _DEFAULT_SEARCH_MODE,
+    turn: int = 1,
+    max_tool_turns: int = 12,
+    max_queries_per_turn: int = 4,
+    oracle_context: str = "",
+) -> str:
+    normalized_search_mode = _normalize_search_mode(search_mode)
+    parts = [f"## Question\n{item['question']}"]
+    if oracle_context.strip():
+        parts.append(f"## Oracle Parsed Pages\n{oracle_context.strip()}")
+    if normalized_search_mode == _DEFAULT_SEARCH_MODE:
+        file_block = "\n".join(f"- {path}" for path in (candidate_files or [])[:20]) or "- none resolved"
+        if corpus_note.strip():
+            parts.append(f"## Document Corpus\n{corpus_note.strip()}")
+        parts.append(f"## Candidate Files\n{file_block}")
+    if item.get("source_docs"):
+        parts.append("## Source Hints\n" + "\n".join(f"- {hint}" for hint in item["source_docs"]))
+    if normalized_search_mode != _DEFAULT_SEARCH_MODE and item.get("source_files"):
+        parts.append("## File Hints\n" + "\n".join(f"- {hint}" for hint in item["source_files"]))
+    if diagnostic_mode and diagnostic_instruction.strip():
+        parts.append(f"## Training Readout\n{diagnostic_instruction.strip()}")
+    if normalized_search_mode == _CUSTOM_SEARCH_MODE:
+        parts.append(
+            _build_round_instruction(
+                turn=turn,
+                max_tool_turns=max_tool_turns,
+                max_queries_per_turn=max_queries_per_turn,
+            )
+        )
+        parts.append(
+            "## Output Format\n"
+            "If you need more evidence, return only `<search_queries>[...]</search_queries>`.\n"
+            "If you are ready to answer, return only `<answer>...</answer>`."
+        )
+        parts.append(
+            "Use only the provided oracle parsed pages and controller-provided custom search evidence. "
+            "Do not rely on any built-in web search capability."
+        )
+    elif normalized_search_mode == _AZURE_SEARCH_MODE:
+        parts.append("Use the model's built-in web search tool when needed. Return the final answer inside <answer>...</answer>.")
+    return "\n\n".join(parts)
+def _extract_answer(text: str) -> str:
+    match = _FINAL_RE.search(text)
+    if match:
+        return match.group(1).strip()
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    return lines[-1] if lines else text.strip()
+def _extract_search_queries(text: str) -> list[str]:
+    match = _SEARCH_RE.search(text or "")
+    if not match:
+        return []
+    raw = match.group(1).strip()
+    if not raw:
+        return []
+    parsed_queries: list[str] = []
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        parsed = None
+    if isinstance(parsed, dict):
+        for key in ("queries", "search_queries", "query"):
+            value = parsed.get(key)
+            if isinstance(value, str) and value.strip():
+                parsed_queries = [value.strip()]
+                break
+            if isinstance(value, list):
+                parsed_queries = [str(item).strip() for item in value if str(item).strip()]
+                break
+    elif isinstance(parsed, list):
+        parsed_queries = [str(item).strip() for item in parsed if str(item).strip()]
+    elif isinstance(parsed, str) and parsed.strip():
+        parsed_queries = [parsed.strip()]
+    if not parsed_queries:
+        raw_lines = [line.strip(" -*\t\r\n\"'") for line in raw.splitlines()]
+        parsed_queries = [line for line in raw_lines if line]
+    if len(parsed_queries) <= 1 and parsed_queries:
+        multi = [part.strip(" \"'") for part in re.split(r"[;,]", parsed_queries[0]) if part.strip(" \"'")]
+        if len(multi) > 1:
+            parsed_queries = multi
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for query in parsed_queries:
+        normalized = query.strip()
+        if not normalized or normalized in seen:
+            continue
+        seen.add(normalized)
+        deduped.append(normalized)
+    return deduped
+def _docs_link_targets(docs_roots: list[str]) -> list[tuple[str, str]]:
+    return [(root, os.path.join("docs", f"root_{idx}")) for idx, root in enumerate(docs_roots, start=1)]
+def _workspace_doc_path(path: str, docs_roots: list[str]) -> str:
+    resolved_path = os.path.realpath(path)
+    for idx, root in enumerate(docs_roots, start=1):
+        resolved_root = os.path.realpath(root)
+        if resolved_path == resolved_root or resolved_path.startswith(resolved_root + os.sep):
+            rel_path = os.path.relpath(resolved_path, resolved_root)
+            return os.path.join("docs", f"root_{idx}", rel_path)
+    return path
+def _build_codex_skill(skill_content: str) -> str:
+    return render_skill_md(
+        skill_content,
+        description="Dynamic ReflACT skill for solving the current OfficeQA local-document question.",
+        preamble=(
+            "Use this skill when answering the current OfficeQA question.\n"
+            "Inspect the provided local document excerpts or files, ground the answer in the evidence,\n"
+            "and return the final answer inside <answer>...</answer>."
+        ),
+    )
+def _run_codex_once(
+    *,
+    pred_dir: str,
+    item: dict,
+    skill_content: str,
+    candidate_files: list[str],
+    docs_roots: list[str],
+    model: str,
+    timeout: int,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+    previous_response: str = "",
+    oracle_context: str = "",
+) -> tuple[str, str, str, str]:
+    rel_files = [_workspace_doc_path(path, docs_roots) for path in candidate_files[:20]]
+    corpus_note = (
+        "The full OfficeQA document corpus is available under `docs/`. "
+        "The candidate files below are source hints or likely starting points; search the full corpus if needed."
+    )
+    user = _build_user(
+        item,
+        rel_files,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        corpus_note=corpus_note,
+        oracle_context=oracle_context,
+    )
+    task_parts = [user]
+    if previous_response:
+        task_parts.append(
+            "## Previous Attempt\n"
+            f"{previous_response}\n\n"
+            "Review the local documents again and correct the answer if needed."
+        )
+    task_text = "\n\n".join(task_parts)
+    skill_md = _build_codex_skill(skill_content)
+    work_dir = os.path.join(pred_dir, "codex_exec")
+    prepare_workspace(
+        work_dir=work_dir,
+        skill_md=skill_md,
+        task_text=task_text,
+        link_dirs=_docs_link_targets(docs_roots),
+    )
+    prompt = (
+        "Use the `skillopt-target` skill available in this workspace.\n"
+        "Read `task.md`, inspect or search the full OfficeQA corpus under `docs/`, and answer the question.\n"
+        "Treat candidate files in `task.md` as hints, not an access limit.\n"
+        "Return the final answer inside <answer>...</answer>."
+    )
+    final_message, raw = run_target_exec(
+        work_dir=work_dir,
+        prompt=prompt,
+        model=model,
+        timeout=timeout,
+        data_dirs=docs_roots,
+    )
+    return final_message or raw, raw, skill_md, task_text
+def _execute_custom_search_round(
+    queries: list[str],
+    *,
+    api_url: str,
+    auth_env: str,
+    provider: str,
+    max_num_results: int,
+    timeout: int,
+) -> str:
+    blocks = []
+    for index, query in enumerate(queries, start=1):
+        try:
+            result = custom_search(
+                query,
+                api_url=api_url,
+                auth_env=auth_env,
+                provider=provider,
+                max_num_results=max_num_results,
+                timeout=timeout,
+            )
+        except Exception as search_error:  # noqa: BLE001
+            result = f"Query: {query}\n\n[search error: {search_error}]"
+        blocks.append(f"## Query {index}\n{result}")
+    return "\n\n".join(blocks)
+def _run_custom_search_process(
+    item: dict,
+    skill_content: str,
+    *,
+    max_tool_turns: int,
+    max_completion_tokens: int,
+    max_queries_per_turn: int,
+    diagnostic_mode: bool,
+    diagnostic_instruction: str,
+    search_api_url: str,
+    search_auth_env: str,
+    search_provider: str,
+    search_max_num_results: int,
+    search_timeout_seconds: int,
+    oracle_context: str = "",
+) -> tuple[str, str, str, str, list[dict], str, dict]:
+    if not str(search_api_url or "").strip():
+        raise ValueError("custom_search mode requires a non-empty search_api_url")
+    if not os.environ.get(search_auth_env, "").strip():
+        raise ValueError(f"custom_search mode requires auth token env var {search_auth_env}")
+    if get_target_backend() not in {"openai_chat", "qwen_chat"}:
+        raise ValueError("custom_search mode is only supported with target_backend='openai_chat' or 'qwen_chat'")
+    system = _build_system(
+        skill_content,
+        search_mode=_CUSTOM_SEARCH_MODE,
+        max_tool_turns=max_tool_turns,
+        max_queries_per_turn=max_queries_per_turn,
+    )
+    initial_user = _build_user(
+        item,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        search_mode=_CUSTOM_SEARCH_MODE,
+        turn=1,
+        max_tool_turns=max_tool_turns,
+        max_queries_per_turn=max_queries_per_turn,
+        oracle_context=oracle_context,
+    )
+    latest_user = initial_user
+    messages: list[dict] = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": initial_user},
+    ]
+    conversation: list[dict] = [{"role": "user", "content": initial_user}]
+    final_response = ""
+    final_answer = ""
+    fail_reason = ""
+    last_response_metadata: dict = {}
+    for turn in range(1, max_tool_turns + 1):
+        message, _ = chat_target_messages(
+            messages=messages,
+            max_completion_tokens=max_completion_tokens,
+            retries=5,
+            stage="rollout",
+            return_message=True,
+        )
+        response = message.content or ""
+        final_response = response
+        last_response_metadata = _message_debug_metadata(message)
+        messages.append({"role": "assistant", "content": response})
+        message_event = {"type": "message", "turn": turn, "content": response}
+        if last_response_metadata:
+            message_event["response_metadata"] = last_response_metadata
+        conversation.append(message_event)
+        if "<answer>" in response.lower():
+            final_answer = _extract_answer(response)
+            return system, latest_user, final_response, final_answer, conversation, "", last_response_metadata
+        if turn == max_tool_turns:
+            fail_reason = f"Final round ({max_tool_turns}) ended without <answer>...</answer>"
+            break
+        queries = _extract_search_queries(response)[:max_queries_per_turn]
+        if not queries:
+            fail_reason = "Model neither produced search queries nor a final answer"
+            break
+        results_text = _execute_custom_search_round(
+            queries,
+            api_url=search_api_url,
+            auth_env=search_auth_env,
+            provider=search_provider,
+            max_num_results=search_max_num_results,
+            timeout=search_timeout_seconds,
+        )
+        conversation.append({"type": "tool_call", "turn": turn, "cmd": f"custom_search({queries!r})", "obs": results_text})
+        latest_user = (
+            f"## Search Results Round {turn}\n{results_text}\n\n"
+            + _build_round_instruction(
+                turn=turn + 1,
+                max_tool_turns=max_tool_turns,
+                max_queries_per_turn=max_queries_per_turn,
+            )
+            + "\n\nFollow the round policy above exactly."
+        )
+        messages.append({"role": "user", "content": latest_user})
+        conversation.append({"role": "user", "turn": turn + 1, "content": latest_user})
+    return system, latest_user, final_response, final_answer, conversation, fail_reason, last_response_metadata
+def _run_azure_search_process(
+    item: dict,
+    skill_content: str,
+    *,
+    max_completion_tokens: int,
+    diagnostic_mode: bool,
+    diagnostic_instruction: str,
+) -> tuple[str, str, str, str, list[dict], str, dict]:
+    if get_target_backend() != "openai_chat":
+        raise ValueError("azure_search mode is only supported with target_backend='openai_chat'")
+    system = _build_system(skill_content, search_mode=_AZURE_SEARCH_MODE)
+    user = _build_user(
+        item,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        search_mode=_AZURE_SEARCH_MODE,
+    )
+    messages = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": user},
+    ]
+    conversation: list[dict] = [{"role": "user", "content": user}]
+    message, _ = chat_target_messages(
+        messages=messages,
+        max_completion_tokens=max_completion_tokens,
+        retries=5,
+        stage="rollout",
+        return_message=True,
+        tools=[{"type": "web_search"}],
+    )
+    response = message.content or ""
+    last_response_metadata = _message_debug_metadata(message)
+    message_event = {"type": "message", "content": response}
+    if last_response_metadata:
+        message_event["response_metadata"] = last_response_metadata
+    conversation.append(message_event)
+    if "<answer>" in response.lower():
+        return system, user, response, _extract_answer(response), conversation, "", last_response_metadata
+    return system, user, response, "", conversation, "Model did not produce a final answer", last_response_metadata
+def _run_offline_no_tools_process(
+    item: dict,
+    skill_content: str,
+    *,
+    max_completion_tokens: int,
+    diagnostic_mode: bool,
+    diagnostic_instruction: str,
+    candidate_files: list[str],
+    oracle_context: str = "",
+) -> tuple[str, str, str, str, list[dict], str, dict]:
+    system = _build_system(skill_content, search_mode=_DEFAULT_SEARCH_MODE, use_local_tools=False)
+    user = _build_user(
+        item,
+        candidate_files,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        search_mode=_DEFAULT_SEARCH_MODE,
+        oracle_context=oracle_context,
+    )
+    messages = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": user},
+    ]
+    conversation: list[dict] = [{"role": "user", "content": user}]
+    message, _ = chat_target_messages(
+        messages=messages,
+        max_completion_tokens=max_completion_tokens,
+        retries=5,
+        stage="rollout",
+        return_message=True,
+    )
+    response = message.content or ""
+    last_response_metadata = _message_debug_metadata(message)
+    message_event = {"type": "message", "content": response}
+    if last_response_metadata:
+        message_event["response_metadata"] = last_response_metadata
+    conversation.append(message_event)
+    if "<answer>" in response.lower():
+        return system, user, response, _extract_answer(response), conversation, "", last_response_metadata
+    return system, user, response, "", conversation, "Model did not produce a final answer", last_response_metadata
+def process_one(
+    item: dict,
+    out_root: str,
+    skill_content: str,
+    *,
+    max_tool_turns: int = 12,
+    max_completion_tokens: int = 16384,
+    search_mode: str = _DEFAULT_SEARCH_MODE,
+    max_queries_per_turn: int = 4,
+    search_api_url: str = "",
+    search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH",
+    search_provider: str = "duckduckgo",
+    search_max_num_results: int = 4,
+    search_timeout_seconds: int = 20,
+    use_local_tools: bool = True,
+    data_dirs: list[str] | str | None = None,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+) -> dict:
+    item_id = str(item["id"])
+    pred_dir = os.path.join(out_root, "predictions", item_id)
+    os.makedirs(pred_dir, exist_ok=True)
+    normalized_search_mode = _normalize_search_mode(search_mode)
+    docs_roots: list[str] = []
+    candidate_files: list[str] = []
+    oracle_context = ""
+    if normalized_search_mode == _DEFAULT_SEARCH_MODE:
+        docs_roots = resolve_docs_roots(data_dirs)
+        candidate_files = resolve_candidate_files(item.get("source_files", []), docs_roots)
+        oracle_context = build_oracle_parsed_pages_context(
+            item.get("source_files", []),
+            item.get("source_docs", []),
+            docs_roots,
+            evidence_note=(
+                "Treat it as primary document evidence and combine it with local document tool evidence when useful."
+                if use_local_tools
+                else "Treat it as primary document evidence for answering the question."
+            ),
+        )
+    elif normalized_search_mode == _CUSTOM_SEARCH_MODE:
+        docs_roots = resolve_docs_roots(data_dirs)
+        if item.get("source_files"):
+            candidate_files = resolve_candidate_files(item.get("source_files", []), docs_roots)
+        oracle_context = build_oracle_parsed_pages_context(
+            item.get("source_files", []),
+            item.get("source_docs", []),
+            docs_roots,
+        )
+    system = _build_system(
+        skill_content,
+        search_mode=normalized_search_mode,
+        use_local_tools=use_local_tools,
+        max_tool_turns=max_tool_turns,
+        max_queries_per_turn=max_queries_per_turn,
+    )
+    user = _build_user(
+        item,
+        candidate_files if normalized_search_mode == _DEFAULT_SEARCH_MODE else None,
+        diagnostic_mode=diagnostic_mode,
+        diagnostic_instruction=diagnostic_instruction,
+        search_mode=normalized_search_mode,
+        max_tool_turns=max_tool_turns,
+        max_queries_per_turn=max_queries_per_turn,
+        oracle_context=oracle_context,
+    )
+    conversation: list[dict] = [{"role": "user", "content": user}]
+    final_response = ""
+    final_answer = ""
+    fail_reason = ""
+    last_response_metadata: dict = {}
+    allowed_files = [os.path.basename(path) for path in candidate_files]
+    try:
+        if normalized_search_mode == _CUSTOM_SEARCH_MODE:
+            system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_custom_search_process(
+                item,
+                skill_content,
+                max_tool_turns=max_tool_turns,
+                max_completion_tokens=max_completion_tokens,
+                max_queries_per_turn=max_queries_per_turn,
+                diagnostic_mode=diagnostic_mode,
+                diagnostic_instruction=diagnostic_instruction,
+                search_api_url=search_api_url,
+                search_auth_env=search_auth_env,
+                search_provider=search_provider,
+                search_max_num_results=search_max_num_results,
+                search_timeout_seconds=search_timeout_seconds,
+                oracle_context=oracle_context,
+            )
+        elif normalized_search_mode == _AZURE_SEARCH_MODE:
+            system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_azure_search_process(
+                item,
+                skill_content,
+                max_completion_tokens=max_completion_tokens,
+                diagnostic_mode=diagnostic_mode,
+                diagnostic_instruction=diagnostic_instruction,
+            )
+        elif not use_local_tools:
+            system, user, final_response, final_answer, conversation, fail_reason, last_response_metadata = _run_offline_no_tools_process(
+                item,
+                skill_content,
+                max_completion_tokens=max_completion_tokens,
+                diagnostic_mode=diagnostic_mode,
+                diagnostic_instruction=diagnostic_instruction,
+                candidate_files=candidate_files,
+                oracle_context=oracle_context,
+            )
+        elif is_target_exec_backend():
+            from skillopt.model import azure_openai as _llm
+            response = ""
+            system = ""
+            user = ""
+            for turn in range(1, max_tool_turns + 1):
+                response, _raw, system, user = _run_codex_once(
+                    pred_dir=pred_dir,
+                    item=item,
+                    skill_content=skill_content,
+                    candidate_files=candidate_files,
+                    docs_roots=docs_roots,
+                    model=_llm.TARGET_DEPLOYMENT,
+                    timeout=180,
+                    diagnostic_mode=diagnostic_mode if turn == 1 else False,
+                    diagnostic_instruction=diagnostic_instruction if turn == 1 else "",
+                    previous_response=response if turn > 1 else "",
+                    oracle_context=oracle_context,
+                )
+                final_response = response
+                conversation.append({"type": "message", "turn": turn, "content": response})
+                if "<answer>" in response.lower():
+                    final_answer = _extract_answer(response)
+                    break
+            if not final_answer:
+                fail_reason = f"Exceeded codex turn budget ({max_tool_turns})"
+            system = system or _build_codex_skill(skill_content)
+            user = user or _build_user(item, [_workspace_doc_path(path, docs_roots) for path in candidate_files])
+        else:
+            messages: list[dict] = [
+                {"role": "system", "content": system},
+                {"role": "user", "content": user},
+            ]
+            for turn in range(1, max_tool_turns + 1):
+                message, _ = chat_target_messages(
+                    messages=messages,
+                    max_completion_tokens=max_completion_tokens,
+                    retries=5,
+                    stage="rollout",
+                    tools=_TOOL_SCHEMAS,
+                    tool_choice="auto",
+                    return_message=True,
+                )
+                response = message.content or ""
+                final_response = response
+                assistant_message = {"role": "assistant", "content": response}
+                if getattr(message, "tool_calls", None):
+                    assistant_message["tool_calls"] = [tool_call.model_dump(mode="json") for tool_call in message.tool_calls]
+                messages.append(assistant_message)
+                conversation.append({"type": "message", "content": response})
+                if getattr(message, "tool_calls", None):
+                    for tool_call in message.tool_calls:
+                        tool_name = tool_call.function.name
+                        arguments = json.loads(tool_call.function.arguments) if tool_call.function.arguments else {}
+                        cmd, obs = run_tool(tool_name, arguments, allowed_roots=docs_roots, allowed_files=allowed_files)
+                        conversation.append({"type": "tool_call", "cmd": cmd, "obs": obs})
+                        messages.append({
+                            "role": "tool",
+                            "tool_call_id": tool_call.id,
+                            "content": obs,
+                        })
+                    continue
+                if "<answer>" in response.lower():
+                    final_answer = _extract_answer(response)
+                    break
+                if turn == max_tool_turns:
+                    fail_reason = f"Exceeded tool-turn budget ({max_tool_turns})"
+                else:
+                    fail_reason = "Model neither produced a tool request nor a final answer"
+                    break
+    except Exception as e:  # noqa: BLE001
+        fail_reason = f"error: {e}"
+    with open(os.path.join(pred_dir, "target_system_prompt.txt"), "w", encoding="utf-8") as f:
+        f.write(system)
+    with open(os.path.join(pred_dir, "target_user_prompt.txt"), "w", encoding="utf-8") as f:
+        f.write(user)
+    with open(os.path.join(pred_dir, "conversation.json"), "w", encoding="utf-8") as f:
+        json.dump(conversation, f, ensure_ascii=False, indent=2)
+    eval_result = evaluate(final_answer, item.get("ground_truth", "")) if final_answer else {"em": 0.0, "f1": 0.0, "predicted_answer": "", "gold_answer": item.get("ground_truth", "")}
+    result = {
+        "id": item_id,
+        "question": item.get("question", ""),
+        "task_type": item.get("task_type", "officeqa"),
+        "task_description": item.get("question", ""),
+        "predicted_answer": eval_result["predicted_answer"],
+        "response": final_response,
+        "ground_truth": item.get("ground_truth", ""),
+        "source_files": item.get("source_files", []),
+        "resolved_source_paths": candidate_files,
+        "oracle_parsed_pages_included": bool(oracle_context),
+        "oracle_parsed_pages_chars": len(oracle_context),
+        "use_local_tools": bool(use_local_tools),
+        "hard": int(eval_result["em"]),
+        "soft": eval_result["f1"],
+        "fail_reason": fail_reason or ("" if eval_result["em"] else f"predicted '{eval_result['predicted_answer']}' but expected '{item.get('ground_truth', '')}'"),
+        "agent_ok": not fail_reason,
+        "n_turns": len(conversation),
+        "last_finish_reason": last_response_metadata.get("finish_reason", ""),
+        "target_system_prompt": system,
+        "target_user_prompt": user,
+    }
+    return result
+def run_batch(
+    items: list[dict],
+    out_root: str,
+    skill_content: str,
+    *,
+    workers: int = 8,
+    max_tool_turns: int = 12,
+    max_completion_tokens: int = 16384,
+    search_mode: str = _DEFAULT_SEARCH_MODE,
+    max_queries_per_turn: int = 4,
+    search_api_url: str = "",
+    search_auth_env: str = "OFFICEQA_CUSTOM_SEARCH_AUTH",
+    search_provider: str = "duckduckgo",
+    search_max_num_results: int = 4,
+    search_timeout_seconds: int = 20,
+    use_local_tools: bool = True,
+    data_dirs: list[str] | str | None = None,
+    diagnostic_mode: bool = False,
+    diagnostic_instruction: str = "",
+) -> list[dict]:
+    results_path = os.path.join(out_root, "results.jsonl")
+    os.makedirs(out_root, exist_ok=True)
+    done_ids: set[str] = set()
+    existing: list[dict] = []
+    if os.path.exists(results_path):
+        with open(results_path, encoding="utf-8") as f:
+            for line in f:
+                try:
+                    row = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                done_ids.add(str(row.get("id")))
+                existing.append(row)
+    pending = [item for item in items if str(item["id"]) not in done_ids]
+    if not pending:
+        return existing
+    total = len(existing) + len(pending)
+    completed = len(existing)
+    correct_count = sum(1 for r in existing if r.get("hard", 0))
+    if existing:
+        print(f"    [rollout] resuming: {completed}/{total} already done", flush=True)
+
+    results = list(existing)
+    with open(results_path, "a", encoding="utf-8") as outf, ThreadPoolExecutor(max_workers=workers) as ex:
+        futs = {
+            ex.submit(
+                process_one,
+                item,
+                out_root,
+                skill_content,
+                max_tool_turns=max_tool_turns,
+                max_completion_tokens=max_completion_tokens,
+                search_mode=search_mode,
+                max_queries_per_turn=max_queries_per_turn,
+                search_api_url=search_api_url,
+                search_auth_env=search_auth_env,
+                search_provider=search_provider,
+                search_max_num_results=search_max_num_results,
+                search_timeout_seconds=search_timeout_seconds,
+                use_local_tools=use_local_tools,
+                data_dirs=data_dirs,
+                diagnostic_mode=diagnostic_mode,
+                diagnostic_instruction=diagnostic_instruction,
+            ): item
+            for item in pending
+        }
+        for fut in as_completed(futs):
+            res = fut.result()
+            results.append(res)
+            completed += 1
+            if res.get("hard", 0):
+                correct_count += 1
+            acc = correct_count / completed if completed else 0
+            print(
+                f"    [rollout] {completed}/{total} "
+                f"(acc={acc:.3f}) id={res.get('id', '?')} "
+                f"hard={res.get('hard', '?')}",
+                flush=True,
+            )
+            outf.write(json.dumps(res, ensure_ascii=False) + "\n")
+            outf.flush()
+    return results