hidevs-eval 0.1.1__tar.gz → 0.1.2__tar.gz

{hidevs_eval-0.1.1 → hidevs_eval-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hidevs-eval
-Version: 0.1.1
+Version: 0.1.2
 Summary: HiDevs quest eval client. Runs the local agent and uploads replies for scoring.
 Requires-Python: >=3.11
 Requires-Dist: pyyaml>=6

{hidevs_eval-0.1.1 → hidevs_eval-0.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hidevs-eval"
-version = "0.1.1"
+version = "0.1.2"
 description = "HiDevs quest eval client. Runs the local agent and uploads replies for scoring."
 requires-python = ">=3.11"
 dependencies = [

{hidevs_eval-0.1.1 → hidevs_eval-0.1.2}/src/hidevs_eval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hidevs-eval
-Version: 0.1.1
+Version: 0.1.2
 Summary: HiDevs quest eval client. Runs the local agent and uploads replies for scoring.
 Requires-Python: >=3.11
 Requires-Dist: pyyaml>=6

{hidevs_eval-0.1.1 → hidevs_eval-0.1.2}/src/hidevs_eval.egg-info/SOURCES.txt

@@ -8,4 +8,5 @@ src/hidevs_eval.egg-info/requires.txt
 src/hidevs_eval.egg-info/top_level.txt
 src/hidevs_quest/__init__.py
 src/hidevs_quest/cli.py
+src/hidevs_quest/graphify.py
 src/hidevs_quest/runner.py

hidevs_eval-0.1.2/src/hidevs_quest/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.2"

{hidevs_eval-0.1.1 → hidevs_eval-0.1.2}/src/hidevs_quest/cli.py

@@ -273,10 +273,10 @@ def evaluate(args: argparse.Namespace) -> None:
         if started.get("result"):
             reports.append(started["result"])
             continue
-        results = run_cases(config, root, started["cases"])
+        results, ast_graph = run_cases(config, root, started["cases"])
         reports.append(api(
             f"{session['server']}/v1/runs/{started['run_id']}",
-            {"results": results},
+            {"results": results, "ast_graph": ast_graph},
             session["token"],
         ))
     print_result(reports)

hidevs_eval-0.1.2/src/hidevs_quest/graphify.py

@@ -0,0 +1,782 @@
+"""Graphify: anti-gaming AST call/dataflow graph + runtime execution/argument tracing."""
+
+from __future__ import annotations
+
+import ast
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+
+# ==============================================================================
+# Strategy A — Static AST Call Graph + Argument / Dataflow Integrity
+# ==============================================================================
+
+
+def _call_name(node: ast.AST) -> str | None:
+    if isinstance(node, ast.Name):
+        return node.id
+    if isinstance(node, ast.Attribute):
+        return node.attr
+    return None
+
+
+def _is_literal(node: ast.AST) -> bool:
+    if isinstance(node, ast.Constant):
+        return True
+    if isinstance(node, (ast.List, ast.Tuple, ast.Set)):
+        return all(_is_literal(elt) for elt in node.elts)
+    if isinstance(node, ast.Dict):
+        return all(
+            (k is None or _is_literal(k)) and _is_literal(v)
+            for k, v in zip(node.keys, node.values)
+        )
+    if isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)):
+        return _is_literal(node.operand)
+    return False
+
+
+def _names_used(node: ast.AST) -> set[str]:
+    return {n.id for n in ast.walk(node) if isinstance(n, ast.Name)}
+
+
+class ASTGraphVisitor(ast.NodeVisitor):
+    def __init__(self, file_path: str):
+        self.file_path = file_path
+        self.current_function: str | None = None
+        self.functions: dict[str, dict[str, Any]] = {}
+        self.calls: list[dict[str, Any]] = []
+        self.tool_call_sites: list[dict[str, Any]] = []
+        self.hardcoded_trajectory_sites: list[dict[str, Any]] = []
+
+    def visit_FunctionDef(self, node: ast.FunctionDef):
+        old = self.current_function
+        self.current_function = f"{self.file_path}:{node.name}"
+        params = [a.arg for a in node.args.args if a.arg not in {"self", "cls"}]
+        self.functions[self.current_function] = {
+            "name": node.name,
+            "params": params,
+            "file": self.file_path,
+            "lineno": node.lineno,
+            "calls": [],
+        }
+
+        for child in ast.walk(node):
+            if isinstance(child, ast.Return) and isinstance(child.value, ast.Dict):
+                self._inspect_return_dict(child.value, node.name)
+
+        self.generic_visit(node)
+        self.current_function = old
+
+    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef):
+        self.visit_FunctionDef(node)  # type: ignore[arg-type]
+
+    def _inspect_return_dict(self, value: ast.Dict, fn_name: str) -> None:
+        for key, val in zip(value.keys, value.values):
+            key_name = key.value if isinstance(key, ast.Constant) else None
+            if key_name != "trajectory":
+                continue
+            if isinstance(val, ast.List) and val.elts and all(
+                isinstance(elt, ast.Constant) and isinstance(elt.value, str) for elt in val.elts
+            ):
+                self.hardcoded_trajectory_sites.append({
+                    "function": fn_name,
+                    "file": self.file_path,
+                    "lineno": getattr(val, "lineno", 0),
+                    "values": [elt.value for elt in val.elts],  # type: ignore[union-attr]
+                })
+
+    def visit_Call(self, node: ast.Call):
+        if not self.current_function:
+            self.generic_visit(node)
+            return
+
+        func_name = _call_name(node.func)
+        if not func_name:
+            self.generic_visit(node)
+            return
+
+        arg_nodes = list(node.args) + [kw.value for kw in node.keywords]
+        literal_args = sum(1 for a in arg_nodes if _is_literal(a))
+        dynamic_names = set()
+        for a in arg_nodes:
+            if not _is_literal(a):
+                dynamic_names |= _names_used(a)
+
+        site = {
+            "caller": self.current_function,
+            "callee_short": func_name,
+            "lineno": getattr(node, "lineno", 0),
+            "arg_count": len(arg_nodes),
+            "literal_arg_count": literal_args,
+            "all_args_literal": bool(arg_nodes) and literal_args == len(arg_nodes),
+            "dynamic_names": sorted(dynamic_names),
+            "literal_values": [
+                getattr(a, "value", None) for a in arg_nodes if isinstance(a, ast.Constant)
+            ],
+        }
+        self.calls.append(site)
+        self.tool_call_sites.append(site)
+        self.functions[self.current_function]["calls"].append(func_name)
+        self.generic_visit(node)
+
+
+def build_ast_graph(
+    sources: dict[str, str],
+    entry_spec: str = "",
+    tools: list[dict] | None = None,
+) -> dict[str, Any]:
+    """Build AST call graph and flag hardcoded tool args / fake trajectories."""
+    all_functions: dict[str, dict[str, Any]] = {}
+    all_calls: list[dict[str, Any]] = []
+    all_sites: list[dict[str, Any]] = []
+    hardcoded_trajectories: list[dict[str, Any]] = []
+
+    for path, code in sources.items():
+        try:
+            tree = ast.parse(code)
+        except SyntaxError:
+            continue
+        visitor = ASTGraphVisitor(path)
+        visitor.visit(tree)
+        all_functions.update(visitor.functions)
+        all_calls.extend(visitor.calls)
+        all_sites.extend(visitor.tool_call_sites)
+        hardcoded_trajectories.extend(visitor.hardcoded_trajectory_sites)
+
+    nodes = list(all_functions.keys())
+    edges = []
+    for call in all_calls:
+        caller = call["caller"]
+        callee_short = call["callee_short"]
+        matched = [fn for fn in nodes if fn.endswith(f":{callee_short}")]
+        if matched:
+            for target in matched:
+                edges.append({"source": caller, "target": target, "type": "call"})
+        else:
+            edges.append({"source": caller, "target": callee_short, "type": "external_call"})
+
+    entry_node = None
+    entry_params: list[str] = []
+    if entry_spec:
+        parts = entry_spec.split(":")
+        method = None
+        # Prefer explicit method from config if present in entry_spec class form:
+        # "module:Class" — method resolved separately by caller via tools + "run" fallback.
+        fn_candidates = [parts[-1], "run", "execute", "call", "handle"]
+        for candidate in fn_candidates:
+            for fn_key in nodes:
+                if fn_key.endswith(f":{candidate}"):
+                    entry_node = fn_key
+                    entry_params = list(all_functions[fn_key].get("params") or [])
+                    break
+            if entry_node:
+                break
+
+    reachable: set[str] = set()
+    if entry_node:
+        queue = [entry_node]
+        reachable.add(entry_node)
+        while queue:
+            curr = queue.pop(0)
+            for edge in edges:
+                tgt = edge["target"]
+                if edge["source"] == curr and tgt not in reachable:
+                    reachable.add(tgt)
+                    queue.append(tgt)
+
+    tool_functions: dict[str, str] = {}
+    for tool in tools or []:
+        spec = tool.get("function") or ""
+        cap = tool.get("capability") or spec
+        if ":" in spec:
+            tool_functions[cap] = spec.split(":", 1)[1]
+
+    referenced_names: set[str] = set()
+    for _path, code in sources.items():
+        try:
+            tree = ast.parse(code)
+        except SyntaxError:
+            continue
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Name) and isinstance(getattr(node, "ctx", None), ast.Load):
+                referenced_names.add(node.id)
+
+    tool_reachability: dict[str, Any] = {}
+    hardcoded_tool_args: list[dict[str, Any]] = []
+    input_disconnected_tools: list[dict[str, Any]] = []
+
+    for cap, fn_name in tool_functions.items():
+        matched_tools = [n for n in nodes if n.endswith(f":{fn_name}")]
+        is_reachable = any(n in reachable for n in matched_tools)
+        is_bound = fn_name in referenced_names
+        # LLM agents often dispatch via a tool registry dict; a Load binding counts.
+        tool_reachability[cap] = {
+            "function": fn_name,
+            "exists_in_ast": bool(matched_tools),
+            "reachable_from_entry": is_reachable or (bool(matched_tools) and is_bound),
+            "call_graph_reachable": is_reachable,
+            "name_bound_in_source": is_bound,
+        }
+
+        # Call sites that invoke this tool from entry-reachable callers
+        sites = [
+            s for s in all_sites
+            if s["callee_short"] == fn_name and s["caller"] in reachable
+        ]
+        for site in sites:
+            if site["arg_count"] > 0 and site["all_args_literal"]:
+                hardcoded_tool_args.append({
+                    "capability": cap,
+                    "function": fn_name,
+                    "caller": site["caller"],
+                    "lineno": site["lineno"],
+                    "literal_values": site["literal_values"],
+                })
+            # Numeric/string tool args must touch entry params or derived names when entry has params
+            elif entry_params and site["arg_count"] > 0:
+                dyn = set(site.get("dynamic_names") or [])
+                # Allow derived locals (any non-literal name). Fail only if NO dynamic names
+                # and some literals mixed in (partial hardcode).
+                if site["literal_arg_count"] > 0 and not dyn:
+                    hardcoded_tool_args.append({
+                        "capability": cap,
+                        "function": fn_name,
+                        "caller": site["caller"],
+                        "lineno": site["lineno"],
+                        "literal_values": site["literal_values"],
+                        "reason": "mixed_or_literal_without_dynamic_names",
+                    })
+                # Input disconnection: dynamic names exist but none relate to entry params
+                # and none look like cleaned/parsed derivatives of the input param names.
+                if dyn and entry_params:
+                    related = dyn & set(entry_params)
+                    # Common derivative names still count if they come from assignment
+                    # of entry params — tracked lightly via shared tokens.
+                    if not related:
+                        # Soft signal: if literals dominate, already caught; if only
+                        # unrelated names AND there are numeric literals, flag.
+                        if site["literal_arg_count"] > 0:
+                            input_disconnected_tools.append({
+                                "capability": cap,
+                                "function": fn_name,
+                                "caller": site["caller"],
+                                "lineno": site["lineno"],
+                                "dynamic_names": sorted(dyn),
+                                "entry_params": entry_params,
+                                "literal_values": site["literal_values"],
+                            })
+
+    return {
+        "nodes": nodes,
+        "edges": edges,
+        "entry_node": entry_node,
+        "entry_params": entry_params,
+        "reachable_nodes": list(reachable),
+        "tool_reachability": tool_reachability,
+        "hardcoded_tool_args": hardcoded_tool_args,
+        "input_disconnected_tools": input_disconnected_tools,
+        "hardcoded_trajectory_sites": hardcoded_trajectories,
+        "has_hardcoded_tool_args": bool(hardcoded_tool_args),
+        "has_hardcoded_trajectory": bool(hardcoded_trajectories),
+        "has_hardcoded_literals": bool(hardcoded_tool_args) or bool(hardcoded_trajectories),
+    }
+
+
+# ==============================================================================
+# Strategy B — Runtime Execution Trace + Argument Capture
+# ==============================================================================
+
+LLM_PATH_MARKERS = (
+    "google/genai",
+    "google\\genai",
+    "google/generativeai",
+    "google/ai/generativelanguage",
+    "langchain_google_genai",
+    "langchain_openai",
+    "langchain_anthropic",
+    "langchain_community/chat_models",
+    "langchain_community\\chat_models",
+    "/openai/",
+    "\\openai\\",
+    "/anthropic/",
+    "\\anthropic\\",
+    "litellm",
+    "/lyzr",
+    "\\lyzr",
+)
+
+LLM_METHOD_NAMES = {
+    "generate_content",
+    "generate_content_stream",
+    "create",
+    "complete",
+    "acomplete",
+    "invoke",
+    "ainvoke",
+    "run_agent",
+    "_generate",
+    "_agenerate",
+    "generate",
+}
+
+
+def _extract_model_from_frame(frame) -> str | None:
+    locals_map = frame.f_locals
+    for key in ("model", "model_name", "engine", "model_id"):
+        value = locals_map.get(key)
+        if isinstance(value, str) and value:
+            return value
+    kwargs = locals_map.get("kwargs")
+    if isinstance(kwargs, dict):
+        for key in ("model", "model_name", "engine", "model_id"):
+            value = kwargs.get(key)
+            if isinstance(value, str) and value:
+                return value
+    self_obj = locals_map.get("self")
+    if self_obj is not None:
+        for key in ("model", "model_name", "model_id", "engine"):
+            value = getattr(self_obj, key, None)
+            if isinstance(value, str) and value:
+                return value
+    return None
+
+
+def _is_llm_sdk_file(filename: str) -> bool:
+    if not filename or filename.startswith("<"):
+        return False
+    lowered = filename.replace("\\", "/").lower()
+    return any(marker in lowered for marker in LLM_PATH_MARKERS)
+
+
+class ExecutionTracer:
+    """Record user-code call DAG + LLM SDK invokes via sys.settrace."""
+
+    _active = None
+
+    def __init__(self, root_dir: Path):
+        self.root_dir = root_dir.resolve()
+        self.call_stack: list[dict[str, Any]] = []
+        self.recorded_events: list[dict[str, Any]] = []
+        self.nodes: set[str] = set()
+        self.edges: list[dict[str, str]] = []
+        self.llm_calls: list[dict[str, Any]] = []
+        self.start_time = 0.0
+        self.old_trace = None
+        self._previous_active = None
+
+    def _is_user_file(self, filename: str) -> bool:
+        if not filename or filename.startswith("<") or "importlib" in filename:
+            return False
+        try:
+            p = Path(filename).resolve()
+            return p.is_relative_to(self.root_dir) and not any(
+                part in {".venv", "venv", "__pycache__", ".hidevs", ".eval_artifacts"}
+                for part in p.parts
+            )
+        except (ValueError, OSError):
+            return False
+
+    def trace_calls(self, frame, event, arg):
+        if event not in ("call", "return"):
+            return self.trace_calls
+        code = frame.f_code
+        filename = code.co_filename
+        fn_name = code.co_name
+
+        # Track provider SDK LLM methods even outside user project root
+        if event == "call" and _is_llm_sdk_file(filename):
+            path_l = filename.replace("\\", "/").lower()
+            is_llm_method = fn_name in LLM_METHOD_NAMES or (
+                fn_name == "create"
+                and any(tok in path_l for tok in ("chat/completions", "/messages", "completions"))
+            )
+            if is_llm_method:
+                model = _extract_model_from_frame(frame)
+                self.llm_calls.append({
+                    "method": fn_name,
+                    "file": filename,
+                    "model": model,
+                    "t": time.monotonic() - self.start_time,
+                })
+                node_id = f"llm:{fn_name}"
+                self.nodes.add(node_id)
+                if self.call_stack:
+                    parent = self.call_stack[-1]["node_id"]
+                    edge = {"source": parent, "target": node_id}
+                    if edge not in self.edges:
+                        self.edges.append(edge)
+
+        if not self._is_user_file(filename):
+            return self.trace_calls
+
+        rel = Path(filename).resolve().relative_to(self.root_dir).as_posix()
+        node_id = f"{rel}:{fn_name}"
+
+        if event == "call":
+            self.nodes.add(node_id)
+            if self.call_stack:
+                parent = self.call_stack[-1]["node_id"]
+                edge = {"source": parent, "target": node_id}
+                if edge not in self.edges:
+                    self.edges.append(edge)
+            self.call_stack.append({"node_id": node_id})
+            self.recorded_events.append({"type": "call", "node_id": node_id})
+            # Do NOT count in-project method names as LLM SDK calls.
+            # Real provider calls are captured from SDK file paths above.
+        elif event == "return" and self.call_stack:
+            if self.call_stack[-1]["node_id"] == node_id:
+                self.call_stack.pop()
+                self.recorded_events.append({"type": "return", "node_id": node_id})
+        return self.trace_calls
+
+    def note_tool(self, node_id: str) -> None:
+        """Record a declared tool the harness wrapper actually invoked."""
+        node_id = str(node_id or "")
+        if ":" not in node_id:
+            return
+        self.nodes.add(node_id)
+        self.recorded_events.append({"type": "call", "node_id": node_id})
+
+    def __enter__(self):
+        self.start_time = time.monotonic()
+        self.old_trace = sys.gettrace()
+        self._previous_active = ExecutionTracer._active
+        ExecutionTracer._active = self
+        sys.settrace(self.trace_calls)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        sys.settrace(self.old_trace)
+        ExecutionTracer._active = self._previous_active
+
+    def get_graph(self) -> dict[str, Any]:
+        return {
+            "nodes": list(self.nodes),
+            "edges": self.edges,
+            "call_sequence": [e["node_id"] for e in self.recorded_events if e["type"] == "call"],
+            "total_calls": len(self.edges),
+            "llm_calls": list(self.llm_calls),
+        }
+
+
+def _jsonable(value: Any) -> Any:
+    if isinstance(value, (str, int, float, bool)) or value is None:
+        return value
+    if isinstance(value, (list, tuple)):
+        return [_jsonable(v) for v in value[:20]]
+    if isinstance(value, dict):
+        return {str(k): _jsonable(v) for k, v in list(value.items())[:20]}
+    return repr(value)[:200]
+
+
+def summarize_tool_args(tool_calls: dict[str, list[dict]]) -> dict[str, Any]:
+    """Detect static / input-independent tool arguments across cases."""
+    static_caps: list[str] = []
+    details: dict[str, Any] = {}
+    for cap, rows in tool_calls.items():
+        if len(rows) < 2:
+            details[cap] = {"cases": len(rows), "static_across_cases": False}
+            continue
+        signatures = [row.get("signature") for row in rows]
+        static = len(set(signatures)) == 1 and all(signatures)
+        if static:
+            static_caps.append(cap)
+        details[cap] = {
+            "cases": len(rows),
+            "static_across_cases": static,
+            "signatures": signatures[:5],
+        }
+    return {"static_capabilities": static_caps, "by_capability": details}
+
+
+# ==============================================================================
+# Graph Matcher — fail closed on gaming patterns
+# ==============================================================================
+
+
+def verify_graphify(
+    ast_graph: dict,
+    runtime_graph: dict,
+    required_tools: list[str],
+    tools_config: list[dict] | None = None,
+    case_inputs: list[dict] | None = None,
+) -> dict[str, Any]:
+    """Fail closed: tools must be reachable, executed, input-driven, non-hardcoded."""
+    reasons: list[str] = []
+    checks_passed = True
+
+    tool_fn_map: dict[str, str] = {}
+    for t in tools_config or []:
+        cap = t.get("capability") or t.get("function")
+        func_spec = t.get("function") or ""
+        if ":" in func_spec:
+            tool_fn_map[cap] = func_spec.split(":", 1)[1]
+        elif func_spec:
+            tool_fn_map[cap] = func_spec
+
+    # 1) AST reachability
+    for cap in required_tools:
+        reach = (ast_graph.get("tool_reachability") or {}).get(cap) or {}
+        if not reach.get("reachable_from_entry"):
+            reasons.append(f"AST: required tool not reachable from entry: {cap}")
+            checks_passed = False
+
+    # 2) Hardcoded tool arguments in source
+    hardcoded = ast_graph.get("hardcoded_tool_args") or []
+    if hardcoded:
+        sample = hardcoded[0]
+        reasons.append(
+            "AST: tool called with hardcoded literal arguments "
+            f"({sample.get('capability')}: {sample.get('literal_values')} at line {sample.get('lineno')})"
+        )
+        checks_passed = False
+
+    # 3) Input-disconnected numeric tool calls
+    disconnected = ast_graph.get("input_disconnected_tools") or []
+    if disconnected:
+        sample = disconnected[0]
+        reasons.append(
+            "AST: tool args include literals not derived from entry input "
+            f"({sample.get('capability')}: literals={sample.get('literal_values')})"
+        )
+        checks_passed = False
+
+    # 4) Fake hardcoded trajectory list in return
+    if ast_graph.get("hardcoded_trajectory_sites") or ast_graph.get("has_hardcoded_trajectory"):
+        sites = ast_graph.get("hardcoded_trajectory_sites") or []
+        values = sites[0].get("values") if sites else []
+        reasons.append(
+            f"AST: hardcoded trajectory list detected in return (self-reported fake tool trace): {values}"
+        )
+        checks_passed = False
+
+    # 5) Runtime execution of required tools (graph nodes or harness counts)
+    executed_nodes = [str(node) for node in (runtime_graph.get("nodes") or [])]
+    counts = runtime_graph.get("counts") if isinstance(runtime_graph.get("counts"), dict) else {}
+    missing_execution: list[str] = []
+    for cap in required_tools:
+        fn_name = tool_fn_map.get(cap, cap.split(".")[-1])
+        in_nodes = any(node.endswith(f":{fn_name}") for node in executed_nodes)
+        try:
+            in_counts = int(counts.get(cap) or 0) > 0
+        except (TypeError, ValueError):
+            in_counts = False
+        if not in_nodes and not in_counts:
+            reasons.append(f"Runtime: required tool was not executed: {cap}")
+            checks_passed = False
+            missing_execution.append(cap)
+
+    # 6) Runtime tool args static across differing inputs
+    arg_summary = runtime_graph.get("tool_arg_summary") or {}
+    static_caps = arg_summary.get("static_capabilities") or []
+    if static_caps and case_inputs and len(case_inputs) >= 2:
+        # Only fail when case inputs actually differ
+        input_sigs = [
+            repr(sorted((c.get("input") or c).items())) if isinstance(c, dict) else repr(c)
+            for c in case_inputs
+        ]
+        if len(set(input_sigs)) > 1:
+            reasons.append(
+                "Runtime: tool arguments identical across different case inputs "
+                f"(static gaming): {static_caps}"
+            )
+            checks_passed = False
+
+    # 7) Empty graph only when the harness also recorded no calls
+    if required_tools and not executed_nodes and missing_execution:
+        reasons.append("Runtime: empty execution graph")
+        checks_passed = False
+
+    return {
+        "passed": checks_passed,
+        "score": 1.0 if checks_passed else 0.0,
+        "reasons": reasons,
+        "ast_nodes_count": len(ast_graph.get("nodes") or []),
+        "runtime_nodes_count": len(executed_nodes),
+        "executed_tool_nodes": executed_nodes,
+        "hardcoded_tool_args": hardcoded,
+        "hardcoded_trajectory_sites": ast_graph.get("hardcoded_trajectory_sites") or [],
+        "tool_arg_summary": arg_summary,
+    }
+
+
+# ==============================================================================
+# L1.12 — LLM Integrity (real provider SDK call, not rule-only)
+# ==============================================================================
+
+LLM_PROVIDER_IMPORTS = {
+    "google": ("google.genai", "google.generativeai", "genai", "langchain_google_genai"),
+    "openai": ("openai", "langchain_openai"),
+    "anthropic": ("anthropic", "langchain_anthropic"),
+    "lyzr": ("lyzr",),
+    "litellm": ("litellm",),
+    "other": (),
+}
+
+LLM_CALL_MARKERS = (
+    "generate_content",
+    "generate_content_stream",
+    "chat",
+    "completions",
+    "messages",
+    "complete",
+    "acomplete",
+    "invoke",
+    "ainvoke",
+    "run_agent",
+)
+
+
+def analyze_llm_ast(sources: dict[str, str], llm_claim: dict | None) -> dict[str, Any]:
+    """Static check: declared LLM import/call present; model string referenced."""
+    claim = llm_claim or {}
+    provider = claim.get("provider") or ""
+    model = claim.get("model") or ""
+    import_name = claim.get("import") or ""
+    call_path = claim.get("call") or ""
+    call_tail = call_path.split(".")[-1] if call_path else ""
+
+    import_found = False
+    call_found = False
+    model_literal_found = False
+    provider_sdk_seen = False
+    evidence: list[str] = []
+
+    for path, text in sources.items():
+        if import_name and (f"import {import_name}" in text or f"from {import_name}" in text or f"from {import_name.split('.')[0]}" in text):
+            import_found = True
+            evidence.append(f"import:{path}")
+        for marker in LLM_PROVIDER_IMPORTS.get(provider, ()):
+            if marker and (f"import {marker}" in text or f"from {marker}" in text):
+                provider_sdk_seen = True
+        if call_path and call_path in text:
+            call_found = True
+            evidence.append(f"call_path:{path}")
+        if call_tail and call_tail in text:
+            # weaker signal — method name appears
+            if any(token in text for token in (f".{call_tail}(", f"{call_tail}(")):
+                call_found = True
+                evidence.append(f"call_tail:{path}:{call_tail}")
+        if model and (f'"{model}"' in text or f"'{model}'" in text):
+            model_literal_found = True
+            evidence.append(f"model_literal:{path}")
+        # generic LLM call markers in source
+        for marker in LLM_CALL_MARKERS:
+            if f".{marker}(" in text or f"{marker}(" in text:
+                if provider in {"google", "openai", "anthropic", "lyzr", "litellm", "other"}:
+                    evidence.append(f"marker:{path}:{marker}")
+
+    return {
+        "provider": provider,
+        "model": model,
+        "import_found": import_found or provider_sdk_seen,
+        "call_found": call_found,
+        "model_literal_found": model_literal_found,
+        "provider_sdk_seen": provider_sdk_seen,
+        "evidence": evidence[:20],
+    }
+
+
+def verify_llm_integrity(
+    sources: dict[str, str],
+    config: dict,
+    runtime_graph: dict,
+    suite: dict | None = None,
+) -> dict[str, Any]:
+    """Fail if no real LLM declaration / AST usage / runtime LLM invoke."""
+    reasons: list[str] = []
+    checks_passed = True
+    suite = suite or {}
+    require = suite.get("required_llm", True)
+
+    llm_claim = ((config.get("requirements") or {}).get("llm")) or {}
+    if require and not llm_claim:
+        reasons.append("requirements.llm is missing (provider, model, import, call required)")
+        checks_passed = False
+        return {
+            "passed": False,
+            "score": 0.0,
+            "reasons": reasons,
+            "ast": {},
+            "runtime_llm_calls": [],
+        }
+
+    ast_info = analyze_llm_ast(sources, llm_claim)
+    if require and not ast_info["import_found"]:
+        reasons.append(
+            f"AST: declared LLM import not found for provider={ast_info['provider']} import={llm_claim.get('import')}"
+        )
+        checks_passed = False
+    if require and not ast_info["call_found"]:
+        reasons.append(
+            f"AST: declared LLM call not found in sources: {llm_claim.get('call')}"
+        )
+        checks_passed = False
+
+    runtime_calls = list(runtime_graph.get("llm_calls") or [])
+    provider = (llm_claim.get("provider") or "").strip()
+    provider_markers = {
+        "google": (
+            "google/genai",
+            "google\\genai",
+            "google/generativeai",
+            "google/ai/generativelanguage",
+            "langchain_google_genai",
+        ),
+        "openai": ("/openai/", "\\openai\\", "openai/resources", "langchain_openai"),
+        "anthropic": ("/anthropic/", "\\anthropic\\", "langchain_anthropic"),
+        "lyzr": ("/lyzr", "\\lyzr"),
+        "litellm": ("litellm",),
+        "other": (),
+    }.get(provider, ())
+
+    def _matches_provider(call: dict) -> bool:
+        path = str(call.get("file") or "").replace("\\", "/").lower()
+        if not provider_markers:
+            return bool(path)
+        return any(marker.replace("\\", "/").lower() in path for marker in provider_markers)
+
+    provider_calls = [c for c in runtime_calls if _matches_provider(c)]
+    # Ignore no-op stubs: provider SDK call should come from installed package path
+    if require and not provider_calls:
+        reasons.append(
+            "Runtime: no real LLM provider SDK call observed for declared provider "
+            f"{provider!r} (rule-based / mock agents fail L1.12)"
+        )
+        checks_passed = False
+
+    declared_model = (llm_claim.get("model") or "").strip()
+    observed_models = sorted({
+        str(c.get("model")) for c in provider_calls if c.get("model")
+    })
+    if require and provider_calls and declared_model and observed_models:
+        if declared_model not in observed_models and not any(
+            declared_model in m or m in declared_model for m in observed_models
+        ):
+            reasons.append(
+                f"Runtime: declared model {declared_model!r} not observed in LLM calls "
+                f"(observed={observed_models})"
+            )
+            checks_passed = False
+    elif require and provider_calls and declared_model and not observed_models and provider in {"google", "openai", "anthropic"}:
+        reasons.append(
+            f"Runtime: LLM SDK was called but model id was not observed (declared={declared_model!r})"
+        )
+        checks_passed = False
+
+    # Instant local-only pattern: zero LLM calls already failed; also flag if claim says llm but only tools ran
+    return {
+        "passed": checks_passed,
+        "score": 1.0 if checks_passed else 0.0,
+        "reasons": reasons,
+        "ast": ast_info,
+        "runtime_llm_calls": runtime_calls[:20],
+        "declared": {
+            "provider": llm_claim.get("provider"),
+            "model": llm_claim.get("model"),
+            "import": llm_claim.get("import"),
+            "call": llm_claim.get("call"),
+        },
+    }

{hidevs_eval-0.1.1 → hidevs_eval-0.1.2}/src/hidevs_quest/runner.py

@@ -134,13 +134,33 @@ def invoke(config: dict, root: Path, case: dict) -> Any:
 
 
 class Scripted:
-    def __init__(self, original, mode: str):
+    def __init__(self, original, mode: str, node_id: str = ""):
         self.original = original
         self.mode = mode
         self.calls = []
+        self.arg_records = []
+        self.node_id = node_id
+
+    def _note(self) -> None:
+        if not self.node_id:
+            return
+        from hidevs_quest.graphify import ExecutionTracer
+
+        tracer = getattr(ExecutionTracer, "_active", None)
+        if tracer is not None:
+            tracer.note_tool(self.node_id)
 
     def __call__(self, *args, **kwargs):
+        from hidevs_quest.graphify import _jsonable
+
+        record = {
+            "args": [_jsonable(a) for a in args],
+            "kwargs": {str(k): _jsonable(v) for k, v in kwargs.items()},
+        }
+        record["signature"] = repr((record["args"], sorted(record["kwargs"].items())))
         self.calls.append(True)
+        self.arg_records.append(record)
+        self._note()
         if self.mode == "raise":
             raise RuntimeError("injected tool failure")
         if self.mode == "timeout":
@@ -161,6 +181,17 @@ def modules_under(root: Path):
             continue
 
 
+def _tool_node_id(root: Path, module, module_name: str, fn_name: str) -> str:
+    path = getattr(module, "__file__", None)
+    if path:
+        try:
+            rel = Path(path).resolve().relative_to(Path(root).resolve()).as_posix()
+            return f"{rel}:{fn_name}"
+        except (ValueError, OSError):
+            pass
+    return f"{module_name.replace('.', '/')}.py:{fn_name}"
+
+
 def patch_tools(root: Path, config: dict, mode: str):
     wrappers = {}
     restores = []
@@ -174,18 +205,18 @@ def patch_tools(root: Path, config: dict, mode: str):
         module_name, fn_name = spec.split(":", 1)
         module = __import__(module_name, fromlist=[fn_name])
         original = getattr(module, fn_name)
-        wrapper = Scripted(original, mode)
+        wrapper = Scripted(original, mode, _tool_node_id(root, module, module_name, fn_name))
         wrappers[capability] = wrapper
         setattr(module, fn_name, wrapper)
         restores.append((module, fn_name, original))
         for loaded in modules_under(root):
             for attr, value in list(vars(loaded).items()):
-                if value is original:
+                if value is original or (isinstance(value, Scripted) and value.original is original):
                     setattr(loaded, attr, wrapper)
                     restores.append((loaded, attr, original))
                 if isinstance(value, dict):
                     for key, item in list(value.items()):
-                        if item is original:
+                        if item is original or (isinstance(item, Scripted) and item.original is original):
                             value[key] = wrapper
                             restores.append((value, key, original))
     return wrappers, restores
@@ -199,28 +230,80 @@ def restore(restores) -> None:
             setattr(owner, key, original)
 
 
-def run_cases(config: dict, root: Path, cases: list[dict]) -> list[dict]:
+from hidevs_quest.graphify import ExecutionTracer, build_ast_graph, summarize_tool_args
+
+
+def run_cases(config: dict, root: Path, cases: list[dict]) -> tuple[list[dict], dict]:
     results = []
+    all_runtime_graphs = {}
+    tool_calls_by_cap: dict[str, list[dict]] = {}
+
+    artifacts_dir = root / ".eval_artifacts"
+    artifacts_dir.mkdir(exist_ok=True)
+
+    sources = collect_sources(root)
+    entry = config.get("entry") or {}
+    entry_spec = entry.get("class") or entry.get("callable") or ""
+    # Prefer method name for entry node resolution (class:Method style via method field)
+    method = entry.get("method") or ""
+    if method and ":" in entry_spec:
+        # Keep class for import; pass method hint by appending candidate via tools build
+        pass
+    ast_graph = build_ast_graph(sources, entry_spec if not method else f"{entry_spec}:{method}", config.get("tools"))
+    # If method provided, force entry candidates to include method
+    if method and not (ast_graph.get("entry_node") or "").endswith(f":{method}"):
+        ast_graph = build_ast_graph(sources, method, config.get("tools"))
+    (artifacts_dir / "ast_graph.json").write_text(json.dumps(ast_graph, indent=2))
+
     for case in cases:
         mode = case.get("tool_mode") or "ok"
         wrappers, restores = patch_tools(root, config, mode)
+        runtime_graph = {}
         try:
             try:
-                payload = as_dict(invoke(config, root, case))
+                with ExecutionTracer(root) as tracer:
+                    payload = as_dict(invoke(config, root, case))
+                runtime_graph = tracer.get_graph()
                 error = ""
             except Exception:
                 payload = {}
                 error = traceback.format_exc()
         finally:
             restore(restores)
+
+        tool_arg_calls = {}
+        for name, wrapper in wrappers.items():
+            tool_arg_calls[name] = list(wrapper.arg_records)
+            tool_calls_by_cap.setdefault(name, []).extend(
+                {**rec, "case_id": case["id"]} for rec in wrapper.arg_records
+            )
+        runtime_graph["tool_arg_calls"] = tool_arg_calls
+
         counts = {name: len(wrapper.calls) for name, wrapper in wrappers.items()}
+        runtime_graph = dict(runtime_graph or {})
+        runtime_graph["counts"] = counts
+        all_runtime_graphs[case["id"]] = runtime_graph
         results.append({
             "id": case["id"],
             "payload": payload,
             "error": error[-1500:] if error else "",
             "counts": counts,
+            "runtime_graph": runtime_graph,
         })
-    return results
+
+    tool_arg_summary = summarize_tool_args(tool_calls_by_cap)
+    all_llm_calls = []
+    for graph in all_runtime_graphs.values():
+        graph["tool_arg_summary"] = tool_arg_summary
+        all_llm_calls.extend(graph.get("llm_calls") or [])
+    for row in results:
+        row["runtime_graph"]["tool_arg_summary"] = tool_arg_summary
+        row["runtime_graph"]["llm_calls"] = all_llm_calls
+
+    (artifacts_dir / "execution_trace_graph.json").write_text(json.dumps(all_runtime_graphs, indent=2))
+    (artifacts_dir / "tool_arg_summary.json").write_text(json.dumps(tool_arg_summary, indent=2))
+    (artifacts_dir / "llm_calls.json").write_text(json.dumps(all_llm_calls, indent=2))
+    return results, ast_graph
 
 
 CLIENT_HEADER = "X-Hidevs-Client"

hidevs_eval-0.1.1/src/hidevs_quest/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.1"