hidevs-eval 0.1.2__tar.gz → 0.1.3__tar.gz

{hidevs_eval-0.1.2 → hidevs_eval-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hidevs-eval
-Version: 0.1.2
+Version: 0.1.3
 Summary: HiDevs quest eval client. Runs the local agent and uploads replies for scoring.
 Requires-Python: >=3.11
 Requires-Dist: pyyaml>=6

{hidevs_eval-0.1.2 → hidevs_eval-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hidevs-eval"
-version = "0.1.2"
+version = "0.1.3"
 description = "HiDevs quest eval client. Runs the local agent and uploads replies for scoring."
 requires-python = ">=3.11"
 dependencies = [

{hidevs_eval-0.1.2 → hidevs_eval-0.1.3}/src/hidevs_eval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hidevs-eval
-Version: 0.1.2
+Version: 0.1.3
 Summary: HiDevs quest eval client. Runs the local agent and uploads replies for scoring.
 Requires-Python: >=3.11
 Requires-Dist: pyyaml>=6

{hidevs_eval-0.1.2 → hidevs_eval-0.1.3}/src/hidevs_eval.egg-info/SOURCES.txt

@@ -9,4 +9,5 @@ src/hidevs_eval.egg-info/top_level.txt
 src/hidevs_quest/__init__.py
 src/hidevs_quest/cli.py
 src/hidevs_quest/graphify.py
+src/hidevs_quest/llm_probe.py
 src/hidevs_quest/runner.py

hidevs_eval-0.1.3/src/hidevs_quest/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.3"

{hidevs_eval-0.1.2 → hidevs_eval-0.1.3}/src/hidevs_quest/graphify.py

@@ -398,6 +398,8 @@ class ExecutionTracer:
                     "method": fn_name,
                     "file": filename,
                     "model": model,
+                    "via": "sdk",
+                    "proven": True,
                     "t": time.monotonic() - self.start_time,
                 })
                 node_id = f"llm:{fn_name}"

hidevs_eval-0.1.3/src/hidevs_quest/llm_probe.py

@@ -0,0 +1,436 @@
+"""Prove a declared model call actually hit a model.
+
+A config entry such as ``meetagent.provider.chat_completion`` is wrapped at eval
+time. The wrapper counts only when, inside that call, an HTTP request looks like
+a chat/completions call and the value returned to the agent was taken from the
+response. A function that returns a fixed string, ignores the response, or echoes
+the prompt is not recorded.
+"""
+
+from __future__ import annotations
+
+import ast
+import hashlib
+import importlib
+import inspect
+import json
+import re
+import sys
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+SECRET_TEXT = re.compile(
+    r"mongodb(?:\+srv)?://\S+|AIza[0-9A-Za-z_\-]{20,}|sk-[0-9A-Za-z]{16,}|\b(?:\d[ -]?){13,16}\b",
+    re.IGNORECASE,
+)
+
+_LLM_HINTS = (
+    "chat/completions",
+    "/completions",
+    "generatecontent",
+    "/api/chat",
+    "/api/generate",
+    "/v1/messages",
+    "/v1/responses",
+    "generativelanguage",
+)
+_BODY_KEYS = {"messages", "contents", "prompt", "input"}
+_HOOKS: list[tuple[Any, str, Any]] = []
+
+
+def redact_text(value: str) -> str:
+    return SECRET_TEXT.sub("[redacted]", value)
+
+
+def clip_value(value: Any, depth: int = 0) -> Any:
+    if isinstance(value, str):
+        return redact_text(value)[:800]
+    if isinstance(value, (int, float, bool)) or value is None:
+        return value
+    if depth >= 4:
+        return redact_text(repr(value))[:200]
+    if isinstance(value, dict):
+        return {str(k)[:80]: clip_value(v, depth + 1) for k, v in list(value.items())[:12]}
+    if isinstance(value, (list, tuple)):
+        return [clip_value(v, depth + 1) for v in list(value)[:8]]
+    return redact_text(repr(value))[:200]
+
+
+def resolve_llm_callable(root: Path, call: str):
+    """Return (module, attr, fn) for ``pkg.mod.fn`` or ``pkg.mod:fn``. Otherwise None."""
+    call = (call or "").strip()
+    if not call or call in {"invoke", "llm.invoke"} or call.endswith(".invoke"):
+        # Instance methods such as llm.invoke are observed through the SDK tracer.
+        if ":" not in call:
+            return None
+    root_s = str(root)
+    if root_s not in sys.path:
+        sys.path.insert(0, root_s)
+    if ":" in call:
+        module_name, attr = call.split(":", 1)
+        if not module_name or "." in attr or not attr.isidentifier():
+            return None
+        try:
+            module = importlib.import_module(module_name)
+        except Exception:
+            return None
+        fn = getattr(module, attr, None)
+        if callable(fn):
+            return module, attr, fn
+        return None
+    parts = call.split(".")
+    if len(parts) < 2:
+        return None
+    attr = parts[-1]
+    module_name = ".".join(parts[:-1])
+    if not attr.isidentifier() or not module_name:
+        return None
+    try:
+        module = importlib.import_module(module_name)
+    except Exception:
+        return None
+    fn = getattr(module, attr, None)
+    if callable(fn):
+        return module, attr, fn
+    return None
+
+
+def _fingerprint(text: str) -> str:
+    normalized = " ".join((text or "").split())[:2000]
+    return hashlib.sha256(normalized.encode("utf-8", errors="ignore")).hexdigest()[:16]
+
+
+def _result_text(result: Any) -> str:
+    if isinstance(result, str):
+        return result.strip()
+    if isinstance(result, dict):
+        for key in ("content", "text", "output", "answer"):
+            value = result.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip()
+    content = getattr(result, "content", None)
+    if isinstance(content, str):
+        return content.strip()
+    return ""
+
+
+def _return_constants(source: str) -> set[str]:
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        return set()
+    found: set[str] = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Return) and isinstance(node.value, ast.Constant):
+            if isinstance(node.value.value, str) and node.value.value.strip():
+                found.add(node.value.value.strip())
+    return found
+
+
+def _request_bytes(body: Any) -> bytes:
+    if body is None:
+        return b""
+    if isinstance(body, bytes):
+        return body
+    if isinstance(body, str):
+        return body.encode("utf-8", errors="ignore")
+    if isinstance(body, dict):
+        return json.dumps(body).encode("utf-8")
+    return str(body).encode("utf-8", errors="ignore")
+
+
+def _loads(body: bytes | str | None) -> Any:
+    if body is None:
+        return None
+    if isinstance(body, dict):
+        return body
+    try:
+        text = body.decode("utf-8", errors="ignore") if isinstance(body, bytes) else str(body)
+        return json.loads(text)
+    except (json.JSONDecodeError, UnicodeError, TypeError):
+        return None
+
+
+def _url_of(url: Any) -> str:
+    full = getattr(url, "full_url", None)
+    return str(full or url or "")
+
+
+def classify_exchange(url: str, request_body: bytes) -> dict[str, Any] | None:
+    """Return a small description when the request looks like a model call."""
+    lowered = (url or "").lower().split("?", 1)[0]
+    data = _loads(request_body)
+    shaped = any(hint in lowered for hint in _LLM_HINTS)
+    if isinstance(data, dict) and _BODY_KEYS.intersection(data):
+        shaped = True
+    if not shaped:
+        return None
+    kind = "chat_completions"
+    if "generatecontent" in lowered or "generativelanguage" in lowered:
+        kind = "generate_content"
+    elif "/v1/messages" in lowered:
+        kind = "messages"
+    elif "/api/chat" in lowered:
+        kind = "api_chat"
+    elif "/api/generate" in lowered:
+        kind = "api_generate"
+    elif "/v1/responses" in lowered:
+        kind = "responses"
+    model = ""
+    user_bits: list[str] = []
+    if isinstance(data, dict):
+        if isinstance(data.get("model"), str):
+            model = data["model"].strip()
+        messages = data.get("messages")
+        if isinstance(messages, list):
+            for item in messages:
+                if isinstance(item, dict) and item.get("role") != "system":
+                    content = item.get("content")
+                    if isinstance(content, str):
+                        user_bits.append(content)
+        for key in ("prompt", "input"):
+            if isinstance(data.get(key), str):
+                user_bits.append(data[key])
+        contents = data.get("contents")
+        if isinstance(contents, list):
+            for item in contents:
+                parts = item.get("parts") if isinstance(item, dict) else None
+                if isinstance(parts, list):
+                    for part in parts:
+                        if isinstance(part, dict) and isinstance(part.get("text"), str):
+                            user_bits.append(part["text"])
+    host = ""
+    try:
+        host = urllib.parse.urlparse(url).hostname or ""
+    except Exception:
+        host = ""
+    return {
+        "url_kind": kind,
+        "url_host": host.lower(),
+        "model": model,
+        "user_text": "\n".join(user_bits)[:4000],
+    }
+
+
+def _response_texts(body: bytes) -> list[str]:
+    texts: list[str] = []
+    raw = body.decode("utf-8", errors="ignore") if isinstance(body, bytes) else str(body or "")
+    if raw.strip():
+        texts.append(raw)
+    data = _loads(body)
+    if isinstance(data, dict):
+        try:
+            content = data["choices"][0]["message"]["content"]
+            if isinstance(content, str):
+                texts.append(content)
+        except (KeyError, IndexError, TypeError):
+            pass
+        try:
+            content = data["candidates"][0]["content"]["parts"][0]["text"]
+            if isinstance(content, str):
+                texts.append(content)
+        except (KeyError, IndexError, TypeError):
+            pass
+        response = data.get("response")
+        if isinstance(response, str):
+            texts.append(response)
+        message = data.get("message")
+        if isinstance(message, dict) and isinstance(message.get("content"), str):
+            texts.append(message["content"])
+    return texts
+
+
+def _used_response(result: str, response_body: bytes) -> bool:
+    normalized = " ".join(result.split())
+    if len(normalized) < 8:
+        return False
+    for text in _response_texts(response_body):
+        other = " ".join(text.split())
+        if not other:
+            continue
+        if normalized == other or normalized in other or (len(other) >= 8 and other in normalized):
+            return True
+    return False
+
+
+def _is_echo(result: str, user_text: str) -> bool:
+    normalized = " ".join(result.split())
+    user = " ".join((user_text or "").split())
+    if not normalized or not user:
+        return False
+    return normalized == user or (len(normalized) > 40 and normalized in user)
+
+
+def _active_span(tracer: Any) -> dict | None:
+    spans = getattr(tracer, "_llm_spans", None) if tracer is not None else None
+    if not spans:
+        return None
+    return spans[-1]
+
+
+def observe_http(url: str, request_body: Any, status: int, response_body: bytes) -> None:
+    from hidevs_quest.graphify import ExecutionTracer
+
+    tracer = ExecutionTracer._active
+    span = _active_span(tracer)
+    if span is None:
+        return
+    info = classify_exchange(url, _request_bytes(request_body))
+    if info is None:
+        return
+    if not isinstance(response_body, (bytes, bytearray)):
+        response_body = _request_bytes(response_body)
+    # Keep only what the proof needs. Do not store the prompt or the completion.
+    span["http"].append({
+        **info,
+        "status": int(status or 0),
+        "response_body": bytes(response_body)[:200_000],
+    })
+
+
+def _prove(span: dict, result: Any, probe: "LlmCallProbe") -> dict | None:
+    text = _result_text(result)
+    if not text or text in probe.constants:
+        return None
+    for event in span.get("http") or []:
+        if not (200 <= int(event.get("status") or 0) < 300):
+            continue
+        if _is_echo(text, event.get("user_text") or ""):
+            continue
+        if not _used_response(text, event.get("response_body") or b""):
+            continue
+        return {
+            "method": probe.name,
+            "file": probe.filename,
+            "model": event.get("model") or None,
+            "via": "http",
+            "proven": True,
+            "declared": True,
+            "url_kind": event.get("url_kind") or "",
+            "url_host": event.get("url_host") or "",
+            "input_fingerprint": _fingerprint(event.get("user_text") or ""),
+            "output_fingerprint": _fingerprint(text),
+        }
+    return None
+
+
+class LlmCallProbe:
+    def __init__(self, original, declared_model: str = ""):
+        self.original = original
+        self.name = getattr(original, "__name__", "call")
+        self.filename = str(getattr(original, "__code__", None) and original.__code__.co_filename or "")
+        self.declared_model = declared_model
+        try:
+            self.constants = _return_constants(inspect.getsource(original))
+        except (OSError, TypeError):
+            self.constants = set()
+
+    def __call__(self, *args, **kwargs):
+        from hidevs_quest.graphify import ExecutionTracer
+
+        tracer = ExecutionTracer._active
+        span = {"http": []}
+        if tracer is not None:
+            tracer._llm_spans = getattr(tracer, "_llm_spans", [])
+            tracer._llm_spans.append(span)
+        try:
+            result = self.original(*args, **kwargs)
+        except BaseException:
+            if tracer is not None and tracer._llm_spans:
+                tracer._llm_spans.pop()
+            raise
+        if tracer is not None and tracer._llm_spans:
+            tracer._llm_spans.pop()
+            record = _prove(span, result, self)
+            if record is not None:
+                tracer.llm_calls.append(record)
+                tracer.nodes.add(f"llm:{self.name}")
+        return result
+
+
+def _wrap_read(response, url: str, request_body: bytes):
+    original_read = response.read
+
+    def read(*args, **kwargs):
+        blob = original_read(*args, **kwargs)
+        try:
+            status = int(getattr(response, "status", None) or getattr(response, "code", None) or 200)
+        except (TypeError, ValueError):
+            status = 200
+        payload = blob if isinstance(blob, (bytes, bytearray)) else _request_bytes(blob)
+        observe_http(url, request_body, status, payload)
+        return blob
+
+    response.read = read
+    return response
+
+
+def _wrapped_urlopen(original):
+    def urlopen(url, data=None, *args, **kwargs):
+        request_url = _url_of(url)
+        request_body = _request_bytes(getattr(url, "data", None) if data is None else data)
+        response = original(url, data, *args, **kwargs)
+        return _wrap_read(response, request_url, request_body)
+
+    return urlopen
+
+
+def install_http_hooks() -> None:
+    """Patch HTTP clients so a declared model call can be proved. Idempotent."""
+    if _HOOKS:
+        return
+    original = urllib.request.urlopen
+    urllib.request.urlopen = _wrapped_urlopen(original)
+    _HOOKS.append((urllib.request, "urlopen", original))
+
+    try:
+        import httpx
+    except Exception:
+        httpx = None
+    if httpx is not None and hasattr(httpx.Client, "send"):
+        original_send = httpx.Client.send
+
+        def send(self, request, *args, **kwargs):
+            response = original_send(self, request, *args, **kwargs)
+            try:
+                observe_http(
+                    str(request.url),
+                    getattr(request, "content", b"") or b"",
+                    int(getattr(response, "status_code", 0) or 0),
+                    response.content or b"",
+                )
+            except Exception:
+                pass
+            return response
+
+        httpx.Client.send = send
+        _HOOKS.append((httpx.Client, "send", original_send))
+
+    try:
+        import requests
+    except Exception:
+        requests = None
+    if requests is not None:
+        original_request = requests.sessions.Session.request
+
+        def request(self, method, url, *args, **kwargs):
+            response = original_request(self, method, url, *args, **kwargs)
+            body = kwargs.get("data")
+            if body is None and kwargs.get("json") is not None:
+                body = kwargs.get("json")
+            try:
+                observe_http(str(response.url or url), body, int(response.status_code or 0), response.content or b"")
+            except Exception:
+                pass
+            return response
+
+        requests.sessions.Session.request = request
+        _HOOKS.append((requests.sessions.Session, "request", original_request))
+
+
+def remove_http_hooks() -> None:
+    while _HOOKS:
+        owner, name, original = _HOOKS.pop()
+        setattr(owner, name, original)

{hidevs_eval-0.1.2 → hidevs_eval-0.1.3}/src/hidevs_quest/runner.py

@@ -139,6 +139,7 @@ class Scripted:
         self.mode = mode
         self.calls = []
         self.arg_records = []
+        self.results = []
         self.node_id = node_id
 
     def _note(self) -> None:
@@ -165,7 +166,11 @@ class Scripted:
             raise RuntimeError("injected tool failure")
         if self.mode == "timeout":
             raise TimeoutError("injected tool timeout")
-        return self.original(*args, **kwargs)
+        result = self.original(*args, **kwargs)
+        from hidevs_quest.llm_probe import clip_value
+
+        self.results.append(clip_value(result))
+        return result
 
 
 def modules_under(root: Path):
@@ -230,6 +235,33 @@ def restore(restores) -> None:
             setattr(owner, key, original)
 
 
+def patch_llm(root: Path, config: dict):
+    """Wrap requirements.llm.call when it is a real module function.
+
+    Instance calls such as llm.invoke stay on the SDK tracer. The wrapper
+    records a call only after llm_probe proves a model response was used.
+    """
+    from hidevs_quest.llm_probe import LlmCallProbe, resolve_llm_callable
+
+    claim = ((config.get("requirements") or {}).get("llm")) or {}
+    found = resolve_llm_callable(root, str(claim.get("call") or ""))
+    if not found:
+        return []
+    module, name, original = found
+    if isinstance(original, LlmCallProbe):
+        original = original.original
+    probe = LlmCallProbe(original, declared_model=str(claim.get("model") or ""))
+    restores = [(module, name, original)]
+    setattr(module, name, probe)
+    for loaded in modules_under(root):
+        for attr, value in list(vars(loaded).items()):
+            current = value.original if isinstance(value, LlmCallProbe) else value
+            if current is original:
+                setattr(loaded, attr, probe)
+                restores.append((loaded, attr, original))
+    return restores
+
+
 from hidevs_quest.graphify import ExecutionTracer, build_ast_graph, summarize_tool_args
 
 
@@ -255,50 +287,61 @@ def run_cases(config: dict, root: Path, cases: list[dict]) -> tuple[list[dict],
         ast_graph = build_ast_graph(sources, method, config.get("tools"))
     (artifacts_dir / "ast_graph.json").write_text(json.dumps(ast_graph, indent=2))
 
-    for case in cases:
-        mode = case.get("tool_mode") or "ok"
-        wrappers, restores = patch_tools(root, config, mode)
-        runtime_graph = {}
-        try:
+    from hidevs_quest.llm_probe import install_http_hooks, remove_http_hooks
+
+    install_http_hooks()
+    try:
+        for case in cases:
+            mode = case.get("tool_mode") or "ok"
+            wrappers, restores = patch_tools(root, config, mode)
+            restores.extend(patch_llm(root, config))
+            runtime_graph = {}
             try:
-                with ExecutionTracer(root) as tracer:
-                    payload = as_dict(invoke(config, root, case))
-                runtime_graph = tracer.get_graph()
-                error = ""
-            except Exception:
-                payload = {}
-                error = traceback.format_exc()
-        finally:
-            restore(restores)
-
-        tool_arg_calls = {}
-        for name, wrapper in wrappers.items():
-            tool_arg_calls[name] = list(wrapper.arg_records)
-            tool_calls_by_cap.setdefault(name, []).extend(
-                {**rec, "case_id": case["id"]} for rec in wrapper.arg_records
-            )
-        runtime_graph["tool_arg_calls"] = tool_arg_calls
-
-        counts = {name: len(wrapper.calls) for name, wrapper in wrappers.items()}
-        runtime_graph = dict(runtime_graph or {})
-        runtime_graph["counts"] = counts
-        all_runtime_graphs[case["id"]] = runtime_graph
-        results.append({
-            "id": case["id"],
-            "payload": payload,
-            "error": error[-1500:] if error else "",
-            "counts": counts,
-            "runtime_graph": runtime_graph,
-        })
-
-    tool_arg_summary = summarize_tool_args(tool_calls_by_cap)
-    all_llm_calls = []
-    for graph in all_runtime_graphs.values():
-        graph["tool_arg_summary"] = tool_arg_summary
-        all_llm_calls.extend(graph.get("llm_calls") or [])
-    for row in results:
-        row["runtime_graph"]["tool_arg_summary"] = tool_arg_summary
-        row["runtime_graph"]["llm_calls"] = all_llm_calls
+                try:
+                    with ExecutionTracer(root) as tracer:
+                        payload = as_dict(invoke(config, root, case))
+                    runtime_graph = tracer.get_graph()
+                    error = ""
+                except Exception:
+                    payload = {}
+                    error = traceback.format_exc()
+            finally:
+                restore(restores)
+
+            tool_arg_calls = {}
+            for name, wrapper in wrappers.items():
+                tool_arg_calls[name] = list(wrapper.arg_records)
+                tool_calls_by_cap.setdefault(name, []).extend(
+                    {**rec, "case_id": case["id"]} for rec in wrapper.arg_records
+                )
+            runtime_graph["tool_arg_calls"] = tool_arg_calls
+
+            counts = {name: len(wrapper.calls) for name, wrapper in wrappers.items()}
+            runtime_graph = dict(runtime_graph or {})
+            runtime_graph["counts"] = counts
+            runtime_graph["tool_results"] = {
+                name: list(wrapper.results[:3])
+                for name, wrapper in wrappers.items()
+                if wrapper.results
+            }
+            all_runtime_graphs[case["id"]] = runtime_graph
+            results.append({
+                "id": case["id"],
+                "payload": payload,
+                "error": error[-1500:] if error else "",
+                "counts": counts,
+                "runtime_graph": runtime_graph,
+            })
+
+        tool_arg_summary = summarize_tool_args(tool_calls_by_cap)
+        all_llm_calls = []
+        for graph in all_runtime_graphs.values():
+            graph["tool_arg_summary"] = tool_arg_summary
+            all_llm_calls.extend(graph.get("llm_calls") or [])
+        for row in results:
+            row["runtime_graph"]["tool_arg_summary"] = tool_arg_summary
+    finally:
+        remove_http_hooks()
 
     (artifacts_dir / "execution_trace_graph.json").write_text(json.dumps(all_runtime_graphs, indent=2))
     (artifacts_dir / "tool_arg_summary.json").write_text(json.dumps(tool_arg_summary, indent=2))

hidevs_eval-0.1.2/src/hidevs_quest/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.1.2"