evalrun 0.4.0__py3-none-any.whl

framework/utils.py

@@ -0,0 +1,41 @@
+"""Shared helper utility functions for the framework."""
+
+import json
+import re
+from typing import Any
+
+
+def parse_json_markdown(text: str) -> Any:
+    """Extracts and parses a JSON structure from raw LLM output.
+
+    Handles wrapping blocks like ```json ... ``` robustly.
+
+    Args:
+        text: Raw text generated by the model.
+
+    Returns:
+        The parsed Python dictionary, list, or primitive.
+
+    Raises:
+        ValueError: If JSON decoding fails.
+    """
+    cleaned = text.strip()
+    if cleaned.startswith("```json"):
+        cleaned = cleaned[7:]
+    elif cleaned.startswith("```"):
+        cleaned = cleaned[3:]
+    if cleaned.endswith("```"):
+        cleaned = cleaned[:-3]
+    normalized = cleaned.strip()
+    try:
+        return json.loads(normalized, strict=False)
+    except json.JSONDecodeError as first_error:
+        # LLM judges occasionally emit a trailing comma before `}` or `]`.
+        # Repair only that conservative, unambiguous formatting issue.
+        repaired = re.sub(r",(\s*[}\]])", r"\1", normalized)
+        if repaired == normalized:
+            raise ValueError(f"Invalid JSON string: {first_error}") from first_error
+        try:
+            return json.loads(repaired, strict=False)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON string: {e}") from e

framework/verification/__init__.py

@@ -0,0 +1,13 @@
+"""Package containing interfaces and structures for factual verification."""
+
+from framework.verification.models import (
+    ClaimType,
+    VerificationStatus,
+    Claim,
+    Evidence,
+    VerificationReport,
+)
+from framework.verification.base import BaseVerifier
+from framework.verification.local import LocalKnowledgeBaseVerifier
+from framework.verification.extractor import ClaimExtractor
+from framework.verification.pipeline import VerificationPipeline

framework/verification/base.py

@@ -0,0 +1,32 @@
+"""Base interface for factual claims verifiers."""
+
+from abc import ABC, abstractmethod
+from typing import List
+from .models import Claim, Evidence
+
+
+class BaseVerifier(ABC):
+    """Abstract base class for factual claims verifiers."""
+
+    @abstractmethod
+    def verify(self, claim: Claim) -> Evidence:
+        """Verifies a single claim against a ground-truth source.
+
+        Args:
+            claim: The claim to verify.
+
+        Returns:
+            An Evidence dataclass containing the outcome status and source reference.
+        """
+        pass
+
+    def verify_all(self, claims: List[Claim]) -> List[Evidence]:
+        """Batch verifies a list of claims sequentially.
+
+        Args:
+            claims: The list of claims to verify.
+
+        Returns:
+            A list of Evidence outcomes.
+        """
+        return [self.verify(c) for c in claims]

framework/verification/extractor.py

@@ -0,0 +1,105 @@
+"""Factual claim extraction from agent output using LLM."""
+
+from typing import List
+from framework.exceptions import VerificationError
+from framework.llms import BaseLLM, Message
+from framework.models import AgentOutput
+from framework.utils import parse_json_markdown
+from .models import Claim, ClaimType
+from .prompts import CLAIM_EXTRACTION_PROMPT
+
+
+class ClaimExtractor:
+    """Extracts factual claims from agent output text using an LLM."""
+
+    def __init__(self, llm: BaseLLM):
+        """Initializes the ClaimExtractor.
+
+        Args:
+            llm: The LLM client to use for structured extraction.
+        """
+        self.llm = llm
+
+    def extract(self, output: AgentOutput) -> List[Claim]:
+        """Extracts verifiable claims from the agent output.
+
+        Args:
+            output: The output to analyze.
+
+        Returns:
+            A list of extracted Claim objects.
+
+        Raises:
+            VerificationError: If extraction or JSON parsing fails.
+        """
+        prompt_messages = self._build_messages(output.content)
+
+        try:
+            response = self.llm.generate(prompt_messages)
+            raw_text = response.text
+        except Exception as e:
+            raise VerificationError(
+                f"LLM claims extraction generation failed: {e}"
+            ) from e
+
+        raw_claims = self._parse_response(raw_text)
+
+        claims = []
+        for idx, item in enumerate(raw_claims):
+            claim = self._validate_and_build_claim(item, idx)
+            claims.append(claim)
+
+        return claims
+
+    def _build_messages(self, content: str) -> List[Message]:
+        """Constructs prompt messages for claims extraction."""
+        return [
+            Message(role="system", content=CLAIM_EXTRACTION_PROMPT),
+            Message(
+                role="user",
+                content=f"Extract all factual claims from the following text:\n\n{content}",
+            ),
+        ]
+
+    def _parse_response(self, raw_text: str) -> List[dict]:
+        """Cleans and parses the LLM JSON response."""
+        try:
+            parsed = parse_json_markdown(raw_text)
+        except Exception as e:
+            raise VerificationError(
+                f"Failed to parse extractor LLM output as JSON. Raw response:\n{raw_text}"
+            ) from e
+
+        if not isinstance(parsed, list):
+            raise VerificationError(
+                f"Expected JSON array of claims, got: {type(parsed)}"
+            )
+
+        return parsed
+
+    def _validate_and_build_claim(self, raw_item: dict, index: int) -> Claim:
+        """Validates raw dict fields and builds a Claim object."""
+        if not isinstance(raw_item, dict):
+            raise VerificationError(
+                f"Expected claim dictionary at index {index}, got: {type(raw_item)}"
+            )
+
+        for field in ["subject", "predicate", "value", "claim_type"]:
+            if field not in raw_item:
+                raise VerificationError(
+                    f"Missing required field '{field}' in claim at index {index}: {raw_item}"
+                )
+
+        # Map type dynamically with validation fallback in python instead of LLM direct mapping
+        raw_type = str(raw_item["claim_type"]).lower()
+        try:
+            claim_type = ClaimType(raw_type)
+        except ValueError:
+            claim_type = ClaimType.OTHER
+
+        return Claim(
+            subject=str(raw_item["subject"]),
+            predicate=str(raw_item["predicate"]),
+            value=str(raw_item["value"]),
+            claim_type=claim_type,
+        )

framework/verification/local.py

@@ -0,0 +1,122 @@
+"""Local knowledge base claim verifier."""
+
+import json
+from pathlib import Path
+from typing import Any, Dict
+from framework.exceptions import KnowledgeBaseLoadError
+from .base import BaseVerifier
+from .models import Claim, Evidence, VerificationStatus
+from .utils import normalize_key, values_are_equivalent
+
+
+class LocalKnowledgeBaseVerifier(BaseVerifier):
+    """Verifies claims against a local JSON knowledge base of truth facts."""
+
+    def __init__(self, kb_path: str):
+        """Initializes the LocalKnowledgeBaseVerifier.
+
+        Args:
+            kb_path: Path to the JSON file containing ground-truth data.
+        """
+        self.kb_path = Path(kb_path)
+        self._kb: Dict[str, Dict[str, Any]] = {}
+        self._load_kb()
+
+    def _load_kb(self) -> None:
+        """Loads and normalizes the knowledge base keys."""
+        if not self.kb_path.exists():
+            raise KnowledgeBaseLoadError(
+                f"Knowledge base file does not exist: {self.kb_path}"
+            )
+
+        try:
+            with open(self.kb_path, "r", encoding="utf-8") as f:
+                raw_data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise KnowledgeBaseLoadError(
+                f"Malformed JSON in knowledge base at {self.kb_path}: {e}"
+            ) from e
+        except Exception as e:
+            raise KnowledgeBaseLoadError(
+                f"Failed to read knowledge base at {self.kb_path}: {e}"
+            ) from e
+
+        if not isinstance(raw_data, dict):
+            raise KnowledgeBaseLoadError(
+                f"Knowledge base JSON must root-level represent a dictionary, got {type(raw_data)}"
+            )
+
+        # Normalize the database subject and predicate keys for robust matching
+        for subject, predicates in raw_data.items():
+            if not isinstance(predicates, dict):
+                raise KnowledgeBaseLoadError(
+                    f"Expected subject '{subject}' predicates to be a dictionary, got {type(predicates)}"
+                )
+            norm_subj = normalize_key(subject)
+            if norm_subj not in self._kb:
+                self._kb[norm_subj] = {}
+            for pred, val in predicates.items():
+                self._kb[norm_subj][normalize_key(pred)] = val
+
+    def verify(self, claim: Claim) -> Evidence:
+        """Verifies a single claim against the local normalized knowledge base.
+
+        Args:
+            claim: The claim to verify.
+
+        Returns:
+            An Evidence object containing the outcome.
+        """
+        norm_subj = normalize_key(claim.subject)
+        norm_pred = normalize_key(claim.predicate)
+        source_str = str(self.kb_path)
+
+        # 1. Subject check (exact normalized match)
+        if norm_subj not in self._kb:
+            return Evidence(
+                claim=claim,
+                status=VerificationStatus.NOT_FOUND,
+                expected_value=None,
+                actual_value=claim.value,
+                source=source_str,
+                # Local KB is treated as authoritative.
+                confidence=1.0,
+                reason=f"Subject '{claim.subject}' was not found in the ground-truth records.",
+            )
+
+        subject_kb = self._kb[norm_subj]
+
+        # 2. Predicate check (exact normalized match)
+        if norm_pred not in subject_kb:
+            return Evidence(
+                claim=claim,
+                status=VerificationStatus.UNKNOWN,
+                expected_value=None,
+                actual_value=claim.value,
+                source=source_str,
+                # Local KB is treated as authoritative.
+                confidence=1.0,
+                reason=f"Subject '{claim.subject}' is verified, but attribute '{claim.predicate}' is not documented in the ground-truth records.",
+            )
+
+        expected_val = str(subject_kb[norm_pred])
+        actual_val = claim.value
+
+        # 3. Value check
+        if values_are_equivalent(expected_val, actual_val):
+            status = VerificationStatus.VERIFIED
+            reason_str = f"Factual claim matches ground-truth record for '{claim.subject}' ('{claim.predicate}' = '{expected_val}')."
+        else:
+            status = VerificationStatus.REFUTED
+            reason_str = f"Factual discrepancy: ground-truth records list '{expected_val}' for '{claim.subject}' attribute '{claim.predicate}', but agent claimed '{actual_val}'."
+
+        return Evidence(
+            claim=claim,
+            status=status,
+            expected_value=expected_val,
+            actual_value=actual_val,
+            source=source_str,
+            # Local KB is treated as authoritative.
+            confidence=1.0,
+            reason=reason_str,
+        )

framework/verification/models.py

@@ -0,0 +1,112 @@
+"""Data structures for the factual claim verification subsystem."""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import List, Optional
+
+
+class ClaimType(str, Enum):
+    """Supported types of factual claims for travel itineraries."""
+
+    PRICE = "price"
+    TIMING = "timing"
+    EXISTENCE = "existence"
+    DISTANCE = "distance"
+    WEATHER = "weather"
+    OTHER = "other"
+
+
+class VerificationStatus(str, Enum):
+    """Possible outcomes of verifying a claim against ground-truth data."""
+
+    VERIFIED = "verified"
+    REFUTED = "refuted"
+    NOT_FOUND = "not_found"
+    UNKNOWN = "unknown"
+
+
+@dataclass
+class Claim:
+    """Represents a single testable factual assertion extracted from agent output."""
+
+    subject: str
+    predicate: str
+    value: str
+    claim_type: ClaimType
+
+
+@dataclass
+class Evidence:
+    """Represents the outcome of verifying a Claim against a ground-truth source."""
+
+    claim: Claim
+    status: VerificationStatus
+    expected_value: Optional[str]
+    actual_value: Optional[str]
+    source: str
+    confidence: float = 1.0  # Confidence in verification result on a scale from 0.0 to 1.0.
+    reason: Optional[str] = None  # Contextual reasoning explaining findings or discrepancy details.
+
+
+@dataclass
+class VerificationReport:
+    """Consolidated verification outcomes for all claims in a scenario."""
+
+    claims: List[Claim]
+    evidence: List[Evidence]
+
+    @property
+    def total_claims(self) -> int:
+        return len(self.claims)
+
+    @property
+    def verified_count(self) -> int:
+        return sum(
+            1 for e in self.evidence if e.status == VerificationStatus.VERIFIED
+        )
+
+    @property
+    def refuted_count(self) -> int:
+        return sum(
+            1 for e in self.evidence if e.status == VerificationStatus.REFUTED
+        )
+
+    @property
+    def unknown_count(self) -> int:
+        return sum(
+            1 for e in self.evidence if e.status == VerificationStatus.UNKNOWN
+        )
+
+    @property
+    def not_found_count(self) -> int:
+        return sum(
+            1 for e in self.evidence if e.status == VerificationStatus.NOT_FOUND
+        )
+
+    def to_markdown(self) -> str:
+        """Formats verification report evidence metrics and findings into a markdown string."""
+        evidence_lines = []
+        evidence_lines.append("Verification Summary:")
+        evidence_lines.append(f"- Total Claims Extracted: {self.total_claims}")
+        evidence_lines.append(f"- Verified: {self.verified_count}")
+        evidence_lines.append(f"- Refuted: {self.refuted_count}")
+        evidence_lines.append(f"- Unknown Predicates: {self.unknown_count}")
+        evidence_lines.append(f"- Not Found Subjects: {self.not_found_count}\n")
+        evidence_lines.append("Detailed Findings:")
+
+        if not self.evidence:
+            evidence_lines.append("No testable factual claims were extracted or verified.")
+        else:
+            for idx, ev in enumerate(self.evidence):
+                line = (
+                    f"- Claim {idx+1}: {ev.claim.subject} -> {ev.claim.predicate}: {ev.claim.value} "
+                    f"(Type: {ev.claim.claim_type.value})\n"
+                    f"  Status: {ev.status.value.upper()}\n"
+                    f"  Expected: {ev.expected_value or 'N/A'}, Actual: {ev.actual_value or 'N/A'}\n"
+                    f"  Source: {ev.source} (Confidence: {ev.confidence})\n"
+                    f"  Reason: {ev.reason or 'N/A'}"
+                )
+                evidence_lines.append(line)
+
+        return "\n".join(evidence_lines)
+

framework/verification/pipeline.py

@@ -0,0 +1,36 @@
+"""Verification pipeline orchestrating claim extraction and verifications."""
+
+from framework.models import AgentOutput
+from .base import BaseVerifier
+from .extractor import ClaimExtractor
+from .models import VerificationReport
+
+
+class VerificationPipeline:
+    """Orchestrates factual claim extraction and ground-truth verification.
+
+    Wraps the pipeline execution: Agent Output -> Extraction -> Verifications -> VerificationReport.
+    """
+
+    def __init__(self, extractor: ClaimExtractor, verifier: BaseVerifier):
+        """Initializes the VerificationPipeline.
+
+        Args:
+            extractor: The extractor instance to fetch claims.
+            verifier: The verifier instance to verify claims.
+        """
+        self.extractor = extractor
+        self.verifier = verifier
+
+    def run(self, output: AgentOutput) -> VerificationReport:
+        """Executes the verification pipeline on the agent output.
+
+        Args:
+            output: The output to analyze.
+
+        Returns:
+            A VerificationReport containing claims and evidence metrics.
+        """
+        claims = self.extractor.extract(output)
+        evidence = self.verifier.verify_all(claims)
+        return VerificationReport(claims=claims, evidence=evidence)

framework/verification/prompts.py

@@ -0,0 +1,24 @@
+"""Prompts for the factual verification subsystem."""
+
+CLAIM_EXTRACTION_PROMPT = (
+    "You are a precise factual information extraction assistant.\n"
+    "Your task is to analyze the agent's travel itinerary output and extract a focused "
+    "set of testable factual claims (LIMIT to a maximum of 10 total claims).\n\n"
+    "CRITICAL RULES:\n"
+    "1. Do NOT extract repetitive or trivial claims. Do NOT assert 'exists' or 'distance' for every "
+    "location or attraction listed in the itinerary.\n"
+    "2. Focus strictly on the most important concrete factual assertions: transit durations (e.g. Shinkansen duration), "
+    "specific ticket/hotel costs, closed days, or operating schedules.\n"
+    "3. Do NOT extract subjective opinions (e.g., 'beautiful view', 'great atmosphere').\n\n"
+    "You must return a raw JSON array of objects. Do not include markdown headers, "
+    "conversational filler, or explanations. Return ONLY the valid JSON block.\n\n"
+    "JSON Schema:\n"
+    "[\n"
+    "  {\n"
+    "    \"subject\": \"string (e.g., 'Shinkansen Tokyo to Kyoto', 'Gyeongbokgung Palace')\",\n"
+    "    \"predicate\": \"string (e.g., 'travel_duration_minutes', 'closed_days', 'ticket_price')\",\n"
+    "    \"value\": \"string (the asserted fact, e.g., '140', 'Tuesday', '3000 KRW')\",\n"
+    "    \"claim_type\": \"string (must be one of: 'price', 'timing', 'existence', 'distance', 'weather', 'other')\"\n"
+    "  }\n"
+    "]"
+)

framework/verification/utils.py

@@ -0,0 +1,54 @@
+"""Verification subsystem helper utilities."""
+
+
+def normalize_key(text: str) -> str:
+    """Normalizes keys for robust string matching (lowercase alphanumeric)."""
+    return "".join(c for c in text.lower() if c.isalnum())
+
+
+def _clean_numeric_value(val: str) -> str:
+    """Removes common currency symbols, commas, and currency codes, leaving alphanumeric/numeric chars."""
+    val = val.upper()
+    for clean_target in [
+        "JPY",
+        "INR",
+        "USD",
+        "KRW",
+        "EUR",
+        "GBP",
+        "YEN",
+        "RUPEES",
+        "¥",
+        "₹",
+        "$",
+        "€",
+        "£",
+        ",",
+    ]:
+        val = val.replace(clean_target, "")
+    return val.strip()
+
+
+def values_are_equivalent(expected: str, actual: str) -> bool:
+    """Checks if two values are equivalent, handling numeric/currency normalizations."""
+    # 1. Direct normalized string check
+    norm_expected = expected.strip().lower()
+    norm_actual = actual.strip().lower()
+    if norm_expected == norm_actual:
+        return True
+
+    # 2. Try cleaning numeric values (e.g., "3000 JPY" vs "¥3,000")
+    clean_exp = _clean_numeric_value(expected)
+    clean_act = _clean_numeric_value(actual)
+
+    if clean_exp == clean_act:
+        return True
+
+    # 3. Try parsing as float if they are numeric
+    try:
+        if float(clean_exp) == float(clean_act):
+            return True
+    except ValueError:
+        pass
+
+    return False

ui/__init__.py

@@ -0,0 +1 @@
+"""Guided local web UI package for EvalRun."""