evalrun 0.4.0__py3-none-any.whl

framework/models.py

@@ -0,0 +1,83 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class Benchmark:
+    """Represents a single evaluation scenario.
+
+    Stores the parsed details of the benchmark scenario (e.g., from markdown).
+    """
+
+    benchmark_id: str
+    name: str
+    description: str
+    prompt: str
+    constraints: Dict[str, Any]
+    expected_behavior: List[str]
+    evaluation_criteria: Dict[str, List[str]]
+    pass_criteria: List[str]
+    failure_conditions: List[str]
+    notes: List[str]
+    profile: str = "travel-agent"
+
+
+@dataclass
+class AgentOutput:
+    """Represents the raw response/output produced by an AI agent."""
+
+    content: Any
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class DimensionScore:
+    """Stores the score and justification for a single evaluation dimension."""
+
+    dimension: str
+    score: float
+    reason: str
+
+
+@dataclass
+class EvaluationResult:
+    """Represents the final evaluation outcome for a benchmark scenario."""
+
+    benchmark_id: str
+    benchmark_name: str
+    overall_score: float
+    dimension_scores: List[DimensionScore]
+    passed: bool
+    agent_metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class EvaluationProfile:
+    """Defines the name and dimension weights for a specific evaluation domain."""
+
+    name: str
+    weights: Dict[str, float]
+    pass_threshold: float = 75.0
+
+
+@dataclass
+class ParsedSections:
+    """Represents the raw, un-transformed markdown sections parsed from a file."""
+
+    description: str
+    prompt: str
+    constraints: str
+    expected_behavior: str
+    evaluation_criteria: str
+    pass_criteria: str
+    failure_conditions: str
+    notes: Optional[str] = None
+
+
+@dataclass
+class ParsedBenchmark:
+    """Combines raw metadata and raw parsed sections before transformation."""
+
+    metadata: Dict[str, Any]
+    sections: ParsedSections
+

framework/parser.py

@@ -0,0 +1,45 @@
+"""Public API for parsing benchmark scenarios."""
+
+from pathlib import Path
+from typing import Union
+from framework.models import Benchmark, ParsedBenchmark
+from framework.parsers import parse_frontmatter, parse_sections, map_to_benchmark
+
+
+def parse_benchmark_text(text: str) -> Benchmark:
+    """Parses raw markdown text and maps it to a Benchmark dataclass.
+
+    Args:
+        text: The raw markdown text containing YAML frontmatter.
+
+    Returns:
+        A Benchmark dataclass instance.
+
+    Raises:
+        FrontmatterParseError: If YAML frontmatter parsing fails.
+        MarkdownParseError: If markdown section parsing fails.
+        MapperError: If mapping parsed content to Benchmark fails.
+    """
+    metadata, body = parse_frontmatter(text)
+    parsed_sections = parse_sections(body)
+    parsed_benchmark = ParsedBenchmark(metadata=metadata, sections=parsed_sections)
+    return map_to_benchmark(parsed_benchmark)
+
+
+def parse_benchmark(path: Union[str, Path]) -> Benchmark:
+    """Parses a markdown benchmark file on disk and maps it to a Benchmark dataclass.
+
+    Args:
+        path: Path to the markdown file (string or pathlib.Path).
+
+    Returns:
+        A Benchmark dataclass instance.
+
+    Raises:
+        FileNotFoundError: If the file does not exist.
+        FrontmatterParseError: If YAML frontmatter parsing fails.
+        MarkdownParseError: If markdown section parsing fails.
+        MapperError: If mapping parsed content to Benchmark fails.
+    """
+    text = Path(path).read_text(encoding="utf-8")
+    return parse_benchmark_text(text)

framework/parsers/__init__.py

@@ -0,0 +1,12 @@
+"""Package containing evaluation scenario parsers."""
+
+from framework.parsers.frontmatter import parse_frontmatter
+from framework.parsers.markdown import parse_sections
+from framework.parsers.mapper import map_to_benchmark
+from framework.parsers.transformers import parse_list, parse_nested_lists, parse_yaml_block
+from framework.exceptions import (
+    FrontmatterParseError,
+    MarkdownParseError,
+    TransformerError,
+    MapperError,
+)

framework/parsers/frontmatter.py

@@ -0,0 +1,28 @@
+"""Module for parsing YAML frontmatter from markdown documents."""
+
+from typing import Any, Dict, Tuple
+import frontmatter
+from framework.exceptions import FrontmatterParseError
+
+
+def parse_frontmatter(content: str) -> Tuple[Dict[str, Any], str]:
+    """Parses YAML frontmatter from a markdown string.
+
+    Args:
+        content: The raw markdown content, which may contain YAML frontmatter.
+
+    Returns:
+        A tuple of (metadata, remaining_content) where:
+            - metadata is a dictionary containing the parsed frontmatter fields.
+            - remaining_content is the markdown body without the frontmatter block.
+
+    Raises:
+        FrontmatterParseError: If frontmatter parsing fails due to malformed YAML.
+    """
+    try:
+        post = frontmatter.loads(content)
+        return post.metadata, post.content
+    except Exception as e:
+        raise FrontmatterParseError(
+            f"Failed to parse YAML frontmatter: {e}"
+        ) from e

framework/parsers/mapper.py

@@ -0,0 +1,66 @@
+"""Module for mapping parsed frontmatter and sections to a Benchmark dataclass."""
+
+from typing import Any, Dict
+from framework.exceptions import MapperError
+from framework.models import Benchmark, ParsedBenchmark
+from framework.parsers.transformers import parse_list, parse_nested_lists, parse_yaml_block
+
+
+def map_to_benchmark(parsed_benchmark: ParsedBenchmark) -> Benchmark:
+    """Maps a ParsedBenchmark (metadata + raw sections) to a Benchmark dataclass.
+
+    Args:
+        parsed_benchmark: The raw parsed benchmark components.
+
+    Returns:
+        A Benchmark dataclass instance.
+
+    Raises:
+        MapperError: If required metadata is missing or if transformations fail.
+    """
+    metadata = parsed_benchmark.metadata
+    sections = parsed_benchmark.sections
+
+    # 1. Validate and extract metadata
+    benchmark_id = metadata.get("benchmark_id")
+    if not benchmark_id:
+        raise MapperError("Metadata is missing required field: 'benchmark_id'")
+    if not isinstance(benchmark_id, str):
+        raise MapperError("'benchmark_id' must be a string")
+
+    name = metadata.get("name")
+    if not name:
+        raise MapperError("Metadata is missing required field: 'name'")
+    if not isinstance(name, str):
+        raise MapperError("'name' must be a string")
+
+    # 2. Map and transform sections using the transformers module
+    description = sections.description.strip()
+    prompt = sections.prompt.strip()
+
+    try:
+        constraints = parse_yaml_block(sections.constraints)
+    except Exception as e:
+        raise MapperError(f"Failed to parse constraints YAML: {e}") from e
+
+    expected_behavior = parse_list(sections.expected_behavior)
+    evaluation_criteria = parse_nested_lists(sections.evaluation_criteria)
+    pass_criteria = parse_list(sections.pass_criteria)
+    failure_conditions = parse_list(sections.failure_conditions)
+
+    notes = parse_list(sections.notes) if sections.notes else []
+    profile = metadata.get("profile", "travel-agent")
+
+    return Benchmark(
+        benchmark_id=benchmark_id,
+        name=name,
+        description=description,
+        prompt=prompt,
+        constraints=constraints,
+        expected_behavior=expected_behavior,
+        evaluation_criteria=evaluation_criteria,
+        pass_criteria=pass_criteria,
+        failure_conditions=failure_conditions,
+        notes=notes,
+        profile=profile,
+    )

framework/parsers/markdown.py

@@ -0,0 +1,122 @@
+"""Module for parsing sections from markdown bodies."""
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+# pyrefly: ignore [missing-import]
+from markdown_it import MarkdownIt
+from framework.exceptions import MarkdownParseError
+from framework.models import ParsedSections
+
+# Single shared parser instance at the module level to avoid repeated construction
+_MD = MarkdownIt()
+
+
+@dataclass
+class Heading:
+    """Represents a heading parsed from markdown."""
+
+    level: int
+    text: str
+    start_line: int
+    end_line: int
+
+
+def _require_section(sections: Dict[str, str], name: str) -> str:
+    """Ensures a required section is present in sections dictionary.
+
+    Args:
+        sections: Dictionary of headings to raw contents.
+        name: Name of the required section.
+
+    Returns:
+        The content of the required section.
+
+    Raises:
+        MarkdownParseError: If the section is missing.
+    """
+    if name not in sections:
+        raise MarkdownParseError(f"Missing required section: '{name}'")
+    return sections[name]
+
+
+def parse_sections(content: str) -> ParsedSections:
+    """Splits a markdown document body into sections and returns ParsedSections.
+
+    Automatically detects the highest heading level (H1, H2, etc.) present in
+    the document and splits the text into sections using those headings.
+
+    Args:
+        content: The raw markdown content body (without frontmatter).
+
+    Returns:
+        A ParsedSections dataclass containing raw markdown text for each section.
+
+    Raises:
+        MarkdownParseError: If any required section is missing.
+    """
+    tokens = _MD.parse(content)
+    lines = content.splitlines()
+
+    # Gather all headings and their line offsets using a simple stateful pass
+    headings: List[Heading] = []
+    current_heading: Optional[Heading] = None
+
+    for token in tokens:
+        if token.type == "heading_open":
+            if not token.map:
+                continue
+            current_heading = Heading(
+                level=int(token.tag[1]),
+                text="",
+                start_line=token.map[0],
+                end_line=token.map[1],
+            )
+        elif token.type == "inline" and current_heading is not None:
+            current_heading.text += token.content
+        elif token.type == "heading_close" and current_heading is not None:
+            # We match the tag (e.g. h1, h2) to ensure correct nesting closure
+            if token.tag == f"h{current_heading.level}":
+                headings.append(current_heading)
+                current_heading = None
+
+    if not headings:
+        raise MarkdownParseError("No headings found in the markdown body")
+
+    # Identify the primary (highest-level) headings to use as section splitters
+    primary_level = min(h.level for h in headings)
+    primary_headings = [h for h in headings if h.level == primary_level]
+
+    sections: Dict[str, str] = {}
+    for idx, heading in enumerate(primary_headings):
+        # The section content starts immediately after the heading
+        content_start = heading.end_line
+
+        # The section content ends where the next primary heading begins
+        if idx + 1 < len(primary_headings):
+            content_end = primary_headings[idx + 1].start_line
+        else:
+            content_end = len(lines)
+
+        section_content = "\n".join(lines[content_start:content_end]).strip()
+        sections[heading.text] = section_content
+
+    # Validate and map to ParsedSections dataclass
+    description = _require_section(sections, "Description")
+    prompt = _require_section(sections, "User Prompt")
+    constraints = _require_section(sections, "Extracted Constraints")
+    expected_behavior = _require_section(sections, "Expected Behaviour")
+    evaluation_criteria = _require_section(sections, "Evaluation Criteria")
+    pass_criteria = _require_section(sections, "Pass Criteria")
+    failure_conditions = _require_section(sections, "Failure Conditions")
+    notes = sections.get("Notes")
+
+    return ParsedSections(
+        description=description,
+        prompt=prompt,
+        constraints=constraints,
+        expected_behavior=expected_behavior,
+        evaluation_criteria=evaluation_criteria,
+        pass_criteria=pass_criteria,
+        failure_conditions=failure_conditions,
+        notes=notes,
+    )

framework/parsers/transformers.py

@@ -0,0 +1,112 @@
+"""Module for transforming raw markdown content into python structures."""
+
+from typing import Any, Dict, List
+import yaml
+# pyrefly: ignore [missing-import]
+from markdown_it import MarkdownIt
+from framework.exceptions import TransformerError
+
+# Single shared parser instance at the module level to avoid repeated construction
+_MD = MarkdownIt()
+
+
+def parse_list(content: str) -> List[str]:
+    """Parses all bulleted or numbered list items in markdown content.
+
+    Args:
+        content: The raw markdown content of the list section.
+
+    Returns:
+        A list of strings representing the text of each list item.
+    """
+    tokens = _MD.parse(content)
+    items: List[str] = []
+    current_item: List[str] = []
+    in_item = False
+
+    for token in tokens:
+        if token.type == "list_item_open":
+            in_item = True
+            current_item = []
+        elif token.type == "inline" and in_item:
+            current_item.append(token.content)
+        elif token.type == "list_item_close" and in_item:
+            items.append("".join(current_item).strip())
+            in_item = False
+
+    return items
+
+
+def parse_nested_lists(content: str) -> Dict[str, List[str]]:
+    """Parses subheadings and the list items directly beneath them.
+
+    Args:
+        content: The raw markdown containing headings and list items.
+
+    Returns:
+        A dictionary mapping each subheading's text to its list items.
+    """
+    tokens = _MD.parse(content)
+    result: Dict[str, List[str]] = {}
+
+    active_heading: List[str] = []
+    in_heading = False
+    active_item: List[str] = []
+    in_item = False
+    current_list: List[str] = []
+
+    for token in tokens:
+        if token.type == "heading_open":
+            # Save the previous heading and its list items if we had one
+            if active_heading:
+                result["".join(active_heading).strip()] = current_list
+            active_heading = []
+            in_heading = True
+            current_list = []
+        elif token.type == "inline" and in_heading:
+            active_heading.append(token.content)
+        elif token.type == "heading_close":
+            in_heading = False
+        elif token.type == "list_item_open" and active_heading:
+            in_item = True
+            active_item = []
+        elif token.type == "inline" and in_item:
+            active_item.append(token.content)
+        elif token.type == "list_item_close" and in_item:
+            current_list.append("".join(active_item).strip())
+            in_item = False
+
+    # Save the trailing heading
+    if active_heading:
+        result["".join(active_heading).strip()] = current_list
+
+    return result
+
+
+def parse_yaml_block(content: str) -> Dict[str, Any]:
+    """Finds and parses the first YAML code fence block in the markdown content.
+
+    Args:
+        content: The raw markdown content.
+
+    Returns:
+        A dictionary representation of the parsed YAML content.
+
+    Raises:
+        TransformerError: If YAML parsing fails.
+    """
+    tokens = _MD.parse(content)
+
+    for token in tokens:
+        if token.type == "fence" and token.info == "yaml":
+            try:
+                result = yaml.safe_load(token.content)
+                if result is None:
+                    return {}
+                if not isinstance(result, dict):
+                    raise TransformerError("YAML block content must represent a dictionary")
+                return result
+            except Exception as e:
+                raise TransformerError(f"Failed to parse YAML block: {e}") from e
+
+    return {}

framework/profiles/__init__.py

@@ -0,0 +1,28 @@
+"""Evaluation profiles package."""
+
+from framework.profiles.travel import (
+    TRAVEL_PROFILE,
+    TRAVEL_ROUTE_OPTIMIZATION_PROFILE,
+    TRAVEL_REMOTE_WORKER_TIMEZONES_PROFILE,
+    TRAVEL_MID_TRIP_REPLANNING_PROFILE,
+    TRAVEL_INFORMATION_GATHERING_UNCERTAINTY_PROFILE,
+    PROFILE_REGISTRY,
+)
+from framework.profiles.support import SUPPORT_TRIAGE_PROFILE
+
+# Keep one lookup used by BenchmarkRunner while allowing each domain to own
+# its profile definition.
+PROFILE_REGISTRY = {
+    **PROFILE_REGISTRY,
+    SUPPORT_TRIAGE_PROFILE.name: SUPPORT_TRIAGE_PROFILE,
+}
+
+__all__ = [
+    "TRAVEL_PROFILE",
+    "TRAVEL_ROUTE_OPTIMIZATION_PROFILE",
+    "TRAVEL_REMOTE_WORKER_TIMEZONES_PROFILE",
+    "TRAVEL_MID_TRIP_REPLANNING_PROFILE",
+    "TRAVEL_INFORMATION_GATHERING_UNCERTAINTY_PROFILE",
+    "SUPPORT_TRIAGE_PROFILE",
+    "PROFILE_REGISTRY",
+]

framework/profiles/registry.py

@@ -0,0 +1,104 @@
+"""Custom Evaluation Profiles and Evaluators Plugin Registry."""
+
+import importlib
+import json
+from pathlib import Path
+from typing import Any, Dict, Optional
+from framework.evaluation.base import BaseEvaluator
+from framework.models import EvaluationProfile
+
+
+_DYNAMIC_PROFILE_REGISTRY: Dict[str, EvaluationProfile] = {}
+_DYNAMIC_EVALUATOR_REGISTRY: Dict[str, BaseEvaluator] = {}
+
+
+def register_profile(name: str, profile: EvaluationProfile) -> None:
+    """Registers a custom evaluation profile into the global profile registry."""
+    _DYNAMIC_PROFILE_REGISTRY[name.lower()] = profile
+
+
+def get_custom_profile(name: str) -> Optional[EvaluationProfile]:
+    """Retrieves a registered profile by name."""
+    return _DYNAMIC_PROFILE_REGISTRY.get(name.lower())
+
+
+def register_evaluator_plugin(dimension_name: str, evaluator: BaseEvaluator) -> None:
+    """Registers a custom evaluator plugin for a dimension name."""
+    _DYNAMIC_EVALUATOR_REGISTRY[dimension_name] = evaluator
+    _DYNAMIC_EVALUATOR_REGISTRY[dimension_name.lower()] = evaluator
+
+
+def get_evaluator_plugin(dimension_name: str) -> Optional[BaseEvaluator]:
+    """Retrieves a registered custom evaluator plugin by dimension name."""
+    return _DYNAMIC_EVALUATOR_REGISTRY.get(dimension_name) or _DYNAMIC_EVALUATOR_REGISTRY.get(dimension_name.lower())
+
+
+def load_evaluator_plugin_from_spec(spec: str, judge_llm: Any = None) -> BaseEvaluator:
+    """Dynamically loads and instantiates a custom evaluator plugin from a specifier.
+
+    Args:
+        spec: Import specifier string ('module:Class' or 'module:factory').
+        judge_llm: Optional judge LLM instance to pass to the evaluator constructor.
+
+    Returns:
+        Instantiated BaseEvaluator.
+    """
+    if ":" not in spec:
+        raise ValueError(f"Invalid evaluator specifier '{spec}'. Expected 'module:Class' or 'module:factory'.")
+
+    mod_name, attr_name = spec.split(":", 1)
+    module = importlib.import_module(mod_name)
+    target = getattr(module, attr_name)
+
+    if inspect_is_class(target):
+        try:
+            return target(llm=judge_llm)
+        except TypeError:
+            try:
+                return target(judge_llm=judge_llm)
+            except TypeError:
+                return target()
+    elif callable(target):
+        try:
+            return target(llm=judge_llm)
+        except TypeError:
+            return target()
+    else:
+        raise ValueError(f"Evaluator specifier '{spec}' resolved to non-callable object.")
+
+
+def inspect_is_class(obj: Any) -> bool:
+    import inspect
+    return inspect.isclass(obj)
+
+
+def load_profile_from_file(filepath: str) -> EvaluationProfile:
+    """Loads an EvaluationProfile from a JSON configuration file."""
+    path = Path(filepath)
+    if not path.exists():
+        raise FileNotFoundError(f"Custom profile file '{filepath}' not found.")
+
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    profile_id = data.get("profile_id") or path.stem
+    name = data.get("name", profile_id)
+    pass_threshold = float(data.get("pass_threshold", 75.0))
+
+    raw_weights = data.get("dimension_weights") or data.get("weights") or {}
+    weights: Dict[str, float] = {}
+    if isinstance(raw_weights, list):
+        for item in raw_weights:
+            if isinstance(item, dict) and "dimension" in item:
+                weights[item["dimension"]] = float(item.get("weight", 1.0))
+    elif isinstance(raw_weights, dict):
+        weights = {str(k): float(v) for k, v in raw_weights.items()}
+
+    profile = EvaluationProfile(
+        name=name,
+        weights=weights,
+        pass_threshold=pass_threshold,
+    )
+
+    register_profile(profile_id, profile)
+    return profile

framework/profiles/support.py

@@ -0,0 +1,27 @@
+"""Evaluation profile for the minimal support-triage benchmark."""
+
+from framework.evaluation.dimensions import (
+    ADAPTABILITY,
+    CONSTRAINT_SATISFACTION,
+    INFORMATION_ACCURACY,
+    PERSONALIZATION,
+    PLANNING_QUALITY,
+)
+from framework.models import EvaluationProfile
+
+
+# The existing dimensions are intentionally reused for this small proof of
+# generalization. Their support interpretation is documented in the scenario:
+# constraints=SLA/policy, planning=triage actions, accuracy=category/facts,
+# personalization=tone, adaptability=escalation judgment.
+SUPPORT_TRIAGE_PROFILE = EvaluationProfile(
+    name="support-triage",
+    weights={
+        CONSTRAINT_SATISFACTION: 30.0,
+        INFORMATION_ACCURACY: 25.0,
+        PLANNING_QUALITY: 20.0,
+        ADAPTABILITY: 15.0,
+        PERSONALIZATION: 10.0,
+    },
+    pass_threshold=75.0,
+)