harness-eval 5.0.0__py3-none-any.whl

harness_eval/__init__.py

@@ -0,0 +1,3 @@
+"""harness-eval: Evaluate and compare AI agent setups."""
+
+__version__ = "0.1.0"

harness_eval/analysis/budget.py

@@ -0,0 +1,74 @@
+"""Token budget analysis for the setup as a whole."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from harness_eval.core.types import ComponentType, ParsedComponent, Setup
+
+
+@dataclass
+class BudgetReport:
+    """Token budget breakdown for a setup."""
+
+    total_tokens: int = 0
+    always_loaded_tokens: int = 0
+    on_demand_tokens: int = 0
+    always_loaded_ratio: float = 0.0
+    by_type: dict[str, int] = field(default_factory=dict)
+    by_component: list[tuple[str, str, int]] = field(default_factory=list)
+    heaviest_component_name: str = ""
+    heaviest_component_ratio: float = 0.0
+
+
+_EXCLUDED_FROM_BUDGET = {ComponentType.UNCATEGORIZED}
+
+
+def _is_always_loaded(comp: ParsedComponent) -> bool:
+    if comp.component_type == ComponentType.HOOKS:
+        return True
+    if comp.component_type != ComponentType.CLAUDE_MD:
+        return False
+    if comp.source_tool != "cursor":
+        return True
+    return bool(comp.frontmatter and comp.frontmatter.get("alwaysApply") is True)
+
+
+def analyze_budget(setup: Setup) -> BudgetReport:
+    """Analyze token budget distribution across a setup."""
+    by_type: dict[str, int] = {}
+    by_component: list[tuple[str, str, int]] = []
+    always_loaded = 0
+    on_demand = 0
+    heaviest_name = ""
+    heaviest_tokens = 0
+
+    for comp in setup.components:
+        if comp.component_type in _EXCLUDED_FROM_BUDGET:
+            continue
+        type_key = comp.component_type.value
+        by_type[type_key] = by_type.get(type_key, 0) + comp.token_count
+        by_component.append((type_key, comp.name, comp.token_count))
+
+        if _is_always_loaded(comp):
+            always_loaded += comp.token_count
+        else:
+            on_demand += comp.token_count
+
+        if comp.token_count > heaviest_tokens:
+            heaviest_tokens = comp.token_count
+            heaviest_name = f"{type_key}/{comp.name}"
+
+    total = always_loaded + on_demand
+    by_component.sort(key=lambda x: x[2], reverse=True)
+
+    return BudgetReport(
+        total_tokens=total,
+        always_loaded_tokens=always_loaded,
+        on_demand_tokens=on_demand,
+        always_loaded_ratio=always_loaded / total if total else 0.0,
+        by_type=by_type,
+        by_component=by_component,
+        heaviest_component_name=heaviest_name,
+        heaviest_component_ratio=heaviest_tokens / total if total else 0.0,
+    )

harness_eval/analysis/context_utilization.py

@@ -0,0 +1,90 @@
+"""Context utilization analysis: how much of each model's context window the setup consumes."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from harness_eval.analysis.budget import BudgetReport
+from harness_eval.core.types import Setup
+
+
+@dataclass
+class ModelSpec:
+    """A model with a known context window size."""
+
+    name: str
+    context_window: int
+
+
+@dataclass
+class ModelUtilization:
+    """Context utilization for one model."""
+
+    model: str
+    context_window: int
+    always_loaded_pct: float
+    peak_load_pct: float
+    remaining_pct: float
+    warning: bool
+
+
+@dataclass
+class ContextUtilizationReport:
+    """Context utilization across all target models."""
+
+    always_loaded_tokens: int = 0
+    peak_tokens: int = 0
+    models: list[ModelUtilization] = field(default_factory=list)
+
+
+PEAK_WARNING_THRESHOLD = 0.20
+PEAK_CRITICAL_THRESHOLD = 0.50
+ALWAYS_LOADED_WARNING_THRESHOLD = 0.10
+
+DEFAULT_MODELS: list[ModelSpec] = [
+    ModelSpec("claude-haiku-4.5", 200_000),
+    ModelSpec("claude-sonnet-4.6", 200_000),
+    ModelSpec("claude-opus-4.6", 200_000),
+    ModelSpec("claude-opus-4.7", 200_000),
+    ModelSpec("claude-opus-4.8", 200_000),
+    ModelSpec("claude-opus-4.6-1m", 1_000_000),
+    ModelSpec("claude-opus-4.7-1m", 1_000_000),
+    ModelSpec("claude-opus-4.8-1m", 1_000_000),
+]
+
+
+def analyze_context_utilization(
+    setup: Setup,
+    budget: BudgetReport,
+    models: list[ModelSpec] | None = None,
+) -> ContextUtilizationReport:
+    """Compute context window utilization per model."""
+    target_models = models if models is not None else DEFAULT_MODELS
+    always_loaded = budget.always_loaded_tokens
+    peak = budget.total_tokens
+
+    utilizations: list[ModelUtilization] = []
+    for spec in target_models:
+        if spec.context_window <= 0:
+            continue
+        al_pct = always_loaded / spec.context_window
+        peak_pct = peak / spec.context_window
+        remaining = max(0.0, 1.0 - peak_pct)
+        utilizations.append(
+            ModelUtilization(
+                model=spec.name,
+                context_window=spec.context_window,
+                always_loaded_pct=al_pct,
+                peak_load_pct=peak_pct,
+                remaining_pct=remaining,
+                warning=peak_pct > PEAK_WARNING_THRESHOLD,
+            )
+        )
+
+    utilizations.sort(key=lambda m: m.peak_load_pct, reverse=True)
+
+    return ContextUtilizationReport(
+        always_loaded_tokens=always_loaded,
+        peak_tokens=peak,
+        models=utilizations,
+    )

harness_eval/analysis/dependencies.py

@@ -0,0 +1,111 @@
+"""Dependency mapping: which components reference which, and are any broken?"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from harness_eval.core.types import ComponentType, Setup
+
+
+@dataclass
+class DependencyEdge:
+    """A reference from one component to another."""
+
+    source_type: str
+    source_name: str
+    target_type: str
+    target_name: str
+    exists: bool
+
+
+@dataclass
+class DependencyReport:
+    """Dependency analysis results."""
+
+    edges: list[DependencyEdge] = field(default_factory=list)
+    broken_refs: list[DependencyEdge] = field(default_factory=list)
+    orphan_components: list[str] = field(default_factory=list)
+
+
+_SKILL_REF_PATTERNS = [
+    re.compile(r"skills?[/\\](\w[\w-]*)"),
+    re.compile(r"/(\w[\w-]*)\s+skill"),
+    re.compile(r"skill\s+['\"](\w[\w-]*)['\"]"),
+]
+
+
+def analyze_dependencies(setup: Setup) -> DependencyReport:
+    """Map dependencies between components and find broken references."""
+    known_skills = {c.name for c in setup.by_type(ComponentType.SKILL)}
+    known_commands = {c.name for c in setup.by_type(ComponentType.COMMAND)}
+    all_known = known_skills | known_commands
+
+    edges: list[DependencyEdge] = []
+
+    for comp in setup.by_type(ComponentType.AGENT):
+        if comp.frontmatter:
+            referenced = comp.frontmatter.get("skills", []) or []
+            if isinstance(referenced, str):
+                referenced = [s.strip() for s in referenced.split(",")]
+            for skill_name in referenced:
+                if not skill_name:
+                    continue
+                exists = skill_name in known_skills
+                edges.append(
+                    DependencyEdge(
+                        source_type="agent",
+                        source_name=comp.name,
+                        target_type="skill",
+                        target_name=skill_name,
+                        exists=exists,
+                    )
+                )
+
+    for comp in setup.components:
+        if comp.component_type in (ComponentType.MCP_CONFIG, ComponentType.HOOKS):
+            continue
+        for pattern in _SKILL_REF_PATTERNS:
+            for match in pattern.finditer(comp.content):
+                ref_name = match.group(1)
+                if ref_name in all_known and ref_name != comp.name:
+                    edges.append(
+                        DependencyEdge(
+                            source_type=comp.component_type.value,
+                            source_name=comp.name,
+                            target_type="skill" if ref_name in known_skills else "command",
+                            target_name=ref_name,
+                            exists=True,
+                        )
+                    )
+
+    seen = set()
+    deduped: list[DependencyEdge] = []
+    for e in edges:
+        key = (e.source_type, e.source_name, e.target_type, e.target_name)
+        if key not in seen:
+            seen.add(key)
+            deduped.append(e)
+
+    broken = [e for e in deduped if not e.exists]
+
+    referenced_names = set()
+    for e in deduped:
+        referenced_names.add(e.target_name)
+        referenced_names.add(e.source_name)
+
+    orphans = []
+    # Skills are activated by description matching, not by explicit references,
+    # so an unreferenced skill is not orphaned. Only flag commands and agents.
+    orphan_types = (ComponentType.COMMAND, ComponentType.AGENT)
+    for comp in setup.components:
+        if comp.component_type not in orphan_types:
+            continue
+        if comp.name not in referenced_names and len(setup.components) > 3:
+            orphans.append(f"{comp.component_type.value}/{comp.name}")
+
+    return DependencyReport(
+        edges=deduped,
+        broken_refs=broken,
+        orphan_components=orphans,
+    )

harness_eval/analysis/system.py

@@ -0,0 +1,95 @@
+"""System-level analysis: evaluate the setup as a whole, not component-by-component."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from harness_eval.analysis.budget import BudgetReport, analyze_budget
+from harness_eval.analysis.context_utilization import (
+    ContextUtilizationReport,
+    analyze_context_utilization,
+)
+from harness_eval.analysis.dependencies import DependencyReport, analyze_dependencies
+from harness_eval.analysis.triggers import TriggerReport, analyze_triggers
+from harness_eval.core.types import Setup
+
+
+@dataclass
+class SystemReport:
+    """Complete system-level analysis of a setup."""
+
+    setup_name: str
+    component_count: int
+    budget: BudgetReport
+    triggers: TriggerReport
+    dependencies: DependencyReport
+    context_utilization: ContextUtilizationReport
+    findings: list[str] = field(default_factory=list)
+    uncategorized_files: list[str] = field(default_factory=list)
+    detected_tools: tuple[str, ...] = ()
+
+
+def analyze_system(setup: Setup) -> SystemReport:
+    """Run all system-level analyses on a setup."""
+    budget = analyze_budget(setup)
+    triggers = analyze_triggers(setup)
+    dependencies = analyze_dependencies(setup)
+    context_utilization = analyze_context_utilization(setup, budget)
+
+    findings: list[str] = []
+
+    tool_set = set(setup.detected_tools)
+    tool_set.discard("Third-party modules")
+    if not tool_set:
+        always_label = "system instructions"
+    elif tool_set == {"Cursor"}:
+        always_label = "cursor rules"
+    elif len(tool_set) > 1:
+        always_label = "system instructions"
+    elif "Gemini CLI" in tool_set:
+        always_label = "GEMINI.md"
+    elif "OpenCode" in tool_set:
+        always_label = "AGENTS.md"
+    elif "Copilot" in tool_set:
+        always_label = "system instructions"
+    else:
+        always_label = "CLAUDE.md"
+
+    if budget.always_loaded_ratio > 0.7:
+        findings.append(
+            f"Inverted budget: {budget.always_loaded_ratio:.0%} of tokens are "
+            f"always-loaded ({always_label}). Most content should be in on-demand skills."
+        )
+    elif budget.always_loaded_ratio > 0.5:
+        findings.append(
+            f"Heavy always-loaded ratio: {budget.always_loaded_ratio:.0%} of tokens "
+            f"are always-loaded. Consider moving content to on-demand skills."
+        )
+
+    if budget.heaviest_component_ratio > 0.5:
+        findings.append(
+            f"One component ({budget.heaviest_component_name}) uses "
+            f"{budget.heaviest_component_ratio:.0%} of the total token budget."
+        )
+
+    for pair in triggers.overlap_pairs:
+        findings.append(
+            f"Trigger overlap: '{pair[0]}' and '{pair[1]}' have "
+            f"{pair[2]:.0%} description similarity, may load together."
+        )
+
+    from harness_eval.core.types import ComponentType as CT
+
+    uncategorized = [c.name for c in setup.by_type(CT.UNCATEGORIZED)]
+
+    return SystemReport(
+        setup_name=setup.name,
+        component_count=len([c for c in setup.components if c.component_type != CT.UNCATEGORIZED]),
+        budget=budget,
+        triggers=triggers,
+        dependencies=dependencies,
+        context_utilization=context_utilization,
+        findings=findings,
+        uncategorized_files=uncategorized,
+        detected_tools=setup.detected_tools,
+    )

harness_eval/analysis/triggers.py

@@ -0,0 +1,63 @@
+"""Trigger overlap analysis: detect skills that would load simultaneously."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from harness_eval.core.types import ComponentType, Setup
+from harness_eval.utils.similarity import tfidf_similarity
+
+TRIGGER_OVERLAP_THRESHOLD = 0.50
+
+
+@dataclass
+class TriggerReport:
+    """Trigger overlap analysis results."""
+
+    skill_count: int = 0
+    skills_with_description: int = 0
+    skills_without_description: int = 0
+    overlap_pairs: list[tuple[str, str, float]] = field(default_factory=list)
+    missing_use_when: list[str] = field(default_factory=list)
+
+
+def analyze_triggers(setup: Setup) -> TriggerReport:
+    """Analyze skill trigger descriptions for overlap and quality."""
+    skills = setup.by_type(ComponentType.SKILL)
+    report = TriggerReport(skill_count=len(skills))
+
+    descriptions: dict[str, str] = {}
+
+    for comp in skills:
+        desc = ""
+        if comp.frontmatter and isinstance(comp.frontmatter.get("description"), str):
+            desc = comp.frontmatter["description"]
+
+        if desc:
+            report.skills_with_description += 1
+            descriptions[comp.name] = desc
+
+            desc_lower = desc.lower()
+            use_when_phrases = [
+                "use when",
+                "use for",
+                "applies to",
+                "relevant for",
+                "triggered by",
+                "invoke when",
+            ]
+            if not any(phrase in desc_lower for phrase in use_when_phrases):
+                report.missing_use_when.append(comp.name)
+        else:
+            report.skills_without_description += 1
+
+    names = list(descriptions.keys())
+    for i in range(len(names)):
+        for j in range(i + 1, len(names)):
+            sim = tfidf_similarity(descriptions[names[i]], descriptions[names[j]])
+            if sim >= TRIGGER_OVERLAP_THRESHOLD:
+                report.overlap_pairs.append((names[i], names[j], sim))
+
+    report.overlap_pairs.sort(key=lambda x: x[2], reverse=True)
+
+    return report

harness_eval/analysis/types.py

@@ -0,0 +1,40 @@
+"""Types for cross-analysis."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class SetupComparison:
+    """Comparison of inspection results between two setups."""
+
+    setup_a: str
+    setup_b: str
+    shared_components: list[str] = field(default_factory=list)
+    only_in_a: list[str] = field(default_factory=list)
+    only_in_b: list[str] = field(default_factory=list)
+    score_deltas: dict[str, int] = field(default_factory=dict)
+
+
+@dataclass
+class Correlation:
+    """A correlation between a component difference and an inspection score difference."""
+
+    component_name: str
+    component_type: str
+    setup_name: str
+    error_count: int
+    warning_count: int
+    impact: str  # "positive", "negative", "neutral"
+    explanation: str
+
+
+@dataclass
+class CrossAnalysisResult:
+    """Complete cross-analysis result."""
+
+    comparison: SetupComparison
+    correlations: list[Correlation] = field(default_factory=list)
+    summary: str = ""
+    setup_scores: dict[str, dict[str, int]] = field(default_factory=dict)

harness_eval/cli/__init__.py

@@ -0,0 +1,13 @@
+"""CLI entry point for harness-eval."""
+
+from __future__ import annotations
+
+import click
+
+cli = click.Group(name="harness-eval", help="Evaluate AI agent setups.")
+click.version_option(package_name="harness-eval")(cli)
+
+from harness_eval.cli import lint as _lint  # noqa: E402, F401
+from harness_eval.cli import review as _review  # noqa: E402, F401
+from harness_eval.cli import security as _security  # noqa: E402, F401
+from harness_eval.cli import skill as _skill  # noqa: E402, F401

harness_eval/cli/_helpers.py

@@ -0,0 +1,14 @@
+"""Shared CLI helpers."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import click
+
+
+def emit_output(text: str, output_path: str | None) -> None:
+    if output_path:
+        Path(output_path).write_text(text)
+    else:
+        click.echo(text)