mcpbr 0.4.16__py3-none-any.whl → 0.6.0__py3-none-any.whl

mcpbr/task_scheduler.py

@@ -0,0 +1,468 @@
+"""Task prioritization and scheduling for benchmark evaluations.
+
+Provides intelligent task ordering strategies to optimize benchmark runs
+for speed, cost, coverage diversity, or custom scoring functions. Tasks
+can be reordered before execution to get faster feedback, reduce costs,
+or ensure diverse coverage across repositories and categories.
+
+Addresses GitHub issue #92: Task Prioritization and Scheduling.
+"""
+
+from collections import defaultdict
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class SchedulingStrategy(Enum):
+    """Strategy for ordering benchmark tasks before execution.
+
+    Attributes:
+        DEFAULT: Preserve original task order (no reordering).
+        SPEED_FIRST: Run fastest tasks first for quick feedback.
+        COST_FIRST: Run cheapest tasks first to minimize early spend.
+        COVERAGE_FIRST: Round-robin across categories/repos for diverse early results.
+        CUSTOM: Use a user-provided scoring function.
+    """
+
+    DEFAULT = "default"
+    SPEED_FIRST = "speed"
+    COST_FIRST = "cost"
+    COVERAGE_FIRST = "coverage"
+    CUSTOM = "custom"
+
+
+@dataclass
+class TaskPriority:
+    """Priority metadata for a single benchmark task.
+
+    Attributes:
+        task_id: Unique identifier for the task (e.g., instance_id).
+        priority_score: Computed priority score (lower = higher priority / runs first).
+        estimated_time_seconds: Rough estimate of task execution time in seconds.
+        estimated_cost_usd: Rough estimate of task cost in USD.
+        category: Category or grouping key (e.g., repo name, difficulty level).
+        metadata: Additional metadata associated with the task.
+    """
+
+    task_id: str
+    priority_score: float = 0.0
+    estimated_time_seconds: float | None = None
+    estimated_cost_usd: float | None = None
+    category: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+# Default token-per-character ratio for cost estimation.
+# Based on empirical observation that ~4 characters is roughly 1 token for English text.
+_DEFAULT_CHARS_PER_TOKEN = 4
+
+# Default assumed output-to-input token ratio for agent tasks.
+# Agents typically produce 3-5x more output than input for coding tasks.
+_DEFAULT_OUTPUT_INPUT_RATIO = 4.0
+
+# Baseline seconds per 1000 characters of problem statement.
+# Longer problems tend to require more exploration and tool calls.
+_DEFAULT_SECONDS_PER_KCHAR = 30.0
+
+# Minimum estimated time for any task (seconds).
+_MIN_ESTIMATED_TIME = 30.0
+
+# Minimum estimated cost for any task (USD).
+_MIN_ESTIMATED_COST = 0.001
+
+
+class TaskScheduler:
+    """Scheduler that reorders benchmark tasks based on a chosen strategy.
+
+    The scheduler assigns priority scores to tasks and returns them in
+    sorted order. It supports preset strategies (speed, cost, coverage)
+    and custom scoring functions.
+
+    Args:
+        strategy: The scheduling strategy to use.
+        custom_scorer: A callable that takes a task dict and returns a float
+            priority score (lower = runs first). Required when strategy is CUSTOM.
+
+    Raises:
+        ValueError: If strategy is CUSTOM but no custom_scorer is provided.
+
+    Example:
+        >>> scheduler = TaskScheduler(strategy=SchedulingStrategy.SPEED_FIRST)
+        >>> ordered = scheduler.schedule(tasks)
+        >>> print(scheduler.preview(tasks))
+    """
+
+    def __init__(
+        self,
+        strategy: SchedulingStrategy = SchedulingStrategy.DEFAULT,
+        custom_scorer: Callable[[dict[str, Any]], float] | None = None,
+    ) -> None:
+        if strategy == SchedulingStrategy.CUSTOM and custom_scorer is None:
+            raise ValueError(
+                "custom_scorer is required when strategy is CUSTOM. "
+                "Provide a callable that takes a task dict and returns a float score."
+            )
+        self._strategy = strategy
+        self._custom_scorer = custom_scorer
+
+    @property
+    def strategy(self) -> SchedulingStrategy:
+        """The active scheduling strategy."""
+        return self._strategy
+
+    def schedule(self, tasks: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        """Reorder tasks according to the active scheduling strategy.
+
+        Args:
+            tasks: List of task dictionaries to schedule. Each task should have
+                at least an ``instance_id`` key. Additional keys like
+                ``problem_statement``, ``repo``, and ``category`` improve
+                estimation accuracy.
+
+        Returns:
+            A new list of task dictionaries in the scheduled order.
+            The original list is not mutated.
+        """
+        if not tasks:
+            return []
+
+        if self._strategy == SchedulingStrategy.DEFAULT:
+            return list(tasks)
+
+        priorities = self._compute_priorities(tasks)
+
+        # Sort by priority_score ascending (lower = runs first)
+        priorities.sort(key=lambda p: p.priority_score)
+
+        # Build task lookup by id for efficient reordering
+        task_by_id: dict[str, dict[str, Any]] = {}
+        for task in tasks:
+            tid = task.get("instance_id", str(id(task)))
+            task_by_id[tid] = task
+
+        return [task_by_id[p.task_id] for p in priorities]
+
+    def preview(self, tasks: list[dict[str, Any]]) -> str:
+        """Generate a human-readable preview of the scheduled task order.
+
+        Args:
+            tasks: List of task dictionaries to preview.
+
+        Returns:
+            A formatted string showing the scheduled order with priority
+            details, suitable for display before execution.
+        """
+        if not tasks:
+            return "No tasks to schedule."
+
+        scheduled = self.schedule(tasks)
+        priorities = self._compute_priorities(tasks)
+        priorities.sort(key=lambda p: p.priority_score)
+
+        # Build a lookup for priority info
+        priority_by_id: dict[str, TaskPriority] = {p.task_id: p for p in priorities}
+
+        lines: list[str] = []
+        lines.append(f"Schedule Preview (strategy: {self._strategy.value})")
+        lines.append(f"Total tasks: {len(scheduled)}")
+        lines.append("-" * 70)
+        lines.append(f"{'#':<4} {'Task ID':<35} {'Score':<8} {'Est. Time':<12} {'Est. Cost':<10}")
+        lines.append("-" * 70)
+
+        for i, task in enumerate(scheduled, start=1):
+            tid = task.get("instance_id", str(id(task)))
+            priority = priority_by_id.get(tid)
+
+            if priority is not None:
+                score_str = f"{priority.priority_score:.2f}"
+                time_str = (
+                    f"{priority.estimated_time_seconds:.0f}s"
+                    if priority.estimated_time_seconds is not None
+                    else "N/A"
+                )
+                cost_str = (
+                    f"${priority.estimated_cost_usd:.4f}"
+                    if priority.estimated_cost_usd is not None
+                    else "N/A"
+                )
+            else:
+                score_str = "N/A"
+                time_str = "N/A"
+                cost_str = "N/A"
+
+            lines.append(f"{i:<4} {tid:<35} {score_str:<8} {time_str:<12} {cost_str:<10}")
+
+        lines.append("-" * 70)
+        return "\n".join(lines)
+
+    def estimate_task_cost(self, task: dict[str, Any], model: str = "sonnet") -> float:
+        """Estimate the cost of running a single task in USD.
+
+        The estimate is based on the length of the problem statement and
+        the model's pricing. Longer problems produce more tokens and cost more.
+
+        Args:
+            task: Task dictionary, ideally containing a ``problem_statement`` key.
+            model: Model identifier used for pricing lookup (default: ``"sonnet"``).
+
+        Returns:
+            Estimated cost in USD. Returns ``_MIN_ESTIMATED_COST`` if pricing
+            data is unavailable or the problem statement is missing.
+        """
+        from .pricing import get_model_pricing
+
+        problem = task.get("problem_statement", "")
+        problem_len = len(problem) if isinstance(problem, str) else 0
+
+        pricing = get_model_pricing(model)
+        if pricing is None:
+            return _MIN_ESTIMATED_COST
+
+        # Estimate input tokens from problem length
+        input_tokens = max(problem_len / _DEFAULT_CHARS_PER_TOKEN, 100)
+
+        # Estimate output tokens as a multiple of input
+        output_tokens = input_tokens * _DEFAULT_OUTPUT_INPUT_RATIO
+
+        # Calculate cost in USD
+        input_cost = (input_tokens / 1_000_000) * pricing.input_price_per_mtok
+        output_cost = (output_tokens / 1_000_000) * pricing.output_price_per_mtok
+
+        return max(input_cost + output_cost, _MIN_ESTIMATED_COST)
+
+    def estimate_task_time(self, task: dict[str, Any]) -> float:
+        """Estimate the execution time for a single task in seconds.
+
+        The estimate is based on the length of the problem statement.
+        Longer problems typically require more exploration time.
+
+        Args:
+            task: Task dictionary, ideally containing a ``problem_statement`` key.
+
+        Returns:
+            Estimated execution time in seconds (minimum ``_MIN_ESTIMATED_TIME``).
+        """
+        problem = task.get("problem_statement", "")
+        problem_len = len(problem) if isinstance(problem, str) else 0
+
+        # Scale linearly with problem length
+        estimated = (problem_len / 1000) * _DEFAULT_SECONDS_PER_KCHAR
+
+        return max(estimated, _MIN_ESTIMATED_TIME)
+
+    def _compute_priorities(self, tasks: list[dict[str, Any]]) -> list[TaskPriority]:
+        """Compute priority scores for all tasks based on the active strategy.
+
+        Args:
+            tasks: List of task dictionaries.
+
+        Returns:
+            List of TaskPriority objects with computed scores.
+        """
+        if self._strategy == SchedulingStrategy.SPEED_FIRST:
+            return self._prioritize_by_speed(tasks)
+        elif self._strategy == SchedulingStrategy.COST_FIRST:
+            return self._prioritize_by_cost(tasks)
+        elif self._strategy == SchedulingStrategy.COVERAGE_FIRST:
+            return self._prioritize_by_coverage(tasks)
+        elif self._strategy == SchedulingStrategy.CUSTOM:
+            return self._prioritize_by_custom(tasks)
+        else:
+            # DEFAULT: preserve original order via index-based scoring
+            return [
+                TaskPriority(
+                    task_id=task.get("instance_id", str(id(task))),
+                    priority_score=float(i),
+                )
+                for i, task in enumerate(tasks)
+            ]
+
+    def _prioritize_by_speed(self, tasks: list[dict[str, Any]]) -> list[TaskPriority]:
+        """Assign priority scores based on estimated execution time (ascending).
+
+        Args:
+            tasks: List of task dictionaries.
+
+        Returns:
+            List of TaskPriority objects scored by estimated time.
+        """
+        priorities: list[TaskPriority] = []
+        for task in tasks:
+            tid = task.get("instance_id", str(id(task)))
+            est_time = self.estimate_task_time(task)
+            est_cost = self.estimate_task_cost(task)
+            category = _extract_category(task)
+
+            priorities.append(
+                TaskPriority(
+                    task_id=tid,
+                    priority_score=est_time,
+                    estimated_time_seconds=est_time,
+                    estimated_cost_usd=est_cost,
+                    category=category,
+                )
+            )
+        return priorities
+
+    def _prioritize_by_cost(self, tasks: list[dict[str, Any]]) -> list[TaskPriority]:
+        """Assign priority scores based on estimated cost (ascending).
+
+        Args:
+            tasks: List of task dictionaries.
+
+        Returns:
+            List of TaskPriority objects scored by estimated cost.
+        """
+        priorities: list[TaskPriority] = []
+        for task in tasks:
+            tid = task.get("instance_id", str(id(task)))
+            est_time = self.estimate_task_time(task)
+            est_cost = self.estimate_task_cost(task)
+            category = _extract_category(task)
+
+            priorities.append(
+                TaskPriority(
+                    task_id=tid,
+                    priority_score=est_cost,
+                    estimated_time_seconds=est_time,
+                    estimated_cost_usd=est_cost,
+                    category=category,
+                )
+            )
+        return priorities
+
+    def _prioritize_by_coverage(self, tasks: list[dict[str, Any]]) -> list[TaskPriority]:
+        """Assign priority scores using round-robin across categories.
+
+        Tasks are grouped by category (repo, difficulty, or explicit category),
+        then interleaved so that early execution covers diverse categories.
+
+        Args:
+            tasks: List of task dictionaries.
+
+        Returns:
+            List of TaskPriority objects with interleaved category ordering.
+        """
+        # Group tasks by category
+        groups: dict[str, list[dict[str, Any]]] = defaultdict(list)
+        for task in tasks:
+            category = _extract_category(task) or "_uncategorized_"
+            groups[category].append(task)
+
+        # Sort group keys for deterministic ordering
+        sorted_keys = sorted(groups.keys())
+
+        # Round-robin interleave: take one task from each category in turn
+        result: list[TaskPriority] = []
+        score = 0.0
+        max_group_len = max(len(g) for g in groups.values()) if groups else 0
+
+        for round_idx in range(max_group_len):
+            for key in sorted_keys:
+                group = groups[key]
+                if round_idx < len(group):
+                    task = group[round_idx]
+                    tid = task.get("instance_id", str(id(task)))
+                    est_time = self.estimate_task_time(task)
+                    est_cost = self.estimate_task_cost(task)
+
+                    result.append(
+                        TaskPriority(
+                            task_id=tid,
+                            priority_score=score,
+                            estimated_time_seconds=est_time,
+                            estimated_cost_usd=est_cost,
+                            category=key,
+                        )
+                    )
+                    score += 1.0
+
+        return result
+
+    def _prioritize_by_custom(self, tasks: list[dict[str, Any]]) -> list[TaskPriority]:
+        """Assign priority scores using the user-provided custom scorer.
+
+        Args:
+            tasks: List of task dictionaries.
+
+        Returns:
+            List of TaskPriority objects scored by the custom function.
+
+        Raises:
+            RuntimeError: If custom_scorer is None (should not happen due to
+                __init__ validation).
+        """
+        if self._custom_scorer is None:
+            raise RuntimeError("custom_scorer is None but strategy is CUSTOM")
+
+        priorities: list[TaskPriority] = []
+        for task in tasks:
+            tid = task.get("instance_id", str(id(task)))
+            score = self._custom_scorer(task)
+            est_time = self.estimate_task_time(task)
+            est_cost = self.estimate_task_cost(task)
+            category = _extract_category(task)
+
+            priorities.append(
+                TaskPriority(
+                    task_id=tid,
+                    priority_score=score,
+                    estimated_time_seconds=est_time,
+                    estimated_cost_usd=est_cost,
+                    category=category,
+                )
+            )
+        return priorities
+
+
+def _extract_category(task: dict[str, Any]) -> str | None:
+    """Extract a category label from a task dictionary.
+
+    Checks common fields in order of preference: ``category``, ``repo``,
+    ``difficulty``. Returns the first non-empty string found, or None.
+
+    Args:
+        task: Task dictionary.
+
+    Returns:
+        Category string, or None if no category field is found.
+    """
+    for key in ("category", "repo", "difficulty"):
+        value = task.get(key)
+        if value is not None:
+            return str(value)
+    return None
+
+
+def create_scheduler(preset: str, **kwargs: Any) -> TaskScheduler:
+    """Create a TaskScheduler from a preset name.
+
+    Convenience factory function that maps human-readable preset names to
+    scheduling strategies.
+
+    Args:
+        preset: One of ``"default"``, ``"speed"``, ``"cost"``, ``"coverage"``.
+        **kwargs: Additional keyword arguments passed to TaskScheduler
+            (e.g., ``custom_scorer``).
+
+    Returns:
+        Configured TaskScheduler instance.
+
+    Raises:
+        ValueError: If the preset name is not recognized.
+    """
+    preset_map: dict[str, SchedulingStrategy] = {
+        "default": SchedulingStrategy.DEFAULT,
+        "speed": SchedulingStrategy.SPEED_FIRST,
+        "cost": SchedulingStrategy.COST_FIRST,
+        "coverage": SchedulingStrategy.COVERAGE_FIRST,
+    }
+
+    strategy = preset_map.get(preset.lower())
+    if strategy is None:
+        valid_presets = ", ".join(sorted(preset_map.keys()))
+        raise ValueError(f"Unknown scheduling preset: '{preset}'. Valid presets: {valid_presets}")
+
+    return TaskScheduler(strategy=strategy, **kwargs)

{mcpbr-0.4.16.dist-info → mcpbr-0.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mcpbr
-Version: 0.4.16
+Version: 0.6.0
 Summary: Model Context Protocol Benchmark Runner - evaluate MCP servers against software engineering benchmarks
 Project-URL: Homepage, https://github.com/greynewell/mcpbr
 Project-URL: Repository, https://github.com/greynewell/mcpbr
@@ -30,6 +30,9 @@ Requires-Dist: pydantic>=2.0.0
 Requires-Dist: pyyaml>=6.0.0
 Requires-Dist: requests>=2.31.0
 Requires-Dist: rich>=13.0.0
+Provides-Extra: all-providers
+Requires-Dist: google-generativeai>=0.3.0; extra == 'all-providers'
+Requires-Dist: openai>=1.0.0; extra == 'all-providers'
 Provides-Extra: dev
 Requires-Dist: pre-commit>=3.0.0; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
@@ -40,6 +43,10 @@ Requires-Dist: mkdocs-material>=9.5.0; extra == 'docs'
 Requires-Dist: mkdocs-minify-plugin>=0.7.0; extra == 'docs'
 Requires-Dist: mkdocs>=1.5.0; extra == 'docs'
 Requires-Dist: mkdocstrings[python]>=0.24.0; extra == 'docs'
+Provides-Extra: gemini
+Requires-Dist: google-generativeai>=0.3.0; extra == 'gemini'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
 Description-Content-Type: text/markdown
 
 # mcpbr

{mcpbr-0.4.16.dist-info → mcpbr-0.6.0.dist-info}/RECORD

@@ -1,40 +1,56 @@
-mcpbr/__init__.py,sha256=3vhpKV9kVECjuPapKpCPEHTjlOsyhuoiLZxBv9O1eL0,152
+mcpbr/__init__.py,sha256=uTZA0md1SKeI-aFAFCGdnfYD8NVOI70BQhxNN_EsyxA,447
 mcpbr/__main__.py,sha256=WmeQsAqtW_9tMTNKArH1m76DPBokZpXuy6dMZp13gXA,132
 mcpbr/agent.py,sha256=aSFH2S3ExKZfdVfMbzk6D1nRhpKt4JmpRzmF4Vi6Gmo,5795
 mcpbr/cache.py,sha256=YiP13omwMbXLb6NhNocJvL58enXEx9J8OrvTZnWUkw4,13254
 mcpbr/cli.py,sha256=xvh7gpJx0LzjV3g-Te4FF7BfHubGzDxOiYQsSeQnCEc,68276
-mcpbr/config.py,sha256=7lWV0ZtzyD6WZ07IR4yhT9lyBBPONzlanaO4XHm9OoE,18952
+mcpbr/config.py,sha256=KEuOzo-hhGwbejljlUpQn7gWC42Y5doe07hqeLwQIYA,20171
 mcpbr/config_inheritance.py,sha256=0EV9Tv62UFNgZoc8mY7yYjHEbnMM_R5EAhSeuK7ajAA,6617
+mcpbr/config_migration.py,sha256=vTs52uYLO0DusB07nHZT2Y27-_eZdZKhaXYWhDFcnJI,16098
 mcpbr/config_validator.py,sha256=ZMEIeK4y6fSwyY46Xv5dK5v3jM4HDKcYkosnIcn7iyI,20488
+mcpbr/config_wizard.py,sha256=IPk5a2xI_6CTvo9tIDe2MLGewbc_iVYg3k_NOTC6A6c,22522
 mcpbr/custom_metrics.py,sha256=4pMO9-BPpeQ_GUTnZ18TQXINFScAMH3cIYm0HG-C51o,13213
+mcpbr/dashboard.py,sha256=wt2A-yFgDvQc94wgPPJlz70gFAkyUi41xgfqPL9xRQY,21884
+mcpbr/dataset_streaming.py,sha256=XwQSdvy97yurlcAC5hUwto8bLuCf2A9FSMcwjTD_Tho,16720
 mcpbr/dataset_versioning.py,sha256=Y_ZSGhl8ihl6Kgee_p7VbkNwGhgwIdMZPlRunvk4knY,7149
-mcpbr/docker_env.py,sha256=_45OUZKjUevE9O3YLF_1uvQtdOyJ7yZIYWmSvXN3cFw,31794
+mcpbr/docker_cache.py,sha256=jn_9Ak2d8omNmedSCBwA7wrswtEQvB-Bu8TIP2cm-F0,18704
+mcpbr/docker_env.py,sha256=EQmVfQNyvWmmMErRJeEpsvJp1dNiLyIG2EDAmjZVMTI,31795
+mcpbr/docker_prewarm.py,sha256=DCB5-h-Uu7KjsfMCGsZOTpTjLhshMQ8QfpbQh-T1PxU,12639
+mcpbr/dry_run.py,sha256=rlohv2QjtaFvLeJ7JOAPIH95GGOFoenFLcmBk-xX8x4,18219
 mcpbr/env_expansion.py,sha256=Rkhth-tWV8CptQlSSk9exuMsUaSTTW9hj69z4snZd_U,6122
 mcpbr/evaluation.py,sha256=EjPREWv7hBRqhBhNan0ERh2imqMBegT0Y2cgZlTxRGk,12765
 mcpbr/failure_analysis.py,sha256=N5xp9YPe2d7P9fTa2LVSHsPgB1WOQtWMeClq3bOv4_c,19883
 mcpbr/few_shot.py,sha256=bFDdes_kgZAFWoFZQEfZG5Z2Es9rmkB1jsxSMp4aCCM,11684
-mcpbr/gpu_support.py,sha256=eroBiLkt1A3Q2ODJDSyqrd_BzcMh8tFkjtPn7PsvJJc,5070
-mcpbr/harness.py,sha256=8-qmcPR2CDFuoBib9g6lPx7aMOK-5PuZgpWhpGs-Ils,51419
+mcpbr/formatting.py,sha256=lwZcb4fD5osBzJlerICyvAVb4KHSm_nRTBg1dVfD6Lo,14193
+mcpbr/gpu_support.py,sha256=6f_t_bGoADnu0uPsGDwSWYD7xHN_e553x1J6ceQUHi4,5071
+mcpbr/graceful_degradation.py,sha256=QPFQ0BCLERvH4e68rQkePHUW9BDJmXttlIQaqCOP5gw,8992
+mcpbr/harness.py,sha256=Rc6CqzZOMJyuHqfuOIDisLOoPka-cqAqYiL7zr7ALFg,53193
 mcpbr/harnesses.py,sha256=h9iDp4qkPABNwO9OXbJ61qcD4n0oAUTU7AQksxRKLcg,47335
 mcpbr/incremental_save.py,sha256=1dm3pGiEIhP8cVk_Y6XF_cAdo3B_vyRc6CO8Wt-MyIA,4830
 mcpbr/junit_reporter.py,sha256=M_02zJbFbA3VoIYG5oR7VDecqWHEpIee-JOUShWNuLU,9261
+mcpbr/languages.py,sha256=LrWjEsaH9nl37LZtRxPwC8J3zETMbbcxRI16gjMx2iw,8087
 mcpbr/latency_metrics.py,sha256=xNMaUzGMSbOIfuoyZGyIfyMk5uAmoj6K65ZAs5D6Z8c,10476
 mcpbr/log_formatter.py,sha256=d2jWH7z4IRSbr8-PbnEt3TmLAqk8vgdPT38uTnTCN5c,21488
-mcpbr/models.py,sha256=zsrBrwFeOfNKgThUbT1oPkF5pdRjL1QJjMte0vXjcbk,3710
+mcpbr/logging_config.py,sha256=0Q5qNhT_Pz1mYwj1rRENuDlIdiHBBNN7N-cSWAg5B6M,6823
+mcpbr/models.py,sha256=epRVhs1ryGO74ehDRppArUfeOTxnHYyctEo0fD8Dwvw,5698
 mcpbr/output_validator.py,sha256=TUoBtDjjXvR6MACbWV6uNOsxM_n4C0Jbn5in35HH4K8,1750
-mcpbr/preflight.py,sha256=UE185nnXqXP1hfVDia0sPHfR2pMXohBkbAPuZfDG_Z0,8914
-mcpbr/pricing.py,sha256=WFnVS3L8z9PSEXuEkGWFdeSJvmnUKqaQGE6-AgUEzdA,7214
+mcpbr/preflight.py,sha256=owWPOCEIN8vSY1Z2NB1T9ow7UzY6O6mLh5KPK1ZxEGo,8915
+mcpbr/pricing.py,sha256=L-HE-C8LWCQ2iWEqWHkBtyLzPPBdr05bVaCmEy9EQsw,9693
 mcpbr/profiler.py,sha256=SRXLKf2TOlpnMbQpGvjRy1Agv-XaEz6lDmBa5WGNv8c,15954
-mcpbr/providers.py,sha256=ebrnH6RXODxX4Ma9r7Is5VBHYFNP5LwCs-vpLbbHP8o,6598
+mcpbr/providers.py,sha256=p8gsPbcM9HZjSFztCbBUBvbWqc0hA_lprNh-vtDwDiE,16721
 mcpbr/regression.py,sha256=xm_ago8ZP3RAOrDNjtINwyRUvzKWJcJDWbzf3hp6LlU,12827
 mcpbr/reporting.py,sha256=Odzb7EgpimW-qh01VQedhb2X594ACrOcGe4jshgiwTg,56111
+mcpbr/resource_limits.py,sha256=MnmKcJLPCD-cyydGyiZz1TOPsQSpmdsmrbAvYWlxM5s,16594
+mcpbr/result_streaming.py,sha256=uXKCPNSXhBd8VV-Sc5aldC7zSM1MG8pAiMgsMP7ZZOA,16602
 mcpbr/sampling.py,sha256=Hpgh2TayI3QGcno-Np9eYi8sklxKEZQXyhpaQlc9T4Q,6248
 mcpbr/schema.py,sha256=fdjiKmp1au2oN5aXcPRoCbyvwm2XeMD5DmeWSurMk4A,6858
-mcpbr/smoke_test.py,sha256=srYGOn_auspRbt_a6ebYDDDq_nujA_iZGman5nU1ikU,14925
+mcpbr/sdk.py,sha256=uxX7lHzR0SkZe3lmTT7zi_C0nPPQY5wGtN1USIGf-8M,8772
+mcpbr/smoke_test.py,sha256=SCWR26kVmHZj-smWziwMh24UIO_MtIaik-REuHZ0pbc,14926
 mcpbr/state_tracker.py,sha256=rIP9LIHtQg6oBsLIxnwRjE865Kw6U7DMO_GzzuMRC0E,10790
 mcpbr/statistics.py,sha256=Ny8TMdBrIpS4KfKCJcuFfTeaGuTmEkS1G_uHBlboYdA,19134
 mcpbr/streaming.py,sha256=XPhkXO1R1EsWtkoPvCpyy4TehEom7hkuOeP-00joX3o,13853
 mcpbr/swebench_test_specs.py,sha256=Mh_BPjcexkgDT3p4zT2p31925b8w5tgsxxRpYZQZalM,1390
+mcpbr/task_batching.py,sha256=KFw_RxZUvVgjRr3pc0ehBX2GAoPT5yTP4VdM43e1_Dk,15275
+mcpbr/task_scheduler.py,sha256=HBR8ebJJ-ZDzM2nnWaIHhOqdbdqRjYzbEVGYjGiVYEw,17183
 mcpbr/templates.py,sha256=dqwboVB-yfE06w2rgDOvuWJB4Hx5duH_W-jvLBqmlKg,10683
 mcpbr/benchmarks/__init__.py,sha256=2-7Ebg6-wHo1QGfVKWjjbREcLG_A-6Q0XfZGiyXrOeE,4489
 mcpbr/benchmarks/adversarial.py,sha256=69VBTZv6BhR1JwjQepA_YwAu3b--vJviGd6IWs2h1QA,12357
@@ -80,15 +96,15 @@ mcpbr/infrastructure/azure_health.py,sha256=xITmIa9IfYIwxcVhY0sJ81a-6WNKiT8kSQTd
 mcpbr/infrastructure/base.py,sha256=Olj6uiNBeGoUqltZI1NHZfa26kzT-6jfp8YIXSykFKM,3037
 mcpbr/infrastructure/local.py,sha256=VK6UAg7Dzvb9v1LAJgNGA_s0blQKrHAQEXBAC75zAL8,4237
 mcpbr/infrastructure/manager.py,sha256=j0T7U1Tbajmfve4SNfhYKikvL9kgSVT01fYKMC-sH-s,4796
-mcpbr-0.4.16.data/data/mcpbr/data/templates/brave-search.yaml,sha256=PYHXJOaDqYKoqdJc3JV1WbaL-BacrdkQPck1eKGbMPo,1098
-mcpbr-0.4.16.data/data/mcpbr/data/templates/filesystem.yaml,sha256=1p6Z6ChViFYHAODYD71JFst6gdhR5y5rnWNf7Pp5zOY,1091
-mcpbr-0.4.16.data/data/mcpbr/data/templates/github.yaml,sha256=uzPwq5_loFegvH6RNov1MQclbBiFBgYWzpiKLfEN9H4,1133
-mcpbr-0.4.16.data/data/mcpbr/data/templates/google-maps.yaml,sha256=ldR7E9UmuAA-3nJZ1SShD7PhG0_AwDJOSYuy19hQ6cI,1116
-mcpbr-0.4.16.data/data/mcpbr/data/templates/postgres.yaml,sha256=r6R1069BhV4ADQGPZ-T9r6xMNwbr2yrNh8-IHPb4XiI,1178
-mcpbr-0.4.16.data/data/mcpbr/data/templates/slack.yaml,sha256=dBn_YqlFJMJai_55sRDb4hXClgxRpcyYTlWl4LBkpuo,1072
-mcpbr-0.4.16.data/data/mcpbr/data/templates/sqlite.yaml,sha256=UR5yN9f8v_BC6oskny2xMldHWzZrB9b_PpFSmv5eccg,1080
-mcpbr-0.4.16.dist-info/METADATA,sha256=GeSnMZw0x7-XPhblIu50aCO7NXaNfjgVScnBOp6ZaOA,55069
-mcpbr-0.4.16.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-mcpbr-0.4.16.dist-info/entry_points.txt,sha256=lLL8icujqBF36V9bF4gfaB2at4cFKCiv2IdJ1i5hT9U,41
-mcpbr-0.4.16.dist-info/licenses/LICENSE,sha256=mcXLPreEXzD-816yLKmocCPr9_k3gFFo62TjrSuKkIQ,1075
-mcpbr-0.4.16.dist-info/RECORD,,
+mcpbr-0.6.0.data/data/mcpbr/data/templates/brave-search.yaml,sha256=PYHXJOaDqYKoqdJc3JV1WbaL-BacrdkQPck1eKGbMPo,1098
+mcpbr-0.6.0.data/data/mcpbr/data/templates/filesystem.yaml,sha256=1p6Z6ChViFYHAODYD71JFst6gdhR5y5rnWNf7Pp5zOY,1091
+mcpbr-0.6.0.data/data/mcpbr/data/templates/github.yaml,sha256=uzPwq5_loFegvH6RNov1MQclbBiFBgYWzpiKLfEN9H4,1133
+mcpbr-0.6.0.data/data/mcpbr/data/templates/google-maps.yaml,sha256=ldR7E9UmuAA-3nJZ1SShD7PhG0_AwDJOSYuy19hQ6cI,1116
+mcpbr-0.6.0.data/data/mcpbr/data/templates/postgres.yaml,sha256=r6R1069BhV4ADQGPZ-T9r6xMNwbr2yrNh8-IHPb4XiI,1178
+mcpbr-0.6.0.data/data/mcpbr/data/templates/slack.yaml,sha256=dBn_YqlFJMJai_55sRDb4hXClgxRpcyYTlWl4LBkpuo,1072
+mcpbr-0.6.0.data/data/mcpbr/data/templates/sqlite.yaml,sha256=UR5yN9f8v_BC6oskny2xMldHWzZrB9b_PpFSmv5eccg,1080
+mcpbr-0.6.0.dist-info/METADATA,sha256=sNd6FYrOQ1tgz6bum5D03w373dEJfXODVLfmLsiy5hk,55376
+mcpbr-0.6.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+mcpbr-0.6.0.dist-info/entry_points.txt,sha256=lLL8icujqBF36V9bF4gfaB2at4cFKCiv2IdJ1i5hT9U,41
+mcpbr-0.6.0.dist-info/licenses/LICENSE,sha256=mcXLPreEXzD-816yLKmocCPr9_k3gFFo62TjrSuKkIQ,1075
+mcpbr-0.6.0.dist-info/RECORD,,