roller-agent 0.2.0__py3-none-any.whl

roller/benches/qwenclaw/__init__.py

@@ -0,0 +1,49 @@
+"""QwenClawBench adapter package."""
+
+from roller.benches.qwenclaw.adapter import (
+    DATASET_DIR_NAME,
+    TASK_SCHEMA,
+    QwenClawBenchAdapter,
+)
+from roller.benches.qwenclaw.grader_runtime import (
+    AUTO_PENALTY_THRESHOLD,
+    GRADE_TIMEOUT_SECONDS,
+    JUDGE_ENV_NAMES,
+    OUTPUT_DIR_NAME,
+    WORKSPACE_CONTAINER_PATH,
+    QwenClawGradedHarness,
+    build_grader_source,
+)
+from roller.benches.qwenclaw.report import aggregate_samples
+from roller.benches.qwenclaw.task_parser import (
+    DEFAULT_TIMEOUT_SECONDS,
+    QwenClawTask,
+    parse_task_markdown,
+    split_sections,
+)
+from roller.benches.qwenclaw.workspace import (
+    materialize_workspace,
+    workspace_compatibility_script,
+)
+from roller.benches.transcripts import session_trajectory
+
+__all__ = [
+    "AUTO_PENALTY_THRESHOLD",
+    "DATASET_DIR_NAME",
+    "DEFAULT_TIMEOUT_SECONDS",
+    "GRADE_TIMEOUT_SECONDS",
+    "JUDGE_ENV_NAMES",
+    "OUTPUT_DIR_NAME",
+    "TASK_SCHEMA",
+    "WORKSPACE_CONTAINER_PATH",
+    "QwenClawBenchAdapter",
+    "QwenClawGradedHarness",
+    "QwenClawTask",
+    "aggregate_samples",
+    "build_grader_source",
+    "materialize_workspace",
+    "parse_task_markdown",
+    "session_trajectory",
+    "split_sections",
+    "workspace_compatibility_script",
+]

roller/benches/qwenclaw/adapter.py

@@ -0,0 +1,167 @@
+"""QwenClawBench adapter assembly."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable, Sequence
+from dataclasses import replace
+from pathlib import Path
+from typing import Any
+
+from roller.benches.base import (
+    BenchAdapter,
+    BenchBinding,
+    BenchFormatError,
+    GradeSpec,
+    validate_timeout_multiplier,
+)
+from roller.benches.qwenclaw.grader_runtime import (
+    GRADE_TIMEOUT_SECONDS,
+    JUDGE_ENV_NAMES,
+    OUTPUT_DIR_NAME,
+    WORKSPACE_CONTAINER_PATH,
+    QwenClawGradedHarness,
+    build_grader_source,
+)
+from roller.benches.qwenclaw.report import aggregate_samples
+from roller.benches.qwenclaw.task_parser import parse_task_markdown
+from roller.benches.qwenclaw.workspace import materialize_workspace
+
+TASK_SCHEMA = "roller.task/v1"
+DATASET_DIR_NAME = "qwenclawbench-v1.1-100"
+
+
+class QwenClawBenchAdapter(BenchAdapter):
+    name = "qwenclawbench"
+    description = "QwenClawBench — 100 real-user OpenClaw tasks across 8 domains"
+
+    def __init__(self, cache_root: Path | None = None) -> None:
+        self.cache_root = cache_root
+
+    def load(
+        self,
+        root: Path,
+        *,
+        categories: Sequence[str] | None = None,
+        task_ids: Sequence[str] | None = None,
+        variant: str | None = None,
+        timeout_multiplier: float = 1.0,
+    ) -> list[dict[str, Any]]:
+        if variant is not None:
+            raise BenchFormatError(f"{self.name}: this bench has no variants")
+        multiplier = validate_timeout_multiplier(timeout_multiplier)
+        dataset = self._dataset_root(root)
+        tasks_root = dataset / "tasks"
+        assets_root = dataset / "assets"
+        wanted_categories = set(categories or ())
+        wanted_ids = set(task_ids or ())
+        rows: list[dict[str, Any]] = []
+        seen_ids: set[str] = set()
+        available_categories: set[str] = set()
+
+        for task_path in sorted(tasks_root.glob("task_*.md")):
+            task = parse_task_markdown(
+                task_path.read_text(encoding="utf-8"),
+                str(task_path),
+                source_path=task_path,
+            )
+            if task.task_id in seen_ids:
+                raise BenchFormatError(
+                    f"duplicate QwenClawBench task id: {task.task_id}"
+                )
+            seen_ids.add(task.task_id)
+            if task.category:
+                available_categories.add(task.category)
+            if wanted_categories and task.category not in wanted_categories:
+                continue
+            if wanted_ids and task.task_id not in wanted_ids:
+                continue
+
+            workspace = materialize_workspace(task, assets_root, self.cache_root)
+            env_names = (
+                JUDGE_ENV_NAMES if task.grading_type in {"llm_judge", "hybrid"} else ()
+            )
+            binding = BenchBinding(
+                name=self.name,
+                category=task.category or None,
+                metadata={
+                    "upstream_task_id": task.task_id,
+                    "name": task.name,
+                    "subcategory": task.subcategory,
+                    "grading_type": task.grading_type,
+                    "grading_weights": task.grading_weights,
+                    "input_modality": task.input_modality,
+                    "external_dependency": task.external_dependency,
+                    "run_index": 1,
+                    "runs_per_task": 1,
+                },
+                grade=GradeSpec(
+                    code=build_grader_source(task),
+                    entrypoint="roller_grade",
+                    env_names=env_names,
+                    timeout_seconds=GRADE_TIMEOUT_SECONDS,
+                    needs_transcript=True,
+                    workspace_path=WORKSPACE_CONTAINER_PATH,
+                    score_key="overall_score",
+                ),
+            )
+            rows.append(
+                {
+                    "schema_version": TASK_SCHEMA,
+                    "id": task.task_id,
+                    "harness": "openclaw",
+                    "prompt": task.prompt,
+                    "workspace": str(workspace),
+                    "files": {"output_dir": OUTPUT_DIR_NAME},
+                    "timeout_seconds": max(
+                        1, math.ceil(task.timeout_seconds * multiplier)
+                    ),
+                    "bench": binding.as_dict(),
+                }
+            )
+
+        missing_categories = wanted_categories - available_categories
+        if missing_categories:
+            raise BenchFormatError(
+                "unknown QwenClawBench categories: "
+                + ", ".join(sorted(missing_categories))
+            )
+        missing_ids = wanted_ids - seen_ids
+        if missing_ids:
+            raise BenchFormatError(
+                "unknown QwenClawBench task ids: " + ", ".join(sorted(missing_ids))
+            )
+        return rows
+
+    def wrap_harness(self, harness, binding, *, grading_semaphore=None):
+        if binding.grade is None:
+            return harness
+        grade = replace(
+            binding.grade, transcript_path="/tmp/roller-qwenclaw-transcript.json"
+        )
+        return QwenClawGradedHarness(
+            harness,
+            replace(binding, grade=grade),
+            grading_semaphore=grading_semaphore,
+        )
+
+    def aggregate(self, samples: Iterable[dict[str, Any]]) -> dict[str, Any]:
+        return aggregate_samples(samples)
+
+    @staticmethod
+    def _dataset_root(root: Path) -> Path:
+        candidates = [
+            root,
+            root / "data" / DATASET_DIR_NAME,
+        ]
+        data_root = root / "data"
+        if data_root.is_dir():
+            candidates.extend(
+                path for path in sorted(data_root.iterdir()) if path.is_dir()
+            )
+        for candidate in candidates:
+            if (candidate / "tasks").is_dir() and (candidate / "assets").is_dir():
+                return candidate.resolve()
+        raise BenchFormatError(
+            f"{root}: expected tasks/ and assets/, or data/{DATASET_DIR_NAME}/"
+        )

roller/benches/qwenclaw/grader_runtime.py

@@ -0,0 +1,395 @@
+"""Self-contained QwenClawBench grader generation and harness integration."""
+
+from __future__ import annotations
+
+import ast
+import json
+import re
+import tempfile
+
+from roller.benches.grading import (
+    GradedHarness,
+    GradeOutcome,
+    workspace_container_path,
+)
+from roller.benches.qwenclaw.task_parser import QwenClawTask
+from roller.benches.qwenclaw.workspace import workspace_compatibility_script
+from roller.benches.transcripts import grader_messages, session_trajectory
+
+IMPORT_LINE_RE = re.compile(
+    r"^(?P<indent>[ \t]*)import\s+(?P<names>[A-Za-z0-9_., \t]+)$",
+    re.MULTILINE,
+)
+
+WORKSPACE_CONTAINER_PATH = "/home/node/workspace"
+OUTPUT_DIR_NAME = ".roller_outputs"
+GRADE_TIMEOUT_SECONDS = 1800
+AUTO_PENALTY_THRESHOLD = 0.75
+
+JUDGE_ENV_NAMES = (
+    "QWENCLAW_JUDGE_BASE_URL",
+    "QWENCLAW_JUDGE_API_KEY",
+    "QWENCLAW_JUDGE_MODEL",
+    "QWENCLAW_JUDGE_MAX_RETRIES",
+    "JUDGE_BASE_URL",
+    "JUDGE_API_KEY",
+    "JUDGE_MODEL",
+)
+
+_GRADER_RUNTIME = r"""
+import json as _roller_json
+import math as _roller_math
+import os as _roller_os
+import re as _roller_re
+import shutil as _roller_shutil
+import time as _roller_time
+import urllib.error as _roller_urlerror
+import urllib.request as _roller_urlrequest
+from pathlib import Path as _RollerPath
+
+_roller_automated_grade = globals().get("grade")
+_ROLLER_GRADING_TYPE = __ROLLER_GRADING_TYPE__
+_ROLLER_PROMPT = __ROLLER_PROMPT__
+_ROLLER_EXPECTED = __ROLLER_EXPECTED__
+_ROLLER_RUBRIC = __ROLLER_RUBRIC__
+_ROLLER_WEIGHTS = __ROLLER_WEIGHTS__
+_ROLLER_OUTPUT_DIR = __ROLLER_OUTPUT_DIR__
+
+def _roller_numeric(scores):
+    result = {}
+    for key, value in (scores or {}).items():
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            result[str(key)] = float(value)
+    return result
+
+def _roller_average(scores):
+    values = list(_roller_numeric(scores).values())
+    return sum(values) / len(values) if values else 0.0
+
+def _roller_text(content):
+    if isinstance(content, str):
+        return content
+    if isinstance(content, dict):
+        for key in ("text", "thinking", "reasoning"):
+            value = content.get(key)
+            if isinstance(value, str):
+                return value
+        return ""
+    if isinstance(content, list):
+        return "\n".join(filter(None, (_roller_text(item) for item in content)))
+    return ""
+
+def _roller_summary(transcript):
+    parts = []
+    for event in transcript or []:
+        if not isinstance(event, dict) or event.get("type") != "message":
+            continue
+        message = event.get("message") or {}
+        role = message.get("role")
+        content = message.get("content") or []
+        if role == "assistant":
+            for item in content:
+                item_type = item.get("type") if isinstance(item, dict) else None
+                if isinstance(item, str) or item_type == "text":
+                    text = _roller_text(item)
+                    if text:
+                        parts.append("Assistant: %s" % text[:2000])
+                elif item_type in ("thinking", "reasoning"):
+                    text = _roller_text(item)
+                    if text:
+                        parts.append("Reasoning: %s" % text[:2000])
+                elif isinstance(item, dict) and item.get("type") in ("toolCall", "tool_use"):
+                    arguments = item.get("arguments", item.get("input", {}))
+                    parts.append("Tool: %s(%s)" % (
+                        item.get("name"),
+                        _roller_json.dumps(arguments, ensure_ascii=False),
+                    ))
+        elif role == "toolResult":
+            text = _roller_text(content)
+            if text:
+                parts.append("Result: %s" % text[:2000])
+        elif role == "user":
+            text = _roller_text(content)
+            if text:
+                parts.append("User: %s" % text[:2000])
+    return "\n".join(parts)
+
+def _roller_judge_prompt(transcript):
+    return (
+        "You are a grading function. Your ONLY job is to output a single JSON object.\n\n"
+        "CRITICAL RULES:\n"
+        "- Do NOT use any tools (no Read, Write, exec, or any other tool calls)\n"
+        "- Do NOT create files or run commands\n"
+        "- Do NOT write any prose, explanation, or commentary outside the JSON\n"
+        "- Respond with ONLY a JSON object — nothing else\n\n"
+        "Be a strict evaluator. Reserve 1.0 for genuinely excellent performance. "
+        "An average acceptable completion should score around 0.6-0.7. "
+        "Deduct points for unnecessary steps, verbose output, and inefficient tool usage.\n\n"
+        "## Task\n%s\n\n## Expected Behavior\n%s\n\n"
+        "## Agent Transcript (summarized)\n%s\n\n## Grading Rubric\n%s\n\n"
+        "Score each criterion from 0.0 to 1.0.\n\n"
+        "Respond with ONLY this JSON structure (no markdown, no code fences, no extra text):\n"
+        '{"scores": {"criterion_name": 0.0}, "total": 0.0, "notes": "brief justification"}'
+    ) % (_ROLLER_PROMPT, _ROLLER_EXPECTED, _roller_summary(transcript), _ROLLER_RUBRIC)
+
+def _roller_json_objects(text):
+    objects = []
+    depth = 0
+    current = []
+    for char in text:
+        if char == "{":
+            if depth == 0:
+                current = []
+            depth += 1
+        if depth:
+            current.append(char)
+        if char == "}" and depth:
+            depth -= 1
+            if depth == 0:
+                objects.append("".join(current))
+    return objects
+
+def _roller_parse_judge(text):
+    text = (text or "").strip()
+    fenced = _roller_re.search(r"```(?:json)?\s*(.*?)\s*```", text, _roller_re.DOTALL)
+    candidates = ([fenced.group(1)] if fenced else []) + list(reversed(_roller_json_objects(text)))
+    parsed = {}
+    for candidate in candidates:
+        try:
+            value = _roller_json.loads(candidate)
+        except Exception:
+            continue
+        if isinstance(value, dict):
+            parsed = value
+            if "scores" in value:
+                break
+    scores = {}
+    source_scores = parsed.get("scores", parsed.get("criteria_scores", {}))
+    if isinstance(source_scores, dict):
+        for key, value in source_scores.items():
+            if isinstance(value, dict):
+                value = value.get("score")
+            if isinstance(value, (int, float)) and not isinstance(value, bool):
+                if not _roller_math.isfinite(value):
+                    raise ValueError("judge criterion score must be finite: %s" % key)
+                scores[str(key)] = float(value)
+    total = parsed.get("total", parsed.get("score"))
+    if not isinstance(total, (int, float)) or isinstance(total, bool):
+        if not scores:
+            raise ValueError(
+                "judge response did not contain a numeric total or criterion score"
+            )
+        total = _roller_average(scores)
+    if not _roller_math.isfinite(total):
+        raise ValueError("judge total score must be finite")
+    return scores, max(0.0, min(1.0, float(total))), str(
+        parsed.get("notes", parsed.get("justification", parsed.get("reasoning", "")))
+    )
+
+def _roller_call_judge(transcript):
+    base = (
+        _roller_os.environ.get("QWENCLAW_JUDGE_BASE_URL")
+        or _roller_os.environ.get("JUDGE_BASE_URL")
+        or ""
+    ).strip()
+    key = (
+        _roller_os.environ.get("QWENCLAW_JUDGE_API_KEY")
+        or _roller_os.environ.get("JUDGE_API_KEY")
+        or ""
+    ).strip()
+    model = (
+        _roller_os.environ.get("QWENCLAW_JUDGE_MODEL")
+        or _roller_os.environ.get("JUDGE_MODEL")
+        or "claude-opus-4-5-20251101"
+    ).strip()
+    if not base or not key:
+        raise RuntimeError(
+            "LLM judge requires QWENCLAW_JUDGE_BASE_URL and "
+            "QWENCLAW_JUDGE_API_KEY (JUDGE_BASE_URL/JUDGE_API_KEY also accepted)"
+        )
+    endpoint = base.rstrip("/")
+    if not endpoint.endswith("/chat/completions"):
+        endpoint += "/chat/completions"
+    payload = _roller_json.dumps({
+        "model": model.split("/", 1)[-1] if "/" in model else model,
+        "messages": [{"role": "user", "content": _roller_judge_prompt(transcript)}],
+        "temperature": 0.0,
+        "max_tokens": 20480,
+    }).encode("utf-8")
+    retries = int(_roller_os.environ.get("QWENCLAW_JUDGE_MAX_RETRIES", "3"))
+    retries = max(1, min(100, retries))
+    last_error = None
+    for attempt in range(retries):
+        request = _roller_urlrequest.Request(
+            endpoint,
+            data=payload,
+            headers={"Content-Type": "application/json", "Authorization": "Bearer " + key},
+            method="POST",
+        )
+        try:
+            with _roller_urlrequest.urlopen(request, timeout=600) as response:
+                body = _roller_json.loads(response.read().decode("utf-8"))
+            choices = body.get("choices") or []
+            if not choices:
+                raise RuntimeError("judge API returned no choices")
+            return _roller_parse_judge((choices[0].get("message") or {}).get("content", ""))
+        except Exception as exc:
+            last_error = exc
+            if attempt + 1 < retries:
+                _roller_time.sleep(min(5 * (2 ** attempt), 30))
+    raise RuntimeError("judge API failed after %d attempt(s): %s" % (retries, last_error))
+
+def _roller_snapshot(workspace_path):
+    source = _RollerPath(workspace_path)
+    destination = source / _ROLLER_OUTPUT_DIR
+    if destination.exists():
+        _roller_shutil.rmtree(destination)
+    destination.mkdir(parents=True)
+    for item in source.iterdir():
+        if item.name in {_ROLLER_OUTPUT_DIR, ".git"}:
+            continue
+        target = destination / item.name
+        if item.is_symlink():
+            continue
+        if item.is_dir():
+            _roller_shutil.copytree(item, target, symlinks=True)
+        elif item.is_file():
+            _roller_shutil.copy2(item, target)
+
+def roller_grade(transcript, workspace_path):
+    try:
+        automated_scores = {}
+        automated_score = 0.0
+        if _ROLLER_GRADING_TYPE in {"automated", "hybrid"}:
+            if not callable(_roller_automated_grade):
+                raise RuntimeError("automated grader function is missing")
+            automated_scores = _roller_numeric(
+                _roller_automated_grade(transcript, workspace_path)
+            )
+            automated_score = _roller_average(automated_scores)
+        judge_scores = {}
+        judge_score = 0.0
+        notes = ""
+        if _ROLLER_GRADING_TYPE in {"llm_judge", "hybrid"}:
+            judge_scores, judge_score, notes = _roller_call_judge(transcript)
+        if _ROLLER_GRADING_TYPE == "automated":
+            overall = simple = automated_score
+        elif _ROLLER_GRADING_TYPE == "llm_judge":
+            overall = simple = judge_score
+        else:
+            auto_weight = float(_ROLLER_WEIGHTS.get("automated", 0.5))
+            judge_weight = float(_ROLLER_WEIGHTS.get("llm_judge", 0.5))
+            weight = auto_weight + judge_weight
+            if weight <= 0:
+                auto_weight = judge_weight = 0.5
+                weight = 1.0
+            simple = (automated_score * auto_weight + judge_score * judge_weight) / weight
+            adjusted = 0.0 if automated_score < 0.75 else judge_score
+            overall = (automated_score * auto_weight + adjusted * judge_weight) / weight
+        return {
+            "overall_score": overall,
+            "score_simple": simple,
+            "automated_score": automated_score,
+            "llm_judge_score": judge_score,
+            "automated": automated_scores,
+            "llm_judge": judge_scores,
+            "notes": notes,
+        }
+    finally:
+        _roller_snapshot(workspace_path)
+"""
+
+
+def _remove_unused_yaml_import(code: str) -> str:
+    tree = ast.parse(code)
+    if any(
+        isinstance(node, ast.Name)
+        and node.id == "yaml"
+        and isinstance(node.ctx, ast.Load)
+        for node in ast.walk(tree)
+    ):
+        return code
+
+    def remove_yaml(match: re.Match[str]) -> str:
+        names = [name.strip() for name in match.group("names").split(",")]
+        if "yaml" not in names:
+            return match.group(0)
+        names.remove("yaml")
+        if not names:
+            return ""
+        return f"{match.group('indent')}import {', '.join(names)}"
+
+    return IMPORT_LINE_RE.sub(remove_yaml, code)
+
+
+def build_grader_source(task: QwenClawTask) -> str:
+    automated = _remove_unused_yaml_import(task.automated_code)
+    runtime = (
+        _GRADER_RUNTIME.replace("__ROLLER_GRADING_TYPE__", repr(task.grading_type))
+        .replace("__ROLLER_PROMPT__", repr(task.prompt))
+        .replace("__ROLLER_EXPECTED__", repr(task.expected_behavior))
+        .replace("__ROLLER_RUBRIC__", repr(task.llm_judge_rubric))
+        .replace("__ROLLER_WEIGHTS__", repr(task.grading_weights))
+        .replace("__ROLLER_OUTPUT_DIR__", repr(OUTPUT_DIR_NAME))
+    )
+    return (automated + "\n\n" if automated else "") + runtime.strip() + "\n"
+
+
+def _session_trajectory(session):
+    # Preserve the old module-level monkeypatch point after moving this class
+    # into a submodule.
+    from roller.benches import qwenclaw
+
+    return qwenclaw.session_trajectory(session)
+
+
+class QwenClawGradedHarness(GradedHarness):
+    """Normalize each native journal before the upstream grader sees it."""
+
+    def _prepare_session(self, session) -> None:
+        super()._prepare_session(session)
+        workspace = workspace_container_path(session)
+        result = self.inner.runtime.run(
+            [
+                "exec",
+                session.container,
+                "/bin/sh",
+                "-c",
+                workspace_compatibility_script(workspace),
+            ],
+            timeout=30,
+        )
+        if result.returncode != 0:
+            detail = (result.stderr or result.stdout or "").strip()[-500:]
+            raise RuntimeError(
+                "QwenClawBench workspace compatibility setup failed"
+                + (f": {detail}" if detail else "")
+            )
+
+    def _grade(self, session, request, status):
+        if not getattr(session, "container_started", False):
+            return super()._grade(session, request, status)
+        try:
+            events = grader_messages(_session_trajectory(session))
+            with tempfile.NamedTemporaryFile(
+                "w", suffix=".json", encoding="utf-8"
+            ) as handle:
+                json.dump(events, handle, ensure_ascii=False)
+                handle.flush()
+                copied = self.inner.runtime.run(
+                    [
+                        "cp",
+                        handle.name,
+                        f"{session.container}:/tmp/roller-qwenclaw-transcript.json",
+                    ],
+                    timeout=60,
+                )
+                if copied.returncode:
+                    raise RuntimeError(
+                        "QwenClawBench normalized transcript staging failed"
+                    )
+            return super()._grade(session, request, status)
+        except Exception as exc:  # noqa: BLE001 - grading failures become score metadata
+            return GradeOutcome(
+                {"overall_score": 0.0, "error": str(exc)}, 0.0, str(exc)
+            )

roller/benches/qwenclaw/report.py

@@ -0,0 +1,49 @@
+"""QwenClawBench sample aggregation."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+from roller.benches.report import mean_aggregate, sample_score
+
+
+def aggregate_samples(samples: Iterable[dict[str, Any]]) -> dict[str, Any]:
+    rows = list(samples)
+    report = mean_aggregate(rows)
+    grouped: dict[str, list[float]] = {}
+    for row in rows:
+        metadata = row.get("bench_metadata")
+        upstream_id = (
+            metadata.get("upstream_task_id")
+            if isinstance(metadata, dict)
+            else row.get("task_id")
+        )
+        scores = row.get("bench_scores")
+        value = (
+            sample_score(scores, row.get("bench_score_key") or "overall_score")
+            if isinstance(scores, dict)
+            else None
+        )
+        if isinstance(upstream_id, str):
+            grouped.setdefault(upstream_id, []).append(
+                value if value is not None else 0.0
+            )
+    task_means = {
+        task_id: sum(values) / len(values) for task_id, values in grouped.items()
+    }
+    perfect = sum(value >= 1.0 - 1e-9 for value in task_means.values())
+    report["average"] = (
+        round(sum(task_means.values()) / len(task_means), 6)
+        if task_means
+        else None
+    )
+    report["upstream_task_count"] = len(task_means)
+    report["strict_accuracy"] = (
+        round(perfect / len(task_means), 6) if task_means else None
+    )
+    report["perfect_task_count"] = perfect
+    report["runs_per_task"] = {
+        task_id: len(values) for task_id, values in sorted(grouped.items())
+    }
+    return report