benchflow 0.6.8.dev1913__tar.gz → 0.6.9.dev1920__tar.gz

{benchflow-0.6.8.dev1913 → benchflow-0.6.9.dev1920}/CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## [Unreleased]
 
+### Changed
+- **Trajectory uploads require contributor provenance.** The single
+  `bench traj upload` command now requires `--github-id` and `--email`; both
+  values are validated locally and server-side and stored in `manifest.json`
+  under the structured `contributor` field. Existing 0.6.8 manifests remain
+  readable by the validator during the client transition.
+
+## 0.6.8 — 2026-08-15
+
 ### Added
 - **Public trajectory contribution.** `bench traj upload PATH` validates and
   structurally redacts trajectory JSONL, creates a content-addressed manifest,
@@ -37,6 +46,29 @@
   eval-config `environment:` docker/daytona selector) is a different
   subsystem and is unchanged.
 
+### Fixed
+- **Coherent integration coverage for code-and-fixture changes.** The
+  credential-free fixture job now tests pull-request source together with its
+  task fixtures, while the secret-bearing smoke job retains trusted fixtures;
+  the release gate requires both results. (#969)
+- **Retryable Daytona transport failures stay retryable.** Transient SDK
+  transport errors are stamped while their vendor type is still available,
+  empty exception messages retain useful type information, and permanent
+  errors remain outside the retry policy. (#970)
+- **Live token counters no longer freeze behind callback-log reads.** Larger,
+  bounded ranged reads distinguish EOF from read failure, expose lag, and keep
+  terminal phase labels from moving backwards. (#971)
+- **Fresh OpenClaw installs and GPT-5.4 calls use compatible limits.** OpenClaw
+  is pinned to a Node-compatible runtime, and raw plus ACP-alias GPT-5.4 model
+  IDs clamp output tokens to the provider's 128,000-token limit. (#977, #986)
+- **Anthropic Vertex routes include their required runtime.** The gateway now
+  installs the Google Cloud Vertex AI SDK used by LiteLLM's Anthropic Vertex
+  path. (#978)
+- **ACP subprocess diagnostics survive normal teardown.** A bounded, redacted
+  stderr tail is retained even when stdout remains open; stderr drain failures
+  cannot mask the structured transport diagnosis, and repeated process close
+  calls are idempotent. (#980)
+
 ## 0.6.7 — 2026-08-09
 
 ### Added

{benchflow-0.6.8.dev1913 → benchflow-0.6.9.dev1920}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: benchflow
-Version: 0.6.8.dev1913
+Version: 0.6.9.dev1920
 Summary: Multi-turn agent benchmarking with ACP — run any agent, any model, any provider.
 Project-URL: Homepage, https://github.com/benchflow-ai/benchflow
 Project-URL: Repository, https://github.com/benchflow-ai/benchflow

{benchflow-0.6.8.dev1913 → benchflow-0.6.9.dev1920}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benchflow"
-version = "0.6.8.dev1913"
+version = "0.6.9.dev1920"
 description = "Multi-turn agent benchmarking with ACP — run any agent, any model, any provider."
 readme = "README.md"
 requires-python = ">=3.12"

{benchflow-0.6.8.dev1913 → benchflow-0.6.9.dev1920}/src/benchflow/cli/traj.py

@@ -29,6 +29,14 @@ def register_traj(app: typer.Typer) -> None:
             Path,
             typer.Argument(help="Trajectory JSONL file, directory, or trial directory"),
         ],
+        github_id: Annotated[
+            str,
+            typer.Option("--github-id", help="Contributor GitHub username"),
+        ],
+        email: Annotated[
+            str,
+            typer.Option("--email", help="Contributor email stored in the manifest"),
+        ],
         source_id: Annotated[
             str | None,
             typer.Option("--source-id", help="Stable contributor source identifier"),
@@ -84,6 +92,8 @@ def register_traj(app: typer.Typer) -> None:
                 path,
                 source_id=selected_source_id,
                 uploaded_by=os.environ.get("BENCHFLOW_TRAJ_UPLOADED_BY"),
+                github_id=github_id,
+                email=email,
             ) as staged:
                 if dry_run:
                     _print_dry_run(staged, destination=destination, direct=direct)

{benchflow-0.6.8.dev1913 → benchflow-0.6.9.dev1920}/src/benchflow/publish/broker.py

@@ -42,6 +42,8 @@ def upload_capture_via_broker(
         "uploaded_by": staged.manifest["uploaded_by"],
         "artifacts": artifacts,
     }
+    if contributor := staged.manifest.get("contributor"):
+        request_body["contributor"] = contributor
     manager = nullcontext(http_client) if http_client is not None else httpx.Client()
     try:
         with _quiet_httpx_request_logging(), manager as client:

{benchflow-0.6.8.dev1913 → benchflow-0.6.9.dev1920}/src/benchflow/publish/traj_capture.py

@@ -23,11 +23,18 @@ MAX_CAPTURE_BYTES = 2 * MAX_FILE_BYTES
 MAX_MANIFEST_BYTES = 1024**2
 MAX_RUN_METADATA_BYTES = 1024**2
 MAX_UPLOADED_BY_LENGTH = 256
+MAX_GITHUB_ID_LENGTH = 39
+MAX_EMAIL_LENGTH = 254
 MAX_JSONL_RECORD_BYTES = 8 * 1024**2
 MAX_JSON_NESTING = 100
 MAX_JSON_NODES = 100_000
 SOURCE_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$")
 ARTIFACT_NAME_PATTERN = re.compile(r"^trajectory/[A-Za-z0-9._-]{1,128}\.jsonl$")
+GITHUB_ID_PATTERN = re.compile(
+    rf"^[A-Za-z0-9](?:[A-Za-z0-9-]{{0,{MAX_GITHUB_ID_LENGTH - 2}}}[A-Za-z0-9])?$"
+)
+EMAIL_LOCAL_PATTERN = re.compile(r"^[A-Za-z0-9.!#$%&'*+/=?^_`{|}~-]+$")
+EMAIL_DOMAIN_LABEL_PATTERN = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$")
 
 
 def _reject_duplicate_json_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
@@ -102,6 +109,42 @@ def validate_source_id(source_id: str) -> str:
     return normalized
 
 
+def validate_github_id(github_id: str) -> str:
+    """Validate a self-asserted GitHub username stored as contributor provenance."""
+    normalized = github_id.strip()
+    if not GITHUB_ID_PATTERN.fullmatch(normalized) or "--" in normalized:
+        raise ValueError(
+            "invalid GitHub ID; use the 1-39 character username without '@'"
+        )
+    return normalized
+
+
+def validate_email(email: str) -> str:
+    """Validate a bounded ASCII contributor email address."""
+    normalized = email.strip()
+    if len(normalized) > MAX_EMAIL_LENGTH or normalized.count("@") != 1:
+        raise ValueError("invalid contributor email address")
+    local, domain = normalized.rsplit("@", 1)
+    labels = domain.split(".")
+    if (
+        not local
+        or len(local) > 64
+        or local.startswith(".")
+        or local.endswith(".")
+        or ".." in local
+        or not EMAIL_LOCAL_PATTERN.fullmatch(local)
+        or len(labels) < 2
+        or any(
+            not label
+            or len(label) > 63
+            or not EMAIL_DOMAIN_LABEL_PATTERN.fullmatch(label)
+            for label in labels
+        )
+    ):
+        raise ValueError("invalid contributor email address")
+    return normalized
+
+
 def validate_artifact_name(name: str) -> str:
     """Require the canonical public trajectory object namespace."""
     if not ARTIFACT_NAME_PATTERN.fullmatch(name):
@@ -119,6 +162,8 @@ def stage_trajectory_capture(
     source_id: str,
     redact: bool = True,
     uploaded_by: str | None = None,
+    github_id: str | None = None,
+    email: str | None = None,
 ) -> Iterator[StagedCapture]:
     """Validate and stage a trajectory capture without mutating its source."""
     source_id = validate_source_id(source_id)
@@ -126,6 +171,14 @@ def stage_trajectory_capture(
         raise ValueError(
             f"trajectory contributor label exceeds {MAX_UPLOADED_BY_LENGTH} characters"
         )
+    contributor: dict[str, str] | None = None
+    if github_id is not None or email is not None:
+        if github_id is None or email is None:
+            raise ValueError("GitHub ID and email must be provided together")
+        contributor = {
+            "github_id": validate_github_id(github_id),
+            "email": validate_email(email),
+        }
     resolved = _resolve_input(path)
     if len(resolved.files) > MAX_ARTIFACTS:
         raise ValueError(
@@ -176,6 +229,7 @@ def stage_trajectory_capture(
             payloads=payloads,
             metadata_dir=resolved.metadata_dir,
             uploaded_by=uploaded_by,
+            contributor=contributor,
             redact=redact,
             replacement_count=replacement_count,
         )
@@ -183,6 +237,8 @@ def stage_trajectory_capture(
             redacted_manifest, manifest_replacements = redact_value(manifest)
             if redacted_manifest["source_id"] != manifest["source_id"]:
                 raise ValueError("source id contains a secret-like value")
+            if redacted_manifest.get("contributor") != manifest.get("contributor"):
+                raise ValueError("contributor metadata resembles a secret-like value")
             replacement_count += manifest_replacements
             redacted_manifest["redaction"]["replacements"] = replacement_count
             manifest = redacted_manifest
@@ -384,11 +440,12 @@ def _build_manifest(
     payloads: list[StagedFile],
     metadata_dir: Path | None,
     uploaded_by: str | None,
+    contributor: dict[str, str] | None,
     redact: bool,
     replacement_count: int,
 ) -> dict[str, Any]:
-    return {
-        "schema_version": "1.0.0",
+    manifest = {
+        "schema_version": "1.1.0" if contributor is not None else "1.0.0",
         "kind": "bronze.trajectory",
         "created_at": datetime.now(UTC)
         .replace(microsecond=0)
@@ -405,6 +462,9 @@ def _build_manifest(
         ],
         "redaction": {"applied": redact, "replacements": replacement_count},
     }
+    if contributor is not None:
+        manifest["contributor"] = contributor
+    return manifest
 
 
 def _load_run_metadata(metadata_dir: Path | None) -> dict[str, Any]:

{benchflow-0.6.8.dev1913 → benchflow-0.6.9.dev1920}/src/benchflow/sandbox/process/_base.py

@@ -156,6 +156,26 @@ class SubprocessLiveProcess(LiveProcess):
             if len(self._stderr_tail) > _STDERR_TAIL_LIMIT:
                 del self._stderr_tail[:-_STDERR_TAIL_LIMIT]
 
+    async def _finish_stderr_drain(self, *, cancel_on_timeout: bool) -> None:
+        """Bound the stderr drain without leaking its transport failures."""
+        stderr_task = getattr(self, "_stderr_task", None)
+        if not stderr_task or stderr_task.cancelled():
+            return
+        try:
+            await asyncio.wait_for(
+                asyncio.shield(stderr_task), timeout=_STDERR_DRAIN_TIMEOUT_SEC
+            )
+        except asyncio.CancelledError:
+            if not stderr_task.cancelled():
+                raise
+        except TimeoutError:
+            if cancel_on_timeout:
+                stderr_task.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await stderr_task
+        except Exception:
+            logger.debug("Could not finish draining subprocess stderr", exc_info=True)
+
     @property
     def stderr_tail(self) -> str:
         """Bounded stderr captured while the subprocess was alive."""
@@ -176,10 +196,7 @@ class SubprocessLiveProcess(LiveProcess):
         if not line:
             stderr_task = getattr(self, "_stderr_task", None)
             if stderr_task:
-                with contextlib.suppress(TimeoutError):
-                    await asyncio.wait_for(
-                        asyncio.shield(stderr_task), timeout=_STDERR_DRAIN_TIMEOUT_SEC
-                    )
+                await self._finish_stderr_drain(cancel_on_timeout=False)
                 stderr_text = self.stderr_tail.strip()
             else:
                 stderr_text = ""
@@ -257,16 +274,7 @@ class SubprocessLiveProcess(LiveProcess):
                 except TimeoutError:
                     self._process.kill()
                     await self._process.wait()
-            stderr_task = getattr(self, "_stderr_task", None)
-            if stderr_task:
-                try:
-                    await asyncio.wait_for(
-                        asyncio.shield(stderr_task), timeout=_STDERR_DRAIN_TIMEOUT_SEC
-                    )
-                except TimeoutError:
-                    stderr_task.cancel()
-                    with contextlib.suppress(asyncio.CancelledError):
-                        await stderr_task
+            await self._finish_stderr_drain(cancel_on_timeout=True)
             logger.info("Process terminated")
 
     @property

{benchflow-0.6.8.dev1913 → benchflow-0.6.9.dev1920}/tests/test_process.py

@@ -46,6 +46,40 @@ async def test_subprocess_close_cancels_stderr_drain_that_never_reaches_eof(
     )
     await process.close()
     assert process._stderr_task.cancelled()
+    await process.close()
+
+
+@pytest.mark.asyncio
+async def test_subprocess_readline_keeps_typed_error_when_stderr_drain_fails() -> None:
+    """Guards PR #980 against stderr failures masking transport diagnostics."""
+    from benchflow.diagnostics import TransportClosedError
+    from benchflow.sandbox.process import SubprocessLiveProcess
+
+    class _LiveProcess(SubprocessLiveProcess):
+        async def start(self, command, env=None, cwd=None) -> None:
+            pass
+
+    async def fail_stderr_drain() -> None:
+        raise ConnectionResetError("stderr transport reset")
+
+    stdout = MagicMock()
+    stdout.readline = AsyncMock(return_value=b"")
+    child = MagicMock(
+        stdout=stdout,
+        stderr=None,
+        returncode=1,
+        pid=12345,
+    )
+    process = _LiveProcess()
+    process._process = child
+    process._stderr_tail = bytearray(b"diagnostic before reset")
+    process._stderr_task = asyncio.create_task(fail_stderr_drain())
+
+    with pytest.raises(TransportClosedError) as exc_info:
+        await process.readline()
+
+    assert exc_info.value.diagnostic.stderr_snippet == "diagnostic before reset"
+    assert exc_info.value.diagnostic.transport_diagnosis == "process_exited"
 
 
 class _FakeStdin:

{benchflow-0.6.8.dev1913 → benchflow-0.6.9.dev1920}/tests/test_publish_azure_blob.py

@@ -271,6 +271,53 @@ def test_staging_enforces_contributor_label_bound(tmp_path: Path) -> None:
         ).__enter__()
 
 
+def test_manifest_contains_validated_contributor_provenance(tmp_path: Path) -> None:
+    """CLI contributor parameters survive in the manifest uploaded last."""
+    trial = _trial(tmp_path)
+    contributor = {
+        "github_id": "benchflow-ai",
+        "email": "contributor+demo@benchflow.ai",
+    }
+
+    with stage_trajectory_capture(
+        trial,
+        source_id="demo",
+        github_id=contributor["github_id"],
+        email=contributor["email"],
+    ) as staged:
+        assert staged.manifest["schema_version"] == "1.1.0"
+        assert staged.manifest["contributor"] == contributor
+        persisted = json.loads(staged.files[-1].local_path.read_text(encoding="utf-8"))
+        assert persisted["contributor"] == contributor
+
+
+@pytest.mark.parametrize(
+    ("github_id", "email", "message"),
+    [
+        ("@benchflow-ai", "user@benchflow.ai", "GitHub ID"),
+        ("benchflow--ai", "user@benchflow.ai", "GitHub ID"),
+        ("benchflow-ai", "not-an-email", "email"),
+        ("benchflow-ai", "user@localhost", "email"),
+        (None, "user@benchflow.ai", "provided together"),
+        ("benchflow-ai", None, "provided together"),
+    ],
+)
+def test_staging_rejects_invalid_or_partial_contributor_provenance(
+    tmp_path: Path,
+    github_id: str | None,
+    email: str | None,
+    message: str,
+) -> None:
+    """Malformed contributor metadata never reaches a transport."""
+    with pytest.raises(ValueError, match=message):
+        stage_trajectory_capture(
+            _trial(tmp_path),
+            source_id="demo",
+            github_id=github_id,
+            email=email,
+        ).__enter__()
+
+
 @pytest.mark.parametrize(
     "filename", ["capture with space.jsonl", "capture.JSONL", "x" * 129 + ".jsonl"]
 )

{benchflow-0.6.8.dev1913 → benchflow-0.6.9.dev1920}/tests/test_traj_upload_cli.py

@@ -7,6 +7,7 @@ import logging
 from pathlib import Path
 from types import SimpleNamespace
 
+import click
 import httpx
 import pytest
 from typer.testing import CliRunner
@@ -16,6 +17,8 @@ from benchflow.publish.broker import upload_capture_via_broker
 from benchflow.publish.traj_capture import stage_trajectory_capture
 
 runner = CliRunner()
+GITHUB_ID = "benchflow-user"
+EMAIL = "user@example.com"
 
 
 def _trial(tmp_path: Path) -> Path:
@@ -28,6 +31,19 @@ def _trial(tmp_path: Path) -> Path:
     return trial
 
 
+def _upload_command(path: Path, *args: str) -> list[str]:
+    return [
+        "traj",
+        "upload",
+        str(path),
+        "--github-id",
+        GITHUB_ID,
+        "--email",
+        EMAIL,
+        *args,
+    ]
+
+
 def test_stock_cli_has_the_verified_public_broker() -> None:
     """A wheel install can contribute without private endpoint configuration."""
     from benchflow.cli.traj import DEFAULT_TRAJ_BROKER_URL
@@ -72,12 +88,13 @@ def test_dry_run_stages_without_constructing_a_transport(
         raise AssertionError("network client constructed during --dry-run")
 
     monkeypatch.setattr(httpx, "Client", fail_client)
-    result = runner.invoke(app, ["traj", "upload", str(trial), "--dry-run"])
+    result = runner.invoke(app, _upload_command(trial, "--dry-run"))
 
     assert result.exit_code == 0, result.output
     assert "sha256:" in result.output
     assert "trajectory/acp_trajectory.jsonl" in result.output
     assert "manifest.json" in result.output
+    assert EMAIL not in result.output
 
 
 def test_direct_mode_reports_azure_destination(
@@ -87,6 +104,10 @@ def test_direct_mode_reports_azure_destination(
     trial = _trial(tmp_path)
 
     def fake_upload(staged, *, container_url):
+        assert staged.manifest["contributor"] == {
+            "github_id": GITHUB_ID,
+            "email": EMAIL,
+        }
         return SimpleNamespace(
             url=f"{container_url}/sources/demo/{staged.traj_digest}/",
             uploaded=("payload", "manifest"),
@@ -98,14 +119,12 @@ def test_direct_mode_reports_azure_destination(
     )
     result = runner.invoke(
         app,
-        [
-            "traj",
-            "upload",
-            str(trial),
+        _upload_command(
+            trial,
             "--direct",
             "--container-url",
             "https://tasksminerdata.blob.core.windows.net/bronze",
-        ],
+        ),
     )
 
     assert result.exit_code == 0, result.output
@@ -130,9 +149,20 @@ def test_broker_mode_uses_exact_manifest_and_server_order(
                 "source_id",
                 "traj_digest",
                 "uploaded_by",
+                "contributor",
                 "artifacts",
             }
+            assert body["contributor"] == {
+                "github_id": GITHUB_ID,
+                "email": EMAIL,
+            }
+            assert body["schema_version"] == "1.1.0"
             return httpx.Response(200, json=_broker_payload(request))
+        if request.url.path.endswith("manifest.json"):
+            assert json.loads(request.content)["contributor"] == {
+                "github_id": GITHUB_ID,
+                "email": EMAIL,
+            }
         assert request.headers["x-ms-blob-type"] == "BlockBlob"
         assert request.headers["if-none-match"] == "*"
         return httpx.Response(201)
@@ -140,7 +170,7 @@ def test_broker_mode_uses_exact_manifest_and_server_order(
     client = httpx.Client(transport=httpx.MockTransport(handler))
     monkeypatch.setattr("benchflow.publish.broker.httpx.Client", lambda: client)
     monkeypatch.setenv("BENCHFLOW_TRAJ_BROKER_URL", "https://broker.test")
-    result = runner.invoke(app, ["traj", "upload", str(trial)])
+    result = runner.invoke(app, _upload_command(trial))
 
     assert result.exit_code == 0, result.output
     assert [request.method for request in requests] == ["POST", "PUT", "PUT"]
@@ -198,12 +228,12 @@ def test_broker_conflict_is_success_and_rate_limit_is_actionable(
         )
     )
     monkeypatch.setattr("benchflow.publish.broker.httpx.Client", lambda: conflict)
-    result = runner.invoke(app, ["traj", "upload", str(trial)])
+    result = runner.invoke(app, _upload_command(trial))
     assert result.exit_code == 0, result.output
     assert "Already uploaded" in result.output
 
     monkeypatch.setattr("benchflow.publish.broker.httpx.Client", lambda: limited)
-    result = runner.invoke(app, ["traj", "upload", str(trial)])
+    result = runner.invoke(app, _upload_command(trial))
     assert result.exit_code == 1
     assert "retry after 60" in result.output
 
@@ -215,7 +245,7 @@ def test_missing_broker_names_both_available_modes(
     trial = _trial(tmp_path)
     monkeypatch.delenv("BENCHFLOW_TRAJ_BROKER_URL", raising=False)
     monkeypatch.setattr("benchflow.cli.traj.DEFAULT_TRAJ_BROKER_URL", None)
-    result = runner.invoke(app, ["traj", "upload", str(trial)])
+    result = runner.invoke(app, _upload_command(trial))
 
     assert result.exit_code == 1
     assert "BENCHFLOW_TRAJ_BROKER_URL" in result.output
@@ -231,7 +261,7 @@ def test_validation_failure_names_the_bad_file(
     path = trial / "trajectory" / "acp_trajectory.jsonl"
     path.write_text("{bad\n", encoding="utf-8")
     monkeypatch.setenv("BENCHFLOW_TRAJ_BROKER_URL", "https://broker.test")
-    result = runner.invoke(app, ["traj", "upload", str(trial)])
+    result = runner.invoke(app, _upload_command(trial))
 
     assert result.exit_code == 1
     assert "acp_trajectory.jsonl" in result.output.replace("\n", "")
@@ -290,7 +320,54 @@ def test_broker_put_conflicts_are_cloud_neutral_skips(
 
 
 def test_help_exposes_only_the_planned_upload_command() -> None:
-    """The trajectory CLI surface remains a single clean command."""
+    """Guards PR #992 while ignoring Rich's environment-specific ANSI styling."""
+    traj_group = next(group for group in app.registered_groups if group.name == "traj")
+    assert {
+        command.name for command in traj_group.typer_instance.registered_commands
+    } == {"upload"}
+
     result = runner.invoke(app, ["traj", "--help"])
     assert result.exit_code == 0
     assert "upload" in result.output
+
+    upload_help = runner.invoke(app, ["traj", "upload", "--help"])
+    assert upload_help.exit_code == 0
+    upload_help_output = click.unstyle(upload_help.output)
+    assert "--github-id" in upload_help_output
+    assert "--email" in upload_help_output
+
+
+def test_upload_requires_github_id_and_email_parameters(tmp_path: Path) -> None:
+    """Guards PR #992: every CLI upload must name its contributor."""
+    trial = _trial(tmp_path)
+    result = runner.invoke(app, ["traj", "upload", str(trial)])
+
+    assert result.exit_code == 2
+    assert "--github-id" in click.unstyle(result.output)
+
+    result = runner.invoke(
+        app,
+        ["traj", "upload", str(trial), "--github-id", GITHUB_ID],
+    )
+    assert result.exit_code == 2
+    assert "--email" in click.unstyle(result.output)
+
+
+@pytest.mark.parametrize(
+    ("args", "message"),
+    [
+        (("--github-id", "@not-a-github-id", "--email", EMAIL), "GitHub ID"),
+        (("--github-id", GITHUB_ID, "--email", "not-an-email"), "email"),
+    ],
+)
+def test_upload_validates_contributor_parameters_locally(
+    tmp_path: Path, args: tuple[str, ...], message: str
+) -> None:
+    """Malformed contributor provenance fails before the upload handshake."""
+    result = runner.invoke(
+        app,
+        ["traj", "upload", str(_trial(tmp_path)), *args, "--dry-run"],
+    )
+
+    assert result.exit_code == 1
+    assert message in result.output