codna 0.1.36__tar.gz → 0.1.37__tar.gz

{codna-0.1.36 → codna-0.1.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codna
-Version: 0.1.36
+Version: 0.1.37
 Summary: Local-first repo intelligence and agent fixing for Codna.
 Author: Thyn
 Maintainer: Thyn

{codna-0.1.36 → codna-0.1.37}/codna/__init__.py

@@ -5,4 +5,4 @@ This package is the `codna` CLI. Users never touch "algenta" directly — the en
 is an internal dependency reached over HTTP.
 """
 
-__version__ = "0.1.36"
+__version__ = "0.1.37"

{codna-0.1.36 → codna-0.1.37}/codna/fix_inputs.py

@@ -5,7 +5,9 @@ extraction and PR-body rendering. Split out of ``cli.py`` to keep it under the m
 from __future__ import annotations
 
 import os
+import shutil
 import subprocess
+import tempfile
 
 
 class FixInputError(Exception):
@@ -65,7 +67,79 @@ def _discover_failing_tests_for_fix(repo_dir: str, test_cmd: str | None):
         return discover_failing_tests(worktree.path, test_cmd)
 
 
-def resolve_issue(args):
+def _is_remote_repo_spec(repo: str) -> bool:
+    clean = (repo or "").strip()
+    return clean.startswith(("http://", "https://", "git@")) or clean.endswith(".git")
+
+
+def _run_git_for_remote_tests(args: list[str], *, cwd: str | None, env: dict[str, str] | None) -> None:
+    try:
+        proc = subprocess.run(
+            ["git", *args],
+            cwd=cwd,
+            env=env,
+            capture_output=True,
+            text=True,
+            timeout=120,
+            check=False,
+        )
+    except (OSError, subprocess.SubprocessError) as exc:
+        raise FixInputError(f"could not prepare remote checkout for --tests: {exc}") from exc
+    if proc.returncode != 0:
+        stderr = (proc.stderr or proc.stdout or "").strip()[-1000:]
+        raise FixInputError(f"could not prepare remote checkout for --tests: {stderr}")
+
+
+def _checkout_remote_ref(checkout: str, ref: str, *, repository_url: str, token: str | None) -> None:
+    from .packaged_git import _git_auth_env, _validated_ref
+
+    clean_ref = _validated_ref(ref)
+    env = _git_auth_env(repository_url, token)
+    fetch = subprocess.run(
+        ["git", "fetch", "--depth=1", "origin", clean_ref],
+        cwd=checkout,
+        env=env,
+        capture_output=True,
+        text=True,
+        timeout=120,
+        check=False,
+    )
+    target = "FETCH_HEAD" if fetch.returncode == 0 else clean_ref
+    _run_git_for_remote_tests(["checkout", "--detach", target], cwd=checkout, env=env)
+    _run_git_for_remote_tests(["reset", "--hard", "HEAD"], cwd=checkout, env=env)
+    _run_git_for_remote_tests(["clean", "-fdx"], cwd=checkout, env=env)
+
+
+def _discover_remote_failing_tests_for_fix(args, github_token: str | None):
+    from .packaged_git import PackagedGitError, _git_auth_env, _reject_embedded_credentials
+
+    repo_url = getattr(args, "repo", "") or ""
+    try:
+        _reject_embedded_credentials(repo_url)
+        env = _git_auth_env(repo_url, github_token)
+    except PackagedGitError as exc:
+        raise FixInputError(str(exc)) from exc
+
+    root = tempfile.mkdtemp(prefix="codna-remote-test-discovery-")
+    checkout = os.path.join(root, "repo")
+    try:
+        clone_args = ["clone", "--no-tags"]
+        ref = getattr(args, "ref", None)
+        base_branch = getattr(args, "base_branch", None)
+        if not ref:
+            clone_args.append("--depth=1")
+            if base_branch:
+                clone_args.extend(["--single-branch", "--branch", base_branch])
+        clone_args.extend([repo_url, checkout])
+        _run_git_for_remote_tests(clone_args, cwd=None, env=env)
+        if ref:
+            _checkout_remote_ref(checkout, ref, repository_url=repo_url, token=github_token)
+        return _discover_failing_tests_for_fix(checkout, getattr(args, "test_cmd", None))
+    finally:
+        shutil.rmtree(root, ignore_errors=True)
+
+
+def resolve_issue(args, github_token: str | None = None):
     """Resolve ``(issue_text, failing_test_ids)`` from the fix args. Raises FixInputError on bad input.
 
     Precedence: ``--from-junit`` and ``--failing-test`` are explicit inputs; ``--tests`` auto-discovers
@@ -80,9 +154,13 @@ def resolve_issue(args):
         return (args.issue or issue), merged
 
     if getattr(args, "tests", False):
-        repo_dir = os.path.abspath(os.path.expanduser(getattr(args, "repo", ".") or "."))
         try:
-            issue, failing = _discover_failing_tests_for_fix(repo_dir, getattr(args, "test_cmd", None))
+            repo = getattr(args, "repo", ".") or "."
+            if _is_remote_repo_spec(repo):
+                issue, failing = _discover_remote_failing_tests_for_fix(args, github_token)
+            else:
+                repo_dir = os.path.abspath(os.path.expanduser(repo))
+                issue, failing = _discover_failing_tests_for_fix(repo_dir, getattr(args, "test_cmd", None))
         except Exception as exc:  # noqa: BLE001 - surface any discovery failure as a clean CLI error
             raise FixInputError(str(exc)) from exc
         merged = list(dict.fromkeys([*explicit_failing, *failing]))

{codna-0.1.36 → codna-0.1.37}/codna/fix_run.py

@@ -272,13 +272,6 @@ def run_fix(c, args) -> dict:
     from . import cli as _cli
     from .cli_errors import format_cli_error
 
-    try:
-        issue, failing = resolve_issue(args)
-    except FixInputError as exc:
-        _cli._die(str(exc))
-    if not issue and not failing:
-        _cli._die("`codna fix` needs --issue (or --from-junit / --tests) describing what's broken.")
-
     gh_token = (args.github_token or os.environ.get("CODNA_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN"))
     open_pr = bool(getattr(args, "open_pr", False))
     apply = bool(getattr(args, "apply", False))
@@ -288,6 +281,13 @@ def run_fix(c, args) -> dict:
         if not gh_token:
             _cli._die("--open-pr needs a write token: pass --github-token or set GITHUB_TOKEN.")
 
+    try:
+        issue, failing = resolve_issue(args, github_token=gh_token)
+    except FixInputError as exc:
+        _cli._die(str(exc))
+    if not issue and not failing:
+        _cli._die("`codna fix` needs --issue (or --from-junit / --tests) describing what's broken.")
+
     verify = (bool(getattr(args, "tests", False)) or bool(getattr(args, "test_cmd", None))) and apply and not open_pr
     max_iterations = max(1, int(getattr(args, "max_iterations", 1) or 1)) if verify else 1
 

{codna-0.1.36 → codna-0.1.37}/codna/memory.py

@@ -623,17 +623,33 @@ def _semantic_rerank(query: str, cands: list, mem) -> list:
 
 def repo_id_for(repo_path: str) -> str:
     """Stable repo identity for partitioning: ``github.com/owner/repo`` if there's an origin remote,
-    else the absolute path (so memory works on any local checkout, not only GitHub repos)."""
+    else the absolute path (so memory works on any local checkout, not only GitHub repos).
+
+    Git deliberately walks up parent directories. That is correct for commands run at a repo root, but
+    unsafe for arbitrary local paths: a temporary non-git repo under this source checkout must not inherit
+    Codna's own GitHub remote and collide with other paths in a shared collection.
+    """
+    local = os.path.abspath(os.path.expanduser(repo_path))
     try:
-        out = subprocess.run(["git", "-C", repo_path, "remote", "get-url", "origin"],
-                             capture_output=True, text=True, timeout=5)
-        if out.returncode == 0:
+        root = subprocess.run(
+            ["git", "-C", local, "rev-parse", "--show-toplevel"],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        git_root = os.path.abspath(os.path.expanduser(root.stdout.strip())) if root.returncode == 0 else ""
+        if git_root and os.path.realpath(git_root) == os.path.realpath(local):
+            out = subprocess.run(["git", "-C", local, "remote", "get-url", "origin"],
+                                 capture_output=True, text=True, timeout=5)
+        else:
+            out = None
+        if out is not None and out.returncode == 0:
             m = re.search(r"github\.com[:/]([^/]+/[^/.]+)", out.stdout)
             if m:
                 return f"github.com/{m.group(1)}"
     except (OSError, subprocess.SubprocessError):
         pass
-    return os.path.abspath(os.path.expanduser(repo_path))
+    return local
 
 
 def _git(repo_path: str, *args: str) -> str | None:

{codna-0.1.36 → codna-0.1.37}/codna/packaged_agent_runner.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import shutil
 import subprocess
 from dataclasses import dataclass
@@ -187,6 +188,14 @@ def _project_guidance(request: PackagedAgentRunRequest) -> str | None:
 def _issue_text_with_guidance(request: PackagedAgentRunRequest) -> str:
     """The Cline agent's task, with project guidance appended so the agent honors it while editing."""
     allowed = _allowed_write_paths(request)
+    test_guidance = ""
+    if _is_test_driven_fix(request.signals) and not _explicit_test_edit_request(request.signals):
+        test_guidance = (
+            "\n\n[CODNA FAILING-TEST FIX CONTRACT]\n"
+            "The failing tests are verification signals, not the target fix. "
+            "Do not modify test/spec files unless the user explicitly states that the test expectation is wrong. "
+            "Fix the production/source code that makes the tests fail."
+        )
     path_guidance = ""
     if allowed:
         path_guidance = (
@@ -198,10 +207,10 @@ def _issue_text_with_guidance(request: PackagedAgentRunRequest) -> str:
         )
     guidance = _project_guidance(request)
     if not guidance:
-        return request.issue_text + path_guidance
+        return request.issue_text + test_guidance + path_guidance
     return (f"{request.issue_text}\n\n"
             "[PROJECT GUIDANCE — honor these project rules while making changes]\n"
-            f"{guidance}{path_guidance}")
+            f"{guidance}{test_guidance}{path_guidance}")
 
 
 def _injected_context(request: PackagedAgentRunRequest) -> dict[str, Any]:
@@ -241,6 +250,9 @@ def _allowed_write_paths(request: PackagedAgentRunRequest) -> list[str]:
     for item in request.evidence_bundle.get("evidence_items", []):
         if isinstance(item, dict):
             _append_path(output, item.get("file_path"))
+    if _is_test_driven_fix(request.signals) and not _explicit_test_edit_request(request.signals):
+        non_test_paths = [path for path in output if not _is_test_path(path)]
+        return non_test_paths
     return output
 
 
@@ -255,6 +267,36 @@ def _append_path(output: list[str], value: Any) -> None:
         output.append(normalized)
 
 
+def _is_test_driven_fix(signals: Mapping[str, Any]) -> bool:
+    if _string_list(signals.get("failing_tests")):
+        return True
+    issue = signals.get("issue_text")
+    return isinstance(issue, str) and bool(
+        re.search(r"\b(failing test|tests? are failing|unittest|pytest|junit|assertionerror)\b", issue, re.I)
+    )
+
+
+def _explicit_test_edit_request(signals: Mapping[str, Any]) -> bool:
+    issue = signals.get("issue_text")
+    if not isinstance(issue, str):
+        return False
+    return bool(
+        re.search(r"\b(update|change|rewrite|correct)\s+(the\s+)?(test|tests|assertion|expectation)s?\b", issue, re.I)
+        or re.search(r"\b(test|tests|assertion|expectation)s?\s+(is|are)\s+(wrong|incorrect|outdated)\b", issue, re.I)
+    )
+
+
+def _is_test_path(path: str) -> bool:
+    normalized = path.replace("\\", "/").lower()
+    parts = normalized.split("/")
+    name = parts[-1] if parts else normalized
+    return (
+        any(part in {"test", "tests", "__tests__", "spec", "specs"} for part in parts[:-1])
+        or name.startswith("test_")
+        or name.endswith(("_test.py", "_spec.py", ".test.js", ".test.ts", ".spec.js", ".spec.ts"))
+    )
+
+
 def _dynamic_timeout_ms(snapshot: Mapping[str, Any]) -> int:
     raw_tokens = _int_value(snapshot.get("raw_repo_token_estimate"))
     files = _int_value(snapshot.get("snapshot_file_count") or snapshot.get("file_count"))

{codna-0.1.36 → codna-0.1.37}/codna/packaged_repository_advanced.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import hashlib
 import json
 import os
+import re
 import subprocess
 from dataclasses import dataclass
 from datetime import datetime, timezone
@@ -102,13 +103,21 @@ class PackagedRepositoryAdvanced:
                     "session_id": result.session_id,
                 },
             )
+        changed_files = _changed_files_from_patch(patch_diff)
+        _validate_patch_scope_for_test_driven_fix(
+            signals=run_request.signals,
+            changed_files=changed_files,
+            repository_id=repository_id,
+            snapshot_id=str(snapshot["snapshot_id"]),
+            agent_run_id=result.agent_run_id,
+            session_id=result.session_id,
+        )
         created_at = _utc_now()
         patch_id = "patch_" + _sha256_text(
             f"{repository_id}|{snapshot['snapshot_id']}|{patch_diff}"
         )[:24]
         patch_path = self._patch_dir / f"{patch_id}.diff"
         _atomic_write_text(patch_path, patch_diff + "\n")
-        changed_files = _changed_files_from_patch(patch_diff)
         plan_id = "plan_" + _sha256_text(
             f"{repository_id}|{snapshot['snapshot_id']}|{patch_id}|{issue_text}"
         )[:24]
@@ -533,6 +542,71 @@ def _changed_files_from_patch(patch_diff: str) -> list[str]:
     return files
 
 
+def _validate_patch_scope_for_test_driven_fix(
+    *,
+    signals: Mapping[str, Any],
+    changed_files: list[str],
+    repository_id: str,
+    snapshot_id: str,
+    agent_run_id: str | None,
+    session_id: str | None,
+) -> None:
+    if not _is_test_driven_fix(signals) or not changed_files:
+        return
+    if _explicit_test_edit_request(signals):
+        return
+    if any(not _is_test_path(path) for path in changed_files):
+        return
+    raise PackagedRepositoryAdvancedError(
+        "local_repository_agent_test_only_patch",
+        "Codna refused a test-only patch for a failing-test fix. Fix source code, not the test, "
+        "or pass an explicit --issue stating that the test expectation itself is wrong.",
+        {
+            "repository_id": repository_id,
+            "snapshot_id": snapshot_id,
+            "changed_files": changed_files,
+            "agent_run_id": agent_run_id,
+            "session_id": session_id,
+        },
+    )
+
+
+def _is_test_driven_fix(signals: Mapping[str, Any]) -> bool:
+    if _string_list(signals.get("failing_tests")):
+        return True
+    issue = signals.get("issue_text")
+    return isinstance(issue, str) and bool(
+        re.search(r"\b(failing test|tests? are failing|unittest|pytest|junit|assertionerror)\b", issue, re.I)
+    )
+
+
+def _explicit_test_edit_request(signals: Mapping[str, Any]) -> bool:
+    issue = signals.get("issue_text")
+    if not isinstance(issue, str):
+        return False
+    return bool(
+        re.search(r"\b(update|change|rewrite|correct)\s+(the\s+)?(test|tests|assertion|expectation)s?\b", issue, re.I)
+        or re.search(r"\b(test|tests|assertion|expectation)s?\s+(is|are)\s+(wrong|incorrect|outdated)\b", issue, re.I)
+    )
+
+
+def _is_test_path(path: str) -> bool:
+    normalized = path.replace("\\", "/").lower()
+    parts = normalized.split("/")
+    name = parts[-1] if parts else normalized
+    return (
+        any(part in {"test", "tests", "__tests__", "spec", "specs"} for part in parts[:-1])
+        or name.startswith("test_")
+        or name.endswith(("_test.py", "_spec.py", ".test.js", ".test.ts", ".spec.js", ".spec.ts"))
+    )
+
+
+def _string_list(value: Any) -> list[str]:
+    if not isinstance(value, list):
+        return []
+    return [item for item in value if isinstance(item, str) and item.strip()]
+
+
 def _changed_line_count(patch_diff: str) -> int:
     count = 0
     for line in patch_diff.splitlines():

{codna-0.1.36 → codna-0.1.37}/codna/packaged_repository_backend.py

@@ -402,7 +402,12 @@ class PackagedRepositoryBackend:
         signals = request.get("signals") if isinstance(request.get("signals"), dict) else {}
         query = _query_text(signals)
         changed_files = _string_list(signals.get("changed_files"))
-        ranked = self._rank_files(snapshot, query, changed_files)
+        ranked = self._rank_files(
+            snapshot,
+            query,
+            changed_files,
+            test_driven=_is_test_driven_fix(signals),
+        )
         evidence_items = [
             _evidence_item(rank=index + 1, entry=entry, score=score, query=query)
             for index, (entry, score) in enumerate(ranked[:MAX_EVIDENCE_ITEMS])
@@ -550,6 +555,8 @@ class PackagedRepositoryBackend:
         snapshot: Mapping[str, Any],
         query: str,
         changed_files: list[str],
+        *,
+        test_driven: bool = False,
     ) -> list[tuple[FileEntry, float]]:
         changed = set(changed_files) | set(_string_list(snapshot.get("focus_paths")))
         terms = _query_terms(query)
@@ -565,7 +572,7 @@ class PackagedRepositoryBackend:
                 sample=str(item.get("sample") or ""),
                 sample_sha256=str(item.get("sample_sha256") or ""),
             )
-            ranked.append((entry, _score_entry(entry, terms, changed)))
+            ranked.append((entry, _score_entry(entry, terms, changed, test_driven=test_driven)))
         return sorted(ranked, key=lambda row: (-row[1], row[0].path))
 
     def _load_snapshot(self, snapshot_id: str) -> dict[str, Any]:
@@ -695,7 +702,7 @@ def _normalized_suffix(path: Path) -> str:
     return path.suffix.lower()
 
 
-def _score_entry(entry: FileEntry, terms: set[str], changed_files: set[str]) -> float:
+def _score_entry(entry: FileEntry, terms: set[str], changed_files: set[str], *, test_driven: bool = False) -> float:
     if entry.path in changed_files:
         return 10_000.0
     path_text = entry.path.lower().replace("_", "-")
@@ -713,13 +720,35 @@ def _score_entry(entry: FileEntry, terms: set[str], changed_files: set[str]) ->
         score -= 12.0
     if any(part in {"src", "lib", "app", "packages", "cmd", "internal"} for part in entry.path.split("/")):
         score += 12.0
-    if any(part in {"test", "tests", "__tests__", "spec"} for part in entry.path.split("/")):
+    if _is_test_path(entry.path):
         score -= 4.0
+        if test_driven:
+            score -= 180.0
     if entry.path.endswith((".json", ".lock", ".snap")):
         score -= 10.0
     return score
 
 
+def _is_test_driven_fix(signals: Mapping[str, Any]) -> bool:
+    if _string_list(signals.get("failing_tests")):
+        return True
+    issue = signals.get("issue_text")
+    return isinstance(issue, str) and bool(
+        re.search(r"\b(failing test|tests? are failing|unittest|pytest|junit|assertionerror)\b", issue, re.I)
+    )
+
+
+def _is_test_path(path: str) -> bool:
+    normalized = path.replace("\\", "/").lower()
+    parts = normalized.split("/")
+    name = parts[-1] if parts else normalized
+    return (
+        any(part in {"test", "tests", "__tests__", "spec", "specs"} for part in parts[:-1])
+        or name.startswith("test_")
+        or name.endswith(("_test.py", "_spec.py", ".test.js", ".test.ts", ".spec.js", ".spec.ts"))
+    )
+
+
 def _evidence_item(*, rank: int, entry: FileEntry, score: float, query: str) -> dict[str, Any]:
     selection = _snippet_selection(entry.sample, _query_terms(query))
     snippet = selection.snippet

{codna-0.1.36 → codna-0.1.37}/codna/webhook_github.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import json
 import os
-from typing import Any
+from typing import Any, Mapping
 
 from .webhook import WebhookError
 

{codna-0.1.36 → codna-0.1.37}/codna.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: codna
-Version: 0.1.36
+Version: 0.1.37
 Summary: Local-first repo intelligence and agent fixing for Codna.
 Author: Thyn
 Maintainer: Thyn

{codna-0.1.36 → codna-0.1.37}/codna.egg-info/SOURCES.txt

@@ -147,6 +147,7 @@ tests/test_netjail.py
 tests/test_packaged_confidence.py
 tests/test_packaged_guidance.py
 tests/test_packaged_remote_pr.py
+tests/test_packaged_test_fix_scope.py
 tests/test_packaging.py
 tests/test_patchgen.py
 tests/test_policy.py

{codna-0.1.36 → codna-0.1.37}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "codna"
-version = "0.1.36"
+version = "0.1.37"
 description = "Local-first repo intelligence and agent fixing for Codna."
 readme = "README.md"
 requires-python = ">=3.10"

{codna-0.1.36 → codna-0.1.37}/tests/test_cli_surface_parity.py

@@ -67,6 +67,26 @@ def test_resolve_issue_tests_discovers(monkeypatch):
     assert failing == ["p::t1", "p::t2"] and "failing" in issue
 
 
+def test_resolve_issue_tests_materializes_remote_repo_for_discovery(monkeypatch):
+    captured = {}
+
+    def fake_remote_discovery(args, github_token):
+        captured["repo"] = args.repo
+        captured["github_token"] = github_token
+        return "1 failing test(s)", ["remote::test"]
+
+    monkeypatch.setattr("codna.fix_inputs._discover_remote_failing_tests_for_fix", fake_remote_discovery)
+
+    issue, failing = resolve_issue(
+        _Args(tests=True, repo="https://github.com/owner/repo.git"),
+        github_token="ghs_test",
+    )
+
+    assert issue == "1 failing test(s)"
+    assert failing == ["remote::test"]
+    assert captured == {"repo": "https://github.com/owner/repo.git", "github_token": "ghs_test"}
+
+
 def test_resolve_issue_tests_no_failures_and_no_issue_raises(monkeypatch):
     monkeypatch.setattr("codna.testrun.discover_failing_tests", lambda repo, cmd: (None, []))
     with pytest.raises(FixInputError):

{codna-0.1.36 → codna-0.1.37}/tests/test_fix_run.py

@@ -179,6 +179,49 @@ def test_apply_local_branch(monkeypatch):
     assert result["verified"] is None  # no --tests → no verification
 
 
+def test_open_pr_with_tests_keeps_remote_repo_as_fix_target(monkeypatch):
+    captured = {}
+
+    def fake_resolve_issue(args, github_token=None):
+        captured["resolve_token"] = github_token
+        return "1 failing test(s)", ["remote::test"]
+
+    def fake_plan_once(_client, **kwargs):
+        captured["plan"] = kwargs
+        return "rid", {"snapshot_id": "s"}, _plan()
+
+    monkeypatch.setattr(fix_run, "resolve_issue", fake_resolve_issue)
+    monkeypatch.setattr(fix_run, "_plan_once", fake_plan_once)
+    monkeypatch.setattr(
+        fix_run,
+        "_apply_once",
+        lambda *a, **k: {
+            "mode": "remote_pr",
+            "pull_request_url": "https://github.com/owner/repo/pull/1",
+            "status": "created",
+        },
+    )
+
+    result = fix_run.run_fix(
+        object(),
+        _Args(
+            repo="https://github.com/owner/repo.git",
+            open_pr=True,
+            tests=True,
+            issue=None,
+            github_token="ghs_test",
+        ),
+    )
+
+    assert captured["resolve_token"] == "ghs_test"
+    assert captured["plan"]["repo"] == "https://github.com/owner/repo.git"
+    assert captured["plan"]["open_pr"] is True
+    assert captured["plan"]["gh_token"] == "ghs_test"
+    assert captured["plan"]["failing"] == ["remote::test"]
+    assert result["pull_request_url"] == "https://github.com/owner/repo/pull/1"
+    assert result["verified"] is None
+
+
 # discover_failing_tests contract: GREEN is (None, []); a failure is (issue_text, [ids]) OR
 # (issue_text, []) when the run failed but produced no per-test ids.
 _FAIL = ("1 failing test(s): pkg::t1", ["t1"])