atlas-init 0.1.0__py3-none-any.whl → 0.1.4__py3-none-any.whl

atlas_init/cli_tf/debug_logs.py

@@ -0,0 +1,221 @@
+import json
+import logging
+import re
+from contextlib import suppress
+from typing import Any, NamedTuple, Self
+
+from model_lib import Entity
+from pydantic import ValidationError, model_validator
+
+logger = logging.getLogger(__name__)
+
+
+def parsed(payload: str) -> tuple[dict[str, Any], list]:
+    with suppress(ValueError):
+        resp = json.loads(payload)
+        if isinstance(resp, dict):
+            return resp, []
+        if isinstance(resp, list):
+            return {}, resp
+    raise ValueError(f"Could not parse payload: {payload}")
+
+
+class PathHeadersPayload(Entity):
+    method: str
+    path: str
+    http_protocol: str
+    headers: dict[str, str]
+    text: str
+
+    @property
+    def expect_list_response(self) -> bool:
+        return self.method == "GET" and self.path.endswith("s") and all(not c.isdigit() for c in self.path)
+
+
+def parse_request(request_lines: list[str]) -> PathHeadersPayload:
+    path_line, *header_lines_payload = request_lines
+    headers_end = header_lines_payload.index("")
+    header_lines = header_lines_payload[:headers_end]
+    payload_lines = header_lines_payload[headers_end + 1 :]
+    payload_end = payload_lines.index("")
+    payload_lines = payload_lines[:payload_end]
+    method, path, http_protocol = path_line.split(" ")
+    return PathHeadersPayload(
+        method=method,
+        http_protocol=http_protocol,
+        path=path,
+        headers=dict(header_line.split(": ", 1) for header_line in header_lines),
+        text="\n".join(payload_lines),
+    )
+
+
+class StatusHeadersResponse(Entity):
+    http_protocol: str
+    status: int
+    status_text: str
+    headers: dict[str, str]
+    text: str
+
+
+def parse_response(response_lines: list[str]) -> StatusHeadersResponse:
+    http_protocol_status, *header_lines_response = response_lines
+    http_protocol, status, status_text = http_protocol_status.split(" ", 2)
+    headers_end = header_lines_response.index("")
+    header_lines = header_lines_response[:headers_end]
+    response = header_lines_response[headers_end + 1 :]
+    return StatusHeadersResponse(
+        http_protocol=http_protocol,
+        status=status,  # type: ignore
+        status_text=status_text,
+        headers=dict(header_line.split(": ", 1) for header_line in header_lines),
+        text="\n".join(response),
+    )
+
+
+# application/vnd.atlas.2024-08-05+json;charset=utf-8
+
+_version_date_pattern = re.compile(r"(\d{4}-\d{2}-\d{2})")
+
+
+def extract_version(content_type: str) -> str:
+    if match := _version_date_pattern.search(content_type):
+        return match.group(1)
+    raise ValueError(f"Could not extract version from {content_type} header")
+
+
+class SDKRoundtrip(Entity):
+    request: PathHeadersPayload
+    response: StatusHeadersResponse
+    resp_index: int
+    step_number: int
+
+    @property
+    def id(self) -> str:
+        return f"{self.request.method}_{self.request.path}_{self.version}"
+
+    @property
+    def version(self) -> str:
+        content_type = self.response.headers.get("Content-Type", "v1")
+        try:
+            return extract_version(content_type)
+        except ValueError:
+            logger.warning(f"failed to extract version from response header ({content_type}), trying request")
+        content_type = self.request.headers.get("Accept", "v1")
+        return extract_version(content_type)
+
+    @model_validator(mode="after")
+    def ensure_match(self) -> Self:
+        req = self.request
+        resp = self.response
+        _, resp_payload_list = parsed(resp.text)
+        if req.expect_list_response and not resp_payload_list:
+            raise ValueError(f"Expected list response but got dict: {resp.text}")
+        return self
+
+
+MARKER_END = "-----------------------------------"
+MARKER_REQUEST_START = "---[ REQUEST ]"
+MARKER_RESPONSE_START = "---[ RESPONSE ]----"
+MARKER_START_STEP = "Starting TestStep: "
+MARKER_TEST = "Starting TestSteps: "
+
+
+class FileRef(NamedTuple):
+    index: int
+    line_start: int
+    line_end: int
+
+
+def parse_http_requests(logs: str) -> list[SDKRoundtrip]:
+    """
+    Problem: With requests that are done in parallel.
+    An alternative is to use parallel 1 but it will be slow
+    Methods: (rejected)
+    1. Look for match from `path` to the something in the payload
+    2. Use the X-Java-Method header to match the response with the path
+    3. X-Envoy-Upstream-Service-Time to match it
+
+    Method: (accepted)
+    Can say that expected payload is either a list or a dict and if it ends with an identifier it is higher chance for a dict
+    """
+    test_count = logs.count(MARKER_TEST)
+    assert test_count == 1, f"Only one test is supported, found {test_count}"
+    requests, responses = parse_raw_req_responses(logs)
+    tf_step_starts = [i for i, line in enumerate(logs.splitlines()) if MARKER_START_STEP in line]
+    used_responses: set[int] = set()
+    responses_list: list[StatusHeadersResponse] = list(responses.values())
+    sdk_roundtrips = []
+    for ref, request in requests.items():
+        roundtrip = match_request(used_responses, responses_list, ref, request, tf_step_starts)
+        sdk_roundtrips.append(roundtrip)
+        used_responses.add(roundtrip.resp_index)
+    return sdk_roundtrips
+
+
+def find_step_number(ref: FileRef, step_starts: list[int]) -> int:
+    for i, step_start in enumerate(reversed(step_starts)):
+        if step_start < ref.line_start:
+            return len(step_starts) - i
+    logger.warning(f"Could not find step start for {ref}")
+    return 0
+
+
+def match_request(
+    used_responses: set[int],
+    responses_list: list[StatusHeadersResponse],
+    ref: FileRef,
+    request: PathHeadersPayload,
+    step_starts: list[int],
+) -> SDKRoundtrip:
+    for i, response in enumerate(responses_list):
+        if i in used_responses:
+            continue
+        with suppress(ValidationError):
+            step_number = find_step_number(ref, step_starts)
+            return SDKRoundtrip(request=request, response=response, resp_index=i, step_number=step_number)
+    remaining_responses = [resp for i, resp in enumerate(responses_list) if i not in used_responses]
+    err_msg = f"Could not match request {ref} with any response\n\n{request}\n\n\nThere are #{len(remaining_responses)} responses left that doesn't match\n{'-'*80}\n{'\n'.join(r.text for r in remaining_responses)}"
+    raise ValueError(err_msg)
+
+
+def parse_raw_req_responses(
+    logs: str,
+) -> tuple[dict[FileRef, PathHeadersPayload], dict[FileRef, StatusHeadersResponse]]:
+    # sourcery skip: dict-comprehension
+    request_count = 0
+    response_count = 0
+    in_request = False
+    in_response = False
+    current_start = 0
+    requests: dict[FileRef, list[str]] = {}
+    responses: dict[FileRef, list[str]] = {}
+    log_lines = logs.splitlines()
+    for i, line in enumerate(log_lines):
+        if line.startswith(MARKER_REQUEST_START):
+            in_request = True
+            current_start = i + 1
+        elif line.startswith(MARKER_RESPONSE_START):
+            in_response = True
+            current_start = i + 1
+        if in_request and line.startswith(MARKER_END):
+            key = FileRef(index=request_count, line_start=current_start, line_end=i)
+            requests[key] = log_lines[current_start:i]
+            request_count += 1
+            in_request = False
+        if in_response and line.startswith(MARKER_END):
+            key = FileRef(index=request_count, line_start=current_start, line_end=i)
+            responses[key] = log_lines[current_start:i]
+            response_count += 1
+            in_response = False
+    assert not in_request, "Request not closed"
+    assert not in_response, "Response not closed"
+    assert (
+        request_count == response_count
+    ), f"Mismatch in request and response count: {request_count} != {response_count}"
+    parsed_requests = {}
+    for ref, request_lines in requests.items():
+        parsed_requests[ref] = parse_request(request_lines)
+    parsed_responses = {}
+    for ref, response_lines in responses.items():
+        parsed_responses[ref] = parse_response(response_lines)
+    return parsed_requests, parsed_responses

atlas_init/cli_tf/debug_logs_test_data.py

@@ -0,0 +1,253 @@
+import json
+import logging
+from collections.abc import Callable
+from typing import NamedTuple
+
+from model_lib import Entity
+from pydantic import Field, model_validator
+
+from atlas_init.cli_tf.debug_logs import SDKRoundtrip
+
+logger = logging.getLogger(__name__)
+
+
+class StatusText(Entity):
+    status: int
+    text: str
+
+    @property
+    def id(self):
+        return f"{self.status}_{self.text}"
+
+
+class RequestInfo(Entity):
+    version: str
+    method: str
+    path: str
+    text: str
+    responses: list[StatusText] = Field(default_factory=list)
+
+    @property
+    def id(self):
+        return "__".join(  # noqa: FLY002
+            [
+                self.method,
+                self.path,
+                self.version,
+                self.text,
+            ]  # need to include text to differentiate between requests
+        )
+
+
+class StepRequests(Entity):
+    diff_requests: list[RequestInfo] = Field(default_factory=list)
+    request_responses: list[RequestInfo] = Field(default_factory=list)
+
+    def existing_request(self, info: RequestInfo) -> RequestInfo | None:
+        return next((r for r in self.request_responses if r.id == info.id), None)
+
+    def add_request(
+        self,
+        path: str,
+        method: str,
+        version: str,
+        status: int,
+        text: str,
+        text_response: str,
+        is_diff: bool,
+    ):
+        status_text = StatusText(status=status, text=text_response)
+        info = RequestInfo(
+            path=path,
+            method=method,
+            version=version,
+            text=text,
+            responses=[status_text],
+        )
+        if is_diff:
+            self.diff_requests.append(info)
+        if existing := self.existing_request(info):
+            existing.responses.append(status_text)
+        else:
+            self.request_responses.append(info)
+
+
+class RTModifier(Entity):
+    version: str
+    method: str
+    path: str
+    modification: Callable[[SDKRoundtrip], None]
+
+    def match(self, rt: SDKRoundtrip, normalized_path: str) -> bool:
+        return rt.request.method == self.method and normalized_path == self.path and rt.version == self.version
+
+
+class MockRequestData(Entity):
+    step_count: int
+    steps: list[StepRequests] = Field(default_factory=list, init=False)
+    variables: dict[str, str] = Field(default_factory=dict)
+
+    @model_validator(mode="after")
+    def set_steps(self):
+        self.steps = [StepRequests() for _ in range(self.step_count)]
+        return self
+
+    def add_roundtrip(
+        self,
+        rt: SDKRoundtrip,
+        normalized_path: str,
+        normalized_text: str,
+        normalized_response_text: str,
+        is_diff: bool,
+    ):
+        step = self.steps[rt.step_number - 1]
+        if rt.request.method == "PATCH":
+            logger.info(f"PATCH: {rt.request.path}")
+        step.add_request(
+            normalized_path,
+            rt.request.method,
+            rt.version,
+            rt.response.status,
+            normalized_text,
+            normalized_response_text,
+            is_diff,
+        )
+
+    def update_variables(self, variables: dict[str, str]) -> None:
+        if missing_value := sorted(name for name, value in variables.items() if not value):
+            err_msg = f"Missing values for variables: {missing_value}"
+            raise ValueError(err_msg)
+        changes: list[VariableChange] = []
+        for name, value in variables.items():
+            old_value = self.variables.get(name)
+            if old_value and old_value != value:
+                for suffix in range(2, 10):
+                    new_name = f"{name}{suffix}"
+                    old_value2 = self.variables.get(new_name, "")
+                    if old_value2 and old_value2 != value:
+                        continue
+                    if not old_value2:
+                        logger.warning(f"Adding variable {name} to {new_name}={value}")
+                    change = VariableChange(name, new_name, old_value, value)
+                    changes.append(change)
+                    self.variables[new_name] = value
+                    break
+                else:
+                    raise ValueError(f"Too many variables with the same name and different values: {name}")
+            else:
+                self.variables[name] = value
+        if changes:
+            raise VariablesChangedError(changes)
+
+    def prune_duplicate_responses(self):
+        for step in self.steps:
+            for request in step.request_responses:
+                pruned_responses = []
+                seen_response_ids = set()
+                before_len = len(request.responses)
+                for response in request.responses:
+                    if response.id in seen_response_ids:
+                        continue
+                    seen_response_ids.add(response.id)
+                    pruned_responses.append(response)
+                request.responses = pruned_responses
+                after_len = len(request.responses)
+                if before_len != after_len:
+                    logger.info(f"Pruned {before_len - after_len} duplicate responses from {request.id}")
+
+
+class ApiSpecPath(Entity):
+    path: str
+
+    def variables(self, path: str) -> dict[str, str]:
+        return {
+            var[1:-1]: default
+            for var, default in zip(self.path.split("/"), path.split("/"), strict=False)
+            if var.startswith("{") and var.endswith("}")
+        }
+
+    def match(self, path: str) -> bool:
+        parts_expected = self.path.split("/")
+        parts_actual = path.split("/")
+        if len(parts_expected) != len(parts_actual):
+            return False
+        for expected, actual in zip(parts_expected, parts_actual, strict=False):
+            if expected == actual:
+                continue
+            if expected.startswith("{") and expected.endswith("}"):
+                continue
+            return False
+        return True
+
+
+def find_normalized_path(path: str, api_spec_paths: list[ApiSpecPath]) -> ApiSpecPath:
+    if "?" in path:
+        path = path.split("?")[0]
+    path = path.rstrip("/")  # remove trailing slash
+    for api_spec_path in api_spec_paths:
+        if api_spec_path.match(path):
+            return api_spec_path
+    raise ValueError(f"Could not find path: {path}")
+
+
+def normalize_text(text: str, variables: dict[str, str]) -> str:
+    for var, value in variables.items():
+        text = text.replace(value, f"{{{var}}}")
+    if not text:
+        return text
+    try:
+        parsed_text = json.loads(text)
+        return json.dumps(parsed_text, indent=1, sort_keys=True)
+    except json.JSONDecodeError:
+        logger.warning(f"Could not parse text: {text}")
+        return text
+
+
+def default_is_diff(rt: SDKRoundtrip) -> bool:
+    return rt.request.method not in {"DELETE", "GET"}
+
+
+class VariableChange(NamedTuple):
+    var_name: str
+    new_var_name: str
+    old: str
+    new: str
+
+
+class VariablesChangedError(Exception):
+    def __init__(self, changes: list[VariableChange]) -> None:
+        super().__init__(f"Variables changed: {changes}")
+        self.changes = changes
+
+
+def create_mock_data(
+    roundtrips: list[SDKRoundtrip],
+    api_spec_paths: dict[str, list[ApiSpecPath]],
+    is_diff: Callable[[SDKRoundtrip], bool] | None = None,
+    modifiers: list[RTModifier] | None = None,
+) -> MockRequestData:
+    steps = max(rt.step_number for rt in roundtrips)
+    mock_data = MockRequestData(step_count=steps)
+    is_diff = is_diff or default_is_diff
+    modifiers = modifiers or []
+    for rt in roundtrips:
+        request_path = rt.request.path
+        method = rt.request.method
+        spec_path = find_normalized_path(request_path, api_spec_paths[method])
+        rt_variables = spec_path.variables(request_path)
+        normalized_path = spec_path.path
+        try:
+            mock_data.update_variables(rt_variables)
+        except VariablesChangedError as e:
+            for change in e.changes:
+                rt_variables.pop(change.var_name)
+                rt_variables[change.new_var_name] = change.new
+            normalized_path = normalize_text(request_path, rt_variables)
+        for modifier in modifiers:
+            if modifier.match(rt, normalized_path):
+                modifier.modification(rt)
+        normalized_text = normalize_text(rt.request.text, rt_variables)
+        normalized_response_text = normalize_text(rt.response.text, rt_variables)
+        mock_data.add_roundtrip(rt, normalized_path, normalized_text, normalized_response_text, is_diff(rt))
+    # requests.prune_duplicate_responses() better to keep duplicates to stay KISS
+    return mock_data

atlas_init/cli_tf/github_logs.py

@@ -0,0 +1,229 @@
+import logging
+import os
+from collections import defaultdict
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor, wait
+from datetime import datetime
+from functools import lru_cache
+from pathlib import Path
+from typing import NamedTuple
+
+import requests
+from github import Auth, Github
+from github.Repository import Repository
+from github.WorkflowJob import WorkflowJob
+from github.WorkflowRun import WorkflowRun
+from github.WorkflowStep import WorkflowStep
+from zero_3rdparty import datetime_utils, file_utils
+
+from atlas_init.cli_tf.go_test_run import GoTestRun, parse
+from atlas_init.repos.path import (
+    GH_OWNER_TERRAFORM_PROVIDER_MONGODBATLAS,
+)
+from atlas_init.settings.path import (
+    DEFAULT_GITHUB_CI_RUN_LOGS,
+    DEFAULT_GITHUB_SUMMARY_DIR,
+)
+
+logger = logging.getLogger(__name__)
+
+GH_TOKEN_ENV_NAME = "GH_TOKEN"  # noqa: S105
+GITHUB_CI_RUN_LOGS_ENV_NAME = "GITHUB_CI_RUN_LOGS"
+GITHUB_CI_SUMMARY_DIR_ENV_NAME = "GITHUB_CI_SUMMARY_DIR_ENV_NAME"
+REQUIRED_GH_ENV_VARS = [GH_TOKEN_ENV_NAME, GITHUB_CI_RUN_LOGS_ENV_NAME]
+MAX_DOWNLOADS = 5
+
+
+@lru_cache
+def get_auth() -> Auth.Auth:
+    token = os.environ[GH_TOKEN_ENV_NAME]
+    return Auth.Token(token)
+
+
+@lru_cache
+def get_repo(repo_id: str) -> Repository:
+    auth = get_auth()
+    g = Github(auth=auth)
+    logger.info(f"logged in as: {g.get_user().login}")
+    return g.get_repo(repo_id)
+
+
+_DEFAULT_FILESTEMS = {
+    "test-suite",
+    "terraform-compatibility-matrix",
+    # "acceptance-tests",
+}
+
+
+def include_filestems(stems: set[str]) -> Callable[[WorkflowRun], bool]:
+    def inner(run: WorkflowRun) -> bool:
+        workflow_stem = stem_name(run.path)
+        return workflow_stem in stems
+
+    return inner
+
+
+def stem_name(workflow_path: str) -> str:
+    return Path(workflow_path).stem
+
+
+def tf_repo() -> Repository:
+    return get_repo(GH_OWNER_TERRAFORM_PROVIDER_MONGODBATLAS)
+
+
+class WorkflowJobId(NamedTuple):
+    workflow_id: int
+    job_id: int
+
+
+def find_test_runs(
+    since: datetime,
+    include_workflow: Callable[[WorkflowRun], bool] | None = None,
+    include_job: Callable[[WorkflowJob], bool] | None = None,
+    branch: str = "master",
+) -> dict[WorkflowJobId, list[GoTestRun]]:
+    include_workflow = include_workflow or include_filestems(_DEFAULT_FILESTEMS)
+    include_job = include_job or include_test_jobs()
+    jobs_found = defaultdict(list)
+    repository = tf_repo()
+    for workflow in repository.get_workflow_runs(
+        created=f">{since.strftime('%Y-%m-%d')}",
+        branch=branch,
+        exclude_pull_requests=True,  # type: ignore
+    ):
+        if not include_workflow(workflow):
+            continue
+        workflow_dir = workflow_logs_dir(workflow)
+        paginated_jobs = workflow.jobs("all")
+        worker_count = min(paginated_jobs.totalCount, 10) or 1
+        with ThreadPoolExecutor(max_workers=worker_count) as pool:
+            futures: dict[Future[list[GoTestRun]], WorkflowJob] = {}
+            for job in paginated_jobs:
+                if not include_job(job):
+                    continue
+                future = pool.submit(find_job_test_runs, workflow_dir, job)
+                futures[future] = job
+            done, not_done = wait(futures.keys(), timeout=300)
+            for f in not_done:
+                logger.warning(f"timeout to find go tests for job = {futures[f].html_url}")
+        workflow_id = workflow.id
+        for f in done:
+            job = futures[f]
+            try:
+                go_test_runs: list[GoTestRun] = f.result()
+            except Exception:
+                job_log_path = logs_file(workflow_dir, job)
+                logger.exception(
+                    f"failed to find go tests for job: {job.html_url}, error 👆, local_path: {job_log_path}"
+                )
+                continue
+            jobs_found[WorkflowJobId(workflow_id, job.id)].extend(go_test_runs)
+    return jobs_found
+
+
+def find_job_test_runs(workflow_dir: Path, job: WorkflowJob) -> list[GoTestRun]:
+    jobs_log_path = download_job_safely(workflow_dir, job)
+    return [] if jobs_log_path is None else parse_job_logs(job, jobs_log_path)
+
+
+def parse_job_logs(job: WorkflowJob, logs_path: Path) -> list[GoTestRun]:
+    step, logs_lines = select_step_and_log_content(job, logs_path)
+    return list(parse(logs_lines, job, step))
+
+
+def download_job_safely(workflow_dir: Path, job: WorkflowJob) -> Path | None:
+    path = logs_file(workflow_dir, job)
+    job_summary = f"found test job: {job.name}, attempt {job.run_attempt}, {job.created_at}, url: {job.html_url}"
+    if path.exists():
+        logger.info(f"{job_summary} exist @ {path}")
+        return path
+    logger.info(f"{job_summary}\n\t\t downloading to {path}")
+    try:
+        logs_response = requests.get(job.logs_url(), timeout=60)
+        logs_response.raise_for_status()
+    except Exception as e:  # noqa: BLE001
+        logger.warning(f"failed to download logs for {job.html_url}, e={e!r}")
+        return None
+    file_utils.ensure_parents_write_text(path, logs_response.text)
+    return path
+
+
+def logs_dir() -> Path:
+    logs_dir_str = os.environ.get(GITHUB_CI_RUN_LOGS_ENV_NAME)
+    if not logs_dir_str:
+        logger.warning(f"using {DEFAULT_GITHUB_CI_RUN_LOGS} to store github ci logs!")
+        return DEFAULT_GITHUB_CI_RUN_LOGS
+    return Path(logs_dir_str)
+
+
+def summary_dir(summary_name: str) -> Path:
+    summary_dir_str = os.environ.get(GITHUB_CI_SUMMARY_DIR_ENV_NAME)
+    if not summary_dir_str:
+        logger.warning(f"using {DEFAULT_GITHUB_SUMMARY_DIR / summary_name} to store summaries")
+        return DEFAULT_GITHUB_SUMMARY_DIR / summary_name
+    return Path(summary_dir_str) / summary_name
+
+
+def workflow_logs_dir(workflow: WorkflowRun) -> Path:
+    dt = workflow.created_at
+    date_str = datetime_utils.get_date_as_rfc3339_without_time(dt)
+    workflow_name = stem_name(workflow.path)
+    return logs_dir() / f"{date_str}/{workflow.id}_{workflow_name}"
+
+
+def logs_file(workflow_dir: Path, job: WorkflowJob) -> Path:
+    if job.run_attempt != 1:
+        workflow_dir = workflow_dir.with_name(f"{workflow_dir.name}_attempt{job.run_attempt}")
+    filename = f"{job.id}_" + job.name.replace(" ", "").replace("/", "_").replace("__", "_") + ".txt"
+    return workflow_dir / filename
+
+
+def as_test_group(job_name: str) -> str:
+    """tests-1.8.x-latest / tests-1.8.x-latest-dev / config"""
+    return "" if "/" not in job_name else job_name.split("/")[-1].strip()
+
+
+def include_test_jobs(test_group: str = "") -> Callable[[WorkflowJob], bool]:
+    def inner(job: WorkflowJob) -> bool:
+        job_name = job.name
+        if test_group:
+            return is_test_job(job_name) and as_test_group(job_name) == test_group
+        return is_test_job(job.name)
+
+    return inner
+
+
+def is_test_job(job_name: str) -> bool:
+    """
+    >>> is_test_job("tests-1.8.x-latest / tests-1.8.x-latest-dev / config")
+    True
+    """
+    if "-before" in job_name or "-after" in job_name:
+        return False
+    return "tests-" in job_name and not job_name.endswith(("get-provider-version", "change-detection"))
+
+
+def select_step_and_log_content(job: WorkflowJob, logs_path: Path) -> tuple[int, list[str]]:
+    full_text = logs_path.read_text()
+    step = test_step(job.steps)
+    last_step_start = current_step_start = 1
+    # there is always an extra setup job step, so starting at 1
+    current_step = 1
+    lines = full_text.splitlines()
+    for line_index, line in enumerate(lines, 0):
+        if "##[group]Run " in line:
+            current_step += 1
+            last_step_start, current_step_start = current_step_start, line_index
+            if current_step == step + 1:
+                return step, lines[last_step_start:current_step_start]
+    assert step == current_step, f"didn't find enough step in logs for {job.html_url}"
+    return step, lines[current_step_start:]
+
+
+def test_step(steps: list[WorkflowStep]) -> int:
+    for i, step in enumerate(steps, 1):
+        if "test" in step.name.lower():
+            return i
+    last_step = len(steps)
+    logger.warning(f"using {last_step} as final step, unable to find 'test' in {steps}")
+    return last_step