atlas-init 0.3.7__py3-none-any.whl → 0.4.1__py3-none-any.whl

atlas_init/cli_helper/run_manager.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import logging
+import os
+import signal
+import subprocess  # nosec
+import sys
+import threading
+import time
+from collections import deque
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass, field
+from logging import Logger
+from pathlib import Path
+from time import monotonic
+
+from atlas_init.cli_helper.run import LOG_CMD_PREFIX, find_binary_on_path
+
+default_logger = logging.getLogger(__name__)
+
+
+class ResultInProgressError(Exception):
+    pass
+
+
+class ResultDoneError(Exception):
+    pass
+
+
+class LogTextNotFoundError(Exception):
+    def __init__(self, store: ResultStore) -> None:
+        self.store = store
+        super().__init__(store)
+
+
+@dataclass
+class WaitOnText:
+    line: str
+    timeout: float
+
+
+@dataclass
+class ResultStore:
+    wait_condition: WaitOnText | None = None
+    _recent_lines: deque = field(default_factory=lambda: deque(maxlen=1000))
+
+    result: list[str] = field(default_factory=list)
+    exit_code: int | None = None
+    _aborted: bool = False
+    _terminated: bool = False
+    _killed: bool = False
+
+    @property
+    def result_str(self) -> str:
+        return "".join(self.result)
+
+    @property
+    def is_ok(self) -> bool:
+        if self.in_progress():
+            raise ResultInProgressError
+        return self.exit_code == 0
+
+    def _add_line(self, line: str) -> None:
+        self._recent_lines.append(line)
+        self.result.append(line)
+
+    def unexpected_error(self) -> bool:
+        if self.in_progress():
+            raise ResultInProgressError
+        return self.exit_code != 0 and not self._aborted
+
+    def force_stopped(self) -> bool:
+        if self.in_progress():
+            raise ResultInProgressError
+        return self._killed or self._terminated
+
+    def in_progress(self) -> bool:
+        return self.exit_code is None
+
+    def wait(self) -> None:
+        condition = self.wait_condition
+        if not condition:
+            return
+        timeout = condition.timeout
+        start = monotonic()
+        while monotonic() - start < timeout:
+            if not self.in_progress():
+                raise LogTextNotFoundError(self)
+            while self._recent_lines:
+                line = self._recent_lines.popleft()
+                if condition.line in line:
+                    return
+            time.sleep(0.1)
+        raise LogTextNotFoundError(self)
+
+    def _abort(self) -> None:
+        self._aborted = True
+
+    def _terminate(self) -> None:
+        self._terminated = True
+
+    def _kill(self) -> None:
+        self._killed = True
+
+
+class RunManager:
+    def __init__(
+        self,
+        worker_count: int = 100,
+        signal_int_timeout_s: float = 0.2,
+        signal_term_timeout_s: float = 0.2,
+        signal_kill_timeout_s: float = 0.2,
+        *,
+        dry_run: bool = False,
+    ):
+        """
+        Args:
+            worker_count: the number of workers to run in parallel
+            terminate_read_timeout: the time to wait after terminating a process before closing the output
+        """
+        self.processes: dict[int, subprocess.Popen] = {}
+        self.results: dict[int, ResultStore] = {}
+        self.lock = threading.RLock()
+        self.pool = ThreadPoolExecutor(max_workers=worker_count)
+        self.signal_int_timeout_s = signal_int_timeout_s
+        self.signal_term_timeout_s = signal_term_timeout_s
+        self.signal_kill_timeout_s = signal_kill_timeout_s
+        self.dry_run = dry_run
+
+    def set_timeouts(self, timeout: float):
+        self.signal_int_timeout_s = timeout
+        self.signal_term_timeout_s = timeout
+        self.signal_kill_timeout_s = timeout
+
+    def __enter__(self):
+        self.pool.__enter__()
+        return self
+
+    def run_process_wait_on_log(
+        self,
+        command: str,
+        cwd: Path,
+        logger: Logger | None,
+        env: dict | None = None,
+        result_store: ResultStore | None = None,
+        *,
+        line_in_log: str,
+        timeout: float,
+        binary: str = "",
+    ) -> Future[ResultStore]:
+        command = self._resolve_command(binary, command, logger)
+        store = result_store or ResultStore()
+        store.wait_condition = WaitOnText(line=line_in_log, timeout=timeout)
+        future = self.pool.submit(self._run, command, cwd, logger, env, store)
+        if not self.dry_run:
+            store.wait()
+        return future
+
+    def run_process(
+        self,
+        command: str,
+        cwd: Path,
+        logger: Logger | None,
+        env: dict | None = None,
+        result_store: ResultStore | None = None,
+        *,
+        binary: str = "",
+    ) -> Future[ResultStore]:
+        command = self._resolve_command(binary, command, logger)
+        return self.pool.submit(self._run, command, cwd, logger, env, result_store)
+
+    def _resolve_command(self, binary: str, command: str, logger: Logger | None):
+        if binary:
+            binary_path = find_binary_on_path(binary, logger or default_logger, allow_missing=self.dry_run)
+            command = f"{binary_path} {command}"
+        return command
+
+    def _run(
+        self,
+        command: str,
+        cwd: Path,
+        logger: Logger | None,
+        env: dict | None = None,
+        result: ResultStore | None = None,
+    ) -> ResultStore:
+        result = result or ResultStore()
+        logger = logger or default_logger
+
+        def read_output(process: subprocess.Popen):
+            for line in process.stdout:  # type: ignore
+                result._add_line(line)
+
+        sys_stderr = sys.stderr
+
+        def read_stderr(process: subprocess.Popen):
+            for line in process.stderr:  # type: ignore
+                sys_stderr.write(line)
+                result._add_line(line)
+
+        logger.info(f"{LOG_CMD_PREFIX}{command}' from '{cwd}'")
+        if self.dry_run:
+            result.exit_code = 0
+            result.result.append(f"DRY RUN: {command}")
+            return result
+        with subprocess.Popen(
+            command,
+            cwd=cwd,
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            stdin=sys.stdin,
+            start_new_session=True,
+            shell=True,  # noqa: S602 # We control the calls to this function and don't suspect any shell injection #nosec
+            bufsize=0,
+            text=True,  # This makes it return strings instead of bytes
+        ) as process:
+            with self.lock:
+                self.processes[threading.get_ident()] = process
+                self.results[threading.get_ident()] = result
+            read_future_out = self.pool.submit(read_output, process)
+            read_future_err = self.pool.submit(read_stderr, process)
+            try:
+                process.wait()
+            except Exception:
+                logger.exception(f"failed to run command: {command}")
+            finally:
+                for std_name, future in zip(["stdout", "stderr"], [read_future_out, read_future_err], strict=False):
+                    try:
+                        future.result(1)
+                    except BaseException:
+                        logger.exception(f"failed to read output ({std_name}) for command: {command}")
+                with self.lock:
+                    del self.processes[threading.get_ident()]
+                    del self.results[threading.get_ident()]
+        result.exit_code = process.returncode
+        if result.unexpected_error():
+            logger.error(f"command failed '{command}', error code: {result.exit_code}")
+        if result.force_stopped():
+            logger.error(f"command killed '{command}'")
+        return result
+
+    def __exit__(self, *_):
+        self.pool.shutdown(wait=False, cancel_futures=True)  # wait happens in __exit__, avoid new futures starting
+        self.terminate_all()
+        self.pool.__exit__(None, None, None)
+
+    def terminate_all(self):
+        self._send_signal_to_all(signal.SIGINT, ResultStore._abort)
+        self.wait_for_processes_ok(self.signal_int_timeout_s)
+        self._send_signal_to_all(signal.SIGTERM, ResultStore._terminate)
+        self.wait_for_processes_ok(self.signal_term_timeout_s)
+        self._send_signal_to_all(signal.SIGKILL, ResultStore._kill)
+        self.wait_for_processes_ok(self.signal_kill_timeout_s)
+
+    def _send_signal_to_all(self, signal_type: signal.Signals, result_call: Callable[[ResultStore], None]):
+        with self.lock:
+            for pid, process in self.processes.items():
+                result_call(self.results[pid])
+                gpid = os.getpgid(process.pid)
+                os.killpg(gpid, signal_type)
+
+    def wait_for_processes_ok(self, timeout: float):
+        start = monotonic()
+        if not self.processes:
+            return True
+        while monotonic() - start < timeout:
+            with self.lock:
+                if not any(result.in_progress() for result in self.results.values()):
+                    return True
+            time.sleep(0.1)
+        return False

atlas_init/cli_helper/tf_runner.py

@@ -8,8 +8,7 @@ from model_lib import dump
 from zero_3rdparty.file_utils import copy, iter_paths_and_relative
 
 from atlas_init.cli_helper.run import (
-    add_to_clipboard,
-    run_command_is_ok,
+    run_binary_command_is_ok,
     run_command_receive_result,
 )
 from atlas_init.settings.config import TerraformVars, TestSuite
@@ -31,7 +30,7 @@ def get_tf_vars(settings: AtlasInitSettings, active_groups: list[TestSuite]) ->
         "project_name": settings.project_name,
         "out_dir": settings.profile_dir,
         "extra_env_vars": settings.manual_env_vars,
-        **settings.cfn_config(),
+        **settings.tf_vars(),
         **tf_vars.as_configs(),
     }
 
@@ -63,34 +62,28 @@ def run_terraform(settings: AtlasInitSettings, command: str, extra_args: list[st
 
 def _run_terraform(settings: AtlasInitSettings, command: str, extra_args: list[str]):
     command_parts = [
-        "terraform",
         command,
         "-var-file",
         str(settings.tf_vars_path),
         *extra_args,
     ]
-    is_ok = run_command_is_ok(
-        command_parts,
+    is_ok = run_binary_command_is_ok(
+        "terraform",
+        " ".join(command_parts),
         env=os.environ | {"TF_DATA_DIR": settings.tf_data_dir},
         cwd=settings.tf_path,
         logger=logger,
     )
     if not is_ok:
         raise TerraformRunError
-    if settings.skip_copy:
-        return
-    env_generated = settings.env_vars_generated
-    if env_generated.exists():
-        clipboard_content = "\n".join(f"export {line}" for line in env_generated.read_text().splitlines())
-        add_to_clipboard(clipboard_content, logger)
-        logger.info("loaded env-vars to clipboard ✅")
 
 
 def dump_tf_vars(settings: AtlasInitSettings, tf_vars: dict[str, Any]):
     tf_vars_path = settings.tf_vars_path
     tf_vars_path.parent.mkdir(exist_ok=True, parents=True)
     tf_vars_str = dump(tf_vars, "pretty_json")
-    logger.info(f"writing tf vars to {tf_vars_path}: \n{tf_vars_str}")
+    logger.info(f"writing tf vars to {tf_vars_path}")
+    logger.debug(f"tf vars:\n{tf_vars_str}")
     tf_vars_path.write_text(tf_vars_str)
 
 

atlas_init/cli_root/__init__.py

@@ -0,0 +1,10 @@
+_dry_run = False
+
+
+def is_dry_run():
+    return _dry_run
+
+
+def set_dry_run(dry_run: bool):
+    global _dry_run  # noqa: PLW0603 # instead of passing dry_run everywhere we can use this global variable
+    _dry_run = dry_run

atlas_init/cli_root/go_test.py

@@ -26,6 +26,7 @@ def go_test(
     names: list[str] = typer.Option(
         ..., "-n", "--names", default_factory=list, help="run only the tests with these names"
     ),
+    use_replay_mode: bool = typer.Option(False, "--replay", help="use replay mode and stored responses"),
 ):
     if export_mock_tf_log and mode != GoTestMode.individual:
         err_msg = "exporting mock-tf-log is only supported for individual tests"
@@ -54,6 +55,7 @@ def go_test(
                 re_run=re_run,
                 env_vars=env_method,
                 names=set(names),
+                use_replay_mode=use_replay_mode,
             )
         case _:
             raise NotImplementedError

atlas_init/cli_root/trigger.py

@@ -1,10 +1,16 @@
 import logging
 
 import requests
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 from zero_3rdparty.id_creator import simple_id
 
 from atlas_init.settings.env_vars import init_settings
-from atlas_init.settings.path import dump_dotenv, dump_vscode_dotenv
+from atlas_init.settings.env_vars_generated import (
+    EnvVarsGenerated,
+    RealmSettings,
+    TFModuleCluster,
+)
+from atlas_init.settings.path import dump_dotenv
 from atlas_init.typer_app import app_command
 
 logger = logging.getLogger(__name__)
@@ -12,67 +18,78 @@ logger = logging.getLogger(__name__)
 
 @app_command()
 def trigger_app():
+    create_realm_app()
+
+
+def create_realm_app():
     settings = init_settings()
-    login_req = {
-        "username": settings.MONGODB_ATLAS_PUBLIC_KEY,
-        "apiKey": settings.MONGODB_ATLAS_PRIVATE_KEY,
-    }
-    response = requests.post(
-        "https://realm-dev.mongodb.com/api/admin/v3.0/auth/providers/mongodb-cloud/login",
-        json=login_req,
-        headers={"Accept": "application/json", "Content-Type": "application/json"},
-        timeout=10,
-    )
-    response.raise_for_status()
-    token_response = response.json()
-    access_token = token_response["access_token"]
-    logger.info(f"token: {access_token}")
-    auth_headers = {"Authorization": f"Bearer {access_token}"}
-    env_vars = settings.load_env_vars_generated()
-    project_id = env_vars["MONGODB_ATLAS_PROJECT_ID"]
-    cluster_name = env_vars["MONGODB_ATLAS_CLUSTER_NAME"]
-    apps = list_apps(auth_headers, project_id)
-    if apps:
+    base_url = settings.realm_url
+    project_id = settings.env_vars_cls(EnvVarsGenerated).MONGODB_ATLAS_PROJECT_ID
+    cluster_name = settings.env_vars_cls(TFModuleCluster).MONGODB_ATLAS_CLUSTER_NAME
+    auth_headers = login_to_realm(settings, base_url)
+    realm_settings = settings.env_vars_cls_or_none(RealmSettings, path=settings.env_vars_trigger)
+    if realm_settings and function_exists(
+        base_url,
+        auth_headers,
+        project_id,
+        realm_settings.MONGODB_REALM_APP_ID,
+        realm_settings.MONGODB_REALM_FUNCTION_ID,
+    ):
+        logger.info(f"function {realm_settings.MONGODB_REALM_FUNCTION_NAME} already exists ✅")
+        settings.include_extra_env_vars_in_vscode(realm_settings.model_dump())
+        return
+    logger.info("creating new realm app")
+    if apps := list_apps(base_url, auth_headers, project_id):
         logger.info(f"got apps: {apps}")
         app_id = apps[0]["_id"]
     else:
         logger.info("no apps found, creating one")
-        app = create_app(auth_headers, project_id, cluster_name)
+        app = create_app(base_url, auth_headers, project_id, cluster_name, settings.AWS_REGION)
         logger.info(f"created app: {app}")
         app_id = app["_id"]
     logger.info(f"using app_id: {app_id}")
     suffix = simple_id(length=5)
-    service = create_service(auth_headers, project_id, cluster_name, app_id, suffix)
+    service = create_service(base_url, auth_headers, project_id, cluster_name, app_id, suffix)
     logger.info(f"new service: {service}")
     service_id = service["_id"]
     logger.info(f"using service_id: {service_id}")
-    func_response = create_function(auth_headers, project_id, app_id, suffix)
+    func_response = create_function(base_url, auth_headers, project_id, app_id, suffix)
     logger.info(f"new function: {func_response}")
     func_id = func_response["_id"]
     func_name = func_response["name"]
     logger.info(f"using func_id: {func_id}")
-    extra_env_vars = {
-        "MONGODB_REALM_APP_ID": app_id,
-        "MONGODB_REALM_SERVICE_ID": service_id,
-        "MONGODB_REALM_FUNCTION_ID": func_id,
-        "MONGODB_REALM_FUNCTION_NAME": func_name,
-        "MONGODB_REALM_BASE_URL": "https://realm-dev.mongodb.com/",
-    }
+    realm_settings = RealmSettings(
+        MONGODB_REALM_APP_ID=app_id,
+        MONGODB_REALM_SERVICE_ID=service_id,
+        MONGODB_REALM_FUNCTION_ID=func_id,
+        MONGODB_REALM_FUNCTION_NAME=func_name,
+        MONGODB_REALM_BASE_URL=base_url,
+    )
+    extra_env_vars = realm_settings.model_dump()
     dump_dotenv(settings.env_vars_trigger, extra_env_vars)
     logger.info(f"done {settings.env_vars_trigger} created with trigger env-vars ✅")
+    settings.include_extra_env_vars_in_vscode(extra_env_vars)
 
-    generated_env_vars = settings.load_env_vars_generated()
-    generated_env_vars.update(extra_env_vars)
-    dump_dotenv(settings.env_vars_generated, generated_env_vars)
-    logger.info(f"done {settings.env_vars_generated} updated  with trigger env-vars ✅")
 
-    dump_vscode_dotenv(settings.env_vars_generated, settings.env_vars_vs_code, **extra_env_vars)
-    logger.info(f"done {settings.env_vars_vs_code} updated with trigger env-vars ✅")
+def login_to_realm(settings, base_url):
+    login_req = {
+        "username": settings.MONGODB_ATLAS_PUBLIC_KEY,
+        "apiKey": settings.MONGODB_ATLAS_PRIVATE_KEY,
+    }
+    token_response = _request_post_call(
+        f"{base_url}api/admin/v3.0/auth/providers/mongodb-cloud/login",
+        data=login_req,
+        headers={"Accept": "application/json", "Content-Type": "application/json"},
+        timeout=10,
+    )
+    access_token = token_response["access_token"]
+    logger.debug(f"token: {access_token}")
+    return {"Authorization": f"Bearer {access_token}"}
 
 
-def list_apps(auth_headers: dict[str, str], project_id: str) -> list[dict]:
+def list_apps(base_url: str, auth_headers: dict[str, str], project_id: str) -> list[dict]:
     existing_apps_response = requests.get(
-        f"https://realm-dev.mongodb.com/api/admin/v3.0/groups/{project_id}/apps",
+        f"{base_url}api/admin/v3.0/groups/{project_id}/apps",
         headers=auth_headers,
         timeout=10,
     )
@@ -82,13 +99,30 @@ def list_apps(auth_headers: dict[str, str], project_id: str) -> list[dict]:
     return apps
 
 
-def create_app(auth_headers: dict[str, str], project_id: str, cluster_name: str) -> dict:
+# https://www.mongodb.com/docs/atlas/app-services/apps/deployment-models-and-regions/#cloud-deployment-regions
+_cloud_deployment_regions = {
+    "aws-eu-west-1": "IE",
+    "aws-us-west-2": "US-OR",
+    "aws-ap-southeast-2": "AU",
+    "aws-us-east-1": "US-VA",
+}
+
+
+def create_app(
+    base_url: str, auth_headers: dict[str, str], project_id: str, cluster_name: str, aws_region: str
+) -> dict:
+    provider_region = f"aws-{aws_region}"
+    location = _cloud_deployment_regions.get(provider_region)
+    if not location:
+        raise ValueError(
+            f"unknown location for provider_region: {provider_region}, only supports: {_cloud_deployment_regions}"
+        )
     create_app_req = {
         "name": "atlas-init-app",
-        "location": "US-VA",
+        "location": location,
         "deployment_model": "GLOBAL",
         "environment": "production",
-        "provider_region": "aws-us-east-1",
+        "provider_region": provider_region,
         "data_source": {
             "name": "mongodb-atlas",
             "type": "mongodb-atlas",
@@ -99,19 +133,25 @@ def create_app(auth_headers: dict[str, str], project_id: str, cluster_name: str)
             },
         },
     }
-    create_app_response = requests.post(
-        f"https://realm-dev.mongodb.com/api/admin/v3.0/groups/{project_id}/apps",
-        json=create_app_req,
+    app_response = _request_post_call(
+        f"{base_url}api/admin/v3.0/groups/{project_id}/apps",
+        data=create_app_req,
         headers=auth_headers,
         timeout=10,
+        log_data_on_failure=True,
     )
-    create_app_response.raise_for_status()
-    app = create_app_response.json()
-    assert isinstance(app, dict), f"expected dict, got: {app!r}"
-    return app
+    assert isinstance(app_response, dict), f"expected dict, got: {app_response!r}"
+    return app_response
 
 
-def create_service(auth_headers: dict[str, str], project_id: str, cluster_name: str, app_id: str, suffix: str) -> dict:
+def create_service(
+    base_url: str,
+    auth_headers: dict[str, str],
+    project_id: str,
+    cluster_name: str,
+    app_id: str,
+    suffix: str,
+) -> dict:
     create_service_req = {
         "name": f"atlas-init-{suffix}",
         "type": "mongodb-atlas",
@@ -121,19 +161,24 @@ def create_service(auth_headers: dict[str, str], project_id: str, cluster_name:
             "wireProtocolEnabled": True,
         },
     }
-    create_service_response = requests.post(
-        f"https://realm-dev.mongodb.com/api/admin/v3.0/groups/{project_id}/apps/{app_id}/services",
-        json=create_service_req,
+    service = _request_post_call(
+        f"{base_url}api/admin/v3.0/groups/{project_id}/apps/{app_id}/services",
+        data=create_service_req,
         headers=auth_headers,
         timeout=10,
+        log_data_on_failure=True,
     )
-    create_service_response.raise_for_status()
-    service = create_service_response.json()
     assert isinstance(service, dict), f"expected dict, got: {service}"
     return service
 
 
-def create_function(auth_headers: dict[str, str], project_id: str, app_id: str, suffix: str) -> dict:
+def create_function(
+    base_url: str,
+    auth_headers: dict[str, str],
+    project_id: str,
+    app_id: str,
+    suffix: str,
+) -> dict:
     create_func_req = {
         "can_evaluate": {},
         "name": f"testfunc-{suffix}",
@@ -141,13 +186,56 @@ def create_function(auth_headers: dict[str, str], project_id: str, app_id: str,
         "source": 'exports = function(changeEvent) {console.log("New Document Inserted")};',
         "run_as_system": True,
     }
-    create_func_response = requests.post(
-        f"https://realm-dev.mongodb.com/api/admin/v3.0/groups/{project_id}/apps/{app_id}/functions",
-        json=create_func_req,
+    func = _request_post_call(
+        f"{base_url}api/admin/v3.0/groups/{project_id}/apps/{app_id}/functions",
+        data=create_func_req,
         headers=auth_headers,
         timeout=10,
+        log_data_on_failure=True,
     )
-    create_func_response.raise_for_status()
-    func = create_func_response.json()
     assert isinstance(func, dict), f"expected dict, got: {func}"
     return func
+
+
+def function_exists(
+    base_url: str,
+    auth_headers: dict[str, str],
+    project_id: str,
+    app_id: str,
+    func_id: str,
+) -> bool:
+    # https://services.cloud.mongodb.com/api/admin/v3.0/groups/{groupId}/apps/{appId}/functions/{functionId}
+    get_func_response = requests.get(
+        f"{base_url}api/admin/v3.0/groups/{project_id}/apps/{app_id}/functions/{func_id}",
+        headers=auth_headers,
+        timeout=10,
+    )
+    if get_func_response.status_code == 404:  # noqa: PLR2004
+        return False
+    get_func_response.raise_for_status()
+    func = get_func_response.json()
+    assert isinstance(func, dict), f"expected dict response, got: {func}"
+    return True
+
+
+class _RetryPostRequestError(Exception):
+    pass
+
+
+@retry(
+    stop=stop_after_attempt(5),
+    wait=wait_fixed(10),
+    retry=retry_if_exception_type(),
+    reraise=True,
+)
+def _request_post_call(
+    url: str, data: dict, headers: dict[str, str], timeout: int, *, log_data_on_failure: bool = False
+) -> dict:
+    response = requests.post(url, json=data, headers=headers, timeout=timeout)
+    if response.status_code >= 500:  # noqa: PLR2004
+        logger.warning(f"failed to post to {url}, status_code: {response.status_code}, response: {response.text}")
+        if log_data_on_failure:
+            logger.warning(f"data: {data}")
+        raise _RetryPostRequestError(f"status_code: {response.status_code}, response: {response.text}")
+    response.raise_for_status()
+    return response.json()

atlas_init/cli_tf/debug_logs.py

@@ -154,7 +154,7 @@ MARKER_TEST = "Starting TestSteps: "
 
 
 class FileRef(NamedTuple):
-    index: int
+    request_index: int
     line_start: int
     line_end: int
 
@@ -249,12 +249,12 @@ def parse_raw_req_responses(
             in_response = True
             current_start = i + 1
         if in_request and line.startswith(MARKER_END):
-            key = FileRef(index=request_count, line_start=current_start, line_end=i)
+            key = FileRef(request_index=request_count, line_start=current_start, line_end=i)
             requests[key] = log_lines[current_start:i]
             request_count += 1
             in_request = False
         if in_response and line.startswith(MARKER_END):
-            key = FileRef(index=request_count, line_start=current_start, line_end=i)
+            key = FileRef(request_index=request_count, line_start=current_start, line_end=i)
             responses[key] = log_lines[current_start:i]
             response_count += 1
             in_response = False