atlas-init 0.1.8__py3-none-any.whl → 0.2.0__py3-none-any.whl

atlas_init/__init__.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-VERSION = "0.1.8"
+VERSION = "0.2.0"
 
 
 def running_in_repo() -> bool:

atlas_init/cli_args.py

@@ -1,8 +1,11 @@
 from typing import Any
 
+import typer
 from model_lib import parse_payload
 from zero_3rdparty.iter_utils import key_equal_value_to_dict
 
+option_sdk_repo_path = typer.Option("", "-sdk", "--sdk-repo-path", help="the path to the sdk repo")
+
 
 def parse_key_values(params: list[str]) -> dict[str, str]:
     return key_equal_value_to_dict(params)

atlas_init/cli_tf/app.py

@@ -9,6 +9,7 @@ import typer
 from zero_3rdparty.datetime_utils import utc_now
 from zero_3rdparty.file_utils import clean_dir
 
+from atlas_init.cli_args import option_sdk_repo_path
 from atlas_init.cli_helper.run import (
     add_to_clipboard,
     run_binary_command_is_ok,
@@ -27,6 +28,7 @@ from atlas_init.cli_tf.go_test_summary import (
     create_detailed_summary,
     create_short_summary,
 )
+from atlas_init.cli_tf.mock_tf_log import mock_tf_log_cmd
 from atlas_init.cli_tf.schema import (
     download_admin_api,
     dump_generator_config,
@@ -45,6 +47,7 @@ from atlas_init.settings.env_vars import init_settings
 from atlas_init.settings.interactive import confirm
 
 app = typer.Typer(no_args_is_help=True)
+app.command(name="mock-tf-log")(mock_tf_log_cmd)
 logger = logging.getLogger(__name__)
 
 
@@ -224,7 +227,7 @@ def schema2(
     ),
     config_path: Path = typer.Option("", "-c", "--config", help="the path to the SchemaV2 config"),
     replace: bool = typer.Option(False, "-r", "--replace", help="replace the existing schema file"),
-    sdk_repo_path_str: str = typer.Option("", "-sdk", "--sdk-repo-path", help="the path to the sdk repo"),
+    sdk_repo_path_str: str = option_sdk_repo_path,
 ):
     repo_path = current_repo_path(Repo.TF)
     config_path = config_path or repo_path / "schema_v2.yaml"

atlas_init/cli_tf/debug_logs.py

@@ -2,6 +2,7 @@ import json
 import logging
 import re
 from contextlib import suppress
+from functools import total_ordering
 from typing import Any, NamedTuple, Self
 
 from model_lib import Entity
@@ -83,6 +84,7 @@ def extract_version(content_type: str) -> str:
     raise ValueError(f"Could not extract version from {content_type} header")
 
 
+@total_ordering
 class SDKRoundtrip(Entity):
     request: PathHeadersPayload
     response: StatusHeadersResponse
@@ -112,6 +114,11 @@ class SDKRoundtrip(Entity):
             raise ValueError(f"Expected list response but got dict: {resp.text}")
         return self
 
+    def __lt__(self, other) -> bool:
+        if not isinstance(other, SDKRoundtrip):
+            raise TypeError
+        return self.resp_index < other.resp_index
+
 
 MARKER_END = "-----------------------------------"
 MARKER_REQUEST_START = "---[ REQUEST ]"
@@ -126,6 +133,19 @@ class FileRef(NamedTuple):
     line_end: int
 
 
+_name_extract = re.compile(r"test_name=(\S+)")
+
+
+def parse_test_name(logs: str) -> str:
+    test_count = logs.count(MARKER_TEST)
+    assert test_count == 1, f"Only one test is supported, found {test_count}"
+    test_start = logs.index(MARKER_TEST)
+    full_line = logs[test_start:].split("\n", maxsplit=1)[0]
+    if match := _name_extract.search(full_line):
+        return match.group(1)
+    raise ValueError(f"Could not extract test name from {full_line}")
+
+
 def parse_http_requests(logs: str) -> list[SDKRoundtrip]:
     """
     Problem: With requests that are done in parallel.
@@ -138,8 +158,8 @@ def parse_http_requests(logs: str) -> list[SDKRoundtrip]:
     Method: (accepted)
     Can say that expected payload is either a list or a dict and if it ends with an identifier it is higher chance for a dict
     """
-    test_count = logs.count(MARKER_TEST)
-    assert test_count == 1, f"Only one test is supported, found {test_count}"
+    test_name = parse_test_name(logs)
+    logger.info(f"Finding http requests for test name: {test_name}")
     requests, responses = parse_raw_req_responses(logs)
     tf_step_starts = [i for i, line in enumerate(logs.splitlines()) if MARKER_START_STEP in line]
     used_responses: set[int] = set()
@@ -149,7 +169,7 @@ def parse_http_requests(logs: str) -> list[SDKRoundtrip]:
         roundtrip = match_request(used_responses, responses_list, ref, request, tf_step_starts)
         sdk_roundtrips.append(roundtrip)
         used_responses.add(roundtrip.resp_index)
-    return sdk_roundtrips
+    return sorted(sdk_roundtrips)
 
 
 def find_step_number(ref: FileRef, step_starts: list[int]) -> int:
@@ -172,7 +192,12 @@ def match_request(
             continue
         with suppress(ValidationError):
             step_number = find_step_number(ref, step_starts)
-            return SDKRoundtrip(request=request, response=response, resp_index=i, step_number=step_number)
+            return SDKRoundtrip(
+                request=request,
+                response=response,
+                resp_index=i,
+                step_number=step_number,
+            )
     remaining_responses = [resp for i, resp in enumerate(responses_list) if i not in used_responses]
     err_msg = f"Could not match request {ref} with any response\n\n{request}\n\n\nThere are #{len(remaining_responses)} responses left that doesn't match\n{'-'*80}\n{'\n'.join(r.text for r in remaining_responses)}"
     raise ValueError(err_msg)

atlas_init/cli_tf/debug_logs_test_data.py

@@ -12,8 +12,10 @@ logger = logging.getLogger(__name__)
 
 
 class StatusText(Entity):
+    response_index: int
     status: int
     text: str
+    duplicate_responses: int | None = None
 
     @property
     def id(self):
@@ -21,9 +23,9 @@ class StatusText(Entity):
 
 
 class RequestInfo(Entity):
-    version: str
-    method: str
     path: str
+    method: str
+    version: str
     text: str
     responses: list[StatusText] = Field(default_factory=list)
 
@@ -43,6 +45,10 @@ class StepRequests(Entity):
     diff_requests: list[RequestInfo] = Field(default_factory=list)
     request_responses: list[RequestInfo] = Field(default_factory=list)
 
+    @property
+    def all_requests(self):
+        return self.diff_requests + self.request_responses
+
     def existing_request(self, info: RequestInfo) -> RequestInfo | None:
         return next((r for r in self.request_responses if r.id == info.id), None)
 
@@ -55,8 +61,9 @@ class StepRequests(Entity):
         text: str,
         text_response: str,
         is_diff: bool,
+        response_index: int,
     ):
-        status_text = StatusText(status=status, text=text_response)
+        status_text = StatusText(status=status, text=text_response, response_index=response_index)
         info = RequestInfo(
             path=path,
             method=method,
@@ -111,6 +118,7 @@ class MockRequestData(Entity):
             normalized_text,
             normalized_response_text,
             is_diff,
+            rt.resp_index,
         )
 
     def update_variables(self, variables: dict[str, str]) -> None:
@@ -139,21 +147,32 @@ class MockRequestData(Entity):
         if changes:
             raise VariablesChangedError(changes)
 
+    def replace_text_variables(self):
+        for step in self.steps:
+            for request in step.all_requests:
+                request.text = normalize_text(request.text, self.variables)
+                for response in request.responses:
+                    response.text = normalize_text(response.text, self.variables)
+
     def prune_duplicate_responses(self):
         for step in self.steps:
             for request in step.request_responses:
                 pruned_responses = []
-                seen_response_ids = set()
+                seen_response_ids: dict[str, StatusText] = {}
                 before_len = len(request.responses)
                 for response in request.responses:
-                    if response.id in seen_response_ids:
+                    if existing_response := seen_response_ids.get(response.id):
+                        if existing_response.duplicate_responses is None:
+                            existing_response.duplicate_responses = 0
+                        existing_response.duplicate_responses += 1
                         continue
-                    seen_response_ids.add(response.id)
+                    seen_response_ids[response.id] = response
                     pruned_responses.append(response)
                 request.responses = pruned_responses
                 after_len = len(request.responses)
-                if before_len != after_len:
-                    logger.info(f"Pruned {before_len - after_len} duplicate responses from {request.id}")
+                duplicate_responses = before_len - after_len
+                if duplicate_responses > 0:
+                    logger.info(f"Pruned {duplicate_responses} duplicate responses from {request.id}")
 
 
 class ApiSpecPath(Entity):
@@ -204,7 +223,7 @@ def normalize_text(text: str, variables: dict[str, str]) -> str:
 
 
 def default_is_diff(rt: SDKRoundtrip) -> bool:
-    return rt.request.method not in {"DELETE", "GET"}
+    return rt.request.method not in {"GET"}
 
 
 class VariableChange(NamedTuple):
@@ -225,6 +244,8 @@ def create_mock_data(
     api_spec_paths: dict[str, list[ApiSpecPath]],
     is_diff: Callable[[SDKRoundtrip], bool] | None = None,
     modifiers: list[RTModifier] | None = None,
+    *,
+    prune_duplicates: bool = True,
 ) -> MockRequestData:
     steps = max(rt.step_number for rt in roundtrips)
     mock_data = MockRequestData(step_count=steps)
@@ -246,8 +267,10 @@ def create_mock_data(
         for modifier in modifiers:
             if modifier.match(rt, normalized_path):
                 modifier.modification(rt)
-        normalized_text = normalize_text(rt.request.text, rt_variables)
-        normalized_response_text = normalize_text(rt.response.text, rt_variables)
+        normalized_text = normalize_text(rt.request.text, mock_data.variables)
+        normalized_response_text = normalize_text(rt.response.text, mock_data.variables)
         mock_data.add_roundtrip(rt, normalized_path, normalized_text, normalized_response_text, is_diff(rt))
-    # requests.prune_duplicate_responses() better to keep duplicates to stay KISS
+    mock_data.replace_text_variables()
+    if prune_duplicates:
+        mock_data.prune_duplicate_responses()  # better to keep duplicates to stay KISS
     return mock_data

atlas_init/cli_tf/mock_tf_log.py

@@ -0,0 +1,85 @@
+import json
+import logging
+from pathlib import Path
+from typing import Self
+
+import typer
+from model_lib import Entity, dump
+from pydantic import Field, model_validator
+from zero_3rdparty import file_utils
+
+from atlas_init.cli_args import option_sdk_repo_path
+from atlas_init.cli_tf.debug_logs import (
+    SDKRoundtrip,
+    parse_http_requests,
+    parse_test_name,
+)
+from atlas_init.cli_tf.debug_logs_test_data import create_mock_data, default_is_diff
+from atlas_init.repos.go_sdk import parse_api_spec_paths
+
+logger = logging.getLogger(__name__)
+
+
+class MockTFLog(Entity):
+    log_path: Path
+    output_dir: Path
+    sdk_path: Path
+    diff_skip_suffixes: list[str] = Field(default_factory=list)
+    keep_duplicates: bool = False
+
+    @model_validator(mode="after")
+    def ensure_paths_exist(self) -> Self:
+        if not self.log_path.exists():
+            raise ValueError(f"log_path: {self.log_path} doesn't exist")
+        if not self.sdk_path.exists():
+            raise ValueError(f"sdk_path: {self.sdk_path} doesn't exist")
+        if not self.output_dir.exists():
+            raise ValueError(f"output_dir: {self.output_dir} doesn't exist")
+        assert self.output_dir.name == "testdata", "output_path should be a directory named testdata"
+        return self
+
+    def differ(self, rt: SDKRoundtrip) -> bool:
+        return default_is_diff(rt) and not any(rt.request.path.endswith(suffix) for suffix in self.diff_skip_suffixes)
+
+
+def mock_tf_log(req: MockTFLog) -> None:
+    log_file_text = req.log_path.read_text()
+    test_name = parse_test_name(log_file_text)
+    roundtrips = parse_http_requests(log_file_text)
+    api_spec_paths = parse_api_spec_paths(req.sdk_path)
+    data = create_mock_data(
+        roundtrips,
+        api_spec_paths,
+        is_diff=req.differ,
+        prune_duplicates=not req.keep_duplicates,
+    )
+    # avoid anchors
+    data_yaml = dump(json.loads(dump(data, "json")), "yaml")
+    output_path = req.output_dir / f"{test_name}.yaml"
+    logger.info(f"Writing to {output_path}")
+    file_utils.ensure_parents_write_text(output_path, data_yaml)
+
+
+def mock_tf_log_cmd(
+    log_path: str = typer.Argument(..., help="the path to the log file generated with TF_LOG_PATH"),
+    sdk_repo_path_str: str = option_sdk_repo_path,
+    output_testdir: str = typer.Option(
+        "",
+        "-o",
+        "--output-testdir",
+        help="the path to the output test directory, for example: internal/service/advancedclustertpf/testdata/",
+    ),
+    diff_skip_suffixes: list[str] = typer.Option(..., "-s", "--skip-suffixes", default_factory=list),
+    keep_duplicates: bool = typer.Option(False, "-keep", "--keep-duplicates", help="keep duplicate requests"),
+):
+    cwd = Path.cwd()
+    default_sdk_path = cwd.parent / "atlas-sdk-go"
+    default_testdir = cwd.parent / "testdata"
+    event_in = MockTFLog(
+        log_path=Path(log_path),
+        output_dir=Path(output_testdir) if output_testdir else default_testdir,
+        sdk_path=Path(sdk_repo_path_str) if sdk_repo_path_str else default_sdk_path,
+        diff_skip_suffixes=diff_skip_suffixes,
+        keep_duplicates=keep_duplicates,
+    )
+    mock_tf_log(event_in)

atlas_init/repos/go_sdk.py

@@ -1,5 +1,11 @@
+from collections import defaultdict
 from pathlib import Path
 
+from model_lib import parse_model
+
+from atlas_init.cli_tf.debug_logs_test_data import ApiSpecPath
+from atlas_init.cli_tf.schema_v2_api_parsing import OpenapiSchema
+
 
 def go_sdk_breaking_changes(repo_path: Path, go_sdk_rel_path: str = "../atlas-sdk-go") -> Path:
     rel_path = "tools/releaser/breaking_changes"
@@ -7,3 +13,13 @@ def go_sdk_breaking_changes(repo_path: Path, go_sdk_rel_path: str = "../atlas-sd
     breaking_changes_dir = breaking_changes_dir.absolute()
     assert breaking_changes_dir.exists(), f"not found breaking_changes={breaking_changes_dir}"
     return breaking_changes_dir
+
+
+def parse_api_spec_paths(sdk_repo_path: Path) -> dict[str, list[ApiSpecPath]]:
+    api_spec_path = sdk_repo_path / "openapi/atlas-api-transformed.yaml"
+    model = parse_model(api_spec_path, t=OpenapiSchema)
+    paths: dict[str, list[ApiSpecPath]] = defaultdict(list)
+    for path, path_dict in model.paths.items():
+        for method in path_dict:
+            paths[method.upper()].append(ApiSpecPath(path=path))
+    return paths

{atlas_init-0.1.8.dist-info → atlas_init-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: atlas-init
-Version: 0.1.8
+Version: 0.2.0
 Project-URL: Documentation, https://github.com/EspenAlbert/atlas-init#readme
 Project-URL: Issues, https://github.com/EspenAlbert/atlas-init/issues
 Project-URL: Source, https://github.com/EspenAlbert/atlas-init

{atlas_init-0.1.8.dist-info → atlas_init-0.2.0.dist-info}/RECORD

@@ -1,8 +1,8 @@
-atlas_init/__init__.py,sha256=iuwrP7v_WWz_sVz-k0_a5k5HduJTimtVRY4WP2RNiz4,372
+atlas_init/__init__.py,sha256=bW9zb8obruloS6_54VzQHbSGtnnhhbEBuu5fipDvPcc,372
 atlas_init/__main__.py,sha256=dY1dWWvwxRZMmnOFla6RSfti-hMeLeKdoXP7SVYqMUc,52
 atlas_init/atlas_init.yaml,sha256=GMyJVhKKRc7WzEu7fafmWgeTsDaExTLv7QvXOmE_Brg,1907
 atlas_init/cli.py,sha256=IiOEC_Jry6vrSDH3_OvsU50F-_3iVIS4tV6-R7659fY,9642
-atlas_init/cli_args.py,sha256=6ucjRGLGde1XcZIfUCGrWbmQLJB9bWZaXxyAsYXMSSw,427
+atlas_init/cli_args.py,sha256=tiwUYAE0JBSl9lHV6VJ41vFCU90ChBZ4mKvi-YoF_HY,541
 atlas_init/humps.py,sha256=l0ZXXuI34wwd9TskXhCjULfGbUyK-qNmiyC6_2ow6kU,7339
 atlas_init/terraform.yaml,sha256=qPrnbzBEP-JAQVkYadHsggRnDmshrOJyiv0ckyZCxwY,2734
 atlas_init/typer_app.py,sha256=irgBK7WSewg7on25KKHY9FNUDvmXgavmc0Z39yhYNjI,2507
@@ -21,14 +21,15 @@ atlas_init/cli_helper/tf_runner.py,sha256=OYdC-Y6i-xRh8_LCudKdtP7CEYEO9e67nVhhol
 atlas_init/cli_root/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/cli_root/trigger.py,sha256=oEgqb_l25tyYgUaFHEuChcOCJA7k3mnRa4D-Myz-Igs,5789
 atlas_init/cli_tf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-atlas_init/cli_tf/app.py,sha256=2lrlbuXPSfgL_k4BeZ4jiWNkRuSemCp5xle1yxCiU4I,11255
+atlas_init/cli_tf/app.py,sha256=-50lyNwG5eh8nVCN6mCcIB_b4xd5aJjhEoK-XVHD8XU,11359
 atlas_init/cli_tf/changelog.py,sha256=biWYKf1pZvXZ-jEgcZ5q9sY7nTGrL2PuI0h9mCILf_g,3181
-atlas_init/cli_tf/debug_logs.py,sha256=J8MLryNNkRivo2-Z3fnqf5pjPit9xndbsqNSbATTtUg,8047
-atlas_init/cli_tf/debug_logs_test_data.py,sha256=AOysldCVSBtqW8ZhRnxcOPctng9OodQy0zWWcYbmpoA,8577
+atlas_init/cli_tf/debug_logs.py,sha256=lnB5BpcEooVzGd2RLxbwAVQs0ZYXzRKy5sHa0hftHI8,8799
+atlas_init/cli_tf/debug_logs_test_data.py,sha256=CP1FbRvaFXOOr5m0u-Y6YAvWpqn6WzH-XoPYBJZN0bE,9594
 atlas_init/cli_tf/github_logs.py,sha256=VD7qhlXNuG21eTuJ5VI7rsflp5WHSodfngkRVgQlumw,8114
 atlas_init/cli_tf/go_test_run.py,sha256=ZoQSvIasmWauFxZJrWL0ObFX-P0k-D3c_ep3OnPY4zs,5842
 atlas_init/cli_tf/go_test_run_format.py,sha256=OUd6QPHDeTzbwVuh6MhP-xXgjOOGP9W_sCLJ8KylBTs,1201
 atlas_init/cli_tf/go_test_summary.py,sha256=agr4SITgxchjgOzRpScoTUk-iG38QDLkpnsMtTW9GTY,5382
+atlas_init/cli_tf/mock_tf_log.py,sha256=tefWI5mS5lwOWCUidI_p7LeTaoyYCQl951LFgLYGEy8,3157
 atlas_init/cli_tf/schema.py,sha256=q0RUHb-6ORbQ41u2bZXaWWQLB-xE8Q_O7U1AFq2Gfng,12398
 atlas_init/cli_tf/schema_go_parser.py,sha256=PiRfFFVnkhltxcGFfOCgH53wwzIEynw2BXmSfaINLL8,8294
 atlas_init/cli_tf/schema_inspection.py,sha256=ujLvGfg3baByND4nRD0drZoI45STxo3VfYvim-PfVOc,1764
@@ -49,7 +50,7 @@ atlas_init/cloud/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 atlas_init/cloud/aws.py,sha256=97kkURWHFAKDIw4704aFmyoeAfQKL11IXMyaQbZUt80,2473
 atlas_init/repos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/repos/cfn.py,sha256=rjyVVxRhWL65tdAbEHT72UReK2h99Bj6RA4O2pBO-bc,2466
-atlas_init/repos/go_sdk.py,sha256=6_ENYMP6yQufo1dXJbLJFDxCuq-8oeTYiGAPC61Uauc,424
+atlas_init/repos/go_sdk.py,sha256=nh3lw9iw4lDGdHnhC8KK0PZTDMUGKvCHTMTuEtIUKNg,1058
 atlas_init/repos/path.py,sha256=wrT8e01OBoAHj8iMrxqutgqWu-BHPe9-bEWtcZRu238,4187
 atlas_init/settings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/settings/config.py,sha256=HIytZom8RRvpLGy6u8CpZ83tmFXI6v1tO3iSiuo08kc,6259
@@ -85,7 +86,7 @@ atlas_init/tf/modules/vpc_peering/vpc_peering.tf,sha256=hJ3KJdGbLpOQednUpVuiJ0Cq
 atlas_init/tf/modules/vpc_privatelink/atlas-privatelink.tf,sha256=FloaaX1MNDvoMZxBnEopeLKyfIlq6kaX2dmx8WWlXNU,1298
 atlas_init/tf/modules/vpc_privatelink/variables.tf,sha256=gktHCDYD4rz6CEpLg5aiXcFbugw4L5S2Fqc52QYdJyc,255
 atlas_init/tf/modules/vpc_privatelink/versions.tf,sha256=G0u5V_Hvvrkux_tqfOY05pA-GzSp_qILpfx1dZaTGDc,237
-atlas_init-0.1.8.dist-info/METADATA,sha256=rY36JD0IN5OQGE9wpSx8uIdtFfkJOBZu6mg3DJzQF5A,6167
-atlas_init-0.1.8.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-atlas_init-0.1.8.dist-info/entry_points.txt,sha256=6Ycq_NZ7cym1BCA9YiVhYylJzy6woREjt7QSCSLm-1o,54
-atlas_init-0.1.8.dist-info/RECORD,,
+atlas_init-0.2.0.dist-info/METADATA,sha256=DY-E8A0r-DLm9H9hX4Nc60Wjb1fcLs0QWLU-vdJ8LjA,6167
+atlas_init-0.2.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+atlas_init-0.2.0.dist-info/entry_points.txt,sha256=6Ycq_NZ7cym1BCA9YiVhYylJzy6woREjt7QSCSLm-1o,54
+atlas_init-0.2.0.dist-info/RECORD,,