atlas-init 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. atlas_init/__init__.py +1 -1
  2. atlas_init/cli_args.py +19 -1
  3. atlas_init/cli_tf/ci_tests.py +116 -24
  4. atlas_init/cli_tf/go_test_run.py +14 -2
  5. atlas_init/cli_tf/go_test_summary.py +334 -82
  6. atlas_init/cli_tf/go_test_tf_error.py +20 -12
  7. atlas_init/cli_tf/hcl/modifier2.py +120 -0
  8. atlas_init/cli_tf/openapi.py +10 -6
  9. atlas_init/html_out/__init__.py +0 -0
  10. atlas_init/html_out/md_export.py +143 -0
  11. atlas_init/sdk_ext/__init__.py +0 -0
  12. atlas_init/sdk_ext/go.py +102 -0
  13. atlas_init/sdk_ext/typer_app.py +18 -0
  14. atlas_init/settings/env_vars.py +13 -1
  15. atlas_init/settings/env_vars_generated.py +2 -0
  16. atlas_init/tf/.terraform.lock.hcl +33 -33
  17. atlas_init/tf/modules/aws_s3/provider.tf +1 -1
  18. atlas_init/tf/modules/aws_vpc/provider.tf +1 -1
  19. atlas_init/tf/modules/cloud_provider/provider.tf +1 -1
  20. atlas_init/tf/modules/cluster/provider.tf +1 -1
  21. atlas_init/tf/modules/encryption_at_rest/provider.tf +1 -1
  22. atlas_init/tf/modules/federated_vars/federated_vars.tf +1 -2
  23. atlas_init/tf/modules/federated_vars/provider.tf +1 -1
  24. atlas_init/tf/modules/project_extra/provider.tf +1 -1
  25. atlas_init/tf/modules/stream_instance/provider.tf +1 -1
  26. atlas_init/tf/modules/vpc_peering/provider.tf +1 -1
  27. atlas_init/tf/modules/vpc_privatelink/versions.tf +1 -1
  28. atlas_init/tf/providers.tf +1 -1
  29. atlas_init/tf_ext/__init__.py +0 -0
  30. atlas_init/tf_ext/__main__.py +3 -0
  31. atlas_init/tf_ext/api_call.py +325 -0
  32. atlas_init/tf_ext/args.py +17 -0
  33. atlas_init/tf_ext/constants.py +3 -0
  34. atlas_init/tf_ext/models.py +106 -0
  35. atlas_init/tf_ext/paths.py +126 -0
  36. atlas_init/tf_ext/settings.py +39 -0
  37. atlas_init/tf_ext/tf_dep.py +324 -0
  38. atlas_init/tf_ext/tf_modules.py +394 -0
  39. atlas_init/tf_ext/tf_vars.py +173 -0
  40. atlas_init/tf_ext/typer_app.py +24 -0
  41. {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/METADATA +3 -2
  42. {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/RECORD +45 -28
  43. atlas_init-0.7.0.dist-info/entry_points.txt +5 -0
  44. atlas_init-0.6.0.dist-info/entry_points.txt +0 -2
  45. {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/WHEEL +0 -0
  46. {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/licenses/LICENSE +0 -0
@@ -2,7 +2,7 @@ terraform {
2
2
  required_providers {
3
3
  mongodbatlas = {
4
4
  source = "mongodb/mongodbatlas"
5
- version = "1.33"
5
+ version = ">=1.33"
6
6
  }
7
7
  }
8
8
  }
@@ -2,7 +2,7 @@ terraform {
2
2
  required_providers {
3
3
  mongodbatlas = {
4
4
  source = "mongodb/mongodbatlas"
5
- version = "1.33"
5
+ version = ">=1.33"
6
6
  }
7
7
  }
8
8
  }
@@ -2,7 +2,7 @@ terraform {
2
2
  required_providers {
3
3
  mongodbatlas = {
4
4
  source = "mongodb/mongodbatlas"
5
- version = "1.33"
5
+ version = ">=1.33"
6
6
  }
7
7
  }
8
8
 
@@ -6,7 +6,7 @@ terraform {
6
6
  }
7
7
  mongodbatlas = {
8
8
  source = "mongodb/mongodbatlas"
9
- version = "1.33"
9
+ version = ">=1.33"
10
10
  }
11
11
  }
12
12
 
@@ -2,7 +2,7 @@ terraform {
2
2
  required_providers {
3
3
  mongodbatlas = {
4
4
  source = "mongodb/mongodbatlas"
5
- version = "1.33"
5
+ version = ">=1.33"
6
6
  }
7
7
  aws = {
8
8
  source = "hashicorp/aws"
File without changes
@@ -0,0 +1,3 @@
1
+ from atlas_init.tf_ext.typer_app import typer_main
2
+
3
+ typer_main()
@@ -0,0 +1,325 @@
1
+ from collections import defaultdict
2
+ from concurrent.futures import Future, as_completed
3
+ from functools import lru_cache
4
+ import json
5
+ import logging
6
+ import os
7
+ from pathlib import Path
8
+
9
+ from ask_shell import new_task, print_to_live, run_pool
10
+ from model_lib import dump, parse_model
11
+ from pydantic import BaseModel, Field, model_validator
12
+ import requests
13
+ from rich.markdown import Markdown
14
+ import typer
15
+ from requests.auth import HTTPDigestAuth
16
+ from zero_3rdparty.file_utils import ensure_parents_write_text
17
+ from zero_3rdparty.str_utils import ensure_prefix, ensure_suffix, instance_repr
18
+
19
+ from atlas_init.cli_tf.mock_tf_log import resolve_admin_api_path
20
+ from atlas_init.cli_tf.openapi import OpenapiSchema
21
+ from atlas_init.settings.env_vars import init_settings
22
+ from atlas_init.settings.env_vars_generated import AtlasSettingsWithProject
23
+ from atlas_init.settings.env_vars_modules import (
24
+ TFModuleCluster,
25
+ TFModuleFederated_Vars,
26
+ TFModuleProject_Extra,
27
+ TFModuleStream_Instance,
28
+ )
29
+ from atlas_init.settings.path import load_dotenv
30
+ from atlas_init.tf_ext.settings import TfDepSettings
31
+
32
+ logger = logging.getLogger(__name__)
33
+
34
+ ALLOWED_MISSING_VARS: set[str] = {
35
+ "alertConfigId",
36
+ "alertId",
37
+ "clientId",
38
+ "cloudProvider",
39
+ "invoiceId",
40
+ # "name",
41
+ "pipelineName",
42
+ "processId",
43
+ "username",
44
+ }
45
+ ALLOWED_ERROR_CODES: set[str] = {
46
+ "CANNOT_USE_CLUSTER_IN_SERVERLESS_INSTANCE_API",
47
+ "VALIDATION_ERROR",
48
+ "UNEXPECTED_ERROR",
49
+ "CANNOT_USE_NON_FLEX_CLUSTER_IN_FLEX_API",
50
+ "CHECKPOINTS_ONLY_ON_CONTINOUS_BACKUP",
51
+ "INCORRECT_BACKUP_API_ENDPOINT",
52
+ }
53
+
54
+
55
+ # export ATLAS_INIT_TEST_SUITES=clusterm10,s3,federated,project,stream_connection
56
+ def resolve_path_variables() -> dict[str, str]:
57
+ settings = init_settings()
58
+ env_vars_full = load_dotenv(settings.env_vars_vs_code)
59
+ atlas_settings = AtlasSettingsWithProject(**env_vars_full)
60
+ cluster_settings = TFModuleCluster(**env_vars_full)
61
+ project_settings = TFModuleProject_Extra(**env_vars_full)
62
+ stream_settings = TFModuleStream_Instance(**env_vars_full)
63
+ federated_settings = TFModuleFederated_Vars(**env_vars_full)
64
+ return {
65
+ "orgId": atlas_settings.MONGODB_ATLAS_ORG_ID,
66
+ "cloudProvider": "AWS",
67
+ "federationSettingsId": federated_settings.MONGODB_ATLAS_FEDERATION_SETTINGS_ID,
68
+ "clusterName": cluster_settings.MONGODB_ATLAS_CLUSTER_NAME,
69
+ "name": cluster_settings.MONGODB_ATLAS_CLUSTER_NAME,
70
+ "groupId": atlas_settings.MONGODB_ATLAS_PROJECT_ID,
71
+ "teamId": project_settings.MONGODB_ATLAS_TEAM_ID,
72
+ "tenantName": stream_settings.MONGODB_ATLAS_STREAM_INSTANCE_NAME,
73
+ "apiUserId": atlas_settings.MONGODB_ATLAS_PROJECT_OWNER_ID,
74
+ "username": atlas_settings.MONGODB_ATLAS_USER_EMAIL,
75
+ }
76
+
77
+
78
+ class ApiCall(BaseModel):
79
+ operation_id: str
80
+ path: str
81
+ accept_header: str = "application/vnd.atlas.2023-01-01+json"
82
+ query_args: dict[str, str] = Field(default_factory=dict)
83
+
84
+ def __str__(self):
85
+ return instance_repr(self, ["operation_id", "path"])
86
+
87
+ def path_with_variables(self, path_variables: dict[str, str]):
88
+ return self.path.format(**path_variables)
89
+
90
+ @model_validator(mode="after")
91
+ def check_path_variables(self):
92
+ self.accept_header = ensure_prefix(self.accept_header, "application/vnd.atlas.")
93
+ self.accept_header = ensure_suffix(self.accept_header, "+json")
94
+ return self
95
+
96
+
97
+ class UnresolvedPathsError(Exception):
98
+ def __init__(self, missing_var_paths: dict[str, list[str]]) -> None:
99
+ self.missing_var_paths = missing_var_paths
100
+ missing_vars_formatted = "\n".join(f"{var}: {paths}" for var, paths in missing_var_paths.items())
101
+ super().__init__(f"Failed to resolve path variables:\nMissing vars: {missing_vars_formatted}")
102
+
103
+
104
+ class ApiCalls(BaseModel):
105
+ calls: list[ApiCall] = Field(default_factory=list)
106
+ ignored_calls: list[ApiCall] = Field(default_factory=list)
107
+ path_variables: dict[str, str] = Field(default_factory=resolve_path_variables)
108
+ skip_validation: bool = False
109
+
110
+ @model_validator(mode="after")
111
+ def check_path_variables(self):
112
+ if self.skip_validation:
113
+ return self
114
+ missing_vars_paths: dict[str, list[str]] = defaultdict(list)
115
+ ok_calls = []
116
+ for call in self.calls:
117
+ try:
118
+ call.path_with_variables(self.path_variables)
119
+ ok_calls.append(call)
120
+ except KeyError as e:
121
+ missing_vars_paths[str(e).strip("'")].append(f"{call.operation_id} {call.path}")
122
+ self.ignored_calls.append(call)
123
+ continue
124
+ for allowed_missing in sorted(ALLOWED_MISSING_VARS):
125
+ if allowed_missing in missing_vars_paths:
126
+ logger.info(f"Allowed missing variable {allowed_missing}: {missing_vars_paths[allowed_missing]}")
127
+ del missing_vars_paths[allowed_missing]
128
+ if missing_vars_paths:
129
+ raise UnresolvedPathsError(missing_var_paths=missing_vars_paths)
130
+ self.calls = ok_calls
131
+ return self
132
+
133
+ def dump_to_dict(self) -> dict:
134
+ return {
135
+ "calls": [call.model_dump(exclude_defaults=True, exclude_unset=True) for call in self.calls],
136
+ }
137
+
138
+
139
+ @lru_cache
140
+ def _public_private_key() -> tuple[str, str]:
141
+ public_key = os.environ.get("MONGODB_ATLAS_PUBLIC_KEY")
142
+ private_key = os.environ.get("MONGODB_ATLAS_PRIVATE_KEY")
143
+ if not public_key or not private_key:
144
+ raise ValueError("MONGODB_ATLAS_PUBLIC_KEY and MONGODB_ATLAS_PRIVATE_KEY must be set in environment variables.")
145
+ return public_key, private_key
146
+
147
+
148
+ class APICallError(Exception):
149
+ def __init__(self, api_call: ApiCall, json_response: dict, error: requests.exceptions.HTTPError):
150
+ self.api_call = api_call
151
+ self.json_response = json_response
152
+ super().__init__(f"Failed to make API call {api_call}:\njson={json_response}\n{error}")
153
+
154
+ @property
155
+ def error_code(self) -> str:
156
+ return self.json_response.get("errorCode", "")
157
+
158
+
159
+ def call_api(api_call: ApiCall, path_variables: dict[str, str]) -> dict:
160
+ resolved_path = api_call.path_with_variables(path_variables)
161
+ response = requests.get(
162
+ f"https://cloud-dev.mongodb.com/{resolved_path.lstrip('/')}",
163
+ params=api_call.query_args,
164
+ headers={"Accept": api_call.accept_header, "Content-Type": "application/json"},
165
+ auth=HTTPDigestAuth(*_public_private_key()),
166
+ timeout=30,
167
+ )
168
+ try:
169
+ response_json = response.json()
170
+ except requests.exceptions.JSONDecodeError as e:
171
+ logger.error(f"Failed to parse_json {api_call}: {e}")
172
+ response_json = {}
173
+ try:
174
+ response.raise_for_status()
175
+ except requests.exceptions.HTTPError as e:
176
+ raise APICallError(api_call, response_json, e) from e
177
+ return response_json
178
+
179
+
180
+ class NoSelfLinkError(Exception):
181
+ def __init__(self, json_response: dict) -> None:
182
+ self.json_response = json_response
183
+ super().__init__("No self link found in response")
184
+
185
+
186
+ def parse_href_response(json_response: dict) -> str:
187
+ for ref in json_response.get("links", []):
188
+ if ref.get("rel") == "self":
189
+ return ref.get("href")
190
+ raise NoSelfLinkError(json_response)
191
+
192
+
193
+ def api_config(
194
+ config_path_str: str = typer.Option("", "-p", "--path", help="Path to the API config file"),
195
+ query_args_str: str = typer.Option(
196
+ '{"pageNum": "0", "itemsPerPage": "0"}', "-q", "--query-args", help="Query arguments for the API call"
197
+ ),
198
+ verbose: bool = typer.Option(False, "-v", "--verbose", help="Verbose output"),
199
+ ):
200
+ query_args: dict[str, str] = json.loads(query_args_str)
201
+ if config_path_str == "":
202
+ with new_task("Find API Calls that use pagination"):
203
+ config_path = dump_config_path(query_args)
204
+ else:
205
+ config_path = Path(config_path_str)
206
+ assert config_path.exists(), f"Config file {config_path} does not exist."
207
+ model = parse_model(config_path, t=ApiCalls)
208
+ total_calls = len(model.calls)
209
+ assert _public_private_key(), "Public and private keys must be set in environment variables."
210
+ path_variables = model.path_variables
211
+ op_id_path_self_qstring: dict[tuple[str, str], str] = {}
212
+ with run_pool(
213
+ task_name="make API calls", max_concurrent_submits=10, threads_used_per_submit=1, total=total_calls
214
+ ) as pool:
215
+ futures: dict[Future, ApiCall] = {
216
+ pool.submit(call_api, api_call, path_variables): api_call for api_call in model.calls
217
+ }
218
+ for future in as_completed(futures):
219
+ api_call = futures[future]
220
+ try:
221
+ result = future.result()
222
+ except APICallError as e:
223
+ if e.error_code in ALLOWED_ERROR_CODES:
224
+ logger.info(f"Allowed error code {e.error_code} in response for {api_call}")
225
+ model.ignored_calls.append(api_call)
226
+ continue
227
+ raise
228
+ except Exception as e:
229
+ logger.error(e)
230
+ continue
231
+ try:
232
+ href = parse_href_response(result)
233
+ op_id_path_self_qstring[(api_call.operation_id, api_call.path)] = href.split("?")[-1]
234
+ except NoSelfLinkError as e:
235
+ logger.error(f"{api_call} did not have a self link in the response:\n{e.json_response}")
236
+ continue
237
+ logger.info(f"API call {api_call} completed successfully with self ref:\n{href}")
238
+ if verbose:
239
+ logger.info(f"Response for {api_call.query_args} was:\n{dump(result, 'pretty_json')}")
240
+ query_args_str = "&".join(f"{key}={value}" for key, value in query_args.items())
241
+ md_report: list[str] = [
242
+ f"# Pagination Report for query_args='{query_args_str}'",
243
+ "",
244
+ "## Checked endpoints",
245
+ "",
246
+ "Operation ID | Path | SelfQueryString",
247
+ "--- | --- | ---",
248
+ *[
249
+ f"{operation_id} | {path} | {self_query_string}"
250
+ for (operation_id, path), self_query_string in op_id_path_self_qstring.items()
251
+ ],
252
+ "",
253
+ "## Ignored endpoints (not checked)",
254
+ "",
255
+ "Operation ID | Path",
256
+ "--- | ---",
257
+ *[f"{call.operation_id} | {call.path}" for call in model.ignored_calls],
258
+ ]
259
+ md_content = "\n".join(md_report)
260
+ md = Markdown(md_content)
261
+ print_to_live(md)
262
+ output_path = TfDepSettings.from_env().pagination_output_path(query_args_str)
263
+ ensure_parents_write_text(output_path, md_content)
264
+ logger.info(f"Pagination report saved to {output_path}")
265
+ return md
266
+
267
+
268
+ def api(
269
+ path: str = typer.Option("-p", "--path", help="Path to the API endpoint"),
270
+ query_string: str = typer.Option("", "-q", "--query-string", help="Query string for the API call"),
271
+ ):
272
+ assert path, "Path must be provided."
273
+ accept_header = "application/vnd.atlas.2023-01-01+json"
274
+ url = f"https://cloud-dev.mongodb.com/{path.lstrip('/')}?{query_string}"
275
+ logger.info(f"Calling {url}")
276
+ try:
277
+ r = requests.get(
278
+ url,
279
+ headers={"Accept": accept_header, "Content-Type": "application/json"},
280
+ auth=HTTPDigestAuth(*_public_private_key()),
281
+ timeout=30,
282
+ )
283
+ print(r.text)
284
+ r.raise_for_status()
285
+ except requests.exceptions.HTTPError as e:
286
+ print(e)
287
+ print(e.response)
288
+
289
+
290
+ def dump_config_path(query_args: dict[str, str]) -> Path:
291
+ settings = TfDepSettings.from_env()
292
+ latest_api_spec = resolve_admin_api_path()
293
+ model = parse_model(latest_api_spec, t=OpenapiSchema)
294
+ paginated_paths: list[ApiCall] = []
295
+ path_versions = list(model.path_method_api_versions())
296
+
297
+ for (path, method, code), versions in path_versions:
298
+ if method != "get" or code != "200":
299
+ continue
300
+ assert len(versions) == 1, f"{path} {method} {code} has multiple versions: {versions}"
301
+ get_method = model.get_method(path)
302
+ if not get_method:
303
+ continue
304
+ parameters = get_method.get("parameters", [])
305
+ for param in parameters:
306
+ if param_ref := param.get("$ref"):
307
+ if param_ref.endswith("itemsPerPage"):
308
+ version = versions[0].strftime("%Y-%m-%d")
309
+ paginated_paths.append(
310
+ ApiCall(
311
+ path=path,
312
+ query_args=query_args,
313
+ accept_header=f"application/vnd.atlas.{version}+json",
314
+ operation_id=get_method["operationId"],
315
+ )
316
+ )
317
+ config_path = settings.api_calls_path
318
+ calls = ApiCalls(
319
+ calls=paginated_paths,
320
+ skip_validation=True,
321
+ )
322
+ calls_yaml = dump(calls.dump_to_dict(), "yaml")
323
+ logger.info(f"Dumped {len(paginated_paths)} API calls to {config_path}")
324
+ ensure_parents_write_text(config_path, calls_yaml)
325
+ return config_path
@@ -0,0 +1,17 @@
1
+ import typer
2
+
3
+
4
+ def default_skippped_directories() -> list[str]:
5
+ return [
6
+ "prometheus-and-teams", # Provider registry.terraform.io/hashicorp/template v2.2.0 does not have a package available for your current platform, darwin_arm64.
7
+ ]
8
+
9
+
10
+ REPO_PATH_ARG = typer.Argument(help, help="Path to the mongodbatlas-terraform-provider repository")
11
+ SKIP_EXAMPLES_DIRS_OPTION = typer.Option(
12
+ ...,
13
+ "--skip-examples",
14
+ help="Skip example directories with these names",
15
+ default_factory=default_skippped_directories,
16
+ show_default=True,
17
+ )
@@ -0,0 +1,3 @@
1
+ DEFAULT_EXTERNAL_SUBSTRINGS = ["aws", "azure", "google", "gcp"]
2
+ DEFAULT_INTERNAL_SUBSTRINGS = ["atlas", "mongo", "aws_region", "gcp_region", "azure_region", "cidr"]
3
+ ATLAS_PROVIDER_NAME = "mongodbatlas"
@@ -0,0 +1,106 @@
1
+ from __future__ import annotations
2
+ from typing import Self
3
+
4
+ from model_lib import Entity
5
+ from pydantic import Field, RootModel, model_validator
6
+
7
+ from atlas_init.tf_ext.tf_dep import AtlasGraph
8
+
9
+ _emojii_list = [
10
+ "1️⃣",
11
+ "2️⃣",
12
+ "3️⃣",
13
+ "4️⃣",
14
+ "5️⃣",
15
+ "6️⃣",
16
+ "7️⃣",
17
+ "8️⃣",
18
+ "9️⃣",
19
+ "🔟",
20
+ "1️⃣1️⃣",
21
+ "1️⃣2️⃣",
22
+ ]
23
+ _emoji_counter = 0
24
+
25
+
26
+ def choose_next_emoji() -> str:
27
+ global _emoji_counter
28
+ emoji = _emojii_list[_emoji_counter]
29
+ _emoji_counter += 1
30
+ return emoji
31
+
32
+
33
+ class ModuleState(Entity):
34
+ resource_types: set[str] = Field(default_factory=set, description="Set of resource types in the module.")
35
+
36
+
37
+ def default_allowed_multi_parents() -> set[str]:
38
+ return {
39
+ "mongodbatlas_project",
40
+ }
41
+
42
+
43
+ class ModuleConfig(Entity):
44
+ name: str = Field(..., description="Name of the module.")
45
+ root_resource_types: list[str] = Field(..., description="List of root resource types for the module.")
46
+ force_include_children: list[str] = Field(
47
+ default_factory=list, description="List of resource types that should always be included as children."
48
+ )
49
+ emojii: str = Field(init=False, default_factory=choose_next_emoji)
50
+ allowed_multi_parents: set[str] = Field(
51
+ default_factory=default_allowed_multi_parents,
52
+ description="Set of parents that a child resource type can have in addition to the root_resource_type.",
53
+ )
54
+ allow_external_dependencies: bool = Field(
55
+ default=False, description="Whether to allow external dependencies for the module."
56
+ )
57
+ extra_nested_resource_types: list[str] = Field(
58
+ default_factory=list,
59
+ description="List of additional nested resource types that should be included in the module.",
60
+ )
61
+
62
+ state: ModuleState = Field(default_factory=ModuleState, description="Internal state of the module.")
63
+
64
+ @model_validator(mode="after")
65
+ def update_state(self) -> Self:
66
+ self.state.resource_types.update(self.root_resource_types)
67
+ return self
68
+
69
+ @property
70
+ def tree_label(self) -> str:
71
+ return f"{self.emojii} {self.name}"
72
+
73
+ def include_child(self, child: str, atlas_graph: AtlasGraph) -> bool:
74
+ if child in atlas_graph.deprecated_resource_types:
75
+ return False
76
+ if child in self.force_include_children or child in self.extra_nested_resource_types:
77
+ self.state.resource_types.add(child)
78
+ return True
79
+ has_external_dependencies = len(atlas_graph.external_parents.get(child, [])) > 0
80
+ if self.allow_external_dependencies and has_external_dependencies:
81
+ has_external_dependencies = False
82
+ is_a_parent = bool(atlas_graph.parent_child_edges.get(child))
83
+ extra_parents = (
84
+ set(atlas_graph.all_parents(child))
85
+ - self.allowed_multi_parents
86
+ - set(self.root_resource_types)
87
+ - set(self.extra_nested_resource_types)
88
+ )
89
+ has_extra_parents = len(extra_parents) > 0
90
+ if has_external_dependencies or is_a_parent or has_extra_parents:
91
+ return False
92
+ self.state.resource_types.add(child)
93
+ return True
94
+
95
+
96
+ class ModuleConfigs(RootModel[dict[str, ModuleConfig]]):
97
+ def module_emoji_prefix(self, resource_type: str) -> str:
98
+ """Get the emoji prefix for a resource type based on its module."""
99
+ return next(
100
+ (
101
+ module_config.emojii
102
+ for module_config in self.root.values()
103
+ if resource_type in module_config.state.resource_types
104
+ ),
105
+ "",
106
+ )
@@ -0,0 +1,126 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ from collections import defaultdict
5
+ from pathlib import Path
6
+ from typing import Self
7
+
8
+ from model_lib import Entity
9
+ from pydantic import Field, RootModel
10
+ from zero_3rdparty.file_utils import iter_paths
11
+
12
+ from atlas_init.cli_tf.hcl.modifier2 import resource_types_vars_usage, safe_parse, variable_reader, variable_usages
13
+ from atlas_init.tf_ext.constants import ATLAS_PROVIDER_NAME, DEFAULT_EXTERNAL_SUBSTRINGS, DEFAULT_INTERNAL_SUBSTRINGS
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+
18
+ def find_example_dirs(repo_path: Path) -> list[Path]:
19
+ example_dirs: set[Path] = {
20
+ tf_file.parent for tf_file in (repo_path / "examples").rglob("*.tf") if ".terraform" not in tf_file.parts
21
+ }
22
+ return sorted(example_dirs)
23
+
24
+
25
+ def get_example_directories(repo_path: Path, skip_names: list[str]):
26
+ example_dirs = find_example_dirs(repo_path)
27
+ logger.info(f"Found {len(example_dirs)} exaple directories in {repo_path}")
28
+ if skip_names:
29
+ len_before = len(example_dirs)
30
+ example_dirs = [d for d in example_dirs if d.name not in skip_names]
31
+ logger.info(f"Skipped {len_before - len(example_dirs)} example directories with names: {skip_names}")
32
+ return example_dirs
33
+
34
+
35
+ def find_variables(variables_tf: Path) -> dict[str, str | None]:
36
+ if not variables_tf.exists():
37
+ return {}
38
+ tree = safe_parse(variables_tf)
39
+ if not tree:
40
+ logger.warning(f"Failed to parse {variables_tf}")
41
+ return {}
42
+ return variable_reader(tree)
43
+
44
+
45
+ def find_variable_resource_type_usages(variables: set[str], example_dir: Path) -> dict[str, set[str]]:
46
+ usages = defaultdict(set)
47
+ for path in example_dir.glob("*.tf"):
48
+ tree = safe_parse(path)
49
+ if not tree:
50
+ logger.warning(f"Failed to parse {path}")
51
+ continue
52
+ path_usages = variable_usages(variables, tree)
53
+ for variable, resources in path_usages.items():
54
+ usages[variable].update(resources)
55
+ return usages
56
+
57
+
58
+ class ResourceVarUsage(Entity):
59
+ var_name: str
60
+ attribute_path: str
61
+
62
+
63
+ def is_variable_name_external(
64
+ name: str, external_substrings: list[str] | None = None, internal_substrings: list[str] | None = None
65
+ ) -> bool:
66
+ external_substrings = external_substrings or DEFAULT_EXTERNAL_SUBSTRINGS
67
+ internal_substrings = internal_substrings or DEFAULT_INTERNAL_SUBSTRINGS
68
+ if any(substring in name for substring in internal_substrings):
69
+ return False
70
+ return any(substring in name for substring in external_substrings)
71
+
72
+
73
+ class ResourceTypeUsage(Entity):
74
+ name: str
75
+ example_files: list[Path] = Field(default_factory=list)
76
+ variable_usage: list[ResourceVarUsage] = Field(default_factory=list)
77
+
78
+ def add_usage(self, example_files: list[Path], variable_usages: list[ResourceVarUsage]):
79
+ for example_file in example_files:
80
+ if example_file not in self.example_files:
81
+ self.example_files.append(example_file)
82
+ self.variable_usage.extend(variable_usages)
83
+
84
+ @property
85
+ def external_var_usages(self) -> list[str]:
86
+ return [usage.var_name for usage in self.variable_usage if is_variable_name_external(usage.var_name)]
87
+
88
+
89
+ class ResourceTypes(RootModel[dict[str, ResourceTypeUsage]]):
90
+ def add_resource_type(self, resource_type: str, example_files: list[Path], variable_usages: list[ResourceVarUsage]):
91
+ if resource_type not in self.root:
92
+ self.root[resource_type] = ResourceTypeUsage(name=resource_type)
93
+ resource_type_usage = self.root[resource_type]
94
+ resource_type_usage.add_usage(example_files, variable_usages)
95
+
96
+ def atlas_resource_type_with_external_var_usages(self) -> Self:
97
+ return type(self)(
98
+ root={
99
+ name: usage
100
+ for name, usage in self.root.items()
101
+ if name.startswith(ATLAS_PROVIDER_NAME) and usage.external_var_usages
102
+ }
103
+ )
104
+
105
+ def dump_with_external_vars(self) -> dict[str, dict]:
106
+ return {
107
+ name: usages.model_dump() | {"external_var_usages": usages.external_var_usages}
108
+ for name, usages in self.root.items()
109
+ }
110
+
111
+
112
+ def find_resource_types_with_usages(example_dir: Path):
113
+ output = ResourceTypes(root={})
114
+ for path in iter_paths(example_dir, "*.tf", exclude_folder_names=[".terraform"]):
115
+ tree = safe_parse(path)
116
+ if not tree:
117
+ logger.warning(f"Failed to parse {path}")
118
+ continue
119
+ type_var_usages = resource_types_vars_usage(tree)
120
+ for resource_type, var_usages in type_var_usages.items():
121
+ variable_usages = [
122
+ ResourceVarUsage(var_name=variable_name, attribute_path=attribute_path)
123
+ for variable_name, attribute_path in var_usages.items()
124
+ ]
125
+ output.add_resource_type(resource_type, example_files=[path], variable_usages=variable_usages)
126
+ return output
@@ -0,0 +1,39 @@
1
+ from pathlib import Path
2
+ from model_lib import StaticSettings
3
+
4
+
5
+ class TfDepSettings(StaticSettings):
6
+ @property
7
+ def atlas_graph_path(self) -> Path:
8
+ return self.static_root / "atlas_graph.yaml"
9
+
10
+ @property
11
+ def vars_file_path(self) -> Path:
12
+ return self.static_root / "tf_vars.yaml"
13
+
14
+ @property
15
+ def vars_external_file_path(self) -> Path:
16
+ return self.static_root / "tf_vars_external.yaml"
17
+
18
+ @property
19
+ def resource_types_file_path(self) -> Path:
20
+ return self.static_root / "tf_resource_types.yaml"
21
+
22
+ @property
23
+ def resource_types_external_file_path(self) -> Path:
24
+ return self.static_root / "tf_resource_types_external.yaml"
25
+
26
+ @property
27
+ def schema_resource_types_path(self) -> Path:
28
+ return self.static_root / "tf_schema_resource_types.yaml"
29
+
30
+ @property
31
+ def schema_resource_types_deprecated_path(self) -> Path:
32
+ return self.static_root / "tf_schema_resource_types_deprecated.yaml"
33
+
34
+ @property
35
+ def api_calls_path(self) -> Path:
36
+ return self.static_root / "tf_api_calls.yaml"
37
+
38
+ def pagination_output_path(self, query_string: str) -> Path:
39
+ return self.static_root / "pagination_output" / f"query_is_{query_string or 'empty'}.md"