atlas-init 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atlas_init/__init__.py +1 -1
- atlas_init/cli_args.py +19 -1
- atlas_init/cli_tf/ci_tests.py +116 -24
- atlas_init/cli_tf/go_test_run.py +14 -2
- atlas_init/cli_tf/go_test_summary.py +334 -82
- atlas_init/cli_tf/go_test_tf_error.py +20 -12
- atlas_init/cli_tf/hcl/modifier2.py +120 -0
- atlas_init/cli_tf/openapi.py +10 -6
- atlas_init/html_out/__init__.py +0 -0
- atlas_init/html_out/md_export.py +143 -0
- atlas_init/sdk_ext/__init__.py +0 -0
- atlas_init/sdk_ext/go.py +102 -0
- atlas_init/sdk_ext/typer_app.py +18 -0
- atlas_init/settings/env_vars.py +13 -1
- atlas_init/settings/env_vars_generated.py +2 -0
- atlas_init/tf/.terraform.lock.hcl +33 -33
- atlas_init/tf/modules/aws_s3/provider.tf +1 -1
- atlas_init/tf/modules/aws_vpc/provider.tf +1 -1
- atlas_init/tf/modules/cloud_provider/provider.tf +1 -1
- atlas_init/tf/modules/cluster/provider.tf +1 -1
- atlas_init/tf/modules/encryption_at_rest/provider.tf +1 -1
- atlas_init/tf/modules/federated_vars/federated_vars.tf +1 -2
- atlas_init/tf/modules/federated_vars/provider.tf +1 -1
- atlas_init/tf/modules/project_extra/provider.tf +1 -1
- atlas_init/tf/modules/stream_instance/provider.tf +1 -1
- atlas_init/tf/modules/vpc_peering/provider.tf +1 -1
- atlas_init/tf/modules/vpc_privatelink/versions.tf +1 -1
- atlas_init/tf/providers.tf +1 -1
- atlas_init/tf_ext/__init__.py +0 -0
- atlas_init/tf_ext/__main__.py +3 -0
- atlas_init/tf_ext/api_call.py +325 -0
- atlas_init/tf_ext/args.py +17 -0
- atlas_init/tf_ext/constants.py +3 -0
- atlas_init/tf_ext/models.py +106 -0
- atlas_init/tf_ext/paths.py +126 -0
- atlas_init/tf_ext/settings.py +39 -0
- atlas_init/tf_ext/tf_dep.py +324 -0
- atlas_init/tf_ext/tf_modules.py +394 -0
- atlas_init/tf_ext/tf_vars.py +173 -0
- atlas_init/tf_ext/typer_app.py +24 -0
- {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/METADATA +3 -2
- {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/RECORD +45 -28
- atlas_init-0.7.0.dist-info/entry_points.txt +5 -0
- atlas_init-0.6.0.dist-info/entry_points.txt +0 -2
- {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/WHEEL +0 -0
- {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/licenses/LICENSE +0 -0
atlas_init/tf/providers.tf
CHANGED
File without changes
|
@@ -0,0 +1,325 @@
|
|
1
|
+
from collections import defaultdict
|
2
|
+
from concurrent.futures import Future, as_completed
|
3
|
+
from functools import lru_cache
|
4
|
+
import json
|
5
|
+
import logging
|
6
|
+
import os
|
7
|
+
from pathlib import Path
|
8
|
+
|
9
|
+
from ask_shell import new_task, print_to_live, run_pool
|
10
|
+
from model_lib import dump, parse_model
|
11
|
+
from pydantic import BaseModel, Field, model_validator
|
12
|
+
import requests
|
13
|
+
from rich.markdown import Markdown
|
14
|
+
import typer
|
15
|
+
from requests.auth import HTTPDigestAuth
|
16
|
+
from zero_3rdparty.file_utils import ensure_parents_write_text
|
17
|
+
from zero_3rdparty.str_utils import ensure_prefix, ensure_suffix, instance_repr
|
18
|
+
|
19
|
+
from atlas_init.cli_tf.mock_tf_log import resolve_admin_api_path
|
20
|
+
from atlas_init.cli_tf.openapi import OpenapiSchema
|
21
|
+
from atlas_init.settings.env_vars import init_settings
|
22
|
+
from atlas_init.settings.env_vars_generated import AtlasSettingsWithProject
|
23
|
+
from atlas_init.settings.env_vars_modules import (
|
24
|
+
TFModuleCluster,
|
25
|
+
TFModuleFederated_Vars,
|
26
|
+
TFModuleProject_Extra,
|
27
|
+
TFModuleStream_Instance,
|
28
|
+
)
|
29
|
+
from atlas_init.settings.path import load_dotenv
|
30
|
+
from atlas_init.tf_ext.settings import TfDepSettings
|
31
|
+
|
32
|
+
logger = logging.getLogger(__name__)
|
33
|
+
|
34
|
+
ALLOWED_MISSING_VARS: set[str] = {
|
35
|
+
"alertConfigId",
|
36
|
+
"alertId",
|
37
|
+
"clientId",
|
38
|
+
"cloudProvider",
|
39
|
+
"invoiceId",
|
40
|
+
# "name",
|
41
|
+
"pipelineName",
|
42
|
+
"processId",
|
43
|
+
"username",
|
44
|
+
}
|
45
|
+
ALLOWED_ERROR_CODES: set[str] = {
|
46
|
+
"CANNOT_USE_CLUSTER_IN_SERVERLESS_INSTANCE_API",
|
47
|
+
"VALIDATION_ERROR",
|
48
|
+
"UNEXPECTED_ERROR",
|
49
|
+
"CANNOT_USE_NON_FLEX_CLUSTER_IN_FLEX_API",
|
50
|
+
"CHECKPOINTS_ONLY_ON_CONTINOUS_BACKUP",
|
51
|
+
"INCORRECT_BACKUP_API_ENDPOINT",
|
52
|
+
}
|
53
|
+
|
54
|
+
|
55
|
+
# export ATLAS_INIT_TEST_SUITES=clusterm10,s3,federated,project,stream_connection
|
56
|
+
def resolve_path_variables() -> dict[str, str]:
|
57
|
+
settings = init_settings()
|
58
|
+
env_vars_full = load_dotenv(settings.env_vars_vs_code)
|
59
|
+
atlas_settings = AtlasSettingsWithProject(**env_vars_full)
|
60
|
+
cluster_settings = TFModuleCluster(**env_vars_full)
|
61
|
+
project_settings = TFModuleProject_Extra(**env_vars_full)
|
62
|
+
stream_settings = TFModuleStream_Instance(**env_vars_full)
|
63
|
+
federated_settings = TFModuleFederated_Vars(**env_vars_full)
|
64
|
+
return {
|
65
|
+
"orgId": atlas_settings.MONGODB_ATLAS_ORG_ID,
|
66
|
+
"cloudProvider": "AWS",
|
67
|
+
"federationSettingsId": federated_settings.MONGODB_ATLAS_FEDERATION_SETTINGS_ID,
|
68
|
+
"clusterName": cluster_settings.MONGODB_ATLAS_CLUSTER_NAME,
|
69
|
+
"name": cluster_settings.MONGODB_ATLAS_CLUSTER_NAME,
|
70
|
+
"groupId": atlas_settings.MONGODB_ATLAS_PROJECT_ID,
|
71
|
+
"teamId": project_settings.MONGODB_ATLAS_TEAM_ID,
|
72
|
+
"tenantName": stream_settings.MONGODB_ATLAS_STREAM_INSTANCE_NAME,
|
73
|
+
"apiUserId": atlas_settings.MONGODB_ATLAS_PROJECT_OWNER_ID,
|
74
|
+
"username": atlas_settings.MONGODB_ATLAS_USER_EMAIL,
|
75
|
+
}
|
76
|
+
|
77
|
+
|
78
|
+
class ApiCall(BaseModel):
|
79
|
+
operation_id: str
|
80
|
+
path: str
|
81
|
+
accept_header: str = "application/vnd.atlas.2023-01-01+json"
|
82
|
+
query_args: dict[str, str] = Field(default_factory=dict)
|
83
|
+
|
84
|
+
def __str__(self):
|
85
|
+
return instance_repr(self, ["operation_id", "path"])
|
86
|
+
|
87
|
+
def path_with_variables(self, path_variables: dict[str, str]):
|
88
|
+
return self.path.format(**path_variables)
|
89
|
+
|
90
|
+
@model_validator(mode="after")
|
91
|
+
def check_path_variables(self):
|
92
|
+
self.accept_header = ensure_prefix(self.accept_header, "application/vnd.atlas.")
|
93
|
+
self.accept_header = ensure_suffix(self.accept_header, "+json")
|
94
|
+
return self
|
95
|
+
|
96
|
+
|
97
|
+
class UnresolvedPathsError(Exception):
|
98
|
+
def __init__(self, missing_var_paths: dict[str, list[str]]) -> None:
|
99
|
+
self.missing_var_paths = missing_var_paths
|
100
|
+
missing_vars_formatted = "\n".join(f"{var}: {paths}" for var, paths in missing_var_paths.items())
|
101
|
+
super().__init__(f"Failed to resolve path variables:\nMissing vars: {missing_vars_formatted}")
|
102
|
+
|
103
|
+
|
104
|
+
class ApiCalls(BaseModel):
|
105
|
+
calls: list[ApiCall] = Field(default_factory=list)
|
106
|
+
ignored_calls: list[ApiCall] = Field(default_factory=list)
|
107
|
+
path_variables: dict[str, str] = Field(default_factory=resolve_path_variables)
|
108
|
+
skip_validation: bool = False
|
109
|
+
|
110
|
+
@model_validator(mode="after")
|
111
|
+
def check_path_variables(self):
|
112
|
+
if self.skip_validation:
|
113
|
+
return self
|
114
|
+
missing_vars_paths: dict[str, list[str]] = defaultdict(list)
|
115
|
+
ok_calls = []
|
116
|
+
for call in self.calls:
|
117
|
+
try:
|
118
|
+
call.path_with_variables(self.path_variables)
|
119
|
+
ok_calls.append(call)
|
120
|
+
except KeyError as e:
|
121
|
+
missing_vars_paths[str(e).strip("'")].append(f"{call.operation_id} {call.path}")
|
122
|
+
self.ignored_calls.append(call)
|
123
|
+
continue
|
124
|
+
for allowed_missing in sorted(ALLOWED_MISSING_VARS):
|
125
|
+
if allowed_missing in missing_vars_paths:
|
126
|
+
logger.info(f"Allowed missing variable {allowed_missing}: {missing_vars_paths[allowed_missing]}")
|
127
|
+
del missing_vars_paths[allowed_missing]
|
128
|
+
if missing_vars_paths:
|
129
|
+
raise UnresolvedPathsError(missing_var_paths=missing_vars_paths)
|
130
|
+
self.calls = ok_calls
|
131
|
+
return self
|
132
|
+
|
133
|
+
def dump_to_dict(self) -> dict:
|
134
|
+
return {
|
135
|
+
"calls": [call.model_dump(exclude_defaults=True, exclude_unset=True) for call in self.calls],
|
136
|
+
}
|
137
|
+
|
138
|
+
|
139
|
+
@lru_cache
|
140
|
+
def _public_private_key() -> tuple[str, str]:
|
141
|
+
public_key = os.environ.get("MONGODB_ATLAS_PUBLIC_KEY")
|
142
|
+
private_key = os.environ.get("MONGODB_ATLAS_PRIVATE_KEY")
|
143
|
+
if not public_key or not private_key:
|
144
|
+
raise ValueError("MONGODB_ATLAS_PUBLIC_KEY and MONGODB_ATLAS_PRIVATE_KEY must be set in environment variables.")
|
145
|
+
return public_key, private_key
|
146
|
+
|
147
|
+
|
148
|
+
class APICallError(Exception):
|
149
|
+
def __init__(self, api_call: ApiCall, json_response: dict, error: requests.exceptions.HTTPError):
|
150
|
+
self.api_call = api_call
|
151
|
+
self.json_response = json_response
|
152
|
+
super().__init__(f"Failed to make API call {api_call}:\njson={json_response}\n{error}")
|
153
|
+
|
154
|
+
@property
|
155
|
+
def error_code(self) -> str:
|
156
|
+
return self.json_response.get("errorCode", "")
|
157
|
+
|
158
|
+
|
159
|
+
def call_api(api_call: ApiCall, path_variables: dict[str, str]) -> dict:
|
160
|
+
resolved_path = api_call.path_with_variables(path_variables)
|
161
|
+
response = requests.get(
|
162
|
+
f"https://cloud-dev.mongodb.com/{resolved_path.lstrip('/')}",
|
163
|
+
params=api_call.query_args,
|
164
|
+
headers={"Accept": api_call.accept_header, "Content-Type": "application/json"},
|
165
|
+
auth=HTTPDigestAuth(*_public_private_key()),
|
166
|
+
timeout=30,
|
167
|
+
)
|
168
|
+
try:
|
169
|
+
response_json = response.json()
|
170
|
+
except requests.exceptions.JSONDecodeError as e:
|
171
|
+
logger.error(f"Failed to parse_json {api_call}: {e}")
|
172
|
+
response_json = {}
|
173
|
+
try:
|
174
|
+
response.raise_for_status()
|
175
|
+
except requests.exceptions.HTTPError as e:
|
176
|
+
raise APICallError(api_call, response_json, e) from e
|
177
|
+
return response_json
|
178
|
+
|
179
|
+
|
180
|
+
class NoSelfLinkError(Exception):
|
181
|
+
def __init__(self, json_response: dict) -> None:
|
182
|
+
self.json_response = json_response
|
183
|
+
super().__init__("No self link found in response")
|
184
|
+
|
185
|
+
|
186
|
+
def parse_href_response(json_response: dict) -> str:
|
187
|
+
for ref in json_response.get("links", []):
|
188
|
+
if ref.get("rel") == "self":
|
189
|
+
return ref.get("href")
|
190
|
+
raise NoSelfLinkError(json_response)
|
191
|
+
|
192
|
+
|
193
|
+
def api_config(
|
194
|
+
config_path_str: str = typer.Option("", "-p", "--path", help="Path to the API config file"),
|
195
|
+
query_args_str: str = typer.Option(
|
196
|
+
'{"pageNum": "0", "itemsPerPage": "0"}', "-q", "--query-args", help="Query arguments for the API call"
|
197
|
+
),
|
198
|
+
verbose: bool = typer.Option(False, "-v", "--verbose", help="Verbose output"),
|
199
|
+
):
|
200
|
+
query_args: dict[str, str] = json.loads(query_args_str)
|
201
|
+
if config_path_str == "":
|
202
|
+
with new_task("Find API Calls that use pagination"):
|
203
|
+
config_path = dump_config_path(query_args)
|
204
|
+
else:
|
205
|
+
config_path = Path(config_path_str)
|
206
|
+
assert config_path.exists(), f"Config file {config_path} does not exist."
|
207
|
+
model = parse_model(config_path, t=ApiCalls)
|
208
|
+
total_calls = len(model.calls)
|
209
|
+
assert _public_private_key(), "Public and private keys must be set in environment variables."
|
210
|
+
path_variables = model.path_variables
|
211
|
+
op_id_path_self_qstring: dict[tuple[str, str], str] = {}
|
212
|
+
with run_pool(
|
213
|
+
task_name="make API calls", max_concurrent_submits=10, threads_used_per_submit=1, total=total_calls
|
214
|
+
) as pool:
|
215
|
+
futures: dict[Future, ApiCall] = {
|
216
|
+
pool.submit(call_api, api_call, path_variables): api_call for api_call in model.calls
|
217
|
+
}
|
218
|
+
for future in as_completed(futures):
|
219
|
+
api_call = futures[future]
|
220
|
+
try:
|
221
|
+
result = future.result()
|
222
|
+
except APICallError as e:
|
223
|
+
if e.error_code in ALLOWED_ERROR_CODES:
|
224
|
+
logger.info(f"Allowed error code {e.error_code} in response for {api_call}")
|
225
|
+
model.ignored_calls.append(api_call)
|
226
|
+
continue
|
227
|
+
raise
|
228
|
+
except Exception as e:
|
229
|
+
logger.error(e)
|
230
|
+
continue
|
231
|
+
try:
|
232
|
+
href = parse_href_response(result)
|
233
|
+
op_id_path_self_qstring[(api_call.operation_id, api_call.path)] = href.split("?")[-1]
|
234
|
+
except NoSelfLinkError as e:
|
235
|
+
logger.error(f"{api_call} did not have a self link in the response:\n{e.json_response}")
|
236
|
+
continue
|
237
|
+
logger.info(f"API call {api_call} completed successfully with self ref:\n{href}")
|
238
|
+
if verbose:
|
239
|
+
logger.info(f"Response for {api_call.query_args} was:\n{dump(result, 'pretty_json')}")
|
240
|
+
query_args_str = "&".join(f"{key}={value}" for key, value in query_args.items())
|
241
|
+
md_report: list[str] = [
|
242
|
+
f"# Pagination Report for query_args='{query_args_str}'",
|
243
|
+
"",
|
244
|
+
"## Checked endpoints",
|
245
|
+
"",
|
246
|
+
"Operation ID | Path | SelfQueryString",
|
247
|
+
"--- | --- | ---",
|
248
|
+
*[
|
249
|
+
f"{operation_id} | {path} | {self_query_string}"
|
250
|
+
for (operation_id, path), self_query_string in op_id_path_self_qstring.items()
|
251
|
+
],
|
252
|
+
"",
|
253
|
+
"## Ignored endpoints (not checked)",
|
254
|
+
"",
|
255
|
+
"Operation ID | Path",
|
256
|
+
"--- | ---",
|
257
|
+
*[f"{call.operation_id} | {call.path}" for call in model.ignored_calls],
|
258
|
+
]
|
259
|
+
md_content = "\n".join(md_report)
|
260
|
+
md = Markdown(md_content)
|
261
|
+
print_to_live(md)
|
262
|
+
output_path = TfDepSettings.from_env().pagination_output_path(query_args_str)
|
263
|
+
ensure_parents_write_text(output_path, md_content)
|
264
|
+
logger.info(f"Pagination report saved to {output_path}")
|
265
|
+
return md
|
266
|
+
|
267
|
+
|
268
|
+
def api(
|
269
|
+
path: str = typer.Option("-p", "--path", help="Path to the API endpoint"),
|
270
|
+
query_string: str = typer.Option("", "-q", "--query-string", help="Query string for the API call"),
|
271
|
+
):
|
272
|
+
assert path, "Path must be provided."
|
273
|
+
accept_header = "application/vnd.atlas.2023-01-01+json"
|
274
|
+
url = f"https://cloud-dev.mongodb.com/{path.lstrip('/')}?{query_string}"
|
275
|
+
logger.info(f"Calling {url}")
|
276
|
+
try:
|
277
|
+
r = requests.get(
|
278
|
+
url,
|
279
|
+
headers={"Accept": accept_header, "Content-Type": "application/json"},
|
280
|
+
auth=HTTPDigestAuth(*_public_private_key()),
|
281
|
+
timeout=30,
|
282
|
+
)
|
283
|
+
print(r.text)
|
284
|
+
r.raise_for_status()
|
285
|
+
except requests.exceptions.HTTPError as e:
|
286
|
+
print(e)
|
287
|
+
print(e.response)
|
288
|
+
|
289
|
+
|
290
|
+
def dump_config_path(query_args: dict[str, str]) -> Path:
|
291
|
+
settings = TfDepSettings.from_env()
|
292
|
+
latest_api_spec = resolve_admin_api_path()
|
293
|
+
model = parse_model(latest_api_spec, t=OpenapiSchema)
|
294
|
+
paginated_paths: list[ApiCall] = []
|
295
|
+
path_versions = list(model.path_method_api_versions())
|
296
|
+
|
297
|
+
for (path, method, code), versions in path_versions:
|
298
|
+
if method != "get" or code != "200":
|
299
|
+
continue
|
300
|
+
assert len(versions) == 1, f"{path} {method} {code} has multiple versions: {versions}"
|
301
|
+
get_method = model.get_method(path)
|
302
|
+
if not get_method:
|
303
|
+
continue
|
304
|
+
parameters = get_method.get("parameters", [])
|
305
|
+
for param in parameters:
|
306
|
+
if param_ref := param.get("$ref"):
|
307
|
+
if param_ref.endswith("itemsPerPage"):
|
308
|
+
version = versions[0].strftime("%Y-%m-%d")
|
309
|
+
paginated_paths.append(
|
310
|
+
ApiCall(
|
311
|
+
path=path,
|
312
|
+
query_args=query_args,
|
313
|
+
accept_header=f"application/vnd.atlas.{version}+json",
|
314
|
+
operation_id=get_method["operationId"],
|
315
|
+
)
|
316
|
+
)
|
317
|
+
config_path = settings.api_calls_path
|
318
|
+
calls = ApiCalls(
|
319
|
+
calls=paginated_paths,
|
320
|
+
skip_validation=True,
|
321
|
+
)
|
322
|
+
calls_yaml = dump(calls.dump_to_dict(), "yaml")
|
323
|
+
logger.info(f"Dumped {len(paginated_paths)} API calls to {config_path}")
|
324
|
+
ensure_parents_write_text(config_path, calls_yaml)
|
325
|
+
return config_path
|
@@ -0,0 +1,17 @@
|
|
1
|
+
import typer
|
2
|
+
|
3
|
+
|
4
|
+
def default_skippped_directories() -> list[str]:
|
5
|
+
return [
|
6
|
+
"prometheus-and-teams", # Provider registry.terraform.io/hashicorp/template v2.2.0 does not have a package available for your current platform, darwin_arm64.
|
7
|
+
]
|
8
|
+
|
9
|
+
|
10
|
+
REPO_PATH_ARG = typer.Argument(help, help="Path to the mongodbatlas-terraform-provider repository")
|
11
|
+
SKIP_EXAMPLES_DIRS_OPTION = typer.Option(
|
12
|
+
...,
|
13
|
+
"--skip-examples",
|
14
|
+
help="Skip example directories with these names",
|
15
|
+
default_factory=default_skippped_directories,
|
16
|
+
show_default=True,
|
17
|
+
)
|
@@ -0,0 +1,106 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
from typing import Self
|
3
|
+
|
4
|
+
from model_lib import Entity
|
5
|
+
from pydantic import Field, RootModel, model_validator
|
6
|
+
|
7
|
+
from atlas_init.tf_ext.tf_dep import AtlasGraph
|
8
|
+
|
9
|
+
_emojii_list = [
|
10
|
+
"1️⃣",
|
11
|
+
"2️⃣",
|
12
|
+
"3️⃣",
|
13
|
+
"4️⃣",
|
14
|
+
"5️⃣",
|
15
|
+
"6️⃣",
|
16
|
+
"7️⃣",
|
17
|
+
"8️⃣",
|
18
|
+
"9️⃣",
|
19
|
+
"🔟",
|
20
|
+
"1️⃣1️⃣",
|
21
|
+
"1️⃣2️⃣",
|
22
|
+
]
|
23
|
+
_emoji_counter = 0
|
24
|
+
|
25
|
+
|
26
|
+
def choose_next_emoji() -> str:
|
27
|
+
global _emoji_counter
|
28
|
+
emoji = _emojii_list[_emoji_counter]
|
29
|
+
_emoji_counter += 1
|
30
|
+
return emoji
|
31
|
+
|
32
|
+
|
33
|
+
class ModuleState(Entity):
|
34
|
+
resource_types: set[str] = Field(default_factory=set, description="Set of resource types in the module.")
|
35
|
+
|
36
|
+
|
37
|
+
def default_allowed_multi_parents() -> set[str]:
|
38
|
+
return {
|
39
|
+
"mongodbatlas_project",
|
40
|
+
}
|
41
|
+
|
42
|
+
|
43
|
+
class ModuleConfig(Entity):
|
44
|
+
name: str = Field(..., description="Name of the module.")
|
45
|
+
root_resource_types: list[str] = Field(..., description="List of root resource types for the module.")
|
46
|
+
force_include_children: list[str] = Field(
|
47
|
+
default_factory=list, description="List of resource types that should always be included as children."
|
48
|
+
)
|
49
|
+
emojii: str = Field(init=False, default_factory=choose_next_emoji)
|
50
|
+
allowed_multi_parents: set[str] = Field(
|
51
|
+
default_factory=default_allowed_multi_parents,
|
52
|
+
description="Set of parents that a child resource type can have in addition to the root_resource_type.",
|
53
|
+
)
|
54
|
+
allow_external_dependencies: bool = Field(
|
55
|
+
default=False, description="Whether to allow external dependencies for the module."
|
56
|
+
)
|
57
|
+
extra_nested_resource_types: list[str] = Field(
|
58
|
+
default_factory=list,
|
59
|
+
description="List of additional nested resource types that should be included in the module.",
|
60
|
+
)
|
61
|
+
|
62
|
+
state: ModuleState = Field(default_factory=ModuleState, description="Internal state of the module.")
|
63
|
+
|
64
|
+
@model_validator(mode="after")
|
65
|
+
def update_state(self) -> Self:
|
66
|
+
self.state.resource_types.update(self.root_resource_types)
|
67
|
+
return self
|
68
|
+
|
69
|
+
@property
|
70
|
+
def tree_label(self) -> str:
|
71
|
+
return f"{self.emojii} {self.name}"
|
72
|
+
|
73
|
+
def include_child(self, child: str, atlas_graph: AtlasGraph) -> bool:
|
74
|
+
if child in atlas_graph.deprecated_resource_types:
|
75
|
+
return False
|
76
|
+
if child in self.force_include_children or child in self.extra_nested_resource_types:
|
77
|
+
self.state.resource_types.add(child)
|
78
|
+
return True
|
79
|
+
has_external_dependencies = len(atlas_graph.external_parents.get(child, [])) > 0
|
80
|
+
if self.allow_external_dependencies and has_external_dependencies:
|
81
|
+
has_external_dependencies = False
|
82
|
+
is_a_parent = bool(atlas_graph.parent_child_edges.get(child))
|
83
|
+
extra_parents = (
|
84
|
+
set(atlas_graph.all_parents(child))
|
85
|
+
- self.allowed_multi_parents
|
86
|
+
- set(self.root_resource_types)
|
87
|
+
- set(self.extra_nested_resource_types)
|
88
|
+
)
|
89
|
+
has_extra_parents = len(extra_parents) > 0
|
90
|
+
if has_external_dependencies or is_a_parent or has_extra_parents:
|
91
|
+
return False
|
92
|
+
self.state.resource_types.add(child)
|
93
|
+
return True
|
94
|
+
|
95
|
+
|
96
|
+
class ModuleConfigs(RootModel[dict[str, ModuleConfig]]):
|
97
|
+
def module_emoji_prefix(self, resource_type: str) -> str:
|
98
|
+
"""Get the emoji prefix for a resource type based on its module."""
|
99
|
+
return next(
|
100
|
+
(
|
101
|
+
module_config.emojii
|
102
|
+
for module_config in self.root.values()
|
103
|
+
if resource_type in module_config.state.resource_types
|
104
|
+
),
|
105
|
+
"",
|
106
|
+
)
|
@@ -0,0 +1,126 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import logging
|
4
|
+
from collections import defaultdict
|
5
|
+
from pathlib import Path
|
6
|
+
from typing import Self
|
7
|
+
|
8
|
+
from model_lib import Entity
|
9
|
+
from pydantic import Field, RootModel
|
10
|
+
from zero_3rdparty.file_utils import iter_paths
|
11
|
+
|
12
|
+
from atlas_init.cli_tf.hcl.modifier2 import resource_types_vars_usage, safe_parse, variable_reader, variable_usages
|
13
|
+
from atlas_init.tf_ext.constants import ATLAS_PROVIDER_NAME, DEFAULT_EXTERNAL_SUBSTRINGS, DEFAULT_INTERNAL_SUBSTRINGS
|
14
|
+
|
15
|
+
logger = logging.getLogger(__name__)
|
16
|
+
|
17
|
+
|
18
|
+
def find_example_dirs(repo_path: Path) -> list[Path]:
|
19
|
+
example_dirs: set[Path] = {
|
20
|
+
tf_file.parent for tf_file in (repo_path / "examples").rglob("*.tf") if ".terraform" not in tf_file.parts
|
21
|
+
}
|
22
|
+
return sorted(example_dirs)
|
23
|
+
|
24
|
+
|
25
|
+
def get_example_directories(repo_path: Path, skip_names: list[str]):
|
26
|
+
example_dirs = find_example_dirs(repo_path)
|
27
|
+
logger.info(f"Found {len(example_dirs)} exaple directories in {repo_path}")
|
28
|
+
if skip_names:
|
29
|
+
len_before = len(example_dirs)
|
30
|
+
example_dirs = [d for d in example_dirs if d.name not in skip_names]
|
31
|
+
logger.info(f"Skipped {len_before - len(example_dirs)} example directories with names: {skip_names}")
|
32
|
+
return example_dirs
|
33
|
+
|
34
|
+
|
35
|
+
def find_variables(variables_tf: Path) -> dict[str, str | None]:
|
36
|
+
if not variables_tf.exists():
|
37
|
+
return {}
|
38
|
+
tree = safe_parse(variables_tf)
|
39
|
+
if not tree:
|
40
|
+
logger.warning(f"Failed to parse {variables_tf}")
|
41
|
+
return {}
|
42
|
+
return variable_reader(tree)
|
43
|
+
|
44
|
+
|
45
|
+
def find_variable_resource_type_usages(variables: set[str], example_dir: Path) -> dict[str, set[str]]:
|
46
|
+
usages = defaultdict(set)
|
47
|
+
for path in example_dir.glob("*.tf"):
|
48
|
+
tree = safe_parse(path)
|
49
|
+
if not tree:
|
50
|
+
logger.warning(f"Failed to parse {path}")
|
51
|
+
continue
|
52
|
+
path_usages = variable_usages(variables, tree)
|
53
|
+
for variable, resources in path_usages.items():
|
54
|
+
usages[variable].update(resources)
|
55
|
+
return usages
|
56
|
+
|
57
|
+
|
58
|
+
class ResourceVarUsage(Entity):
|
59
|
+
var_name: str
|
60
|
+
attribute_path: str
|
61
|
+
|
62
|
+
|
63
|
+
def is_variable_name_external(
|
64
|
+
name: str, external_substrings: list[str] | None = None, internal_substrings: list[str] | None = None
|
65
|
+
) -> bool:
|
66
|
+
external_substrings = external_substrings or DEFAULT_EXTERNAL_SUBSTRINGS
|
67
|
+
internal_substrings = internal_substrings or DEFAULT_INTERNAL_SUBSTRINGS
|
68
|
+
if any(substring in name for substring in internal_substrings):
|
69
|
+
return False
|
70
|
+
return any(substring in name for substring in external_substrings)
|
71
|
+
|
72
|
+
|
73
|
+
class ResourceTypeUsage(Entity):
|
74
|
+
name: str
|
75
|
+
example_files: list[Path] = Field(default_factory=list)
|
76
|
+
variable_usage: list[ResourceVarUsage] = Field(default_factory=list)
|
77
|
+
|
78
|
+
def add_usage(self, example_files: list[Path], variable_usages: list[ResourceVarUsage]):
|
79
|
+
for example_file in example_files:
|
80
|
+
if example_file not in self.example_files:
|
81
|
+
self.example_files.append(example_file)
|
82
|
+
self.variable_usage.extend(variable_usages)
|
83
|
+
|
84
|
+
@property
|
85
|
+
def external_var_usages(self) -> list[str]:
|
86
|
+
return [usage.var_name for usage in self.variable_usage if is_variable_name_external(usage.var_name)]
|
87
|
+
|
88
|
+
|
89
|
+
class ResourceTypes(RootModel[dict[str, ResourceTypeUsage]]):
|
90
|
+
def add_resource_type(self, resource_type: str, example_files: list[Path], variable_usages: list[ResourceVarUsage]):
|
91
|
+
if resource_type not in self.root:
|
92
|
+
self.root[resource_type] = ResourceTypeUsage(name=resource_type)
|
93
|
+
resource_type_usage = self.root[resource_type]
|
94
|
+
resource_type_usage.add_usage(example_files, variable_usages)
|
95
|
+
|
96
|
+
def atlas_resource_type_with_external_var_usages(self) -> Self:
|
97
|
+
return type(self)(
|
98
|
+
root={
|
99
|
+
name: usage
|
100
|
+
for name, usage in self.root.items()
|
101
|
+
if name.startswith(ATLAS_PROVIDER_NAME) and usage.external_var_usages
|
102
|
+
}
|
103
|
+
)
|
104
|
+
|
105
|
+
def dump_with_external_vars(self) -> dict[str, dict]:
|
106
|
+
return {
|
107
|
+
name: usages.model_dump() | {"external_var_usages": usages.external_var_usages}
|
108
|
+
for name, usages in self.root.items()
|
109
|
+
}
|
110
|
+
|
111
|
+
|
112
|
+
def find_resource_types_with_usages(example_dir: Path):
|
113
|
+
output = ResourceTypes(root={})
|
114
|
+
for path in iter_paths(example_dir, "*.tf", exclude_folder_names=[".terraform"]):
|
115
|
+
tree = safe_parse(path)
|
116
|
+
if not tree:
|
117
|
+
logger.warning(f"Failed to parse {path}")
|
118
|
+
continue
|
119
|
+
type_var_usages = resource_types_vars_usage(tree)
|
120
|
+
for resource_type, var_usages in type_var_usages.items():
|
121
|
+
variable_usages = [
|
122
|
+
ResourceVarUsage(var_name=variable_name, attribute_path=attribute_path)
|
123
|
+
for variable_name, attribute_path in var_usages.items()
|
124
|
+
]
|
125
|
+
output.add_resource_type(resource_type, example_files=[path], variable_usages=variable_usages)
|
126
|
+
return output
|
@@ -0,0 +1,39 @@
|
|
1
|
+
from pathlib import Path
|
2
|
+
from model_lib import StaticSettings
|
3
|
+
|
4
|
+
|
5
|
+
class TfDepSettings(StaticSettings):
|
6
|
+
@property
|
7
|
+
def atlas_graph_path(self) -> Path:
|
8
|
+
return self.static_root / "atlas_graph.yaml"
|
9
|
+
|
10
|
+
@property
|
11
|
+
def vars_file_path(self) -> Path:
|
12
|
+
return self.static_root / "tf_vars.yaml"
|
13
|
+
|
14
|
+
@property
|
15
|
+
def vars_external_file_path(self) -> Path:
|
16
|
+
return self.static_root / "tf_vars_external.yaml"
|
17
|
+
|
18
|
+
@property
|
19
|
+
def resource_types_file_path(self) -> Path:
|
20
|
+
return self.static_root / "tf_resource_types.yaml"
|
21
|
+
|
22
|
+
@property
|
23
|
+
def resource_types_external_file_path(self) -> Path:
|
24
|
+
return self.static_root / "tf_resource_types_external.yaml"
|
25
|
+
|
26
|
+
@property
|
27
|
+
def schema_resource_types_path(self) -> Path:
|
28
|
+
return self.static_root / "tf_schema_resource_types.yaml"
|
29
|
+
|
30
|
+
@property
|
31
|
+
def schema_resource_types_deprecated_path(self) -> Path:
|
32
|
+
return self.static_root / "tf_schema_resource_types_deprecated.yaml"
|
33
|
+
|
34
|
+
@property
|
35
|
+
def api_calls_path(self) -> Path:
|
36
|
+
return self.static_root / "tf_api_calls.yaml"
|
37
|
+
|
38
|
+
def pagination_output_path(self, query_string: str) -> Path:
|
39
|
+
return self.static_root / "pagination_output" / f"query_is_{query_string or 'empty'}.md"
|