atlas-init 0.6.0__py3-none-any.whl → 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atlas_init/__init__.py +1 -1
- atlas_init/cli_args.py +19 -1
- atlas_init/cli_tf/ci_tests.py +116 -24
- atlas_init/cli_tf/go_test_run.py +14 -2
- atlas_init/cli_tf/go_test_summary.py +334 -82
- atlas_init/cli_tf/go_test_tf_error.py +20 -12
- atlas_init/cli_tf/hcl/modifier2.py +120 -0
- atlas_init/cli_tf/openapi.py +10 -6
- atlas_init/html_out/__init__.py +0 -0
- atlas_init/html_out/md_export.py +143 -0
- atlas_init/sdk_ext/__init__.py +0 -0
- atlas_init/sdk_ext/go.py +102 -0
- atlas_init/sdk_ext/typer_app.py +18 -0
- atlas_init/settings/env_vars.py +13 -1
- atlas_init/settings/env_vars_generated.py +2 -0
- atlas_init/tf/.terraform.lock.hcl +33 -33
- atlas_init/tf/modules/aws_s3/provider.tf +1 -1
- atlas_init/tf/modules/aws_vpc/provider.tf +1 -1
- atlas_init/tf/modules/cloud_provider/provider.tf +1 -1
- atlas_init/tf/modules/cluster/provider.tf +1 -1
- atlas_init/tf/modules/encryption_at_rest/provider.tf +1 -1
- atlas_init/tf/modules/federated_vars/federated_vars.tf +1 -2
- atlas_init/tf/modules/federated_vars/provider.tf +1 -1
- atlas_init/tf/modules/project_extra/provider.tf +1 -1
- atlas_init/tf/modules/stream_instance/provider.tf +1 -1
- atlas_init/tf/modules/vpc_peering/provider.tf +1 -1
- atlas_init/tf/modules/vpc_privatelink/versions.tf +1 -1
- atlas_init/tf/providers.tf +1 -1
- atlas_init/tf_ext/__init__.py +0 -0
- atlas_init/tf_ext/__main__.py +3 -0
- atlas_init/tf_ext/api_call.py +325 -0
- atlas_init/tf_ext/args.py +17 -0
- atlas_init/tf_ext/constants.py +3 -0
- atlas_init/tf_ext/models.py +106 -0
- atlas_init/tf_ext/paths.py +126 -0
- atlas_init/tf_ext/settings.py +39 -0
- atlas_init/tf_ext/tf_dep.py +324 -0
- atlas_init/tf_ext/tf_modules.py +394 -0
- atlas_init/tf_ext/tf_vars.py +173 -0
- atlas_init/tf_ext/typer_app.py +24 -0
- {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/METADATA +3 -2
- {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/RECORD +45 -28
- atlas_init-0.7.0.dist-info/entry_points.txt +5 -0
- atlas_init-0.6.0.dist-info/entry_points.txt +0 -2
- {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/WHEEL +0 -0
- {atlas_init-0.6.0.dist-info → atlas_init-0.7.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,394 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
import logging
|
3
|
+
from dataclasses import dataclass
|
4
|
+
from pathlib import Path
|
5
|
+
from typing import ClassVar, Iterable
|
6
|
+
|
7
|
+
import pydot
|
8
|
+
import typer
|
9
|
+
from ask_shell import new_task, print_to_live
|
10
|
+
from model_lib import parse_list, parse_model
|
11
|
+
from rich.tree import Tree
|
12
|
+
from zero_3rdparty.iter_utils import flat_map
|
13
|
+
|
14
|
+
from atlas_init.tf_ext.constants import ATLAS_PROVIDER_NAME
|
15
|
+
from atlas_init.tf_ext.models import ModuleConfig, ModuleConfigs
|
16
|
+
from atlas_init.tf_ext.settings import TfDepSettings
|
17
|
+
from atlas_init.tf_ext.tf_dep import FORCE_INTERNAL_NODES, SKIP_NODES, AtlasGraph, edge_src_dest
|
18
|
+
|
19
|
+
logger = logging.getLogger(__name__)
|
20
|
+
|
21
|
+
|
22
|
+
def default_modules() -> list[str]:
|
23
|
+
return [
|
24
|
+
"mongodbatlas_advanced_cluster",
|
25
|
+
"mongodbatlas_cloud_provider_access_authorization",
|
26
|
+
"mongodbatlas_project",
|
27
|
+
"mongodbatlas_organization",
|
28
|
+
]
|
29
|
+
|
30
|
+
|
31
|
+
def default_skippped_module_resource_types() -> list[str]:
|
32
|
+
return [
|
33
|
+
"mongodbatlas_cluster",
|
34
|
+
"mongodbatlas_flex_cluster",
|
35
|
+
]
|
36
|
+
|
37
|
+
|
38
|
+
def default_module_configs() -> ModuleConfigs:
|
39
|
+
return ModuleConfigs(
|
40
|
+
root={
|
41
|
+
"alerts": ModuleConfig(
|
42
|
+
name="Alerts",
|
43
|
+
root_resource_types=[
|
44
|
+
"mongodbatlas_alert_configuration",
|
45
|
+
],
|
46
|
+
),
|
47
|
+
"auth": ModuleConfig(
|
48
|
+
name="Authentication",
|
49
|
+
root_resource_types=[
|
50
|
+
"mongodbatlas_api_key",
|
51
|
+
"mongodbatlas_custom_db_role",
|
52
|
+
"mongodbatlas_database_user",
|
53
|
+
"mongodbatlas_project_api_key",
|
54
|
+
"mongodbatlas_team",
|
55
|
+
"mongodbatlas_x509_authentication_database_user",
|
56
|
+
],
|
57
|
+
force_include_children=[
|
58
|
+
"mongodbatlas_access_list_api_key",
|
59
|
+
],
|
60
|
+
),
|
61
|
+
"ldap": ModuleConfig(
|
62
|
+
name="LDAP",
|
63
|
+
root_resource_types=[
|
64
|
+
"mongodbatlas_ldap_configuration",
|
65
|
+
"mongodbatlas_ldap_verify",
|
66
|
+
],
|
67
|
+
),
|
68
|
+
"federated auth": ModuleConfig(
|
69
|
+
name="Federated Authentication",
|
70
|
+
root_resource_types=[
|
71
|
+
"mongodbatlas_federated_settings_identity_provider",
|
72
|
+
],
|
73
|
+
force_include_children=[
|
74
|
+
"mongodbatlas_federated_settings_org_config",
|
75
|
+
],
|
76
|
+
),
|
77
|
+
"federated DB": ModuleConfig(
|
78
|
+
name="Federated Database",
|
79
|
+
root_resource_types=[
|
80
|
+
"mongodbatlas_federated_database_instance",
|
81
|
+
],
|
82
|
+
),
|
83
|
+
"network": ModuleConfig(
|
84
|
+
name="Network",
|
85
|
+
root_resource_types=[
|
86
|
+
"mongodbatlas_encryption_at_rest_private_endpoint",
|
87
|
+
"mongodbatlas_network_container",
|
88
|
+
"mongodbatlas_private_endpoint_regional_mode",
|
89
|
+
"mongodbatlas_privatelink_endpoint_service_data_federation_online_archive",
|
90
|
+
"mongodbatlas_privatelink_endpoint_service",
|
91
|
+
"mongodbatlas_privatelink_endpoint",
|
92
|
+
"mongodbatlas_stream_privatelink_endpoint",
|
93
|
+
],
|
94
|
+
force_include_children=["mongodbatlas_network_peering"],
|
95
|
+
),
|
96
|
+
"cloud_provider": ModuleConfig(
|
97
|
+
name="Cloud Provider",
|
98
|
+
root_resource_types=["mongodbatlas_cloud_provider_access_setup"],
|
99
|
+
allow_external_dependencies=True,
|
100
|
+
extra_nested_resource_types=["mongodbatlas_cloud_provider_access_authorization"],
|
101
|
+
force_include_children=["mongodbatlas_encryption_at_rest"],
|
102
|
+
),
|
103
|
+
"streams": ModuleConfig(
|
104
|
+
name="Streams",
|
105
|
+
root_resource_types=["mongodbatlas_stream_instance"],
|
106
|
+
extra_nested_resource_types=["mongodbatlas_stream_connection"],
|
107
|
+
force_include_children=["mongodbatlas_stream_processor"],
|
108
|
+
),
|
109
|
+
"cluster": ModuleConfig(
|
110
|
+
name="Cluster",
|
111
|
+
root_resource_types=["mongodbatlas_advanced_cluster"],
|
112
|
+
),
|
113
|
+
"project": ModuleConfig(
|
114
|
+
name="Project",
|
115
|
+
root_resource_types=["mongodbatlas_project"],
|
116
|
+
force_include_children=[
|
117
|
+
"mongodbatlas_project_ip_access_list" # the external aws_vpc dependency is not really needed
|
118
|
+
],
|
119
|
+
),
|
120
|
+
"organization": ModuleConfig(
|
121
|
+
name="Organization",
|
122
|
+
root_resource_types=["mongodbatlas_organization"],
|
123
|
+
),
|
124
|
+
"cloud_backup_actions": ModuleConfig(
|
125
|
+
name="Cloud Backup Actions",
|
126
|
+
root_resource_types=[
|
127
|
+
"mongodbatlas_cloud_backup_snapshot",
|
128
|
+
"mongodbatlas_cloud_backup_snapshot_export_bucket",
|
129
|
+
],
|
130
|
+
),
|
131
|
+
}
|
132
|
+
)
|
133
|
+
|
134
|
+
|
135
|
+
def tf_modules(
|
136
|
+
skipped_module_resource_types: list[str] = typer.Option(
|
137
|
+
...,
|
138
|
+
"-s",
|
139
|
+
"--skip-resource-types",
|
140
|
+
help="List of resource types to skip when creating module graphs",
|
141
|
+
default_factory=default_skippped_module_resource_types,
|
142
|
+
show_default=True,
|
143
|
+
),
|
144
|
+
):
|
145
|
+
settings = TfDepSettings.from_env()
|
146
|
+
atlas_graph = parse_atlas_graph(settings)
|
147
|
+
output_dir = settings.static_root
|
148
|
+
with new_task("Write graphs"):
|
149
|
+
color_coder_internal = color_coder(atlas_graph, keep_provider_name=False)
|
150
|
+
internal_graph = create_internal_dependencies(atlas_graph, color_coder=color_coder_internal)
|
151
|
+
add_unused_nodes_to_graph(settings, atlas_graph, color_coder_internal, internal_graph)
|
152
|
+
write_graph(internal_graph, output_dir, "atlas_internal.png")
|
153
|
+
write_graph(create_external_dependencies(atlas_graph), output_dir, "atlas_external.png")
|
154
|
+
with new_task("Write module graphs"):
|
155
|
+
modules = generate_module_graphs(skipped_module_resource_types, settings, atlas_graph)
|
156
|
+
with new_task("Internal Graph with Module Numbers"):
|
157
|
+
module_color_coder = ModuleColorCoder(
|
158
|
+
atlas_graph,
|
159
|
+
keep_provider_name=False,
|
160
|
+
modules=modules,
|
161
|
+
)
|
162
|
+
internal_graph_with_numbers = create_internal_dependencies(atlas_graph, module_color_coder)
|
163
|
+
add_unused_nodes_to_graph(settings, atlas_graph, module_color_coder, internal_graph_with_numbers)
|
164
|
+
write_graph(internal_graph_with_numbers, settings.static_root, "atlas_internal_with_numbers.png")
|
165
|
+
with new_task("Missing modules"):
|
166
|
+
all_resources: list[str] = parse_list(settings.schema_resource_types_path, format="yaml")
|
167
|
+
missing_resources = [
|
168
|
+
resource_type
|
169
|
+
for resource_type in all_resources
|
170
|
+
if modules.module_emoji_prefix(resource_type) == ""
|
171
|
+
and resource_type not in skipped_module_resource_types
|
172
|
+
and resource_type not in atlas_graph.deprecated_resource_types
|
173
|
+
]
|
174
|
+
logger.info(f"Missing modules: \n{'\n'.join(missing_resources)}")
|
175
|
+
|
176
|
+
|
177
|
+
def generate_module_graphs(skipped_module_resource_types, settings, atlas_graph):
|
178
|
+
tree = Tree(
|
179
|
+
"Module graphs",
|
180
|
+
)
|
181
|
+
used_resource_types: set[str] = set(
|
182
|
+
skipped_module_resource_types
|
183
|
+
) # avoid the same resource_type in multiple module graphs
|
184
|
+
modules = default_module_configs()
|
185
|
+
for name, module_config in modules.root.items():
|
186
|
+
internal_graph, external_graph = create_module_graphs(
|
187
|
+
atlas_graph,
|
188
|
+
module_config,
|
189
|
+
color_coder_internal=color_coder(atlas_graph, keep_provider_name=False),
|
190
|
+
color_coder_external=color_coder(atlas_graph, keep_provider_name=True),
|
191
|
+
used_resource_types=used_resource_types,
|
192
|
+
)
|
193
|
+
module_tree = tree.add(module_config.tree_label)
|
194
|
+
module_trees: dict[str, Tree] = {
|
195
|
+
resource_type: module_tree.add(remove_provider_name(resource_type))
|
196
|
+
for resource_type in module_config.root_resource_types
|
197
|
+
}
|
198
|
+
|
199
|
+
def get_tree(resource_type: str) -> Tree | None:
|
200
|
+
return next(
|
201
|
+
tree
|
202
|
+
for src, tree in module_trees.items()
|
203
|
+
if src.endswith(resource_type) # provider name might be removed
|
204
|
+
)
|
205
|
+
|
206
|
+
def prefer_root_src_over_nested(src_dest: tuple[str, str]) -> tuple[bool, str, str]:
|
207
|
+
src, dest = src_dest
|
208
|
+
is_root = any(root.endswith(src) for root in module_config.root_resource_types)
|
209
|
+
return (not is_root, src, dest) # sort by whether src is a
|
210
|
+
|
211
|
+
for src, dest in sorted(
|
212
|
+
(edge_src_dest(edge) for edge in internal_graph.get_edge_list()), key=prefer_root_src_over_nested
|
213
|
+
):
|
214
|
+
try:
|
215
|
+
tree_src = get_tree(src)
|
216
|
+
except StopIteration:
|
217
|
+
resource_type = next(
|
218
|
+
root
|
219
|
+
for root in module_config.root_resource_types + module_config.extra_nested_resource_types
|
220
|
+
if root.endswith(src) # provider name might be removed
|
221
|
+
)
|
222
|
+
tree_src = module_tree.add(src)
|
223
|
+
module_trees[resource_type] = tree_src
|
224
|
+
assert tree_src is not None, f"Source {src} not found in module tree"
|
225
|
+
module_trees[dest] = tree_src.add(dest)
|
226
|
+
write_graph(internal_graph, settings.static_root, f"{name}_internal.png")
|
227
|
+
write_graph(external_graph, settings.static_root, f"{name}_external.png")
|
228
|
+
print_to_live(tree)
|
229
|
+
return modules
|
230
|
+
|
231
|
+
|
232
|
+
def parse_atlas_graph(settings: TfDepSettings) -> AtlasGraph:
|
233
|
+
atlas_graph = parse_model(settings.atlas_graph_path, t=AtlasGraph)
|
234
|
+
deprecated_resources = parse_list(settings.schema_resource_types_deprecated_path, format="yaml")
|
235
|
+
atlas_graph.deprecated_resource_types.update(deprecated_resources)
|
236
|
+
atlas_graph.parent_child_edges["mongodbatlas_project"].add("mongodbatlas_auditing")
|
237
|
+
atlas_graph.parent_child_edges["mongodbatlas_project"].add("mongodbatlas_custom_dns_configuration_cluster_aws")
|
238
|
+
atlas_graph.parent_child_edges["mongodbatlas_advanced_cluster"].add("mongodbatlas_global_cluster_config")
|
239
|
+
return atlas_graph
|
240
|
+
|
241
|
+
|
242
|
+
def add_unused_nodes_to_graph(
|
243
|
+
settings: TfDepSettings, atlas_graph: AtlasGraph, color_coder: ColorCoder, internal_graph: pydot.Dot
|
244
|
+
):
|
245
|
+
schema_resource_types: list[str] = parse_list(settings.schema_resource_types_path, format="yaml")
|
246
|
+
all_nodes = atlas_graph.all_internal_nodes
|
247
|
+
for resource_type in schema_resource_types:
|
248
|
+
if resource_type not in all_nodes:
|
249
|
+
internal_graph.add_node(color_coder.create_node(resource_type, is_unused=True))
|
250
|
+
|
251
|
+
|
252
|
+
class NodeSkippedError(Exception):
|
253
|
+
"""Raised when a node is skipped during graph creation."""
|
254
|
+
|
255
|
+
def __init__(self, resource_type: str):
|
256
|
+
self.resource_type = resource_type
|
257
|
+
super().__init__(f"Node skipped: {resource_type}. This is expected for some resource types.")
|
258
|
+
|
259
|
+
|
260
|
+
@dataclass
|
261
|
+
class ColorCoder:
|
262
|
+
graph: AtlasGraph
|
263
|
+
keep_provider_name: bool
|
264
|
+
|
265
|
+
ATLAS_EXTERNAL_COLOR: ClassVar[str] = "red"
|
266
|
+
ATLAS_INTERNAL_COLOR: ClassVar[str] = "green"
|
267
|
+
ATLAS_INTERNAL_UNUSED_COLOR: ClassVar[str] = "gray"
|
268
|
+
ATLAS_DEPRECATED_COLOR: ClassVar[str] = "orange"
|
269
|
+
EXTERNAL_COLOR: ClassVar[str] = "purple"
|
270
|
+
|
271
|
+
def create_node(self, resource_type: str, *, is_unused: bool = False) -> pydot.Node:
|
272
|
+
if resource_type in self.graph.deprecated_resource_types:
|
273
|
+
color = self.ATLAS_DEPRECATED_COLOR
|
274
|
+
elif is_unused:
|
275
|
+
color = self.ATLAS_INTERNAL_UNUSED_COLOR
|
276
|
+
elif resource_type.startswith(ATLAS_PROVIDER_NAME):
|
277
|
+
color = (
|
278
|
+
"red"
|
279
|
+
if resource_type in self.graph.all_external_nodes and resource_type not in FORCE_INTERNAL_NODES
|
280
|
+
else "green"
|
281
|
+
)
|
282
|
+
else:
|
283
|
+
color = self.EXTERNAL_COLOR
|
284
|
+
return pydot.Node(self.node_name(resource_type), shape="box", style="filled", fillcolor=color)
|
285
|
+
|
286
|
+
def node_name(self, resource_type: str) -> str:
|
287
|
+
if resource_type in SKIP_NODES:
|
288
|
+
raise NodeSkippedError(resource_type)
|
289
|
+
return resource_type if self.keep_provider_name else remove_provider_name(resource_type)
|
290
|
+
|
291
|
+
|
292
|
+
def color_coder(atlas_graph: AtlasGraph, keep_provider_name: bool = False) -> ColorCoder:
|
293
|
+
return ColorCoder(atlas_graph, keep_provider_name=keep_provider_name)
|
294
|
+
|
295
|
+
|
296
|
+
@dataclass
|
297
|
+
class ModuleColorCoder(ColorCoder):
|
298
|
+
modules: ModuleConfigs
|
299
|
+
|
300
|
+
def node_name(self, resource_type: str) -> str:
|
301
|
+
if emoji_prefix := self.modules.module_emoji_prefix(resource_type):
|
302
|
+
return f"{emoji_prefix} {super().node_name(resource_type)}"
|
303
|
+
return super().node_name(resource_type)
|
304
|
+
|
305
|
+
|
306
|
+
def remove_provider_name(resource_type: str) -> str:
|
307
|
+
return resource_type.split("_", 1)[-1]
|
308
|
+
|
309
|
+
|
310
|
+
def write_graph(dot_graph: pydot.Dot, out_path: Path, filename: str):
|
311
|
+
out_path.mkdir(parents=True, exist_ok=True)
|
312
|
+
dot_graph.write_png(out_path / filename) # type: ignore
|
313
|
+
|
314
|
+
|
315
|
+
def as_nodes(edges: Iterable[tuple[str, str]]) -> set[str]:
|
316
|
+
return set(flat_map((parent, child) for parent, child in edges))
|
317
|
+
|
318
|
+
|
319
|
+
def create_dot_graph(name: str, edges: Iterable[tuple[str, str]], *, color_coder: ColorCoder) -> pydot.Dot:
|
320
|
+
edges = sorted(edges)
|
321
|
+
graph = pydot.Dot(name, graph_type="graph")
|
322
|
+
nodes = as_nodes(edges)
|
323
|
+
for node in nodes:
|
324
|
+
try:
|
325
|
+
graph.add_node(color_coder.create_node(node))
|
326
|
+
except NodeSkippedError:
|
327
|
+
continue
|
328
|
+
for src, dst in edges:
|
329
|
+
try:
|
330
|
+
graph.add_edge(pydot.Edge(color_coder.node_name(src), color_coder.node_name(dst), color="blue"))
|
331
|
+
except NodeSkippedError:
|
332
|
+
continue
|
333
|
+
return graph
|
334
|
+
|
335
|
+
|
336
|
+
def create_module_graphs(
|
337
|
+
atlas_graph: AtlasGraph,
|
338
|
+
module_config: ModuleConfig,
|
339
|
+
*,
|
340
|
+
color_coder_internal: ColorCoder,
|
341
|
+
color_coder_external: ColorCoder,
|
342
|
+
used_resource_types: set[str],
|
343
|
+
) -> tuple[pydot.Dot, pydot.Dot]:
|
344
|
+
used_resource_types = used_resource_types or set()
|
345
|
+
"""Create two graphs: one for internal-only module dependencies and one for all module dependencies."""
|
346
|
+
child_edges = [
|
347
|
+
(root_resource_type, child)
|
348
|
+
for root_resource_type in module_config.root_resource_types
|
349
|
+
for child in atlas_graph.parent_child_edges.get(root_resource_type, [])
|
350
|
+
if child not in used_resource_types
|
351
|
+
]
|
352
|
+
child_edges.extend(
|
353
|
+
(nested_resource_type, child)
|
354
|
+
for nested_resource_type in module_config.extra_nested_resource_types
|
355
|
+
for child in atlas_graph.parent_child_edges.get(nested_resource_type, [])
|
356
|
+
if child not in used_resource_types
|
357
|
+
)
|
358
|
+
internal_only_edges = [
|
359
|
+
(resource_type, child)
|
360
|
+
for resource_type, child in child_edges
|
361
|
+
if module_config.include_child(child, atlas_graph)
|
362
|
+
]
|
363
|
+
module_name = module_config.name
|
364
|
+
internal_graph = create_dot_graph(
|
365
|
+
f"{module_name} Internal Only Dependencies",
|
366
|
+
internal_only_edges,
|
367
|
+
color_coder=color_coder_internal,
|
368
|
+
)
|
369
|
+
external_edges = [
|
370
|
+
(parent, child)
|
371
|
+
for child, parents in atlas_graph.external_parents.items()
|
372
|
+
if child in child_edges
|
373
|
+
for parent in parents
|
374
|
+
]
|
375
|
+
external_graph = create_dot_graph(
|
376
|
+
f"{module_name} External Dependencies",
|
377
|
+
child_edges + external_edges,
|
378
|
+
color_coder=color_coder_external,
|
379
|
+
)
|
380
|
+
used_resource_types.update(module_config.root_resource_types) # in case a root_resource_type doesn't have children
|
381
|
+
used_resource_types |= as_nodes(internal_only_edges)
|
382
|
+
return internal_graph, external_graph
|
383
|
+
|
384
|
+
|
385
|
+
def create_internal_dependencies(atlas_graph: AtlasGraph, color_coder: ColorCoder) -> pydot.Dot:
|
386
|
+
graph_name = "Atlas Internal Dependencies"
|
387
|
+
return create_dot_graph(graph_name, atlas_graph.iterate_internal_edges(), color_coder=color_coder)
|
388
|
+
|
389
|
+
|
390
|
+
def create_external_dependencies(atlas_graph: AtlasGraph) -> pydot.Dot:
|
391
|
+
graph_name = "Atlas External Dependencies"
|
392
|
+
return create_dot_graph(
|
393
|
+
graph_name, atlas_graph.iterate_external_edges(), color_coder=color_coder(atlas_graph, keep_provider_name=True)
|
394
|
+
)
|
@@ -0,0 +1,173 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import logging
|
4
|
+
from pathlib import Path
|
5
|
+
from typing import ClassVar, NamedTuple
|
6
|
+
|
7
|
+
from ask_shell import new_task, run_and_wait
|
8
|
+
from model_lib import IgnoreFalsy, dump
|
9
|
+
from pydantic import Field, RootModel
|
10
|
+
from zero_3rdparty.file_utils import ensure_parents_write_text
|
11
|
+
from zero_3rdparty.str_utils import instance_repr
|
12
|
+
|
13
|
+
from atlas_init.tf_ext.args import REPO_PATH_ARG, SKIP_EXAMPLES_DIRS_OPTION
|
14
|
+
from atlas_init.tf_ext.constants import ATLAS_PROVIDER_NAME
|
15
|
+
from atlas_init.tf_ext.paths import (
|
16
|
+
ResourceTypes,
|
17
|
+
find_resource_types_with_usages,
|
18
|
+
find_variables,
|
19
|
+
get_example_directories,
|
20
|
+
is_variable_name_external,
|
21
|
+
)
|
22
|
+
from atlas_init.tf_ext.settings import TfDepSettings
|
23
|
+
|
24
|
+
logger = logging.getLogger(__name__)
|
25
|
+
|
26
|
+
|
27
|
+
def tf_vars(
|
28
|
+
repo_path: Path = REPO_PATH_ARG,
|
29
|
+
skip_names: list[str] = SKIP_EXAMPLES_DIRS_OPTION,
|
30
|
+
):
|
31
|
+
settings = TfDepSettings.from_env()
|
32
|
+
logger.info(f"Analyzing Terraform variables in repository: {repo_path}")
|
33
|
+
example_dirs = get_example_directories(repo_path, skip_names)
|
34
|
+
assert example_dirs, "No example directories found. Please check the repository path and skip names."
|
35
|
+
with new_task("Parsing provider schema") as task:
|
36
|
+
resource_types, resource_types_deprecated = parse_schema_resource_types(example_dirs[0])
|
37
|
+
ensure_parents_write_text(settings.schema_resource_types_path, dump(sorted(resource_types), format="yaml"))
|
38
|
+
logger.info(f"Provider schema resource types written to {settings.schema_resource_types_path}")
|
39
|
+
ensure_parents_write_text(
|
40
|
+
settings.schema_resource_types_deprecated_path, dump(sorted(resource_types_deprecated), format="yaml")
|
41
|
+
)
|
42
|
+
logger.info(
|
43
|
+
f"Provider schema deprecated resource types written to {settings.schema_resource_types_deprecated_path}"
|
44
|
+
)
|
45
|
+
logger.info(f"Found {len(resource_types)} resource types in the provider schema.: {', '.join(resource_types)}")
|
46
|
+
with new_task("Parsing variables from examples") as task:
|
47
|
+
update_variables(settings, example_dirs, task)
|
48
|
+
with new_task("Parsing resource types from examples", total=len(example_dirs)) as task:
|
49
|
+
example_resource_types = update_resource_types(settings, example_dirs, task)
|
50
|
+
if missing_example_resource_types := set(resource_types) - set(example_resource_types.root):
|
51
|
+
logger.warning(f"Missing resource types in examples:\n{'\n'.join(sorted(missing_example_resource_types))}")
|
52
|
+
|
53
|
+
|
54
|
+
def parse_provider_resurce_schema(schema: dict, provider_name: str) -> dict:
|
55
|
+
schemas = schema.get("provider_schemas", {})
|
56
|
+
for provider_url, provider_schema in schemas.items():
|
57
|
+
if provider_url.endswith(provider_name):
|
58
|
+
return provider_schema.get("resource_schemas", {})
|
59
|
+
raise ValueError(f"Provider '{provider_name}' not found in schema.")
|
60
|
+
|
61
|
+
|
62
|
+
class TfVarUsage(IgnoreFalsy):
|
63
|
+
name: str = Field(..., description="Name of the Terraform variable.")
|
64
|
+
descriptions: set[str] = Field(default_factory=set, description="Set of descriptions for the variable.")
|
65
|
+
example_paths: list[Path] = Field(
|
66
|
+
default_factory=list, description="List of example files where the variable is used."
|
67
|
+
)
|
68
|
+
|
69
|
+
PARENT_DIR: ClassVar[str] = "examples"
|
70
|
+
|
71
|
+
def update(self, variable_description: str | None, example_dir: Path):
|
72
|
+
if variable_description and variable_description not in self.descriptions:
|
73
|
+
self.descriptions.add(variable_description)
|
74
|
+
assert f"/{self.PARENT_DIR}/" in str(example_dir), "Example directory must be under 'examples/'"
|
75
|
+
if example_dir not in self.example_paths:
|
76
|
+
self.example_paths.append(example_dir)
|
77
|
+
|
78
|
+
@property
|
79
|
+
def paths_str(self) -> str:
|
80
|
+
return ", ".join(str(path).split(self.PARENT_DIR)[1] for path in self.example_paths)
|
81
|
+
|
82
|
+
def __str__(self):
|
83
|
+
return instance_repr(self, ["name", "descriptions", "paths_str"])
|
84
|
+
|
85
|
+
def dump_dict_modifier(self, payload: dict) -> dict:
|
86
|
+
payload["descriptions"] = sorted(self.descriptions)
|
87
|
+
payload["example_paths"] = sorted(self.example_paths)
|
88
|
+
return payload
|
89
|
+
|
90
|
+
|
91
|
+
class TfVarsUsage(RootModel[dict[str, TfVarUsage]]):
|
92
|
+
def add_variable(self, variable: str, variable_description: str | None, example_dir: Path):
|
93
|
+
if variable not in self.root:
|
94
|
+
self.root[variable] = TfVarUsage(name=variable, example_paths=[])
|
95
|
+
self.root[variable].update(variable_description, example_dir)
|
96
|
+
|
97
|
+
def external_vars(self) -> TfVarsUsage:
|
98
|
+
return type(self)(root={name: usage for name, usage in self.root.items() if is_variable_name_external(name)})
|
99
|
+
|
100
|
+
|
101
|
+
def vars_usage_dumping(variables: TfVarsUsage) -> str:
|
102
|
+
vars_model = variables.model_dump()
|
103
|
+
vars_model = dict(sorted(vars_model.items()))
|
104
|
+
return dump(vars_model, format="yaml")
|
105
|
+
|
106
|
+
|
107
|
+
def update_resource_types(settings: TfDepSettings, example_dirs: list[Path], task: new_task) -> ResourceTypes:
|
108
|
+
resource_types = ResourceTypes(root={})
|
109
|
+
for example_dir in example_dirs:
|
110
|
+
example_resources = find_resource_types_with_usages(example_dir)
|
111
|
+
for resource_type, usages in example_resources.root.items():
|
112
|
+
resource_types.add_resource_type(resource_type, usages.example_files, usages.variable_usage)
|
113
|
+
task.update(advance=1)
|
114
|
+
logger.info(f"Found {len(resource_types.root)} resource types in the examples.")
|
115
|
+
resource_types_yaml = resource_types_dumping(resource_types)
|
116
|
+
ensure_parents_write_text(settings.resource_types_file_path, resource_types_yaml)
|
117
|
+
logger.info(f"Resource types usage written to {settings.resource_types_file_path}")
|
118
|
+
atlas_external_resource_types = resource_types.atlas_resource_type_with_external_var_usages()
|
119
|
+
logger.info(f"Found {len(atlas_external_resource_types.root)} Atlas resource types with external variable usages.")
|
120
|
+
atlas_external_resource_types_yaml = resource_types_dumping(atlas_external_resource_types, with_external=True)
|
121
|
+
ensure_parents_write_text(settings.resource_types_external_file_path, atlas_external_resource_types_yaml)
|
122
|
+
logger.info(
|
123
|
+
f"Atlas resource types with external variable usages written to {settings.resource_types_external_file_path}"
|
124
|
+
)
|
125
|
+
return resource_types
|
126
|
+
|
127
|
+
|
128
|
+
def resource_types_dumping(resource_types: ResourceTypes, with_external: bool = False) -> str:
|
129
|
+
resource_types_model = resource_types.dump_with_external_vars() if with_external else resource_types.model_dump()
|
130
|
+
return dump(dict(sorted(resource_types_model.items())), format="yaml")
|
131
|
+
|
132
|
+
|
133
|
+
def update_variables(settings: TfDepSettings, example_dirs: list[Path], task: new_task):
|
134
|
+
variables = parse_all_variables(example_dirs, task)
|
135
|
+
logger.info(f"Found {len(variables.root)} variables in the examples.")
|
136
|
+
vars_yaml = vars_usage_dumping(variables)
|
137
|
+
ensure_parents_write_text(settings.vars_file_path, vars_yaml)
|
138
|
+
logger.info(f"Variables usage written to {settings.vars_file_path}")
|
139
|
+
external_vars = variables.external_vars()
|
140
|
+
if external_vars.root:
|
141
|
+
logger.info(f"Found {len(external_vars.root)} external variables: {', '.join(external_vars.root.keys())}")
|
142
|
+
external_vars_yaml = vars_usage_dumping(external_vars)
|
143
|
+
ensure_parents_write_text(settings.vars_external_file_path, external_vars_yaml)
|
144
|
+
logger.info(f"External variables usage written to {settings.vars_external_file_path}")
|
145
|
+
|
146
|
+
|
147
|
+
class ResourceTypesSchema(NamedTuple):
|
148
|
+
resource_types: list[str]
|
149
|
+
deprecated_resource_types: list[str]
|
150
|
+
|
151
|
+
|
152
|
+
def parse_schema_resource_types(example_dir: Path) -> ResourceTypesSchema:
|
153
|
+
schema_run = run_and_wait("terraform providers schema -json", cwd=example_dir, ansi_content=False)
|
154
|
+
parsed = schema_run.parse_output(dict, output_format="json")
|
155
|
+
resource_schema = parse_provider_resurce_schema(parsed, ATLAS_PROVIDER_NAME)
|
156
|
+
|
157
|
+
def is_deprecated(resource_details: dict) -> bool:
|
158
|
+
return resource_details["block"].get("deprecated", False)
|
159
|
+
|
160
|
+
deprecated_resource_types = [name for name, details in resource_schema.items() if is_deprecated(details)]
|
161
|
+
return ResourceTypesSchema(sorted(resource_schema.keys()), sorted(deprecated_resource_types))
|
162
|
+
|
163
|
+
|
164
|
+
def parse_all_variables(examples_dirs: list[Path], task: new_task) -> TfVarsUsage:
|
165
|
+
variables_usage = TfVarsUsage(root={})
|
166
|
+
for example_dir in examples_dirs:
|
167
|
+
variables_tf = example_dir / "variables.tf"
|
168
|
+
if not variables_tf.exists():
|
169
|
+
continue
|
170
|
+
for variable, variable_desc in find_variables(variables_tf).items():
|
171
|
+
variables_usage.add_variable(variable, variable_desc, example_dir)
|
172
|
+
task.update(advance=1)
|
173
|
+
return variables_usage
|
@@ -0,0 +1,24 @@
|
|
1
|
+
from ask_shell import configure_logging
|
2
|
+
from typer import Typer
|
3
|
+
|
4
|
+
from atlas_init.tf_ext import api_call
|
5
|
+
|
6
|
+
|
7
|
+
def typer_main():
|
8
|
+
from atlas_init.tf_ext import tf_dep, tf_modules, tf_vars
|
9
|
+
|
10
|
+
app = Typer(
|
11
|
+
name="tf-ext",
|
12
|
+
help="Terraform extension commands for Atlas Init",
|
13
|
+
)
|
14
|
+
app.command(name="dep-graph")(tf_dep.tf_dep_graph)
|
15
|
+
app.command(name="vars")(tf_vars.tf_vars)
|
16
|
+
app.command(name="modules")(tf_modules.tf_modules)
|
17
|
+
app.command(name="api")(api_call.api)
|
18
|
+
app.command(name="api-config")(api_call.api_config)
|
19
|
+
configure_logging(app)
|
20
|
+
app()
|
21
|
+
|
22
|
+
|
23
|
+
if __name__ == "__main__":
|
24
|
+
typer_main()
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: atlas-init
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.7.0
|
4
4
|
Project-URL: Documentation, https://github.com/EspenAlbert/atlas-init#readme
|
5
5
|
Project-URL: Issues, https://github.com/EspenAlbert/atlas-init/issues
|
6
6
|
Project-URL: Source, https://github.com/EspenAlbert/atlas-init
|
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.13
|
13
13
|
Requires-Python: >=3.13
|
14
14
|
Requires-Dist: appdirs==1.4.4
|
15
|
-
Requires-Dist: ask-shell>=0.0.
|
15
|
+
Requires-Dist: ask-shell>=0.0.4
|
16
16
|
Requires-Dist: boto3==1.35.92
|
17
17
|
Requires-Dist: gitpython==3.1.42
|
18
18
|
Requires-Dist: humanize==4.9.0
|
@@ -21,6 +21,7 @@ Requires-Dist: motor==3.7.1
|
|
21
21
|
Requires-Dist: mypy-boto3-cloudformation==1.37.22
|
22
22
|
Requires-Dist: orjson==3.10.13
|
23
23
|
Requires-Dist: pydantic-settings==2.7.1
|
24
|
+
Requires-Dist: pydot==4.0.1
|
24
25
|
Requires-Dist: pygithub==2.6.1
|
25
26
|
Requires-Dist: python-hcl2==7.1.0
|
26
27
|
Requires-Dist: questionary==2.1.0
|