atlas-init 0.4.5__py3-none-any.whl → 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. atlas_init/__init__.py +1 -1
  2. atlas_init/cli.py +2 -0
  3. atlas_init/cli_args.py +19 -1
  4. atlas_init/cli_cfn/cfn_parameter_finder.py +59 -51
  5. atlas_init/cli_cfn/example.py +8 -16
  6. atlas_init/cli_helper/go.py +6 -10
  7. atlas_init/cli_root/mms_released.py +46 -0
  8. atlas_init/cli_tf/app.py +3 -84
  9. atlas_init/cli_tf/ci_tests.py +585 -0
  10. atlas_init/cli_tf/codegen/__init__.py +0 -0
  11. atlas_init/cli_tf/codegen/models.py +97 -0
  12. atlas_init/cli_tf/codegen/openapi_minimal.py +74 -0
  13. atlas_init/cli_tf/github_logs.py +7 -94
  14. atlas_init/cli_tf/go_test_run.py +395 -130
  15. atlas_init/cli_tf/go_test_summary.py +589 -10
  16. atlas_init/cli_tf/go_test_tf_error.py +388 -0
  17. atlas_init/cli_tf/hcl/modifier.py +14 -12
  18. atlas_init/cli_tf/hcl/modifier2.py +207 -0
  19. atlas_init/cli_tf/mock_tf_log.py +1 -1
  20. atlas_init/cli_tf/{schema_v2_api_parsing.py → openapi.py} +101 -19
  21. atlas_init/cli_tf/schema_v2.py +43 -1
  22. atlas_init/crud/__init__.py +0 -0
  23. atlas_init/crud/mongo_client.py +115 -0
  24. atlas_init/crud/mongo_dao.py +296 -0
  25. atlas_init/crud/mongo_utils.py +239 -0
  26. atlas_init/html_out/__init__.py +0 -0
  27. atlas_init/html_out/md_export.py +143 -0
  28. atlas_init/repos/go_sdk.py +12 -3
  29. atlas_init/repos/path.py +110 -7
  30. atlas_init/sdk_ext/__init__.py +0 -0
  31. atlas_init/sdk_ext/go.py +102 -0
  32. atlas_init/sdk_ext/typer_app.py +18 -0
  33. atlas_init/settings/config.py +3 -6
  34. atlas_init/settings/env_vars.py +18 -2
  35. atlas_init/settings/env_vars_generated.py +2 -0
  36. atlas_init/settings/interactive2.py +134 -0
  37. atlas_init/tf/.terraform.lock.hcl +59 -59
  38. atlas_init/tf/always.tf +5 -5
  39. atlas_init/tf/main.tf +3 -3
  40. atlas_init/tf/modules/aws_kms/aws_kms.tf +1 -1
  41. atlas_init/tf/modules/aws_s3/provider.tf +2 -1
  42. atlas_init/tf/modules/aws_vpc/provider.tf +2 -1
  43. atlas_init/tf/modules/cfn/cfn.tf +0 -8
  44. atlas_init/tf/modules/cfn/kms.tf +5 -5
  45. atlas_init/tf/modules/cfn/provider.tf +7 -0
  46. atlas_init/tf/modules/cfn/variables.tf +1 -1
  47. atlas_init/tf/modules/cloud_provider/cloud_provider.tf +1 -1
  48. atlas_init/tf/modules/cloud_provider/provider.tf +2 -1
  49. atlas_init/tf/modules/cluster/cluster.tf +31 -31
  50. atlas_init/tf/modules/cluster/provider.tf +2 -1
  51. atlas_init/tf/modules/encryption_at_rest/provider.tf +2 -1
  52. atlas_init/tf/modules/federated_vars/federated_vars.tf +2 -3
  53. atlas_init/tf/modules/federated_vars/provider.tf +2 -1
  54. atlas_init/tf/modules/project_extra/project_extra.tf +1 -10
  55. atlas_init/tf/modules/project_extra/provider.tf +8 -0
  56. atlas_init/tf/modules/stream_instance/provider.tf +8 -0
  57. atlas_init/tf/modules/stream_instance/stream_instance.tf +0 -9
  58. atlas_init/tf/modules/vpc_peering/provider.tf +10 -0
  59. atlas_init/tf/modules/vpc_peering/vpc_peering.tf +0 -10
  60. atlas_init/tf/modules/vpc_privatelink/versions.tf +2 -1
  61. atlas_init/tf/outputs.tf +1 -0
  62. atlas_init/tf/providers.tf +1 -1
  63. atlas_init/tf/variables.tf +7 -7
  64. atlas_init/tf_ext/__init__.py +0 -0
  65. atlas_init/tf_ext/__main__.py +3 -0
  66. atlas_init/tf_ext/api_call.py +325 -0
  67. atlas_init/tf_ext/args.py +17 -0
  68. atlas_init/tf_ext/constants.py +3 -0
  69. atlas_init/tf_ext/models.py +106 -0
  70. atlas_init/tf_ext/paths.py +126 -0
  71. atlas_init/tf_ext/settings.py +39 -0
  72. atlas_init/tf_ext/tf_dep.py +324 -0
  73. atlas_init/tf_ext/tf_modules.py +394 -0
  74. atlas_init/tf_ext/tf_vars.py +173 -0
  75. atlas_init/tf_ext/typer_app.py +24 -0
  76. atlas_init/typer_app.py +4 -8
  77. {atlas_init-0.4.5.dist-info → atlas_init-0.7.0.dist-info}/METADATA +8 -4
  78. atlas_init-0.7.0.dist-info/RECORD +138 -0
  79. atlas_init-0.7.0.dist-info/entry_points.txt +5 -0
  80. atlas_init-0.4.5.dist-info/RECORD +0 -105
  81. atlas_init-0.4.5.dist-info/entry_points.txt +0 -2
  82. {atlas_init-0.4.5.dist-info → atlas_init-0.7.0.dist-info}/WHEEL +0 -0
  83. {atlas_init-0.4.5.dist-info → atlas_init-0.7.0.dist-info}/licenses/LICENSE +0 -0
@@ -0,0 +1,388 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ from dataclasses import dataclass
5
+ from enum import StrEnum
6
+ from functools import total_ordering
7
+ from typing import ClassVar, Literal, NamedTuple, Self, TypeAlias
8
+
9
+ import humanize
10
+ from model_lib import Entity, utc_datetime_ms
11
+ from pydantic import Field, model_validator
12
+ from zero_3rdparty import iter_utils
13
+ from zero_3rdparty.datetime_utils import utc_now
14
+ from zero_3rdparty.str_utils import instance_repr
15
+
16
+ from atlas_init.cli_tf.go_test_run import GoTestRun
17
+ from atlas_init.repos.go_sdk import ApiSpecPaths
18
+
19
+
20
+ class GoTestErrorClass(StrEnum):
21
+ """Goal of each error class to be actionable."""
22
+
23
+ FLAKY_400 = "flaky_400"
24
+ FLAKY_500 = "flaky_500"
25
+ FLAKY_CHECK = "flaky_check"
26
+ FLAKY_CLIENT = "flaky_client"
27
+ OUT_OF_CAPACITY = "out_of_capacity"
28
+ PROJECT_LIMIT_EXCEEDED = "project_limit_exceeded"
29
+ DANGLING_RESOURCE = "dangling_resource"
30
+ REAL_TEST_FAILURE = "real_test_failure"
31
+ TIMEOUT = "timeout"
32
+ UNKNOWN = "unknown"
33
+ PROVIDER_DOWNLOAD = "provider_download"
34
+ UNCLASSIFIED = "unclassified"
35
+
36
+ __ACTIONS__ = {
37
+ FLAKY_400: "retry",
38
+ FLAKY_500: "retry",
39
+ FLAKY_CHECK: "retry",
40
+ FLAKY_CLIENT: "retry",
41
+ PROVIDER_DOWNLOAD: "retry",
42
+ OUT_OF_CAPACITY: "retry_later",
43
+ PROJECT_LIMIT_EXCEEDED: "clean_project",
44
+ DANGLING_RESOURCE: "update_cleanup_script",
45
+ REAL_TEST_FAILURE: "investigate",
46
+ TIMEOUT: "investigate",
47
+ UNKNOWN: "investigate",
48
+ }
49
+ __CONTAINS_MAPPING__ = {
50
+ OUT_OF_CAPACITY: ("OUT_OF_CAPACITY",),
51
+ FLAKY_500: ("HTTP 500", "UNEXPECTED_ERROR", "503 Service Unavailable"),
52
+ FLAKY_CLIENT: ("dial tcp: lookup", "i/o timeout"),
53
+ PROVIDER_DOWNLOAD: [
54
+ "mongodbatlas: failed to retrieve authentication checksums for provider",
55
+ "Error: Failed to install provider github.com: bad response",
56
+ ],
57
+ TIMEOUT: ("timeout while waiting for",),
58
+ }
59
+
60
+ @classmethod
61
+ def auto_classification(cls, output: str) -> GoTestErrorClass | None:
62
+ def contains(output: str, contains_part: str) -> bool:
63
+ if " " in contains_part:
64
+ return all(part in output for part in contains_part.split())
65
+ return contains_part in output
66
+
67
+ return next(
68
+ (
69
+ error_class
70
+ for error_class, contains_list in cls.__CONTAINS_MAPPING__.items()
71
+ if any(contains(output, contains_part) for contains_part in contains_list)
72
+ ),
73
+ None,
74
+ ) # type: ignore
75
+
76
+
77
+ API_METHODS = ["GET", "POST", "PUT", "DELETE", "PATCH"]
78
+
79
+
80
+ class GoTestAPIError(Entity):
81
+ type: Literal["api_error"] = "api_error"
82
+ api_error_code_str: str
83
+ api_path: str
84
+ api_method: Literal["GET", "POST", "PUT", "DELETE", "PATCH"]
85
+ api_response_code: int
86
+ tf_resource_name: str = ""
87
+ tf_resource_type: str = ""
88
+ step_nr: int = -1
89
+
90
+ api_path_normalized: str = Field(init=False, default="")
91
+
92
+ @model_validator(mode="after")
93
+ def strip_path_chars(self) -> GoTestAPIError:
94
+ self.api_path = self.api_path.rstrip(":/")
95
+ return self
96
+
97
+ def add_info_fields(self, info: DetailsInfo) -> None:
98
+ if api_paths := info.paths:
99
+ self.api_path_normalized = api_paths.normalize_path(self.api_method, self.api_path)
100
+
101
+ def __str__(self) -> str:
102
+ resource_part = f"{self.tf_resource_type} " if self.tf_resource_type else ""
103
+ if self.api_path_normalized:
104
+ return f"{resource_part}{self.api_error_code_str} {self.api_method} {self.api_path_normalized} {self.api_response_code}"
105
+ return f"{resource_part}{self.api_error_code_str} {self.api_method} {self.api_path} {self.api_response_code}"
106
+
107
+
108
+ @total_ordering
109
+ class CheckError(Entity):
110
+ attribute: str = ""
111
+ expected: str = ""
112
+ got: str = ""
113
+ check_nr: int = -1
114
+
115
+ def __lt__(self, other) -> bool:
116
+ if not isinstance(other, CheckError):
117
+ raise TypeError
118
+ return (self.check_nr, self.attribute) < (other.check_nr, other.attribute)
119
+
120
+ def __str__(self) -> str:
121
+ if self.attribute and self.expected and self.got:
122
+ return f"{self.check_nr}({self.attribute}:expected:{self.expected}, got: {self.got})"
123
+ return f"{self.check_nr}"
124
+
125
+ @classmethod
126
+ def parse_from_output(cls, output: str) -> list[Self]:
127
+ return [
128
+ cls(**check_match.groupdict()) # type: ignore
129
+ for check_match in check_pattern.finditer(output)
130
+ ]
131
+
132
+
133
+ class GoTestResourceCheckError(Entity):
134
+ type: Literal["check_error"] = "check_error"
135
+ tf_resource_name: str
136
+ tf_resource_type: str
137
+ step_nr: int = -1
138
+ check_errors: list[CheckError] = Field(default_factory=list)
139
+ test_name: str = ""
140
+
141
+ def add_info_fields(self, info: DetailsInfo) -> None:
142
+ self.test_name = info.run.name
143
+
144
+ def __str__(self) -> str:
145
+ return f"{self.tf_resource_type} {self.tf_resource_name} {self.step_nr} {self.check_errors}"
146
+
147
+ @property
148
+ def check_numbers_str(self) -> str:
149
+ return ",".join(str(check.check_nr) for check in sorted(self.check_errors))
150
+
151
+ def check_errors_match(self, other_check_errors: list[CheckError]) -> bool:
152
+ if len(self.check_errors) != len(other_check_errors):
153
+ return False
154
+ return all(
155
+ any(
156
+ check.check_nr == other_check.check_nr and check.attribute == other_check.attribute
157
+ for other_check in other_check_errors
158
+ )
159
+ for check in self.check_errors
160
+ )
161
+
162
+
163
+ class GoTestGeneralCheckError(Entity):
164
+ type: Literal["general_check_error"] = "general_check_error"
165
+ step_nr: int = -1
166
+ check_errors: list[CheckError] = Field(default_factory=list)
167
+ error_check_str: str
168
+ test_name: str = ""
169
+
170
+ def add_info_fields(self, info: DetailsInfo) -> None:
171
+ self.test_name = info.run.name
172
+
173
+ def check_errors_str(self) -> str:
174
+ return ",".join(str(check) for check in sorted(self.check_errors))
175
+
176
+ def __str__(self) -> str:
177
+ return f"Step {self.step_nr} {self.check_errors_str()}"
178
+
179
+
180
+ @dataclass
181
+ class DetailsInfo:
182
+ run: GoTestRun
183
+ paths: ApiSpecPaths | None = None
184
+
185
+
186
+ class GoTestDefaultError(Entity):
187
+ type: Literal["default_error"] = "default_error"
188
+ error_str: str
189
+
190
+ def add_info_fields(self, _: DetailsInfo) -> None:
191
+ pass
192
+
193
+
194
+ ErrorDetailsT: TypeAlias = GoTestAPIError | GoTestResourceCheckError | GoTestDefaultError | GoTestGeneralCheckError
195
+
196
+
197
+ class ErrorClassified(NamedTuple):
198
+ classified: dict[GoTestErrorClass, list[GoTestError]]
199
+ unclassified: list[GoTestError]
200
+
201
+
202
+ class ErrorClassAuthor(StrEnum):
203
+ AUTO = "auto"
204
+ HUMAN = "human"
205
+ LLM = "llm"
206
+ SIMILAR = "similar"
207
+
208
+
209
+ class GoTestErrorClassification(Entity):
210
+ error_class: GoTestErrorClass = GoTestErrorClass.UNCLASSIFIED
211
+ ts: utc_datetime_ms = Field(default_factory=utc_now)
212
+ author: ErrorClassAuthor
213
+ confidence: float = 0.0
214
+ test_output: str = ""
215
+ details: ErrorDetailsT
216
+ run_id: str
217
+ test_name: str
218
+
219
+ STR_COLUMNS: ClassVar[list[str]] = ["error_class", "author", "run_id", "confidence", "ts_when"]
220
+
221
+ def needs_classification(self, confidence_threshold: float = 1.0) -> bool:
222
+ return (
223
+ self.error_class in {GoTestErrorClass.UNCLASSIFIED, GoTestErrorClass.UNKNOWN}
224
+ or self.confidence < confidence_threshold
225
+ )
226
+
227
+ @property
228
+ def ts_when(self) -> str:
229
+ return humanize.naturaltime(self.ts)
230
+
231
+ def __str__(self) -> str:
232
+ return instance_repr(self, self.STR_COLUMNS)
233
+
234
+
235
+ @total_ordering
236
+ class GoTestError(Entity):
237
+ details: ErrorDetailsT
238
+ run: GoTestRun
239
+ bot_error_class: GoTestErrorClass = GoTestErrorClass.UNCLASSIFIED
240
+ human_error_class: GoTestErrorClass = GoTestErrorClass.UNCLASSIFIED
241
+
242
+ def __lt__(self, other) -> bool:
243
+ if not isinstance(other, GoTestError):
244
+ raise TypeError
245
+ return self.run < other.run
246
+
247
+ @property
248
+ def run_id(self) -> str:
249
+ return self.run.id
250
+
251
+ @property
252
+ def run_name(self) -> str:
253
+ return self.run.name
254
+
255
+ @property
256
+ def classifications(self) -> tuple[GoTestErrorClass, GoTestErrorClass] | None:
257
+ if (
258
+ self.bot_error_class != GoTestErrorClass.UNCLASSIFIED
259
+ and self.human_error_class != GoTestErrorClass.UNCLASSIFIED
260
+ ):
261
+ return self.bot_error_class, self.human_error_class
262
+ return None
263
+
264
+ def set_human_and_bot_classification(self, chosen_class: GoTestErrorClass) -> None:
265
+ self.human_error_class = chosen_class
266
+ self.bot_error_class = chosen_class
267
+
268
+ def match(self, other: GoTestError) -> bool:
269
+ if self.run.id == other.run.id:
270
+ return True
271
+ details = self.details
272
+ other_details = other.details
273
+ if type(self.details) is not type(other_details):
274
+ return False
275
+ if isinstance(details, GoTestAPIError):
276
+ assert isinstance(other_details, GoTestAPIError)
277
+ return (
278
+ details.api_path_normalized == other_details.api_path_normalized
279
+ and details.api_response_code == other_details.api_response_code
280
+ and details.api_method == other_details.api_method
281
+ and details.api_response_code == other_details.api_response_code
282
+ )
283
+ if isinstance(details, GoTestResourceCheckError):
284
+ assert isinstance(other_details, GoTestResourceCheckError)
285
+ return (
286
+ details.tf_resource_name == other_details.tf_resource_name
287
+ and details.tf_resource_type == other_details.tf_resource_type
288
+ and details.step_nr == other_details.step_nr
289
+ and details.check_numbers_str == other_details.check_numbers_str
290
+ )
291
+ return False
292
+
293
+ @classmethod
294
+ def group_by_classification(
295
+ cls, errors: list[GoTestError], *, classifier: Literal["bot", "human"] = "human"
296
+ ) -> ErrorClassified:
297
+ def get_classification(error: GoTestError) -> GoTestErrorClass:
298
+ if classifier == "bot":
299
+ return error.bot_error_class
300
+ return error.human_error_class
301
+
302
+ grouped_errors: dict[GoTestErrorClass, list[GoTestError]] = iter_utils.group_by_once(
303
+ errors, key=get_classification
304
+ )
305
+ unclassified = grouped_errors.pop(GoTestErrorClass.UNCLASSIFIED, [])
306
+ return ErrorClassified(grouped_errors, unclassified)
307
+
308
+ @classmethod
309
+ def group_by_name_with_package(cls, errors: list[GoTestError]) -> dict[str, list[GoTestError]]:
310
+ def by_name(error: GoTestError) -> str:
311
+ return error.run.name_with_package
312
+
313
+ return iter_utils.group_by_once(errors, key=by_name)
314
+
315
+ @property
316
+ def short_description(self) -> str:
317
+ details = self.details
318
+ return details_short_description(details) if details else ""
319
+
320
+ def header(self, use_ticks: bool = False) -> str:
321
+ name_with_ticks = f"`{self.run.name_with_package}`" if use_ticks else self.run.name_with_package
322
+ if details := self.short_description:
323
+ return f"{name_with_ticks} {details}"
324
+ return f"{name_with_ticks}"
325
+
326
+
327
+ def details_short_description(details: ErrorDetailsT) -> str:
328
+ match details:
329
+ case GoTestGeneralCheckError():
330
+ return str(details)
331
+ case GoTestResourceCheckError():
332
+ return f"CheckFailure for {details.tf_resource_type}.{details.tf_resource_name} at Step: {details.step_nr} Checks: {details.check_numbers_str}"
333
+ case GoTestAPIError(api_path_normalized=api_path_normalized) if api_path_normalized:
334
+ return f"API Error {details.api_error_code_str} {api_path_normalized}"
335
+ case GoTestAPIError(api_path=api_path):
336
+ return f"{details.api_error_code_str} {api_path}"
337
+ return ""
338
+
339
+
340
+ one_of_methods = "|".join(API_METHODS)
341
+
342
+
343
+ check_pattern_str = r"Check (?P<check_nr>\d+)/\d+"
344
+ check_pattern = re.compile(check_pattern_str)
345
+ url_pattern = r"https://cloud(-dev|-qa)?\.mongodb\.com(?P<api_path>\S+)"
346
+ error_check_pattern = re.compile(check_pattern_str + r"\s+error:\s(?P<error_check_str>.+)$", re.MULTILINE)
347
+ detail_patterns: list[re.Pattern] = [
348
+ re.compile(r"Step (?P<step_nr>\d+)/\d+"),
349
+ check_pattern,
350
+ re.compile(r"mongodbatlas_(?P<tf_resource_type>[^\.]+)\.(?P<tf_resource_name>[\w_-]+)"),
351
+ re.compile(rf"(?P<api_method>{one_of_methods})" + r": HTTP (?P<api_response_code>\d+)"),
352
+ re.compile(r'Error code: "(?P<api_error_code_str>[^"]+)"'),
353
+ re.compile(url_pattern),
354
+ ]
355
+
356
+ # Error: error creating MongoDB Cluster: POST https://cloud-dev.mongodb.com/api/atlas/v1.0/groups/680ecbc7122f5b15cc627ba5/clusters: 409 (request "OUT_OF_CAPACITY") The requested region is currently out of capacity for the requested instance size.
357
+ api_error_pattern_missing_details = re.compile(
358
+ rf"(?P<api_method>{one_of_methods})\s+"
359
+ + url_pattern
360
+ + r'\s+(?P<api_response_code>\d+)\s\(request\s"(?P<api_error_code_str>[^"]+)"\)'
361
+ )
362
+
363
+
364
+ def parse_error_details(run: GoTestRun) -> ErrorDetailsT:
365
+ kwargs = {}
366
+ output = run.output_lines_str
367
+ for pattern in detail_patterns:
368
+ if pattern_match := pattern.search(output):
369
+ kwargs |= pattern_match.groupdict()
370
+ match kwargs:
371
+ case {"api_path": _, "api_error_code_str": _}:
372
+ return GoTestAPIError(**kwargs)
373
+ case {"api_path": _} if pattern_match := api_error_pattern_missing_details.search(output):
374
+ kwargs |= pattern_match.groupdict()
375
+ return GoTestAPIError(**kwargs)
376
+ case {"check_nr": _} if all(name in kwargs for name in ("tf_resource_name", "tf_resource_type")):
377
+ kwargs.pop("check_nr")
378
+ check_errors = CheckError.parse_from_output(output)
379
+ return GoTestResourceCheckError(**kwargs, check_errors=check_errors)
380
+ case {"check_nr": _}:
381
+ if error_check_match := error_check_pattern.search(output):
382
+ kwargs.pop("check_nr")
383
+ check_errors = CheckError.parse_from_output(output)
384
+ return GoTestGeneralCheckError(
385
+ **kwargs, error_check_str=error_check_match.group("error_check_str"), check_errors=check_errors
386
+ )
387
+ kwargs.pop("error_check_str", None) # Remove if it was not matched
388
+ return GoTestDefaultError(error_str=run.output_lines_str)
@@ -5,7 +5,9 @@ from pathlib import Path
5
5
  from typing import Callable
6
6
 
7
7
  import hcl2
8
- from lark import Token, Tree, UnexpectedToken
8
+ from lark import Token, Tree
9
+
10
+ from atlas_init.cli_tf.hcl.modifier2 import safe_parse
9
11
 
10
12
  logger = logging.getLogger(__name__)
11
13
 
@@ -14,10 +16,14 @@ BLOCK_TYPE_OUTPUT = "output"
14
16
 
15
17
 
16
18
  def process_token(node: Token, indent=0):
17
- logger.debug(f"[{indent}] (token)\t|", " " * indent, node.type, node.value)
19
+ debug_log(f"token:{node.type}:{node.value}", indent)
18
20
  return deepcopy(node)
19
21
 
20
22
 
23
+ def debug_log(message: str, depth=0):
24
+ logger.debug(" " * depth + message.rstrip("\n"))
25
+
26
+
21
27
  def is_identifier_block_type(tree: Tree | Token, block_type: str) -> bool:
22
28
  if not isinstance(tree, Tree):
23
29
  return False
@@ -43,7 +49,7 @@ def update_description(tree: Tree, new_descriptions: dict[str, str], existing_na
43
49
  existing_names[name].append(old_description)
44
50
  new_description = new_descriptions.get(name, "")
45
51
  if not new_description:
46
- logger.debug(f"no description found for variable {name}")
52
+ debug_log(f"no description found for variable {name}", 0)
47
53
  return tree
48
54
  new_children[2] = update_body_with_description(variable_body, new_description)
49
55
  return Tree(tree.data, new_children)
@@ -112,7 +118,7 @@ def process_generic(
112
118
  depth=0,
113
119
  ):
114
120
  new_children = []
115
- logger.debug(f"[{depth}] (tree)\t|", " " * depth, node.data)
121
+ debug_log(f"tree:{node.data}", depth)
116
122
  for child in node.children:
117
123
  if isinstance(child, Tree):
118
124
  if tree_match(child):
@@ -146,10 +152,8 @@ def process_descriptions(
146
152
 
147
153
 
148
154
  def update_descriptions(tf_path: Path, new_names: dict[str, str], block_type: str) -> tuple[str, dict[str, list[str]]]:
149
- try:
150
- tree = hcl2.parses(tf_path.read_text()) # type: ignore
151
- except UnexpectedToken as e:
152
- logger.warning(f"failed to parse {tf_path}: {e}")
155
+ tree = safe_parse(tf_path)
156
+ if tree is None:
153
157
  return "", {}
154
158
  existing_descriptions = defaultdict(list)
155
159
  new_tree = process_descriptions(
@@ -210,10 +214,8 @@ def _read_object_elem_key(tree_body: Tree) -> str:
210
214
 
211
215
 
212
216
  def read_block_attribute_object_keys(tf_path: Path, block_type: str, block_name: str, block_key: str) -> list[str]:
213
- try:
214
- tree = hcl2.parses(tf_path.read_text()) # type: ignore
215
- except UnexpectedToken as e:
216
- logger.warning(f"failed to parse {tf_path}: {e}")
217
+ tree = safe_parse(tf_path)
218
+ if tree is None:
217
219
  return []
218
220
  env_vars = []
219
221
 
@@ -0,0 +1,207 @@
1
+ from collections import defaultdict
2
+ import logging
3
+ from contextlib import suppress
4
+ from pathlib import Path
5
+ from typing import NamedTuple
6
+ from lark import Token, Transformer, Tree, UnexpectedToken, v_args
7
+ from hcl2.transformer import Attribute, DictTransformer
8
+ from hcl2.api import reverse_transform, writes, parses
9
+ import rich
10
+
11
+ logger = logging.getLogger(__name__)
12
+
13
+
14
+ def update_attribute_object_str_value_for_block(
15
+ tree: Tree, block_name: str, block_transformer: DictTransformer
16
+ ) -> Tree:
17
+ class BlockUpdater(Transformer):
18
+ @v_args(tree=True)
19
+ def block(self, block_tree: Tree) -> Tree:
20
+ current_block_name = _identifier_name(block_tree)
21
+ if current_block_name == block_name:
22
+ tree_dict = block_transformer.transform(tree)
23
+ tree_modified = reverse_transform(tree_dict)
24
+ assert isinstance(tree_modified, Tree)
25
+ body_tree = tree_modified.children[0]
26
+ assert isinstance(body_tree, Tree)
27
+ block_tree = body_tree.children[0]
28
+ assert isinstance(block_tree, Tree)
29
+ return block_tree
30
+ return block_tree
31
+
32
+ return BlockUpdater().transform(tree)
33
+
34
+
35
+ class AttributeChange(NamedTuple):
36
+ attribute_name: str
37
+ old_value: str | None
38
+ new_value: str
39
+
40
+
41
+ def attribute_transfomer(attr_name: str, obj_key: str, new_value: str) -> tuple[DictTransformer, list[AttributeChange]]:
42
+ changes: list[AttributeChange] = []
43
+
44
+ class AttributeTransformer(DictTransformer):
45
+ def attribute(self, args: list) -> Attribute:
46
+ found_attribute = super().attribute(args)
47
+ if found_attribute.key == attr_name:
48
+ attribute_value = found_attribute.value
49
+ if not isinstance(attribute_value, dict):
50
+ raise ValueError(f"Expected a dict for attribute {attr_name}, but got {type(attribute_value)}")
51
+ old_value = attribute_value.get(obj_key)
52
+ if old_value == new_value:
53
+ return found_attribute
54
+ changes.append(AttributeChange(attr_name, old_value, new_value))
55
+ return Attribute(attr_name, found_attribute.value | {obj_key: new_value})
56
+ return found_attribute
57
+
58
+ return AttributeTransformer(with_meta=True), changes
59
+
60
+
61
+ def variable_reader(tree: Tree) -> dict[str, str | None]:
62
+ """
63
+ Reads the variable names from a parsed HCL2 tree.
64
+ Returns a variable_name -> description, None if no description is found.
65
+ """
66
+ variables: dict[str, str | None] = {}
67
+
68
+ class DescriptionReader(DictTransformer):
69
+ def __init__(self, with_meta: bool = False, *, name: str):
70
+ super().__init__(with_meta)
71
+ self.name = name
72
+ self.description: str | None = None
73
+
74
+ def attribute(self, args: list) -> Attribute:
75
+ name = args[0]
76
+ if name == "description":
77
+ description = _parse_attribute_value(args)
78
+ self.description = description
79
+ return super().attribute(args)
80
+
81
+ class BlockReader(Transformer):
82
+ @v_args(tree=True)
83
+ def block(self, block_tree: Tree) -> Tree:
84
+ current_block_name = _identifier_name(block_tree)
85
+ if current_block_name == "variable":
86
+ variable_name = token_name(block_tree.children[1])
87
+ reader = DescriptionReader(name=variable_name)
88
+ reader.transform(block_tree)
89
+ variables[variable_name] = reader.description
90
+ return block_tree
91
+
92
+ BlockReader().transform(tree)
93
+ return variables
94
+
95
+
96
+ def _parse_attribute_value(args: list) -> str:
97
+ description = args[-1]
98
+ return token_name(description) if isinstance(description, Token) else description.strip('"')
99
+
100
+
101
+ def resource_types_vars_usage(tree: Tree) -> dict[str, dict[str, str]]:
102
+ """
103
+ Reads the resource types and their variable usages from a parsed HCL2 tree.
104
+ Returns a dictionary where keys are resource type names and values are dictionaries
105
+ of variable names and the attribute paths they are used in.
106
+ """
107
+ resource_types: dict[str, dict[str, str]] = defaultdict(dict)
108
+
109
+ class ResourceBlockAttributeReader(DictTransformer):
110
+ def __init__(self, with_meta: bool = False, resource_type: str = ""):
111
+ self.resource_type = resource_type
112
+ resource_types.setdefault(self.resource_type, {})
113
+ super().__init__(with_meta)
114
+
115
+ def attribute(self, args: list) -> Attribute:
116
+ try:
117
+ value = _parse_attribute_value(args)
118
+ except AttributeError:
119
+ return super().attribute(args)
120
+ if value.startswith("var."):
121
+ variable_name = value[4:]
122
+ resource_types[self.resource_type][variable_name] = args[0]
123
+ return super().attribute(args)
124
+
125
+ class BlockReader(Transformer):
126
+ @v_args(tree=True)
127
+ def block(self, block_tree: Tree) -> Tree:
128
+ block_resource_name = _block_resource_name(block_tree)
129
+ if block_resource_name is not None:
130
+ ResourceBlockAttributeReader(with_meta=True, resource_type=block_resource_name).transform(block_tree)
131
+ return block_tree
132
+
133
+ BlockReader().transform(tree)
134
+ return resource_types
135
+
136
+
137
+ def variable_usages(variable_names: set[str], tree: Tree) -> dict[str, set[str]]:
138
+ usages = defaultdict(set)
139
+ current_resource_type = None
140
+
141
+ class ResourceBlockAttributeReader(DictTransformer):
142
+ def attribute(self, args: list) -> Attribute:
143
+ attr_value = args[-1]
144
+ if isinstance(attr_value, str) and attr_value.startswith("var."):
145
+ variable_name = attr_value[4:]
146
+ if variable_name in variable_names:
147
+ assert current_resource_type is not None, "current_resource_type should not be None"
148
+ usages[variable_name].add(current_resource_type)
149
+ return super().attribute(args)
150
+
151
+ class BlockReader(Transformer):
152
+ @v_args(tree=True)
153
+ def block(self, block_tree: Tree) -> Tree:
154
+ block_resource_name = _block_resource_name(block_tree)
155
+ if block_resource_name is not None and block_resource_name.startswith("mongodbatlas_"):
156
+ nonlocal current_resource_type
157
+ current_resource_type = block_resource_name
158
+ ResourceBlockAttributeReader().transform(block_tree)
159
+ return block_tree
160
+
161
+ BlockReader().transform(tree)
162
+ return usages
163
+
164
+
165
+ def _identifier_name(tree: Tree) -> str | None:
166
+ with suppress(Exception):
167
+ identifier_tree = tree.children[0]
168
+ assert identifier_tree.data == "identifier"
169
+ name_token = identifier_tree.children[0]
170
+ assert isinstance(name_token, Token)
171
+ if name_token.type == "NAME":
172
+ return name_token.value
173
+
174
+
175
+ def _block_resource_name(tree: Tree) -> str | None:
176
+ block_name = _identifier_name(tree)
177
+ if block_name != "resource":
178
+ return None
179
+ token = tree.children[1]
180
+ return token_name(token)
181
+
182
+
183
+ def token_name(token):
184
+ assert isinstance(token, Token)
185
+ token_value = token.value
186
+ assert isinstance(token_value, str)
187
+ return token_value.strip('"')
188
+
189
+
190
+ def write_tree(tree: Tree) -> str:
191
+ return writes(tree)
192
+
193
+
194
+ def print_tree(path: Path) -> None:
195
+ tree = safe_parse(path)
196
+ if tree is None:
197
+ return
198
+ logger.info("=" * 10 + f"tree START of {path.parent.name}/{path.name}" + "=" * 10)
199
+ rich.print(tree)
200
+ logger.info("=" * 10 + f"tree END of {path.parent.name}/{path.name}" + "=" * 10)
201
+
202
+
203
+ def safe_parse(path: Path) -> Tree | None:
204
+ try:
205
+ return parses(path.read_text()) # type: ignore
206
+ except UnexpectedToken as e:
207
+ logger.warning(f"failed to parse {path}: {e}")
@@ -154,7 +154,7 @@ def is_cache_up_to_date(cache_path: Path, cache_ttl: int) -> bool:
154
154
  return False
155
155
 
156
156
 
157
- def resolve_admin_api_path(sdk_repo_path_str: str, sdk_branch: str, admin_api_path: str) -> Path:
157
+ def resolve_admin_api_path(sdk_repo_path_str: str = "", sdk_branch: str = "main", admin_api_path: str = "") -> Path:
158
158
  if admin_api_path:
159
159
  resolved_admin_api_path = Path(admin_api_path)
160
160
  if not resolved_admin_api_path.exists():