tgwrap 0.10.2__py3-none-any.whl → 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tgwrap/analyze.py +60 -16
- tgwrap/cli.py +23 -11
- tgwrap/main.py +165 -62
- {tgwrap-0.10.2.dist-info → tgwrap-0.11.0.dist-info}/METADATA +3 -1
- tgwrap-0.11.0.dist-info/RECORD +13 -0
- tgwrap-0.10.2.dist-info/RECORD +0 -13
- {tgwrap-0.10.2.dist-info → tgwrap-0.11.0.dist-info}/LICENSE +0 -0
- {tgwrap-0.10.2.dist-info → tgwrap-0.11.0.dist-info}/WHEEL +0 -0
- {tgwrap-0.10.2.dist-info → tgwrap-0.11.0.dist-info}/entry_points.txt +0 -0
tgwrap/analyze.py
CHANGED
@@ -9,7 +9,7 @@ import re
|
|
9
9
|
|
10
10
|
from .printer import Printer
|
11
11
|
|
12
|
-
def run_analyze(config, data, verbose=False):
|
12
|
+
def run_analyze(config, data, ignore_attributes, verbose=False):
|
13
13
|
""" Run the terrasafe validation and return the unauthorized deletions """
|
14
14
|
|
15
15
|
printer = Printer(verbose)
|
@@ -44,11 +44,29 @@ def run_analyze(config, data, verbose=False):
|
|
44
44
|
for criticallity, ts_level in ts_default_levels.items():
|
45
45
|
ts_config[ts_level] = [f"*{key}*" for key, value in config[criticallity].items() if value.get('terrasafe_level', ts_level) == ts_level]
|
46
46
|
|
47
|
+
ignorable_updates = []
|
48
|
+
for resource in detected_changes['updates']:
|
49
|
+
before = resource['change']['before']
|
50
|
+
after = resource['change']['after']
|
51
|
+
|
52
|
+
for i in ignore_attributes:
|
53
|
+
try:
|
54
|
+
before.pop(i)
|
55
|
+
except KeyError:
|
56
|
+
pass
|
57
|
+
try:
|
58
|
+
after.pop(i)
|
59
|
+
except KeyError:
|
60
|
+
pass
|
61
|
+
|
62
|
+
if before == after:
|
63
|
+
ignorable_updates.append(resource['address'])
|
64
|
+
|
47
65
|
for resource in detected_changes['deletions']:
|
48
66
|
resource_address = resource["address"]
|
49
67
|
# Deny has precendence over Allow!
|
50
68
|
if is_resource_match_any(resource_address, ts_config["unauthorized_deletion"]):
|
51
|
-
printer.verbose(f'Resource {resource_address}
|
69
|
+
printer.verbose(f'Resource {resource_address} cannot be destroyed for any reason')
|
52
70
|
elif is_resource_match_any(resource_address, ts_config["ignore_deletion"]):
|
53
71
|
continue
|
54
72
|
elif is_resource_recreate(resource) and is_resource_match_any(
|
@@ -69,8 +87,8 @@ def run_analyze(config, data, verbose=False):
|
|
69
87
|
|
70
88
|
if changes['unauthorized']:
|
71
89
|
printer.verbose("Unauthorized deletion detected for those resources:")
|
72
|
-
for
|
73
|
-
printer.verbose(f" - {
|
90
|
+
for resource in changes['unauthorized']:
|
91
|
+
printer.verbose(f" - {resource}")
|
74
92
|
printer.verbose("If you really want to delete those resources: comment it or export this environment variable:")
|
75
93
|
printer.verbose(f"export TERRASAFE_ALLOW_DELETION=\"{';'.join(changes['unauthorized'])}\"")
|
76
94
|
|
@@ -114,20 +132,26 @@ def run_analyze(config, data, verbose=False):
|
|
114
132
|
|
115
133
|
# now we have the proper lists, calculate the drifts
|
116
134
|
for key, value in {'deletions': 'delete', 'creations': 'create', 'updates': 'update'}.items():
|
117
|
-
|
135
|
+
|
136
|
+
for index, resource in enumerate(detected_changes[key]):
|
118
137
|
resource_address = resource["address"]
|
119
|
-
|
120
|
-
|
121
|
-
if
|
122
|
-
|
123
|
-
dd_class = resource_config[value]
|
124
|
-
changes['drifts'][dd_class] += 1
|
138
|
+
|
139
|
+
# updates might need to be ignored
|
140
|
+
if key == 'updates' and resource_address in ignorable_updates:
|
141
|
+
detected_changes[key].pop(index)
|
125
142
|
else:
|
126
|
-
|
127
|
-
|
128
|
-
|
143
|
+
has_match, resource_config = get_matching_dd_config(resource_address, dd_config)
|
144
|
+
|
145
|
+
if has_match:
|
146
|
+
# so what drift classification do we have?
|
147
|
+
dd_class = resource_config[value]
|
148
|
+
changes['drifts'][dd_class] += 1
|
149
|
+
else:
|
150
|
+
changes['drifts']['unknown'] += 1
|
151
|
+
if resource_address not in changes['unknowns']:
|
152
|
+
changes['unknowns'].append(resource_address)
|
129
153
|
|
130
|
-
|
154
|
+
changes['drifts']['total'] += 1
|
131
155
|
|
132
156
|
# remove ballast from the following lists
|
133
157
|
changes['all'] = [ # ignore read and no-ops
|
@@ -140,9 +164,17 @@ def run_analyze(config, data, verbose=False):
|
|
140
164
|
]
|
141
165
|
changes['creations'] = [resource["address"] for resource in detected_changes['creations']]
|
142
166
|
changes['updates'] = [resource["address"] for resource in detected_changes['updates']]
|
167
|
+
changes['ignorable_updates'] = ignorable_updates
|
143
168
|
|
144
|
-
|
169
|
+
# see if there are output changes
|
170
|
+
output_changes = get_output_changes(data)
|
171
|
+
changes['outputs'] = []
|
172
|
+
relevant_changes = set(['create', 'update', 'delete'])
|
173
|
+
for k,v in output_changes.items():
|
174
|
+
if relevant_changes.intersection(v['actions']):
|
175
|
+
changes['outputs'].append(k)
|
145
176
|
|
177
|
+
return changes, (not changes['unauthorized'])
|
146
178
|
|
147
179
|
def parse_ignored_from_env_var():
|
148
180
|
ignored = os.environ.get("TERRASAFE_ALLOW_DELETION")
|
@@ -170,6 +202,18 @@ def get_resource_actions(data):
|
|
170
202
|
|
171
203
|
return changes
|
172
204
|
|
205
|
+
def get_output_changes(data):
|
206
|
+
# check format version
|
207
|
+
if data["format_version"].split(".")[0] != "0" and data["format_version"].split(".")[0] != "1":
|
208
|
+
raise Exception("Only format major version 0 or 1 is supported")
|
209
|
+
|
210
|
+
if "output_changes" in data:
|
211
|
+
output_changes = data["output_changes"]
|
212
|
+
else:
|
213
|
+
output_changes = {}
|
214
|
+
|
215
|
+
return output_changes
|
216
|
+
|
173
217
|
|
174
218
|
def has_delete_action(resource):
|
175
219
|
return "delete" in resource["change"]["actions"]
|
tgwrap/cli.py
CHANGED
@@ -120,7 +120,7 @@ def main():
|
|
120
120
|
help='dry-run mode, no real actions are executed (only in combination with step-by-step mode)',
|
121
121
|
show_default=True
|
122
122
|
)
|
123
|
-
@click.option('--no-lock', '-n', is_flag=True, default=
|
123
|
+
@click.option('--no-lock', '-n', is_flag=True, default=True,
|
124
124
|
help='Do not apply a lock while executing the command (or set the TGWRAP_NO_LOCK environment variable, only applicable with plan)',
|
125
125
|
envvar='TGWRAP_NO_LOCK', show_default=True,
|
126
126
|
)
|
@@ -189,7 +189,7 @@ def run(command, verbose, debug, dry_run, no_lock, update, upgrade,
|
|
189
189
|
help='dry-run mode, no real actions are executed (only in combination with step-by-step mode)',
|
190
190
|
show_default=True
|
191
191
|
)
|
192
|
-
@click.option('--no-lock', '-n', is_flag=True, default=
|
192
|
+
@click.option('--no-lock', '-n', is_flag=True, default=True,
|
193
193
|
help='Do not apply a lock while executing the command (or set the TGWRAP_NO_LOCK environment variable, only applicable with plan)',
|
194
194
|
envvar='TGWRAP_NO_LOCK', show_default=True,
|
195
195
|
)
|
@@ -335,7 +335,7 @@ def show(verbose, json, working_dir, planfile_dir, terragrunt_args):
|
|
335
335
|
@click.option('--working-dir', '-w', default=None,
|
336
336
|
help='Working directory, when omitted the current directory is used',
|
337
337
|
)
|
338
|
-
@click.option('--no-lock', '-n', is_flag=True, default=
|
338
|
+
@click.option('--no-lock', '-n', is_flag=True, default=True,
|
339
339
|
help='Do not apply a lock while executing the command (or set the TGWRAP_NO_LOCK environment variable, only applicable with plan)',
|
340
340
|
envvar='TGWRAP_NO_LOCK', show_default=True,
|
341
341
|
)
|
@@ -366,7 +366,7 @@ def run_import(address, id, verbose, dry_run, working_dir, no_lock, terragrunt_a
|
|
366
366
|
help='Verbose printing',
|
367
367
|
show_default=True
|
368
368
|
)
|
369
|
-
@click.option('--exclude-external-dependencies
|
369
|
+
@click.option('--exclude-external-dependencies', '-x',
|
370
370
|
is_flag=True, default=True,
|
371
371
|
help='Whether or not external dependencies must be ignored',
|
372
372
|
show_default=True
|
@@ -388,6 +388,11 @@ def run_import(address, id, verbose, dry_run, working_dir, no_lock, terragrunt_a
|
|
388
388
|
@click.option('--parallel-execution', '-p', is_flag=True, default=False,
|
389
389
|
help='Whether or not to use parallel execution',
|
390
390
|
)
|
391
|
+
@click.option('--ignore-attributes', '-i',
|
392
|
+
multiple=True, default=[],
|
393
|
+
help=r'A glob of attributes for which updates can be ignored, this option can be used multiple times.',
|
394
|
+
show_default=True
|
395
|
+
)
|
391
396
|
@click.option('--include-dir', '-I',
|
392
397
|
multiple=True, default=[],
|
393
398
|
help=r'A glob of a directory that needs to be included, this option can be used multiple times. For example: -I "integrations/\*/\*"',
|
@@ -411,7 +416,7 @@ def run_import(address, id, verbose, dry_run, working_dir, no_lock, terragrunt_a
|
|
411
416
|
@click.argument('terragrunt-args', nargs=-1, type=click.UNPROCESSED)
|
412
417
|
@click.version_option(version=__version__)
|
413
418
|
def run_analyze(verbose, exclude_external_dependencies, working_dir, start_at_step,
|
414
|
-
out, analyze_config, parallel_execution, include_dir, exclude_dir,
|
419
|
+
out, analyze_config, parallel_execution, ignore_attributes, include_dir, exclude_dir,
|
415
420
|
planfile_dir, data_collection_endpoint, terragrunt_args):
|
416
421
|
""" Analyzes the plan files """
|
417
422
|
|
@@ -425,6 +430,7 @@ def run_analyze(verbose, exclude_external_dependencies, working_dir, start_at_st
|
|
425
430
|
out=out,
|
426
431
|
analyze_config=analyze_config,
|
427
432
|
parallel_execution=parallel_execution,
|
433
|
+
ignore_attributes=ignore_attributes,
|
428
434
|
include_dirs=include_dir,
|
429
435
|
exclude_dirs=exclude_dir,
|
430
436
|
planfile_dir=planfile_dir,
|
@@ -701,10 +707,15 @@ def deploy(
|
|
701
707
|
ignore_unknown_options=True,
|
702
708
|
),
|
703
709
|
)
|
704
|
-
@click.option('--
|
705
|
-
help='
|
706
|
-
required=True,
|
707
|
-
|
710
|
+
@click.option('--platform-repo-url', '-p',
|
711
|
+
help='URL of the platform git repository',
|
712
|
+
required=True,
|
713
|
+
envvar='TGWRAP_PLATFORM_REPO_URL'
|
714
|
+
)
|
715
|
+
@click.option('--levels-deep', '-l',
|
716
|
+
help='For how many (directory) levels deep must be searched for deployments',
|
717
|
+
required=True, default=5, show_default=True,
|
718
|
+
type=int,
|
708
719
|
)
|
709
720
|
@click.option('--verbose', '-v',
|
710
721
|
help='Verbose printing',
|
@@ -718,14 +729,15 @@ def deploy(
|
|
718
729
|
show_default=True
|
719
730
|
)
|
720
731
|
@click.version_option(version=__version__)
|
721
|
-
def check_deployments(
|
732
|
+
def check_deployments(platform_repo_url, levels_deep, verbose, working_dir, out):
|
722
733
|
""" Check the freshness of deployed configuration versions against the platform repository """
|
723
734
|
|
724
735
|
check_latest_version(verbose)
|
725
736
|
|
726
737
|
tgwrap = TgWrap(verbose=verbose)
|
727
738
|
tgwrap.check_deployments(
|
728
|
-
|
739
|
+
repo_url=platform_repo_url,
|
740
|
+
levels_deep=levels_deep,
|
729
741
|
working_dir=working_dir,
|
730
742
|
out=out,
|
731
743
|
)
|
tgwrap/main.py
CHANGED
@@ -53,7 +53,6 @@ class TgWrap():
|
|
53
53
|
TERRAGRUNT_FILE='terragrunt.hcl'
|
54
54
|
VERSION_FILE="version.hcl"
|
55
55
|
LATEST_VERSION='latest'
|
56
|
-
LOCATE_VERSION_FILE_MAX_LEVELS=3
|
57
56
|
PLANFILE_NAME="planfile"
|
58
57
|
TG_SOURCE_VAR="TERRAGRUNT_SOURCE"
|
59
58
|
TG_SOURCE_MAP_VAR="TERRAGRUNT_SOURCE_MAP"
|
@@ -222,9 +221,12 @@ class TgWrap():
|
|
222
221
|
include_dirs = [dir.lstrip(f'.{os.path.sep}') for dir in include_dirs]
|
223
222
|
exclude_dirs = [dir.lstrip(f'.{os.path.sep}') for dir in exclude_dirs]
|
224
223
|
|
224
|
+
# Below doesn't seem to work, at least when using `analyze`
|
225
|
+
# Not sure it has been added here in the first place
|
226
|
+
|
225
227
|
# if the dir is not ending on '/*', add it
|
226
|
-
include_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in include_dirs]
|
227
|
-
exclude_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in exclude_dirs]
|
228
|
+
# include_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in include_dirs]
|
229
|
+
# exclude_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in exclude_dirs]
|
228
230
|
|
229
231
|
common_path = os.path.commonpath([os.path.abspath(working_dir), os.path.abspath(directory)])
|
230
232
|
self.printer.verbose(f'Common path for dir {directory}: {common_path}')
|
@@ -372,7 +374,7 @@ class TgWrap():
|
|
372
374
|
|
373
375
|
return graph
|
374
376
|
|
375
|
-
def _clone_repo(self,
|
377
|
+
def _clone_repo(self, repo, target_dir, version_tag=None):
|
376
378
|
"""Clones the repo, possibly a specific version, into a temp directory"""
|
377
379
|
|
378
380
|
def get_tags(target_dir):
|
@@ -452,9 +454,7 @@ class TgWrap():
|
|
452
454
|
return is_latest, is_branch, is_tag
|
453
455
|
|
454
456
|
# clone the repo
|
455
|
-
repo = manifest['git_repository']
|
456
457
|
self.printer.verbose(f'Clone repo {repo}')
|
457
|
-
|
458
458
|
cmd = f"git clone {repo} {target_dir}"
|
459
459
|
rc = subprocess.run(
|
460
460
|
shlex.split(cmd),
|
@@ -483,7 +483,7 @@ class TgWrap():
|
|
483
483
|
working_dir=target_dir,
|
484
484
|
)
|
485
485
|
|
486
|
-
self.printer.header(f'
|
486
|
+
self.printer.header(f'Fetch repo using reference {version_tag}')
|
487
487
|
|
488
488
|
if is_latest:
|
489
489
|
pass # nothing to do, we already have latest
|
@@ -961,6 +961,25 @@ class TgWrap():
|
|
961
961
|
planfile, auto_approve, clean, working_dir, terragrunt_args):
|
962
962
|
""" Executes a terragrunt command on a single module """
|
963
963
|
|
964
|
+
def extract_source_value(terragrunt_file_content):
|
965
|
+
# Regular expression to capture the terraform block
|
966
|
+
terraform_block_pattern = re.compile(r'terraform\s*\{(.*?)\n\}', re.DOTALL)
|
967
|
+
|
968
|
+
# Regular expression to capture the 'source' key and its value
|
969
|
+
source_pattern = re.compile(r'source\s*=\s*"(.*?)(?<!\\)"', re.DOTALL)
|
970
|
+
|
971
|
+
# Find the terraform block
|
972
|
+
terraform_block_match = terraform_block_pattern.search(terragrunt_file_content)
|
973
|
+
if terraform_block_match:
|
974
|
+
terraform_block = terraform_block_match.group(1)
|
975
|
+
|
976
|
+
# Search for the 'source' key within the block
|
977
|
+
source_match = source_pattern.search(terraform_block)
|
978
|
+
if source_match:
|
979
|
+
return source_match.group(1) # Return the value of 'source'
|
980
|
+
else:
|
981
|
+
raise ValueError('Could not locate the terragrunt source value')
|
982
|
+
|
964
983
|
self.printer.verbose(f"Attempting to execute 'run {command}'")
|
965
984
|
if terragrunt_args:
|
966
985
|
self.printer.verbose(f"- with additional parameters: {' '.join(terragrunt_args)}")
|
@@ -968,6 +987,7 @@ class TgWrap():
|
|
968
987
|
check_for_file=self.TERRAGRUNT_FILE
|
969
988
|
if working_dir:
|
970
989
|
check_for_file = os.path.join(working_dir, check_for_file)
|
990
|
+
|
971
991
|
if not os.path.isfile(check_for_file):
|
972
992
|
self.printer.error(
|
973
993
|
f"{check_for_file} not found, this seems not to be a terragrunt module directory!"
|
@@ -978,13 +998,15 @@ class TgWrap():
|
|
978
998
|
source_module = None
|
979
999
|
with open(check_for_file, 'r') as file:
|
980
1000
|
try:
|
981
|
-
content =
|
982
|
-
source = content
|
1001
|
+
content = file.read()
|
1002
|
+
source = extract_source_value(content)
|
1003
|
+
|
983
1004
|
# get the source part, typically the last part after the double /.
|
984
1005
|
# also remove a potential version element from it.
|
985
1006
|
source_module = re.sub(r'\${[^}]*}', '', source.split('//')[::-1][0])
|
986
1007
|
except Exception as e:
|
987
|
-
self.printer.
|
1008
|
+
self.printer.warning(f'Could not parse terragrunt.hcl, but we fall back to default behaviour.')
|
1009
|
+
self.printer.verbose(f'error (of type {type(e)}) raised')
|
988
1010
|
pass
|
989
1011
|
|
990
1012
|
cmd = self._construct_command(
|
@@ -1183,7 +1205,7 @@ class TgWrap():
|
|
1183
1205
|
self.printer.verbose(rc)
|
1184
1206
|
|
1185
1207
|
def analyze(self, exclude_external_dependencies, working_dir, start_at_step,
|
1186
|
-
out, analyze_config, parallel_execution, include_dirs, exclude_dirs,
|
1208
|
+
out, analyze_config, parallel_execution, ignore_attributes, include_dirs, exclude_dirs,
|
1187
1209
|
planfile_dir, data_collection_endpoint, terragrunt_args):
|
1188
1210
|
""" Analyzes the plan files """
|
1189
1211
|
|
@@ -1222,11 +1244,7 @@ class TgWrap():
|
|
1222
1244
|
cmd = f"terraform show -json {self.PLANFILE_NAME}"
|
1223
1245
|
|
1224
1246
|
config = None
|
1225
|
-
if
|
1226
|
-
self.printer.warning(
|
1227
|
-
f"Analyze config file is not set, this is required for checking for unauthorized deletions and drift detection scores!"
|
1228
|
-
)
|
1229
|
-
else:
|
1247
|
+
if analyze_config:
|
1230
1248
|
self.printer.verbose(
|
1231
1249
|
f"\nAnalyze using config {analyze_config}"
|
1232
1250
|
)
|
@@ -1234,7 +1252,6 @@ class TgWrap():
|
|
1234
1252
|
|
1235
1253
|
ts_validation_successful = True
|
1236
1254
|
details = {}
|
1237
|
-
drifts = {}
|
1238
1255
|
try:
|
1239
1256
|
# then run it and capture the output
|
1240
1257
|
with tempfile.NamedTemporaryFile(mode='w+', prefix='tgwrap-', delete=False) as f:
|
@@ -1265,7 +1282,6 @@ class TgWrap():
|
|
1265
1282
|
except IndexError:
|
1266
1283
|
self.printer.warning(f'Could not determine planfile: {line[:100]}')
|
1267
1284
|
|
1268
|
-
|
1269
1285
|
try:
|
1270
1286
|
# plan file could be empty (except for new line) if module is skipped
|
1271
1287
|
if len(plan_file) > 1:
|
@@ -1279,7 +1295,9 @@ class TgWrap():
|
|
1279
1295
|
config=config,
|
1280
1296
|
data=data,
|
1281
1297
|
verbose=self.printer.print_verbose,
|
1298
|
+
ignore_attributes=ignore_attributes,
|
1282
1299
|
)
|
1300
|
+
|
1283
1301
|
if not ts_success:
|
1284
1302
|
ts_validation_successful = False
|
1285
1303
|
else:
|
@@ -1296,6 +1314,7 @@ class TgWrap():
|
|
1296
1314
|
"creations": 0,
|
1297
1315
|
"updates": 0,
|
1298
1316
|
"deletions": 0,
|
1317
|
+
"outputs": 0,
|
1299
1318
|
"minor": 0,
|
1300
1319
|
"medium": 0,
|
1301
1320
|
"major": 0,
|
@@ -1306,9 +1325,14 @@ class TgWrap():
|
|
1306
1325
|
|
1307
1326
|
self.printer.header("Analysis results:", print_line_before=True)
|
1308
1327
|
for key, value in details.items():
|
1328
|
+
# if we want to ignore a few attributes
|
1329
|
+
if ignore_attributes:
|
1330
|
+
value['updates'] = [item for item in value['updates'] if item not in value['ignorable_updates']]
|
1331
|
+
|
1309
1332
|
self.printer.header(f'Module: {key}')
|
1310
|
-
if not value["all"]:
|
1333
|
+
if not value["all"] and not value["outputs"]:
|
1311
1334
|
self.printer.success('No changes detected')
|
1335
|
+
|
1312
1336
|
if value["unauthorized"]:
|
1313
1337
|
self.printer.error('Unauthorized deletions:')
|
1314
1338
|
for m in value["unauthorized"]:
|
@@ -1328,30 +1352,48 @@ class TgWrap():
|
|
1328
1352
|
for m in value["updates"]:
|
1329
1353
|
total_drifts["updates"] = total_drifts["updates"] + 1
|
1330
1354
|
self.printer.normal(f'-> {m}')
|
1355
|
+
if value["ignorable_updates"]:
|
1356
|
+
if self.printer.print_verbose:
|
1357
|
+
self.printer.normal('Updates (ignored):')
|
1358
|
+
for m in value["ignorable_updates"]:
|
1359
|
+
self.printer.normal(f'-> {m}')
|
1360
|
+
else:
|
1361
|
+
self.printer.normal(f'Updates (ignored): {len(value["ignorable_updates"])} resources (add --verbose to see them)')
|
1362
|
+
if value["outputs"]:
|
1363
|
+
self.printer.normal('Output changes:')
|
1364
|
+
for m in value["outputs"]:
|
1365
|
+
total_drifts["outputs"] = total_drifts["outputs"] + 1
|
1366
|
+
self.printer.normal(f'-> {m}')
|
1331
1367
|
|
1332
|
-
|
1333
|
-
|
1334
|
-
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
|
1341
|
-
|
1342
|
-
|
1368
|
+
if not analyze_config:
|
1369
|
+
self.printer.error(
|
1370
|
+
f"Analyze config file is not set, this is required for checking for unauthorized deletions and drift detection scores!",
|
1371
|
+
print_line_before=True,
|
1372
|
+
)
|
1373
|
+
else:
|
1374
|
+
for key, value in details.items():
|
1375
|
+
for type in ["minor", "medium", "major", "unknown", "total"]:
|
1376
|
+
total_drifts[type] += value["drifts"][type]
|
1377
|
+
|
1378
|
+
# the formula below is just a way to achieve a numeric results that is coming from the various drift categories
|
1379
|
+
value['drifts']['score'] = calculate_score(
|
1380
|
+
major = value['drifts']['major'],
|
1381
|
+
medium = value['drifts']['medium'],
|
1382
|
+
minor = value['drifts']['minor'],
|
1383
|
+
)
|
1384
|
+
value['drifts']['score'] = value['drifts']['major'] * 10 + value['drifts']['medium'] + value['drifts']['minor'] / 10
|
1343
1385
|
|
1344
|
-
|
1345
|
-
|
1346
|
-
|
1386
|
+
# the formula below is just a way to achieve a numeric results that is coming from the various drift categories
|
1387
|
+
total_drift_score = total_drifts['major'] * 10 + total_drifts['medium'] + total_drifts['minor'] / 10
|
1388
|
+
total_drifts['score'] = total_drift_score
|
1347
1389
|
|
1348
|
-
|
1349
|
-
|
1350
|
-
|
1351
|
-
|
1352
|
-
|
1353
|
-
|
1354
|
-
|
1390
|
+
self.printer.header(f"Drift score: {total_drift_score} ({total_drifts['major']}.{total_drifts['medium']}.{total_drifts['minor']})")
|
1391
|
+
if total_drifts["unknown"] > 0:
|
1392
|
+
self.printer.warning(f"For {total_drifts['unknown']} resources, drift score is not configured, please update configuration!")
|
1393
|
+
self.printer.warning('- Unknowns:')
|
1394
|
+
for key, value in details.items():
|
1395
|
+
for m in value["unknowns"]:
|
1396
|
+
self.printer.warning(f' -> {m}')
|
1355
1397
|
|
1356
1398
|
if out or data_collection_endpoint:
|
1357
1399
|
# in the output we convert the dict of dicts to a list of dicts as it makes processing
|
@@ -1489,7 +1531,7 @@ class TgWrap():
|
|
1489
1531
|
source_config_dir = None
|
1490
1532
|
|
1491
1533
|
version_tag, _, _ = self._clone_repo(
|
1492
|
-
|
1534
|
+
repo=manifest['git_repository'],
|
1493
1535
|
target_dir=temp_dir,
|
1494
1536
|
version_tag=version_tag,
|
1495
1537
|
)
|
@@ -1611,30 +1653,72 @@ class TgWrap():
|
|
1611
1653
|
except Exception:
|
1612
1654
|
pass
|
1613
1655
|
|
1614
|
-
def check_deployments(self,
|
1656
|
+
def check_deployments(self, repo_url, levels_deep, working_dir, out):
|
1615
1657
|
""" Check the freshness of deployed configuration versions against the platform repository """
|
1616
1658
|
|
1617
|
-
def locate_version_files(current_directory, found_files=[], root_directory=None, level=1):
|
1659
|
+
def locate_version_files(current_directory, found_files=[], root_directory=None, level=1, git_status=''):
|
1618
1660
|
" This tries to find a version file in the current directory, or a given number of directories beneath it"
|
1619
1661
|
|
1662
|
+
# do not include hidden directories
|
1663
|
+
if os.path.basename(current_directory).startswith('.'):
|
1664
|
+
return found_files
|
1665
|
+
|
1620
1666
|
if not root_directory:
|
1621
1667
|
root_directory = current_directory
|
1622
1668
|
|
1669
|
+
if not git_status:
|
1670
|
+
self.printer.verbose(f'Check for git status in directory {current_directory}')
|
1671
|
+
# Execute 'git status' to get an overview of the current status
|
1672
|
+
cmd = "git status"
|
1673
|
+
rc = subprocess.run(
|
1674
|
+
shlex.split(cmd),
|
1675
|
+
cwd=current_directory,
|
1676
|
+
universal_newlines=True,
|
1677
|
+
stdout=subprocess.PIPE,
|
1678
|
+
stderr=subprocess.PIPE,
|
1679
|
+
)
|
1680
|
+
output = ('stdout: ' + rc.stdout + 'stderr: ' + rc.stderr).lower()
|
1681
|
+
if 'not a git repository' in output:
|
1682
|
+
pass
|
1683
|
+
elif 'branch is up to date' in output:
|
1684
|
+
git_status = 'up to date; '
|
1685
|
+
elif 'head detached' in output:
|
1686
|
+
git_status = 'head detached; '
|
1687
|
+
elif 'untracked files' in output:
|
1688
|
+
git_status = git_status + 'untracked files; '
|
1689
|
+
elif 'changes to be committed' in output:
|
1690
|
+
git_status = git_status + 'staged changes; '
|
1691
|
+
elif 'changes not staged for commit' in output:
|
1692
|
+
git_status = git_status + 'unstaged changes; '
|
1693
|
+
elif 'branch is ahead of' in output:
|
1694
|
+
git_status = git_status + 'ahead of remote; '
|
1695
|
+
elif 'branch is behind of' in output:
|
1696
|
+
git_status = git_status + 'behind remote; '
|
1697
|
+
elif 'unmerged paths' in output:
|
1698
|
+
git_status = git_status + 'merge conflicts; '
|
1699
|
+
|
1623
1700
|
for entry in os.listdir(current_directory):
|
1624
1701
|
full_entry = os.path.join(current_directory, entry)
|
1625
|
-
|
1702
|
+
|
1703
|
+
if os.path.isdir(full_entry) and level <= levels_deep:
|
1626
1704
|
found_files = locate_version_files(
|
1627
1705
|
current_directory=full_entry,
|
1628
1706
|
found_files=found_files,
|
1629
1707
|
root_directory=root_directory,
|
1630
1708
|
level=level+1,
|
1709
|
+
git_status=git_status,
|
1631
1710
|
)
|
1632
1711
|
elif entry == self.VERSION_FILE:
|
1633
|
-
found_files.append(
|
1712
|
+
found_files.append(
|
1713
|
+
{
|
1714
|
+
'path': os.path.relpath(current_directory, root_directory),
|
1715
|
+
'git_status': git_status,
|
1716
|
+
}
|
1717
|
+
)
|
1634
1718
|
|
1635
1719
|
return found_files
|
1636
1720
|
|
1637
|
-
def
|
1721
|
+
def get_all_versions(repo_dir, min_version=None):
|
1638
1722
|
"Get all the version tags from the repo including their data"
|
1639
1723
|
|
1640
1724
|
# Execute 'git tag' command to get a list of all tags
|
@@ -1665,25 +1749,27 @@ class TgWrap():
|
|
1665
1749
|
try:
|
1666
1750
|
# do we have a working dir?
|
1667
1751
|
working_dir = working_dir if working_dir else os.getcwd()
|
1668
|
-
self.printer.header(f'Check released versions ({
|
1752
|
+
self.printer.header(f'Check released versions (max {levels_deep} levels deep) in directory: {working_dir}')
|
1669
1753
|
|
1670
|
-
|
1754
|
+
found_files = locate_version_files(working_dir)
|
1671
1755
|
|
1672
1756
|
versions = []
|
1673
|
-
for
|
1757
|
+
for result in found_files:
|
1674
1758
|
# Determine the deployed version as defined in the version file
|
1675
|
-
with open(os.path.join(working_dir,
|
1759
|
+
with open(os.path.join(working_dir, result['path'], self.VERSION_FILE), 'r') as file:
|
1760
|
+
# todo: replace this with regex as it is (now) the only reason we use this lib
|
1676
1761
|
content = hcl2.load(file)
|
1677
1762
|
try:
|
1678
1763
|
version_tag = content['locals'][0]['version_tag']
|
1679
1764
|
versions.append(
|
1680
1765
|
{
|
1681
|
-
'path':
|
1766
|
+
'path': result['path'],
|
1767
|
+
'git_status': result['git_status'],
|
1682
1768
|
'tag': version_tag
|
1683
1769
|
}
|
1684
1770
|
)
|
1685
1771
|
except KeyError as e:
|
1686
|
-
versions.append({
|
1772
|
+
versions.append({result: 'unknown'})
|
1687
1773
|
|
1688
1774
|
self.printer.verbose(f'Detected versions: {versions}')
|
1689
1775
|
|
@@ -1700,11 +1786,14 @@ class TgWrap():
|
|
1700
1786
|
self.printer.verbose(f'Detected minimum version {min_version} and maximum version {max_version}')
|
1701
1787
|
|
1702
1788
|
temp_dir = os.path.join(tempfile.mkdtemp(prefix='tgwrap-'), "tg-source")
|
1703
|
-
|
1704
|
-
|
1789
|
+
self._clone_repo(
|
1790
|
+
repo=repo_url,
|
1791
|
+
target_dir=temp_dir,
|
1792
|
+
version_tag='latest',
|
1793
|
+
)
|
1705
1794
|
|
1706
1795
|
# determine the version tag from the repo, including their date
|
1707
|
-
all_versions =
|
1796
|
+
all_versions = get_all_versions(repo_dir=temp_dir, min_version=min_version['tag'])
|
1708
1797
|
|
1709
1798
|
# so now we can determine how old the deployed versions are
|
1710
1799
|
now = datetime.now(timezone.utc)
|
@@ -1718,31 +1807,45 @@ class TgWrap():
|
|
1718
1807
|
version['days_since_release'] = (now - release_date).days
|
1719
1808
|
|
1720
1809
|
self.printer.header(
|
1721
|
-
'Deployed versions:' if len(versions) > 0 else 'No deployed versions detected'
|
1810
|
+
'Deployed versions:' if len(versions) > 0 else 'No deployed versions detected',
|
1811
|
+
print_line_before=True,
|
1722
1812
|
)
|
1723
|
-
|
1813
|
+
|
1814
|
+
# sort the list based on its path
|
1815
|
+
versions = sorted(versions, key=lambda x: x['path'])
|
1816
|
+
|
1724
1817
|
for version in versions:
|
1725
1818
|
days_since_release = version.get("days_since_release", 0)
|
1726
1819
|
message = f'-> {version["path"]}: {version["tag"]} (released {days_since_release} days ago)'
|
1727
1820
|
if version['release_date'] == 'unknown':
|
1728
1821
|
self.printer.normal(message)
|
1729
|
-
elif days_since_release >
|
1730
|
-
self.printer.error(message)
|
1731
|
-
elif days_since_release > 30:
|
1822
|
+
elif days_since_release > 120:
|
1732
1823
|
self.printer.error(message)
|
1733
|
-
elif days_since_release
|
1824
|
+
elif days_since_release > 80:
|
1825
|
+
self.printer.warning(message)
|
1826
|
+
elif days_since_release < 40:
|
1734
1827
|
self.printer.success(message)
|
1735
1828
|
else:
|
1736
1829
|
self.printer.normal(message)
|
1737
1830
|
|
1831
|
+
if version.get('git_status'):
|
1832
|
+
message = f'WARNING: git status: {version["git_status"].strip()}'
|
1833
|
+
if not 'up to date' in message:
|
1834
|
+
self.printer.warning(message)
|
1835
|
+
|
1738
1836
|
self.printer.normal("\n") # just to get an empty line :-/
|
1739
1837
|
self.printer.warning("""
|
1740
1838
|
Note:
|
1741
1839
|
This result only says something about the freshness of the deployed configurations,
|
1742
1840
|
but not whether the actual resources are in sync with these.
|
1841
|
+
|
1743
1842
|
Check the drift of these configurations with the actual deployments by
|
1744
1843
|
planning and analyzing the results.
|
1745
|
-
|
1844
|
+
|
1845
|
+
Also, it uses the locally checked out repositories, make sure these are pulled so that
|
1846
|
+
this reflect the most up to date situation!
|
1847
|
+
""",
|
1848
|
+
print_line_before=True, print_line_after=True)
|
1746
1849
|
|
1747
1850
|
if out:
|
1748
1851
|
# use the regular printer, to avoid it being sent to stderr
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: tgwrap
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.11.0
|
4
4
|
Summary: A (terragrunt) wrapper around a (terraform) wrapper around ....
|
5
5
|
Home-page: https://gitlab.com/lunadata/tgwrap
|
6
6
|
License: MIT
|
@@ -72,6 +72,8 @@ And this was not something a bunch of aliases could solve, hence this wrapper wa
|
|
72
72
|
|
73
73
|
When using the run-all, analyzing what is about to be changed is not going to be easier. Hence we created the `tgwrap analyze` function that lists all the planned changes and (if a config file is availabe) calculates a drift score and runs a [terrasafe](https://pypi.org/project/terrasafe/) style validation check.
|
74
74
|
|
75
|
+
> you can ignore minor changes, such as tag updates, with `tgwrap analyze -i tags`
|
76
|
+
|
75
77
|
It needs a config file as follows:
|
76
78
|
|
77
79
|
```yaml
|
@@ -0,0 +1,13 @@
|
|
1
|
+
tgwrap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
+
tgwrap/analyze.py,sha256=5C0OcDpOfn6U39QNIoc7V9ePjjcXKbJH6cOVNr9jJ9A,10289
|
3
|
+
tgwrap/cli.py,sha256=RYZf7Mb7qJW-q_-qqREi9s2dAu_wpOtvmvB086Y5URQ,32084
|
4
|
+
tgwrap/deploy.py,sha256=-fSk-Ix_HqrXY7KQX_L27TnFzIuhBHYv4xYJW6zRDN4,10243
|
5
|
+
tgwrap/inspector-resources-template.yml,sha256=Mos8NDzzZ3VxdXgeiVL9cmQfRcIXIHMLf79_KLwdXu8,3297
|
6
|
+
tgwrap/inspector.py,sha256=5pW7Ex1lkKRoXY6hZGbCNmSD2iRzgMSfqi9w7gb-AcY,16990
|
7
|
+
tgwrap/main.py,sha256=rCdrsu1J4biQj7IDqF0AaR-s7DL-cQ2DTt50zAfjZ8M,93702
|
8
|
+
tgwrap/printer.py,sha256=frn1PARd8A28mkRCYR6ybN2x0NBULhNOutn4l2U7REY,2754
|
9
|
+
tgwrap-0.11.0.dist-info/LICENSE,sha256=VT-AVxIXt3EQTC-7Hy1uPGnrDNJLqfcgLgJD78fiyx4,1065
|
10
|
+
tgwrap-0.11.0.dist-info/METADATA,sha256=10rGGUHsLlUTxMGsbhltzdi8bd2QNHkmE_uQ9fHMs8Q,17560
|
11
|
+
tgwrap-0.11.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
12
|
+
tgwrap-0.11.0.dist-info/entry_points.txt,sha256=H8X0PMPmd4aW7Y9iyChZ0Ug6RWGXqhRUvHH-6f6Mxz0,42
|
13
|
+
tgwrap-0.11.0.dist-info/RECORD,,
|
tgwrap-0.10.2.dist-info/RECORD
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
tgwrap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
2
|
-
tgwrap/analyze.py,sha256=TJvAKVIbWl8-8oxpTwXVBWU72q_XQKzUTlyMZ25cV2M,8728
|
3
|
-
tgwrap/cli.py,sha256=0SFzLJA-deut81Mpt6Giq77WK26vTDcajyFnPIHXR5c,31649
|
4
|
-
tgwrap/deploy.py,sha256=-fSk-Ix_HqrXY7KQX_L27TnFzIuhBHYv4xYJW6zRDN4,10243
|
5
|
-
tgwrap/inspector-resources-template.yml,sha256=Mos8NDzzZ3VxdXgeiVL9cmQfRcIXIHMLf79_KLwdXu8,3297
|
6
|
-
tgwrap/inspector.py,sha256=5pW7Ex1lkKRoXY6hZGbCNmSD2iRzgMSfqi9w7gb-AcY,16990
|
7
|
-
tgwrap/main.py,sha256=WKxlREE4QmdZKTHivhWa3K5yI40amGlmjBs4y7oCTqw,88748
|
8
|
-
tgwrap/printer.py,sha256=frn1PARd8A28mkRCYR6ybN2x0NBULhNOutn4l2U7REY,2754
|
9
|
-
tgwrap-0.10.2.dist-info/LICENSE,sha256=VT-AVxIXt3EQTC-7Hy1uPGnrDNJLqfcgLgJD78fiyx4,1065
|
10
|
-
tgwrap-0.10.2.dist-info/METADATA,sha256=TMYitOkytrAQdGcEIeCOeiyKLg5yAFtgnmiupyLiJjo,17476
|
11
|
-
tgwrap-0.10.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
12
|
-
tgwrap-0.10.2.dist-info/entry_points.txt,sha256=H8X0PMPmd4aW7Y9iyChZ0Ug6RWGXqhRUvHH-6f6Mxz0,42
|
13
|
-
tgwrap-0.10.2.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|