tgwrap 0.8.11__py3-none-any.whl → 0.11.2__py3-none-any.whl

tgwrap/main.py

@@ -17,6 +17,7 @@ import sys
 import subprocess
 import shlex
 import shutil
+import requests
 import re
 import tempfile
 import json
@@ -24,6 +25,7 @@ import yaml
 import threading
 import queue
 import multiprocessing
+import traceback
 import click
 import networkx as nx
 import hcl2
@@ -34,13 +36,14 @@ from datetime import datetime, timezone
 from .printer import Printer
 from .analyze import run_analyze
 from .deploy import prepare_deploy_config, run_sync
+from .inspector import AzureInspector
+
 class DateTimeEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, datetime):
             return obj.isoformat()
         return super().default(obj)
 
-
 class TgWrap():
     """
     A wrapper around terragrunt with the sole purpose to make it a bit
@@ -50,7 +53,6 @@ class TgWrap():
     TERRAGRUNT_FILE='terragrunt.hcl'
     VERSION_FILE="version.hcl"
     LATEST_VERSION='latest'
-    LOCATE_VERSION_FILE_MAX_LEVELS=3
     PLANFILE_NAME="planfile"
     TG_SOURCE_VAR="TERRAGRUNT_SOURCE"
     TG_SOURCE_MAP_VAR="TERRAGRUNT_SOURCE_MAP"
@@ -83,9 +85,13 @@ class TgWrap():
                     )
                 self.tg_source_indicator = None
 
+        # terragrunt do now prefer opentofu but we want this to be a conscious decision
+        if not os.environ.get('TERRAGRUNT_TFPATH'):
+            os.environ['TERRAGRUNT_TFPATH'] = 'terraform'
+
     def load_yaml_file(self, filepath):
         try:
-            with open(filepath, 'r') as file:
+            with open(filepath.strip(), 'r') as file:
                 return yaml.safe_load(file)
         except yaml.parser.ParserError as e:
             self.printer.error(f'Cannot parse YAML file {filepath}, check syntax please!')
@@ -102,8 +108,8 @@ class TgWrap():
             'info': '{base_command} terragrunt-info --terragrunt-non-interactive {update} {upgrade} {common}',
             'plan': '{base_command} {command} --terragrunt-non-interactive  -out={planfile_name} {lock_level} {update} {parallelism} {common}',
             'apply': '{base_command} {command} {non_interactive} {no_auto_approve} {update} {parallelism} {common} {planfile}',
-            'show': '{base_command} {command} --terragrunt-non-interactive {planfile_name} {common}',
-            'destroy': '{base_command} {command} {non_interactive} {no_auto_approve} {parallelism} {common} {planfile}',
+            'show': '{base_command} {command} --terragrunt-non-interactive {common} {planfile_name}',
+            'destroy': '{base_command} {command} --terragrunt-no-destroy-dependencies-check {non_interactive} {no_auto_approve} {parallelism} {common} {planfile}',
         }
 
         lock_stmt = ''
@@ -215,9 +221,12 @@ class TgWrap():
         include_dirs = [dir.lstrip(f'.{os.path.sep}') for dir in include_dirs]
         exclude_dirs = [dir.lstrip(f'.{os.path.sep}') for dir in exclude_dirs]
 
+        # Below doesn't seem to work, at least when using `analyze`
+        # Not sure it has been added here in the first place
+
         # if the dir is not ending on '/*', add it
-        include_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in include_dirs]
-        exclude_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in exclude_dirs]
+        # include_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in include_dirs]
+        # exclude_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in exclude_dirs]
 
         common_path = os.path.commonpath([os.path.abspath(working_dir), os.path.abspath(directory)])
         self.printer.verbose(f'Common path for dir {directory}: {common_path}')
@@ -365,7 +374,7 @@ class TgWrap():
 
         return graph
 
-    def _clone_repo(self, manifest, target_dir, version_tag=None):
+    def _clone_repo(self, repo, target_dir, version_tag=None):
         """Clones the repo, possibly a specific version, into a temp directory"""
 
         def get_tags(target_dir):
@@ -445,9 +454,7 @@ class TgWrap():
             return is_latest, is_branch, is_tag
 
         # clone the repo
-        repo = manifest['git_repository']
         self.printer.verbose(f'Clone repo {repo}')
-        
         cmd = f"git clone {repo} {target_dir}"
         rc = subprocess.run(
             shlex.split(cmd),
@@ -476,7 +483,7 @@ class TgWrap():
             working_dir=target_dir,
             )
 
-        self.printer.header(f'Deploy using reference {version_tag}')
+        self.printer.header(f'Fetch repo using reference {version_tag}')
 
         if is_latest:
             pass # nothing to do, we already have latest
@@ -656,7 +663,7 @@ class TgWrap():
         self, command, exclude_external_dependencies, start_at_step, dry_run,
         parallel_execution=False, ask_for_confirmation=False, collect_output_file=None,
         backwards=False, working_dir=None, include_dirs=[], exclude_dirs=[],
-        use_native_terraform=False, add_to_workdir=None,
+        use_native_terraform=False, add_to_workdir=None, continue_on_error=False,
         ):
         "Runs the desired command in the directories as defined in the directed graph"
 
@@ -753,7 +760,7 @@ class TgWrap():
                         progress=progress,
                     )
 
-                    if stop_processing:
+                    if stop_processing and not continue_on_error:
                         self.printer.warning(f"Processing needs to be stopped at step {step_nbr}.")
                         self.printer.normal(
                             f"After you've fixed the problem, you can continue where you left off by adding '--start-at-step {step_nbr}'."
@@ -775,10 +782,204 @@ class TgWrap():
             total_items = sum(len(group) for group in groups)
             self.printer.verbose(f'Executed {group_nbr} groups and {total_items} steps')
 
+    def _get_access_token(self):
+        """Retrieve an access token"""
+    
+        #
+        # Everything we do here, can be done using native python. And probably this is preferable as well.
+        # But I have decided to follow (at least for now) the overall approach of the app and that is
+        # executing systems commands.
+        # This does require the az cli to be installed, but that is a fair assumption if you are working
+        # with terragrunt/terraform and want to post the analyze results to an Azure Data Collection Endpoint.
+        # However, not ruling out this will change, but then the change should be transparant.
+        #
+
+        # Get the Azure information
+        rc = subprocess.run(
+            shlex.split('az account show'),
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=sys.stderr,
+        )
+        self.printer.verbose(rc)
+
+        # Do a few checks
+        if rc.returncode != 0:
+            raise Exception(f'Could not get Azure account info')
+        
+        # Get the ouptut
+        output = json.loads(rc.stdout.decode())
+        if output.get('environmentName') != 'AzureCloud':
+            raise Exception(f'Environment is not an Azure cloud:\n{json.dumps(output, indent=2)}')
+        
+        tenant_id = output.get('tenantId')
+        if not tenant_id:
+            raise Exception(f'Could not determine Azure tenant id:\n{json.dumps(output, indent=2)}')
+
+        principal = output.get('user').get('name')
+        if not principal:
+            raise Exception(f'Could not determine principal:\n{json.dumps(output, indent=2)}')
+
+        # TOKEN=$(az account get-access-token --scope "https://monitor.azure.com//.default" | jq -r '.accessToken')
+        # Get the Azure OAUTH token
+        rc = subprocess.run(
+            shlex.split('az account get-access-token --scope "https://monitor.azure.com//.default"'),
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=sys.stderr,
+        )
+        self.printer.verbose(rc.returncode) # do not print the token to output
+
+        # Do a few checks
+        if rc.returncode != 0:
+            raise Exception(f'Could not get Azure OAUTH token')
+        
+        # Get the ouptut
+        output = json.loads(rc.stdout.decode())
+        token = output.get('accessToken')
+        if not token:
+            raise Exception(f'Could not retrieve an access token:\n{json.dumps(output, indent=2)}')
+
+        return principal, token
+
+    def _post_to_dce(self, data_collection_endpoint, payload, token=None):
+
+        if not token:
+            _, token = self._get_access_token()
+
+        # DCE payload must be submitted as an arry
+        if not isinstance(payload, list):
+            dce_payload = [payload]
+        else:
+            dce_payload = payload
+
+        self.printer.verbose('About to log:')
+        self.printer.verbose(f'- to: {data_collection_endpoint}')
+        self.printer.verbose(f'- payload:\n{json.dumps(dce_payload, indent=2)}')
+
+        # now do the actual post
+        try:
+            headers = {
+                'Authorization': f"Bearer {token}",
+                'Content-Type': 'application/json',
+            }
+            resp = requests.post(
+                url=data_collection_endpoint,
+                headers=headers,
+                json=dce_payload,
+            )
+
+            resp.raise_for_status()
+            self.printer.success('Analyze results logged to DCE', print_line_before=True)
+
+        except requests.exceptions.RequestException as e:
+            # we warn but continue
+            self.printer.warning(f'Error while posting the analyze results ({type(e)}): {e}', print_line_before=True)
+        except Exception as e:
+            self.printer.error(f'Unexpected error: {e}')
+            if self.printer.print_verbose:
+                raise(e)
+            sys.exit(1)
+
+    def _post_analyze_results_to_dce(self, data_collection_endpoint:str, payload:object):
+        """
+        Posts the payload to the given (Azure) data collection endpoint
+        """
+
+        def mask_basic_auth(url):
+            # Regular expression to match basic authentication credentials in URL
+            auth_pattern = re.compile(r"(https?://)([^:@]+):([^:@]+)@(.+)")
+            # Return the url without the basic auth part
+            return auth_pattern.sub(r"\1\4", url)
+
+        principal, token = self._get_access_token()
+
+        # Get the repo info
+        rc = subprocess.run(
+            shlex.split('git config --get remote.origin.url'),
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=sys.stderr,
+        )
+        self.printer.verbose(rc)
+
+        # Do a few checks
+        if rc.returncode != 0:
+            raise Exception(f'Could not get git repo info')
+        
+        # Get the ouptut
+        repo = rc.stdout.decode().rstrip('\n')
+        if not repo:
+            raise Exception(f'Could not get git repo info: {repo}')
+
+        # Remove the basic auth info if it is part of the url
+        repo = mask_basic_auth(repo)
+
+        # Get the current path in the repo
+        rc = subprocess.run(
+            shlex.split('git rev-parse --show-prefix'),
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=sys.stderr,
+        )
+        self.printer.verbose(rc)
+
+        # Do a few checks
+        if rc.returncode != 0:
+            raise Exception(f'Could not get current scope')
+        
+        # Get the ouptut
+        scope = rc.stdout.decode().rstrip('\n')
+        if not scope:
+            raise Exception(f'Could not get scope: {scope}')
+
+        # So now we have everything, we can construct the final payload
+        payload = {
+            "scope": scope,
+            "principal": principal,
+            "repo": repo,
+            "creations": payload.get("summary").get("creations"),
+            "updates": payload.get("summary").get("updates"),
+            "deletions": payload.get("summary").get("deletions"),
+            "minor": payload.get("summary").get("minor"),
+            "medium": payload.get("summary").get("medium"),
+            "major": payload.get("summary").get("major"),
+            "unknown": payload.get("summary").get("unknown"),
+            "total": payload.get("summary").get("total"),
+            "score": payload.get("summary").get("score"),
+            "details": payload.get('details'),
+        }
+        self._post_to_dce(
+            payload=payload,
+            data_collection_endpoint=data_collection_endpoint,
+            token=token,
+        )
+
+        self.printer.verbose('Done')
+
     def run(self, command, debug, dry_run, no_lock, update, upgrade,
         planfile, auto_approve, clean, working_dir, terragrunt_args):
         """ Executes a terragrunt command on a single module """
 
+        def extract_source_value(terragrunt_file_content):
+            # Regular expression to capture the terraform block
+            terraform_block_pattern = re.compile(r'terraform\s*\{(.*?)\n\}', re.DOTALL)
+            
+            # Regular expression to capture the 'source' key and its value
+            source_pattern = re.compile(r'source\s*=\s*"(.*?)(?<!\\)"', re.DOTALL)
+            
+            # Find the terraform block
+            terraform_block_match = terraform_block_pattern.search(terragrunt_file_content)
+            if terraform_block_match:
+                terraform_block = terraform_block_match.group(1)
+                
+                # Search for the 'source' key within the block
+                source_match = source_pattern.search(terraform_block)
+                if source_match:
+                    return source_match.group(1)  # Return the value of 'source'
+            else:
+                raise ValueError('Could not locate the terragrunt source value')
+
         self.printer.verbose(f"Attempting to execute 'run {command}'")
         if terragrunt_args:
             self.printer.verbose(f"- with additional parameters: {' '.join(terragrunt_args)}")
@@ -786,6 +987,7 @@ class TgWrap():
         check_for_file=self.TERRAGRUNT_FILE
         if working_dir:
             check_for_file = os.path.join(working_dir, check_for_file)
+
         if not os.path.isfile(check_for_file):
             self.printer.error(
                 f"{check_for_file} not found, this seems not to be a terragrunt module directory!"
@@ -796,13 +998,15 @@ class TgWrap():
             source_module = None
             with open(check_for_file, 'r') as file:
                 try:
-                    content = hcl2.load(file)
-                    source = content['terraform'][0]['source']
+                    content = file.read()
+                    source = extract_source_value(content)
+
                     # get the source part, typically the last part after the double /.
                     # also remove a potential version element from it.
                     source_module = re.sub(r'\${[^}]*}', '', source.split('//')[::-1][0])
                 except Exception as e:
-                    self.printer.verbose(f'Could not parse terragrunt.hcl, error (of type {type(e)}) raised, but we fall back to default behaviour.')
+                    self.printer.warning(f'Could not parse terragrunt.hcl, but we fall back to default behaviour.')
+                    self.printer.verbose(f'error (of type {type(e)}) raised')
                     pass
 
         cmd = self._construct_command(
@@ -884,14 +1088,14 @@ class TgWrap():
         # tgwrap state mv 'azuread_group.this["viewers"]' 'azuread_group.this["readers"]'
         rc = subprocess.run(
             shlex.split(cmd, posix=False),
-            cwd=cwd,
+            cwd=cwd if cwd else None,
             )
         self.printer.verbose(rc)
 
         sys.exit(rc.returncode)
 
     def run_all(self, command, debug, dry_run, no_lock, update, upgrade,
-        exclude_external_dependencies, step_by_step, planfile, auto_approve, clean,
+        exclude_external_dependencies, step_by_step, continue_on_error, planfile, auto_approve, clean,
         working_dir, start_at_step, limit_parallelism, include_dirs, exclude_dirs, terragrunt_args):
         """ Executes a terragrunt command across multiple modules """
 
@@ -903,10 +1107,6 @@ class TgWrap():
         modifying_command = (command.lower() in ['apply', 'destroy'])
         auto_approve = auto_approve if modifying_command else True
 
-        # if the dir is not ending on '/*', add it
-        include_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in include_dirs]
-        exclude_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in exclude_dirs]
-
         cmd = self._construct_command(
             command=command,
             allow_no_run_all=False,
@@ -933,11 +1133,16 @@ class TgWrap():
                 f'This command will be executed for each individual module:\n$ {cmd}'
                 )
 
+            # if the dir is not ending on '/*', add it
+            include_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in include_dirs]
+            exclude_dirs = [dir.rstrip(f'.{os.path.sep}*') + f'{os.path.sep}*' for dir in exclude_dirs]
+
             self._run_di_graph(
                 command=cmd,
                 exclude_external_dependencies=exclude_external_dependencies,
                 working_dir=working_dir,
                 ask_for_confirmation=(not auto_approve),
+                continue_on_error=continue_on_error,
                 dry_run=dry_run,
                 start_at_step=start_at_step,
                 backwards=True if command.lower() in ['destroy'] else False,
@@ -1000,8 +1205,8 @@ class TgWrap():
             self.printer.verbose(rc)
 
     def analyze(self, exclude_external_dependencies, working_dir, start_at_step,
-        out, analyze_config, parallel_execution,
-        include_dirs, exclude_dirs, planfile_dir, terragrunt_args):
+        out, analyze_config, parallel_execution, ignore_attributes, include_dirs, exclude_dirs, 
+        planfile_dir, data_collection_endpoint, terragrunt_args):
         """ Analyzes the plan files """
 
         def calculate_score(major: int, medium: int, minor: int) -> float :
@@ -1039,19 +1244,14 @@ class TgWrap():
             cmd = f"terraform show -json {self.PLANFILE_NAME}"
 
         config = None
-        if not analyze_config:
-            self.printer.warning(
-                f"Analyze  config file is not set, this is required for checking for unauthorized deletions and drift detection scores!"
-                )
-        else:
+        if analyze_config:
             self.printer.verbose(
                 f"\nAnalyze using config {analyze_config}"
                 )
             config = self.load_yaml_file(analyze_config)
 
         ts_validation_successful = True
-        changes = {}
-        drifts = {}
+        details = {}
         try:
             # then run it and capture the output
             with tempfile.NamedTemporaryFile(mode='w+', prefix='tgwrap-', delete=False) as f:
@@ -1082,7 +1282,6 @@ class TgWrap():
                     except IndexError:
                         self.printer.warning(f'Could not determine planfile: {line[:100]}')
 
-
                     try:
                         # plan file could be empty (except for new line) if module is skipped
                         if len(plan_file) > 1:
@@ -1092,11 +1291,13 @@ class TgWrap():
                             if 'exception' in data:
                                 raise Exception(data['exception'])
 
-                            changes[module], ts_success = run_analyze(
+                            details[module], ts_success = run_analyze(
                                 config=config,
                                 data=data,
                                 verbose=self.printer.print_verbose,
+                                ignore_attributes=ignore_attributes,
                                 )
+
                             if not ts_success:
                                 ts_validation_successful = False
                         else:
@@ -1113,6 +1314,7 @@ class TgWrap():
             "creations": 0,
             "updates": 0,
             "deletions": 0,
+            "outputs": 0,
             "minor": 0,
             "medium": 0,
             "major": 0,
@@ -1122,10 +1324,15 @@ class TgWrap():
         }
 
         self.printer.header("Analysis results:", print_line_before=True)
-        for key, value in changes.items():
+        for key, value in details.items():
+            # if we want to ignore a few attributes
+            if ignore_attributes:
+                value['updates'] = [item for item in value['updates'] if item not in value['ignorable_updates']]
+
             self.printer.header(f'Module: {key}')
-            if not value["all"]:
+            if not value["all"] and not value["outputs"]:
                 self.printer.success('No changes detected')
+
             if value["unauthorized"]:
                 self.printer.error('Unauthorized deletions:')
                 for m in value["unauthorized"]:
@@ -1145,45 +1352,70 @@ class TgWrap():
                 for m in value["updates"]:
                     total_drifts["updates"] = total_drifts["updates"] + 1
                     self.printer.normal(f'-> {m}')
+            if value["ignorable_updates"]:
+                if self.printer.print_verbose:
+                    self.printer.normal('Updates (ignored):')
+                    for m in value["ignorable_updates"]:
+                        self.printer.normal(f'-> {m}')
+                else:
+                    self.printer.normal(f'Updates (ignored): {len(value["ignorable_updates"])} resources (add --verbose to see them)')
+            if value["outputs"]:
+                self.printer.normal('Output changes:')
+                for m in value["outputs"]:
+                    total_drifts["outputs"] = total_drifts["outputs"] + 1
+                    self.printer.normal(f'-> {m}')
 
-        for key, value in changes.items():
-            for type in ["minor", "medium", "major", "unknown", "total"]:
-                total_drifts[type] += value["drifts"][type]
-            
-            # the formula below is just a way to achieve a numeric results that is coming from the various drift categories
-            value['drifts']['score'] = calculate_score(
-                major = value['drifts']['major'],
-                medium = value['drifts']['medium'],
-                minor = value['drifts']['minor'],
-            )
-            value['drifts']['score'] = value['drifts']['major'] * 10 + value['drifts']['medium'] + value['drifts']['minor'] / 10
-
-        # the formula below is just a way to achieve a numeric results that is coming from the various drift categories
-        total_drift_score = total_drifts['major'] * 10 + total_drifts['medium'] + total_drifts['minor'] / 10
-        total_drifts['score'] = total_drift_score
-
-        self.printer.header(f"Drift score: {total_drift_score} ({total_drifts['major']}.{total_drifts['medium']}.{total_drifts['minor']})")
-        if total_drifts["unknown"] > 0:
-            self.printer.warning(f"For {total_drifts['unknown']} resources, drift score is not configured, please update configuration!")
-            self.printer.warning('- Unknowns:')
-            for key, value in changes.items():
-                for m in value["unknowns"]:
-                    self.printer.warning(f' -> {m}')
+        if not analyze_config:
+            self.printer.error(
+                f"Analyze config file is not set, this is required for checking for unauthorized deletions and drift detection scores!",
+                print_line_before=True,
+                )
+        else:
+            for key, value in details.items():
+                for type in ["minor", "medium", "major", "unknown", "total"]:
+                    total_drifts[type] += value["drifts"][type]
+                
+                # the formula below is just a way to achieve a numeric results that is coming from the various drift categories
+                value['drifts']['score'] = calculate_score(
+                    major = value['drifts']['major'],
+                    medium = value['drifts']['medium'],
+                    minor = value['drifts']['minor'],
+                )
+                value['drifts']['score'] = value['drifts']['major'] * 10 + value['drifts']['medium'] + value['drifts']['minor'] / 10
 
-        if out:
+            # the formula below is just a way to achieve a numeric results that is coming from the various drift categories
+            total_drift_score = total_drifts['major'] * 10 + total_drifts['medium'] + total_drifts['minor'] / 10
+            total_drifts['score'] = total_drift_score
+
+            self.printer.header(f"Drift score: {total_drift_score} ({total_drifts['major']}.{total_drifts['medium']}.{total_drifts['minor']})")
+            if total_drifts["unknown"] > 0:
+                self.printer.warning(f"For {total_drifts['unknown']} resources, drift score is not configured, please update configuration!")
+                self.printer.warning('- Unknowns:')
+                for key, value in details.items():
+                    for m in value["unknowns"]:
+                        self.printer.warning(f' -> {m}')
+
+        if out or data_collection_endpoint:
             # in the output we convert the dict of dicts to a list of dicts as it makes processing
             # (e.g. by telegraph) easier.
             output = {
-                "changes": [],
+                "details": [],
                 "summary": {},
             }
-            for key, value in changes.items():
+            for key, value in details.items():
                 value['module'] = key
-                output["changes"].append(value)
+                output["details"].append(value)
 
             output["summary"] = total_drifts
 
-            print(json.dumps(output, indent=4))
+            if out:
+                print(json.dumps(output, indent=4))
+
+            if data_collection_endpoint:
+                self._post_analyze_results_to_dce(
+                    data_collection_endpoint=data_collection_endpoint,
+                    payload=output,
+                )
 
         if not ts_validation_successful:
             self.printer.error("Analysis detected unauthorised deletions, please check your configuration!!!")
@@ -1299,7 +1531,7 @@ class TgWrap():
                 source_config_dir = None
 
             version_tag, _, _ = self._clone_repo(
-                manifest=manifest,
+                repo=manifest['git_repository'],
                 target_dir=temp_dir,
                 version_tag=version_tag,
                 )
@@ -1314,6 +1546,10 @@ class TgWrap():
             # and make it unique
             substacks = set(substacks)
 
+            # the manifest file supports both `sub_stacks` and `substack` config name. Annoying to be a bit autistic when it comes to naming :-/
+            substack_configs = manifest.get('sub_stacks', {})
+            substack_configs.update(manifest.get('substacks', {}))
+
             for target_stage in target_stages:
                 target_dir = os.path.join(working_dir, '', target_stage)
                 self.printer.header(f'Deploy stage {target_stage} to {target_dir}...')
@@ -1323,6 +1559,8 @@ class TgWrap():
                     pass
 
                 deploy_actions = {}
+                deploy_global_configs = include_global_config_files
+                target_stage_found = False
                 # now go through the deploy configurations and apply the one that is relevant
                 for key, value in manifest['deploy'].items():
                     if target_stage not in value['applies_to_stages']:
@@ -1337,13 +1575,15 @@ class TgWrap():
                                 target_dir=target_dir,
                                 target_stage=target_stage,
                                 substacks=substacks,
-                                substack_configs=manifest.get('sub_stacks', {}).items(),
+                                substack_configs=substack_configs.items(),
                                 tg_file_name=self.TERRAGRUNT_FILE,
                                 verbose=self.printer.print_verbose,
                             )
-                        ) 
+                        )
+                        deploy_global_configs = value.get('include_global_config_files', deploy_global_configs)
+                        target_stage_found = True
 
-                if include_global_config_files:
+                if target_stage_found and deploy_global_configs:
                     for gc, global_config in manifest.get('global_config_files', {}).items():
                         self.printer.verbose(f'Found global config : {gc}')
 
@@ -1366,42 +1606,46 @@ class TgWrap():
                 else:
                     self.printer.verbose(f'Skipping global configs')
 
-                self.printer.header('Modules to deploy:')
-                self.printer.normal(f'-> git repository: {manifest["git_repository"]}')
-                self.printer.normal(f'-> version tag: {version_tag}')
-                self.printer.normal('Modules:')
-                for key, value in deploy_actions.items():
-                    self.printer.normal(f'--> {key}')
-
-                if not auto_approve:
-                    response = input("\nDo you want to continue? (y/N) ")
-                    if response.lower() != "y":
-                        sys.exit(1)
-
-                for key, value in deploy_actions.items():
-                    run_sync(
-                        source_path=value['source'],
-                        target_path=value['target'],
-                        excludes=value.get('excludes', []),
-                        include_lock_file=True,
-                        auto_approve=True,
-                        dry_run=dry_run,
-                        clean=False,
-                        terragrunt_file=self.TERRAGRUNT_FILE,
-                        verbose=self.printer.print_verbose,
-                    )
+                if deploy_actions:
+                    self.printer.header('Modules to deploy:')
+                    self.printer.normal(f'-> git repository: {manifest["git_repository"]}')
+                    self.printer.normal(f'-> version tag: {version_tag}')
+                    self.printer.normal('Modules:')
+                    for key, value in deploy_actions.items():
+                        self.printer.normal(f'--> {key}')
+
+                    if not auto_approve:
+                        response = input("\nDo you want to continue? (y/N) ")
+                        # if response.lower() != "y":
+                        #     sys.exit(1)
+
+                    if auto_approve or response.lower() == "y":
+                        for key, value in deploy_actions.items():
+                            run_sync(
+                                source_path=value['source'],
+                                target_path=value['target'],
+                                excludes=value.get('excludes', []),
+                                include_lock_file=True,
+                                auto_approve=True,
+                                dry_run=dry_run,
+                                clean=False,
+                                terragrunt_file=self.TERRAGRUNT_FILE,
+                                verbose=self.printer.print_verbose,
+                            )
 
-                if not dry_run:
-                            # write the version file
-                    with open(os.path.join(target_dir, self.VERSION_FILE), 'w') as f:
-                        f.write(f"""
-    locals {{
-        version_tag="{version_tag}"
-    }}
-    """)
+                        if not dry_run:
+                                    # write the version file
+                            with open(os.path.join(target_dir, self.VERSION_FILE), 'w') as f:
+                                f.write(f"""
+locals {{
+  version_tag="{version_tag}"
+}}
+""")
+                else:
+                    self.printer.normal('Nothing to do')
 
-                    # clean up the cache in the deployed directory to avoid strange issues when planning
-                    self.clean(working_dir=target_dir)
+                # clean up the cache in the deployed directory to avoid strange issues when planning
+                self.clean(working_dir=target_dir)
 
         except KeyError as e:
             self.printer.error(f'Error interpreting the manifest file. Please ensure it uses the proper format. Could not find element: {e}')
@@ -1417,30 +1661,72 @@ class TgWrap():
             except Exception:
                 pass
 
-    def check_deployments(self, manifest_file, working_dir, out):
+    def check_deployments(self, repo_url, levels_deep, working_dir, out):
         """ Check the freshness of deployed configuration versions against the platform repository """
 
-        def locate_version_files(current_directory, found_files=[], root_directory=None, level=1):
+        def locate_version_files(current_directory, found_files=[], root_directory=None, level=1, git_status=''):
             " This tries to find a version file in the current directory, or a given number of directories beneath it"
 
+            # do not include hidden directories
+            if os.path.basename(current_directory).startswith('.'):
+                return found_files
+
             if not root_directory:
                 root_directory = current_directory
 
+            if not git_status:
+                self.printer.verbose(f'Check for git status in directory {current_directory}')
+                # Execute 'git status' to get an overview of the current status
+                cmd = "git status"
+                rc = subprocess.run(
+                    shlex.split(cmd),
+                    cwd=current_directory,
+                    universal_newlines=True,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    )
+                output = ('stdout: ' + rc.stdout + 'stderr: ' + rc.stderr).lower()
+                if 'not a git repository' in output:
+                    pass
+                elif 'branch is up to date' in output:
+                    git_status = 'up to date; '
+                elif 'head detached' in output:
+                    git_status = 'head detached; '
+                elif 'untracked files' in output:
+                    git_status = git_status + 'untracked files; '
+                elif 'changes to be committed' in output:
+                    git_status = git_status + 'staged changes; '
+                elif 'changes not staged for commit' in output:
+                    git_status = git_status + 'unstaged changes; '
+                elif 'branch is ahead of' in output:
+                    git_status = git_status + 'ahead of remote; '
+                elif 'branch is behind of' in output:
+                    git_status = git_status + 'behind remote; '
+                elif 'unmerged paths' in output:
+                    git_status = git_status + 'merge conflicts; '
+
             for entry in os.listdir(current_directory):
                 full_entry = os.path.join(current_directory, entry)
-                if os.path.isdir(full_entry) and level < self.LOCATE_VERSION_FILE_MAX_LEVELS:
+
+                if os.path.isdir(full_entry) and level <= levels_deep:
                     found_files = locate_version_files(
                         current_directory=full_entry,
                         found_files=found_files,
                         root_directory=root_directory,
                         level=level+1,
+                        git_status=git_status,
                     )
                 elif entry == self.VERSION_FILE:
-                    found_files.append(os.path.relpath(current_directory, root_directory))
+                    found_files.append(
+                        {
+                            'path': os.path.relpath(current_directory, root_directory),
+                            'git_status': git_status,
+                        }
+                    )
                     
             return found_files
 
-        def get_all_version(repo_dir, min_version=None):
+        def get_all_versions(repo_dir, min_version=None):
             "Get all the version tags from the repo including their data"
 
             # Execute 'git tag' command to get a list of all tags
@@ -1471,25 +1757,27 @@ class TgWrap():
         try:
             # do we have a working dir? 
             working_dir = working_dir if working_dir else os.getcwd()
-            self.printer.header(f'Check released versions ({self.LOCATE_VERSION_FILE_MAX_LEVELS} levels) in directory: {working_dir}')
+            self.printer.header(f'Check released versions (max {levels_deep} levels deep) in directory: {working_dir}')
 
-            result = locate_version_files(working_dir)
+            found_files = locate_version_files(working_dir)
 
             versions = []
-            for location in result:
+            for result in found_files:
                 # Determine the deployed version as defined in the version file
-                with open(os.path.join(working_dir, location, self.VERSION_FILE), 'r') as file:
+                with open(os.path.join(working_dir, result['path'], self.VERSION_FILE), 'r') as file:
+                    # todo: replace this with regex as it is (now) the only reason we use this lib
                     content = hcl2.load(file)
                     try:
                         version_tag = content['locals'][0]['version_tag']
                         versions.append(
                             {
-                                'path': location,
+                                'path': result['path'],
+                                'git_status': result['git_status'],
                                 'tag': version_tag
                             }
                         )
                     except KeyError as e:
-                        versions.append({location: 'unknown'})
+                        versions.append({result: 'unknown'})
 
             self.printer.verbose(f'Detected versions: {versions}')
 
@@ -1506,11 +1794,14 @@ class TgWrap():
             self.printer.verbose(f'Detected minimum version {min_version} and maximum version {max_version}')
 
             temp_dir = os.path.join(tempfile.mkdtemp(prefix='tgwrap-'), "tg-source")
-            manifest = self.load_yaml_file(os.path.join(working_dir, manifest_file))
-            self._clone_repo(manifest=manifest, target_dir=temp_dir)
+            self._clone_repo(
+                repo=repo_url,
+                target_dir=temp_dir,
+                version_tag='latest',
+            )
 
             # determine the version tag from the repo, including their date
-            all_versions = get_all_version(repo_dir=temp_dir, min_version=min_version['tag'])
+            all_versions = get_all_versions(repo_dir=temp_dir, min_version=min_version['tag'])
 
             # so now we can determine how old the deployed versions are
             now = datetime.now(timezone.utc)
@@ -1524,31 +1815,45 @@ class TgWrap():
                     version['days_since_release'] = (now - release_date).days
 
             self.printer.header(
-                'Deployed versions:' if len(versions) > 0 else 'No deployed versions detected'
+                'Deployed versions:' if len(versions) > 0 else 'No deployed versions detected',
+                print_line_before=True,
                 )
-            
+
+            # sort the list based on its path
+            versions = sorted(versions, key=lambda x: x['path'])
+
             for version in versions:
                 days_since_release = version.get("days_since_release", 0)
                 message = f'-> {version["path"]}: {version["tag"]} (released {days_since_release} days ago)'
                 if version['release_date'] == 'unknown':
                     self.printer.normal(message)
-                elif days_since_release > 60:
+                elif days_since_release > 120:
                     self.printer.error(message)
-                elif days_since_release > 30:
-                    self.printer.error(message)
-                elif days_since_release < 7:
+                elif days_since_release > 80:
+                    self.printer.warning(message)
+                elif days_since_release < 40:
                     self.printer.success(message)
                 else:
                     self.printer.normal(message)
 
+                if version.get('git_status'):
+                    message = f'WARNING: git status: {version["git_status"].strip()}'
+                    if not 'up to date' in message:
+                        self.printer.warning(message)
+
             self.printer.normal("\n") # just to get an empty line :-/
             self.printer.warning("""
 Note:
     This result only says something about the freshness of the deployed configurations,
     but not whether the actual resources are in sync with these.
+
     Check the drift of these configurations with the actual deployments by
     planning and analyzing the results.
-            """)
+
+    Also, it uses the locally checked out repositories, make sure these are pulled so that
+    this reflect the most up to date situation!
+            """,
+            print_line_before=True, print_line_after=True)
 
             if out:
                 # use the regular printer, to avoid it being sent to stderr
@@ -1570,15 +1875,88 @@ Note:
             except Exception:
                 pass
 
-    def show_graph(self, backwards, exclude_external_dependencies, working_dir, include_dirs, exclude_dirs, terragrunt_args):
+    def show_graph(self, backwards, exclude_external_dependencies, analyze, working_dir, include_dirs, exclude_dirs, terragrunt_args):
         """ Shows the dependencies of a project """
 
+        def set_json_dumps_default(obj):
+            if isinstance(obj, set):
+                return list(obj)
+            raise TypeError
+
+        def calculate_dependencies(graph):
+            dependencies = {}
+            for node in graph.nodes:
+                out_degree = graph.out_degree(node)
+                in_degree = graph.in_degree(node)
+                total_degree = out_degree + in_degree
+                dependencies[node] = {
+                    'dependencies': out_degree,
+                    'dependent_on_it': in_degree,
+                    'total': total_degree,
+                }
+
+            return dependencies
+
+        def calculate_graph_metrics(graph):
+
+            metrics = {}
+
+            # Degree centrality
+            metric = {
+                'values': dict(sorted(nx.degree_centrality(graph).items(), key=lambda item: item[1], reverse=True)),
+                'description': 'Shows the degree of each node relative to the number of nodes in the graph',
+            }
+            sorted_dependencies = sorted(dependencies.items(), key=lambda x: x[1]['total'], reverse=True)
+            metrics['degree_centrality'] = metric
+            
+            # Betweenness centrality
+            metric = {
+                'values': dict(sorted(nx.betweenness_centrality(graph).items(), key=lambda item: item[1], reverse=True)),
+                'description': 'Indicates nodes that frequently lie on shortest paths between other nodes',
+            }
+            metrics['betweenness_centrality'] = metric
+            
+            # Closeness centrality
+            metric = {
+                'values': dict(sorted(nx.closeness_centrality(graph).items(), key=lambda item: item[1], reverse=True)),
+                'description': 'Reflects how quickly a node can reach other nodes in the graph',
+            }
+            metrics['closeness_centrality'] = metric
+            
+            # Strongly Connected Components (SCC)
+            metric = {
+                'values': list(nx.strongly_connected_components(graph)),
+                'description': 'Lists sets of nodes that are mutually reachable',
+            }
+            metrics['strongly_connected_components'] = metric
+            
+            # Weakly Connected Components (WCC)
+            metric = {
+                'values': list(nx.weakly_connected_components(graph)),
+                'description': 'Lists sets of nodes that are connected disregarding edge directions',
+            }
+            metrics['weakly_connected_components'] = metric
+            
+            # Average Path Length (only if the graph is connected)
+            if nx.is_strongly_connected(graph):
+                metric = {
+                    'values': nx.average_shortest_path_length(graph),
+                    'description': 'Shows the average shortest path length, indicating the graph\'s efficiency',
+                }
+                metrics['average_path_length'] = metric
+            
+            return metrics
+
         self.printer.verbose(f"Attempting to show dependencies")
         if terragrunt_args:
             self.printer.verbose(f"- with additional parameters: {' '.join(terragrunt_args)}")
 
         "Runs the desired command in the directories as defined in the directed graph"
         graph = self._get_di_graph(backwards=backwards, working_dir=working_dir)
+        try:
+            graph.remove_node(r'\n')
+        except nx.exception.NetworkXError:
+            pass
 
         # first go through the groups and clean up where needed
         groups = self._prepare_groups(
@@ -1598,12 +1976,33 @@ Note:
                 for directory in group:
                     self.printer.normal(f"- {directory}")
 
+        if analyze:
+            self.printer.header("Graph analysis", print_line_before=True)
+
+            self.printer.bold("Dependencies counts:", print_line_before=True)
+            dependencies = calculate_dependencies(graph)
+            sorted_dependencies = sorted(dependencies.items(), key=lambda x: x[1]['total'], reverse=True)
+            for node, counts in sorted_dependencies:
+                msg = f"""
+{node} ->
+\ttotal:        {counts['total']}
+\tdependent on: {counts['dependent_on_it']}
+\tdependencies: {counts['dependencies']}
+"""
+                self.printer.normal(msg)
+
+            metrics = calculate_graph_metrics(graph)
+            for metric, item in metrics.items():
+                self.printer.bold(f'Metric: {metric}')
+                self.printer.normal(f'Description: {item["description"]}')
+                self.printer.normal(json.dumps(item['values'], indent=2, default=set_json_dumps_default))
+
     def clean(self, working_dir):
         """ Clean the temporary files of a terragrunt/terraform project """
 
-        cmd = 'find . -name ".terragrunt-cache" -type d -exec rm -rf {} \; ; ' + \
-            'find . -name ".terraform" -type d -exec rm -rf {} \; ; ' + \
-            'find . -name "terragrunt-debug*" -type f -exec rm -rf {} \;'
+        cmd = r'find . -name ".terragrunt-cache" -type d -exec rm -rf {} \; ; ' + \
+            r'find . -name ".terraform" -type d -exec rm -rf {} \; ; ' + \
+            r'find . -name "terragrunt-debug*" -type f -exec rm -rf {} \;'
 
         # we see the behaviour that with cleaning up large directories, it returns errorcode=1 upon first try
         # never to shy away from a questionable solution to make your life easier, we just run it again :-)
@@ -1652,7 +2051,7 @@ Note:
                 current_release = match.group(1)
                 if current_release not in release_commits:
                     # remove the part between ()
-                    pattern = re.compile('\(.*?\) ')
+                    pattern = re.compile(r'\(.*?\) ')
                     updated_entry = pattern.sub('', entry)
                     release_commits[current_release] = [updated_entry]
             elif current_release:
@@ -1708,3 +2107,65 @@ Note:
             # use the regular printer, to avoid it being sent to stderr
             print(changelog)
 
+    def inspect(self, domain:str,substack:str, stage:str, azure_subscription_id:str, config_file:str,
+                out:bool, data_collection_endpoint:str):
+        """ Inspects the status of an Azure deployment """
+
+        inspector = AzureInspector(
+            subscription_id=azure_subscription_id,
+            domain=domain,
+            substack=substack,
+            stage=stage,
+            config_file=config_file,
+            verbose=self.printer.print_verbose,
+        )
+
+        try:
+            results = inspector.inspect()
+
+            # Report the status
+            exit_code = 0
+            self.printer.header('Inspection status:', print_line_before=True)
+            for k,v in results.items():
+                msg = f"""{v['type']}: {k}
+        -> Resource:  {v.get('inspect_status_code', 'NC')} ({v.get('inspect_message', 'not found')})""" # only since python 3.12 you can use things like \t and \n in an f-string
+                if 'rbac_assignment_status_code' in v:
+                    msg = msg + f"""
+        -> RBAC: {v['rbac_assignment_status_code']} ({v.get('rbac_assignment_message')})"
+""" # only since python 3.12 you can use things like \t and \n in an f-string
+                if v['inspect_status_code'] != 'OK' or v.get('rbac_assignment_status_code', 'OK') == 'NOK':
+                    self.printer.error(msg=msg)
+                    exit_code += 1
+                else:
+                    self.printer.success(msg=msg)
+
+            if out or data_collection_endpoint:
+                # convert results to something DCE understands, and add the inputs
+                payload = []
+                for key, value in results.items():
+                    value_with_key = value.copy()
+                    value_with_key["resource_type"] = value_with_key.pop("type")
+                    value_with_key["resource"] = key
+                    value_with_key["domain"] = domain
+                    value_with_key["substack"] = substack
+                    value_with_key["stage"] = stage
+                    value_with_key["subscription_id"] = azure_subscription_id
+                    payload.append(value_with_key)
+
+                if out:
+                    print(json.dumps(payload, indent=2))
+
+                if data_collection_endpoint:
+                    self._post_to_dce(
+                        data_collection_endpoint=data_collection_endpoint,
+                        payload=payload,
+                    )
+
+            return exit_code
+        except Exception as e:
+            self.printer.normal(f'Exception occurred: {e}')
+
+            if self.printer.print_verbose:
+                traceback.print_exc()
+            
+            return -1