tgwrap 0.8.13__py3-none-any.whl → 0.9.0__py3-none-any.whl

tgwrap/cli.py

@@ -292,9 +292,10 @@ def run_all(command, verbose, debug, dry_run, no_lock, update, upgrade, exclude_
 @click.option('--json', '-j', is_flag=True, default=False,
     help='Show output in json format',
     )
-@click.option('--planfile-dir', '-P', default=None,
+@click.option('--planfile-dir', '-P', default='.terragrunt-cache/current',
     help='Relative path to directory with plan file (or set TGWRAP_PLANFILE_DIR environment variable), see README for more details',
     envvar='TGWRAP_PLANFILE_DIR', type=click.Path(),
+    show_default=True,
     )
 @click.argument('terragrunt-args', nargs=-1, type=click.UNPROCESSED)
 @click.version_option(version=__version__)
@@ -402,11 +403,16 @@ def run_import(address, id, verbose, dry_run, working_dir, no_lock, terragrunt_a
     envvar='TGWRAP_PLANFILE_DIR', type=click.Path(),
     show_default=True,
     )
+@click.option('--data-collection-endpoint', '-D', default=None,
+    help='Optional URI of an (Azure) data collection endpoint, to which the analyse results will be sent',
+    envvar='TGWRAP_ANALYZE_DATA_COLLECTION_ENDPOINT',
+    show_default=True,
+    )
 @click.argument('terragrunt-args', nargs=-1, type=click.UNPROCESSED)
 @click.version_option(version=__version__)
 def run_analyze(verbose, exclude_external_dependencies, working_dir, start_at_step,
-            out, analyze_config, parallel_execution,
-            include_dir, exclude_dir, planfile_dir, terragrunt_args):
+            out, analyze_config, parallel_execution, include_dir, exclude_dir,
+            planfile_dir, data_collection_endpoint, terragrunt_args):
     """ Analyzes the plan files """
 
     check_latest_version(verbose)
@@ -422,6 +428,7 @@ def run_analyze(verbose, exclude_external_dependencies, working_dir, start_at_st
         include_dirs=include_dir,
         exclude_dirs=exclude_dir,
         planfile_dir=planfile_dir,
+        data_collection_endpoint=data_collection_endpoint,
         terragrunt_args=terragrunt_args,
     )
 

tgwrap/deploy.py

@@ -179,11 +179,18 @@ def prepare_deploy_config(step, config, source_dir, source_config_dir, target_di
             source_path = os.path.join(
                 source_dir, source_stage, substack['source'], ''
                 )
+            source_modules = {
+                entry:{} for entry in os.listdir(source_path) if os.path.isdir(os.path.join(source_path, entry))
+            }
             target_path = os.path.join(
                 target_dir, substack['target'], ''
                 )
 
-            include_modules = substack['include_modules'] if len(substack.get('include_modules', {})) > 0 else []
+            printer.verbose(f'Include substack modules: {include_modules}')
+
+    # printer.verbose(f'Found modules: {source_modules}')
+        # include_modules = config['include_modules'] if len(config.get('include_modules')) > 0 else source_modules
+            include_modules = substack['include_modules'] if len(substack.get('include_modules', {})) > 0 else source_modules
             printer.verbose(f'Include modules: {include_modules}')
 
             if include_modules:

tgwrap/main.py

@@ -17,6 +17,7 @@ import sys
 import subprocess
 import shlex
 import shutil
+import requests
 import re
 import tempfile
 import json
@@ -34,13 +35,13 @@ from datetime import datetime, timezone
 from .printer import Printer
 from .analyze import run_analyze
 from .deploy import prepare_deploy_config, run_sync
+
 class DateTimeEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, datetime):
             return obj.isoformat()
         return super().default(obj)
 
-
 class TgWrap():
     """
     A wrapper around terragrunt with the sole purpose to make it a bit
@@ -775,6 +776,150 @@ class TgWrap():
             total_items = sum(len(group) for group in groups)
             self.printer.verbose(f'Executed {group_nbr} groups and {total_items} steps')
 
+    def _post_analyze_results_to_dce(self, data_collection_endpoint:str, payload:object):
+        """
+        Posts the payload to the given (Azure) data collection endpoint
+        """
+
+        #
+        # Everything we do here, can be done using native python. And probably this is preferable as well.
+        # But I have decided to follow (at least for now) the overall approach of the app and that is
+        # executing systems commands.
+        # This does require the az cli to be installed, but that is a fair assumption if you are working
+        # with terragrunt/terraform and want to post the analyze results to a Data Collection Endpoint.
+        # However, not ruling out this will change, but then the change should be transparant.
+        #
+
+        # Get the Azure information
+        rc = subprocess.run(
+            shlex.split('az account show'),
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=sys.stderr,
+        )
+        self.printer.verbose(rc)
+
+        # Do a few checks
+        if rc.returncode != 0:
+            raise Exception(f'Could not get Azure account info')
+        
+        # Get the ouptut
+        output = json.loads(rc.stdout.decode())
+        if output.get('environmentName') != 'AzureCloud':
+            raise Exception(f'Environment is not an Azure cloud:\n{json.dumps(output, indent=2)}')
+        
+        tenant_id = output.get('tenantId')
+        if not tenant_id:
+            raise Exception(f'Could not determine Azure tenant id:\n{json.dumps(output, indent=2)}')
+
+        principal = output.get('user').get('name')
+        if not principal:
+            raise Exception(f'Could not determine principal:\n{json.dumps(output, indent=2)}')
+
+        # TOKEN=$(az account get-access-token --scope "https://monitor.azure.com//.default" | jq -r '.accessToken')
+        # Get the Azure OAUTH token
+        rc = subprocess.run(
+            shlex.split('az account get-access-token --scope "https://monitor.azure.com//.default"'),
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=sys.stderr,
+        )
+        self.printer.verbose(rc.returncode) # do not print the token to output
+
+        # Do a few checks
+        if rc.returncode != 0:
+            raise Exception(f'Could not get Azure OAUTH token')
+        
+        # Get the ouptut
+        output = json.loads(rc.stdout.decode())
+        token = output.get('accessToken')
+        if not token:
+            raise Exception(f'Could not retrieve an access token:\n{json.dumps(output, indent=2)}')
+
+        # Get the repo info
+        rc = subprocess.run(
+            shlex.split('git config --get remote.origin.url'),
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=sys.stderr,
+        )
+        self.printer.verbose(rc)
+
+        # Do a few checks
+        if rc.returncode != 0:
+            raise Exception(f'Could not get git repo info')
+        
+        # Get the ouptut
+        repo = rc.stdout.decode().rstrip('\n')
+        if not repo:
+            raise Exception(f'Could not get git repo info: {repo}')
+
+        # Get the current path in the repo
+        rc = subprocess.run(
+            shlex.split('git rev-parse --show-prefix'),
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=sys.stderr,
+        )
+        self.printer.verbose(rc)
+
+        # Do a few checks
+        if rc.returncode != 0:
+            raise Exception(f'Could not get current scope')
+        
+        # Get the ouptut
+        scope = rc.stdout.decode().rstrip('\n')
+        if not scope:
+            raise Exception(f'Could not get scope: {scope}')
+
+        # So now we have everything, we can construct the final payload
+        dce_payload = [
+            {
+                "scope": scope,
+                "principal": principal,
+                "repo": repo,
+                "creations": payload.get("summary").get("creations"),
+                "updates": payload.get("summary").get("updates"),
+                "deletions": payload.get("summary").get("deletions"),
+                "minor": payload.get("summary").get("minor"),
+                "medium": payload.get("summary").get("medium"),
+                "major": payload.get("summary").get("major"),
+                "unknown": payload.get("summary").get("unknown"),
+                "total": payload.get("summary").get("total"),
+                "score": payload.get("summary").get("score"),
+                "details": payload.get('details'),
+            },
+        ]
+        
+        self.printer.verbose('About to log:')
+        self.printer.verbose(f'- to: {data_collection_endpoint}')
+        self.printer.verbose(f'- payload:\n{json.dumps(dce_payload, indent=2)}')
+
+        # now do the actual post
+        try:
+            headers = {
+                'Authorization': f"Bearer {token}",
+                'Content-Type': 'application/json',
+            }
+            resp = requests.post(
+                url=data_collection_endpoint,
+                headers=headers,
+                json=dce_payload,
+            )
+
+            resp.raise_for_status()
+
+        except requests.exceptions.RequestException as e:
+            # we warn but continue
+            self.printer.warning(f'Error while posting the analyze results ({type(e)}): {e}')
+        except Exception as e:
+            self.printer.error(f'Unexpected error: {e}')
+            if self.printer.print_verbose:
+                raise(e)
+            sys.exit(1)
+
+        self.printer.verbose('Done')
+
     def run(self, command, debug, dry_run, no_lock, update, upgrade,
         planfile, auto_approve, clean, working_dir, terragrunt_args):
         """ Executes a terragrunt command on a single module """
@@ -1001,8 +1146,8 @@ class TgWrap():
             self.printer.verbose(rc)
 
     def analyze(self, exclude_external_dependencies, working_dir, start_at_step,
-        out, analyze_config, parallel_execution,
-        include_dirs, exclude_dirs, planfile_dir, terragrunt_args):
+        out, analyze_config, parallel_execution, include_dirs, exclude_dirs, 
+        planfile_dir, data_collection_endpoint, terragrunt_args):
         """ Analyzes the plan files """
 
         def calculate_score(major: int, medium: int, minor: int) -> float :
@@ -1051,7 +1196,7 @@ class TgWrap():
             config = self.load_yaml_file(analyze_config)
 
         ts_validation_successful = True
-        changes = {}
+        details = {}
         drifts = {}
         try:
             # then run it and capture the output
@@ -1093,7 +1238,7 @@ class TgWrap():
                             if 'exception' in data:
                                 raise Exception(data['exception'])
 
-                            changes[module], ts_success = run_analyze(
+                            details[module], ts_success = run_analyze(
                                 config=config,
                                 data=data,
                                 verbose=self.printer.print_verbose,
@@ -1123,7 +1268,7 @@ class TgWrap():
         }
 
         self.printer.header("Analysis results:", print_line_before=True)
-        for key, value in changes.items():
+        for key, value in details.items():
             self.printer.header(f'Module: {key}')
             if not value["all"]:
                 self.printer.success('No changes detected')
@@ -1147,7 +1292,7 @@ class TgWrap():
                     total_drifts["updates"] = total_drifts["updates"] + 1
                     self.printer.normal(f'-> {m}')
 
-        for key, value in changes.items():
+        for key, value in details.items():
             for type in ["minor", "medium", "major", "unknown", "total"]:
                 total_drifts[type] += value["drifts"][type]
             
@@ -1167,24 +1312,31 @@ class TgWrap():
         if total_drifts["unknown"] > 0:
             self.printer.warning(f"For {total_drifts['unknown']} resources, drift score is not configured, please update configuration!")
             self.printer.warning('- Unknowns:')
-            for key, value in changes.items():
+            for key, value in details.items():
                 for m in value["unknowns"]:
                     self.printer.warning(f' -> {m}')
 
-        if out:
+        if out or data_collection_endpoint:
             # in the output we convert the dict of dicts to a list of dicts as it makes processing
             # (e.g. by telegraph) easier.
             output = {
-                "changes": [],
+                "details": [],
                 "summary": {},
             }
-            for key, value in changes.items():
+            for key, value in details.items():
                 value['module'] = key
-                output["changes"].append(value)
+                output["details"].append(value)
 
             output["summary"] = total_drifts
 
-            print(json.dumps(output, indent=4))
+            if out:
+                print(json.dumps(output, indent=4))
+
+            if data_collection_endpoint:
+                self._post_analyze_results_to_dce(
+                    data_collection_endpoint=data_collection_endpoint,
+                    payload=output,
+                )
 
         if not ts_validation_successful:
             self.printer.error("Analysis detected unauthorised deletions, please check your configuration!!!")

{tgwrap-0.8.13.dist-info → tgwrap-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tgwrap
-Version: 0.8.13
+Version: 0.9.0
 Summary: A (terragrunt) wrapper around a (terraform) wrapper around ....
 Home-page: https://gitlab.com/lunadata/tgwrap
 License: MIT
@@ -142,6 +142,56 @@ export TGWRAP_PLANFILE_DIR=".terragrunt-cache/current"
 
 Or pass it along with the `--planfile-dir|-P` option and it will use that.
 
+### Logging the results
+
+`tgwrap` supports logging the analyze results to an [Azure Log Analytics](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/log-analytics-overview) custom table.
+
+For that, the custom table need to be present, including a [data collection endpoint](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/data-collection-endpoint-overview?tabs=portal) and associated [data collection rule](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/data-collection-rule-overview?tabs=portal).
+
+When you want to activate this, just pass `--data-collection-endpoint` (or, more conveniently, set the `TGWRAP_ANALYZE_DATA_COLLECTION_ENDPOINT` environment variable) with the url to which the data can be posted.
+
+> Note that for this to work, `tgwrap` assumes that there is a functioning [azure cli](https://learn.microsoft.com/en-us/cli/azure/) available on the system.
+
+A payload as below will be posted, and the log analytics table should be able to accomodate for that:
+
+```json
+[
+  {
+    "scope": "terragrunt/dlzs/data-platform/global/platform/rbac/",
+    "principal": "myself",
+    "repo": "https://gitlab.com/my-git-repo.git",
+    "creations": 0,
+    "updates": 0,
+    "deletions": 0,
+    "minor": 0,
+    "medium": 0,
+    "major": 0,
+    "unknown": 0,
+    "total": 0,
+    "score": 0.0,
+    "details": [
+      {
+        "drifts": {
+          "minor": 0,
+          "medium": 0,
+          "major": 0,
+          "unknown": 0,
+          "total": 0,
+          "score": 0.0
+        },
+        "all": [],
+        "creations": [],
+        "updates": [],
+        "deletions": [],
+        "unauthorized": [],
+        "unknowns": [],
+        "module": ""
+      }
+    ]
+  }
+]
+```
+
 ## More than a wrapper
 
 Over time, tgwrap became more than a wrapper, blantly violating [#1 of the unix philosophy](https://en.wikipedia.org/wiki/Unix_philosophy#:~:text=The%20Unix%20philosophy%20is%20documented,%2C%20as%20yet%20unknown%2C%20program.): 'Make each program do one thing well'.

tgwrap-0.9.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+tgwrap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tgwrap/analyze.py,sha256=CsSaGv-be6ATy36z9X7x00gpKY59soJys2VbIzD-tmg,8726
+tgwrap/cli.py,sha256=jP7KuZzqwW2693fVsqEChzIto2T3YyPcSc9kW8ElDWI,29727
+tgwrap/deploy.py,sha256=giLjpN9b4TWov2w8fWyB6jinAUbupMTCUoZHVT442DM,10421
+tgwrap/main.py,sha256=FqpCZ3xGD33YD3IesFMq2QI2AvSv0_4txRpnoPX19Z4,80691
+tgwrap/printer.py,sha256=dkcOCPIPB-IP6pn8QMpa06xlcqPFVaDvxnz-QEpDJV0,2663
+tgwrap-0.9.0.dist-info/LICENSE,sha256=VT-AVxIXt3EQTC-7Hy1uPGnrDNJLqfcgLgJD78fiyx4,1065
+tgwrap-0.9.0.dist-info/METADATA,sha256=IYdY5B0k1OswfcfVCbdwFOHs_itQXKq9fgbtjZPusIw,13334
+tgwrap-0.9.0.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+tgwrap-0.9.0.dist-info/entry_points.txt,sha256=H8X0PMPmd4aW7Y9iyChZ0Ug6RWGXqhRUvHH-6f6Mxz0,42
+tgwrap-0.9.0.dist-info/RECORD,,

tgwrap-0.8.13.dist-info/RECORD

@@ -1,11 +0,0 @@
-tgwrap/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tgwrap/analyze.py,sha256=CsSaGv-be6ATy36z9X7x00gpKY59soJys2VbIzD-tmg,8726
-tgwrap/cli.py,sha256=sWUHP4EYUeKpYP3KgU3t2Er4YLlnwfXNVQkgVwmGFPM,29342
-tgwrap/deploy.py,sha256=bJiox_fz8JsoPreX4woW6-EqAebhpJWnKUVLVeGXkrI,10000
-tgwrap/main.py,sha256=Dqqzqe2x7UXHgAgQL764CfoZ6V0JFbk4TphBkBcYcC8,74872
-tgwrap/printer.py,sha256=dkcOCPIPB-IP6pn8QMpa06xlcqPFVaDvxnz-QEpDJV0,2663
-tgwrap-0.8.13.dist-info/LICENSE,sha256=VT-AVxIXt3EQTC-7Hy1uPGnrDNJLqfcgLgJD78fiyx4,1065
-tgwrap-0.8.13.dist-info/METADATA,sha256=EDgJtU6kePAZrn-aft9LWPDImK_5qEYaBoJCGhY_L3s,11616
-tgwrap-0.8.13.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-tgwrap-0.8.13.dist-info/entry_points.txt,sha256=H8X0PMPmd4aW7Y9iyChZ0Ug6RWGXqhRUvHH-6f6Mxz0,42
-tgwrap-0.8.13.dist-info/RECORD,,