cid-cmd 4.2.7__tar.gz → 4.3.0__tar.gz

{cid_cmd-4.2.7/cid_cmd.egg-info → cid_cmd-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cid-cmd
-Version: 4.2.7
+Version: 4.3.0
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-4.3.0/cid/_version.py

@@ -0,0 +1,2 @@
+__version__ = '4.3.0'
+

cid_cmd-4.3.0/cid/builtin/core/data/permissions/data_set_rls_permissions.json

@@ -0,0 +1,8 @@
+{
+     "RowLevelPermissionDataSet": {
+      "Arn": "${RlsDataSetArn}",
+      "FormatVersion": "VERSION_1",
+      "PermissionPolicy": "GRANT_ACCESS",
+      "Status": "${RlsStatus}"
+   }
+}

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid/cli.py

@@ -94,7 +94,6 @@ def map(ctx, **kwargs):
     """
     ctx.obj.map(**kwargs)
 
-
 @click.option('-v', '--verbose', count=True)
 @click.option('-y', '--yes', help='confirm all', is_flag=True, default=False)
 @cid_command
@@ -141,6 +140,8 @@ def deploy(ctx, **kwargs):
      --catalog TEXT                        Comma separated list of catalog files or urls (ex: foundational,advanced )
      --theme TEXT                          A QuickSight Theme (CLASSIC|MIDNIGHT|SEASIDE|RAINIER)
      --currency TEXT                       A currency symbol instead of default USD (USD|GBP|EUR|JPY|KRW|DKK|TWD|INR)
+     --rls TEXT                            Desired RLS status (CLEAR|ENABLED|DISABLED) Clear = No RLS configuration.
+     --rls-dataset-id TEXT                 ID of RLS Dataset that will be used
      --share-with-account  (yes|no)        Make dashboard visible to other users in the same account by default.
      """
     ctx.obj.deploy(**kwargs)
@@ -210,6 +211,8 @@ def update(ctx, dashboard_id, force, recursive, **kwargs):
      --on-drift (show|override)            Action if a drift of view and dataset is discovered. 'override' = override drift(will destroy customization) or 'show' (default) = show a diff. In Unattended mode (without terminal on-drift will have allways override behaviour)
      --theme TEXT                          A QuickSight Theme (CLASSIC|MIDNIGHT|SEASIDE|RAINIER)
      --currency TEXT                       A currency symbol instead of default USD (USD|GBP|EUR|JPY|KRW|DKK|TWD|INR)
+     --rls TEXT                            Desired RLS status (CLEAR|ENABLED|DISABLED) Clear = No RLS configuration.
+     --rls-dataset-id TEXT                 ID of RLS Dataset that will be used
 
     """
     ctx.obj.update(dashboard_id, force=force, recursive=recursive, **kwargs)

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid/common.py

@@ -1291,7 +1291,18 @@ class Cid():
         # If there still datasets missing try automatic creation
         if missing_datasets:
             missing_str = ', '.join(missing_datasets)
-            print(f'\nThere are still {len(missing_datasets)} datasets missing: {missing_str}')
+            cid_print(f'\nThere are still {len(missing_datasets)} datasets missing: {missing_str}')
+
+            # get rls status of existing datasets
+            found_dataset_objects = [self.qs.describe_dataset(ds_id) for ds_id in found_datasets]
+            rls_dataset_arns = [ds.rls_arn for ds in found_dataset_objects if ds.rls_arn]
+            if rls_dataset_arns:
+                rls_dataset_arn = max(set(rls_dataset_arns), key=rls_dataset_arns.count) #get the most frequent
+                if not get_parameters().get('rls-dataset-id') and not get_parameters().get('rls'):
+                    rls_dataset_id = rls_dataset_arn.split('/')[-1]
+                    cid_print('Existing datasets are linked to RLS {rls_dataset_id}. We will reuse it for {missing_str} datasets.')
+                    set_parameters({'rls-dataset-id': rls_dataset_id})
+
             for dataset_name in missing_datasets[:]:
                 dataset_id = known_datasets.get(dataset_name)
                 print(f'Creating dataset: {dataset_name}')
@@ -1633,6 +1644,50 @@ class Cid():
         compiled_dataset = Dataset.patch(dataset=compiled_dataset, custom_fields=custom_fields, athena=self.athena)
         logger.trace(f"compiled_dataset = {json.dumps(compiled_dataset)}")
         found_dataset = self.qs.describe_dataset(compiled_dataset.get('DataSetId'), timeout=0)
+
+        rls_dataset_id = get_parameters().get('rls-dataset-id')
+        rls_dataset_status = get_parameters().get('rls')
+        if rls_dataset_status == 'CLEAR':
+            logger.debug('deleting rls')
+            if "RowLevelPermissionDataSet" in compiled_dataset:
+                del compiled_dataset["RowLevelPermissionDataSet"]
+            if "RowLevelPermissionDataSet" in found_dataset:
+                del found_dataset["RowLevelPermissionDataSet"]
+
+        if rls_dataset_id or rls_dataset_status:
+            if not rls_dataset_id:
+                for ds in self.qs.list_data_sets():
+                    print(ds)
+                choices = {
+                    f"{ds.get('Name')}({ds.get('DataSetId')})":ds.get('DataSetId')
+                    for ds in self.qs.list_data_sets()
+                    if ds.get('UseAs') == 'RLS_RULES'
+                }
+                if not choices:
+                    raise CidCritical(f"Cannot find RLS DataSets")
+                rls_dataset_id = get_parameter('rls-dataset-id', message='Select RLS dataset', choices=choices)
+            rls_dataset = self.qs.describe_dataset(id=rls_dataset_id)
+            if not rls_dataset:
+                raise CidCritical(f"RLS DataSet {rls_dataset_id} not found")
+            if not rls_dataset.is_rls:
+                raise CidCritical(f"DataSet {rls_dataset_id} is not RLS")
+            rls_permissions = {
+                "Arn": rls_dataset.arn,
+                "Status": (rls_dataset_status
+                    or (found_dataset and found_dataset.rls_status)
+                    or "ENABLED"
+                ),
+                "PermissionPolicy": "GRANT_ACCESS",
+            }
+            cols = [c['Name'] for c in rls_dataset.columns]
+            if 'UserName' in cols or 'GroupName' in cols:
+                rls_permissions['FormatVersion'] = "VERSION_1"
+            elif 'UserARN' in cols or 'GroupARN' in cols:
+                rls_permissions['FormatVersion'] = "VERSION_2"
+            else:
+                raise CidCritical(f"DataSet {rls_dataset_id} must have 'UserName'/'GroupName' or 'UserARN'/'GroupARN' columns.")
+            compiled_dataset["RowLevelPermissionDataSet"] = rls_permissions
+
         if isinstance(found_dataset, Dataset):
             update_dataset = False
             if update:

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid/export.py

@@ -362,7 +362,7 @@ def export_analysis(qs, athena, glue):
         # We can potentially reconsider this and use IDs at some point
         'datasets': sorted(list(set(list(datasets.keys()) + resources_datasets)))
     }
-    dashboard_resource['name'] = dashboard_resource.get('name')
+    dashboard_resource['name'] = dashboard_resource.get('name') or analysis['Name']
     dashboard_resource['dashboardId'] = dashboard_id
     dashboard_resource['category'] = get_parameters().get('category', dashboard_resource.get('category', 'Custom'))
 

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid/helpers/athena.py

@@ -601,7 +601,7 @@ class Athena(CidBase):
         ''')
 
     def find_tables_with_columns(self, columns: list, database_name: str=None, catalog_name: str=None, max_items: int=10000):
-        """ This function searches a table with a given set of columns
+        """ Returns an iterator that yields only tables containing all specified columns.
         """
         iterator = self.client.get_paginator('list_table_metadata').paginate(
             DatabaseName=database_name or self.DatabaseName,
@@ -610,6 +610,9 @@ class Athena(CidBase):
                 'MaxItems': max_items, # sometimes customers can have 1'000s of tables (due to a crawler going crazy for example)
             },
         )
-        return iterator.search(f"""
-            TableMetadataList[?{' && '.join(["contains(Columns[].Name, '"+col+"' )" for col in columns])}].Name
-        """)
+        # We cannot rely on search to find directly columns as there might be Nulls. So iterating old fashion.
+        for table in iterator.search('TableMetadataList'):
+            column_names = [c['Name'] for c in table.get('Columns', [])]
+            if all([(col in column_names) for col in columns]):
+                yield table
+

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid/helpers/cur.py

@@ -271,7 +271,7 @@ class CUR(AbstractCUR):
                     columns=self.cur_minimal_required_columns,
                     database_name=database,
                 )
-                all_cur_tables += [(database, table) for table in tables]
+                all_cur_tables += [(database, table['Name']) for table in tables]
             except self.athena.client.exceptions.ClientError as exc:
                 if 'AccessDenied' in str(exc):
                     logger.info(f'Cannot read from athena database {database}')

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid/helpers/quicksight/__init__.py

@@ -14,7 +14,7 @@ from cid.base import CidBase
 from cid.helpers import diff, timezone, randtime
 from cid.helpers.quicksight.dashboard import Dashboard
 from cid.helpers.quicksight.dataset import Dataset
-from cid.helpers.quicksight.dashboard_patching import add_filter_to_dashboard_definition, patch_currency, patch_group_by
+from cid.helpers.quicksight.dashboard_patching import add_filter_to_dashboard_definition, patch_currency, patch_group_by, patch_spaces
 from cid.helpers.quicksight.datasource import Datasource
 from cid.helpers.quicksight.template import Template as CidQsTemplate
 from cid.helpers.quicksight.definition import Definition as CidQsDefinition
@@ -1340,6 +1340,7 @@ class QuickSight(CidBase):
                 create_parameters['Definition'],
                 currency_symbol=get_parameters().get('currency-symbol', 'USD')
             )
+            create_parameters['Definition'] = patch_spaces(create_parameters['Definition'])
             dataset_references = []
             for identifier, arn in definition.get('datasets', {}).items():
                 # Fetch dataset by name (preferably) OR by id

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid/helpers/quicksight/dashboard.py

@@ -323,8 +323,10 @@ class Dashboard(CidQsResource):
         if self.datasets:
             cid_print(f"  <BOLD>Datasets:<END>")
             for dataset_name, dataset_id in  sorted(self.datasets.items()):
+                ds = self.qs.describe_dataset(dataset_id)
+                rls = {'ENABLED': '🔐', 'DISABLED': '🔓', None: ' '}.get(ds.rls_status)
                 status = self.qs.get_dataset_last_ingestion(dataset_id) or '<BLUE>DIRECT<END>' #todo fix this Blue using dataset import type.
-                cid_print(f'    {dataset_name: <36} ({dataset_id: <36}) {status}')
+                cid_print(f'   {rls} {dataset_name: <36} ({dataset_id: <36}) {status}')
 
     def display_url(self, url_template: str, launch: bool = False, **kwargs) -> None:
         url = url_template.format(dashboard_id=self.id, **kwargs)

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid/helpers/quicksight/dashboard_patching.py

@@ -429,6 +429,24 @@ def detect_global_filter_fields(dashboard_definition):
         cols.append(col_name)
 
 
+def patch_spaces(definition):
+    ''' put spaces in places
+    '''
+    def _patch(data):
+        """Recursively set spaces in text fields"""
+        SPACE = '\u00A0' # special space: non-breaking space
+        if isinstance(data, dict):
+            return {k: _patch(v) if k not in ['Expression'] else v for k, v in data.items()} # skip patching code
+        elif isinstance(data, list):
+            return [_patch(item) for item in data]
+        elif isinstance(data, str):
+            text = re.sub(r'([^>\s])(\s*)(\s)(<[^/])', rf'\1{SPACE}\2\3\4', data, re.MULTILINE|re.DOTALL)
+            if text != data:
+                logger.trace(f'patch_spaces: {repr(text)} {repr(data)}')
+            return text
+        return data
+    return _patch(definition)
+
 def patch_currency(definition, currency_symbol):
     ''' patch dashboard by adding currency symbol where the currency is configured already.
     '''

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid/helpers/quicksight/dataset.py

@@ -1,5 +1,5 @@
+import uuid
 import logging
-from uuid import uuid4
 from copy import deepcopy
 
 import yaml
@@ -15,8 +15,14 @@ DATASET_PROPERTIES = [
 ]
 
 
-class Dataset(CidQsResource):
+def string_to_uuid(input_string):
+    """Generate a UUID from a string using hash.
+    """
+    return str(uuid.uuid5(uuid.NAMESPACE_DNS, input_string))
 
+class Dataset(CidQsResource):
+    """ Dataset
+    """
     def __init__(self, raw: dict, qs=None, athena=None) -> None:
         super().__init__(raw)
         self.qs = qs
@@ -31,6 +37,19 @@ class Dataset(CidQsResource):
     def id(self) -> str:
         return self.get_property('DataSetId')
 
+    @property
+    def is_rls(self):
+        return self.get_property('UseAs') == 'RLS_RULES'
+
+    @property
+    def rls_status(self):
+        return (self.get_property('RowLevelPermissionDataSet') or {}).get('Status')
+
+    @property
+    def rls_arn(self):
+        return (self.get_property('RowLevelPermissionDataSet') or {}).get('Arn')
+
+
     @property
     def columns(self) -> list:
         if not 'OutputColumns' in self.raw: self.describe()
@@ -121,11 +140,13 @@ class Dataset(CidQsResource):
                     renames[key] = dt["RenameColumnOperation"]['NewColumnName']
         logger.trace(f'renames = {renames}')
 
-        projected_cols = next( # get the first DataTransform with ProjectOperation
-            ds['ProjectOperation']["ProjectedColumns"]
-            for ds in root_lt['DataTransforms']
-            if 'ProjectOperation' in ds
-        )
+        projected_cols = []
+        if root_lt.get('DataTransforms'):
+            projected_cols = next( # get the first DataTransform with ProjectOperation
+                ds['ProjectOperation']["ProjectedColumns"]
+                for ds in root_lt.get('DataTransforms', [])
+                if 'ProjectOperation' in ds
+            )
 
         # Update each PhysicalTableMap with all columns from athena views
         all_columns = []
@@ -179,7 +200,7 @@ class Dataset(CidQsResource):
                         "Columns": [
                             {
                                 "ColumnName": col_name,
-                                "ColumnId": str(uuid4()),
+                                "ColumnId": string_to_uuid(dataset['Name'] + col_name),
                                 "Expression": expression
                             }
                         ]

{cid_cmd-4.2.7 → cid_cmd-4.3.0/cid_cmd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cid-cmd
-Version: 4.2.7
+Version: 4.3.0
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid_cmd.egg-info/SOURCES.txt

@@ -36,6 +36,7 @@ cid/builtin/core/data/datasets/trends/monthly_bill_by_account.json
 cid/builtin/core/data/permissions/dashboard_permissions.json
 cid/builtin/core/data/permissions/dashboard_permissions_namespace.json
 cid/builtin/core/data/permissions/data_set_permissions.json
+cid/builtin/core/data/permissions/data_set_rls_permissions.json
 cid/builtin/core/data/permissions/data_source_permissions.json
 cid/builtin/core/data/permissions/folder_permissions.json
 cid/builtin/core/data/queries/cid/compute_savings_plan_eligible_spend.sql

{cid_cmd-4.2.7 → cid_cmd-4.3.0}/cid_cmd.egg-info/top_level.txt

@@ -5,4 +5,5 @@ cid
 dashboards
 dist
 docs
-terraform-modules
+legacy-terraform
+terraform

cid_cmd-4.2.7/cid/_version.py

@@ -1,2 +0,0 @@
-__version__ = '4.2.7'
-