cid-cmd 4.2.4__tar.gz → 4.2.6__tar.gz

{cid_cmd-4.2.4/cid_cmd.egg-info → cid_cmd-4.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cid-cmd
-Version: 4.2.4
+Version: 4.2.6
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-4.2.6/cid/_version.py

@@ -0,0 +1,2 @@
+__version__ = '4.2.6'
+

{cid_cmd-4.2.4 → cid_cmd-4.2.6}/cid/common.py

@@ -1668,17 +1668,11 @@ class Cid():
                         update_dataset = True
                     break
 
-            identical = False # check if dataset needs an update
-            if isinstance(found_dataset, Dataset):
-                identical = True
-                for key in 'PhysicalTableMap LogicalTableMap OutputColumns ImportMode DataSetUsageConfiguration RowLevelPermissionDataSet FieldFolders RowLevelPermissionTagConfiguration DatasetParameters'.split():
-                    if found_dataset.raw.get(key) != compiled_dataset.get(key):
-                        logger.trace(f'not identical {key} {found_dataset.raw.get(key)} != {compiled_dataset.get(key)}')
-                        identical = False
-                logger.trace(f'identical to existing = {identical}')
+            identical = Dataset.datasets_are_identical(found_dataset, compiled_dataset) # check if dataset needs an update
 
             if update_dataset and not identical:
-                self.qs.update_dataset(compiled_dataset)
+                merged_dataset = Dataset.merge_datasets(compiled_dataset, found_dataset)
+                self.qs.update_dataset(merged_dataset)
                 if compiled_dataset.get("ImportMode") == "SPICE":
                     dataset_id = compiled_dataset.get('DataSetId')
                     schedules_definitions = []

{cid_cmd-4.2.4 → cid_cmd-4.2.6}/cid/helpers/quicksight/__init__.py

@@ -261,7 +261,9 @@ class QuickSight(CidBase):
             return True
         except self.client.exceptions.AccessDeniedException as exc:
             logger.debug(f'Cannot tag {arn} (AccessDenied).')
-            return False
+        except self.client.exceptions.ClientError as exc:
+            logger.debug(f'Cannot tag {arn} ({exc}).')
+        return False
 
 
     def get_tags(self, arn):

{cid_cmd-4.2.4 → cid_cmd-4.2.6}/cid/helpers/quicksight/dataset.py

@@ -8,6 +8,12 @@ from cid.helpers.quicksight.resource import CidQsResource
 
 logger = logging.getLogger(__name__)
 
+DATASET_PROPERTIES = [
+    'AwsAccountId', 'DataSetId', 'Name', 'PhysicalTableMap', 'LogicalTableMap', 'ImportMode', 'ColumnGroups',
+    'RowLevelPermissionDataSet', 'RowLevelPermissionTagConfiguration', 'FieldFolders', 'ColumnLevelPermissionRules',
+    'DataSetUsageConfiguration', 'DatasetParameters', 'PerformanceConfiguration'
+]
+
 
 class Dataset(CidQsResource):
 
@@ -107,7 +113,15 @@ class Dataset(CidQsResource):
             # take the first one and let's hope it is fine
             root_lt = next(iter(dataset['LogicalTableMap'].values()))
 
-        projected_cols = next( # get the first DataTrasform with ProjectOperation
+        renames = {}
+        for lt in dataset['LogicalTableMap'].values():
+            for dt in lt.get('DataTransforms', []):
+                if "RenameColumnOperation" in dt:
+                    key = lt['Source'].get('PhysicalTableId', '') + '.' +  dt["RenameColumnOperation"]['ColumnName'] 
+                    renames[key] = dt["RenameColumnOperation"]['NewColumnName']
+        logger.trace(f'renames = {renames}')
+
+        projected_cols = next( # get the first DataTransform with ProjectOperation
             ds['ProjectOperation']["ProjectedColumns"]
             for ds in root_lt['DataTransforms']
             if 'ProjectOperation' in ds
@@ -115,7 +129,7 @@ class Dataset(CidQsResource):
 
         # Update each PhysicalTableMap with all columns from athena views
         all_columns = []
-        for pt in dataset['PhysicalTableMap'].values():
+        for pt_id, pt in dataset['PhysicalTableMap'].items():
             table_name = pt['RelationalTable']['Name']
             database = pt['RelationalTable']['Schema']
             columns = _get_athena_columns(table_name, database)
@@ -137,15 +151,20 @@ class Dataset(CidQsResource):
                 if col['Name'].lower() not in dataset_columns_names
             ] # BTW what if col is there but another type?
 
-            for col in new_columns: # alter names for columns that already exist
-                if col['Name'].lower() in projected_cols:
-                    col['Name'] = f"{col['Name']}[{table_name}]" # What if it is alrady there?
+            # FIXME: need to add RenameColumnOperation!  col['Name'] => f"{col['Name']}[{table_name}]"
+            new_columns = [col for col in new_columns if col['Name'].lower() not in projected_cols] # avoid things that are already there (probably need to take renames into account)
+            new_columns = [col for col in new_columns if col['Name'] not in all_columns] # avoid adding 2nd time
 
             logger.trace(f'dataset_columns_to_keep = {dataset_columns_to_keep}')
             if new_columns:
-                logger.trace(f'new_columns = {new_columns}')
+                logger.trace(f'new_columns = {new_columns} from {pt_id}')
             pt['RelationalTable']['InputColumns'] = dataset_columns_to_keep + new_columns
-            all_columns += [col['Name'] for col in pt['RelationalTable']['InputColumns']]
+
+            for col in new_columns:
+                col_name = col['Name']
+                if f'{pt_id}.{col_name}' in renames:
+                    col_name = renames[f'{pt_id}.{col_name}']
+                all_columns.append(col_name)
 
         # Add all needed calc fields
         existing_create_columns = [dt.get("CreateColumnsOperation", {}).get('Columns', [None])[0] for dt in root_lt.get('DataTransforms', []) if dt.get("CreateColumnsOperation")]
@@ -172,12 +191,12 @@ class Dataset(CidQsResource):
         # Add all new cols to projected columns
         for col in set(all_columns):
             if col.lower() not in [c.lower() for c in projected_cols]:
+                logger.trace(f'adding {col}')
                 projected_cols.append(col)
 
         # filter out all columns that cannot be used for dataset creation
-        update_ = {key: value for key, value in dataset.items() if key in 'DataSetId, Name, PhysicalTableMap, LogicalTableMap, ImportMode, ColumnGroups, FieldFolders, RowLevelPermissionDataSet, RowLevelPermissionTagConfiguration, ColumnLevelPermissionRules, DataSetUsageConfiguration, DatasetParameters'.split(', ')}
+        update_ = {key: value for key, value in dataset.items() if key in DATASET_PROPERTIES}
         logger.trace(f'update_ = {update_}')
-
         return update_
 
 
@@ -217,4 +236,41 @@ class Dataset(CidQsResource):
         for alias, join in join_clauses.items():
             if isinstance(data['Data'].get(alias), dict) :
                 data['Data'][alias]['clause'] = join
-        return (yaml.safe_dump(data))
+        return (yaml.safe_dump(data))
+
+    @staticmethod
+    def datasets_are_identical(dataset1, dataset2):
+        ''' Compare 2 datasets and returns True if no difference found
+        '''
+        if (not dataset1 and not dataset2):
+            return True
+        identical = False
+        if (dataset1 and not dataset2) or (not dataset1 and dataset2):
+            return identical
+        dataset1 = dataset1 if isinstance(dataset1, Dataset) else Dataset(dataset1)
+        dataset2 = dataset2 if isinstance(dataset2, Dataset) else Dataset(dataset2)
+        identical = True
+        for key in DATASET_PROPERTIES:
+            if key in ['AwsAccountId', 'DataSetId']:
+                continue
+            if dataset1.raw.get(key) != dataset2.raw.get(key):
+                logger.trace(f'not identical {key} {dataset1.raw.get(key)} != {dataset2.raw.get(key)}')
+                identical = False
+        logger.trace(f'identical to existing = {identical}')
+        return identical
+
+    @staticmethod
+    def merge_datasets(dataset1, dataset2):
+        ''' merge high level 2 datasets. Not a deep merge.
+        '''
+        if not dataset2:
+            return dataset1
+        if not dataset1:
+            return dataset2
+        dataset1 = dataset1 if isinstance(dataset1, Dataset) else Dataset(dataset1)
+        dataset2 = dataset2 if isinstance(dataset2, Dataset) else Dataset(dataset2)
+        result = {}
+        for key in DATASET_PROPERTIES:
+            if dataset1.raw.get(key) or dataset2.raw.get(key):
+                result[key] = dataset1.raw.get(key) or dataset2.raw.get(key)
+        return result

{cid_cmd-4.2.4 → cid_cmd-4.2.6/cid_cmd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cid-cmd
-Version: 4.2.4
+Version: 4.2.6
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-4.2.4/cid/_version.py

@@ -1,2 +0,0 @@
-__version__ = '4.2.4'
-