cid-cmd 4.2.5__tar.gz → 4.2.7__tar.gz

{cid_cmd-4.2.5/cid_cmd.egg-info → cid_cmd-4.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cid-cmd
-Version: 4.2.5
+Version: 4.2.7
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-4.2.7/cid/_version.py

@@ -0,0 +1,2 @@
+__version__ = '4.2.7'
+

{cid_cmd-4.2.5 → cid_cmd-4.2.7}/cid/common.py

@@ -282,16 +282,22 @@ class Cid():
         export_analysis(self.qs, self.athena, glue=self.glue)
 
     def track(self, action, dashboard_id):
-        """ Send dashboard_id and account_id to adoption tracker """
+        """ Send dashboard_id and account_id to CID adoption tracker """
         method = {'created':'PUT', 'updated':'PATCH', 'deleted': 'DELETE'}.get(action, None)
         if not method:
             logger.debug(f"This will not fail the deployment. Logging action {action} is not supported. This issue will be ignored")
             return
-        endpoint = 'https://okakvoavfg.execute-api.eu-west-1.amazonaws.com/'
+        endpoint = 'https://okakvoavfg.execute-api.eu-west-1.amazonaws.com/' # AWS Managed
+        if os.environ.get('AWS_DEPLOYMENT_TYPE'):
+            deployment_type = os.environ.get('AWS_DEPLOYMENT_TYPE')
+        elif os.environ.get('AWS_EXECUTION_ENV', '').startswith('AWS_Lambda'):
+            deployment_type = 'Lambda'
+        else:
+            deployment_type = 'CID'
         payload = {
             'dashboard_id': dashboard_id,
             'account_id': self.base.account_id,
-            action + '_via': 'Lambda' if os.environ.get('AWS_EXECUTION_ENV', '').startswith('AWS_Lambda') else 'CID',
+            action + '_via': deployment_type,
         }
         try:
             res = requests.request(
@@ -470,17 +476,16 @@ class Cid():
             set_defaults(defaults)
 
     def dump_default_parameters(self):
-
         stop_list = ['profile-name', 'region', 'aws-access-key-id', 'aws-secret-access-key', 'aws-session-token', 'athena-database', 'athena-workgroup']
-        current_defaults = self.parameters_controller.load_parameters(
-            context=get_parameters().get('dashboard-id')
-        )
-        current_defaults = current_defaults | get_parameters()
+        current_defaults = get_parameters()
         for key in list(current_defaults.keys()):
             if key in stop_list:
                 del current_defaults[key]
         logger.trace(f'dumping parameters {current_defaults}')
-        self.parameters_controller.dump_parameters(current_defaults, context=get_parameters().get('dashboard-id'))
+        self.parameters_controller.dump_parameters(
+            current_defaults,
+            context=get_parameters().get('dashboard-id')
+        )
 
 
     def ensure_subscription(self):
@@ -1668,17 +1673,11 @@ class Cid():
                         update_dataset = True
                     break
 
-            identical = False # check if dataset needs an update
-            if isinstance(found_dataset, Dataset):
-                identical = True
-                for key in 'PhysicalTableMap LogicalTableMap OutputColumns ImportMode DataSetUsageConfiguration RowLevelPermissionDataSet FieldFolders RowLevelPermissionTagConfiguration DatasetParameters'.split():
-                    if found_dataset.raw.get(key) != compiled_dataset.get(key):
-                        logger.trace(f'not identical {key} {found_dataset.raw.get(key)} != {compiled_dataset.get(key)}')
-                        identical = False
-                logger.trace(f'identical to existing = {identical}')
+            identical = Dataset.datasets_are_identical(found_dataset, compiled_dataset) # check if dataset needs an update
 
             if update_dataset and not identical:
-                self.qs.update_dataset(compiled_dataset)
+                merged_dataset = Dataset.merge_datasets(compiled_dataset, found_dataset)
+                self.qs.update_dataset(merged_dataset)
                 if compiled_dataset.get("ImportMode") == "SPICE":
                     dataset_id = compiled_dataset.get('DataSetId')
                     schedules_definitions = []

cid_cmd-4.2.7/cid/helpers/parameter_store.py

@@ -0,0 +1,88 @@
+import logging
+from datetime import datetime
+
+from cid.exceptions import CidCritical
+
+logger = logging.getLogger(__name__)
+
+class AthenaStore():
+    def __init__(self, athena, view_name='cid_parameters'):
+        self.athena = athena
+        self.view_name = view_name
+
+    def dump(self, data):
+        ''' dump data to athena
+        '''
+        try:
+            # FIXME: make it multi view
+            self.athena.query(self._generate_view_query(data, self.view_name))
+        except (CidCritical, Exception) as exc:
+            logger.debug(f'failed to save parameters store: {exc}')
+
+    def load(self):
+        ''' load from athena
+        '''
+        try:
+            res = self.athena.query(f'''select * from  {self.view_name}''', include_header=True)
+        except CidCritical as exc:
+            if 'TABLE_NOT_FOUND' in str(exc):
+                res = []
+            else:
+                raise
+        return [{k:v for k, v in zip(res[0], row)} for row in res[1:]]
+
+    def _to_sql_str(self, val):
+        if val is None:
+            return "''"
+        return "'" + str(val).replace("'", "''") + "'"
+
+    def _generate_view_query(self, data, name):
+        all_keys = {key for dictionary in data for key in dictionary.keys()}
+        lines = ',\n            '.join([f'''ROW({','.join([self._to_sql_str(line.get(k)) for k in all_keys])})''' for line in data])
+        query = f"""
+            CREATE OR REPLACE VIEW {name} AS
+            SELECT *
+            FROM (
+                VALUES
+                {lines}
+            ) ignored_table_name ({','.join([key for key in all_keys])})
+        """
+        return query
+
+class ParametersController(AthenaStore):
+    def load_parameters(self, context):
+        data = self.load()
+        context_parameters = {}
+        any_parameters = {}
+        # get any context and then override with specific context
+        for line in sorted(data, key=lambda x: x.get('date', '')): # latest should override
+            any_parameters[line.get('parameter')] = line.get('value')
+            if line.get('context') == str(context):
+                context_parameters[line.get('parameter')] = line.get('value')
+        return any_parameters | context_parameters
+
+    def dump_parameters(self, params, context=None):
+        data = self.load()
+        logger.trace(f'loaded parameters {data}')
+        params = dict(params)
+        date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        #Update parameters if they are present already
+        for line in data:
+            for key, val in list(params.items()):
+                if line.get('context') == str(context) and line.get('parameter') == key:
+                    line['value'] = val
+                    line['date'] = date
+                    del params[key]
+
+        #add parameters that are new
+        for key, val in list(params.items()):
+            data.append({
+                'parameter': key,
+                'value': ','.join([str(v) for v in val]) if isinstance(val, list) else str(val),
+                'context': str(context),
+                'date': date
+            })
+
+        logger.trace(f'dumping parameters {data}')
+        self.dump(data)

{cid_cmd-4.2.5 → cid_cmd-4.2.7}/cid/helpers/quicksight/__init__.py

@@ -261,7 +261,9 @@ class QuickSight(CidBase):
             return True
         except self.client.exceptions.AccessDeniedException as exc:
             logger.debug(f'Cannot tag {arn} (AccessDenied).')
-            return False
+        except self.client.exceptions.ClientError as exc:
+            logger.debug(f'Cannot tag {arn} ({exc}).')
+        return False
 
 
     def get_tags(self, arn):
@@ -733,6 +735,7 @@ class QuickSight(CidBase):
                     'AwsAccountId': self.account_id,
                     'DashboardId': dashboard_id
                 }
+                logger.debug(f'describe_dashboard_definition({dashboard_id})')
                 result = self.client.describe_dashboard_definition(**parameters)
                 self._definitions.update({f'{self.account_id}:{self.identityRegion}:{dashboard_id}': CidQsDefinition(result.get('Definition'))})
                 logger.debug(result)
@@ -1287,6 +1290,7 @@ class QuickSight(CidBase):
             'DashboardId': definition.get('dashboardId'),
             'VersionNumber': created_version
         }
+        logger.debug(f"describe_dashboard { definition.get('dashboardId')}")
         dashboard = self.describe_dashboard(poll=True, **describe_parameters)
         logger.debug('dashboard={dashboard}')
         self.set_tags(dashboard.arn, cid_version_tag=dashboard.cid_version) # try to update version tag
@@ -1304,7 +1308,7 @@ class QuickSight(CidBase):
         definition: cid dashboard definition. NOT QS DEFINITION.
         returns: what we need tof create or update API
         """
-
+        cid_print('Preparing dashboard definition')
         create_parameters = {
             'AwsAccountId': self.account_id,
             'DashboardId': definition.get('dashboardId'),

{cid_cmd-4.2.5 → cid_cmd-4.2.7}/cid/helpers/quicksight/dataset.py

@@ -8,6 +8,12 @@ from cid.helpers.quicksight.resource import CidQsResource
 
 logger = logging.getLogger(__name__)
 
+DATASET_PROPERTIES = [
+    'AwsAccountId', 'DataSetId', 'Name', 'PhysicalTableMap', 'LogicalTableMap', 'ImportMode', 'ColumnGroups',
+    'RowLevelPermissionDataSet', 'RowLevelPermissionTagConfiguration', 'FieldFolders', 'ColumnLevelPermissionRules',
+    'DataSetUsageConfiguration', 'DatasetParameters', 'PerformanceConfiguration'
+]
+
 
 class Dataset(CidQsResource):
 
@@ -189,7 +195,7 @@ class Dataset(CidQsResource):
                 projected_cols.append(col)
 
         # filter out all columns that cannot be used for dataset creation
-        update_ = {key: value for key, value in dataset.items() if key in 'DataSetId, Name, PhysicalTableMap, LogicalTableMap, ImportMode, ColumnGroups, FieldFolders, RowLevelPermissionDataSet, RowLevelPermissionTagConfiguration, ColumnLevelPermissionRules, DataSetUsageConfiguration, DatasetParameters'.split(', ')}
+        update_ = {key: value for key, value in dataset.items() if key in DATASET_PROPERTIES}
         logger.trace(f'update_ = {update_}')
         return update_
 
@@ -230,4 +236,41 @@ class Dataset(CidQsResource):
         for alias, join in join_clauses.items():
             if isinstance(data['Data'].get(alias), dict) :
                 data['Data'][alias]['clause'] = join
-        return (yaml.safe_dump(data))
+        return (yaml.safe_dump(data))
+
+    @staticmethod
+    def datasets_are_identical(dataset1, dataset2):
+        ''' Compare 2 datasets and returns True if no difference found
+        '''
+        if (not dataset1 and not dataset2):
+            return True
+        identical = False
+        if (dataset1 and not dataset2) or (not dataset1 and dataset2):
+            return identical
+        dataset1 = dataset1 if isinstance(dataset1, Dataset) else Dataset(dataset1)
+        dataset2 = dataset2 if isinstance(dataset2, Dataset) else Dataset(dataset2)
+        identical = True
+        for key in DATASET_PROPERTIES:
+            if key in ['AwsAccountId', 'DataSetId']:
+                continue
+            if dataset1.raw.get(key) != dataset2.raw.get(key):
+                logger.trace(f'not identical {key} {dataset1.raw.get(key)} != {dataset2.raw.get(key)}')
+                identical = False
+        logger.trace(f'identical to existing = {identical}')
+        return identical
+
+    @staticmethod
+    def merge_datasets(dataset1, dataset2):
+        ''' merge high level 2 datasets. Not a deep merge.
+        '''
+        if not dataset2:
+            return dataset1
+        if not dataset1:
+            return dataset2
+        dataset1 = dataset1 if isinstance(dataset1, Dataset) else Dataset(dataset1)
+        dataset2 = dataset2 if isinstance(dataset2, Dataset) else Dataset(dataset2)
+        result = {}
+        for key in DATASET_PROPERTIES:
+            if dataset1.raw.get(key) or dataset2.raw.get(key):
+                result[key] = dataset1.raw.get(key) or dataset2.raw.get(key)
+        return result

{cid_cmd-4.2.5 → cid_cmd-4.2.7}/cid/utils.py

@@ -276,7 +276,7 @@ def get_parameter(param_name, message, choices=None, default=None, none_as_disab
         if multi:
             default = default or []
             default = default if isinstance(default, list) else [default]
-            default = [c for c in defaults if c in choices]
+            default = [c for c in default if c in choices]
             if not isatty():
                 result = default
             else:

{cid_cmd-4.2.5 → cid_cmd-4.2.7/cid_cmd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cid-cmd
-Version: 4.2.5
+Version: 4.2.7
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-4.2.5/cid/_version.py

@@ -1,2 +0,0 @@
-__version__ = '4.2.5'
-

cid_cmd-4.2.5/cid/helpers/parameter_store.py

@@ -1,51 +0,0 @@
-from cid.exceptions import CidCritical
-
-class AthenaStore():
-    def __init__(self, athena, view_name='cid_parameters'):
-        self.athena = athena
-        self.view_name = view_name
-
-    def dump(self, data):
-        ''' load from athena
-        '''
-        # FIXME: make it multi view
-        self.athena.query(self._generate_view_query(data, self.view_name))
-
-    def load(self):
-        ''' load from athena
-        '''
-        try:
-            res = self.athena.query(f'''select * from  {self.view_name}''', include_header=True)
-        except CidCritical as exc:
-            if 'TABLE_NOT_FOUND' in str(exc):
-                res = []
-            else:
-                raise
-        return [{k:v for k, v in zip(res[0], row)} for row in res[1:]]
-
-    def _to_sql_str(self, val):
-        if val is None:
-            return "''"
-        return "'" + str(val).replace("'", "''") + "'"
-
-    def _generate_view_query(self, data, name):
-        all_keys = {key for dictionary in data for key in dictionary.keys()}
-        lines = ',\n            '.join([f'''ROW({','.join([self._to_sql_str(line.get(k)) for k in all_keys])})''' for line in data])
-        query = f"""
-            CREATE OR REPLACE VIEW {name} AS
-            SELECT *
-            FROM (
-                VALUES
-                {lines}
-            ) ignored_table_name ({','.join([key for key in all_keys])})
-        """
-        return query
-
-class ParametersController(AthenaStore):
-    def load_parameters(self, context):
-        data = self.load()
-        return { line.get('parameter'):line.get('value') for line in data}
-
-    def dump_parameters(self, params, context=None):
-        data = [{'parameter': key, 'value': ','.join(val) if isinstance(val, list) else val, 'context': str(context) } for key, val in params.items()]
-        self.dump(data)