cid-cmd 0.3.4__tar.gz → 0.3.8__tar.gz

{cid_cmd-0.3.4/cid_cmd.egg-info → cid_cmd-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cid-cmd
-Version: 0.3.4
+Version: 0.3.8
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-0.3.8/cid/_version.py

@@ -0,0 +1 @@
+__version__ = '0.3.8'

{cid_cmd-0.3.4 → cid_cmd-0.3.8}/cid/cli.py

@@ -26,10 +26,12 @@ def cid_command(func):
         if len(ctx.args) % 2 != 0:
             print(f"Unknown extra argument, or an option without value {ctx.args}")
             exit(-1)
+        params = {}
         for i in range(0, len(ctx.args), 2):
-            kwargs[ctx.args[i][2:].replace('-', '_')] = ctx.args[i+1]
+            key = ctx.args[i][2:].replace('-', '_')
+            params[key] = ctx.args[i+1]
+        set_parameters(params, all_yes=ctx.obj.all_yes)
 
-        set_parameters(kwargs, all_yes=ctx.obj.all_yes)
         res = None
         try:
             res = func(ctx, **kwargs)
@@ -59,7 +61,7 @@ def cid_command(func):
 
 @click.group()
 @click.option('--profile_name', '--profile', help='AWS Profile name to use', default=None)
-@click.option('--region_name', help="AWS Region (default:'us-east-1')", default=None)
+@click.option('--region_name', help="AWS Region", default=None)
 @click.option('--aws_access_key_id', help='', default=None)
 @click.option('--aws_secret_access_key', help='', default=None)
 @click.option('--aws_session_token', help='', default=None)
@@ -75,7 +77,6 @@ def main(ctx, **kwargs):
 
     ctx.obj = Cid(**kwargs)
 
-
 @click.option('-v', '--verbose', count=True)
 @click.option('-y', '--yes', help='confirm all', is_flag=True, default=False)
 @cid_command
@@ -136,6 +137,7 @@ def deploy(ctx, **kwargs):
      --resources TEXT                      CID resources yaml file or url
      --category TEXT                       Comma separated list of categories of dashboards (ex: foundational,advanced )
      --catalog TEXT                        Comma separated list of catalog files or urls (ex: foundational,advanced )
+     --theme TEXT                          A QuickSight Theme (CLASSIC|MIDNIGHT|SEASIDE|RAINIER)
     """
     ctx.obj.deploy(**kwargs)
 
@@ -201,6 +203,7 @@ def update(ctx, dashboard_id, force, recursive, **kwargs):
     \b
 
      --on-drift (show|override)            Action if a drift of view and dataset is discovered. 'override' = override drift(will destroy customization) or 'show' (default) = show a diff. In Unattended mode (without terminal on-drift will have allways override behaviour)
+     --theme TEXT                          A QuickSight Theme (CLASSIC|MIDNIGHT|SEASIDE|RAINIER)
 
     """
     ctx.obj.update(dashboard_id, force=force, recursive=recursive, **kwargs)

{cid_cmd-0.3.4 → cid_cmd-0.3.8}/cid/common.py

@@ -45,7 +45,8 @@ class Cid():
         self.qs_url = 'https://{region}.quicksight.{domain}/sn/dashboards/{dashboard_id}'
         self.all_yes = kwargs.get('yes')
         self.verbose = kwargs.get('verbose')
-        set_parameters(kwargs, self.all_yes)
+        # save main parameters to global parameters but do not override parameters that were set from outside
+        set_parameters({key: val for key, val in kwargs.items() if key.replace('_', '-') not in get_parameters()}, self.all_yes)
         self._logger = None
         self.catalog_urls = [
             'https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/main/dashboards/catalog.yaml',
@@ -60,9 +61,11 @@ class Cid():
             'aws_session_token': None
         }
         for key in params.keys():
-            value = get_parameters().get(key.replace('_', '-'))
-            if  value != None:
+            value = get_parameters().get(key.replace('_', '-'), '<NO VALUE>')
+            if  value != '<NO VALUE>':
                 params[key] = value
+        if get_parameters().get('region'):
+            params['region_name'] = get_parameters().get('region') # use region as a synonym of region_name
 
         print('Checking AWS environment...')
         try:
@@ -1323,17 +1326,24 @@ class Cid():
             if "<ADD NEW ROLE>" in choice or choice == cid_role_name: # Create or update role
                 # TODO: get buckets from dashboard parameters
                 buckets = [
-                    f'cid-{self.base.account_id}-share',
+                    f'cid-{self.base.account_id}-shared',
+                    f'cid-{self.base.account_id}-data-exports',
                     f'cid-data-{self.base.account_id}',
                     f'costoptimizationdata{self.base.account_id}',
                 ]
                 additional_buckets = get_parameters().get('allow-buckets')
                 if additional_buckets:
-                    buckets += additional_buckets.split(',')
-
+                    buckets += [bucket.strip().replace('{account_id}', self.base.account_id) for bucket in additional_buckets.split(',')]
+
+                databases = set([
+                    "optimization_data",
+                    "cid_data_collection",
+                    "cid_data_export", # prefix for data-exports hardcoded here
+                    self.athena.DatabaseName,
+                ])
                 role_name = self.iam.ensure_data_source_role_exists(
                     role_name=cid_role_name,
-                    database=self.athena.DatabaseName,
+                    databases=databases,
                     workgroup=self.athena.WorkGroup,
                     buckets=buckets,
                     output_location_bucket = self.athena.workgroup_output_location().split('/')[2],
@@ -1681,8 +1691,8 @@ class Cid():
                     pass
             else:
                 self.athena.execute_query(view_query)
-                assert self.athena.wait_for_view(view_name), f"Failed to create a view {view_name}"
-                logger.info(f'View "{view_name}" created')
+            assert self.athena.wait_for_view(view_name), f"Failed to create a view {view_name}"
+            logger.info(f'View "{view_name}" created')
 
         if 'crawler' in view_definition:
             if not ('CREATE EXTERNAL TABLE' in view_query.upper() or view_definition.get('type') == 'Glue_Table'):

{cid_cmd-0.3.4 → cid_cmd-0.3.8}/cid/helpers/athena.py

@@ -73,29 +73,28 @@ class Athena(CidBase):
             except Exception as exc:
                 if 'AccessDeniedException' in str(exc):
                     logger.warning(f'{type(exc)} - Missing athena:GetDatabase permission. Cannot verify existence of {self._DatabaseName} in {self.CatalogName}. Hope you have it there.')
-                    return self._DatabaseName
-                raise
+                else:
+                    raise
+            return self._DatabaseName
         # Get AWS Athena databases
         athena_databases = self.list_databases()
 
-        # Select default database if present
+        # check if we have a default database
         default_databases = [database for database in athena_databases if database['Name'] == self.defaults.get('DatabaseName')]
-        if len(default_databases):
-            # Silently choose an existing default database
-            self._DatabaseName = default_databases.pop().get('Name')
-        else:
-            # Ask user
-            choices = [d['Name'] for d in athena_databases]
-            if self.defaults.get('DatabaseName') not in choices:
-                choices.append(self.defaults.get('DatabaseName') + ' (CREATE NEW)')
-            self._DatabaseName = get_parameter(
-                param_name='athena-database',
-                message="Select AWS Athena database to use",
-                choices=choices,
-            )
-            if self._DatabaseName.endswith( ' (CREATE NEW)'):
-                self._DatabaseName = self.defaults.get('DatabaseName')
-                self.query(f'CREATE DATABASE {self._DatabaseName}')
+
+        # Ask user
+        choices = [d['Name'] for d in athena_databases]
+        if self.defaults.get('DatabaseName') not in choices:
+            choices.append(self.defaults.get('DatabaseName') + ' (CREATE NEW)')
+        self._DatabaseName = get_parameter(
+            param_name='athena-database',
+            message="Select AWS Athena database to use",
+            choices=choices,
+            default=default_databases[0]['Name'] if default_databases else None,
+        )
+        if self._DatabaseName.endswith( ' (CREATE NEW)'):
+            self._DatabaseName = self.defaults.get('DatabaseName')
+            self.query(f'CREATE DATABASE {self._DatabaseName}')
         logger.info(f'Using Athena database: {self._DatabaseName}')
         return self._DatabaseName
 

{cid_cmd-0.3.4 → cid_cmd-0.3.8}/cid/helpers/cur.py

@@ -177,6 +177,8 @@ class CUR(CidBase):
                 self._metadata = self.athena.get_table_metadata(table_name)
             except self.athena.client.exceptions.ResourceNotFoundException as exc:
                 raise CidCritical(f'Provided cur-table-name "{table_name}" is not found. Please make sure the table exists.') from exc
+            except self.athena. athena.exceptions.MetadataException as exc:
+                raise CidCritical(f'Provided cur-table-name "{table_name}" is not found in database "{self.athena.DatabaseName}". Please make sure the table exists. Also if you use LakeFormation, make sure that the user is granted with read rights.') from exc
             res, message = self.table_is_cur(table=self._metadata, return_reason=True)
             if not res:
                 raise CidCritical(f'Table {table_name} does not look like CUR. {message}')

{cid_cmd-0.3.4 → cid_cmd-0.3.8}/cid/helpers/iam.py

@@ -113,7 +113,7 @@ class IAM(CidBase):
             policy_merge_mode='MERGE_RESOURCES',
             managed_policies=[]
         )
-    def ensure_data_source_role_exists(self, role_name, database, workgroup, kms_key_arns='', buckets=[], output_location_bucket=None):
+    def ensure_data_source_role_exists(self, role_name, databases, workgroup, kms_key_arns='', buckets=[], output_location_bucket=None):
         ''' Create or update a role specifically for a QS Datasource
         '''
         return self.ensure_role_with_policy(
@@ -157,9 +157,10 @@ class IAM(CidBase):
                             ],
                             "Resource": [
                                 f"arn:{self.partition}:glue:{self.region}:{self.account_id}:catalog",
-                                f"arn:{self.partition}:glue:{self.region}:{self.account_id}:database/{database}",
-                                f"arn:{self.partition}:glue:{self.region}:{self.account_id}:table/{database}/*",
-                            ],
+                            ] + sum([
+                                [f"arn:{self.partition}:glue:{self.region}:{self.account_id}:database/{database}",
+                                f"arn:{self.partition}:glue:{self.region}:{self.account_id}:table/{database}/*" ]
+                            for database in databases ], []),
                         },
                         {
                             "Sid": "AllowAthena",
@@ -176,7 +177,6 @@ class IAM(CidBase):
                             ],
                             "Resource": [
                                 f"arn:{self.partition}:athena:{self.region}:{self.account_id}:datacatalog/AwsDataCatalog", # TODO: check if this can be variable?
-                                f"arn:{self.partition}:athena:{self.region}:{self.account_id}:database/{database}",
                                 f"arn:{self.partition}:athena:{self.region}:{self.account_id}:workgroup/{workgroup}",
                             ]
                         },

{cid_cmd-0.3.4 → cid_cmd-0.3.8}/cid/helpers/quicksight/__init__.py

@@ -1335,7 +1335,7 @@ class QuickSight(CidBase):
             'Name': definition.get('name'),
             'ValidationStrategy': {'Mode': 'LENIENT'},
         }
-        theme = definition.get('theme')
+        theme = get_parameters().get('theme') or definition.get('theme')
         if theme:
             if not theme.startswith('arn:'):
                 theme_arn = f'arn:{self.partition}:quicksight::aws:theme/' + theme

{cid_cmd-0.3.4 → cid_cmd-0.3.8/cid_cmd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cid-cmd
-Version: 0.3.4
+Version: 0.3.8
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-0.3.4/cid/_version.py

@@ -1 +0,0 @@
-__version__ = '0.3.4'