cid-cmd 0.3.2__tar.gz → 0.3.8__tar.gz

{cid_cmd-0.3.2/cid_cmd.egg-info → cid_cmd-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cid-cmd
-Version: 0.3.2
+Version: 0.3.8
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-0.3.8/cid/_version.py

@@ -0,0 +1 @@
+__version__ = '0.3.8'

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cid/summary_view.sql

@@ -83,5 +83,8 @@
      WHEN ("line_item_line_item_type" <> 'SavingsPlanNegation') THEN "pricing_public_on_demand_cost" ELSE 0 END) "public_cost"
  FROM
   "${cur_table_name}"
- WHERE (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL  '7' MONTH)) AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL  '7' MONTH)))
+ WHERE 
+    (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL  '7' MONTH)) 
+    AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL  '7' MONTH))
+    AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2')) 
  GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,33,34

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cid/summary_view_ri.sql

@@ -73,6 +73,8 @@
      WHEN ("line_item_line_item_type" <> 'SavingsPlanNegation') THEN "pricing_public_on_demand_cost" ELSE 0 END) "public_cost"
  FROM
   "${cur_table_name}"
- WHERE (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL  '7' MONTH)) AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL  '7' MONTH)))
-
+ WHERE 
+    (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL  '7' MONTH)) 
+    AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL  '7' MONTH))
+    AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))
  GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,33,34

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cid/summary_view_sp.sql

@@ -77,6 +77,8 @@
      WHEN ("line_item_line_item_type" <> 'SavingsPlanNegation') THEN "pricing_public_on_demand_cost" ELSE 0 END) "public_cost"
  FROM
  "${cur_table_name}"
- WHERE (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL  '7' MONTH)) AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL  '7' MONTH)))
-
+  WHERE 
+    (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL  '7' MONTH)) 
+    AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL  '7' MONTH))
+    AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))
  GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,33,34

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cid/summary_view_sp_ri.sql

@@ -73,6 +73,8 @@
       WHEN ("line_item_line_item_type" <> 'SavingsPlanNegation') THEN "pricing_public_on_demand_cost" ELSE 0 END) "public_cost"
   FROM
   "${cur_table_name}"
-  WHERE (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL  '7' MONTH)) AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL  '7' MONTH)))
-
+ WHERE 
+    (("bill_billing_period_start_date" >= ("date_trunc"('month', current_timestamp) - INTERVAL  '7' MONTH)) 
+    AND (CAST("concat"("year", '-', "month", '-01') AS date) >= ("date_trunc"('month', current_date) - INTERVAL  '7' MONTH))
+    AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))
   GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cudos/hourly_view.sql

@@ -21,5 +21,8 @@ CREATE OR REPLACE VIEW hourly_view AS
   , "sum"("line_item_usage_amount") "usage_quantity"
   FROM
     "${cur_table_name}"
-  WHERE (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage')))
-  GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+  WHERE 
+    (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) 
+    AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))
+    AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))
+  GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cudos/hourly_view_ri.sql

@@ -21,5 +21,8 @@ CREATE OR REPLACE VIEW hourly_view AS
   , "sum"("line_item_usage_amount") "usage_quantity"
   FROM
     "${cur_table_name}"
-  WHERE (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage')))
-  GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+  WHERE 
+    (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) 
+    AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))
+    AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))  
+  GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cudos/hourly_view_sp.sql

@@ -21,5 +21,8 @@ CREATE OR REPLACE VIEW hourly_view AS
   , "sum"("line_item_usage_amount") "usage_quantity"
   FROM
     "${cur_table_name}"
-  WHERE (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage')))
-  GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+  WHERE 
+    (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) 
+    AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))
+    AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))
+  GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cudos/hourly_view_sp_ri.sql

@@ -21,5 +21,8 @@ CREATE OR REPLACE VIEW hourly_view AS
     , "sum"("line_item_usage_amount") "usage_quantity"
     FROM
       "${cur_table_name}"
-    WHERE (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage')))
-    GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+    WHERE 
+        (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) 
+        AND ((("line_item_line_item_type" = 'Usage') OR ("line_item_line_item_type" = 'SavingsPlanCoveredUsage')) OR ("line_item_line_item_type" = 'DiscountedUsage'))
+        AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))    
+    GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cudos/resource_view.sql

@@ -35,5 +35,8 @@ CREATE OR REPLACE VIEW resource_view AS
       , "sum"("line_item_unblended_cost") unblended_cost
       FROM
         "${cur_table_name}"
-      WHERE (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) AND (line_item_resource_id <> ''))
-      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29
+      WHERE 
+            (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) 
+            AND (line_item_resource_id <> '')
+            AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))
+      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cudos/resource_view_ri.sql

@@ -35,5 +35,8 @@ CREATE OR REPLACE VIEW resource_view AS
       , "sum"("line_item_unblended_cost") unblended_cost
       FROM
         "${cur_table_name}"
-      WHERE (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) AND (line_item_resource_id <> ''))
-      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29
+      WHERE 
+            (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) 
+            AND (line_item_resource_id <> '')
+            AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))
+      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cudos/resource_view_sp.sql

@@ -35,5 +35,8 @@ CREATE OR REPLACE VIEW resource_view AS
       , "sum"("line_item_unblended_cost") unblended_cost
       FROM
         "${cur_table_name}"
-      WHERE (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) AND (line_item_resource_id <> ''))
-      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29
+      WHERE 
+            (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) 
+            AND (line_item_resource_id <> '')
+            AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))
+      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/builtin/core/data/queries/cudos/resource_view_sp_ri.sql

@@ -35,5 +35,8 @@ CREATE OR REPLACE VIEW resource_view AS
       , "sum"("line_item_unblended_cost") unblended_cost
       FROM
         "${cur_table_name}"
-      WHERE (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) AND (line_item_resource_id <> ''))
-      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29
+      WHERE 
+            (((current_date - INTERVAL  '30' DAY) <= line_item_usage_start_date) 
+            AND (line_item_resource_id <> '')
+            AND "line_item_operation" NOT IN ('EKSPod-EC2','ECSTask-EC2'))
+      GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/cli.py

@@ -26,10 +26,12 @@ def cid_command(func):
         if len(ctx.args) % 2 != 0:
             print(f"Unknown extra argument, or an option without value {ctx.args}")
             exit(-1)
+        params = {}
         for i in range(0, len(ctx.args), 2):
-            kwargs[ctx.args[i][2:].replace('-', '_')] = ctx.args[i+1]
+            key = ctx.args[i][2:].replace('-', '_')
+            params[key] = ctx.args[i+1]
+        set_parameters(params, all_yes=ctx.obj.all_yes)
 
-        set_parameters(kwargs, all_yes=ctx.obj.all_yes)
         res = None
         try:
             res = func(ctx, **kwargs)
@@ -59,7 +61,7 @@ def cid_command(func):
 
 @click.group()
 @click.option('--profile_name', '--profile', help='AWS Profile name to use', default=None)
-@click.option('--region_name', help="AWS Region (default:'us-east-1')", default=None)
+@click.option('--region_name', help="AWS Region", default=None)
 @click.option('--aws_access_key_id', help='', default=None)
 @click.option('--aws_secret_access_key', help='', default=None)
 @click.option('--aws_session_token', help='', default=None)
@@ -75,7 +77,6 @@ def main(ctx, **kwargs):
 
     ctx.obj = Cid(**kwargs)
 
-
 @click.option('-v', '--verbose', count=True)
 @click.option('-y', '--yes', help='confirm all', is_flag=True, default=False)
 @cid_command
@@ -125,6 +126,7 @@ def deploy(ctx, **kwargs):
      --cur-table-name TEXT                 CUR table name
      --quicksight-datasource-id TEXT       QuickSight Datasource ARN (if not found one with provided Athena workgroup)
      --quicksight-datasource-role-arn TEXT IAM Role used for DataSource Creation (if not provided, will use the default QS Role). Must have access to Athena and S3 buckets.
+     --allow-buckets                       Comma separated list of buckets names to add to the default Cid QuickSight role
      --quicksight-user TEXT                QuickSight user
      --dataset-{dataset_name}-id TEXT      QuickSight dataset id for a specific dataset
      --view-{view_name}-{parameter} TEXT   a custom parameter for a view creation, can use variable: {account_id}
@@ -135,6 +137,7 @@ def deploy(ctx, **kwargs):
      --resources TEXT                      CID resources yaml file or url
      --category TEXT                       Comma separated list of categories of dashboards (ex: foundational,advanced )
      --catalog TEXT                        Comma separated list of catalog files or urls (ex: foundational,advanced )
+     --theme TEXT                          A QuickSight Theme (CLASSIC|MIDNIGHT|SEASIDE|RAINIER)
     """
     ctx.obj.deploy(**kwargs)
 
@@ -200,6 +203,7 @@ def update(ctx, dashboard_id, force, recursive, **kwargs):
     \b
 
      --on-drift (show|override)            Action if a drift of view and dataset is discovered. 'override' = override drift(will destroy customization) or 'show' (default) = show a diff. In Unattended mode (without terminal on-drift will have allways override behaviour)
+     --theme TEXT                          A QuickSight Theme (CLASSIC|MIDNIGHT|SEASIDE|RAINIER)
 
     """
     ctx.obj.update(dashboard_id, force=force, recursive=recursive, **kwargs)

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/common.py

@@ -45,7 +45,8 @@ class Cid():
         self.qs_url = 'https://{region}.quicksight.{domain}/sn/dashboards/{dashboard_id}'
         self.all_yes = kwargs.get('yes')
         self.verbose = kwargs.get('verbose')
-        set_parameters(kwargs, self.all_yes)
+        # save main parameters to global parameters but do not override parameters that were set from outside
+        set_parameters({key: val for key, val in kwargs.items() if key.replace('_', '-') not in get_parameters()}, self.all_yes)
         self._logger = None
         self.catalog_urls = [
             'https://raw.githubusercontent.com/aws-samples/aws-cudos-framework-deployment/main/dashboards/catalog.yaml',
@@ -60,9 +61,11 @@ class Cid():
             'aws_session_token': None
         }
         for key in params.keys():
-            value = get_parameters().get(key.replace('_', '-'))
-            if  value != None:
+            value = get_parameters().get(key.replace('_', '-'), '<NO VALUE>')
+            if  value != '<NO VALUE>':
                 params[key] = value
+        if get_parameters().get('region'):
+            params['region_name'] = get_parameters().get('region') # use region as a synonym of region_name
 
         print('Checking AWS environment...')
         try:
@@ -1300,11 +1303,11 @@ class Cid():
         if not role_arn:
             role_name = get_parameters().get('quicksight-datasource-role')
             if role_name:
-                role_arn = f'arn:aws:iam::{self.base.account_id}:role/{role_name}'
+                role_arn = self.iam.get_role_arn(role_name)
 
         if not role_arn:
             quicksight_trusted_roles = list(self.iam.iterate_role_names(search="Roles[?AssumeRolePolicyDocument.Statement[?Principal.Service=='quicksight.amazonaws.com']].RoleName"))
-            #quicksight_trusted_roles = [role for role in quicksight_trusted_roles if role not in ('aws-quicksight-secretsmanager-role-v0')] # filter out irrelevant roles
+            quicksight_trusted_roles = [role for role in quicksight_trusted_roles if role not in ('aws-quicksight-secretsmanager-role-v0')] # filter out irrelevant roles
             # TODO: filter only roles with Athena and S3 policies
             cid_role_name = 'CidCmdQuickSightDataSourceRole'
             choices = quicksight_trusted_roles
@@ -1321,25 +1324,36 @@ class Cid():
                 default=default,
             )
             if "<ADD NEW ROLE>" in choice or choice == cid_role_name: # Create or update role
-                # TODO: allow customer add buckets
+                # TODO: get buckets from dashboard parameters
                 buckets = [
-                    f'cid-{self.base.account_id}-share',
+                    f'cid-{self.base.account_id}-shared',
+                    f'cid-{self.base.account_id}-data-exports',
                     f'cid-data-{self.base.account_id}',
                     f'costoptimizationdata{self.base.account_id}',
                 ]
+                additional_buckets = get_parameters().get('allow-buckets')
+                if additional_buckets:
+                    buckets += [bucket.strip().replace('{account_id}', self.base.account_id) for bucket in additional_buckets.split(',')]
+
+                databases = set([
+                    "optimization_data",
+                    "cid_data_collection",
+                    "cid_data_export", # prefix for data-exports hardcoded here
+                    self.athena.DatabaseName,
+                ])
                 role_name = self.iam.ensure_data_source_role_exists(
                     role_name=cid_role_name,
-                    database=self.athena.DatabaseName,
+                    databases=databases,
                     workgroup=self.athena.WorkGroup,
                     buckets=buckets,
                     output_location_bucket = self.athena.workgroup_output_location().split('/')[2],
                 )
                 cid_print(f'Role {role_name} was updated. https://console.aws.amazon.com/iam/home?#/roles/details/{role_name}')
-                role_arn = f'arn:aws:iam::{self.base.account_id}:role/{role_name}'
-            elif 'USE DEFAULT QuickSight ROLE' in choice:
+                role_arn = self.iam.get_role_arn(role_name)
+            elif 'USE DEFAULT QuickSight ROLE' in choice or choice == 'default':
                 role_arn = None
             else:
-                role_arn = choice
+                role_arn = self.iam.get_role_arn(choice)
 
         athena_datasource = self.qs.create_data_source(
             athena_workgroup=self.athena.WorkGroup,
@@ -1677,8 +1691,8 @@ class Cid():
                     pass
             else:
                 self.athena.execute_query(view_query)
-                assert self.athena.wait_for_view(view_name), f"Failed to create a view {view_name}"
-                logger.info(f'View "{view_name}" created')
+            assert self.athena.wait_for_view(view_name), f"Failed to create a view {view_name}"
+            logger.info(f'View "{view_name}" created')
 
         if 'crawler' in view_definition:
             if not ('CREATE EXTERNAL TABLE' in view_query.upper() or view_definition.get('type') == 'Glue_Table'):
@@ -1710,7 +1724,7 @@ class Cid():
             )
 
         if not crawler_role.startswith('arn:'):
-            crawler_role_arn = f"arn:aws:iam::{self.base.account_id}:role/{crawler_role}"
+            crawler_role_arn = f"arn:{self.base.partition}:iam::{self.base.account_id}:role/{crawler_role}"
         else:
             crawler_role_arn = crawler_role
         params = {

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/helpers/athena.py

@@ -73,29 +73,28 @@ class Athena(CidBase):
             except Exception as exc:
                 if 'AccessDeniedException' in str(exc):
                     logger.warning(f'{type(exc)} - Missing athena:GetDatabase permission. Cannot verify existence of {self._DatabaseName} in {self.CatalogName}. Hope you have it there.')
-                    return self._DatabaseName
-                raise
+                else:
+                    raise
+            return self._DatabaseName
         # Get AWS Athena databases
         athena_databases = self.list_databases()
 
-        # Select default database if present
+        # check if we have a default database
         default_databases = [database for database in athena_databases if database['Name'] == self.defaults.get('DatabaseName')]
-        if len(default_databases):
-            # Silently choose an existing default database
-            self._DatabaseName = default_databases.pop().get('Name')
-        else:
-            # Ask user
-            choices = [d['Name'] for d in athena_databases]
-            if self.defaults.get('DatabaseName') not in choices:
-                choices.append(self.defaults.get('DatabaseName') + ' (CREATE NEW)')
-            self._DatabaseName = get_parameter(
-                param_name='athena-database',
-                message="Select AWS Athena database to use",
-                choices=choices,
-            )
-            if self._DatabaseName.endswith( ' (CREATE NEW)'):
-                self._DatabaseName = self.defaults.get('DatabaseName')
-                self.query(f'CREATE DATABASE {self._DatabaseName}')
+
+        # Ask user
+        choices = [d['Name'] for d in athena_databases]
+        if self.defaults.get('DatabaseName') not in choices:
+            choices.append(self.defaults.get('DatabaseName') + ' (CREATE NEW)')
+        self._DatabaseName = get_parameter(
+            param_name='athena-database',
+            message="Select AWS Athena database to use",
+            choices=choices,
+            default=default_databases[0]['Name'] if default_databases else None,
+        )
+        if self._DatabaseName.endswith( ' (CREATE NEW)'):
+            self._DatabaseName = self.defaults.get('DatabaseName')
+            self.query(f'CREATE DATABASE {self._DatabaseName}')
         logger.info(f'Using Athena database: {self._DatabaseName}')
         return self._DatabaseName
 

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/helpers/cur.py

@@ -4,7 +4,7 @@ import json
 import logging
 
 from cid.base import CidBase
-from cid.utils import get_parameter, get_parameters
+from cid.utils import get_parameter, get_parameters, cid_print
 from cid.exceptions import CidCritical
 
 logger = logging.getLogger(__name__)
@@ -135,8 +135,9 @@ class CUR(CidBase):
                 self.athena.query(f'ALTER TABLE {self.table_name} ADD COLUMNS ({column} {column_type})')
             except (self.athena.client.exceptions.ClientError, CidCritical) as exc:
                 raise CidCritical(f'Column {column} is not found in CUR and we were unable to add it. Please check FAQ.') from exc
-            self._metadata = self.athena.get_table_metadata(self.table_name) # refresh table metadata
-            logger.critical(f"Column '{column}' was added to CUR ({self.table_name}). Please make sure crawler do not override that columns. Crawler='{crawler_name}'")
+            # table takes time to update so just adding column to cached data
+            self._metadata.get('Columns', []).append({'Name': column, 'Type': column_type})
+            cid_print(f"Column '{column}' was added to CUR ({self.table_name}).")
             return
 
         # if table cannot be updated, check if it is ri/sp case - let's hope dashboard views can handle it:
@@ -145,7 +146,7 @@ class CUR(CidBase):
             return
 
         # if a required column is not there and not ri/sp -> stop
-        logger.critical(f"Column '{column}' is not in ({self.table_name}). Cannot continue")
+        logger.critical(f"Column '{column}' is not in ({self.table_name}). Cannot continue.")
 
 
     def table_is_cur(self, table: dict=None, name: str=None, return_reason: bool=False) -> bool:
@@ -176,6 +177,8 @@ class CUR(CidBase):
                 self._metadata = self.athena.get_table_metadata(table_name)
             except self.athena.client.exceptions.ResourceNotFoundException as exc:
                 raise CidCritical(f'Provided cur-table-name "{table_name}" is not found. Please make sure the table exists.') from exc
+            except self.athena. athena.exceptions.MetadataException as exc:
+                raise CidCritical(f'Provided cur-table-name "{table_name}" is not found in database "{self.athena.DatabaseName}". Please make sure the table exists. Also if you use LakeFormation, make sure that the user is granted with read rights.') from exc
             res, message = self.table_is_cur(table=self._metadata, return_reason=True)
             if not res:
                 raise CidCritical(f'Table {table_name} does not look like CUR. {message}')

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/helpers/iam.py

@@ -8,7 +8,7 @@ import logging
 import boto3
 from tqdm import tqdm
 
-from cid.exceptions import CidCritical
+from cid.exceptions import CidCritical, CidError
 from cid.base import CidBase
 from cid.utils import get_parameter
 
@@ -61,13 +61,13 @@ class IAM(CidBase):
                             "Sid": "AllowListBucket",
                             "Effect": "Allow",
                             "Action": ["s3:ListBucket"],
-                            "Resource": [f"arn:aws:s3:::{s3bucket}"]
+                            "Resource": [f"arn:{self.partition}:s3:::{s3bucket}"]
                         },
                         {
                             "Sid": "AllowReadFromBucket",
                             "Effect": "Allow",
                             "Action": [ "s3:GetObject"],
-                            "Resource": [f"arn:aws:s3:::{s3bucket}/*"]
+                            "Resource": [f"arn:{self.partition}:s3:::{s3bucket}/*"]
                         },
                         {
                             "Sid": "AllowGlueActions",
@@ -113,7 +113,7 @@ class IAM(CidBase):
             policy_merge_mode='MERGE_RESOURCES',
             managed_policies=[]
         )
-    def ensure_data_source_role_exists(self, role_name, database, workgroup, kms_key_arns='', buckets=[], output_location_bucket=None):
+    def ensure_data_source_role_exists(self, role_name, databases, workgroup, kms_key_arns='', buckets=[], output_location_bucket=None):
         ''' Create or update a role specifically for a QS Datasource
         '''
         return self.ensure_role_with_policy(
@@ -151,14 +151,16 @@ class IAM(CidBase):
                                 "glue:GetPartition",
                                 "glue:GetPartitions",
                                 "glue:GetDatabases",
+                                "glue:GetDatabase",
                                 "glue:GetTable",
                                 "glue:GetTables",
                             ],
                             "Resource": [
                                 f"arn:{self.partition}:glue:{self.region}:{self.account_id}:catalog",
-                                f"arn:{self.partition}:glue:{self.region}:{self.account_id}:database/{database}",
-                                f"arn:{self.partition}:glue:{self.region}:{self.account_id}:table/{database}/*",
-                            ],
+                            ] + sum([
+                                [f"arn:{self.partition}:glue:{self.region}:{self.account_id}:database/{database}",
+                                f"arn:{self.partition}:glue:{self.region}:{self.account_id}:table/{database}/*" ]
+                            for database in databases ], []),
                         },
                         {
                             "Sid": "AllowAthena",
@@ -175,7 +177,6 @@ class IAM(CidBase):
                             ],
                             "Resource": [
                                 f"arn:{self.partition}:athena:{self.region}:{self.account_id}:datacatalog/AwsDataCatalog", # TODO: check if this can be variable?
-                                f"arn:{self.partition}:athena:{self.region}:{self.account_id}:database/{database}",
                                 f"arn:{self.partition}:athena:{self.region}:{self.account_id}:workgroup/{workgroup}",
                             ]
                         },
@@ -323,6 +324,22 @@ class IAM(CidBase):
         except self.client.exceptions.ClientError as exc:
             logger.warning('Failed to read available IEM roles: {exc}. Most likely not fatal.')
 
+    def get_role_arn(self, role_name:str) -> str:
+        """ Get role arn, and try the best guess if no permissions
+        """
+        try:
+            return self.client.get_role(RoleName=role_name)['Role']['Arn']
+        except self.client.exceptions.NoSuchEntityException:
+            raise CidError(f"Role '{role_name}' does not exist.")
+        except self.client.exceptions.ClientError as exc:
+            if "AccessDenied" in str(exc):
+                logger.debug('Got access denied for describing role. Try the best guess')
+                if role_name.startswith('aws-quicksight-service-role'):
+                    return f'arn:aws:iam::{self.account_id}:role/service-role/{role_name}'
+                else:
+                    return f'arn:aws:iam::{self.account_id}:role/{role_name}'
+            raise CidError(f"An error occurred: {exc}")
+
     def ensure_role_does_not_exist(self, role_name):
         """ Remove a role and all policies if they are not used in other roles
         """

{cid_cmd-0.3.2 → cid_cmd-0.3.8}/cid/helpers/quicksight/__init__.py

@@ -1335,10 +1335,10 @@ class QuickSight(CidBase):
             'Name': definition.get('name'),
             'ValidationStrategy': {'Mode': 'LENIENT'},
         }
-        theme = definition.get('theme')
+        theme = get_parameters().get('theme') or definition.get('theme')
         if theme:
             if not theme.startswith('arn:'):
-                theme_arn = 'arn:aws:quicksight::aws:theme/' + theme
+                theme_arn = f'arn:{self.partition}:quicksight::aws:theme/' + theme
             else:
                 raise NotImplementedError('Only standard themes are supported now.')
             create_parameters['ThemeArn'] = theme_arn

{cid_cmd-0.3.2 → cid_cmd-0.3.8/cid_cmd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cid-cmd
-Version: 0.3.2
+Version: 0.3.8
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-0.3.2/cid/_version.py

@@ -1 +0,0 @@
-__version__ = '0.3.2'