cid-cmd 0.2.45__tar.gz → 0.2.47__tar.gz

{cid-cmd-0.2.45/cid_cmd.egg-info → cid-cmd-0.2.47}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cid-cmd
-Version: 0.2.45
+Version: 0.2.47
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid-cmd-0.2.47/cid/_version.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.2.47'

{cid-cmd-0.2.45 → cid-cmd-0.2.47}/cid/base.py

@@ -21,6 +21,12 @@ class CidBase():
     def account_id(self) -> str:
         return self.awsIdentity.get('Account')
 
+    @property
+    def domain(self) -> str:
+        if self.region.startswith('cn-'):
+            return 'amazonaws.cn'
+        return 'aws.amazon.com'
+
     @property
     def awsIdentity(self) -> dict:
         if not self._awsIdentity:

{cid-cmd-0.2.45 → cid-cmd-0.2.47}/cid/cli.py

@@ -1,4 +1,6 @@
+import os
 import logging
+import platform
 
 import click
 
@@ -14,7 +16,6 @@ prog_name="CLOUD INTELLIGENCE DASHBOARDS (CID) CLI"
 print(f'{prog_name} {version}\n')
 
 if __version__ != latest_version and latest_version != 'UNDEFINED':
-    
     print('\033[93mUPDATE AVAILABLE\033[0m')
     print(f'\033[93mA new version {latest_version} is available, please consider update cid-cmd package via pip\033[0m\n\n')
     logger.info(f'A new version {latest_version} is available, please consider update cid-cmd package via pip')
@@ -67,6 +68,11 @@ def cid_command(func):
 @click.option('-y', '--yes', help='confirm all', is_flag=True, default=False)
 @click.pass_context
 def main(ctx, **kwargs):
+
+    # enable color for windows terminal
+    if platform.system() == "Windows":
+        os.system('color') #nosec B605, B607
+
     ctx.obj = Cid(**kwargs)
 
 

{cid-cmd-0.2.45 → cid-cmd-0.2.47}/cid/common.py

@@ -23,7 +23,7 @@ from botocore.exceptions import ClientError, NoCredentialsError, CredentialRetri
 from cid import utils
 from cid.base import CidBase
 from cid.plugin import Plugin
-from cid.utils import get_parameter, get_parameters, set_parameters, unset_parameter, get_yesno_parameter, cid_print, isatty, merge_objects
+from cid.utils import get_parameter, get_parameters, set_parameters, unset_parameter, get_yesno_parameter, cid_print, isatty, merge_objects, IsolatedParameters
 from cid.helpers.account_map import AccountMap
 from cid.helpers import Athena, CUR, Glue, QuickSight, Dashboard, Dataset, Datasource, csv2view, Organizations
 from cid.helpers.quicksight.template import Template as CidQsTemplate
@@ -45,7 +45,7 @@ class Cid():
         self.plugins = self.__loadPlugins()
         self._clients = dict()
         self._visited_views = [] # Views updated in the current session
-        self.qs_url = 'https://{region}.quicksight.aws.amazon.com/sn/dashboards/{dashboard_id}'
+        self.qs_url = 'https://{region}.quicksight.{domain}/sn/dashboards/{dashboard_id}'
         self.all_yes = kwargs.get('yes')
         self.verbose = kwargs.get('verbose')
         set_parameters(kwargs, self.all_yes)
@@ -75,7 +75,8 @@ class Cid():
                 logger.info(f'AWS profile name: {self.base.session.profile_name}')
             self.qs_url_params = {
                 'account_id': self.base.account_id,
-                'region': self.base.session.region_name
+                'region': self.base.session.region_name,
+                'domain': self.base.domain,
             }
         except (NoCredentialsError, CredentialRetrievalError):
             raise CidCritical('Error: Not authenticated, please check AWS credentials')
@@ -630,24 +631,63 @@ class Cid():
     @command
     def status(self, dashboard_id, **kwargs):
         """Check QuickSight dashboard status"""
-
-        if not dashboard_id:
-            if not self.qs.dashboards:
-                print('No deployed dashboards found')
-                return
-            dashboard_id = self.qs.select_dashboard(force=True)
+        next_selection = None
+        while next_selection != 'exit':
             if not dashboard_id:
-                print('No dashboard selected')
-                return
-            dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
-        else:
+                if not self.qs.dashboards:
+                    print('No deployed dashboards found')
+                    return
+                dashboard_id = self.qs.select_dashboard(force=True)
+                if not dashboard_id:
+                    print('No dashboard selected')
+                    return
             dashboard = self.qs.discover_dashboard(dashboardId=dashboard_id)
 
-        if dashboard is not None:
-            dashboard.display_status()
-            dashboard.display_url(self.qs_url, **self.qs_url_params)
-        else:
-            click.echo('not deployed.')
+            if dashboard is not None:
+                dashboard.display_status()
+                dashboard.display_url(self.qs_url, **self.qs_url_params)
+                with IsolatedParameters():
+                    next_selection = get_parameter(
+                        param_name=f'{dashboard.id}',
+                        message="Please make a selection",
+                        choices={
+                            '[◀] Back': 'back',
+                            '[↗] Open': 'open',
+                            '[◴] Refresh datasets': 'refresh',
+                            '[↺] Update dashboard': 'update',
+                            '[✕] Exit': 'exit',
+                        }
+                    )
+                    if next_selection == 'open':
+                        self.open(dashboard.id, **kwargs)
+
+                    elif next_selection == 'refresh':
+                        dashboard.refresh_datasets()
+
+                    elif next_selection == 'update':
+                        if dashboard.latest:
+                            if not get_yesno_parameter(
+                                    param_name=f'redeploy-{dashboard.id}',
+                                    message=f'\nThe selected dashboard {dashboard.id} is already on the latest version.\nDo you want to re-deploy it?',
+                                    default='no'):
+                                logger.info(f'Not re-deploying {dashboard.id} as it is on latest version.\n')
+                                continue
+                        recursive = get_parameter(
+                            param_name='recursive',
+                            message=f'\nRecursive update the Datasets and Views in addition to the Dashboard update?\nATTENTION: This could lead to the loss of dataset customization.\nRecursive update?',
+                            choices={
+                                '[→] Simple Update (only dashboard)': 'simple',
+                                '[⇶] Recursive Update (dashboard and all dependencies)': 'recursive',
+                            }
+                        ) == 'recursive'
+                        logger.info(f'Updating dashboard: {dashboard.id} with Recursive = {recursive}')
+                        self._deploy(dashboard_id, recursive=recursive, update=True)
+                        logger.info('Rediscover dashboards after update')
+                        self.qs.discover_dashboards()
+                self.qs.clear_dashboard_selection()
+                dashboard_id = None
+            else:
+                cid_print('not deployed.')
 
     @command
     def delete(self, dashboard_id, **kwargs):
@@ -1284,7 +1324,7 @@ class Cid():
                 athena_datasource = Datasource(raw={
                     'AthenaParameters':{},
                     "Id": datasource_id,
-                    "Arn": f"arn:aws:quicksight:{self.base.session.region_name}:{self.base.account_id}:datasource/{datasource_id}",
+                    "Arn": f"arn:{self.base.partition}:quicksight:{self.base.session.region_name}:{self.base.account_id}:datasource/{datasource_id}",
                 })
             except Exception as exc:
                 raise CidCritical(

{cid-cmd-0.2.45 → cid-cmd-0.2.47}/cid/export.py

@@ -69,6 +69,16 @@ def choose_analysis(qs):
     return choices[choice]['AnalysisId']
 
 
+def get_theme(analysis):
+    theme_arn = analysis.get('ThemeArn')
+    if not theme_arn:
+        return None
+    if theme_arn.startswith('arn:aws:quicksight::aws:theme/'):
+        return theme_arn.split('/')[-1]
+    logger.warning(f'Theme {theme_arn} is not standard and is not supported yet. Theme will be ignored.')
+    return None
+
+
 def export_analysis(qs, athena):
     """ Export analysis to yaml resource File
     """
@@ -94,7 +104,7 @@ def export_analysis(qs, athena):
     resources = {}
     resources['dashboards'] = {}
     resources['datasets'] = {}
-
+    theme_id = get_theme(analysis)
 
     cur_helper = CUR(session=athena.session)
     resources_datasets = []
@@ -270,6 +280,8 @@ def export_analysis(qs, athena):
     dashboard_resource['name'] = analysis['Name']
     dashboard_resource['dashboardId'] = dashboard_id
     dashboard_resource['category'] = get_parameters().get('category', 'Custom')
+    if theme_id:
+         dashboard_resource['theme'] = theme_id
 
     dashboard_export_method = None
     if get_parameters().get('template-id'):
@@ -344,7 +356,7 @@ def export_analysis(qs, athena):
             TemplateId=template_id,
             GrantPermissions=[
                 {
-                    "Principal": f'arn:aws:iam::{reader_account_id}:root' if reader_account_id != '*' else '*',
+                    "Principal": f'arn:{qs.partition}:iam::{reader_account_id}:root' if reader_account_id != '*' else '*',
                     'Actions': [
                         "quicksight:DescribeTemplate",
                     ]
@@ -363,7 +375,7 @@ def export_analysis(qs, athena):
 
         for dataset in definition.get('DataSetIdentifierDeclarations', []):
             # Hide region and account number of the source account
-            dataset["DataSetArn"] = 'arn:aws:quicksight:::dataset/' + dataset["DataSetArn"].split('/')[-1]
+            dataset["DataSetArn"] = f'arn:{qs.partition}:quicksight:::dataset/' + dataset["DataSetArn"].split('/')[-1]
         dashboard_resource['data'] = yaml.safe_dump(definition)
 
     resources['dashboards'][analysis['Name'].upper()] = dashboard_resource

{cid-cmd-0.2.45 → cid-cmd-0.2.47}/cid/helpers/quicksight/__init__.py

@@ -2,6 +2,7 @@ import re
 import json
 import uuid
 import time
+import datetime
 import logging
 from string import Template
 from typing import Dict, List, Union
@@ -15,7 +16,7 @@ from cid.helpers.quicksight.dashboard import Dashboard
 from cid.helpers.quicksight.dataset import Dataset
 from cid.helpers.quicksight.datasource import Datasource
 from cid.helpers.quicksight.template import Template as CidQsTemplate
-from cid.utils import get_parameter, get_parameters, exec_env, cid_print, ago
+from cid.utils import get_parameter, get_parameters, exec_env, cid_print, ago, unset_parameter
 from cid.exceptions import CidCritical, CidError
 
 logger = logging.getLogger(__name__)
@@ -604,6 +605,10 @@ class QuickSight(CidBase):
             logger.debug(exc, exc_info=True)
             return list()
 
+    def clear_dashboard_selection (self):
+        """ Clears the current dashboard selection. """
+        unset_parameter('dashboard-id')
+
     def select_dashboard(self, force=False) -> str:
         """ Select from a list of discovered dashboards """
         dashboard_id = get_parameters().get('dashboard-id')
@@ -614,8 +619,9 @@ class QuickSight(CidBase):
             return None
         choices = {}
         for dashboard in self.dashboards.values():
-            health = 'healthy' if dashboard.health else 'unhealthy'
-            key = f'{dashboard.name} ({dashboard.arn}, {health}, {dashboard.status})'
+            health = '' if dashboard.health else ' UNHEALTHY'
+            status = '' if dashboard.status == 'up to date' else ' ' + dashboard.status.upper()
+            key = f'{dashboard.name} ({dashboard.arn.split("/")[-1]}){health}{status}'
             notice = dashboard.definition.get('deprecationNotice', '')
             if notice:
                 key = f'{key} {notice}'
@@ -950,6 +956,29 @@ class QuickSight(CidBase):
             logger.debug(exc, exc_info=True)
             logger.info('No datasets found')
 
+    def refresh_dataset(self, dataset_id):
+        """ Refresh the dataset """
+
+        logger.info(f'Starting refresh for dataset: {dataset_id}')
+        status = 'FAILED'
+        try:
+            response = self.client.describe_data_set(
+                AwsAccountId=self.account_id,
+                DataSetId=dataset_id)
+            mode = response.get('DataSet').get('ImportMode')
+            if mode == 'DIRECT_QUERY':
+                return mode, 'DIRECT'
+            response = self.client.create_ingestion(
+                DataSetId=dataset_id,
+                IngestionId=datetime.datetime.now().strftime("%d%m%y-%H%M%S-%f"),
+                AwsAccountId=self.account_id)
+            status = response.get('IngestionStatus')
+        except self.client.exceptions.AccessDeniedException:
+            logger.error(f'Access denied refreshing dataset: {dataset_id}')
+        except Exception as exc:
+            logger.debug(exc, exc_info=True)
+            raise CidError(f'Unable to list refresh dataset {dataset_id}: {str(exc)}') from exc
+        return mode, status
 
     def describe_data_source(self, id: str, update: bool=False) -> Datasource:
         """ Describes an AWS QuickSight DataSource """
@@ -1125,12 +1154,27 @@ class QuickSight(CidBase):
         try:
             existing_schedules = self.get_dataset_refresh_schedules(dataset_id)
         except CidError as exc:
-            logger.debug(f'List refresh schedule throws: {exc}')
-            logger.warning(
-                f'Cannot read dataset schedules for dataset = {dataset_id}. {str(exc)}. Skipping schedule management.'
-                ' Please make sure scheduled refresh is configured manually.'
-            )
-            return
+            # We cannot access schedules, but let's check if there are scheduled ingestions. 
+            ingestions_exist = False
+            try:
+                ingestions_exist = list(
+                    self.client.get_paginator('list_ingestions').paginate(
+                        DataSetId=dataset_id,
+                        AwsAccountId=self.account_id
+                    ).search("Ingestions[?RequestSource=='SCHEDULED']")
+                )
+            except Exception:
+                logger.debug(f'List refresh schedule throws: {exc}')
+                logger.warning(
+                    f'Cannot read dataset schedules for dataset = {dataset_id}. {str(exc)}. Skipping schedule management.'
+                    ' Please make sure scheduled refresh is configured manually.'
+                )
+                return
+            if ingestions_exist:
+                logger.debug(f'We cannot read schedules but there are ingestions. Skipping creation of schedule.')
+                return
+            logger.debug(f'We cannot read schedules but there no ingestions. Continue to creation of schedule.')
+            existing_schedules = []
 
         if schedules:
             if exec_env()['terminal'] in ('lambda'):
@@ -1198,6 +1242,8 @@ class QuickSight(CidBase):
                     logger.error(f'Unable to create refresh schedule with id {schedule["ScheduleId"]}. Dataset {dataset_id} does not exist.')
                 except self.client.exceptions.AccessDeniedException:
                     logger.error(f'Unable to create refresh schedule with id {schedule["ScheduleId"]}. Please add quicksight:CreateDataSet permission.')
+                except self.client.exceptions.ResourceExistsException:
+                    logger.info(f'Schedule with id {schedule["ScheduleId"]} exists. But can have other settings. You better check.')
                 except Exception as exc:
                     logger.error(f'Unable to create refresh schedule with id {schedule["ScheduleId"]} for dataset "{dataset_id}": {str(exc)}')
             else:
@@ -1271,6 +1317,13 @@ class QuickSight(CidBase):
             'Name': definition.get('name'),
             'ValidationStrategy': {'Mode': 'LENIENT'},
         }
+        theme = definition.get('theme')
+        if theme:
+            if not theme.startswith('arn:'):
+                theme_arn = 'arn:aws:quicksight::aws:theme/' + theme
+            else:
+                raise NotImplementedError('Only standard themes are supported now.')
+            create_parameters['ThemeArn'] = theme_arn
 
         if definition.get('sourceTemplate'):
             dataset_references = [

{cid-cmd-0.2.45 → cid-cmd-0.2.47}/cid/helpers/quicksight/dashboard.py

@@ -81,7 +81,7 @@ class Dashboard(CidQsResource):
             if self.version.get('Status') not in ['CREATION_SUCCESSFUL']:
                 self._status = 'broken'
                 self.status_detail = f"{self.version.get('Status')}: {self.version.get('Errors')}"
-            # Not dicovered yet
+            # Not discovered yet
             elif not self.definition:
                 self._status = 'undiscovered'
             # Missing dataset
@@ -147,13 +147,17 @@ class Dashboard(CidQsResource):
         if self.datasets:
             cid_print(f"  <BOLD>Datasets:<END>")
             for dataset_name, dataset_id in  sorted(self.datasets.items()):
-                status = self.qs.get_dataset_last_ingestion(dataset_id) or '<BLUE>DIRECT<END>'
+                status = self.qs.get_dataset_last_ingestion(dataset_id) or '<BLUE>DIRECT<END>' #todo fix this Blue using dataset import type.
                 cid_print(f'    {dataset_name: <36} ({dataset_id: <36}) {status}')
 
-        print('\n')
-        if get_yesno_parameter('display-raw', 'Display dashboard raw data?', default='yes'):
-            print(json.dumps(self.raw, indent=4, sort_keys=True, default=str))
-
     def display_url(self, url_template: str, launch: bool = False, **kwargs) -> None:
         url = url_template.format(dashboard_id=self.id, **kwargs)
         print(f"#######\n####### {self.name} is available at: " + url + "\n#######")
+
+    def refresh_datasets(self) -> None:
+        """Refresh datasets of dashboard"""
+        if self.datasets:
+            cid_print(f"  <BOLD>Refreshing Datasets:<END>")
+            for dataset_name, dataset_id in  sorted(self.datasets.items()):
+                mode, status = self.qs.refresh_dataset(dataset_id)
+                cid_print(f'    {dataset_name: <36} ({dataset_id: <36}) Refresh Status: {status} Mode: {mode}')

cid-cmd-0.2.47/cid/test/python/test_isolated_parameters.py

@@ -0,0 +1,12 @@
+from cid.utils import IsolatedParameters, get_parameters, set_parameters
+
+def test_isolated_parameter_context():
+    """ make sure the isolated_parameter works
+    """
+    set_parameters({'param': 'a'})
+
+    with IsolatedParameters():
+        set_parameters({'param': 'b'})
+        assert get_parameters().get('param') == 'b', 'parameters within context must be B'
+
+    assert get_parameters().get('param') == 'a', 'parameters within context must be A'

{cid-cmd-0.2.45 → cid-cmd-0.2.47}/cid/utils.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import copy
 import math
 import inspect
 import logging
@@ -279,6 +280,15 @@ def ago(time):
     return 'just now'
 
 
+class IsolatedParameters:
+    """A context manager to run something in isolated set of parameters"""
+    def __enter__(self):
+        self.backup = copy.deepcopy(params)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        global params
+        params = self.backup
+
 def merge_objects(obj1, obj2, depth=2):
     """ merging objects with a depth
 

{cid-cmd-0.2.45 → cid-cmd-0.2.47/cid_cmd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: cid-cmd
-Version: 0.2.45
+Version: 0.2.47
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

{cid-cmd-0.2.45 → cid-cmd-0.2.47}/cid_cmd.egg-info/SOURCES.txt

@@ -115,6 +115,7 @@ cid/helpers/quicksight/datasource.py
 cid/helpers/quicksight/resource.py
 cid/helpers/quicksight/template.py
 cid/test/python/test_csv2view.py
+cid/test/python/test_isolated_parameters.py
 cid/test/python/test_merge.py
 cid_cmd.egg-info/PKG-INFO
 cid_cmd.egg-info/SOURCES.txt

cid-cmd-0.2.45/cid/_version.py

@@ -1,2 +0,0 @@
-
-__version__ = '0.2.45'