cid-cmd 4.2.2__tar.gz → 4.2.4__tar.gz

{cid_cmd-4.2.2/cid_cmd.egg-info → cid_cmd-4.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cid-cmd
-Version: 4.2.2
+Version: 4.2.4
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-4.2.4/cid/_version.py

@@ -0,0 +1,2 @@
+__version__ = '4.2.4'
+

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/builtin/core/data/datasets/kpi/kpi_tracker.json

@@ -48,6 +48,11 @@
                         "Type": "DECIMAL",
                         "SubType": "FLOAT"
                     },
+                    {
+                        "Name": "ec2_usage_cost",
+                        "Type": "DECIMAL",
+                        "SubType": "FLOAT"
+                    },
                     {
                         "Name": "ec2_spot_cost",
                         "Type": "DECIMAL",
@@ -397,6 +402,7 @@
                             "linked_account_id",
                             "spend_all_cost",
                             "ec2_all_cost",
+                            "ec2_usage_cost",
                             "ec2_spot_cost",
                             "ec2_spot_potential_savings",
                             "ec2_previous_generation_cost",

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/builtin/core/data/queries/co/idle.json

@@ -82,7 +82,7 @@
                     "Type": "string"
                 },
                 {
-                    "Name": "utilizationetricsebsvolumewriteiopsmaximum",
+                    "Name": "utilizationmetricsebsvolumewriteiopsmaximum",
                     "Type": "string"
                 },
                 {

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/builtin/core/data/queries/co/idle_options.sql

@@ -23,7 +23,7 @@ CREATE OR REPLACE VIEW compute_optimizer_idle_options AS
         utilizationmetricsnetworkinbytespersecondmaximum, ';',
         utilizationmetricsdatabaseconnectionsmaximum, ';',
         utilizationmetricsebsvolumereadiopsmaximum, ';',
-        utilizationetricsebsvolumewriteiopsmaximum, ';',
+        utilizationmetricsebsvolumewriteiopsmaximum, ';',
         utilizationmetricsvolumereadopspersecondmaximum, ';',
         utilizationmetricsvolumewriteopspersecondmaximum
         ) utilizationmetrics

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/builtin/core/data/queries/kpi/last_kpi_tracker_view.sql

@@ -6,6 +6,7 @@ SELECT DISTINCT
 , spend_all.spend_all_cost
 , spend_all.tags_json
 , instance_all.ec2_all_cost
+, instance_all.ec2_usage_cost
 , instance_all.ec2_spot_cost
 , instance_all.ec2_spot_potential_savings
 , instance_all.ec2_previous_generation_cost
@@ -94,6 +95,7 @@ LEFT JOIN (
    , linked_account_id
    , tags_json
    , "sum"("ec2_all_cost") "ec2_all_cost"
+   , "sum"("ec2_usage_cost") "ec2_usage_cost"
    , "sum"("ec2_spot_cost") "ec2_spot_cost"
    , "sum"("ec2_spot_potential_savings") "ec2_spot_potential_savings"
    , "sum"("ec2_previous_generation_cost") "ec2_previous_generation_cost"

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/common.py

@@ -131,10 +131,10 @@ class Cid():
         """ get/create parameters_controller """
         return ParametersController(self.athena)
 
-    def get_cur(self, target_cur_version):
+    def get_cur(self, target_cur_version=None):
         """ get a cur """
         cur_version = self.cur.version
-        if cur_version != target_cur_version or get_parameters().get('use-cur-proxy'):
+        if (target_cur_version and cur_version != target_cur_version) or get_parameters().get('use-cur-proxy'):
             return ProxyCUR(self.cur, target_cur_version=target_cur_version)
         return self.cur
 
@@ -170,7 +170,7 @@ class Cid():
         account_map = AccountMap(
             self.base.session,
             self.athena,
-            self.cur, # can be any CUR. But it is only needed for trends and dummy
+            self.get_cur, # can be any CUR. But it is only needed for trends and dummy
         )
         return account_map.create_or_update(name)
 
@@ -409,9 +409,7 @@ class Cid():
                         res_list = self.athena.query(query)
                     except (self.athena.client.exceptions.ClientError, CidError, CidCritical) as exc:
                         raise CidCritical(f'Failed fetching parameter {prefix}{key}: {exc}.') from exc
-                    if not res_list:
-                        raise CidCritical(f'Failed fetching parameter {prefix}{key}, {value}. Athena returns empty results. {value.get("error")}')
-                    options = ['-'.join(res) for res in res_list]
+                    options = ['-'.join(res) for res in (res_list or [])]
                     params[key] = self.generic_tags_json(
                         param_name=key,
                         options=options,
@@ -1605,9 +1603,15 @@ class Cid():
         # patch dataset for tags
         cur_tags_json_required = False
         for dep_view_name in dataset_definition.get('dependsOn', {}).get('views', []):
-            cur_tags_json_required = 'tags_json' in str()
-            if self.resources['views'].get(dep_view_name, {}).get('dependsOn',{}).get('tags') == 'json' \
-                or self.resources['views'].get(dep_view_name, {}).get('parameters',{}).get('resource-tags'):
+            try:
+                tags_type = self.resources['views'][dep_view_name]['dependsOn']['tags']
+            except (KeyError, TypeError, AttributeError):
+                tags_type = None
+            try:
+                param_res_tag =  self.resources['views'][dep_view_name]['parameters']['resource-tags']
+            except (KeyError, TypeError, AttributeError):
+                param_res_tag = None
+            if tags_type == 'json' or param_res_tag:
                 cur_tags_json_required = True
                 break
         custom_fields = {}

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/export.py

@@ -179,7 +179,7 @@ def export_analysis(qs, athena, glue):
             elif 'CustomSql' in value and 'DataSourceArn' in value['CustomSql']:
                 logger.debug(f"Dataset {dataset.raw['DataSetId']} looks like CustomSql athena dataset")
                 value['CustomSql']['DataSourceArn'] = '${athena_datasource_arn}'
-                databases = [db_['Name'] for db_ in athena.list_databases()]
+                databases = athena.list_databases()
                 for database in databases:
                     if f'{database}.' in value['CustomSql']['SqlQuery'] or f'"{database}".' in value['CustomSql']['SqlQuery']:
                         logger.debug(f"Replacing {database} in text")

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/helpers/account_map.py

@@ -65,8 +65,8 @@ class AccountMap(CidBase):
         }
     }
 
-    def __init__(self, session: Session, athena: Athena, cur=None) -> None:
-        self.cur = cur # Only for trends and dummy
+    def __init__(self, session: Session, athena: Athena, get_cur=None) -> None:
+        self.get_cur = get_cur # get_cur callback Only for trends and dummy
         self.athena = athena
         super().__init__(session)
 
@@ -121,7 +121,7 @@ class AccountMap(CidBase):
             template = Template(
                 resource_string(
                     view_definition.get('providedBy'),
-                    f'data/queries/{view_file}'
+                    f'data/{view_file}'
                 ).decode('utf-8')
             )
         elif view_definition.get('data'):
@@ -133,8 +133,8 @@ class AccountMap(CidBase):
         vars = {
             'metadata_table_name': metadata_table['Name'],
             'metadata_database_name': metadata_table['Database'],
-            'cur_database': self.cur.database, # only for trends in metadata
-            'cur_table_name': self.cur.table_name, # only for trends in metadata
+            'cur_database': self.get_cur().database if name == 'aws_accounts' and self.get_cur else None, # only for trends in metadata
+            'cur_table_name': self.get_cur().table_name if name == 'aws_accounts' and self.get_cur else None, # only for trends in metadata
         }
         for key, val in self.mappings.get(name).get(metadata_table['Name']).items():
             logger.debug(f'Mapping field {key} to {val}')
@@ -151,7 +151,8 @@ class AccountMap(CidBase):
     def get_dummy_account_mapping_sql(self, name) -> list:
         """Create dummy account mapping"""
         logger.info(f'Creating dummy account mapping for {name}')
-        if self.cur.version.startswith('2'):
+        cur = self.get_cur()
+        if cur.version.startswith('2'):
             sql_file = 'data/queries/shared/account_map_cur2.sql'
         else:
             sql_file = 'data/queries/shared/account_map_dummy.sql'
@@ -161,8 +162,8 @@ class AccountMap(CidBase):
         ).decode('utf-8'))
         columns_tpl = {
             'athena_view_name': name,
-            'cur_table_name': self.cur.table_name,
-            'cur_database': self.cur.database,
+            'cur_table_name': cur.table_name,
+            'cur_database': cur.database,
         }
         compiled_query = template.safe_substitute(columns_tpl)
         return compiled_query
@@ -278,20 +279,23 @@ class AccountMap(CidBase):
             except CidError as exc:
                 logger.debug(exc)
 
-        if self.cur.version.startswith('2'):
+        if self.get_cur and self.get_cur().version.startswith('2'):
             cid_print('Looks like CUR2 is used. Will use it for account map.')
             set_parameters({'account-map-source': 'dummy'})
 
         for attempt in range(3):
             try:
+                choices = {
+                    'AWS Organizations (one time account listing)': 'organization',
+                    'CSV file (relative path required)': 'csv',
+                }
+                if self.get_cur:
+                    choices['Dummy (CUR account data, no names for CUR1)'] = 'dummy'
+
                 metadata_source = get_parameter(
                     param_name='account-map-source',
                     message="Please select account metadata collection method",
-                    choices={
-                        'Dummy (CUR account data, no names for CUR1)': 'dummy',
-                        'AWS Organizations (one time account listing)': 'organization',
-                        'CSV file (relative path required)': 'csv',
-                    },
+                    choices=choices,
                 )
                 logger.info(f'Attempt {attempt + 2}' )
                 # Collect account list from different sources of user choice

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/helpers/athena.py

@@ -282,16 +282,18 @@ class Athena(CidBase):
         logger.debug(f'WorkGroups: {result.get("WorkGroups")}')
         return result.get('WorkGroups')
 
-    def get_table_metadata(self, table_name: str, database_name: str=None) -> dict:
-        table_metadata = self._metadata.get(table_name)
-        params = {
-            'CatalogName': self.CatalogName,
-            'DatabaseName': database_name or self.DatabaseName,
-            'TableName': table_name
-        }
+    def get_table_metadata(self, table_name: str, database_name: str=None, no_cache: bool=False) -> dict:
+        table_metadata = None
+        if not no_cache:
+            table_metadata = self._metadata.get(table_name)
         if not table_metadata:
+            params = {
+                'CatalogName': self.CatalogName,
+                'DatabaseName': database_name or self.DatabaseName,
+                'TableName': table_name,
+            }
             table_metadata = self.client.get_table_metadata(**params).get('TableMetadata')
-            self._metadata.update({table_name: table_metadata})
+            self._metadata[table_name] = table_metadata
 
         return table_metadata
 

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/helpers/cur.py

@@ -184,8 +184,10 @@ class AbstractCUR(CidBase):
                 return self._tag_and_cost_category
 
             self._tag_and_cost_category = []
-            number_of_rows_scanned = 100000 # empiric value
-            for tag_type in ["resource_tags", 'cost_category']:
+            number_of_rows_scanned = 500000 # empiric value
+            for tag_type in ['resource_tags', 'cost_category']:
+                if tag_type not in self.fields:
+                    logging.debug(f'skipping {tag_type} scan')
                 cid_print(f'Scanning {tag_type} in {self.table_name}.')
                 try:
                     res = self.athena.query(
@@ -211,7 +213,9 @@ class AbstractCUR(CidBase):
                     cid_print(f' <BOLD>{tag_type:<{max_width}} | Distinct Values <END> ')
                     for line in res:
                         if int(line[1]) > 10:
-                            cid_print(f' <BOLD>{line[0]:<{max_width}}<END> | {line[1]} ')
+                            name = line[0]
+                            name = name.replace('user_', '')
+                            cid_print(f' <BOLD>{name:<{max_width}}<END> | {line[1]} ')
                     self._tag_and_cost_category += sorted([f"{tag_type}['{line[0]}']" for line in res])
                 except (self.athena.client.exceptions.ClientError, CidCritical, ValueError) as exc:
                     logger.error(f'Failed to read {tag_type} from {self.table_name}: "{exc}". Will continue without.')

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/helpers/quicksight/__init__.py

@@ -1205,6 +1205,8 @@ class QuickSight(CidBase):
             schedule["RefreshType"] = schedule.get("RefreshType", "FULL_REFRESH")
             if "providedBy" in schedule:
                 del schedule["providedBy"]
+            if "source" in schedule:
+                del schedule["source"]
 
             if not existing_schedule:
                 # Avoid adding a new schedule  when customer already has put a schedule manually as this can lead to additional charges.

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/helpers/quicksight/dataset.py

@@ -68,33 +68,12 @@ class Dataset(CidQsResource):
         '''
         def _get_athena_columns(table, database=None):
             '''returns athena columns'''
+            metadata = athena.get_table_metadata(table, database_name=database, no_cache=True)
             return [
-                line[0].split(None, 1)
-                for line in athena.query(
-                    f'SHOW COLUMNS FROM {table}',
-                    include_header=True, database=database
-                )
+                (col['Name'], col['Type'])
+                for col in metadata.get('Columns', [])
             ]
 
-        def _replace_columns(existing_columns, new_columns):
-            '''replace columns but keep the order, ignore case changes
-                assert _replace_columns(
-                    [{'Name': 'a'}, {'Name': 'B'}, {'Name': 'c'}],
-                    [{'Name': 'A'}, {'Name': 'b'}, {'Name': 'd'}]) \
-                 == [{'Name': 'a'}, {'Name': 'B'}, {'Name': 'd'}]
-            Different types will be replaced
-            '''
-            existing_columns = [
-                existing_col
-                for existing_col in existing_columns
-                if str(existing_col).lower() in [str(c).lower() for c in new_columns]
-            ] # filter out old
-            for col in new_columns: # add new
-                if col['Name'].lower() not in [c['Name'].lower() for c in existing_columns]:
-                    existing_columns.append(col)
-                #REFACTOR: what if col is there but another type?
-            return existing_columns
-
         def _athena_to_qs_type(col, athena_type):
             '''map athena type to QS type
              The following data types are supported in SPICE: Date, Decimal-fixed, Decimal-float, Integer, and String.
@@ -125,8 +104,14 @@ class Dataset(CidQsResource):
                 root_lt = dataset['LogicalTableMap'][lt]
                 break
         else:
-            raise ValueError(f'Unable to find a root logical table in the dataset {dataset}')
-        projected_cols = next(ds['ProjectOperation']["ProjectedColumns"] for ds in root_lt['DataTransforms'] if 'ProjectOperation' in ds)
+            # take the first one and let's hope it is fine
+            root_lt = next(iter(dataset['LogicalTableMap'].values()))
+
+        projected_cols = next( # get the first DataTrasform with ProjectOperation
+            ds['ProjectOperation']["ProjectedColumns"]
+            for ds in root_lt['DataTransforms']
+            if 'ProjectOperation' in ds
+        )
 
         # Update each PhysicalTableMap with all columns from athena views
         all_columns = []
@@ -135,12 +120,31 @@ class Dataset(CidQsResource):
             database = pt['RelationalTable']['Schema']
             columns = _get_athena_columns(table_name, database)
             logger.trace(f'columns = {columns}')
-
-            new_columns = [_athena_to_qs_type(name, athena_type) for name, athena_type in columns]
-            #for col in new_columns:
-            #    if col['Name'] in [existing_col['Name'] for existing_col in all_columns]: #FIXME not all_columns so far but must be all cols before modification
-            #        col['Name'] = f'{col["Name"]}[{table_name}]'
-            pt['RelationalTable']['InputColumns'] = _replace_columns(pt['RelationalTable']['InputColumns'], new_columns)
+            athena_columns = [
+                _athena_to_qs_type(name, athena_type.lower())
+                for name, athena_type in columns
+            ]
+            logger.trace(f'athena_columns = {columns}')
+            athena_columns_names = [c['Name'].lower() for c in athena_columns]
+            dataset_columns = pt['RelationalTable']['InputColumns']
+            dataset_columns_names = [col['Name'].lower() for col in dataset_columns]
+            dataset_columns_to_keep = [
+                col for col in dataset_columns
+                if col['Name'].lower() in athena_columns_names
+            ]
+            new_columns = [
+                col for col in athena_columns
+                if col['Name'].lower() not in dataset_columns_names
+            ] # BTW what if col is there but another type?
+
+            for col in new_columns: # alter names for columns that already exist
+                if col['Name'].lower() in projected_cols:
+                    col['Name'] = f"{col['Name']}[{table_name}]" # What if it is alrady there?
+
+            logger.trace(f'dataset_columns_to_keep = {dataset_columns_to_keep}')
+            if new_columns:
+                logger.trace(f'new_columns = {new_columns}')
+            pt['RelationalTable']['InputColumns'] = dataset_columns_to_keep + new_columns
             all_columns += [col['Name'] for col in pt['RelationalTable']['InputColumns']]
 
         # Add all needed calc fields

{cid_cmd-4.2.2 → cid_cmd-4.2.4}/cid/utils.py

@@ -190,7 +190,7 @@ def set_defaults(data: dict) -> None:
     if data:
         defaults.update(data)
 
-def get_defaults() -> None:
+def get_defaults() -> dict:
     global defaults
     return dict(defaults)
 
@@ -250,7 +250,7 @@ def get_parameter(param_name, message, choices=None, default=None, none_as_disab
     param_name = param_name.replace('_', '-')
 
     # override defaults from code with outside defaults
-    if param_name in defaults:
+    if param_name in get_defaults():
         default = defaults.get(param_name)
         if multi and isinstance(default, str):
             default = split_respecting_quotes(default)
@@ -276,6 +276,7 @@ def get_parameter(param_name, message, choices=None, default=None, none_as_disab
         if multi:
             default = default or []
             default = default if isinstance(default, list) else [default]
+            default = [c for c in defaults if c in choices]
             if not isatty():
                 result = default
             else:

{cid_cmd-4.2.2 → cid_cmd-4.2.4/cid_cmd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cid-cmd
-Version: 4.2.2
+Version: 4.2.4
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-4.2.2/cid/_version.py

@@ -1,2 +0,0 @@
-__version__ = '4.2.2'
-