cid-cmd 4.2.3__tar.gz → 4.2.5__tar.gz

{cid_cmd-4.2.3/cid_cmd.egg-info → cid_cmd-4.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cid-cmd
-Version: 4.2.3
+Version: 4.2.5
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-4.2.5/cid/_version.py

@@ -0,0 +1,2 @@
+__version__ = '4.2.5'
+

{cid_cmd-4.2.3 → cid_cmd-4.2.5}/cid/builtin/core/data/datasets/kpi/kpi_tracker.json

@@ -48,6 +48,11 @@
                         "Type": "DECIMAL",
                         "SubType": "FLOAT"
                     },
+                    {
+                        "Name": "ec2_usage_cost",
+                        "Type": "DECIMAL",
+                        "SubType": "FLOAT"
+                    },
                     {
                         "Name": "ec2_spot_cost",
                         "Type": "DECIMAL",
@@ -397,6 +402,7 @@
                             "linked_account_id",
                             "spend_all_cost",
                             "ec2_all_cost",
+                            "ec2_usage_cost",
                             "ec2_spot_cost",
                             "ec2_spot_potential_savings",
                             "ec2_previous_generation_cost",

{cid_cmd-4.2.3 → cid_cmd-4.2.5}/cid/builtin/core/data/queries/co/idle.json

@@ -82,7 +82,7 @@
                     "Type": "string"
                 },
                 {
-                    "Name": "utilizationetricsebsvolumewriteiopsmaximum",
+                    "Name": "utilizationmetricsebsvolumewriteiopsmaximum",
                     "Type": "string"
                 },
                 {

{cid_cmd-4.2.3 → cid_cmd-4.2.5}/cid/builtin/core/data/queries/co/idle_options.sql

@@ -23,7 +23,7 @@ CREATE OR REPLACE VIEW compute_optimizer_idle_options AS
         utilizationmetricsnetworkinbytespersecondmaximum, ';',
         utilizationmetricsdatabaseconnectionsmaximum, ';',
         utilizationmetricsebsvolumereadiopsmaximum, ';',
-        utilizationetricsebsvolumewriteiopsmaximum, ';',
+        utilizationmetricsebsvolumewriteiopsmaximum, ';',
         utilizationmetricsvolumereadopspersecondmaximum, ';',
         utilizationmetricsvolumewriteopspersecondmaximum
         ) utilizationmetrics

{cid_cmd-4.2.3 → cid_cmd-4.2.5}/cid/builtin/core/data/queries/kpi/last_kpi_tracker_view.sql

@@ -6,6 +6,7 @@ SELECT DISTINCT
 , spend_all.spend_all_cost
 , spend_all.tags_json
 , instance_all.ec2_all_cost
+, instance_all.ec2_usage_cost
 , instance_all.ec2_spot_cost
 , instance_all.ec2_spot_potential_savings
 , instance_all.ec2_previous_generation_cost
@@ -94,6 +95,7 @@ LEFT JOIN (
    , linked_account_id
    , tags_json
    , "sum"("ec2_all_cost") "ec2_all_cost"
+   , "sum"("ec2_usage_cost") "ec2_usage_cost"
    , "sum"("ec2_spot_cost") "ec2_spot_cost"
    , "sum"("ec2_spot_potential_savings") "ec2_spot_potential_savings"
    , "sum"("ec2_previous_generation_cost") "ec2_previous_generation_cost"

{cid_cmd-4.2.3 → cid_cmd-4.2.5}/cid/common.py

@@ -409,9 +409,7 @@ class Cid():
                         res_list = self.athena.query(query)
                     except (self.athena.client.exceptions.ClientError, CidError, CidCritical) as exc:
                         raise CidCritical(f'Failed fetching parameter {prefix}{key}: {exc}.') from exc
-                    if not res_list:
-                        raise CidCritical(f'Failed fetching parameter {prefix}{key}, {value}. Athena returns empty results. {value.get("error")}')
-                    options = ['-'.join(res) for res in res_list]
+                    options = ['-'.join(res) for res in (res_list or [])]
                     params[key] = self.generic_tags_json(
                         param_name=key,
                         options=options,

{cid_cmd-4.2.3 → cid_cmd-4.2.5}/cid/helpers/athena.py

@@ -282,16 +282,18 @@ class Athena(CidBase):
         logger.debug(f'WorkGroups: {result.get("WorkGroups")}')
         return result.get('WorkGroups')
 
-    def get_table_metadata(self, table_name: str, database_name: str=None) -> dict:
-        table_metadata = self._metadata.get(table_name)
-        params = {
-            'CatalogName': self.CatalogName,
-            'DatabaseName': database_name or self.DatabaseName,
-            'TableName': table_name
-        }
+    def get_table_metadata(self, table_name: str, database_name: str=None, no_cache: bool=False) -> dict:
+        table_metadata = None
+        if not no_cache:
+            table_metadata = self._metadata.get(table_name)
         if not table_metadata:
+            params = {
+                'CatalogName': self.CatalogName,
+                'DatabaseName': database_name or self.DatabaseName,
+                'TableName': table_name,
+            }
             table_metadata = self.client.get_table_metadata(**params).get('TableMetadata')
-            self._metadata.update({table_name: table_metadata})
+            self._metadata[table_name] = table_metadata
 
         return table_metadata
 

{cid_cmd-4.2.3 → cid_cmd-4.2.5}/cid/helpers/cur.py

@@ -184,8 +184,10 @@ class AbstractCUR(CidBase):
                 return self._tag_and_cost_category
 
             self._tag_and_cost_category = []
-            number_of_rows_scanned = 100000 # empiric value
-            for tag_type in ["resource_tags", 'cost_category']:
+            number_of_rows_scanned = 500000 # empiric value
+            for tag_type in ['resource_tags', 'cost_category']:
+                if tag_type not in self.fields:
+                    logging.debug(f'skipping {tag_type} scan')
                 cid_print(f'Scanning {tag_type} in {self.table_name}.')
                 try:
                     res = self.athena.query(
@@ -211,7 +213,9 @@ class AbstractCUR(CidBase):
                     cid_print(f' <BOLD>{tag_type:<{max_width}} | Distinct Values <END> ')
                     for line in res:
                         if int(line[1]) > 10:
-                            cid_print(f' <BOLD>{line[0]:<{max_width}}<END> | {line[1]} ')
+                            name = line[0]
+                            name = name.replace('user_', '')
+                            cid_print(f' <BOLD>{name:<{max_width}}<END> | {line[1]} ')
                     self._tag_and_cost_category += sorted([f"{tag_type}['{line[0]}']" for line in res])
                 except (self.athena.client.exceptions.ClientError, CidCritical, ValueError) as exc:
                     logger.error(f'Failed to read {tag_type} from {self.table_name}: "{exc}". Will continue without.')

{cid_cmd-4.2.3 → cid_cmd-4.2.5}/cid/helpers/quicksight/__init__.py

@@ -1205,6 +1205,8 @@ class QuickSight(CidBase):
             schedule["RefreshType"] = schedule.get("RefreshType", "FULL_REFRESH")
             if "providedBy" in schedule:
                 del schedule["providedBy"]
+            if "source" in schedule:
+                del schedule["source"]
 
             if not existing_schedule:
                 # Avoid adding a new schedule  when customer already has put a schedule manually as this can lead to additional charges.

{cid_cmd-4.2.3 → cid_cmd-4.2.5}/cid/helpers/quicksight/dataset.py

@@ -68,33 +68,12 @@ class Dataset(CidQsResource):
         '''
         def _get_athena_columns(table, database=None):
             '''returns athena columns'''
+            metadata = athena.get_table_metadata(table, database_name=database, no_cache=True)
             return [
-                line[0].split(None, 1)
-                for line in athena.query(
-                    f'SHOW COLUMNS FROM {table}',
-                    include_header=True, database=database
-                )
+                (col['Name'], col['Type'])
+                for col in metadata.get('Columns', [])
             ]
 
-        def _replace_columns(existing_columns, new_columns):
-            '''replace columns but keep the order, ignore case changes
-                assert _replace_columns(
-                    [{'Name': 'a'}, {'Name': 'B'}, {'Name': 'c'}],
-                    [{'Name': 'A'}, {'Name': 'b'}, {'Name': 'd'}]) \
-                 == [{'Name': 'a'}, {'Name': 'B'}, {'Name': 'd'}]
-            Different types will be replaced
-            '''
-            existing_columns = [
-                existing_col
-                for existing_col in existing_columns
-                if str(existing_col).lower() in [str(c).lower() for c in new_columns]
-            ] # filter out old
-            for col in new_columns: # add new
-                if col['Name'].lower() not in [c['Name'].lower() for c in existing_columns]:
-                    existing_columns.append(col)
-                #REFACTOR: what if col is there but another type?
-            return existing_columns
-
         def _athena_to_qs_type(col, athena_type):
             '''map athena type to QS type
              The following data types are supported in SPICE: Date, Decimal-fixed, Decimal-float, Integer, and String.
@@ -125,23 +104,61 @@ class Dataset(CidQsResource):
                 root_lt = dataset['LogicalTableMap'][lt]
                 break
         else:
-            raise ValueError(f'Unable to find a root logical table in the dataset {dataset}')
-        projected_cols = next(ds['ProjectOperation']["ProjectedColumns"] for ds in root_lt['DataTransforms'] if 'ProjectOperation' in ds)
+            # take the first one and let's hope it is fine
+            root_lt = next(iter(dataset['LogicalTableMap'].values()))
+
+        renames = {}
+        for lt in dataset['LogicalTableMap'].values():
+            for dt in lt.get('DataTransforms', []):
+                if "RenameColumnOperation" in dt:
+                    key = lt['Source'].get('PhysicalTableId', '') + '.' +  dt["RenameColumnOperation"]['ColumnName'] 
+                    renames[key] = dt["RenameColumnOperation"]['NewColumnName']
+        logger.trace(f'renames = {renames}')
+
+        projected_cols = next( # get the first DataTransform with ProjectOperation
+            ds['ProjectOperation']["ProjectedColumns"]
+            for ds in root_lt['DataTransforms']
+            if 'ProjectOperation' in ds
+        )
 
         # Update each PhysicalTableMap with all columns from athena views
         all_columns = []
-        for pt in dataset['PhysicalTableMap'].values():
+        for pt_id, pt in dataset['PhysicalTableMap'].items():
             table_name = pt['RelationalTable']['Name']
             database = pt['RelationalTable']['Schema']
             columns = _get_athena_columns(table_name, database)
             logger.trace(f'columns = {columns}')
-
-            new_columns = [_athena_to_qs_type(name, athena_type) for name, athena_type in columns]
-            #for col in new_columns:
-            #    if col['Name'] in [existing_col['Name'] for existing_col in all_columns]: #FIXME not all_columns so far but must be all cols before modification
-            #        col['Name'] = f'{col["Name"]}[{table_name}]'
-            pt['RelationalTable']['InputColumns'] = _replace_columns(pt['RelationalTable']['InputColumns'], new_columns)
-            all_columns += [col['Name'] for col in pt['RelationalTable']['InputColumns']]
+            athena_columns = [
+                _athena_to_qs_type(name, athena_type.lower())
+                for name, athena_type in columns
+            ]
+            logger.trace(f'athena_columns = {columns}')
+            athena_columns_names = [c['Name'].lower() for c in athena_columns]
+            dataset_columns = pt['RelationalTable']['InputColumns']
+            dataset_columns_names = [col['Name'].lower() for col in dataset_columns]
+            dataset_columns_to_keep = [
+                col for col in dataset_columns
+                if col['Name'].lower() in athena_columns_names
+            ]
+            new_columns = [
+                col for col in athena_columns
+                if col['Name'].lower() not in dataset_columns_names
+            ] # BTW what if col is there but another type?
+
+            # FIXME: need to add RenameColumnOperation!  col['Name'] => f"{col['Name']}[{table_name}]"
+            new_columns = [col for col in new_columns if col['Name'].lower() not in projected_cols] # avoid things that are already there (probably need to take renames into account)
+            new_columns = [col for col in new_columns if col['Name'] not in all_columns] # avoid adding 2nd time
+
+            logger.trace(f'dataset_columns_to_keep = {dataset_columns_to_keep}')
+            if new_columns:
+                logger.trace(f'new_columns = {new_columns} from {pt_id}')
+            pt['RelationalTable']['InputColumns'] = dataset_columns_to_keep + new_columns
+
+            for col in new_columns:
+                col_name = col['Name']
+                if f'{pt_id}.{col_name}' in renames:
+                    col_name = renames[f'{pt_id}.{col_name}']
+                all_columns.append(col_name)
 
         # Add all needed calc fields
         existing_create_columns = [dt.get("CreateColumnsOperation", {}).get('Columns', [None])[0] for dt in root_lt.get('DataTransforms', []) if dt.get("CreateColumnsOperation")]
@@ -168,12 +185,12 @@ class Dataset(CidQsResource):
         # Add all new cols to projected columns
         for col in set(all_columns):
             if col.lower() not in [c.lower() for c in projected_cols]:
+                logger.trace(f'adding {col}')
                 projected_cols.append(col)
 
         # filter out all columns that cannot be used for dataset creation
         update_ = {key: value for key, value in dataset.items() if key in 'DataSetId, Name, PhysicalTableMap, LogicalTableMap, ImportMode, ColumnGroups, FieldFolders, RowLevelPermissionDataSet, RowLevelPermissionTagConfiguration, ColumnLevelPermissionRules, DataSetUsageConfiguration, DatasetParameters'.split(', ')}
         logger.trace(f'update_ = {update_}')
-
         return update_
 
 

{cid_cmd-4.2.3 → cid_cmd-4.2.5}/cid/utils.py

@@ -190,7 +190,7 @@ def set_defaults(data: dict) -> None:
     if data:
         defaults.update(data)
 
-def get_defaults() -> None:
+def get_defaults() -> dict:
     global defaults
     return dict(defaults)
 
@@ -250,7 +250,7 @@ def get_parameter(param_name, message, choices=None, default=None, none_as_disab
     param_name = param_name.replace('_', '-')
 
     # override defaults from code with outside defaults
-    if param_name in defaults:
+    if param_name in get_defaults():
         default = defaults.get(param_name)
         if multi and isinstance(default, str):
             default = split_respecting_quotes(default)
@@ -276,6 +276,7 @@ def get_parameter(param_name, message, choices=None, default=None, none_as_disab
         if multi:
             default = default or []
             default = default if isinstance(default, list) else [default]
+            default = [c for c in defaults if c in choices]
             if not isatty():
                 result = default
             else:

{cid_cmd-4.2.3 → cid_cmd-4.2.5/cid_cmd.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cid-cmd
-Version: 4.2.3
+Version: 4.2.5
 Summary: Cloud Intelligence Dashboards deployment helper tool
 Home-page: https://github.com/aws-samples/aws-cudos-framework-deployment
 Author: AWS CUDOS Team

cid_cmd-4.2.3/cid/_version.py

@@ -1,2 +0,0 @@
-__version__ = '4.2.3'
-