cloud-governance 1.1.402__py3-none-any.whl → 1.1.404__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cloud_governance/main/environment_variables.py +6 -2
- cloud_governance/main/main.py +1 -1
- cloud_governance/policy/aws/monthly_report.py +11 -2
- cloud_governance/policy/aws/yearly_savings_report.py +490 -0
- cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py +1 -1
- {cloud_governance-1.1.402.dist-info → cloud_governance-1.1.404.dist-info}/METADATA +2 -2
- {cloud_governance-1.1.402.dist-info → cloud_governance-1.1.404.dist-info}/RECORD +10 -9
- {cloud_governance-1.1.402.dist-info → cloud_governance-1.1.404.dist-info}/WHEEL +1 -1
- {cloud_governance-1.1.402.dist-info → cloud_governance-1.1.404.dist-info}/licenses/LICENSE +0 -0
- {cloud_governance-1.1.402.dist-info → cloud_governance-1.1.404.dist-info}/top_level.txt +0 -0
|
@@ -105,7 +105,8 @@ class EnvironmentVariables:
|
|
|
105
105
|
self._environment_variables_dict['cluster_policies'] = ['zombie_cluster_resource']
|
|
106
106
|
es_index = 'cloud-governance-policy-es-index'
|
|
107
107
|
self._environment_variables_dict['cost_policies'] = ['cost_explorer', 'cost_over_usage', 'cost_billing_reports',
|
|
108
|
-
'cost_explorer_payer_billings', 'spot_savings_analysis'
|
|
108
|
+
'cost_explorer_payer_billings', 'spot_savings_analysis',
|
|
109
|
+
'yearly_savings_report']
|
|
109
110
|
self._environment_variables_dict['ibm_policies'] = ['tag_baremetal', 'tag_vm', 'ibm_cost_report',
|
|
110
111
|
'ibm_cost_over_usage']
|
|
111
112
|
self._environment_variables_dict['azure_policies'] = ['tag_azure_resource_group']
|
|
@@ -147,6 +148,9 @@ class EnvironmentVariables:
|
|
|
147
148
|
# ['User', 'Budget', 'Project', 'Manager']
|
|
148
149
|
self._environment_variables_dict['cost_explorer_tags'] = EnvironmentVariables.get_env('cost_explorer_tags',
|
|
149
150
|
'{}')
|
|
151
|
+
self._environment_variables_dict['yearly_savings_start_date'] = EnvironmentVariables.get_env('yearly_savings_start_date', '')
|
|
152
|
+
self._environment_variables_dict['yearly_savings_end_date'] = EnvironmentVariables.get_env('yearly_savings_end_date', '')
|
|
153
|
+
self._environment_variables_dict['yearly_savings_es_index'] = EnvironmentVariables.get_env('yearly_savings_es_index', 'cloud-governance-yearly-saving')
|
|
150
154
|
# AZURE Credentials
|
|
151
155
|
self._environment_variables_dict['AZURE_ACCOUNT_ID'] = EnvironmentVariables.get_env('AZURE_ACCOUNT_ID', '')
|
|
152
156
|
self._environment_variables_dict['AZURE_CLIENT_ID'] = EnvironmentVariables.get_env('AZURE_CLIENT_ID', '')
|
|
@@ -279,7 +283,7 @@ class EnvironmentVariables:
|
|
|
279
283
|
self._environment_variables_dict['POLICY_ACTIONS_DAYS'] = literal_eval(
|
|
280
284
|
EnvironmentVariables.get_env('POLICY_ACTIONS_DAYS', '[]'))
|
|
281
285
|
self._environment_variables_dict['DEFAULT_ADMINS'] = literal_eval(
|
|
282
|
-
EnvironmentVariables.get_env('DEFAULT_ADMINS', '["yinsong@redhat.com", "ebattat@redhat.com"]'))
|
|
286
|
+
EnvironmentVariables.get_env('DEFAULT_ADMINS', '["yinsong@redhat.com", "ebattat@redhat.com", "pragchau@redhat.com"]'))
|
|
283
287
|
self._environment_variables_dict['KERBEROS_USERS'] = literal_eval(
|
|
284
288
|
EnvironmentVariables.get_env('KERBEROS_USERS', '[]'))
|
|
285
289
|
self._environment_variables_dict['POLICIES_TO_ALERT'] = literal_eval(
|
cloud_governance/main/main.py
CHANGED
|
@@ -244,7 +244,7 @@ def main():
|
|
|
244
244
|
ibm_classic_infrastructure_policy_runner = IBMPolicyRunner()
|
|
245
245
|
|
|
246
246
|
is_cost_explorer_policies_runner = ''
|
|
247
|
-
if environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'AWS':
|
|
247
|
+
if environment_variables_dict.get('PUBLIC_CLOUD_NAME') and environment_variables_dict.get('PUBLIC_CLOUD_NAME').upper() == 'AWS':
|
|
248
248
|
cost_explorer_policies_runner = None
|
|
249
249
|
is_cost_explorer_policies_runner = policy in environment_variables_dict.get('cost_policies')
|
|
250
250
|
if is_cost_explorer_policies_runner:
|
|
@@ -17,8 +17,17 @@ class MonthlyReport:
|
|
|
17
17
|
self._es_index = 'cloud-governance-mail-messages'
|
|
18
18
|
self._es_host = self.__environment_variables_dict.get('es_host', '')
|
|
19
19
|
self._es_port = self.__environment_variables_dict.get('es_port', '')
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
to_mail_str = self.__environment_variables_dict.get('to_mail', '').strip()
|
|
21
|
+
if to_mail_str in ('[]', ''):
|
|
22
|
+
self._to_mail = []
|
|
23
|
+
else:
|
|
24
|
+
self._to_mail = [x.strip() for x in to_mail_str.split(',') if x.strip()]
|
|
25
|
+
|
|
26
|
+
cc_mail_str = self.__environment_variables_dict.get('cc_mail', '').strip()
|
|
27
|
+
if cc_mail_str in ('[]', ''):
|
|
28
|
+
self._to_cc = []
|
|
29
|
+
else:
|
|
30
|
+
self._to_cc = [x.strip() for x in cc_mail_str.split(',') if cc_mail_str.strip()]
|
|
22
31
|
if self._es_host:
|
|
23
32
|
self._elastic_operations = ElasticSearchOperations(es_host=self._es_host, es_port=self._es_port)
|
|
24
33
|
self._postfix_mail = Postfix()
|
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
from datetime import datetime, timezone, date, timedelta
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
from cloud_governance.common.elasticsearch.elasticsearch_operations import ElasticSearchOperations
|
|
5
|
+
from cloud_governance.common.elasticsearch.elastic_upload import ElasticUpload
|
|
6
|
+
from cloud_governance.common.logger.init_logger import logger
|
|
7
|
+
from cloud_governance.common.logger.logger_time_stamp import logger_time_stamp
|
|
8
|
+
from cloud_governance.main.environment_variables import environment_variables
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class YearlySavingsReport:
|
|
12
|
+
"""
|
|
13
|
+
This class aggregates yearly savings and per-month savings for the current year
|
|
14
|
+
from policy execution data and uploads to a dedicated ES index
|
|
15
|
+
Uses month-by-month queries and deduplication to avoid counting resources multiple times
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
NAT_GATEWAY_HOURLY_COST = 0.045
|
|
19
|
+
ELASTIC_IP_HOURLY_COST = 0.005
|
|
20
|
+
|
|
21
|
+
def __init__(self):
|
|
22
|
+
self.__environment_variables_dict = environment_variables.environment_variables_dict
|
|
23
|
+
self.__es_host = self.__environment_variables_dict.get('es_host', '')
|
|
24
|
+
self.__es_port = self.__environment_variables_dict.get('es_port', '')
|
|
25
|
+
self.__elastic_operations = ElasticSearchOperations(es_host=self.__es_host, es_port=self.__es_port) if self.__es_host else None
|
|
26
|
+
self.__elastic_upload = ElasticUpload()
|
|
27
|
+
self.__policy_es_index = self.__environment_variables_dict.get('es_index', 'cloud-governance-policy-es-index')
|
|
28
|
+
self.__yearly_savings_es_index = self.__environment_variables_dict.get('yearly_savings_es_index')
|
|
29
|
+
account = self.__environment_variables_dict.get('account', 'PERF-DEPT')
|
|
30
|
+
self.__account = account.upper().replace('OPENSHIFT-', '').replace('OPENSHIFT', '').strip()
|
|
31
|
+
# Check for custom date range from environment variables. This won't upload to ES.
|
|
32
|
+
self.__custom_start_date = self.__environment_variables_dict.get('yearly_savings_start_date', '')
|
|
33
|
+
self.__custom_end_date = self.__environment_variables_dict.get('yearly_savings_end_date', '')
|
|
34
|
+
|
|
35
|
+
def __get_last_day_of_month(self, year: int, month: int):
|
|
36
|
+
"""
|
|
37
|
+
Get the last day of a month, handling leap years
|
|
38
|
+
@param year: Year
|
|
39
|
+
@param month: Month (1-12)
|
|
40
|
+
@return: Last day of month
|
|
41
|
+
"""
|
|
42
|
+
if month == 2:
|
|
43
|
+
# Handle February (leap year)
|
|
44
|
+
if year % 4 == 0 and (year % 100 != 0 or year % 400 == 0):
|
|
45
|
+
return 29
|
|
46
|
+
else:
|
|
47
|
+
return 28
|
|
48
|
+
elif month in [4, 6, 9, 11]:
|
|
49
|
+
return 30
|
|
50
|
+
else:
|
|
51
|
+
return 31
|
|
52
|
+
|
|
53
|
+
def __get_savings_value(self, captured_date: date, policy_name: str):
|
|
54
|
+
"""
|
|
55
|
+
@param captured_date: date object
|
|
56
|
+
@param policy_name: policy name
|
|
57
|
+
@return: calculated savings value
|
|
58
|
+
"""
|
|
59
|
+
savings = 0
|
|
60
|
+
end_of_year = date(captured_date.year, 12, 31)
|
|
61
|
+
remaining_days = (end_of_year - captured_date).days
|
|
62
|
+
|
|
63
|
+
if policy_name == 'unused_nat_gateway':
|
|
64
|
+
savings = remaining_days * 24 * self.NAT_GATEWAY_HOURLY_COST
|
|
65
|
+
elif policy_name == 'ip_unattached':
|
|
66
|
+
savings = remaining_days * 24 * self.ELASTIC_IP_HOURLY_COST
|
|
67
|
+
|
|
68
|
+
return savings
|
|
69
|
+
|
|
70
|
+
def __process_monthly_query(self, month_start: str, month_end: str):
|
|
71
|
+
"""
|
|
72
|
+
Process a single monthly query and return resource-level results
|
|
73
|
+
@param month_start: Start date string (YYYY-MM-DD)
|
|
74
|
+
@param month_end: End date string (YYYY-MM-DD)
|
|
75
|
+
@return: Dictionary of {resource_id: {captured_date, policy_name, savings}}
|
|
76
|
+
"""
|
|
77
|
+
if not self.__elastic_operations:
|
|
78
|
+
return {}
|
|
79
|
+
|
|
80
|
+
query = {
|
|
81
|
+
"size": 0,
|
|
82
|
+
"query": {
|
|
83
|
+
"bool": {
|
|
84
|
+
"must": [
|
|
85
|
+
{
|
|
86
|
+
"term": {
|
|
87
|
+
"PublicCloud.keyword": {
|
|
88
|
+
"value": "AWS"
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"term": {
|
|
94
|
+
"account.keyword": {
|
|
95
|
+
"value": self.__account
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
],
|
|
100
|
+
"must_not": [
|
|
101
|
+
{
|
|
102
|
+
"terms": {
|
|
103
|
+
"policy.keyword": [
|
|
104
|
+
"zombie_cluster_resource", "instance_run", "ebs_in_use",
|
|
105
|
+
"s3_inactive", "optimize_resources_report", "instance_idle",
|
|
106
|
+
"cluster_run", "skipped_resources", "ec2_idle", "empty_roles",
|
|
107
|
+
"unused_access_key"
|
|
108
|
+
]
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
],
|
|
112
|
+
"filter": [{
|
|
113
|
+
"range": {
|
|
114
|
+
"timestamp": {
|
|
115
|
+
"gte": month_start,
|
|
116
|
+
"lte": month_end,
|
|
117
|
+
"format": "yyyy-MM-dd"
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}]
|
|
121
|
+
}
|
|
122
|
+
},
|
|
123
|
+
"aggs": {
|
|
124
|
+
"PolicyName": {
|
|
125
|
+
"terms": {
|
|
126
|
+
"field": "policy.keyword",
|
|
127
|
+
"size": 20,
|
|
128
|
+
"order": {"_key": "desc"}
|
|
129
|
+
},
|
|
130
|
+
"aggs": {
|
|
131
|
+
"CapturedDate": {
|
|
132
|
+
"terms": {
|
|
133
|
+
"field": "timestamp",
|
|
134
|
+
"size": 10000
|
|
135
|
+
},
|
|
136
|
+
"aggs": {
|
|
137
|
+
"ResourceId": {
|
|
138
|
+
"terms": {
|
|
139
|
+
"field": "ResourceId.keyword",
|
|
140
|
+
"size": 10000,
|
|
141
|
+
"order": {"_key": "desc"}
|
|
142
|
+
},
|
|
143
|
+
"aggs": {
|
|
144
|
+
"Savings": {
|
|
145
|
+
"max": {
|
|
146
|
+
"field": "TotalYearlySavings",
|
|
147
|
+
"missing": 0
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
try:
|
|
160
|
+
response = self.__elastic_operations.post_query(
|
|
161
|
+
query=query,
|
|
162
|
+
es_index=self.__policy_es_index,
|
|
163
|
+
result_agg=True
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
if not response or 'PolicyName' not in response:
|
|
167
|
+
logger.warning(f'No response or unexpected structure for {month_start} to {month_end}')
|
|
168
|
+
return {}
|
|
169
|
+
|
|
170
|
+
monthly_resources = {}
|
|
171
|
+
policy_buckets = response.get('PolicyName', {}).get('buckets', [])
|
|
172
|
+
|
|
173
|
+
logger.debug(f'DEBUG: Found {len(policy_buckets)} policies in query results')
|
|
174
|
+
for policy in policy_buckets:
|
|
175
|
+
logger.debug(f'DEBUG: Policy found: {policy.get("key")} with {len(policy.get("CapturedDate", {}).get("buckets", []))} captured dates')
|
|
176
|
+
|
|
177
|
+
for policy in policy_buckets:
|
|
178
|
+
policy_name = policy.get('key')
|
|
179
|
+
|
|
180
|
+
for es_capture_date_values in policy.get('CapturedDate', {}).get('buckets', []):
|
|
181
|
+
captured_date_str = es_capture_date_values.get('key_as_string', '')
|
|
182
|
+
|
|
183
|
+
if captured_date_str:
|
|
184
|
+
captured_date = datetime.strptime(captured_date_str[:10], "%Y-%m-%d").date()
|
|
185
|
+
else:
|
|
186
|
+
captured_date = datetime.now(timezone.utc).date()
|
|
187
|
+
|
|
188
|
+
for resource in es_capture_date_values.get('ResourceId', {}).get('buckets', []):
|
|
189
|
+
resource_id = resource.get('key')
|
|
190
|
+
savings = resource.get('Savings', {}).get('value', 0)
|
|
191
|
+
|
|
192
|
+
if resource_id in monthly_resources:
|
|
193
|
+
if monthly_resources[resource_id]['captured_date'] > captured_date:
|
|
194
|
+
monthly_resources[resource_id]['captured_date'] = captured_date
|
|
195
|
+
if savings == 0:
|
|
196
|
+
savings = self.__get_savings_value(captured_date, policy_name)
|
|
197
|
+
if monthly_resources[resource_id]['savings'] > savings:
|
|
198
|
+
monthly_resources[resource_id]['savings'] = savings
|
|
199
|
+
else:
|
|
200
|
+
if savings == 0:
|
|
201
|
+
savings = self.__get_savings_value(captured_date, policy_name)
|
|
202
|
+
monthly_resources[resource_id] = {
|
|
203
|
+
'captured_date': captured_date,
|
|
204
|
+
'policy_name': policy_name,
|
|
205
|
+
'savings': savings
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
policy_summary = {}
|
|
209
|
+
for resource_id, resource_data in monthly_resources.items():
|
|
210
|
+
policy_name = resource_data.get('policy_name')
|
|
211
|
+
savings = resource_data.get('savings', 0)
|
|
212
|
+
policy_summary[policy_name] = policy_summary.get(policy_name, 0) + savings
|
|
213
|
+
|
|
214
|
+
logger.info(f'Monthly resources summary by policy: {policy_summary}')
|
|
215
|
+
logger.info(f"Found {len(monthly_resources)} resources for {month_start} to {month_end}")
|
|
216
|
+
return monthly_resources
|
|
217
|
+
|
|
218
|
+
except Exception as err:
|
|
219
|
+
logger.error(f'Error processing month {month_start} to {month_end}: {err}')
|
|
220
|
+
return {}
|
|
221
|
+
|
|
222
|
+
def __calculate_month_savings(self, year: int, month: int, start_day: int, end_day: int):
|
|
223
|
+
"""
|
|
224
|
+
Calculate savings for a specific month
|
|
225
|
+
@param year: Year
|
|
226
|
+
@param month: Month (1-12)
|
|
227
|
+
@param start_day: Start day of month
|
|
228
|
+
@param end_day: End day of month
|
|
229
|
+
@return: Dictionary of {policy_name: savings_value}
|
|
230
|
+
"""
|
|
231
|
+
month_start = date(year, month, start_day).strftime("%Y-%m-%d")
|
|
232
|
+
month_end = date(year, month, end_day).strftime("%Y-%m-%d")
|
|
233
|
+
|
|
234
|
+
logger.info(f'Calculating savings for {month_start} to {month_end}')
|
|
235
|
+
|
|
236
|
+
return self.__get_yearly_savings(start_date=month_start, end_date=month_end)
|
|
237
|
+
|
|
238
|
+
def __get_yearly_savings(self, start_date: str = None, end_date: str = None):
|
|
239
|
+
"""
|
|
240
|
+
This method returns the yearly savings of the policies.
|
|
241
|
+
Queries month-by-month to avoid Elasticsearch overload.
|
|
242
|
+
@param start_date: Start date string (YYYY-MM-DD). If None, defaults to Jan 1 of current year.
|
|
243
|
+
@param end_date: End date string (YYYY-MM-DD). If None, defaults to Dec 31 of current year.
|
|
244
|
+
@return: Dictionary of {policy_name: total_savings}
|
|
245
|
+
"""
|
|
246
|
+
if not start_date:
|
|
247
|
+
start_date = datetime.now(timezone.utc).date().replace(day=1, month=1).strftime("%Y-%m-%d")
|
|
248
|
+
if not end_date:
|
|
249
|
+
end_date = datetime.now(timezone.utc).date().replace(day=31, month=12).strftime("%Y-%m-%d")
|
|
250
|
+
|
|
251
|
+
logger.info(f"Getting yearly savings from {start_date} to {end_date}")
|
|
252
|
+
|
|
253
|
+
monthly_ranges = self.__split_date_range_by_month(start_date, end_date)
|
|
254
|
+
logger.debug(f"Split into {len(monthly_ranges)} monthly queries")
|
|
255
|
+
|
|
256
|
+
all_resources = {}
|
|
257
|
+
|
|
258
|
+
for i, (month_start, month_end) in enumerate(monthly_ranges, 1):
|
|
259
|
+
logger.info(f"Processing month {i}/{len(monthly_ranges)}: {month_start} to {month_end}")
|
|
260
|
+
monthly_resources = self.__process_monthly_query(month_start, month_end)
|
|
261
|
+
|
|
262
|
+
for resource_id, resource_data in monthly_resources.items():
|
|
263
|
+
if resource_id in all_resources:
|
|
264
|
+
if all_resources[resource_id]['captured_date'] > resource_data['captured_date']:
|
|
265
|
+
all_resources[resource_id]['captured_date'] = resource_data['captured_date']
|
|
266
|
+
if all_resources[resource_id]['savings'] > resource_data['savings']:
|
|
267
|
+
all_resources[resource_id]['savings'] = resource_data['savings']
|
|
268
|
+
else:
|
|
269
|
+
all_resources[resource_id] = resource_data
|
|
270
|
+
|
|
271
|
+
if i < len(monthly_ranges):
|
|
272
|
+
time.sleep(0.5)
|
|
273
|
+
|
|
274
|
+
result = self.__get_total_policy_sum(all_resources)
|
|
275
|
+
return result
|
|
276
|
+
|
|
277
|
+
def __split_date_range_by_month(self, start_date: str, end_date: str):
|
|
278
|
+
"""
|
|
279
|
+
Split date range into monthly chunks
|
|
280
|
+
@param start_date: Start date string (YYYY-MM-DD)
|
|
281
|
+
@param end_date: End date string (YYYY-MM-DD)
|
|
282
|
+
@return: List of (month_start, month_end) tuples as strings
|
|
283
|
+
"""
|
|
284
|
+
if not start_date or not end_date:
|
|
285
|
+
raise ValueError(f"Both start_date and end_date must be provided. Got: start_date={start_date}, end_date={end_date}")
|
|
286
|
+
|
|
287
|
+
start = datetime.strptime(start_date, "%Y-%m-%d").date()
|
|
288
|
+
end = datetime.strptime(end_date, "%Y-%m-%d").date()
|
|
289
|
+
|
|
290
|
+
if start > end:
|
|
291
|
+
raise ValueError(f"start_date ({start_date}) must be <= end_date ({end_date})")
|
|
292
|
+
|
|
293
|
+
monthly_ranges = []
|
|
294
|
+
current_start = start
|
|
295
|
+
|
|
296
|
+
while current_start <= end:
|
|
297
|
+
if current_start.month == 12:
|
|
298
|
+
current_end = date(current_start.year + 1, 1, 1) - timedelta(days=1)
|
|
299
|
+
else:
|
|
300
|
+
current_end = date(current_start.year, current_start.month + 1, 1) - timedelta(days=1)
|
|
301
|
+
if current_end > end:
|
|
302
|
+
current_end = end
|
|
303
|
+
monthly_ranges.append((
|
|
304
|
+
current_start.strftime("%Y-%m-%d"),
|
|
305
|
+
current_end.strftime("%Y-%m-%d")
|
|
306
|
+
))
|
|
307
|
+
if current_end.month == 12:
|
|
308
|
+
current_start = date(current_end.year + 1, 1, 1)
|
|
309
|
+
else:
|
|
310
|
+
current_start = date(current_end.year, current_end.month + 1, 1)
|
|
311
|
+
|
|
312
|
+
return monthly_ranges
|
|
313
|
+
|
|
314
|
+
def __get_total_policy_sum(self, all_resources: dict):
|
|
315
|
+
"""
|
|
316
|
+
Calculate total savings by policy
|
|
317
|
+
@param all_resources: dict of {resource_id: {savings, policy_name, captured_date}}
|
|
318
|
+
@return: dict of {policy_name: total_savings}
|
|
319
|
+
"""
|
|
320
|
+
savings_result = {}
|
|
321
|
+
|
|
322
|
+
for resource_id, values in all_resources.items():
|
|
323
|
+
policy_name = values.get('policy_name')
|
|
324
|
+
savings = values.get('savings', 0)
|
|
325
|
+
savings_result[policy_name] = round(savings_result.get(policy_name, 0) + savings, 3)
|
|
326
|
+
|
|
327
|
+
return savings_result
|
|
328
|
+
|
|
329
|
+
def __update_yearly_savings(self, year: int, all_months_data: dict, total_annual_saving: float):
|
|
330
|
+
"""
|
|
331
|
+
Update yearly savings in Elasticsearch with all months data.
|
|
332
|
+
Creates a new document if year doesn't exist, or updates existing one.
|
|
333
|
+
@param year: Year (e.g., 2026)
|
|
334
|
+
@param all_months_data: Dictionary of {month_number: savings_value} for all months
|
|
335
|
+
@param total_annual_saving: Total cumulative savings for the year
|
|
336
|
+
@return: True if successful
|
|
337
|
+
"""
|
|
338
|
+
try:
|
|
339
|
+
current_date = datetime.now(timezone.utc).date()
|
|
340
|
+
year_id = f"{year}-{self.__account}"
|
|
341
|
+
|
|
342
|
+
data = {
|
|
343
|
+
'year': year,
|
|
344
|
+
'total_saving': round(total_annual_saving, 3),
|
|
345
|
+
'last_updated': current_date.isoformat(),
|
|
346
|
+
'timestamp': datetime.now(timezone.utc),
|
|
347
|
+
'policy': 'yearly_savings_report',
|
|
348
|
+
'index_id': year_id,
|
|
349
|
+
'account': self.__account
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
for month_num in range(1, 13):
|
|
353
|
+
data[f'month_{month_num}'] = round(all_months_data.get(month_num, 0), 3)
|
|
354
|
+
|
|
355
|
+
try:
|
|
356
|
+
if self.__elastic_operations.verify_elastic_index_doc_id(index=self.__yearly_savings_es_index, doc_id=year_id):
|
|
357
|
+
self.__elastic_operations.update_elasticsearch_index(
|
|
358
|
+
index=self.__yearly_savings_es_index,
|
|
359
|
+
id=year_id,
|
|
360
|
+
metadata=data
|
|
361
|
+
)
|
|
362
|
+
logger.info(f"Updated yearly savings for year {year}")
|
|
363
|
+
else:
|
|
364
|
+
self.__elastic_operations.upload_to_elasticsearch(
|
|
365
|
+
index=self.__yearly_savings_es_index,
|
|
366
|
+
data=data,
|
|
367
|
+
id=year_id
|
|
368
|
+
)
|
|
369
|
+
logger.info(f"Created yearly savings document for year {year}")
|
|
370
|
+
except Exception as e:
|
|
371
|
+
logger.warning(f"Update check failed, trying create: {e}")
|
|
372
|
+
try:
|
|
373
|
+
self.__elastic_operations.upload_to_elasticsearch(
|
|
374
|
+
index=self.__yearly_savings_es_index,
|
|
375
|
+
data=data,
|
|
376
|
+
id=year_id
|
|
377
|
+
)
|
|
378
|
+
logger.info(f"Created yearly savings document for year {year}")
|
|
379
|
+
except Exception as create_err:
|
|
380
|
+
logger.error(f"Failed to create document: {create_err}")
|
|
381
|
+
raise create_err
|
|
382
|
+
|
|
383
|
+
return True
|
|
384
|
+
|
|
385
|
+
except Exception as err:
|
|
386
|
+
logger.error(f"Error updating yearly savings: {err}")
|
|
387
|
+
raise err
|
|
388
|
+
|
|
389
|
+
@logger_time_stamp
|
|
390
|
+
def run(self, start_date: str = None, end_date: str = None):
|
|
391
|
+
"""
|
|
392
|
+
Main method to run the yearly savings report
|
|
393
|
+
|
|
394
|
+
@param start_date: Optional start date string (YYYY-MM-DD). If None, checks environment variable, then defaults to Jan 1 of current year.
|
|
395
|
+
@param end_date: Optional end date string (YYYY-MM-DD). If None, checks environment variable, then defaults to today.
|
|
396
|
+
@return: dict with summary
|
|
397
|
+
"""
|
|
398
|
+
if not start_date:
|
|
399
|
+
start_date = self.__custom_start_date
|
|
400
|
+
if not end_date:
|
|
401
|
+
end_date = self.__custom_end_date
|
|
402
|
+
|
|
403
|
+
if start_date and end_date:
|
|
404
|
+
try:
|
|
405
|
+
logger.info(f'Using custom date range: {start_date} to {end_date}')
|
|
406
|
+
|
|
407
|
+
month_savings = self.__get_yearly_savings(start_date=start_date, end_date=end_date)
|
|
408
|
+
total_savings = sum(month_savings.values()) if month_savings else 0.0
|
|
409
|
+
|
|
410
|
+
logger.info(f'Custom date range - Policy savings: {month_savings}')
|
|
411
|
+
logger.info(f'Custom date range - Total savings: ${total_savings:,.2f}')
|
|
412
|
+
|
|
413
|
+
return {
|
|
414
|
+
'status': 'success',
|
|
415
|
+
'custom_date_range': True,
|
|
416
|
+
'start_date': start_date,
|
|
417
|
+
'end_date': end_date,
|
|
418
|
+
'policy_savings': month_savings,
|
|
419
|
+
'total_yearly_savings': total_savings
|
|
420
|
+
}
|
|
421
|
+
except ValueError as e:
|
|
422
|
+
logger.error(f'Invalid date format: {e}. Expected YYYY-MM-DD')
|
|
423
|
+
return {'status': 'error', 'message': f'Invalid date format: {e}'}
|
|
424
|
+
|
|
425
|
+
current_date = datetime.now(timezone.utc)
|
|
426
|
+
current_year = current_date.year
|
|
427
|
+
current_month = current_date.month
|
|
428
|
+
today = current_date.date()
|
|
429
|
+
|
|
430
|
+
logger.info(f'Using default date range: current year {current_year}')
|
|
431
|
+
|
|
432
|
+
all_months_data = {}
|
|
433
|
+
|
|
434
|
+
for month in range(1, 13):
|
|
435
|
+
try:
|
|
436
|
+
if month < current_month:
|
|
437
|
+
end_day = self.__get_last_day_of_month(current_year, month)
|
|
438
|
+
month_savings = self.__calculate_month_savings(
|
|
439
|
+
year=current_year,
|
|
440
|
+
month=month,
|
|
441
|
+
start_day=1,
|
|
442
|
+
end_day=end_day
|
|
443
|
+
)
|
|
444
|
+
all_months_data[month] = sum(month_savings.values()) if month_savings else 0.0
|
|
445
|
+
|
|
446
|
+
elif month == current_month:
|
|
447
|
+
month_savings = self.__calculate_month_savings(
|
|
448
|
+
year=current_year,
|
|
449
|
+
month=month,
|
|
450
|
+
start_day=1,
|
|
451
|
+
end_day=today.day
|
|
452
|
+
)
|
|
453
|
+
all_months_data[month] = sum(month_savings.values()) if month_savings else 0.0
|
|
454
|
+
|
|
455
|
+
else:
|
|
456
|
+
all_months_data[month] = 0.0
|
|
457
|
+
|
|
458
|
+
if month < 12:
|
|
459
|
+
time.sleep(0.2)
|
|
460
|
+
|
|
461
|
+
except Exception as e:
|
|
462
|
+
logger.warning(f"Error calculating savings for month {month}: {e}, setting to 0")
|
|
463
|
+
all_months_data[month] = 0.0
|
|
464
|
+
|
|
465
|
+
total_annual_saving = sum(all_months_data.values())
|
|
466
|
+
|
|
467
|
+
logger.info(f'Monthly savings: {all_months_data}')
|
|
468
|
+
logger.info(f'Total annual savings: ${total_annual_saving:,.2f}')
|
|
469
|
+
|
|
470
|
+
if self.__elastic_operations:
|
|
471
|
+
try:
|
|
472
|
+
self.__update_yearly_savings(
|
|
473
|
+
year=current_year,
|
|
474
|
+
all_months_data=all_months_data,
|
|
475
|
+
total_annual_saving=total_annual_saving
|
|
476
|
+
)
|
|
477
|
+
logger.info(f'Successfully uploaded yearly savings to {self.__yearly_savings_es_index}')
|
|
478
|
+
return {
|
|
479
|
+
'status': 'success',
|
|
480
|
+
'records_uploaded': 1,
|
|
481
|
+
'year': current_year,
|
|
482
|
+
'total_yearly_savings': total_annual_saving,
|
|
483
|
+
'monthly_savings': all_months_data
|
|
484
|
+
}
|
|
485
|
+
except Exception as err:
|
|
486
|
+
logger.error(f'Error uploading to ES: {err}')
|
|
487
|
+
return {'status': 'error', 'message': str(err)}
|
|
488
|
+
else:
|
|
489
|
+
logger.warning('ES not configured')
|
|
490
|
+
return {'status': 'no_upload', 'message': 'ES not configured'}
|
cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py
CHANGED
|
@@ -54,7 +54,7 @@ class NonClusterZombiePolicy:
|
|
|
54
54
|
self.__email_alert = self.__environment_variables_dict.get(
|
|
55
55
|
'EMAIL_ALERT') if self.__environment_variables_dict.get('EMAIL_ALERT') else False
|
|
56
56
|
self.__manager_email_alert = self.__environment_variables_dict.get('MANAGER_EMAIL_ALERT')
|
|
57
|
-
self._admins = ['yinsong@redhat.com', 'ebattat@redhat.com']
|
|
57
|
+
self._admins = ['yinsong@redhat.com', 'ebattat@redhat.com', 'pragchau@redhat.com']
|
|
58
58
|
self._es_upload = ElasticUpload()
|
|
59
59
|
self.resource_pricing = ResourcesPricing()
|
|
60
60
|
self._es_operations = ElasticSearchOperations()
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cloud-governance
|
|
3
|
-
Version: 1.1.
|
|
3
|
+
Version: 1.1.404
|
|
4
4
|
Summary: Cloud Governance Tool
|
|
5
5
|
Home-page: https://github.com/redhat-performance/cloud-governance
|
|
6
6
|
Author: Red Hat
|
|
7
|
-
Author-email: ebattat@redhat.com,
|
|
7
|
+
Author-email: ebattat@redhat.com, pragchau@redhat.com
|
|
8
8
|
License: Apache License 2.0
|
|
9
9
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
10
10
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -143,10 +143,10 @@ cloud_governance/common/utils/configs.py,sha256=shFxWt0Kc-GwzcZKYCkHm058ujwdTxn4
|
|
|
143
143
|
cloud_governance/common/utils/json_datetime_encoder.py,sha256=_-jzRTe0UqAKTn2E9qaU8SYIxHUoRA5ElWuVA0Y54Xw,338
|
|
144
144
|
cloud_governance/common/utils/utils.py,sha256=ZUsi4ax2XhDIV-EQ5kJt5Ppd72kmm2psqcg1cNDZrvc,4349
|
|
145
145
|
cloud_governance/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
146
|
-
cloud_governance/main/environment_variables.py,sha256=
|
|
146
|
+
cloud_governance/main/environment_variables.py,sha256=2EkVW6rHosaIiJ1S1sU4XQVoTgvWpGYJDQ8ACkAqxVA,30325
|
|
147
147
|
cloud_governance/main/environment_variables_exceptions.py,sha256=UR0Ith0P0oshsDZdJRlRq8ZUTt0h8jFvUtrnP4m4AIY,437
|
|
148
148
|
cloud_governance/main/es_uploader.py,sha256=6Ify5CS2NtUF1xXZ-rMwpYxVzDKfEZhv2vogWFltt98,10656
|
|
149
|
-
cloud_governance/main/main.py,sha256=
|
|
149
|
+
cloud_governance/main/main.py,sha256=m1tMOZbWPqwAM6-BPxeLpF2EAzrmDS3nTTsHBwfikMQ,18998
|
|
150
150
|
cloud_governance/main/main_common_operations.py,sha256=YbBJF6Smk3YKhEibnn-fIWu1oKP0pSGIM1WnSXFcBuo,366
|
|
151
151
|
cloud_governance/main/run_cloud_resource_orchestration.py,sha256=Jo7-KDqxrIo8uioyTFCohUHse4uqdEt2ZFnHlX2u57g,776
|
|
152
152
|
cloud_governance/main/main_oerations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -161,12 +161,13 @@ cloud_governance/policy/aws/ebs_in_use.py,sha256=7vGV2qobV14rzC7tJnJiafC3oAhnY_T
|
|
|
161
161
|
cloud_governance/policy/aws/ec2_stop.py,sha256=FXQNkHmiMCEw6Pz6CalIIVYFRsV24RT5TewqFazyP8M,9641
|
|
162
162
|
cloud_governance/policy/aws/empty_roles.py,sha256=gEBaribxBnM_seP8rcifaVdf-Gz-JT2sc1aNKl8AcxA,4804
|
|
163
163
|
cloud_governance/policy/aws/ip_unattached.py,sha256=OnHlTIKSgyxwdAgU9jBApdvDQpzHgECjSodN6KoXWqU,3176
|
|
164
|
-
cloud_governance/policy/aws/monthly_report.py,sha256=
|
|
164
|
+
cloud_governance/policy/aws/monthly_report.py,sha256=e0A0fD2sYWVb0Z_HHTJZzQ9qd957d_4PTvQuW4FR17A,7155
|
|
165
165
|
cloud_governance/policy/aws/optimize_resources_report.py,sha256=zG7w8KHF7Z25jxYGgDadyXp0jcxSREsRCOmQP8lZMNc,6003
|
|
166
166
|
cloud_governance/policy/aws/s3_inactive.py,sha256=NXUUGtJhpqohaWYczSebw-q0Q7RKg3XMMj_h7xfGhQ0,2839
|
|
167
167
|
cloud_governance/policy/aws/skipped_resources.py,sha256=D0kbt9dg6Bkl5PgGaqimWvLct6D-JOnerzJ0FkWzGFc,5679
|
|
168
168
|
cloud_governance/policy/aws/spot_savings_analysis.py,sha256=lGG5qtz8pr7xjLo5BrtVSHGTz928MLwYPbSCaDfpTes,5513
|
|
169
169
|
cloud_governance/policy/aws/unused_access_key.py,sha256=T4XwZT9b4IjTuV9JgCGvBPgMHbyu9tr-W9XI7d-emSE,3381
|
|
170
|
+
cloud_governance/policy/aws/yearly_savings_report.py,sha256=qGNqT6L5T2WnZpwp5NlopckHxeDrF19YB6isayb16I4,21758
|
|
170
171
|
cloud_governance/policy/aws/zombie_cluster_resource.py,sha256=Qkd5_Sh74cixaL7yiQgU1OvoEIfGmeKSECb5tNAnQ7I,59675
|
|
171
172
|
cloud_governance/policy/aws/zombie_snapshots.py,sha256=V48cq4GCG2z-MRwUSE4b5wQcGeI_T1Ah99SiHlTwkvY,3834
|
|
172
173
|
cloud_governance/policy/aws/cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -237,7 +238,7 @@ cloud_governance/policy/policy_operations/aws/zombie_cluster/run_zombie_cluster_
|
|
|
237
238
|
cloud_governance/policy/policy_operations/aws/zombie_cluster/validate_zombies.py,sha256=lOCmOTfrF6M1QZl6to7Y1A13Cvf-taNxBOvvhFswOTo,1164
|
|
238
239
|
cloud_governance/policy/policy_operations/aws/zombie_cluster/zombie_cluster_common_methods.py,sha256=VqXk8A0OLZOxBm422kb-n5zfM8DY4FYdkwZO91xwRxQ,14791
|
|
239
240
|
cloud_governance/policy/policy_operations/aws/zombie_non_cluster/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
240
|
-
cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py,sha256=
|
|
241
|
+
cloud_governance/policy/policy_operations/aws/zombie_non_cluster/run_zombie_non_cluster_policies.py,sha256=1HXoteAj0c2nb-ldQPJd6pZUvagKnS7WooWcaiRr1wg,21068
|
|
241
242
|
cloud_governance/policy/policy_operations/aws/zombie_non_cluster/zombie_non_cluster_polices.py,sha256=LbvOgW9JEvXoE0bpsGoqGdZd9ZmP8PCFmfzDD-S_528,2949
|
|
242
243
|
cloud_governance/policy/policy_operations/azure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
243
244
|
cloud_governance/policy/policy_operations/azure/azure_policy_runner.py,sha256=sHvdVuZCY-FlIwe843aBtx99aC6gGNXM7r6s7Uv3xWk,1129
|
|
@@ -265,8 +266,8 @@ cloud_governance/policy/policy_runners/elasticsearch/__init__.py,sha256=47DEQpj8
|
|
|
265
266
|
cloud_governance/policy/policy_runners/elasticsearch/upload_elastic_search.py,sha256=T_A24SeE9oLhBg0O2k1SHYTUAQr8G-_AaeWFibUOi7c,1880
|
|
266
267
|
cloud_governance/policy/policy_runners/ibm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
267
268
|
cloud_governance/policy/policy_runners/ibm/policy_runner.py,sha256=V0E_f7F3hXit0aSq4BlfX1Jd4vjR2NEvOWsJ5upvZ4o,1302
|
|
268
|
-
cloud_governance-1.1.
|
|
269
|
-
cloud_governance-1.1.
|
|
270
|
-
cloud_governance-1.1.
|
|
271
|
-
cloud_governance-1.1.
|
|
272
|
-
cloud_governance-1.1.
|
|
269
|
+
cloud_governance-1.1.404.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
270
|
+
cloud_governance-1.1.404.dist-info/METADATA,sha256=CB4zsZN5ipmaZlyEzv8HEK5djApQgl5tjF1B42HyYd8,11385
|
|
271
|
+
cloud_governance-1.1.404.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
272
|
+
cloud_governance-1.1.404.dist-info/top_level.txt,sha256=jfB1fgj7jvx3YZkZA4G6hFeS1RHO7J7XtnbjuMNMRww,17
|
|
273
|
+
cloud_governance-1.1.404.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|