brynq-sdk-task-scheduler 3.0.10__tar.gz → 4.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/PKG-INFO +1 -1
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/brynq_sdk_task_scheduler/task_scheduler.py +120 -260
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/brynq_sdk_task_scheduler.egg-info/PKG-INFO +1 -1
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/brynq_sdk_task_scheduler.egg-info/requires.txt +2 -2
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/setup.py +3 -3
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/brynq_sdk_task_scheduler/__init__.py +0 -0
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/brynq_sdk_task_scheduler.egg-info/SOURCES.txt +0 -0
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/brynq_sdk_task_scheduler.egg-info/dependency_links.txt +0 -0
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/brynq_sdk_task_scheduler.egg-info/not-zip-safe +0 -0
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/brynq_sdk_task_scheduler.egg-info/top_level.txt +0 -0
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.1}/setup.cfg +0 -0
|
@@ -16,104 +16,73 @@ from brynq_sdk_elastic import Elastic
|
|
|
16
16
|
from brynq_sdk_brynq import BrynQ
|
|
17
17
|
import warnings
|
|
18
18
|
import re
|
|
19
|
-
LOGGING_OPTIONS = Literal['MYSQL', 'ELASTIC']
|
|
20
19
|
|
|
21
20
|
|
|
22
|
-
class TaskScheduler
|
|
21
|
+
class TaskScheduler:
|
|
23
22
|
|
|
24
|
-
def __init__(self,
|
|
23
|
+
def __init__(self, data_interface_id: int = None, loglevel: str = 'INFO', email_after_errors: bool = False):
|
|
25
24
|
"""
|
|
26
25
|
The TaskScheduler is responsible for the logging to the database. Based on this logging, the next reload will
|
|
27
26
|
start or not and warning will be given or not
|
|
28
|
-
:param
|
|
27
|
+
:param data_interface_id: The ID from the task as saved in the task_scheduler table in the customer database
|
|
29
28
|
:param email_after_errors: a True or False value. When True, there will be send an email to a contactperson of the customer (as given in the database) with the number of errors
|
|
30
29
|
:param loglevel: Chose on which level you want to store the logs. Default is INFO. that means that a logline
|
|
31
30
|
:param disable_logging: If the interface is started from a local instance, logs will not be stored by default. If this is set to True, the logs will be stored in the database
|
|
32
31
|
with level DEBUG not is stored
|
|
33
32
|
"""
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
self.
|
|
38
|
-
|
|
39
|
-
|
|
33
|
+
# If the task is started via the task_scheduler, the following parameters will be passed by the scheduler.
|
|
34
|
+
# The distinction between local and non local is made because the scheduler usually sets the scheduler_log table entry and run_id. When running locally, the tasks will not log anything by default.
|
|
35
|
+
if any(flag in sys.argv[0].split('/') for flag in ['opt', 'home']):
|
|
36
|
+
self.started_local = True
|
|
37
|
+
self.run_id = int(round(time.time() * 100000))
|
|
38
|
+
self.task_id = None
|
|
39
|
+
self.data_interface_id = data_interface_id
|
|
40
|
+
# set it into environment variables for reuse in source and target systems
|
|
41
|
+
os.environ['DATA_INTERFACE_ID'] = str(self.data_interface_id)
|
|
40
42
|
self.mysql_enabled = False
|
|
41
43
|
self.elastic_enabled = False
|
|
42
|
-
|
|
43
|
-
# Initialize MySQL
|
|
44
|
-
self.mysql_unreachable = False
|
|
45
|
-
# if self.mysql_enabled:
|
|
46
|
-
try:
|
|
47
|
-
self.mysql = MySQL()
|
|
48
|
-
self.mysql.ping()
|
|
49
|
-
except Exception as e:
|
|
50
|
-
self.mysql_unreachable = True
|
|
51
|
-
self.mysql = None
|
|
52
|
-
print("MySQL is enabled but not reachable, logs will be saved locally if needed.")
|
|
53
|
-
# else:
|
|
54
|
-
# self.mysql = None
|
|
55
|
-
|
|
56
|
-
# Initialize ElasticSearch
|
|
57
|
-
self.elastic_unreachable = False
|
|
58
|
-
if self.elastic_enabled:
|
|
59
|
-
try:
|
|
60
|
-
self.es = Elastic()
|
|
61
|
-
self.es.get_health()
|
|
62
|
-
except Exception as e:
|
|
63
|
-
self.elastic_unreachable = True
|
|
64
|
-
self.es = Elastic(disabled=True)
|
|
65
|
-
print("ElasticSearch is enabled but not reachable, logs will be saved locally if needed.")
|
|
44
|
+
print("You're running locally, so your task won't be started in the platform and logs won't be saved")
|
|
66
45
|
else:
|
|
67
|
-
self.
|
|
68
|
-
|
|
46
|
+
self.started_local = False
|
|
47
|
+
self.customer_db, self.task_id, self.run_id, self.data_interface_id = sys.argv[1:5]
|
|
48
|
+
self.mysql_enabled = True
|
|
49
|
+
self.elastic_enabled = True
|
|
50
|
+
# If the task is started locally, the parameters should be set locally
|
|
51
|
+
|
|
52
|
+
print(f"Run ID: {self.run_id}, Data Interface ID: {self.data_interface_id}, Task ID: {self.task_id}, Started Locally: {self.started_local}")
|
|
53
|
+
self.brynq = BrynQ()
|
|
54
|
+
self.email_after_errors = email_after_errors
|
|
69
55
|
# Set up local log directory
|
|
70
56
|
self.local_log_dir = 'local_logs'
|
|
71
57
|
os.makedirs(self.local_log_dir, exist_ok=True)
|
|
72
58
|
|
|
73
|
-
# Process local logs if services are now reachable
|
|
74
|
-
if self.mysql_enabled and not self.mysql_unreachable:
|
|
75
|
-
self._process_local_mysql_logs()
|
|
76
|
-
if self.elastic_enabled and not self.elastic_unreachable:
|
|
77
|
-
self._process_local_elastic_logs()
|
|
78
|
-
|
|
79
59
|
try:
|
|
80
|
-
self.email_after_errors = email_after_errors
|
|
81
60
|
self.customer_db = os.getenv("MYSQL_DATABASE")
|
|
82
61
|
self.customer = os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_')
|
|
83
62
|
self.partner_id = os.getenv('PARTNER_ID').lower().replace(' ', '_') if os.getenv('PARTNER_ID') else 'brynq'
|
|
84
63
|
self.loglevel = loglevel
|
|
85
64
|
self.started_at = datetime.datetime.now()
|
|
86
|
-
# If the task is started via the task_scheduler, the following 3 parameters will be passed by the scheduler.
|
|
87
|
-
# The distinction between local and non local is made because the scheduler usually sets the scheduler_log table entry and run_id. When running locally, the tasks should do this itself.
|
|
88
|
-
if len(sys.argv[1:4]) > 0:
|
|
89
|
-
self.started_local = False
|
|
90
|
-
self.customer_db, self.task_id, self.run_id = sys.argv[1:4]
|
|
91
|
-
# If the task is started locally, the parameters should be set locally
|
|
92
|
-
else:
|
|
93
|
-
self.started_local = True
|
|
94
|
-
self.run_id = int(round(time.time() * 100000))
|
|
95
|
-
self.task_id = task_id
|
|
96
|
-
print(self.task_id, self.run_id)
|
|
97
65
|
self.error_count = 0
|
|
98
66
|
|
|
99
67
|
# Check if the log tables exists in the customer database1. If not, create them
|
|
100
68
|
# Mysql throws a warning when a table already exists. We don't care so we ignore warnings. (not exceptions!)
|
|
101
69
|
warnings.filterwarnings('ignore')
|
|
102
70
|
|
|
103
|
-
#
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
self.
|
|
107
|
-
self.
|
|
71
|
+
# Initialize MySQL
|
|
72
|
+
self.mysql_reachable = True
|
|
73
|
+
try:
|
|
74
|
+
self.mysql = MySQL()
|
|
75
|
+
self.mysql.ping()
|
|
76
|
+
except Exception as e:
|
|
77
|
+
self.mysql_reachable = False
|
|
78
|
+
self.mysql = None
|
|
79
|
+
print("MySQL is not reachable, logs will be saved locally if needed.")
|
|
108
80
|
|
|
109
81
|
# Start the task and setup the data in the database
|
|
110
|
-
if self.mysql_enabled:
|
|
82
|
+
if self.mysql_enabled and self.mysql_reachable:
|
|
111
83
|
self.customer_id = self.mysql.raw_query(f'SELECT id FROM sc.customers WHERE dbname = \'{self.customer_db}\'')[0][0]
|
|
112
|
-
|
|
113
|
-
self.task_manual_started = self._check_if_task_manual_started()
|
|
84
|
+
self._process_local_mysql_logs()
|
|
114
85
|
self._start_task()
|
|
115
|
-
else:
|
|
116
|
-
self.task_manual_started = True
|
|
117
86
|
except Exception as e:
|
|
118
87
|
self.error_handling(e)
|
|
119
88
|
|
|
@@ -147,22 +116,6 @@ class TaskScheduler(BrynQ):
|
|
|
147
116
|
:return: error (str) or response of mysql
|
|
148
117
|
"""
|
|
149
118
|
warnings.warn("Execution steps are deprecated, please stop calling this method. It does nothing anymore", DeprecationWarning)
|
|
150
|
-
return
|
|
151
|
-
|
|
152
|
-
def _check_if_task_manual_started(self):
|
|
153
|
-
"""
|
|
154
|
-
Check if the task manual is started of on schedule. If it's manual started, that's important for the variables in the db_variables function.
|
|
155
|
-
In that case the dynamic variables should be used instead of the static ones
|
|
156
|
-
:return: True of False
|
|
157
|
-
"""
|
|
158
|
-
# without logging is only possible during dev, so this is always manual
|
|
159
|
-
response = self.mysql.select('task_scheduler', 'run_instant', f'WHERE id = {self.task_id}')[0][0]
|
|
160
|
-
if response == 1:
|
|
161
|
-
# Reset the 1 back to 0 before sending the result
|
|
162
|
-
self.mysql.update('task_scheduler', ['run_instant'], [0], 'WHERE `id` = {}'.format(self.task_id))
|
|
163
|
-
return True
|
|
164
|
-
else:
|
|
165
|
-
return False
|
|
166
119
|
|
|
167
120
|
def _start_task(self):
|
|
168
121
|
"""
|
|
@@ -173,34 +126,21 @@ class TaskScheduler(BrynQ):
|
|
|
173
126
|
if self.started_local:
|
|
174
127
|
self.mysql.raw_query(f"INSERT INTO `task_scheduler_log` (reload_id, task_id, reload_status, started_at, finished_at) VALUES ({self.run_id}, {self.task_id}, 'Running', '{self.started_at}', null)", insert=True)
|
|
175
128
|
|
|
176
|
-
self.mysql.update('task_scheduler', ['status', 'step_nr'], ['RUNNING', 1], 'WHERE `id` = {
|
|
129
|
+
self.mysql.update('task_scheduler', ['status', 'step_nr'], ['RUNNING', 1], f'WHERE `id` = {self.task_id}')
|
|
177
130
|
|
|
178
|
-
def db_variable(self, variable_name: str
|
|
131
|
+
def db_variable(self, variable_name: str):
|
|
179
132
|
"""
|
|
180
|
-
Get a value from the task_variables table corresponding with the given name. If
|
|
133
|
+
Get a value from the task_variables table corresponding with the given name. If temp value is filled, it will
|
|
181
134
|
(run_instant = 1), then the temp_value will be returned. This is to give the possibility for users in the frontend to run
|
|
182
135
|
a task once manual with other values then normal without overwriting the normal values.
|
|
183
136
|
:param variable_name: the name of the variable
|
|
184
|
-
:param default_value_if_temp_is_empty: bool to determine whether default value should be used if temp value is empty when manually started
|
|
185
137
|
:return: the value of the given variable.
|
|
186
138
|
"""
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
response = self.mysql.select('task_variables', 'value',
|
|
193
|
-
f'WHERE name = \'{variable_name}\' AND task_id = {self.task_id}')
|
|
194
|
-
if len(response) == 0:
|
|
195
|
-
raise Exception(f'Variable with name \'{variable_name}\' does not exist')
|
|
196
|
-
else:
|
|
197
|
-
value = response[0][0]
|
|
198
|
-
if value is None and default_value_if_temp_is_empty is True and len(response[0]) > 0:
|
|
199
|
-
value = response[0][1]
|
|
200
|
-
return value
|
|
201
|
-
else:
|
|
202
|
-
value: str = input(f'Your MYSQL connection is not defined, enter the value for the variable {variable_name}: ')
|
|
203
|
-
return value
|
|
139
|
+
warnings.deprecated("Use self.brynq.interfaces.get_variables() instead")
|
|
140
|
+
|
|
141
|
+
variable = self.brynq.interfaces.get_variables(variable_name=variable_name)
|
|
142
|
+
|
|
143
|
+
return variable
|
|
204
144
|
|
|
205
145
|
def write_execution_log(self, message: str, data, loglevel: str = 'INFO', full_extract: bool = False):
|
|
206
146
|
"""
|
|
@@ -214,101 +154,16 @@ class TaskScheduler(BrynQ):
|
|
|
214
154
|
# Validate if the provided loglevel is valid
|
|
215
155
|
allowed_loglevels = ['DEBUG', 'INFO', 'ERROR', 'CRITICAL']
|
|
216
156
|
if loglevel not in allowed_loglevels:
|
|
217
|
-
raise
|
|
157
|
+
raise ValueError(f"You\'ve entered a not allowed loglevel. Choose one of: {','.join(allowed_loglevels)}")
|
|
218
158
|
|
|
219
159
|
# Get the linenumber from where the logline is executed.
|
|
220
160
|
file_name, line_number, function_name = self.__get_caller_info()
|
|
221
|
-
|
|
222
161
|
print('{} at line: {}'.format(message, line_number))
|
|
223
162
|
|
|
224
163
|
# Count the errors for relevant log levels
|
|
225
164
|
if loglevel == 'ERROR' or loglevel == 'CRITICAL':
|
|
226
165
|
self.error_count += 1
|
|
227
166
|
|
|
228
|
-
if self.elastic_enabled:
|
|
229
|
-
# For Elastic, we need to have the data in JSON format. Handling different data types and preparing extra payload information based on the data type
|
|
230
|
-
# If the data is just a series, count rows, columns and cells
|
|
231
|
-
# Put everything together in the payload for ElasticSearch and send it
|
|
232
|
-
payload = {
|
|
233
|
-
'task_id': self.task_id,
|
|
234
|
-
'reload_id': self.run_id,
|
|
235
|
-
'started_at': datetime.datetime.now().isoformat(),
|
|
236
|
-
'partner_id': self.partner_id,
|
|
237
|
-
'customer': self.customer,
|
|
238
|
-
'file_name': file_name,
|
|
239
|
-
'function_name': function_name,
|
|
240
|
-
'line_number': line_number,
|
|
241
|
-
'task_loglevel': self.loglevel,
|
|
242
|
-
'line_loglevel': loglevel,
|
|
243
|
-
'message': message
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
if isinstance(data, pd.Series):
|
|
247
|
-
dataframe = pd.DataFrame(data).T
|
|
248
|
-
extra_payload = {
|
|
249
|
-
'rows': len(dataframe),
|
|
250
|
-
'columns': len(dataframe.columns),
|
|
251
|
-
'cells': len(dataframe) * len(dataframe.columns),
|
|
252
|
-
}
|
|
253
|
-
if not full_extract:
|
|
254
|
-
extra_payload['payload'] = dataframe.to_json(orient='records')
|
|
255
|
-
# If the data is a list, count rows, columns and cells
|
|
256
|
-
elif isinstance(data, dict):
|
|
257
|
-
records = self.__count_keys(data)
|
|
258
|
-
extra_payload = {
|
|
259
|
-
'rows': 1,
|
|
260
|
-
'columns': records,
|
|
261
|
-
'cells': records,
|
|
262
|
-
}
|
|
263
|
-
if not full_extract:
|
|
264
|
-
extra_payload['payload'] = data
|
|
265
|
-
elif isinstance(data, pd.DataFrame):
|
|
266
|
-
extra_payload = {
|
|
267
|
-
'rows': len(data),
|
|
268
|
-
'columns': len(data.columns),
|
|
269
|
-
'cells': len(data) * len(data.columns),
|
|
270
|
-
}
|
|
271
|
-
if not full_extract:
|
|
272
|
-
extra_payload['payload'] = data.to_json(orient='records')
|
|
273
|
-
# If the data is a response from an URL request, also store all the information about the URL request.
|
|
274
|
-
elif isinstance(data, requests.Response):
|
|
275
|
-
records = 1
|
|
276
|
-
if data.request.body is not None:
|
|
277
|
-
records = self.__count_keys(json.loads(data.request.body))
|
|
278
|
-
if isinstance(data.request.body, bytes):
|
|
279
|
-
data.request.body = data.request.body.decode('utf-8')
|
|
280
|
-
extra_payload = {
|
|
281
|
-
'response': data.text,
|
|
282
|
-
'status_code': data.status_code,
|
|
283
|
-
'url': data.url,
|
|
284
|
-
'method': data.request.method,
|
|
285
|
-
'rows': 1,
|
|
286
|
-
'columns': records,
|
|
287
|
-
'cells': records,
|
|
288
|
-
}
|
|
289
|
-
if not full_extract:
|
|
290
|
-
extra_payload['payload'] = data.request.body
|
|
291
|
-
elif data is None:
|
|
292
|
-
extra_payload = {}
|
|
293
|
-
else:
|
|
294
|
-
extra_payload = {
|
|
295
|
-
'data_type': str(type(data)),
|
|
296
|
-
}
|
|
297
|
-
if not full_extract:
|
|
298
|
-
extra_payload['payload'] = data
|
|
299
|
-
|
|
300
|
-
# Modify payload based on 'full_load' flag
|
|
301
|
-
if data is not None and full_extract is True:
|
|
302
|
-
extra_payload['full_load'] = True
|
|
303
|
-
elif data is not None and full_extract is False:
|
|
304
|
-
extra_payload['full_load'] = False
|
|
305
|
-
|
|
306
|
-
payload.update(extra_payload)
|
|
307
|
-
if not self.elastic_unreachable:
|
|
308
|
-
self.es.post_document(index_name=self.es_index, document=payload)
|
|
309
|
-
else:
|
|
310
|
-
self._save_log_locally(payload, 'elastic')
|
|
311
|
-
|
|
312
167
|
# Write the logline to the MYSQL database, depends on the chosen loglevel in the task
|
|
313
168
|
if self.mysql_enabled:
|
|
314
169
|
mysql_log_data = {
|
|
@@ -319,18 +174,14 @@ class TaskScheduler(BrynQ):
|
|
|
319
174
|
'line_number': line_number,
|
|
320
175
|
'message': re.sub("[']", '', message)
|
|
321
176
|
}
|
|
322
|
-
if
|
|
177
|
+
if self.mysql_reachable:
|
|
323
178
|
try:
|
|
324
179
|
query = f"INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({mysql_log_data['reload_id']}, {mysql_log_data['task_id']}, '{mysql_log_data['log_level']}', '{mysql_log_data['created_at']}', {mysql_log_data['line_number']}, '{mysql_log_data['message']}')"
|
|
325
180
|
if self.loglevel == 'DEBUG' or (self.loglevel == 'INFO' and loglevel != 'DEBUG') or (self.loglevel == 'ERROR' and loglevel in ['ERROR', 'CRITICAL']) or (self.loglevel == 'CRITICAL' and loglevel == 'CRITICAL'):
|
|
326
181
|
self.mysql.raw_query(query, insert=True)
|
|
327
|
-
except pymysql.err.OperationalError as e:
|
|
328
|
-
print(f"MySQL connection lost during logging: {e}")
|
|
329
|
-
self.
|
|
330
|
-
self._save_log_locally(mysql_log_data, 'mysql')
|
|
331
|
-
except pymysql.err.InterfaceError as e:
|
|
332
|
-
print(f"MySQL connection closed: {e}")
|
|
333
|
-
self.mysql_unreachable = True
|
|
182
|
+
except (pymysql.err.OperationalError, pymysql.err.InterfaceError) as e:
|
|
183
|
+
print(f"MySQL connection lost or closed during logging: {e}")
|
|
184
|
+
self.mysql_reachable = False
|
|
334
185
|
self._save_log_locally(mysql_log_data, 'mysql')
|
|
335
186
|
except Exception as e:
|
|
336
187
|
print(f"Error during logging to MySQL: {e}")
|
|
@@ -346,7 +197,6 @@ class TaskScheduler(BrynQ):
|
|
|
346
197
|
"""
|
|
347
198
|
# Update the step number in the task_scheduler table
|
|
348
199
|
warnings.warn("Execution steps are deprecated, please stop calling this method. It does nothing anymore", DeprecationWarning)
|
|
349
|
-
return
|
|
350
200
|
|
|
351
201
|
def error_handling(self, e: Exception, breaking=True, send_to_teams=False):
|
|
352
202
|
"""
|
|
@@ -366,58 +216,60 @@ class TaskScheduler(BrynQ):
|
|
|
366
216
|
exc_type, exc_obj, exc_tb = sys.exc_info()
|
|
367
217
|
error = str(e)[:400].replace('\'', '').replace('\"', '') + ' | Line: {}'.format(exc_tb.tb_lineno)
|
|
368
218
|
|
|
369
|
-
if self.elastic_enabled:
|
|
370
|
-
# Preparing the primary payload with error details for upload to elastic and send it
|
|
371
|
-
payload = {
|
|
372
|
-
'task_id': self.task_id,
|
|
373
|
-
'reload_id': self.run_id,
|
|
374
|
-
'started_at': datetime.datetime.now().isoformat(),
|
|
375
|
-
'partner_id': self.partner_id,
|
|
376
|
-
'customer': self.customer,
|
|
377
|
-
'file_name': file_name,
|
|
378
|
-
'function_name': function_name,
|
|
379
|
-
'line_number': line_number,
|
|
380
|
-
'task_loglevel': self.loglevel,
|
|
381
|
-
'line_loglevel': 'CRITICAL',
|
|
382
|
-
'message': str(e),
|
|
383
|
-
'traceback': traceback.format_exc()
|
|
384
|
-
}
|
|
385
|
-
self.es.post_document(index_name=self.es_index, document=payload)
|
|
386
|
-
|
|
387
219
|
if self.mysql_enabled:
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
220
|
+
try:
|
|
221
|
+
now = datetime.datetime.now()
|
|
222
|
+
# Log to log table in the database
|
|
223
|
+
mysql_log_data = {
|
|
224
|
+
'reload_id': self.run_id,
|
|
225
|
+
'task_id': self.task_id,
|
|
226
|
+
'log_level': 'CRITICAL',
|
|
227
|
+
'created_at': now,
|
|
228
|
+
'line_number': exc_tb.tb_lineno,
|
|
229
|
+
'message': error
|
|
230
|
+
}
|
|
231
|
+
self.error_count += 1
|
|
232
|
+
# Get scheduler task details for logging
|
|
233
|
+
task_details = \
|
|
234
|
+
self.mysql.select('task_scheduler, data_interfaces', 'data_interfaces.docker_image, data_interfaces.runfile_path', 'WHERE task_scheduler.data_interface_id = data_interfaces.id AND task_scheduler.id = {}'.format(self.task_id))[0]
|
|
235
|
+
taskname = task_details[0]
|
|
236
|
+
customer = task_details[1].split('/')[-1].split('.')[0]
|
|
237
|
+
|
|
238
|
+
query = f"INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({mysql_log_data['reload_id']}, {mysql_log_data['task_id']}, '{mysql_log_data['log_level']}', '{mysql_log_data['created_at']}', {mysql_log_data['line_number']}, '{mysql_log_data['message']}')"
|
|
399
239
|
self.mysql.raw_query(query, insert=True)
|
|
400
|
-
if send_to_teams:
|
|
401
|
-
Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
|
|
402
|
-
if breaking:
|
|
403
|
-
# Set scheduler status to failed
|
|
404
|
-
self.mysql.update('task_scheduler', ['status', 'last_reload', 'last_error_message', 'step_nr'],
|
|
405
|
-
['IDLE', now, 'Failed', 0],
|
|
406
|
-
'WHERE `id` = {}'.format(self.task_id))
|
|
407
|
-
|
|
408
|
-
self.mysql.update(table='task_scheduler_log',
|
|
409
|
-
columns=['reload_status', 'finished_at'],
|
|
410
|
-
values=['Failed', f'{now}'],
|
|
411
|
-
filter=f'WHERE `reload_id` = {self.run_id}')
|
|
412
|
-
if self.email_after_errors:
|
|
413
|
-
self.email_errors(failed=True)
|
|
414
|
-
# Remove the temp values from the variables table
|
|
415
|
-
self.mysql.raw_query(f'UPDATE `task_variables` SET temp_value = null WHERE task_id = {self.task_id}', insert=True)
|
|
416
240
|
|
|
417
|
-
|
|
418
|
-
|
|
241
|
+
if send_to_teams:
|
|
242
|
+
Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
|
|
243
|
+
if breaking:
|
|
244
|
+
# Set scheduler status to failed
|
|
245
|
+
self.mysql.update('task_scheduler', ['status', 'last_reload', 'last_error_message', 'step_nr'],
|
|
246
|
+
['IDLE', now, 'Failed', 0],
|
|
247
|
+
f'WHERE `id` = {self.task_id}')
|
|
248
|
+
|
|
249
|
+
self.mysql.update(table='task_scheduler_log',
|
|
250
|
+
columns=['reload_status', 'finished_at'],
|
|
251
|
+
values=['Failed', f'{now}'],
|
|
252
|
+
filter=f'WHERE `reload_id` = {self.run_id}')
|
|
253
|
+
if self.email_after_errors:
|
|
254
|
+
self.email_errors(failed=True)
|
|
255
|
+
# Remove the temp values from the variables table
|
|
256
|
+
self.mysql.raw_query(f'UPDATE `task_variables` SET temp_value = null WHERE task_id = {self.task_id}', insert=True)
|
|
257
|
+
|
|
258
|
+
# Start the chained tasks if it there are tasks which should start if this one is failed
|
|
259
|
+
self.start_chained_tasks(finished_task_status='FAILED')
|
|
260
|
+
except (pymysql.err.OperationalError, pymysql.err.InterfaceError) as e:
|
|
261
|
+
print(f"MySQL connection lost or closed during logging: {e}")
|
|
262
|
+
self.mysql_reachable = False
|
|
263
|
+
self._save_log_locally(mysql_log_data, 'mysql')
|
|
264
|
+
except Exception as e:
|
|
265
|
+
print(f"Error during logging to MySQL: {e}")
|
|
266
|
+
self._save_log_locally(mysql_log_data, 'mysql')
|
|
419
267
|
|
|
420
|
-
|
|
268
|
+
# if breaking, reraise for clear traceback (local development) or just print if not breaking
|
|
269
|
+
if breaking:
|
|
270
|
+
raise e
|
|
271
|
+
else:
|
|
272
|
+
print(error)
|
|
421
273
|
|
|
422
274
|
def finish_task(self, reload_instant=False, log_limit: Optional[int] = 10000, log_date_limit: datetime.date = None):
|
|
423
275
|
"""
|
|
@@ -475,7 +327,8 @@ class TaskScheduler(BrynQ):
|
|
|
475
327
|
|
|
476
328
|
def start_chained_tasks(self, finished_task_status: str):
|
|
477
329
|
if self.mysql_enabled:
|
|
478
|
-
|
|
330
|
+
# only start chained tasks when trigger is on other task, otherwise this has changed in the db
|
|
331
|
+
filter = f'WHERE start_after_task_id = \'{self.task_id}\' AND start_after_preceding_task = \'{finished_task_status}\' AND task_type = \'OTHER_TASK\''
|
|
479
332
|
response = self.mysql.select(table='task_scheduler', selection='id', filter=filter)
|
|
480
333
|
if len(response) > 0:
|
|
481
334
|
tasks_to_run = [str(task[0]) for task in response]
|
|
@@ -485,7 +338,7 @@ class TaskScheduler(BrynQ):
|
|
|
485
338
|
|
|
486
339
|
def email_errors(self, failed):
|
|
487
340
|
# The mails to email to should be stored in the task_variables table with the variable email_errors_to
|
|
488
|
-
email_variable = self.
|
|
341
|
+
email_variable = self.brynq.interfaces.get_variables(variable_name='email_errors_to')
|
|
489
342
|
if email_variable is not None:
|
|
490
343
|
email_to = email_variable.split(',')
|
|
491
344
|
if isinstance(email_to, list):
|
|
@@ -546,17 +399,24 @@ class TaskScheduler(BrynQ):
|
|
|
546
399
|
except Exception as e:
|
|
547
400
|
print(f"Error processing MySQL log file {log_file}: {e}")
|
|
548
401
|
|
|
549
|
-
def
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
402
|
+
def _write_log_to_mysql(self, log_entry):
|
|
403
|
+
"""
|
|
404
|
+
Insert a log entry dictionary into the task_execution_log table in MySQL.
|
|
405
|
+
:param log_entry: dict with keys matching the columns of task_execution_log
|
|
406
|
+
"""
|
|
407
|
+
try:
|
|
408
|
+
query = (
|
|
409
|
+
"INSERT INTO `task_execution_log` "
|
|
410
|
+
"(reload_id, task_id, log_level, created_at, line_number, message) "
|
|
411
|
+
"VALUES ({reload_id}, {task_id}, '{log_level}', '{created_at}', {line_number}, '{message}')"
|
|
412
|
+
).format(
|
|
413
|
+
reload_id=log_entry['reload_id'],
|
|
414
|
+
task_id=log_entry['task_id'],
|
|
415
|
+
log_level=log_entry['log_level'],
|
|
416
|
+
created_at=log_entry['created_at'],
|
|
417
|
+
line_number=log_entry['line_number'],
|
|
418
|
+
message=str(log_entry['message']).replace("'", "")
|
|
419
|
+
)
|
|
420
|
+
self.mysql.raw_query(query, insert=True)
|
|
421
|
+
except Exception as e:
|
|
422
|
+
print(f"Error writing log entry to MySQL: {e}")
|
|
@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
|
|
|
2
2
|
|
|
3
3
|
setup(
|
|
4
4
|
name='brynq_sdk_task_scheduler',
|
|
5
|
-
version='
|
|
5
|
+
version='4.0.1',
|
|
6
6
|
description='Code to execute tasks in BrynQ.com with the task scheduler',
|
|
7
7
|
long_description='Code to execute tasks in the BrynQ.com platform with the task scheduler',
|
|
8
8
|
author='BrynQ',
|
|
@@ -10,9 +10,9 @@ setup(
|
|
|
10
10
|
packages=find_namespace_packages(include=['brynq_sdk*']),
|
|
11
11
|
license='BrynQ License',
|
|
12
12
|
install_requires=[
|
|
13
|
-
'brynq-sdk-brynq>=
|
|
13
|
+
'brynq-sdk-brynq>=4,<5',
|
|
14
14
|
'brynq-sdk-functions>=2,<3',
|
|
15
|
-
'brynq-sdk-mysql>=
|
|
15
|
+
'brynq-sdk-mysql>=3,<4',
|
|
16
16
|
'brynq-sdk-mandrill>=2,<3',
|
|
17
17
|
'brynq-sdk-elastic>=3,<4'
|
|
18
18
|
],
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|