brynq-sdk-task-scheduler 3.0.10__tar.gz → 4.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/PKG-INFO +1 -1
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/brynq_sdk_task_scheduler/task_scheduler.py +120 -257
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/brynq_sdk_task_scheduler.egg-info/PKG-INFO +1 -1
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/brynq_sdk_task_scheduler.egg-info/requires.txt +2 -2
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/setup.py +3 -3
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/brynq_sdk_task_scheduler/__init__.py +0 -0
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/brynq_sdk_task_scheduler.egg-info/SOURCES.txt +0 -0
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/brynq_sdk_task_scheduler.egg-info/dependency_links.txt +0 -0
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/brynq_sdk_task_scheduler.egg-info/not-zip-safe +0 -0
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/brynq_sdk_task_scheduler.egg-info/top_level.txt +0 -0
- {brynq_sdk_task_scheduler-3.0.10 → brynq_sdk_task_scheduler-4.0.0}/setup.cfg +0 -0
|
@@ -16,104 +16,76 @@ from brynq_sdk_elastic import Elastic
|
|
|
16
16
|
from brynq_sdk_brynq import BrynQ
|
|
17
17
|
import warnings
|
|
18
18
|
import re
|
|
19
|
-
LOGGING_OPTIONS = Literal['MYSQL', 'ELASTIC']
|
|
20
19
|
|
|
21
20
|
|
|
22
|
-
class TaskScheduler
|
|
21
|
+
class TaskScheduler:
|
|
23
22
|
|
|
24
|
-
def __init__(self,
|
|
23
|
+
def __init__(self, data_interface_id: int = None, loglevel: str = 'INFO', email_after_errors: bool = False):
|
|
25
24
|
"""
|
|
26
25
|
The TaskScheduler is responsible for the logging to the database. Based on this logging, the next reload will
|
|
27
26
|
start or not and warning will be given or not
|
|
28
|
-
:param
|
|
27
|
+
:param data_interface_id: The ID from the task as saved in the task_scheduler table in the customer database
|
|
29
28
|
:param email_after_errors: a True or False value. When True, there will be send an email to a contactperson of the customer (as given in the database) with the number of errors
|
|
30
29
|
:param loglevel: Chose on which level you want to store the logs. Default is INFO. that means that a logline
|
|
31
30
|
:param disable_logging: If the interface is started from a local instance, logs will not be stored by default. If this is set to True, the logs will be stored in the database
|
|
32
31
|
with level DEBUG not is stored
|
|
33
32
|
"""
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
self.
|
|
38
|
-
|
|
39
|
-
|
|
33
|
+
# If the task is started via the task_scheduler, the following parameters will be passed by the scheduler.
|
|
34
|
+
# The distinction between local and non local is made because the scheduler usually sets the scheduler_log table entry and run_id. When running locally, the tasks will not log anything by default.
|
|
35
|
+
if any(flag in sys.argv[0].split('/') for flag in ['opt', 'home']):
|
|
36
|
+
self.started_local = True
|
|
37
|
+
self.run_id = int(round(time.time() * 100000))
|
|
38
|
+
self.task_id = None
|
|
39
|
+
self.data_interface_id = data_interface_id
|
|
40
|
+
# set it into environment variables for reuse in source and target systems
|
|
41
|
+
os.environ['DATA_INTERFACE_ID'] = str(self.data_interface_id)
|
|
40
42
|
self.mysql_enabled = False
|
|
41
43
|
self.elastic_enabled = False
|
|
42
|
-
|
|
43
|
-
# Initialize MySQL
|
|
44
|
-
self.mysql_unreachable = False
|
|
45
|
-
# if self.mysql_enabled:
|
|
46
|
-
try:
|
|
47
|
-
self.mysql = MySQL()
|
|
48
|
-
self.mysql.ping()
|
|
49
|
-
except Exception as e:
|
|
50
|
-
self.mysql_unreachable = True
|
|
51
|
-
self.mysql = None
|
|
52
|
-
print("MySQL is enabled but not reachable, logs will be saved locally if needed.")
|
|
53
|
-
# else:
|
|
54
|
-
# self.mysql = None
|
|
55
|
-
|
|
56
|
-
# Initialize ElasticSearch
|
|
57
|
-
self.elastic_unreachable = False
|
|
58
|
-
if self.elastic_enabled:
|
|
59
|
-
try:
|
|
60
|
-
self.es = Elastic()
|
|
61
|
-
self.es.get_health()
|
|
62
|
-
except Exception as e:
|
|
63
|
-
self.elastic_unreachable = True
|
|
64
|
-
self.es = Elastic(disabled=True)
|
|
65
|
-
print("ElasticSearch is enabled but not reachable, logs will be saved locally if needed.")
|
|
44
|
+
print("You're running locally, so your task won't be started in the platform and logs won't be saved")
|
|
66
45
|
else:
|
|
67
|
-
self.
|
|
68
|
-
|
|
46
|
+
self.started_local = False
|
|
47
|
+
self.customer_db, self.task_id, self.run_id, self.data_interface_id = sys.argv[1:5]
|
|
48
|
+
self.mysql_enabled = True
|
|
49
|
+
self.elastic_enabled = True
|
|
50
|
+
# If the task is started locally, the parameters should be set locally
|
|
51
|
+
|
|
52
|
+
print(f"Run ID: {self.run_id}, Data Interface ID: {self.data_interface_id}, Task ID: {self.task_id}, Started Locally: {self.started_local}")
|
|
53
|
+
self.brynq = BrynQ()
|
|
54
|
+
self.email_after_errors = email_after_errors
|
|
69
55
|
# Set up local log directory
|
|
70
56
|
self.local_log_dir = 'local_logs'
|
|
71
57
|
os.makedirs(self.local_log_dir, exist_ok=True)
|
|
72
58
|
|
|
73
59
|
# Process local logs if services are now reachable
|
|
74
|
-
if self.mysql_enabled and
|
|
60
|
+
if self.mysql_enabled and self.mysql_reachable:
|
|
75
61
|
self._process_local_mysql_logs()
|
|
76
|
-
if self.elastic_enabled and not self.elastic_unreachable:
|
|
77
|
-
self._process_local_elastic_logs()
|
|
78
62
|
|
|
79
63
|
try:
|
|
80
|
-
self.email_after_errors = email_after_errors
|
|
81
64
|
self.customer_db = os.getenv("MYSQL_DATABASE")
|
|
82
65
|
self.customer = os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_')
|
|
83
66
|
self.partner_id = os.getenv('PARTNER_ID').lower().replace(' ', '_') if os.getenv('PARTNER_ID') else 'brynq'
|
|
84
67
|
self.loglevel = loglevel
|
|
85
68
|
self.started_at = datetime.datetime.now()
|
|
86
|
-
# If the task is started via the task_scheduler, the following 3 parameters will be passed by the scheduler.
|
|
87
|
-
# The distinction between local and non local is made because the scheduler usually sets the scheduler_log table entry and run_id. When running locally, the tasks should do this itself.
|
|
88
|
-
if len(sys.argv[1:4]) > 0:
|
|
89
|
-
self.started_local = False
|
|
90
|
-
self.customer_db, self.task_id, self.run_id = sys.argv[1:4]
|
|
91
|
-
# If the task is started locally, the parameters should be set locally
|
|
92
|
-
else:
|
|
93
|
-
self.started_local = True
|
|
94
|
-
self.run_id = int(round(time.time() * 100000))
|
|
95
|
-
self.task_id = task_id
|
|
96
|
-
print(self.task_id, self.run_id)
|
|
97
69
|
self.error_count = 0
|
|
98
70
|
|
|
99
71
|
# Check if the log tables exists in the customer database1. If not, create them
|
|
100
72
|
# Mysql throws a warning when a table already exists. We don't care so we ignore warnings. (not exceptions!)
|
|
101
73
|
warnings.filterwarnings('ignore')
|
|
102
74
|
|
|
103
|
-
#
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
self.
|
|
107
|
-
self.
|
|
75
|
+
# Initialize MySQL
|
|
76
|
+
self.mysql_reachable = True
|
|
77
|
+
try:
|
|
78
|
+
self.mysql = MySQL()
|
|
79
|
+
self.mysql.ping()
|
|
80
|
+
except Exception as e:
|
|
81
|
+
self.mysql_reachable = False
|
|
82
|
+
self.mysql = None
|
|
83
|
+
print("MySQL is enabled but not reachable, logs will be saved locally if needed.")
|
|
108
84
|
|
|
109
85
|
# Start the task and setup the data in the database
|
|
110
|
-
if self.mysql_enabled:
|
|
86
|
+
if self.mysql_enabled and self.mysql_reachable:
|
|
111
87
|
self.customer_id = self.mysql.raw_query(f'SELECT id FROM sc.customers WHERE dbname = \'{self.customer_db}\'')[0][0]
|
|
112
|
-
# Check if the task is started on schedule or manual. store in a variable to use later in the script
|
|
113
|
-
self.task_manual_started = self._check_if_task_manual_started()
|
|
114
88
|
self._start_task()
|
|
115
|
-
else:
|
|
116
|
-
self.task_manual_started = True
|
|
117
89
|
except Exception as e:
|
|
118
90
|
self.error_handling(e)
|
|
119
91
|
|
|
@@ -147,22 +119,6 @@ class TaskScheduler(BrynQ):
|
|
|
147
119
|
:return: error (str) or response of mysql
|
|
148
120
|
"""
|
|
149
121
|
warnings.warn("Execution steps are deprecated, please stop calling this method. It does nothing anymore", DeprecationWarning)
|
|
150
|
-
return
|
|
151
|
-
|
|
152
|
-
def _check_if_task_manual_started(self):
|
|
153
|
-
"""
|
|
154
|
-
Check if the task manual is started of on schedule. If it's manual started, that's important for the variables in the db_variables function.
|
|
155
|
-
In that case the dynamic variables should be used instead of the static ones
|
|
156
|
-
:return: True of False
|
|
157
|
-
"""
|
|
158
|
-
# without logging is only possible during dev, so this is always manual
|
|
159
|
-
response = self.mysql.select('task_scheduler', 'run_instant', f'WHERE id = {self.task_id}')[0][0]
|
|
160
|
-
if response == 1:
|
|
161
|
-
# Reset the 1 back to 0 before sending the result
|
|
162
|
-
self.mysql.update('task_scheduler', ['run_instant'], [0], 'WHERE `id` = {}'.format(self.task_id))
|
|
163
|
-
return True
|
|
164
|
-
else:
|
|
165
|
-
return False
|
|
166
122
|
|
|
167
123
|
def _start_task(self):
|
|
168
124
|
"""
|
|
@@ -173,34 +129,21 @@ class TaskScheduler(BrynQ):
|
|
|
173
129
|
if self.started_local:
|
|
174
130
|
self.mysql.raw_query(f"INSERT INTO `task_scheduler_log` (reload_id, task_id, reload_status, started_at, finished_at) VALUES ({self.run_id}, {self.task_id}, 'Running', '{self.started_at}', null)", insert=True)
|
|
175
131
|
|
|
176
|
-
self.mysql.update('task_scheduler', ['status', 'step_nr'], ['RUNNING', 1], 'WHERE `id` = {
|
|
132
|
+
self.mysql.update('task_scheduler', ['status', 'step_nr'], ['RUNNING', 1], f'WHERE `id` = {self.task_id}')
|
|
177
133
|
|
|
178
|
-
def db_variable(self, variable_name: str
|
|
134
|
+
def db_variable(self, variable_name: str):
|
|
179
135
|
"""
|
|
180
|
-
Get a value from the task_variables table corresponding with the given name. If
|
|
136
|
+
Get a value from the task_variables table corresponding with the given name. If temp value is filled, it will
|
|
181
137
|
(run_instant = 1), then the temp_value will be returned. This is to give the possibility for users in the frontend to run
|
|
182
138
|
a task once manual with other values then normal without overwriting the normal values.
|
|
183
139
|
:param variable_name: the name of the variable
|
|
184
|
-
:param default_value_if_temp_is_empty: bool to determine whether default value should be used if temp value is empty when manually started
|
|
185
140
|
:return: the value of the given variable.
|
|
186
141
|
"""
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
response = self.mysql.select('task_variables', 'value',
|
|
193
|
-
f'WHERE name = \'{variable_name}\' AND task_id = {self.task_id}')
|
|
194
|
-
if len(response) == 0:
|
|
195
|
-
raise Exception(f'Variable with name \'{variable_name}\' does not exist')
|
|
196
|
-
else:
|
|
197
|
-
value = response[0][0]
|
|
198
|
-
if value is None and default_value_if_temp_is_empty is True and len(response[0]) > 0:
|
|
199
|
-
value = response[0][1]
|
|
200
|
-
return value
|
|
201
|
-
else:
|
|
202
|
-
value: str = input(f'Your MYSQL connection is not defined, enter the value for the variable {variable_name}: ')
|
|
203
|
-
return value
|
|
142
|
+
warnings.deprecated("Use self.brynq.interfaces.get_variables() instead")
|
|
143
|
+
|
|
144
|
+
variable = self.brynq.interfaces.get_variables(variable_name=variable_name)
|
|
145
|
+
|
|
146
|
+
return variable
|
|
204
147
|
|
|
205
148
|
def write_execution_log(self, message: str, data, loglevel: str = 'INFO', full_extract: bool = False):
|
|
206
149
|
"""
|
|
@@ -214,101 +157,16 @@ class TaskScheduler(BrynQ):
|
|
|
214
157
|
# Validate if the provided loglevel is valid
|
|
215
158
|
allowed_loglevels = ['DEBUG', 'INFO', 'ERROR', 'CRITICAL']
|
|
216
159
|
if loglevel not in allowed_loglevels:
|
|
217
|
-
raise
|
|
160
|
+
raise ValueError(f"You\'ve entered a not allowed loglevel. Choose one of: {','.join(allowed_loglevels)}")
|
|
218
161
|
|
|
219
162
|
# Get the linenumber from where the logline is executed.
|
|
220
163
|
file_name, line_number, function_name = self.__get_caller_info()
|
|
221
|
-
|
|
222
164
|
print('{} at line: {}'.format(message, line_number))
|
|
223
165
|
|
|
224
166
|
# Count the errors for relevant log levels
|
|
225
167
|
if loglevel == 'ERROR' or loglevel == 'CRITICAL':
|
|
226
168
|
self.error_count += 1
|
|
227
169
|
|
|
228
|
-
if self.elastic_enabled:
|
|
229
|
-
# For Elastic, we need to have the data in JSON format. Handling different data types and preparing extra payload information based on the data type
|
|
230
|
-
# If the data is just a series, count rows, columns and cells
|
|
231
|
-
# Put everything together in the payload for ElasticSearch and send it
|
|
232
|
-
payload = {
|
|
233
|
-
'task_id': self.task_id,
|
|
234
|
-
'reload_id': self.run_id,
|
|
235
|
-
'started_at': datetime.datetime.now().isoformat(),
|
|
236
|
-
'partner_id': self.partner_id,
|
|
237
|
-
'customer': self.customer,
|
|
238
|
-
'file_name': file_name,
|
|
239
|
-
'function_name': function_name,
|
|
240
|
-
'line_number': line_number,
|
|
241
|
-
'task_loglevel': self.loglevel,
|
|
242
|
-
'line_loglevel': loglevel,
|
|
243
|
-
'message': message
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
if isinstance(data, pd.Series):
|
|
247
|
-
dataframe = pd.DataFrame(data).T
|
|
248
|
-
extra_payload = {
|
|
249
|
-
'rows': len(dataframe),
|
|
250
|
-
'columns': len(dataframe.columns),
|
|
251
|
-
'cells': len(dataframe) * len(dataframe.columns),
|
|
252
|
-
}
|
|
253
|
-
if not full_extract:
|
|
254
|
-
extra_payload['payload'] = dataframe.to_json(orient='records')
|
|
255
|
-
# If the data is a list, count rows, columns and cells
|
|
256
|
-
elif isinstance(data, dict):
|
|
257
|
-
records = self.__count_keys(data)
|
|
258
|
-
extra_payload = {
|
|
259
|
-
'rows': 1,
|
|
260
|
-
'columns': records,
|
|
261
|
-
'cells': records,
|
|
262
|
-
}
|
|
263
|
-
if not full_extract:
|
|
264
|
-
extra_payload['payload'] = data
|
|
265
|
-
elif isinstance(data, pd.DataFrame):
|
|
266
|
-
extra_payload = {
|
|
267
|
-
'rows': len(data),
|
|
268
|
-
'columns': len(data.columns),
|
|
269
|
-
'cells': len(data) * len(data.columns),
|
|
270
|
-
}
|
|
271
|
-
if not full_extract:
|
|
272
|
-
extra_payload['payload'] = data.to_json(orient='records')
|
|
273
|
-
# If the data is a response from an URL request, also store all the information about the URL request.
|
|
274
|
-
elif isinstance(data, requests.Response):
|
|
275
|
-
records = 1
|
|
276
|
-
if data.request.body is not None:
|
|
277
|
-
records = self.__count_keys(json.loads(data.request.body))
|
|
278
|
-
if isinstance(data.request.body, bytes):
|
|
279
|
-
data.request.body = data.request.body.decode('utf-8')
|
|
280
|
-
extra_payload = {
|
|
281
|
-
'response': data.text,
|
|
282
|
-
'status_code': data.status_code,
|
|
283
|
-
'url': data.url,
|
|
284
|
-
'method': data.request.method,
|
|
285
|
-
'rows': 1,
|
|
286
|
-
'columns': records,
|
|
287
|
-
'cells': records,
|
|
288
|
-
}
|
|
289
|
-
if not full_extract:
|
|
290
|
-
extra_payload['payload'] = data.request.body
|
|
291
|
-
elif data is None:
|
|
292
|
-
extra_payload = {}
|
|
293
|
-
else:
|
|
294
|
-
extra_payload = {
|
|
295
|
-
'data_type': str(type(data)),
|
|
296
|
-
}
|
|
297
|
-
if not full_extract:
|
|
298
|
-
extra_payload['payload'] = data
|
|
299
|
-
|
|
300
|
-
# Modify payload based on 'full_load' flag
|
|
301
|
-
if data is not None and full_extract is True:
|
|
302
|
-
extra_payload['full_load'] = True
|
|
303
|
-
elif data is not None and full_extract is False:
|
|
304
|
-
extra_payload['full_load'] = False
|
|
305
|
-
|
|
306
|
-
payload.update(extra_payload)
|
|
307
|
-
if not self.elastic_unreachable:
|
|
308
|
-
self.es.post_document(index_name=self.es_index, document=payload)
|
|
309
|
-
else:
|
|
310
|
-
self._save_log_locally(payload, 'elastic')
|
|
311
|
-
|
|
312
170
|
# Write the logline to the MYSQL database, depends on the chosen loglevel in the task
|
|
313
171
|
if self.mysql_enabled:
|
|
314
172
|
mysql_log_data = {
|
|
@@ -319,18 +177,14 @@ class TaskScheduler(BrynQ):
|
|
|
319
177
|
'line_number': line_number,
|
|
320
178
|
'message': re.sub("[']", '', message)
|
|
321
179
|
}
|
|
322
|
-
if
|
|
180
|
+
if self.mysql_reachable:
|
|
323
181
|
try:
|
|
324
182
|
query = f"INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({mysql_log_data['reload_id']}, {mysql_log_data['task_id']}, '{mysql_log_data['log_level']}', '{mysql_log_data['created_at']}', {mysql_log_data['line_number']}, '{mysql_log_data['message']}')"
|
|
325
183
|
if self.loglevel == 'DEBUG' or (self.loglevel == 'INFO' and loglevel != 'DEBUG') or (self.loglevel == 'ERROR' and loglevel in ['ERROR', 'CRITICAL']) or (self.loglevel == 'CRITICAL' and loglevel == 'CRITICAL'):
|
|
326
184
|
self.mysql.raw_query(query, insert=True)
|
|
327
|
-
except pymysql.err.OperationalError as e:
|
|
328
|
-
print(f"MySQL connection lost during logging: {e}")
|
|
329
|
-
self.
|
|
330
|
-
self._save_log_locally(mysql_log_data, 'mysql')
|
|
331
|
-
except pymysql.err.InterfaceError as e:
|
|
332
|
-
print(f"MySQL connection closed: {e}")
|
|
333
|
-
self.mysql_unreachable = True
|
|
185
|
+
except (pymysql.err.OperationalError, pymysql.err.InterfaceError) as e:
|
|
186
|
+
print(f"MySQL connection lost or closed during logging: {e}")
|
|
187
|
+
self.mysql_reachable = False
|
|
334
188
|
self._save_log_locally(mysql_log_data, 'mysql')
|
|
335
189
|
except Exception as e:
|
|
336
190
|
print(f"Error during logging to MySQL: {e}")
|
|
@@ -346,7 +200,6 @@ class TaskScheduler(BrynQ):
|
|
|
346
200
|
"""
|
|
347
201
|
# Update the step number in the task_scheduler table
|
|
348
202
|
warnings.warn("Execution steps are deprecated, please stop calling this method. It does nothing anymore", DeprecationWarning)
|
|
349
|
-
return
|
|
350
203
|
|
|
351
204
|
def error_handling(self, e: Exception, breaking=True, send_to_teams=False):
|
|
352
205
|
"""
|
|
@@ -366,58 +219,60 @@ class TaskScheduler(BrynQ):
|
|
|
366
219
|
exc_type, exc_obj, exc_tb = sys.exc_info()
|
|
367
220
|
error = str(e)[:400].replace('\'', '').replace('\"', '') + ' | Line: {}'.format(exc_tb.tb_lineno)
|
|
368
221
|
|
|
369
|
-
if self.elastic_enabled:
|
|
370
|
-
# Preparing the primary payload with error details for upload to elastic and send it
|
|
371
|
-
payload = {
|
|
372
|
-
'task_id': self.task_id,
|
|
373
|
-
'reload_id': self.run_id,
|
|
374
|
-
'started_at': datetime.datetime.now().isoformat(),
|
|
375
|
-
'partner_id': self.partner_id,
|
|
376
|
-
'customer': self.customer,
|
|
377
|
-
'file_name': file_name,
|
|
378
|
-
'function_name': function_name,
|
|
379
|
-
'line_number': line_number,
|
|
380
|
-
'task_loglevel': self.loglevel,
|
|
381
|
-
'line_loglevel': 'CRITICAL',
|
|
382
|
-
'message': str(e),
|
|
383
|
-
'traceback': traceback.format_exc()
|
|
384
|
-
}
|
|
385
|
-
self.es.post_document(index_name=self.es_index, document=payload)
|
|
386
|
-
|
|
387
222
|
if self.mysql_enabled:
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
223
|
+
try:
|
|
224
|
+
now = datetime.datetime.now()
|
|
225
|
+
# Log to log table in the database
|
|
226
|
+
mysql_log_data = {
|
|
227
|
+
'reload_id': self.run_id,
|
|
228
|
+
'task_id': self.task_id,
|
|
229
|
+
'log_level': 'CRITICAL',
|
|
230
|
+
'created_at': now,
|
|
231
|
+
'line_number': exc_tb.tb_lineno,
|
|
232
|
+
'message': error
|
|
233
|
+
}
|
|
234
|
+
self.error_count += 1
|
|
235
|
+
# Get scheduler task details for logging
|
|
236
|
+
task_details = \
|
|
237
|
+
self.mysql.select('task_scheduler, data_interfaces', 'data_interfaces.docker_image, data_interfaces.runfile_path', 'WHERE task_scheduler.data_interface_id = data_interfaces.id AND task_scheduler.id = {}'.format(self.task_id))[0]
|
|
238
|
+
taskname = task_details[0]
|
|
239
|
+
customer = task_details[1].split('/')[-1].split('.')[0]
|
|
240
|
+
|
|
241
|
+
query = f"INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({mysql_log_data['reload_id']}, {mysql_log_data['task_id']}, '{mysql_log_data['log_level']}', '{mysql_log_data['created_at']}', {mysql_log_data['line_number']}, '{mysql_log_data['message']}')"
|
|
399
242
|
self.mysql.raw_query(query, insert=True)
|
|
400
|
-
if send_to_teams:
|
|
401
|
-
Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
|
|
402
|
-
if breaking:
|
|
403
|
-
# Set scheduler status to failed
|
|
404
|
-
self.mysql.update('task_scheduler', ['status', 'last_reload', 'last_error_message', 'step_nr'],
|
|
405
|
-
['IDLE', now, 'Failed', 0],
|
|
406
|
-
'WHERE `id` = {}'.format(self.task_id))
|
|
407
|
-
|
|
408
|
-
self.mysql.update(table='task_scheduler_log',
|
|
409
|
-
columns=['reload_status', 'finished_at'],
|
|
410
|
-
values=['Failed', f'{now}'],
|
|
411
|
-
filter=f'WHERE `reload_id` = {self.run_id}')
|
|
412
|
-
if self.email_after_errors:
|
|
413
|
-
self.email_errors(failed=True)
|
|
414
|
-
# Remove the temp values from the variables table
|
|
415
|
-
self.mysql.raw_query(f'UPDATE `task_variables` SET temp_value = null WHERE task_id = {self.task_id}', insert=True)
|
|
416
243
|
|
|
417
|
-
|
|
418
|
-
|
|
244
|
+
if send_to_teams:
|
|
245
|
+
Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
|
|
246
|
+
if breaking:
|
|
247
|
+
# Set scheduler status to failed
|
|
248
|
+
self.mysql.update('task_scheduler', ['status', 'last_reload', 'last_error_message', 'step_nr'],
|
|
249
|
+
['IDLE', now, 'Failed', 0],
|
|
250
|
+
f'WHERE `id` = {self.task_id}')
|
|
251
|
+
|
|
252
|
+
self.mysql.update(table='task_scheduler_log',
|
|
253
|
+
columns=['reload_status', 'finished_at'],
|
|
254
|
+
values=['Failed', f'{now}'],
|
|
255
|
+
filter=f'WHERE `reload_id` = {self.run_id}')
|
|
256
|
+
if self.email_after_errors:
|
|
257
|
+
self.email_errors(failed=True)
|
|
258
|
+
# Remove the temp values from the variables table
|
|
259
|
+
self.mysql.raw_query(f'UPDATE `task_variables` SET temp_value = null WHERE task_id = {self.task_id}', insert=True)
|
|
260
|
+
|
|
261
|
+
# Start the chained tasks if it there are tasks which should start if this one is failed
|
|
262
|
+
self.start_chained_tasks(finished_task_status='FAILED')
|
|
263
|
+
except (pymysql.err.OperationalError, pymysql.err.InterfaceError) as e:
|
|
264
|
+
print(f"MySQL connection lost or closed during logging: {e}")
|
|
265
|
+
self.mysql_reachable = False
|
|
266
|
+
self._save_log_locally(mysql_log_data, 'mysql')
|
|
267
|
+
except Exception as e:
|
|
268
|
+
print(f"Error during logging to MySQL: {e}")
|
|
269
|
+
self._save_log_locally(mysql_log_data, 'mysql')
|
|
419
270
|
|
|
420
|
-
|
|
271
|
+
# if breaking, reraise for clear traceback (local development) or just print if not breaking
|
|
272
|
+
if breaking:
|
|
273
|
+
raise e
|
|
274
|
+
else:
|
|
275
|
+
print(error)
|
|
421
276
|
|
|
422
277
|
def finish_task(self, reload_instant=False, log_limit: Optional[int] = 10000, log_date_limit: datetime.date = None):
|
|
423
278
|
"""
|
|
@@ -475,7 +330,8 @@ class TaskScheduler(BrynQ):
|
|
|
475
330
|
|
|
476
331
|
def start_chained_tasks(self, finished_task_status: str):
|
|
477
332
|
if self.mysql_enabled:
|
|
478
|
-
|
|
333
|
+
# only start chained tasks when trigger is on other task, otherwise this has changed in the db
|
|
334
|
+
filter = f'WHERE start_after_task_id = \'{self.task_id}\' AND start_after_preceding_task = \'{finished_task_status}\' AND task_type = \'OTHER_TASK\''
|
|
479
335
|
response = self.mysql.select(table='task_scheduler', selection='id', filter=filter)
|
|
480
336
|
if len(response) > 0:
|
|
481
337
|
tasks_to_run = [str(task[0]) for task in response]
|
|
@@ -485,7 +341,7 @@ class TaskScheduler(BrynQ):
|
|
|
485
341
|
|
|
486
342
|
def email_errors(self, failed):
|
|
487
343
|
# The mails to email to should be stored in the task_variables table with the variable email_errors_to
|
|
488
|
-
email_variable = self.
|
|
344
|
+
email_variable = self.brynq.interfaces.get_variables(variable_name='email_errors_to')
|
|
489
345
|
if email_variable is not None:
|
|
490
346
|
email_to = email_variable.split(',')
|
|
491
347
|
if isinstance(email_to, list):
|
|
@@ -546,17 +402,24 @@ class TaskScheduler(BrynQ):
|
|
|
546
402
|
except Exception as e:
|
|
547
403
|
print(f"Error processing MySQL log file {log_file}: {e}")
|
|
548
404
|
|
|
549
|
-
def
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
405
|
+
def _write_log_to_mysql(self, log_entry):
|
|
406
|
+
"""
|
|
407
|
+
Insert a log entry dictionary into the task_execution_log table in MySQL.
|
|
408
|
+
:param log_entry: dict with keys matching the columns of task_execution_log
|
|
409
|
+
"""
|
|
410
|
+
try:
|
|
411
|
+
query = (
|
|
412
|
+
"INSERT INTO `task_execution_log` "
|
|
413
|
+
"(reload_id, task_id, log_level, created_at, line_number, message) "
|
|
414
|
+
"VALUES ({reload_id}, {task_id}, '{log_level}', '{created_at}', {line_number}, '{message}')"
|
|
415
|
+
).format(
|
|
416
|
+
reload_id=log_entry['reload_id'],
|
|
417
|
+
task_id=log_entry['task_id'],
|
|
418
|
+
log_level=log_entry['log_level'],
|
|
419
|
+
created_at=log_entry['created_at'],
|
|
420
|
+
line_number=log_entry['line_number'],
|
|
421
|
+
message=str(log_entry['message']).replace("'", "")
|
|
422
|
+
)
|
|
423
|
+
self.mysql.raw_query(query, insert=True)
|
|
424
|
+
except Exception as e:
|
|
425
|
+
print(f"Error writing log entry to MySQL: {e}")
|
|
@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
|
|
|
2
2
|
|
|
3
3
|
setup(
|
|
4
4
|
name='brynq_sdk_task_scheduler',
|
|
5
|
-
version='
|
|
5
|
+
version='4.0.0',
|
|
6
6
|
description='Code to execute tasks in BrynQ.com with the task scheduler',
|
|
7
7
|
long_description='Code to execute tasks in the BrynQ.com platform with the task scheduler',
|
|
8
8
|
author='BrynQ',
|
|
@@ -10,9 +10,9 @@ setup(
|
|
|
10
10
|
packages=find_namespace_packages(include=['brynq_sdk*']),
|
|
11
11
|
license='BrynQ License',
|
|
12
12
|
install_requires=[
|
|
13
|
-
'brynq-sdk-brynq>=
|
|
13
|
+
'brynq-sdk-brynq>=4,<5',
|
|
14
14
|
'brynq-sdk-functions>=2,<3',
|
|
15
|
-
'brynq-sdk-mysql>=
|
|
15
|
+
'brynq-sdk-mysql>=3,<4',
|
|
16
16
|
'brynq-sdk-mandrill>=2,<3',
|
|
17
17
|
'brynq-sdk-elastic>=3,<4'
|
|
18
18
|
],
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|