brynq-sdk-task-scheduler 3.0.10__tar.gz → 4.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.0
2
2
  Name: brynq_sdk_task_scheduler
3
- Version: 3.0.10
3
+ Version: 4.0.1
4
4
  Summary: Code to execute tasks in BrynQ.com with the task scheduler
5
5
  Home-page: UNKNOWN
6
6
  Author: BrynQ
@@ -16,104 +16,73 @@ from brynq_sdk_elastic import Elastic
16
16
  from brynq_sdk_brynq import BrynQ
17
17
  import warnings
18
18
  import re
19
- LOGGING_OPTIONS = Literal['MYSQL', 'ELASTIC']
20
19
 
21
20
 
22
- class TaskScheduler(BrynQ):
21
+ class TaskScheduler:
23
22
 
24
- def __init__(self, task_id: int = None, loglevel: str = 'INFO', email_after_errors: bool = False, logging: List[LOGGING_OPTIONS] | None = ["MYSQL", "ELASTIC"]):
23
+ def __init__(self, data_interface_id: int = None, loglevel: str = 'INFO', email_after_errors: bool = False):
25
24
  """
26
25
  The TaskScheduler is responsible for the logging to the database. Based on this logging, the next reload will
27
26
  start or not and warning will be given or not
28
- :param task_id: The ID from the task as saved in the task_scheduler table in the customer database
27
+ :param data_interface_id: The ID from the task as saved in the task_scheduler table in the customer database
29
28
  :param email_after_errors: a True or False value. When True, there will be send an email to a contactperson of the customer (as given in the database) with the number of errors
30
29
  :param loglevel: Chose on which level you want to store the logs. Default is INFO. that means that a logline
31
30
  :param disable_logging: If the interface is started from a local instance, logs will not be stored by default. If this is set to True, the logs will be stored in the database
32
31
  with level DEBUG not is stored
33
32
  """
34
- super().__init__()
35
- try:
36
- self.mysql_enabled = 'MYSQL' in logging
37
- self.elastic_enabled = 'ELASTIC' in logging
38
- except Exception as e:
39
- print("Error parsing logging options, disabling logging. Error is: " + str(e))
33
+ # If the task is started via the task_scheduler, the following parameters will be passed by the scheduler.
34
+ # The distinction between local and non local is made because the scheduler usually sets the scheduler_log table entry and run_id. When running locally, the tasks will not log anything by default.
35
+ if any(flag in sys.argv[0].split('/') for flag in ['opt', 'home']):
36
+ self.started_local = True
37
+ self.run_id = int(round(time.time() * 100000))
38
+ self.task_id = None
39
+ self.data_interface_id = data_interface_id
40
+ # set it into environment variables for reuse in source and target systems
41
+ os.environ['DATA_INTERFACE_ID'] = str(self.data_interface_id)
40
42
  self.mysql_enabled = False
41
43
  self.elastic_enabled = False
42
-
43
- # Initialize MySQL
44
- self.mysql_unreachable = False
45
- # if self.mysql_enabled:
46
- try:
47
- self.mysql = MySQL()
48
- self.mysql.ping()
49
- except Exception as e:
50
- self.mysql_unreachable = True
51
- self.mysql = None
52
- print("MySQL is enabled but not reachable, logs will be saved locally if needed.")
53
- # else:
54
- # self.mysql = None
55
-
56
- # Initialize ElasticSearch
57
- self.elastic_unreachable = False
58
- if self.elastic_enabled:
59
- try:
60
- self.es = Elastic()
61
- self.es.get_health()
62
- except Exception as e:
63
- self.elastic_unreachable = True
64
- self.es = Elastic(disabled=True)
65
- print("ElasticSearch is enabled but not reachable, logs will be saved locally if needed.")
44
+ print("You're running locally, so your task won't be started in the platform and logs won't be saved")
66
45
  else:
67
- self.es = Elastic(disabled=True)
68
-
46
+ self.started_local = False
47
+ self.customer_db, self.task_id, self.run_id, self.data_interface_id = sys.argv[1:5]
48
+ self.mysql_enabled = True
49
+ self.elastic_enabled = True
50
+ # If the task is started locally, the parameters should be set locally
51
+
52
+ print(f"Run ID: {self.run_id}, Data Interface ID: {self.data_interface_id}, Task ID: {self.task_id}, Started Locally: {self.started_local}")
53
+ self.brynq = BrynQ()
54
+ self.email_after_errors = email_after_errors
69
55
  # Set up local log directory
70
56
  self.local_log_dir = 'local_logs'
71
57
  os.makedirs(self.local_log_dir, exist_ok=True)
72
58
 
73
- # Process local logs if services are now reachable
74
- if self.mysql_enabled and not self.mysql_unreachable:
75
- self._process_local_mysql_logs()
76
- if self.elastic_enabled and not self.elastic_unreachable:
77
- self._process_local_elastic_logs()
78
-
79
59
  try:
80
- self.email_after_errors = email_after_errors
81
60
  self.customer_db = os.getenv("MYSQL_DATABASE")
82
61
  self.customer = os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_')
83
62
  self.partner_id = os.getenv('PARTNER_ID').lower().replace(' ', '_') if os.getenv('PARTNER_ID') else 'brynq'
84
63
  self.loglevel = loglevel
85
64
  self.started_at = datetime.datetime.now()
86
- # If the task is started via the task_scheduler, the following 3 parameters will be passed by the scheduler.
87
- # The distinction between local and non local is made because the scheduler usually sets the scheduler_log table entry and run_id. When running locally, the tasks should do this itself.
88
- if len(sys.argv[1:4]) > 0:
89
- self.started_local = False
90
- self.customer_db, self.task_id, self.run_id = sys.argv[1:4]
91
- # If the task is started locally, the parameters should be set locally
92
- else:
93
- self.started_local = True
94
- self.run_id = int(round(time.time() * 100000))
95
- self.task_id = task_id
96
- print(self.task_id, self.run_id)
97
65
  self.error_count = 0
98
66
 
99
67
  # Check if the log tables exists in the customer database1. If not, create them
100
68
  # Mysql throws a warning when a table already exists. We don't care so we ignore warnings. (not exceptions!)
101
69
  warnings.filterwarnings('ignore')
102
70
 
103
- # Creates Elasticsearch index and data view if not exists
104
- if self.elastic_enabled:
105
- self.es_index = f"task_execution_log_{self.customer_db}"
106
- self.es.create_index(index_name=self.es_index)
107
- self.es.create_data_view(space_name='interfaces', view_name=f'task_execution_log_{self.customer_db}', name=f'Task execution log {self.customer_db}', time_field='started_at')
71
+ # Initialize MySQL
72
+ self.mysql_reachable = True
73
+ try:
74
+ self.mysql = MySQL()
75
+ self.mysql.ping()
76
+ except Exception as e:
77
+ self.mysql_reachable = False
78
+ self.mysql = None
79
+ print("MySQL is not reachable, logs will be saved locally if needed.")
108
80
 
109
81
  # Start the task and setup the data in the database
110
- if self.mysql_enabled:
82
+ if self.mysql_enabled and self.mysql_reachable:
111
83
  self.customer_id = self.mysql.raw_query(f'SELECT id FROM sc.customers WHERE dbname = \'{self.customer_db}\'')[0][0]
112
- # Check if the task is started on schedule or manual. store in a variable to use later in the script
113
- self.task_manual_started = self._check_if_task_manual_started()
84
+ self._process_local_mysql_logs()
114
85
  self._start_task()
115
- else:
116
- self.task_manual_started = True
117
86
  except Exception as e:
118
87
  self.error_handling(e)
119
88
 
@@ -147,22 +116,6 @@ class TaskScheduler(BrynQ):
147
116
  :return: error (str) or response of mysql
148
117
  """
149
118
  warnings.warn("Execution steps are deprecated, please stop calling this method. It does nothing anymore", DeprecationWarning)
150
- return
151
-
152
- def _check_if_task_manual_started(self):
153
- """
154
- Check if the task manual is started of on schedule. If it's manual started, that's important for the variables in the db_variables function.
155
- In that case the dynamic variables should be used instead of the static ones
156
- :return: True of False
157
- """
158
- # without logging is only possible during dev, so this is always manual
159
- response = self.mysql.select('task_scheduler', 'run_instant', f'WHERE id = {self.task_id}')[0][0]
160
- if response == 1:
161
- # Reset the 1 back to 0 before sending the result
162
- self.mysql.update('task_scheduler', ['run_instant'], [0], 'WHERE `id` = {}'.format(self.task_id))
163
- return True
164
- else:
165
- return False
166
119
 
167
120
  def _start_task(self):
168
121
  """
@@ -173,34 +126,21 @@ class TaskScheduler(BrynQ):
173
126
  if self.started_local:
174
127
  self.mysql.raw_query(f"INSERT INTO `task_scheduler_log` (reload_id, task_id, reload_status, started_at, finished_at) VALUES ({self.run_id}, {self.task_id}, 'Running', '{self.started_at}', null)", insert=True)
175
128
 
176
- self.mysql.update('task_scheduler', ['status', 'step_nr'], ['RUNNING', 1], 'WHERE `id` = {}'.format(self.task_id))
129
+ self.mysql.update('task_scheduler', ['status', 'step_nr'], ['RUNNING', 1], f'WHERE `id` = {self.task_id}')
177
130
 
178
- def db_variable(self, variable_name: str, default_value_if_temp_is_empty: bool = False):
131
+ def db_variable(self, variable_name: str):
179
132
  """
180
- Get a value from the task_variables table corresponding with the given name. If the task is manually started
133
+ Get a value from the task_variables table corresponding with the given name. If temp value is filled, it will
181
134
  (run_instant = 1), then the temp_value will be returned. This is to give the possibility for users in the frontend to run
182
135
  a task once manual with other values then normal without overwriting the normal values.
183
136
  :param variable_name: the name of the variable
184
- :param default_value_if_temp_is_empty: bool to determine whether default value should be used if temp value is empty when manually started
185
137
  :return: the value of the given variable.
186
138
  """
187
- if self.mysql_enabled:
188
- if self.task_manual_started or self.started_local:
189
- response = self.mysql.select('task_variables', 'temp_value, value',
190
- f'WHERE name = \'{variable_name}\' AND task_id = {self.task_id}')
191
- else:
192
- response = self.mysql.select('task_variables', 'value',
193
- f'WHERE name = \'{variable_name}\' AND task_id = {self.task_id}')
194
- if len(response) == 0:
195
- raise Exception(f'Variable with name \'{variable_name}\' does not exist')
196
- else:
197
- value = response[0][0]
198
- if value is None and default_value_if_temp_is_empty is True and len(response[0]) > 0:
199
- value = response[0][1]
200
- return value
201
- else:
202
- value: str = input(f'Your MYSQL connection is not defined, enter the value for the variable {variable_name}: ')
203
- return value
139
+ warnings.deprecated("Use self.brynq.interfaces.get_variables() instead")
140
+
141
+ variable = self.brynq.interfaces.get_variables(variable_name=variable_name)
142
+
143
+ return variable
204
144
 
205
145
  def write_execution_log(self, message: str, data, loglevel: str = 'INFO', full_extract: bool = False):
206
146
  """
@@ -214,101 +154,16 @@ class TaskScheduler(BrynQ):
214
154
  # Validate if the provided loglevel is valid
215
155
  allowed_loglevels = ['DEBUG', 'INFO', 'ERROR', 'CRITICAL']
216
156
  if loglevel not in allowed_loglevels:
217
- raise Exception('You\'ve entered a not allowed loglevel. Choose one of: {}'.format(allowed_loglevels))
157
+ raise ValueError(f"You\'ve entered a not allowed loglevel. Choose one of: {','.join(allowed_loglevels)}")
218
158
 
219
159
  # Get the linenumber from where the logline is executed.
220
160
  file_name, line_number, function_name = self.__get_caller_info()
221
-
222
161
  print('{} at line: {}'.format(message, line_number))
223
162
 
224
163
  # Count the errors for relevant log levels
225
164
  if loglevel == 'ERROR' or loglevel == 'CRITICAL':
226
165
  self.error_count += 1
227
166
 
228
- if self.elastic_enabled:
229
- # For Elastic, we need to have the data in JSON format. Handling different data types and preparing extra payload information based on the data type
230
- # If the data is just a series, count rows, columns and cells
231
- # Put everything together in the payload for ElasticSearch and send it
232
- payload = {
233
- 'task_id': self.task_id,
234
- 'reload_id': self.run_id,
235
- 'started_at': datetime.datetime.now().isoformat(),
236
- 'partner_id': self.partner_id,
237
- 'customer': self.customer,
238
- 'file_name': file_name,
239
- 'function_name': function_name,
240
- 'line_number': line_number,
241
- 'task_loglevel': self.loglevel,
242
- 'line_loglevel': loglevel,
243
- 'message': message
244
- }
245
-
246
- if isinstance(data, pd.Series):
247
- dataframe = pd.DataFrame(data).T
248
- extra_payload = {
249
- 'rows': len(dataframe),
250
- 'columns': len(dataframe.columns),
251
- 'cells': len(dataframe) * len(dataframe.columns),
252
- }
253
- if not full_extract:
254
- extra_payload['payload'] = dataframe.to_json(orient='records')
255
- # If the data is a list, count rows, columns and cells
256
- elif isinstance(data, dict):
257
- records = self.__count_keys(data)
258
- extra_payload = {
259
- 'rows': 1,
260
- 'columns': records,
261
- 'cells': records,
262
- }
263
- if not full_extract:
264
- extra_payload['payload'] = data
265
- elif isinstance(data, pd.DataFrame):
266
- extra_payload = {
267
- 'rows': len(data),
268
- 'columns': len(data.columns),
269
- 'cells': len(data) * len(data.columns),
270
- }
271
- if not full_extract:
272
- extra_payload['payload'] = data.to_json(orient='records')
273
- # If the data is a response from an URL request, also store all the information about the URL request.
274
- elif isinstance(data, requests.Response):
275
- records = 1
276
- if data.request.body is not None:
277
- records = self.__count_keys(json.loads(data.request.body))
278
- if isinstance(data.request.body, bytes):
279
- data.request.body = data.request.body.decode('utf-8')
280
- extra_payload = {
281
- 'response': data.text,
282
- 'status_code': data.status_code,
283
- 'url': data.url,
284
- 'method': data.request.method,
285
- 'rows': 1,
286
- 'columns': records,
287
- 'cells': records,
288
- }
289
- if not full_extract:
290
- extra_payload['payload'] = data.request.body
291
- elif data is None:
292
- extra_payload = {}
293
- else:
294
- extra_payload = {
295
- 'data_type': str(type(data)),
296
- }
297
- if not full_extract:
298
- extra_payload['payload'] = data
299
-
300
- # Modify payload based on 'full_load' flag
301
- if data is not None and full_extract is True:
302
- extra_payload['full_load'] = True
303
- elif data is not None and full_extract is False:
304
- extra_payload['full_load'] = False
305
-
306
- payload.update(extra_payload)
307
- if not self.elastic_unreachable:
308
- self.es.post_document(index_name=self.es_index, document=payload)
309
- else:
310
- self._save_log_locally(payload, 'elastic')
311
-
312
167
  # Write the logline to the MYSQL database, depends on the chosen loglevel in the task
313
168
  if self.mysql_enabled:
314
169
  mysql_log_data = {
@@ -319,18 +174,14 @@ class TaskScheduler(BrynQ):
319
174
  'line_number': line_number,
320
175
  'message': re.sub("[']", '', message)
321
176
  }
322
- if not self.mysql_unreachable:
177
+ if self.mysql_reachable:
323
178
  try:
324
179
  query = f"INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({mysql_log_data['reload_id']}, {mysql_log_data['task_id']}, '{mysql_log_data['log_level']}', '{mysql_log_data['created_at']}', {mysql_log_data['line_number']}, '{mysql_log_data['message']}')"
325
180
  if self.loglevel == 'DEBUG' or (self.loglevel == 'INFO' and loglevel != 'DEBUG') or (self.loglevel == 'ERROR' and loglevel in ['ERROR', 'CRITICAL']) or (self.loglevel == 'CRITICAL' and loglevel == 'CRITICAL'):
326
181
  self.mysql.raw_query(query, insert=True)
327
- except pymysql.err.OperationalError as e:
328
- print(f"MySQL connection lost during logging: {e}")
329
- self.mysql_unreachable = True
330
- self._save_log_locally(mysql_log_data, 'mysql')
331
- except pymysql.err.InterfaceError as e:
332
- print(f"MySQL connection closed: {e}")
333
- self.mysql_unreachable = True
182
+ except (pymysql.err.OperationalError, pymysql.err.InterfaceError) as e:
183
+ print(f"MySQL connection lost or closed during logging: {e}")
184
+ self.mysql_reachable = False
334
185
  self._save_log_locally(mysql_log_data, 'mysql')
335
186
  except Exception as e:
336
187
  print(f"Error during logging to MySQL: {e}")
@@ -346,7 +197,6 @@ class TaskScheduler(BrynQ):
346
197
  """
347
198
  # Update the step number in the task_scheduler table
348
199
  warnings.warn("Execution steps are deprecated, please stop calling this method. It does nothing anymore", DeprecationWarning)
349
- return
350
200
 
351
201
  def error_handling(self, e: Exception, breaking=True, send_to_teams=False):
352
202
  """
@@ -366,58 +216,60 @@ class TaskScheduler(BrynQ):
366
216
  exc_type, exc_obj, exc_tb = sys.exc_info()
367
217
  error = str(e)[:400].replace('\'', '').replace('\"', '') + ' | Line: {}'.format(exc_tb.tb_lineno)
368
218
 
369
- if self.elastic_enabled:
370
- # Preparing the primary payload with error details for upload to elastic and send it
371
- payload = {
372
- 'task_id': self.task_id,
373
- 'reload_id': self.run_id,
374
- 'started_at': datetime.datetime.now().isoformat(),
375
- 'partner_id': self.partner_id,
376
- 'customer': self.customer,
377
- 'file_name': file_name,
378
- 'function_name': function_name,
379
- 'line_number': line_number,
380
- 'task_loglevel': self.loglevel,
381
- 'line_loglevel': 'CRITICAL',
382
- 'message': str(e),
383
- 'traceback': traceback.format_exc()
384
- }
385
- self.es.post_document(index_name=self.es_index, document=payload)
386
-
387
219
  if self.mysql_enabled:
388
- self.error_count += 1
389
- # Get scheduler task details for logging
390
- task_details = \
391
- self.mysql.select('task_scheduler, data_interfaces', 'data_interfaces.docker_image, data_interfaces.runfile_path', 'WHERE task_scheduler.data_interface_id = data_interfaces.id AND task_scheduler.id = {}'.format(self.task_id))[0]
392
- taskname = task_details[0]
393
- customer = task_details[1].split('/')[-1].split('.')[0]
394
- now = datetime.datetime.now()
395
-
396
- # Log to log table in the database
397
- if self.mysql_enabled:
398
- query = "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, 'CRITICAL', '{}', {}, '{}')".format(self.run_id, self.task_id, now, exc_tb.tb_lineno, error)
220
+ try:
221
+ now = datetime.datetime.now()
222
+ # Log to log table in the database
223
+ mysql_log_data = {
224
+ 'reload_id': self.run_id,
225
+ 'task_id': self.task_id,
226
+ 'log_level': 'CRITICAL',
227
+ 'created_at': now,
228
+ 'line_number': exc_tb.tb_lineno,
229
+ 'message': error
230
+ }
231
+ self.error_count += 1
232
+ # Get scheduler task details for logging
233
+ task_details = \
234
+ self.mysql.select('task_scheduler, data_interfaces', 'data_interfaces.docker_image, data_interfaces.runfile_path', 'WHERE task_scheduler.data_interface_id = data_interfaces.id AND task_scheduler.id = {}'.format(self.task_id))[0]
235
+ taskname = task_details[0]
236
+ customer = task_details[1].split('/')[-1].split('.')[0]
237
+
238
+ query = f"INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({mysql_log_data['reload_id']}, {mysql_log_data['task_id']}, '{mysql_log_data['log_level']}', '{mysql_log_data['created_at']}', {mysql_log_data['line_number']}, '{mysql_log_data['message']}')"
399
239
  self.mysql.raw_query(query, insert=True)
400
- if send_to_teams:
401
- Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
402
- if breaking:
403
- # Set scheduler status to failed
404
- self.mysql.update('task_scheduler', ['status', 'last_reload', 'last_error_message', 'step_nr'],
405
- ['IDLE', now, 'Failed', 0],
406
- 'WHERE `id` = {}'.format(self.task_id))
407
-
408
- self.mysql.update(table='task_scheduler_log',
409
- columns=['reload_status', 'finished_at'],
410
- values=['Failed', f'{now}'],
411
- filter=f'WHERE `reload_id` = {self.run_id}')
412
- if self.email_after_errors:
413
- self.email_errors(failed=True)
414
- # Remove the temp values from the variables table
415
- self.mysql.raw_query(f'UPDATE `task_variables` SET temp_value = null WHERE task_id = {self.task_id}', insert=True)
416
240
 
417
- # Start the chained tasks if it there are tasks which should start if this one is failed
418
- self.start_chained_tasks(finished_task_status='FAILED')
241
+ if send_to_teams:
242
+ Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
243
+ if breaking:
244
+ # Set scheduler status to failed
245
+ self.mysql.update('task_scheduler', ['status', 'last_reload', 'last_error_message', 'step_nr'],
246
+ ['IDLE', now, 'Failed', 0],
247
+ f'WHERE `id` = {self.task_id}')
248
+
249
+ self.mysql.update(table='task_scheduler_log',
250
+ columns=['reload_status', 'finished_at'],
251
+ values=['Failed', f'{now}'],
252
+ filter=f'WHERE `reload_id` = {self.run_id}')
253
+ if self.email_after_errors:
254
+ self.email_errors(failed=True)
255
+ # Remove the temp values from the variables table
256
+ self.mysql.raw_query(f'UPDATE `task_variables` SET temp_value = null WHERE task_id = {self.task_id}', insert=True)
257
+
258
+ # Start the chained tasks if it there are tasks which should start if this one is failed
259
+ self.start_chained_tasks(finished_task_status='FAILED')
260
+ except (pymysql.err.OperationalError, pymysql.err.InterfaceError) as e:
261
+ print(f"MySQL connection lost or closed during logging: {e}")
262
+ self.mysql_reachable = False
263
+ self._save_log_locally(mysql_log_data, 'mysql')
264
+ except Exception as e:
265
+ print(f"Error during logging to MySQL: {e}")
266
+ self._save_log_locally(mysql_log_data, 'mysql')
419
267
 
420
- raise Exception(error)
268
+ # if breaking, reraise for clear traceback (local development) or just print if not breaking
269
+ if breaking:
270
+ raise e
271
+ else:
272
+ print(error)
421
273
 
422
274
  def finish_task(self, reload_instant=False, log_limit: Optional[int] = 10000, log_date_limit: datetime.date = None):
423
275
  """
@@ -475,7 +327,8 @@ class TaskScheduler(BrynQ):
475
327
 
476
328
  def start_chained_tasks(self, finished_task_status: str):
477
329
  if self.mysql_enabled:
478
- filter = f'WHERE start_after_task_id = \'{self.task_id}\' AND start_after_preceding_task = \'{finished_task_status}\''
330
+ # only start chained tasks when trigger is on other task, otherwise this has changed in the db
331
+ filter = f'WHERE start_after_task_id = \'{self.task_id}\' AND start_after_preceding_task = \'{finished_task_status}\' AND task_type = \'OTHER_TASK\''
479
332
  response = self.mysql.select(table='task_scheduler', selection='id', filter=filter)
480
333
  if len(response) > 0:
481
334
  tasks_to_run = [str(task[0]) for task in response]
@@ -485,7 +338,7 @@ class TaskScheduler(BrynQ):
485
338
 
486
339
  def email_errors(self, failed):
487
340
  # The mails to email to should be stored in the task_variables table with the variable email_errors_to
488
- email_variable = self.db_variable('email_errors_to')
341
+ email_variable = self.brynq.interfaces.get_variables(variable_name='email_errors_to')
489
342
  if email_variable is not None:
490
343
  email_to = email_variable.split(',')
491
344
  if isinstance(email_to, list):
@@ -546,17 +399,24 @@ class TaskScheduler(BrynQ):
546
399
  except Exception as e:
547
400
  print(f"Error processing MySQL log file {log_file}: {e}")
548
401
 
549
- def _process_local_elastic_logs(self):
550
- elastic_log_files = [f for f in os.listdir(self.local_log_dir) if f.startswith('elastic_log_')]
551
- for log_file in elastic_log_files:
552
- log_file_path = os.path.join(self.local_log_dir, log_file)
553
- try:
554
- with open(log_file_path, 'r') as f:
555
- logs = json.load(f)
556
- # Process logs
557
- for log_entry in logs:
558
- self.es.post_document(index_name=self.es_index, document=log_entry)
559
- # Remove the log file after processing
560
- os.remove(log_file_path)
561
- except Exception as e:
562
- print(f"Error processing ElasticSearch log file {log_file}: {e}")
402
+ def _write_log_to_mysql(self, log_entry):
403
+ """
404
+ Insert a log entry dictionary into the task_execution_log table in MySQL.
405
+ :param log_entry: dict with keys matching the columns of task_execution_log
406
+ """
407
+ try:
408
+ query = (
409
+ "INSERT INTO `task_execution_log` "
410
+ "(reload_id, task_id, log_level, created_at, line_number, message) "
411
+ "VALUES ({reload_id}, {task_id}, '{log_level}', '{created_at}', {line_number}, '{message}')"
412
+ ).format(
413
+ reload_id=log_entry['reload_id'],
414
+ task_id=log_entry['task_id'],
415
+ log_level=log_entry['log_level'],
416
+ created_at=log_entry['created_at'],
417
+ line_number=log_entry['line_number'],
418
+ message=str(log_entry['message']).replace("'", "")
419
+ )
420
+ self.mysql.raw_query(query, insert=True)
421
+ except Exception as e:
422
+ print(f"Error writing log entry to MySQL: {e}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.0
2
2
  Name: brynq-sdk-task-scheduler
3
- Version: 3.0.10
3
+ Version: 4.0.1
4
4
  Summary: Code to execute tasks in BrynQ.com with the task scheduler
5
5
  Home-page: UNKNOWN
6
6
  Author: BrynQ
@@ -1,5 +1,5 @@
1
- brynq-sdk-brynq<4,>=2
1
+ brynq-sdk-brynq<5,>=4
2
2
  brynq-sdk-functions<3,>=2
3
- brynq-sdk-mysql<3,>=2
3
+ brynq-sdk-mysql<4,>=3
4
4
  brynq-sdk-mandrill<3,>=2
5
5
  brynq-sdk-elastic<4,>=3
@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
2
2
 
3
3
  setup(
4
4
  name='brynq_sdk_task_scheduler',
5
- version='3.0.10',
5
+ version='4.0.1',
6
6
  description='Code to execute tasks in BrynQ.com with the task scheduler',
7
7
  long_description='Code to execute tasks in the BrynQ.com platform with the task scheduler',
8
8
  author='BrynQ',
@@ -10,9 +10,9 @@ setup(
10
10
  packages=find_namespace_packages(include=['brynq_sdk*']),
11
11
  license='BrynQ License',
12
12
  install_requires=[
13
- 'brynq-sdk-brynq>=2,<4',
13
+ 'brynq-sdk-brynq>=4,<5',
14
14
  'brynq-sdk-functions>=2,<3',
15
- 'brynq-sdk-mysql>=2,<3',
15
+ 'brynq-sdk-mysql>=3,<4',
16
16
  'brynq-sdk-mandrill>=2,<3',
17
17
  'brynq-sdk-elastic>=3,<4'
18
18
  ],