brynq-sdk-task-scheduler 3.0.10__tar.gz → 4.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.0
2
2
  Name: brynq_sdk_task_scheduler
3
- Version: 3.0.10
3
+ Version: 4.0.0
4
4
  Summary: Code to execute tasks in BrynQ.com with the task scheduler
5
5
  Home-page: UNKNOWN
6
6
  Author: BrynQ
@@ -16,104 +16,76 @@ from brynq_sdk_elastic import Elastic
16
16
  from brynq_sdk_brynq import BrynQ
17
17
  import warnings
18
18
  import re
19
- LOGGING_OPTIONS = Literal['MYSQL', 'ELASTIC']
20
19
 
21
20
 
22
- class TaskScheduler(BrynQ):
21
+ class TaskScheduler:
23
22
 
24
- def __init__(self, task_id: int = None, loglevel: str = 'INFO', email_after_errors: bool = False, logging: List[LOGGING_OPTIONS] | None = ["MYSQL", "ELASTIC"]):
23
+ def __init__(self, data_interface_id: int = None, loglevel: str = 'INFO', email_after_errors: bool = False):
25
24
  """
26
25
  The TaskScheduler is responsible for the logging to the database. Based on this logging, the next reload will
27
26
  start or not and warning will be given or not
28
- :param task_id: The ID from the task as saved in the task_scheduler table in the customer database
27
+ :param data_interface_id: The ID from the task as saved in the task_scheduler table in the customer database
29
28
  :param email_after_errors: a True or False value. When True, there will be send an email to a contactperson of the customer (as given in the database) with the number of errors
30
29
  :param loglevel: Chose on which level you want to store the logs. Default is INFO. that means that a logline
31
30
  :param disable_logging: If the interface is started from a local instance, logs will not be stored by default. If this is set to True, the logs will be stored in the database
32
31
  with level DEBUG not is stored
33
32
  """
34
- super().__init__()
35
- try:
36
- self.mysql_enabled = 'MYSQL' in logging
37
- self.elastic_enabled = 'ELASTIC' in logging
38
- except Exception as e:
39
- print("Error parsing logging options, disabling logging. Error is: " + str(e))
33
+ # If the task is started via the task_scheduler, the following parameters will be passed by the scheduler.
34
+ # The distinction between local and non local is made because the scheduler usually sets the scheduler_log table entry and run_id. When running locally, the tasks will not log anything by default.
35
+ if any(flag in sys.argv[0].split('/') for flag in ['opt', 'home']):
36
+ self.started_local = True
37
+ self.run_id = int(round(time.time() * 100000))
38
+ self.task_id = None
39
+ self.data_interface_id = data_interface_id
40
+ # set it into environment variables for reuse in source and target systems
41
+ os.environ['DATA_INTERFACE_ID'] = str(self.data_interface_id)
40
42
  self.mysql_enabled = False
41
43
  self.elastic_enabled = False
42
-
43
- # Initialize MySQL
44
- self.mysql_unreachable = False
45
- # if self.mysql_enabled:
46
- try:
47
- self.mysql = MySQL()
48
- self.mysql.ping()
49
- except Exception as e:
50
- self.mysql_unreachable = True
51
- self.mysql = None
52
- print("MySQL is enabled but not reachable, logs will be saved locally if needed.")
53
- # else:
54
- # self.mysql = None
55
-
56
- # Initialize ElasticSearch
57
- self.elastic_unreachable = False
58
- if self.elastic_enabled:
59
- try:
60
- self.es = Elastic()
61
- self.es.get_health()
62
- except Exception as e:
63
- self.elastic_unreachable = True
64
- self.es = Elastic(disabled=True)
65
- print("ElasticSearch is enabled but not reachable, logs will be saved locally if needed.")
44
+ print("You're running locally, so your task won't be started in the platform and logs won't be saved")
66
45
  else:
67
- self.es = Elastic(disabled=True)
68
-
46
+ self.started_local = False
47
+ self.customer_db, self.task_id, self.run_id, self.data_interface_id = sys.argv[1:5]
48
+ self.mysql_enabled = True
49
+ self.elastic_enabled = True
50
+ # If the task is started locally, the parameters should be set locally
51
+
52
+ print(f"Run ID: {self.run_id}, Data Interface ID: {self.data_interface_id}, Task ID: {self.task_id}, Started Locally: {self.started_local}")
53
+ self.brynq = BrynQ()
54
+ self.email_after_errors = email_after_errors
69
55
  # Set up local log directory
70
56
  self.local_log_dir = 'local_logs'
71
57
  os.makedirs(self.local_log_dir, exist_ok=True)
72
58
 
73
59
  # Process local logs if services are now reachable
74
- if self.mysql_enabled and not self.mysql_unreachable:
60
+ if self.mysql_enabled and self.mysql_reachable:
75
61
  self._process_local_mysql_logs()
76
- if self.elastic_enabled and not self.elastic_unreachable:
77
- self._process_local_elastic_logs()
78
62
 
79
63
  try:
80
- self.email_after_errors = email_after_errors
81
64
  self.customer_db = os.getenv("MYSQL_DATABASE")
82
65
  self.customer = os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_')
83
66
  self.partner_id = os.getenv('PARTNER_ID').lower().replace(' ', '_') if os.getenv('PARTNER_ID') else 'brynq'
84
67
  self.loglevel = loglevel
85
68
  self.started_at = datetime.datetime.now()
86
- # If the task is started via the task_scheduler, the following 3 parameters will be passed by the scheduler.
87
- # The distinction between local and non local is made because the scheduler usually sets the scheduler_log table entry and run_id. When running locally, the tasks should do this itself.
88
- if len(sys.argv[1:4]) > 0:
89
- self.started_local = False
90
- self.customer_db, self.task_id, self.run_id = sys.argv[1:4]
91
- # If the task is started locally, the parameters should be set locally
92
- else:
93
- self.started_local = True
94
- self.run_id = int(round(time.time() * 100000))
95
- self.task_id = task_id
96
- print(self.task_id, self.run_id)
97
69
  self.error_count = 0
98
70
 
99
71
  # Check if the log tables exists in the customer database1. If not, create them
100
72
  # Mysql throws a warning when a table already exists. We don't care so we ignore warnings. (not exceptions!)
101
73
  warnings.filterwarnings('ignore')
102
74
 
103
- # Creates Elasticsearch index and data view if not exists
104
- if self.elastic_enabled:
105
- self.es_index = f"task_execution_log_{self.customer_db}"
106
- self.es.create_index(index_name=self.es_index)
107
- self.es.create_data_view(space_name='interfaces', view_name=f'task_execution_log_{self.customer_db}', name=f'Task execution log {self.customer_db}', time_field='started_at')
75
+ # Initialize MySQL
76
+ self.mysql_reachable = True
77
+ try:
78
+ self.mysql = MySQL()
79
+ self.mysql.ping()
80
+ except Exception as e:
81
+ self.mysql_reachable = False
82
+ self.mysql = None
83
+ print("MySQL is enabled but not reachable, logs will be saved locally if needed.")
108
84
 
109
85
  # Start the task and setup the data in the database
110
- if self.mysql_enabled:
86
+ if self.mysql_enabled and self.mysql_reachable:
111
87
  self.customer_id = self.mysql.raw_query(f'SELECT id FROM sc.customers WHERE dbname = \'{self.customer_db}\'')[0][0]
112
- # Check if the task is started on schedule or manual. store in a variable to use later in the script
113
- self.task_manual_started = self._check_if_task_manual_started()
114
88
  self._start_task()
115
- else:
116
- self.task_manual_started = True
117
89
  except Exception as e:
118
90
  self.error_handling(e)
119
91
 
@@ -147,22 +119,6 @@ class TaskScheduler(BrynQ):
147
119
  :return: error (str) or response of mysql
148
120
  """
149
121
  warnings.warn("Execution steps are deprecated, please stop calling this method. It does nothing anymore", DeprecationWarning)
150
- return
151
-
152
- def _check_if_task_manual_started(self):
153
- """
154
- Check if the task manual is started of on schedule. If it's manual started, that's important for the variables in the db_variables function.
155
- In that case the dynamic variables should be used instead of the static ones
156
- :return: True of False
157
- """
158
- # without logging is only possible during dev, so this is always manual
159
- response = self.mysql.select('task_scheduler', 'run_instant', f'WHERE id = {self.task_id}')[0][0]
160
- if response == 1:
161
- # Reset the 1 back to 0 before sending the result
162
- self.mysql.update('task_scheduler', ['run_instant'], [0], 'WHERE `id` = {}'.format(self.task_id))
163
- return True
164
- else:
165
- return False
166
122
 
167
123
  def _start_task(self):
168
124
  """
@@ -173,34 +129,21 @@ class TaskScheduler(BrynQ):
173
129
  if self.started_local:
174
130
  self.mysql.raw_query(f"INSERT INTO `task_scheduler_log` (reload_id, task_id, reload_status, started_at, finished_at) VALUES ({self.run_id}, {self.task_id}, 'Running', '{self.started_at}', null)", insert=True)
175
131
 
176
- self.mysql.update('task_scheduler', ['status', 'step_nr'], ['RUNNING', 1], 'WHERE `id` = {}'.format(self.task_id))
132
+ self.mysql.update('task_scheduler', ['status', 'step_nr'], ['RUNNING', 1], f'WHERE `id` = {self.task_id}')
177
133
 
178
- def db_variable(self, variable_name: str, default_value_if_temp_is_empty: bool = False):
134
+ def db_variable(self, variable_name: str):
179
135
  """
180
- Get a value from the task_variables table corresponding with the given name. If the task is manually started
136
+ Get a value from the task_variables table corresponding with the given name. If temp value is filled, it will
181
137
  (run_instant = 1), then the temp_value will be returned. This is to give the possibility for users in the frontend to run
182
138
  a task once manual with other values then normal without overwriting the normal values.
183
139
  :param variable_name: the name of the variable
184
- :param default_value_if_temp_is_empty: bool to determine whether default value should be used if temp value is empty when manually started
185
140
  :return: the value of the given variable.
186
141
  """
187
- if self.mysql_enabled:
188
- if self.task_manual_started or self.started_local:
189
- response = self.mysql.select('task_variables', 'temp_value, value',
190
- f'WHERE name = \'{variable_name}\' AND task_id = {self.task_id}')
191
- else:
192
- response = self.mysql.select('task_variables', 'value',
193
- f'WHERE name = \'{variable_name}\' AND task_id = {self.task_id}')
194
- if len(response) == 0:
195
- raise Exception(f'Variable with name \'{variable_name}\' does not exist')
196
- else:
197
- value = response[0][0]
198
- if value is None and default_value_if_temp_is_empty is True and len(response[0]) > 0:
199
- value = response[0][1]
200
- return value
201
- else:
202
- value: str = input(f'Your MYSQL connection is not defined, enter the value for the variable {variable_name}: ')
203
- return value
142
+ warnings.deprecated("Use self.brynq.interfaces.get_variables() instead")
143
+
144
+ variable = self.brynq.interfaces.get_variables(variable_name=variable_name)
145
+
146
+ return variable
204
147
 
205
148
  def write_execution_log(self, message: str, data, loglevel: str = 'INFO', full_extract: bool = False):
206
149
  """
@@ -214,101 +157,16 @@ class TaskScheduler(BrynQ):
214
157
  # Validate if the provided loglevel is valid
215
158
  allowed_loglevels = ['DEBUG', 'INFO', 'ERROR', 'CRITICAL']
216
159
  if loglevel not in allowed_loglevels:
217
- raise Exception('You\'ve entered a not allowed loglevel. Choose one of: {}'.format(allowed_loglevels))
160
+ raise ValueError(f"You\'ve entered a not allowed loglevel. Choose one of: {','.join(allowed_loglevels)}")
218
161
 
219
162
  # Get the linenumber from where the logline is executed.
220
163
  file_name, line_number, function_name = self.__get_caller_info()
221
-
222
164
  print('{} at line: {}'.format(message, line_number))
223
165
 
224
166
  # Count the errors for relevant log levels
225
167
  if loglevel == 'ERROR' or loglevel == 'CRITICAL':
226
168
  self.error_count += 1
227
169
 
228
- if self.elastic_enabled:
229
- # For Elastic, we need to have the data in JSON format. Handling different data types and preparing extra payload information based on the data type
230
- # If the data is just a series, count rows, columns and cells
231
- # Put everything together in the payload for ElasticSearch and send it
232
- payload = {
233
- 'task_id': self.task_id,
234
- 'reload_id': self.run_id,
235
- 'started_at': datetime.datetime.now().isoformat(),
236
- 'partner_id': self.partner_id,
237
- 'customer': self.customer,
238
- 'file_name': file_name,
239
- 'function_name': function_name,
240
- 'line_number': line_number,
241
- 'task_loglevel': self.loglevel,
242
- 'line_loglevel': loglevel,
243
- 'message': message
244
- }
245
-
246
- if isinstance(data, pd.Series):
247
- dataframe = pd.DataFrame(data).T
248
- extra_payload = {
249
- 'rows': len(dataframe),
250
- 'columns': len(dataframe.columns),
251
- 'cells': len(dataframe) * len(dataframe.columns),
252
- }
253
- if not full_extract:
254
- extra_payload['payload'] = dataframe.to_json(orient='records')
255
- # If the data is a list, count rows, columns and cells
256
- elif isinstance(data, dict):
257
- records = self.__count_keys(data)
258
- extra_payload = {
259
- 'rows': 1,
260
- 'columns': records,
261
- 'cells': records,
262
- }
263
- if not full_extract:
264
- extra_payload['payload'] = data
265
- elif isinstance(data, pd.DataFrame):
266
- extra_payload = {
267
- 'rows': len(data),
268
- 'columns': len(data.columns),
269
- 'cells': len(data) * len(data.columns),
270
- }
271
- if not full_extract:
272
- extra_payload['payload'] = data.to_json(orient='records')
273
- # If the data is a response from an URL request, also store all the information about the URL request.
274
- elif isinstance(data, requests.Response):
275
- records = 1
276
- if data.request.body is not None:
277
- records = self.__count_keys(json.loads(data.request.body))
278
- if isinstance(data.request.body, bytes):
279
- data.request.body = data.request.body.decode('utf-8')
280
- extra_payload = {
281
- 'response': data.text,
282
- 'status_code': data.status_code,
283
- 'url': data.url,
284
- 'method': data.request.method,
285
- 'rows': 1,
286
- 'columns': records,
287
- 'cells': records,
288
- }
289
- if not full_extract:
290
- extra_payload['payload'] = data.request.body
291
- elif data is None:
292
- extra_payload = {}
293
- else:
294
- extra_payload = {
295
- 'data_type': str(type(data)),
296
- }
297
- if not full_extract:
298
- extra_payload['payload'] = data
299
-
300
- # Modify payload based on 'full_load' flag
301
- if data is not None and full_extract is True:
302
- extra_payload['full_load'] = True
303
- elif data is not None and full_extract is False:
304
- extra_payload['full_load'] = False
305
-
306
- payload.update(extra_payload)
307
- if not self.elastic_unreachable:
308
- self.es.post_document(index_name=self.es_index, document=payload)
309
- else:
310
- self._save_log_locally(payload, 'elastic')
311
-
312
170
  # Write the logline to the MYSQL database, depends on the chosen loglevel in the task
313
171
  if self.mysql_enabled:
314
172
  mysql_log_data = {
@@ -319,18 +177,14 @@ class TaskScheduler(BrynQ):
319
177
  'line_number': line_number,
320
178
  'message': re.sub("[']", '', message)
321
179
  }
322
- if not self.mysql_unreachable:
180
+ if self.mysql_reachable:
323
181
  try:
324
182
  query = f"INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({mysql_log_data['reload_id']}, {mysql_log_data['task_id']}, '{mysql_log_data['log_level']}', '{mysql_log_data['created_at']}', {mysql_log_data['line_number']}, '{mysql_log_data['message']}')"
325
183
  if self.loglevel == 'DEBUG' or (self.loglevel == 'INFO' and loglevel != 'DEBUG') or (self.loglevel == 'ERROR' and loglevel in ['ERROR', 'CRITICAL']) or (self.loglevel == 'CRITICAL' and loglevel == 'CRITICAL'):
326
184
  self.mysql.raw_query(query, insert=True)
327
- except pymysql.err.OperationalError as e:
328
- print(f"MySQL connection lost during logging: {e}")
329
- self.mysql_unreachable = True
330
- self._save_log_locally(mysql_log_data, 'mysql')
331
- except pymysql.err.InterfaceError as e:
332
- print(f"MySQL connection closed: {e}")
333
- self.mysql_unreachable = True
185
+ except (pymysql.err.OperationalError, pymysql.err.InterfaceError) as e:
186
+ print(f"MySQL connection lost or closed during logging: {e}")
187
+ self.mysql_reachable = False
334
188
  self._save_log_locally(mysql_log_data, 'mysql')
335
189
  except Exception as e:
336
190
  print(f"Error during logging to MySQL: {e}")
@@ -346,7 +200,6 @@ class TaskScheduler(BrynQ):
346
200
  """
347
201
  # Update the step number in the task_scheduler table
348
202
  warnings.warn("Execution steps are deprecated, please stop calling this method. It does nothing anymore", DeprecationWarning)
349
- return
350
203
 
351
204
  def error_handling(self, e: Exception, breaking=True, send_to_teams=False):
352
205
  """
@@ -366,58 +219,60 @@ class TaskScheduler(BrynQ):
366
219
  exc_type, exc_obj, exc_tb = sys.exc_info()
367
220
  error = str(e)[:400].replace('\'', '').replace('\"', '') + ' | Line: {}'.format(exc_tb.tb_lineno)
368
221
 
369
- if self.elastic_enabled:
370
- # Preparing the primary payload with error details for upload to elastic and send it
371
- payload = {
372
- 'task_id': self.task_id,
373
- 'reload_id': self.run_id,
374
- 'started_at': datetime.datetime.now().isoformat(),
375
- 'partner_id': self.partner_id,
376
- 'customer': self.customer,
377
- 'file_name': file_name,
378
- 'function_name': function_name,
379
- 'line_number': line_number,
380
- 'task_loglevel': self.loglevel,
381
- 'line_loglevel': 'CRITICAL',
382
- 'message': str(e),
383
- 'traceback': traceback.format_exc()
384
- }
385
- self.es.post_document(index_name=self.es_index, document=payload)
386
-
387
222
  if self.mysql_enabled:
388
- self.error_count += 1
389
- # Get scheduler task details for logging
390
- task_details = \
391
- self.mysql.select('task_scheduler, data_interfaces', 'data_interfaces.docker_image, data_interfaces.runfile_path', 'WHERE task_scheduler.data_interface_id = data_interfaces.id AND task_scheduler.id = {}'.format(self.task_id))[0]
392
- taskname = task_details[0]
393
- customer = task_details[1].split('/')[-1].split('.')[0]
394
- now = datetime.datetime.now()
395
-
396
- # Log to log table in the database
397
- if self.mysql_enabled:
398
- query = "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, 'CRITICAL', '{}', {}, '{}')".format(self.run_id, self.task_id, now, exc_tb.tb_lineno, error)
223
+ try:
224
+ now = datetime.datetime.now()
225
+ # Log to log table in the database
226
+ mysql_log_data = {
227
+ 'reload_id': self.run_id,
228
+ 'task_id': self.task_id,
229
+ 'log_level': 'CRITICAL',
230
+ 'created_at': now,
231
+ 'line_number': exc_tb.tb_lineno,
232
+ 'message': error
233
+ }
234
+ self.error_count += 1
235
+ # Get scheduler task details for logging
236
+ task_details = \
237
+ self.mysql.select('task_scheduler, data_interfaces', 'data_interfaces.docker_image, data_interfaces.runfile_path', 'WHERE task_scheduler.data_interface_id = data_interfaces.id AND task_scheduler.id = {}'.format(self.task_id))[0]
238
+ taskname = task_details[0]
239
+ customer = task_details[1].split('/')[-1].split('.')[0]
240
+
241
+ query = f"INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({mysql_log_data['reload_id']}, {mysql_log_data['task_id']}, '{mysql_log_data['log_level']}', '{mysql_log_data['created_at']}', {mysql_log_data['line_number']}, '{mysql_log_data['message']}')"
399
242
  self.mysql.raw_query(query, insert=True)
400
- if send_to_teams:
401
- Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
402
- if breaking:
403
- # Set scheduler status to failed
404
- self.mysql.update('task_scheduler', ['status', 'last_reload', 'last_error_message', 'step_nr'],
405
- ['IDLE', now, 'Failed', 0],
406
- 'WHERE `id` = {}'.format(self.task_id))
407
-
408
- self.mysql.update(table='task_scheduler_log',
409
- columns=['reload_status', 'finished_at'],
410
- values=['Failed', f'{now}'],
411
- filter=f'WHERE `reload_id` = {self.run_id}')
412
- if self.email_after_errors:
413
- self.email_errors(failed=True)
414
- # Remove the temp values from the variables table
415
- self.mysql.raw_query(f'UPDATE `task_variables` SET temp_value = null WHERE task_id = {self.task_id}', insert=True)
416
243
 
417
- # Start the chained tasks if it there are tasks which should start if this one is failed
418
- self.start_chained_tasks(finished_task_status='FAILED')
244
+ if send_to_teams:
245
+ Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
246
+ if breaking:
247
+ # Set scheduler status to failed
248
+ self.mysql.update('task_scheduler', ['status', 'last_reload', 'last_error_message', 'step_nr'],
249
+ ['IDLE', now, 'Failed', 0],
250
+ f'WHERE `id` = {self.task_id}')
251
+
252
+ self.mysql.update(table='task_scheduler_log',
253
+ columns=['reload_status', 'finished_at'],
254
+ values=['Failed', f'{now}'],
255
+ filter=f'WHERE `reload_id` = {self.run_id}')
256
+ if self.email_after_errors:
257
+ self.email_errors(failed=True)
258
+ # Remove the temp values from the variables table
259
+ self.mysql.raw_query(f'UPDATE `task_variables` SET temp_value = null WHERE task_id = {self.task_id}', insert=True)
260
+
261
+ # Start the chained tasks if it there are tasks which should start if this one is failed
262
+ self.start_chained_tasks(finished_task_status='FAILED')
263
+ except (pymysql.err.OperationalError, pymysql.err.InterfaceError) as e:
264
+ print(f"MySQL connection lost or closed during logging: {e}")
265
+ self.mysql_reachable = False
266
+ self._save_log_locally(mysql_log_data, 'mysql')
267
+ except Exception as e:
268
+ print(f"Error during logging to MySQL: {e}")
269
+ self._save_log_locally(mysql_log_data, 'mysql')
419
270
 
420
- raise Exception(error)
271
+ # if breaking, reraise for clear traceback (local development) or just print if not breaking
272
+ if breaking:
273
+ raise e
274
+ else:
275
+ print(error)
421
276
 
422
277
  def finish_task(self, reload_instant=False, log_limit: Optional[int] = 10000, log_date_limit: datetime.date = None):
423
278
  """
@@ -475,7 +330,8 @@ class TaskScheduler(BrynQ):
475
330
 
476
331
  def start_chained_tasks(self, finished_task_status: str):
477
332
  if self.mysql_enabled:
478
- filter = f'WHERE start_after_task_id = \'{self.task_id}\' AND start_after_preceding_task = \'{finished_task_status}\''
333
+ # only start chained tasks when trigger is on other task, otherwise this has changed in the db
334
+ filter = f'WHERE start_after_task_id = \'{self.task_id}\' AND start_after_preceding_task = \'{finished_task_status}\' AND task_type = \'OTHER_TASK\''
479
335
  response = self.mysql.select(table='task_scheduler', selection='id', filter=filter)
480
336
  if len(response) > 0:
481
337
  tasks_to_run = [str(task[0]) for task in response]
@@ -485,7 +341,7 @@ class TaskScheduler(BrynQ):
485
341
 
486
342
  def email_errors(self, failed):
487
343
  # The mails to email to should be stored in the task_variables table with the variable email_errors_to
488
- email_variable = self.db_variable('email_errors_to')
344
+ email_variable = self.brynq.interfaces.get_variables(variable_name='email_errors_to')
489
345
  if email_variable is not None:
490
346
  email_to = email_variable.split(',')
491
347
  if isinstance(email_to, list):
@@ -546,17 +402,24 @@ class TaskScheduler(BrynQ):
546
402
  except Exception as e:
547
403
  print(f"Error processing MySQL log file {log_file}: {e}")
548
404
 
549
- def _process_local_elastic_logs(self):
550
- elastic_log_files = [f for f in os.listdir(self.local_log_dir) if f.startswith('elastic_log_')]
551
- for log_file in elastic_log_files:
552
- log_file_path = os.path.join(self.local_log_dir, log_file)
553
- try:
554
- with open(log_file_path, 'r') as f:
555
- logs = json.load(f)
556
- # Process logs
557
- for log_entry in logs:
558
- self.es.post_document(index_name=self.es_index, document=log_entry)
559
- # Remove the log file after processing
560
- os.remove(log_file_path)
561
- except Exception as e:
562
- print(f"Error processing ElasticSearch log file {log_file}: {e}")
405
+ def _write_log_to_mysql(self, log_entry):
406
+ """
407
+ Insert a log entry dictionary into the task_execution_log table in MySQL.
408
+ :param log_entry: dict with keys matching the columns of task_execution_log
409
+ """
410
+ try:
411
+ query = (
412
+ "INSERT INTO `task_execution_log` "
413
+ "(reload_id, task_id, log_level, created_at, line_number, message) "
414
+ "VALUES ({reload_id}, {task_id}, '{log_level}', '{created_at}', {line_number}, '{message}')"
415
+ ).format(
416
+ reload_id=log_entry['reload_id'],
417
+ task_id=log_entry['task_id'],
418
+ log_level=log_entry['log_level'],
419
+ created_at=log_entry['created_at'],
420
+ line_number=log_entry['line_number'],
421
+ message=str(log_entry['message']).replace("'", "")
422
+ )
423
+ self.mysql.raw_query(query, insert=True)
424
+ except Exception as e:
425
+ print(f"Error writing log entry to MySQL: {e}")
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 1.0
2
2
  Name: brynq-sdk-task-scheduler
3
- Version: 3.0.10
3
+ Version: 4.0.0
4
4
  Summary: Code to execute tasks in BrynQ.com with the task scheduler
5
5
  Home-page: UNKNOWN
6
6
  Author: BrynQ
@@ -1,5 +1,5 @@
1
- brynq-sdk-brynq<4,>=2
1
+ brynq-sdk-brynq<5,>=4
2
2
  brynq-sdk-functions<3,>=2
3
- brynq-sdk-mysql<3,>=2
3
+ brynq-sdk-mysql<4,>=3
4
4
  brynq-sdk-mandrill<3,>=2
5
5
  brynq-sdk-elastic<4,>=3
@@ -2,7 +2,7 @@ from setuptools import setup, find_namespace_packages
2
2
 
3
3
  setup(
4
4
  name='brynq_sdk_task_scheduler',
5
- version='3.0.10',
5
+ version='4.0.0',
6
6
  description='Code to execute tasks in BrynQ.com with the task scheduler',
7
7
  long_description='Code to execute tasks in the BrynQ.com platform with the task scheduler',
8
8
  author='BrynQ',
@@ -10,9 +10,9 @@ setup(
10
10
  packages=find_namespace_packages(include=['brynq_sdk*']),
11
11
  license='BrynQ License',
12
12
  install_requires=[
13
- 'brynq-sdk-brynq>=2,<4',
13
+ 'brynq-sdk-brynq>=4,<5',
14
14
  'brynq-sdk-functions>=2,<3',
15
- 'brynq-sdk-mysql>=2,<3',
15
+ 'brynq-sdk-mysql>=3,<4',
16
16
  'brynq-sdk-mandrill>=2,<3',
17
17
  'brynq-sdk-elastic>=3,<4'
18
18
  ],