brynq-sdk-task-scheduler 1.2.1__tar.gz → 2.0.0__tar.gz

{brynq_sdk_task_scheduler-1.2.1 → brynq_sdk_task_scheduler-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: brynq_sdk_task_scheduler
-Version: 1.2.1
+Version: 2.0.0
 Summary: Code to execute tasks in BrynQ.com with the task scheduler
 Home-page: UNKNOWN
 Author: BrynQ

{brynq_sdk_task_scheduler-1.2.1 → brynq_sdk_task_scheduler-2.0.0}/brynq_sdk/task_scheduler/task_scheduler.py

@@ -35,7 +35,9 @@ class TaskScheduler(BrynQ):
             self.email_after_errors = email_after_errors
             self.customer_db = self.mysql.database
             self.customer_id = self.mysql.raw_query(f'SELECT id FROM sc.customers WHERE dbname = \'{self.customer_db}\'')[0][0]
+            self.customer = os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_')
             self.partner_id = os.getenv('PARTNER_ID').lower().replace(' ', '_') if os.getenv('PARTNER_ID') else 'brynq'
+            self.write_logs_to_mysql = os.getenv('WRITE_LOGS_TO_MSYQL') if os.getenv('WRITE_LOGS_TO_MSYQL') else False
             self.task_id = task_id
             self.loglevel = loglevel
             self.started_at = datetime.datetime.now()
@@ -58,7 +60,7 @@ class TaskScheduler(BrynQ):
             self.task_manual_started = self.check_if_task_manual_started()
 
             # Creates Elasticsearch index and data view if not exists
-            self.es_index = f"task_execution_log_{self.customer_db}_{self.started_at.strftime('%Y_%m')}"
+            self.es_index = f"task_execution_log_{self.customer_db}"
             self.es.create_index(index_name=self.es_index)
             self.es.create_data_view(space_name='interfaces', view_name=f'task_execution_log_{self.customer_db}', name=f'Task execution log {self.customer_db}', time_field='started_at')
 
@@ -173,6 +175,10 @@ class TaskScheduler(BrynQ):
         if loglevel not in allowed_loglevels:
             raise Exception('You\'ve entered a not allowed loglevel. Choose one of: {}'.format(allowed_loglevels))
 
+        # Count the errors for relevant log levels
+        if loglevel == 'ERROR' or loglevel == 'CRITICAL':
+            self.error_count += 1
+
         # For Elastic, we need to have the data in JSON format. Handling different data types and preparing extra payload information based on the data type
         # If the data is just a series, count rows, columns and cells
         if isinstance(data, pd.Series):
@@ -245,7 +251,7 @@ class TaskScheduler(BrynQ):
             'started_at': datetime.datetime.now().isoformat(),
             'partner_id': self.partner_id,
             'customer_id': self.customer_id,
-            'customer': os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_'),
+            'customer': self.customer,
             'file_name': file_name,
             'function_name': function_name,
             'line_number': line_number,
@@ -258,31 +264,13 @@ class TaskScheduler(BrynQ):
 
         # Write the logline to the MYSQL database, depends on the chosen loglevel in the task
         print('{} at line: {}'.format(message, line_number))
-        # Remove quotes from message since these break the query
-        message = re.sub("[']", '', message)
-        if self.loglevel == 'DEBUG':
-            # Count the errors
-            if loglevel == 'ERROR' or loglevel == 'CRITICAL':
-                self.error_count += 1
-            return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
-            return self.mysql.update(table='task_execution_log',
-                                     columns=['reload_id', 'task_id', 'log_level', 'created_at', 'line_number', 'message'],
-                                     values=[self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message])
-        elif self.loglevel == 'INFO' and (loglevel == 'INFO' or loglevel == 'ERROR' or loglevel == 'CRITICAL'):
-            # Count the errors
-            if loglevel == 'ERROR' or loglevel == 'CRITICAL':
-                self.error_count += 1
-            return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
-        elif self.loglevel == 'ERROR' and (loglevel == 'ERROR' or loglevel == 'CRITICAL'):
-            self.error_count += 1
-            return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
-        elif self.loglevel == 'CRITICAL' and loglevel == 'CRITICAL':
-            self.error_count += 1
-            return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
+
+        if self.write_logs_to_mysql:
+            # Remove quotes from message since these break the query
+            message = re.sub("[']", '', message)
+            query = "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message)
+            if self.loglevel == 'DEBUG' or (self.loglevel == 'INFO' and loglevel != 'DEBUG') or (self.loglevel == 'ERROR' and loglevel in ['ERROR', 'CRITICAL']) or (self.loglevel == 'CRITICAL' and loglevel == 'CRITICAL'):
+                return self.mysql.raw_query(query, insert=True)
 
     def update_execution_step(self, step_number: int):
         """
@@ -304,10 +292,6 @@ class TaskScheduler(BrynQ):
         :param started_at: Give the time the task is started
         :return: nothing
         """
-        # Format error to a somewhat readable format
-        exc_type, exc_obj, exc_tb = sys.exc_info()
-        error = str(e)[:400].replace('\'', '').replace('\"', '') + ' | Line: {}'.format(exc_tb.tb_lineno)
-
         # Get the linenumber from where the logline is executed.
         file_name, line_number, function_name = self.__get_caller_info()
 
@@ -318,7 +302,7 @@ class TaskScheduler(BrynQ):
             'started_at': datetime.datetime.now().isoformat(),
             'partner_id': self.partner_id,
             'customer_id': self.customer_id,
-            'customer': os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_'),
+            'customer': self.customer,
             'file_name': file_name,
             'function_name': function_name,
             'line_number': line_number,
@@ -329,33 +313,34 @@ class TaskScheduler(BrynQ):
         }
         self.es.post_document(index_name=self.es_index, document=payload)
 
-
+        self.error_count += 1
         # Get scheduler task details for logging
         task_details = \
             self.mysql.select('task_scheduler, data_interfaces', 'data_interfaces.docker_image, data_interfaces.runfile_path', 'WHERE task_scheduler.data_interface_id = data_interfaces.id AND task_scheduler.id = {}'.format(self.task_id))[0]
         taskname = task_details[0]
         customer = task_details[1].split('/')[-1].split('.')[0]
+        now = datetime.datetime.now()
+
+        # Format error to a somewhat readable format
+        exc_type, exc_obj, exc_tb = sys.exc_info()
+        error = str(e)[:400].replace('\'', '').replace('\"', '') + ' | Line: {}'.format(exc_tb.tb_lineno)
 
+        # Log to log table in the database
+        if self.write_logs_to_mysql:
+            query = "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, 'CRITICAL', '{}', {}, '{}')".format(self.run_id, self.task_id, now, exc_tb.tb_lineno, error)
+            self.mysql.raw_query(query, insert=True)
+        if send_to_teams:
+            Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
         if breaking:
             # Set scheduler status to failed
             self.mysql.update('task_scheduler', ['status', 'last_reload', 'last_error_message', 'step_nr'],
-                              ['IDLE', datetime.datetime.now(), 'Failed', 0],
+                              ['IDLE', now, 'Failed', 0],
                               'WHERE `id` = {}'.format(self.task_id))
-            # Log to database
-            self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, 'CRITICAL', '{}', {}, '{}')".format(self.run_id,
-                                                                                                                                                                        self.task_id,
-                                                                                                                                                                        datetime.datetime.now(),
-                                                                                                                                                                        exc_tb.tb_lineno,
-                                                                                                                                                                        error),
-                insert=True)
+
             self.mysql.update(table='task_scheduler_log',
                               columns=['reload_status', 'finished_at'],
-                              values=['Failed', f'{datetime.datetime.now()}'],
+                              values=['Failed', f'{now}'],
                               filter=f'WHERE `reload_id` = {self.run_id}')
-            # Notify users on Teams and If the variable self.send_mail_after_errors is set to True, send an email with the message that the task is failed
-            if send_to_teams:
-                Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
             if self.email_after_errors:
                 self.email_errors(failed=True)
             # Remove the temp values from the variables table
@@ -365,22 +350,15 @@ class TaskScheduler(BrynQ):
             self.start_chained_tasks(finished_task_status='FAILED')
 
             raise Exception(error)
-        else:
-            self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, 'CRITICAL', '{}', {}, '{}')".format(self.run_id,
-                                                                                                                                                                        self.task_id,
-                                                                                                                                                                        datetime.datetime.now(),
-                                                                                                                                                                        exc_tb.tb_lineno,
-                                                                                                                                                                        error),
-                insert=True)
-            if send_to_teams:
-                Functions.send_error_to_teams(database=customer, task_number=self.task_id, task_title=taskname)
-        self.error_count += 1
+
 
     def finish_task(self, reload_instant=False, log_limit: typing.Optional[int] = 10000, log_date_limit: datetime.date = None):
         """
         At the end of the script, write the outcome to the database. Write if the task is finished with or without errors, Email to a contactperson if this variable is given in the
         variables table. Also clean up the execution_log table when the number of lines is more than 1000
+        :param reload_instant: If the task should start again after it's finished
+        :param log_limit: The maximum number of logs to keep in the database. If the number of logs exceeds this limit, the oldest logs will be deleted.
+        :param log_date_limit: The date from which logs should be kept. If this is set, logs older than this date will be deleted.
         :return:
         """
         # If reload instant is true, this adds an extra field 'run_instant' to the update query, and sets the value to 1. This makes the task reload immediately after it's finished
@@ -414,19 +392,20 @@ class TaskScheduler(BrynQ):
         # Start the new task if it there is a task which should start if this one is finished
         self.start_chained_tasks(finished_task_status='SUCCESS')
 
-        # Clean up execution log
-        # set this date filter above the actual delete filter because of the many uncooperative quotation marks involved in the whole filter
-        log_date_limit_filter = f"AND created_at >= \'{log_date_limit.strftime('%Y-%m-%d')}\'" if log_date_limit is not None else None
-        delete_filter = f"WHERE task_id = {self.task_id} " \
-                        f"AND reload_id NOT IN (SELECT reload_id FROM (SELECT reload_id FROM `task_execution_log` WHERE task_id = {self.task_id} " \
-                        f"AND log_level != 'CRITICAL' " \
-                        f"AND log_level != 'ERROR' " \
-                        f"{log_date_limit_filter if log_date_limit_filter is not None else ''} " \
-                        f"ORDER BY created_at DESC {f' LIMIT {log_limit} ' if log_limit is not None else ''}) temp)"
-
-        resp = self.mysql.delete(table="task_execution_log",
-                                 filter=delete_filter)
-        print(resp)
+        if self.write_logs_to_mysql:
+            # Clean up execution log
+            # set this date filter above the actual delete filter because of the many uncooperative quotation marks involved in the whole filter
+            log_date_limit_filter = f"AND created_at >= \'{log_date_limit.strftime('%Y-%m-%d')}\'" if log_date_limit is not None else None
+            delete_filter = f"WHERE task_id = {self.task_id} " \
+                            f"AND reload_id NOT IN (SELECT reload_id FROM (SELECT reload_id FROM `task_execution_log` WHERE task_id = {self.task_id} " \
+                            f"AND log_level != 'CRITICAL' " \
+                            f"AND log_level != 'ERROR' " \
+                            f"{log_date_limit_filter if log_date_limit_filter is not None else ''} " \
+                            f"ORDER BY created_at DESC {f' LIMIT {log_limit} ' if log_limit is not None else ''}) temp)"
+
+            resp = self.mysql.delete(table="task_execution_log",
+                                     filter=delete_filter)
+            print(resp)
 
     def start_chained_tasks(self, finished_task_status: str):
         filter = f'WHERE start_after_task_id = \'{self.task_id}\' AND start_after_preceding_task = \'{finished_task_status}\''

{brynq_sdk_task_scheduler-1.2.1 → brynq_sdk_task_scheduler-2.0.0}/brynq_sdk_task_scheduler.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: brynq-sdk-task-scheduler
-Version: 1.2.1
+Version: 2.0.0
 Summary: Code to execute tasks in BrynQ.com with the task scheduler
 Home-page: UNKNOWN
 Author: BrynQ

{brynq_sdk_task_scheduler-1.2.1 → brynq_sdk_task_scheduler-2.0.0}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup
 
 setup(
     name='brynq_sdk_task_scheduler',
-    version='1.2.1',
+    version='2.0.0',
     description='Code to execute tasks in BrynQ.com with the task scheduler',
     long_description='Code to execute tasks in the BrynQ.com platform with the task scheduler',
     author='BrynQ',