brynq-sdk-task-scheduler 1.1.4__tar.gz → 1.2.1__tar.gz

{brynq_sdk_task_scheduler-1.1.4 → brynq_sdk_task_scheduler-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: brynq_sdk_task_scheduler
-Version: 1.1.4
+Version: 1.2.1
 Summary: Code to execute tasks in BrynQ.com with the task scheduler
 Home-page: UNKNOWN
 Author: BrynQ

{brynq_sdk_task_scheduler-1.1.4 → brynq_sdk_task_scheduler-1.2.1}/brynq_sdk/task_scheduler/task_scheduler.py

@@ -29,36 +29,43 @@ class TaskScheduler(BrynQ):
         with level DEBUG not is stored
         """
         super().__init__()
-        self.es = Elastic()
         self.mysql = MySQL()
-        self.email_after_errors = email_after_errors
-        self.task_id = task_id
-        self.loglevel = loglevel
-        self.started_at = datetime.datetime.now()
-        # If the task is started via the task_scheduler, the following 3 parameters will be passed by the scheduler
-        if len(sys.argv[1:4]) > 0:
-            self.started_local = False
-            self.customer_db, self.task_id, self.run_id = sys.argv[1:4]
-        # If the task is started locally, the parameters should be set locally
-        else:
-            self.started_local = True
-            self.customer_db = 'placeholder'
-            self.run_id = int(round(time.time() * 100000))
-        print(self.task_id, self.run_id)
-        self.error_count = 0
+        try:
+            self.es = Elastic()
+            self.email_after_errors = email_after_errors
+            self.customer_db = self.mysql.database
+            self.customer_id = self.mysql.raw_query(f'SELECT id FROM sc.customers WHERE dbname = \'{self.customer_db}\'')[0][0]
+            self.partner_id = os.getenv('PARTNER_ID').lower().replace(' ', '_') if os.getenv('PARTNER_ID') else 'brynq'
+            self.task_id = task_id
+            self.loglevel = loglevel
+            self.started_at = datetime.datetime.now()
+            # If the task is started via the task_scheduler, the following 3 parameters will be passed by the scheduler
+            if len(sys.argv[1:4]) > 0:
+                self.started_local = False
+                self.customer_db, self.task_id, self.run_id = sys.argv[1:4]
+            # If the task is started locally, the parameters should be set locally
+            else:
+                self.started_local = True
+                self.run_id = int(round(time.time() * 100000))
+            print(self.task_id, self.run_id)
+            self.error_count = 0
 
-        # Check if the log tables exists in the customer database. If not, create them
-        # Mysql throws a warning when a table already exists. We don't care so we ignore warnings. (not exceptions!)
-        warnings.filterwarnings('ignore')
+            # Check if the log tables exists in the customer database. If not, create them
+            # Mysql throws a warning when a table already exists. We don't care so we ignore warnings. (not exceptions!)
+            warnings.filterwarnings('ignore')
 
-        # Check if the task is started on schedule or manual. store in a variable to use later in the script
-        self.task_manual_started = self.check_if_task_manual_started()
+            # Check if the task is started on schedule or manual. store in a variable to use later in the script
+            self.task_manual_started = self.check_if_task_manual_started()
 
-        # Creates Elasticsearch index and data view if not exists
-        self.es.initialize_customer()
+            # Creates Elasticsearch index and data view if not exists
+            self.es_index = f"task_execution_log_{self.customer_db}_{self.started_at.strftime('%Y_%m')}"
+            self.es.create_index(index_name=self.es_index)
+            self.es.create_data_view(space_name='interfaces', view_name=f'task_execution_log_{self.customer_db}', name=f'Task execution log {self.customer_db}', time_field='started_at')
 
-        # Start the task and setup the data in the database
-        self.start_task()
+            # Start the task and setup the data in the database
+            self.start_task()
+        except Exception as e:
+            self.error_handling(e)
 
     def __count_keys(self, json_obj):
         if not isinstance(json_obj, dict):
@@ -71,6 +78,14 @@ class TaskScheduler(BrynQ):
                 key_count += self.__count_keys(value)  # Recursively count keys in nested dictionaries
         return key_count
 
+    def __get_caller_info(self):
+        stack = inspect.stack()
+        caller_frame = stack[2][0]
+        file_name = caller_frame.f_code.co_filename
+        line_number = caller_frame.f_lineno
+        function_name = stack[2][3]
+        return file_name, line_number, function_name
+
     def create_task_execution_steps(self, step_details: list):
         """
         Check if the given steps already exists in the task_execution_steps table. If not, update or insert the values in the table
@@ -158,7 +173,8 @@ class TaskScheduler(BrynQ):
         if loglevel not in allowed_loglevels:
             raise Exception('You\'ve entered a not allowed loglevel. Choose one of: {}'.format(allowed_loglevels))
 
-        # Handling different data types and preparing extra payload information based on the data type
+        # For Elastic, we need to have the data in JSON format. Handling different data types and preparing extra payload information based on the data type
+        # If the data is just a series, count rows, columns and cells
         if isinstance(data, pd.Series):
             dataframe = pd.DataFrame(data).T
             extra_payload = {
@@ -168,6 +184,7 @@ class TaskScheduler(BrynQ):
             }
             if not full_extract:
                 extra_payload['payload'] = dataframe.to_json(orient='records')
+        # If the data is a list, count rows, columns and cells
         elif isinstance(data, dict):
             records = self.__count_keys(data)
             extra_payload = {
@@ -185,6 +202,7 @@ class TaskScheduler(BrynQ):
             }
             if not full_extract:
                 extra_payload['payload'] = data.to_json(orient='records')
+        # If the data is a response from an URL request, also store all the information about the URL request.
         elif isinstance(data, requests.Response):
             records = 1
             if data.request.body is not None:
@@ -217,41 +235,37 @@ class TaskScheduler(BrynQ):
         elif data is not None and full_extract is False:
             extra_payload['full_load'] = False
 
-        # Preparing the primary payload with log details
+        # Get the linenumber from where the logline is executed.
+        file_name, line_number, function_name = self.__get_caller_info()
+
+        # Put everything together in the payload for ElasticSearch and send it
         payload = {
-            'reload_id': self.run_id,
             'task_id': self.task_id,
-            'customer_id': os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_'),
+            'reload_id': self.run_id,
             'started_at': datetime.datetime.now().isoformat(),
-            'loglevel': loglevel,
+            'partner_id': self.partner_id,
+            'customer_id': self.customer_id,
+            'customer': os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_'),
+            'file_name': file_name,
+            'function_name': function_name,
+            'line_number': line_number,
+            'task_loglevel': self.loglevel,
+            'line_loglevel': loglevel,
             'message': message
         }
         payload.update(extra_payload)
+        self.es.post_document(index_name=self.es_index, document=payload)
 
-        # Sending the payload to ElasticSearch
-        self.es.task_execution_log(payload)
-
-        # Get the linenumber from where the logline is executed. Get the stacktrace of this action, jump 1 file up and pick then the linenumber (second item)
-        linenumber = inspect.getouterframes(inspect.currentframe())[1][2]
-        # Write the logline to the database, depends on the chosen loglevel in the task
-        print('{} at line: {}'.format(message, linenumber))
+        # Write the logline to the MYSQL database, depends on the chosen loglevel in the task
+        print('{} at line: {}'.format(message, line_number))
         # Remove quotes from message since these break the query
         message = re.sub("[']", '', message)
-        timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
-        information = {
-            'reload_id': self.run_id,
-            'task_id': self.task_id,
-            'log_level': loglevel,
-            'line_number': linenumber,
-            'message': message,
-            'created_at': timestamp
-        }
         if self.loglevel == 'DEBUG':
             # Count the errors
             if loglevel == 'ERROR' or loglevel == 'CRITICAL':
                 self.error_count += 1
             return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message), insert=True)
+                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
             return self.mysql.update(table='task_execution_log',
                                      columns=['reload_id', 'task_id', 'log_level', 'created_at', 'line_number', 'message'],
                                      values=[self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message])
@@ -260,15 +274,15 @@ class TaskScheduler(BrynQ):
             if loglevel == 'ERROR' or loglevel == 'CRITICAL':
                 self.error_count += 1
             return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message), insert=True)
+                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
         elif self.loglevel == 'ERROR' and (loglevel == 'ERROR' or loglevel == 'CRITICAL'):
             self.error_count += 1
             return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message), insert=True)
+                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
         elif self.loglevel == 'CRITICAL' and loglevel == 'CRITICAL':
             self.error_count += 1
             return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message), insert=True)
+                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
 
     def update_execution_step(self, step_number: int):
         """
@@ -290,24 +304,32 @@ class TaskScheduler(BrynQ):
         :param started_at: Give the time the task is started
         :return: nothing
         """
+        # Format error to a somewhat readable format
+        exc_type, exc_obj, exc_tb = sys.exc_info()
+        error = str(e)[:400].replace('\'', '').replace('\"', '') + ' | Line: {}'.format(exc_tb.tb_lineno)
+
+        # Get the linenumber from where the logline is executed.
+        file_name, line_number, function_name = self.__get_caller_info()
 
-        # Preparing the primary payload with error details for upload to elastic
+        # Preparing the primary payload with error details for upload to elastic and send it
         payload = {
-            'reload_id': self.run_id,
             'task_id': self.task_id,
-            'customer_id': os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_'),
+            'reload_id': self.run_id,
             'started_at': datetime.datetime.now().isoformat(),
-            'loglevel': 'CRITICAL',
+            'partner_id': self.partner_id,
+            'customer_id': self.customer_id,
+            'customer': os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_'),
+            'file_name': file_name,
+            'function_name': function_name,
+            'line_number': line_number,
+            'task_loglevel': self.loglevel,
+            'line_loglevel': 'CRITICAL',
             'message': str(e),
             'traceback': traceback.format_exc()
         }
+        self.es.post_document(index_name=self.es_index, document=payload)
 
-        # Sending the payload to ElasticSearch
-        self.es.task_execution_log(payload)
 
-        # Format error to a somewhat readable format
-        exc_type, exc_obj, exc_tb = sys.exc_info()
-        error = str(e)[:400].replace('\'', '').replace('\"', '') + ' | Line: {}'.format(exc_tb.tb_lineno)
         # Get scheduler task details for logging
         task_details = \
             self.mysql.select('task_scheduler, data_interfaces', 'data_interfaces.docker_image, data_interfaces.runfile_path', 'WHERE task_scheduler.data_interface_id = data_interfaces.id AND task_scheduler.id = {}'.format(self.task_id))[0]
@@ -320,15 +342,6 @@ class TaskScheduler(BrynQ):
                               ['IDLE', datetime.datetime.now(), 'Failed', 0],
                               'WHERE `id` = {}'.format(self.task_id))
             # Log to database
-            timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
-            information = {
-                'reload_id': self.run_id,
-                'task_id': self.task_id,
-                'log_level': 'CRITICAL',
-                'line_number': exc_tb.tb_lineno,
-                'message': error,
-                'created_at': timestamp
-            }
             self.mysql.raw_query(
                 "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, 'CRITICAL', '{}', {}, '{}')".format(self.run_id,
                                                                                                                                                                         self.task_id,
@@ -353,15 +366,6 @@ class TaskScheduler(BrynQ):
 
             raise Exception(error)
         else:
-            timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
-            information = {
-                'reload_id': self.run_id,
-                'task_id': self.task_id,
-                'log_level': 'CRITICAL',
-                'line_number': exc_tb.tb_lineno,
-                'message': error,
-                'created_at': timestamp
-            }
             self.mysql.raw_query(
                 "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, 'CRITICAL', '{}', {}, '{}')".format(self.run_id,
                                                                                                                                                                         self.task_id,
@@ -442,11 +446,9 @@ class TaskScheduler(BrynQ):
                 for i in email_to:
                     email_list.append({'name': 'BrynQ User', 'mail': i.strip()})
                 # Set the content of the mail and all other stuff
-                task = self.mysql.select(table='task_scheduler', selection='title', filter=f'WHERE id = {self.task_id}')[0][
-                    0]
+                task = self.mysql.select(table='data_interfaces', selection='title', filter=f'WHERE id = {self.task_id}')[0][0]
                 finished_at = \
-                    self.mysql.select(table='task_scheduler', selection='last_reload', filter=f'WHERE id = {self.task_id}')[0][
-                        0]
+                    self.mysql.select(table='task_scheduler', selection='last_reload', filter=f'WHERE data_interface_id = {self.task_id}')[0][0]
                 if failed:
                     subject = f'Task \'{task}\' has failed'
                     content = f'Task \'{task}\' with task ID \'{self.task_id}\' failed during its last run and was stopped at {finished_at}. ' \

{brynq_sdk_task_scheduler-1.1.4 → brynq_sdk_task_scheduler-1.2.1}/brynq_sdk_task_scheduler.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: brynq-sdk-task-scheduler
-Version: 1.1.4
+Version: 1.2.1
 Summary: Code to execute tasks in BrynQ.com with the task scheduler
 Home-page: UNKNOWN
 Author: BrynQ

{brynq_sdk_task_scheduler-1.1.4 → brynq_sdk_task_scheduler-1.2.1}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup
 
 setup(
     name='brynq_sdk_task_scheduler',
-    version='1.1.4',
+    version='1.2.1',
     description='Code to execute tasks in BrynQ.com with the task scheduler',
     long_description='Code to execute tasks in the BrynQ.com platform with the task scheduler',
     author='BrynQ',