brynq-sdk-task-scheduler 1.1.3__tar.gz → 1.2.0__tar.gz

{brynq_sdk_task_scheduler-1.1.3 → brynq_sdk_task_scheduler-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: brynq_sdk_task_scheduler
-Version: 1.1.3
+Version: 1.2.0
 Summary: Code to execute tasks in BrynQ.com with the task scheduler
 Home-page: UNKNOWN
 Author: BrynQ

{brynq_sdk_task_scheduler-1.1.3 → brynq_sdk_task_scheduler-1.2.0}/brynq_sdk/task_scheduler/task_scheduler.py

@@ -32,6 +32,9 @@ class TaskScheduler(BrynQ):
         self.es = Elastic()
         self.mysql = MySQL()
         self.email_after_errors = email_after_errors
+        self.customer_db = self.mysql.database
+        self.customer_id = self.mysql.raw_query(f'SELECT id FROM sc.customers WHERE dbname = \'{self.customer_db}\'')[0][0]
+        self.partner_id = os.getenv('PARTNER_ID').lower().replace(' ', '_') if os.getenv('PARTNER_ID') else 'brynq'
         self.task_id = task_id
         self.loglevel = loglevel
         self.started_at = datetime.datetime.now()
@@ -42,7 +45,6 @@ class TaskScheduler(BrynQ):
         # If the task is started locally, the parameters should be set locally
         else:
             self.started_local = True
-            self.customer_db = 'placeholder'
             self.run_id = int(round(time.time() * 100000))
         print(self.task_id, self.run_id)
         self.error_count = 0
@@ -50,13 +52,14 @@ class TaskScheduler(BrynQ):
         # Check if the log tables exists in the customer database. If not, create them
         # Mysql throws a warning when a table already exists. We don't care so we ignore warnings. (not exceptions!)
         warnings.filterwarnings('ignore')
-        # self.check_if_logging_tables_exists()
 
         # Check if the task is started on schedule or manual. store in a variable to use later in the script
         self.task_manual_started = self.check_if_task_manual_started()
 
         # Creates Elasticsearch index and data view if not exists
-        self.es.initialize_customer()
+        self.es_index = f"task_execution_log_{self.customer_db}_{self.started_at.strftime('%Y_%m')}"
+        self.es.create_index(index_name=self.es_index)
+        self.es.create_data_view(space_name='interfaces', view_name=f'task_execution_log_{self.customer_db}', name=f'Task execution log {self.customer_db}', time_field='started_at')
 
         # Start the task and setup the data in the database
         self.start_task()
@@ -72,99 +75,13 @@ class TaskScheduler(BrynQ):
                 key_count += self.__count_keys(value)  # Recursively count keys in nested dictionaries
         return key_count
 
-    def check_if_logging_tables_exists(self):
-        """
-        This function checks if all the needed tables for the task_scheduler exists. If they don't, this function
-        creates the needed tables
-        :return: nothing
-        """
-        # Check if the table task_scheduler exists. If not, create it
-        new_table_query = 'CREATE TABLE IF NOT EXISTS `task_scheduler` (' \
-                          '`id`                         int(11)                                 NOT NULL AUTO_INCREMENT,' \
-                          '`dashboard_reload`           bool                                    NOT NULL DEFAULT \'0\',' \
-                          '`title`                      varchar(50)                             NOT NULL,' \
-                          '`description`                varchar(255)                            NOT NULL,' \
-                          '`dashboard_guid`             varchar(255)                            NULL DEFAULT NULL,' \
-                          '`docker_image`               varchar(255)                            DEFAULT NULL,' \
-                          '`runfile_path`               varchar(255)                            DEFAULT NULL,' \
-                          '`trigger_type`               enum("MANUAL", "TIME", "OTHER_TASK")    NOT NULL DEFAULT \'MANUAL\',' \
-                          '`next_reload`                timestamp                               NULL DEFAULT NULL,' \
-                          '`timezone`                   enum("Africa/Abidjan", "Africa/Accra", "Africa/Addis_Ababa", "Africa/Algiers", "Africa/Asmara", "Africa/Bamako", "Africa/Bangui", "Africa/Banjul", "Africa/Bissau", "Africa/Blantyre", "Africa/Brazzaville", "Africa/Bujumbura", "Africa/Cairo", "Africa/Casablanca", "Africa/Ceuta", "Africa/Conakry", "Africa/Dakar", "Africa/Dar_es_Salaam", "Africa/Djibouti", "Africa/Douala", "Africa/El_Aaiun", "Africa/Freetown", "Africa/Gaborone", "Africa/Harare", "Africa/Johannesburg", "Africa/Juba", "Africa/Kampala", "Africa/Khartoum", "Africa/Kigali", "Africa/Kinshasa", "Africa/Lagos", "Africa/Libreville", "Africa/Lome", "Africa/Luanda", "Africa/Lubumbashi", "Africa/Lusaka", "Africa/Malabo", "Africa/Maputo", "Africa/Maseru", "Africa/Mbabane", "Africa/Mogadishu", "Africa/Monrovia", "Africa/Nairobi", "Africa/Ndjamena", "Africa/Niamey", "Africa/Nouakchott", "Africa/Ouagadougou", "Africa/Porto-Novo", "Africa/Sao_Tome", "Africa/Tripoli", "Africa/Tunis", "Africa/Windhoek", "America/Adak", "America/Anchorage", "America/Anguilla", "America/Antigua", "America/Araguaina", "America/Argentina/Buenos_Aires", "America/Argentina/Catamarca", "America/Argentina/Cordoba", "America/Argentina/Jujuy", "America/Argentina/La_Rioja", "America/Argentina/Mendoza", "America/Argentina/Rio_Gallegos", "America/Argentina/Salta", "America/Argentina/San_Juan", "America/Argentina/San_Luis", "America/Argentina/Tucuman", "America/Argentina/Ushuaia", "America/Aruba", "America/Asuncion", "America/Atikokan", "America/Bahia", "America/Bahia_Banderas", "America/Barbados", "America/Belem", "America/Belize", "America/Blanc-Sablon", "America/Boa_Vista", "America/Bogota", "America/Boise", "America/Cambridge_Bay", "America/Campo_Grande", "America/Cancun", "America/Caracas", "America/Cayenne", "America/Cayman", "America/Chicago", "America/Chihuahua", "America/Costa_Rica", "America/Creston", "America/Cuiaba", "America/Curacao", "America/Danmarkshavn", "America/Dawson", "America/Dawson_Creek", "America/Denver", "America/Detroit", "America/Dominica", "America/Edmonton", "America/Eirunepe", "America/El_Salvador", "America/Fort_Nelson", "America/Fortaleza", "America/Glace_Bay", "America/Godthab", "America/Goose_Bay", "America/Grand_Turk", "America/Grenada", "America/Guadeloupe", "America/Guatemala", "America/Guayaquil", "America/Guyana", "America/Halifax", "America/Havana", "America/Hermosillo", "America/Indiana/Indianapolis", "America/Indiana/Knox", "America/Indiana/Marengo", "America/Indiana/Petersburg", "America/Indiana/Tell_City", "America/Indiana/Vevay", "America/Indiana/Vincennes", "America/Indiana/Winamac", "America/Inuvik", "America/Iqaluit", "America/Jamaica", "America/Juneau", "America/Kentucky/Louisville", "America/Kentucky/Monticello", "America/Kralendijk", "America/La_Paz", "America/Lima", "America/Los_Angeles", "America/Lower_Princes", "America/Maceio", "America/Managua", "America/Manaus", "America/Marigot", "America/Martinique", "America/Matamoros", "America/Mazatlan", "America/Menominee", "America/Merida", "America/Metlakatla", "America/Mexico_City", "America/Miquelon", "America/Moncton", "America/Monterrey", "America/Montevideo", "America/Montserrat", "America/Nassau", "America/New_York", "America/Nipigon", "America/Nome", "America/Noronha", "America/North_Dakota/Beulah", "America/North_Dakota/Center", "America/North_Dakota/New_Salem", "America/Ojinaga", "America/Panama", "America/Pangnirtung", "America/Paramaribo", "America/Phoenix", "America/Port-au-Prince", "America/Port_of_Spain", "America/Porto_Velho", "America/Puerto_Rico", "America/Punta_Arenas", "America/Rainy_River", "America/Rankin_Inlet", "America/Recife", "America/Regina", "America/Resolute", "America/Rio_Branco", "America/Santarem", "America/Santiago", "America/Santo_Domingo", "America/Sao_Paulo", "America/Scoresbysund", "America/Sitka", "America/St_Barthelemy", "America/St_Johns", "America/St_Kitts", "America/St_Lucia", "America/St_Thomas", "America/St_Vincent", "America/Swift_Current", "America/Tegucigalpa", "America/Thule", "America/Thunder_Bay", "America/Tijuana", "America/Toronto", "America/Tortola", "America/Vancouver", "America/Whitehorse", "America/Winnipeg", "America/Yakutat", "America/Yellowknife", "Antarctica/Casey", "Antarctica/Davis", "Antarctica/DumontDUrville", "Antarctica/Macquarie", "Antarctica/Mawson", "Antarctica/McMurdo", "Antarctica/Palmer", "Antarctica/Rothera", "Antarctica/Syowa", "Antarctica/Troll", "Antarctica/Vostok", "Arctic/Longyearbyen", "Asia/Aden", "Asia/Almaty", "Asia/Amman", "Asia/Anadyr", "Asia/Aqtau", "Asia/Aqtobe", "Asia/Ashgabat", "Asia/Atyrau", "Asia/Baghdad", "Asia/Bahrain", "Asia/Baku", "Asia/Bangkok", "Asia/Barnaul", "Asia/Beirut", "Asia/Bishkek", "Asia/Brunei", "Asia/Chita", "Asia/Choibalsan", "Asia/Colombo", "Asia/Damascus", "Asia/Dhaka", "Asia/Dili", "Asia/Dubai", "Asia/Dushanbe", "Asia/Famagusta", "Asia/Gaza", "Asia/Hebron", "Asia/Ho_Chi_Minh", "Asia/Hong_Kong", "Asia/Hovd", "Asia/Irkutsk", "Asia/Jakarta", "Asia/Jayapura", "Asia/Jerusalem", "Asia/Kabul", "Asia/Kamchatka", "Asia/Karachi", "Asia/Kathmandu", "Asia/Khandyga", "Asia/Kolkata", "Asia/Krasnoyarsk", "Asia/Kuala_Lumpur", "Asia/Kuching", "Asia/Kuwait", "Asia/Macau", "Asia/Magadan", "Asia/Makassar", "Asia/Manila", "Asia/Muscat", "Asia/Nicosia", "Asia/Novokuznetsk", "Asia/Novosibirsk", "Asia/Omsk", "Asia/Oral", "Asia/Phnom_Penh", "Asia/Pontianak", "Asia/Pyongyang", "Asia/Qatar", "Asia/Qostanay", "Asia/Qyzylorda", "Asia/Riyadh", "Asia/Sakhalin", "Asia/Samarkand", "Asia/Seoul", "Asia/Shanghai", "Asia/Singapore", "Asia/Srednekolymsk", "Asia/Taipei", "Asia/Tashkent", "Asia/Tbilisi", "Asia/Tehran", "Asia/Thimphu", "Asia/Tokyo", "Asia/Tomsk", "Asia/Ulaanbaatar", "Asia/Urumqi", "Asia/Ust-Nera", "Asia/Vientiane", "Asia/Vladivostok", "Asia/Yakutsk", "Asia/Yangon", "Asia/Yekaterinburg", "Asia/Yerevan", "Atlantic/Azores", "Atlantic/Bermuda", "Atlantic/Canary", "Atlantic/Cape_Verde", "Atlantic/Faroe", "Atlantic/Madeira", "Atlantic/Reykjavik", "Atlantic/South_Georgia", "Atlantic/St_Helena", "Atlantic/Stanley", "Australia/Adelaide", "Australia/Brisbane", "Australia/Broken_Hill", "Australia/Currie", "Australia/Darwin", "Australia/Eucla", "Australia/Hobart", "Australia/Lindeman", "Australia/Lord_Howe", "Australia/Melbourne", "Australia/Perth", "Australia/Sydney", "Canada/Atlantic", "Canada/Central", "Canada/Eastern", "Canada/Mountain", "Canada/Newfoundland", "Canada/Pacific", "Europe/Amsterdam", "Europe/Andorra", "Europe/Astrakhan", "Europe/Athens", "Europe/Belgrade", "Europe/Berlin", "Europe/Bratislava", "Europe/Brussels", "Europe/Bucharest", "Europe/Budapest", "Europe/Busingen", "Europe/Chisinau", "Europe/Copenhagen", "Europe/Dublin", "Europe/Gibraltar", "Europe/Guernsey", "Europe/Helsinki", "Europe/Isle_of_Man", "Europe/Istanbul", "Europe/Jersey", "Europe/Kaliningrad", "Europe/Kiev", "Europe/Kirov", "Europe/Lisbon", "Europe/Ljubljana", "Europe/London", "Europe/Luxembourg", "Europe/Madrid", "Europe/Malta", "Europe/Mariehamn", "Europe/Minsk", "Europe/Monaco", "Europe/Moscow", "Europe/Oslo", "Europe/Paris", "Europe/Podgorica", "Europe/Prague", "Europe/Riga", "Europe/Rome", "Europe/Samara", "Europe/San_Marino", "Europe/Sarajevo", "Europe/Saratov", "Europe/Simferopol", "Europe/Skopje", "Europe/Sofia", "Europe/Stockholm", "Europe/Tallinn", "Europe/Tirane", "Europe/Ulyanovsk", "Europe/Uzhgorod", "Europe/Vaduz", "Europe/Vatican", "Europe/Vienna", "Europe/Vilnius", "Europe/Volgograd", "Europe/Warsaw", "Europe/Zagreb", "Europe/Zaporozhye", "Europe/Zurich", "GMT", "Indian/Antananarivo", "Indian/Chagos", "Indian/Christmas", "Indian/Cocos", "Indian/Comoro", "Indian/Kerguelen", "Indian/Mahe", "Indian/Maldives", "Indian/Mauritius", "Indian/Mayotte", "Indian/Reunion", "Pacific/Apia", "Pacific/Auckland", "Pacific/Bougainville", "Pacific/Chatham", "Pacific/Chuuk", "Pacific/Easter", "Pacific/Efate", "Pacific/Enderbury", "Pacific/Fakaofo", "Pacific/Fiji", "Pacific/Funafuti", "Pacific/Galapagos", "Pacific/Gambier", "Pacific/Guadalcanal", "Pacific/Guam", "Pacific/Honolulu", "Pacific/Kiritimati", "Pacific/Kosrae", "Pacific/Kwajalein", "Pacific/Majuro", "Pacific/Marquesas", "Pacific/Midway", "Pacific/Nauru", "Pacific/Niue", "Pacific/Norfolk", "Pacific/Noumea", "Pacific/Pago_Pago", "Pacific/Palau", "Pacific/Pitcairn", "Pacific/Pohnpei", "Pacific/Port_Moresby", "Pacific/Rarotonga", "Pacific/Saipan", "Pacific/Tahiti", "Pacific/Tarawa", "Pacific/Tongatapu", "Pacific/Wake", "Pacific/Wallis", "US/Alaska", "US/Arizona", "US/Central", "US/Eastern", "US/Hawaii", "US/Mountain", "US/Pacific", "UTC") CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT \"Europe/Amsterdam\",' \
-                          '`frequency`                  varchar(255)                            DEFAULT \'{"month":0,"day":0,"hour":0,"minute":0}\',' \
-                          '`start_after_task_id`        int                                     DEFAULT NULL,' \
-                          '`start_after_preceding_task` enum("FAILED", "SUCCESS")               DEFAULT NULL,' \
-                          '`last_reload`                timestamp                               NULL DEFAULT NULL,' \
-                          '`last_error_message`         varchar(255)                            DEFAULT NULL,' \
-                          '`status`                     varchar(255)                            DEFAULT \'IDLE\',' \
-                          '`disabled`                   tinyint(4)                              DEFAULT \'1\',' \
-                          '`run_instant`                tinyint(1)                              DEFAULT \'0\',' \
-                          '`sftp_mapping`               varchar(255)                            NOT NULL DEFAULT \'[]\',' \
-                          '`step_nr`                    int                                     NOT NULL DEFAULT \'0\',' \
-                          '`stopped_by_user`            tinyint(1)                              DEFAULT \'0\',' \
-                          '`stop_is_allowed`            bool                                    NOT NULL DEFAULT \'0\',' \
-                          'PRIMARY KEY (`id`),' \
-                          'UNIQUE KEY `task_scheduler_id_uindex` (`id`),' \
-                          'constraint task_scheduler_task_scheduler_id_fk foreign key (start_after_task_id) references task_scheduler (id)' \
-                          ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci'
-        self.mysql.raw_query(new_table_query)
-
-        # Check if the table task_scheduler_log exists. If not, create it
-        new_table_query = 'CREATE TABLE IF NOT EXISTS `task_scheduler_log` (' \
-                          '`reload_id`      bigint          NOT NULL,' \
-                          '`task_id`        int             NULL,' \
-                          '`reload_status`  varchar(255)    NULL,' \
-                          '`started_at`     datetime        NULL,' \
-                          '`finished_at`    datetime        NULL' \
-                          ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci'
-        self.mysql.raw_query(new_table_query)
-
-        # Check if the table check_task_execution_log exists. If not, create it
-        new_table_query = 'CREATE TABLE IF NOT EXISTS `task_execution_log`(' \
-                          '`reload_id`   bigint       NOT NULL,' \
-                          '`task_id`     int          NULL,' \
-                          '`log_level`   varchar(255) NULL,' \
-                          '`created_at`  datetime     NULL,' \
-                          '`line_number` int          NULL,' \
-                          '`message`     longtext     NULL)' \
-                          'ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci'
-        self.mysql.raw_query(new_table_query)
-
-        # Check if the table check_task_execution_steps exists. If not, create it
-        new_table_query = 'CREATE TABLE IF NOT EXISTS `task_execution_steps`(' \
-                          '`id`          bigint       NOT NULL AUTO_INCREMENT,' \
-                          '`task_id`     int          NULL,' \
-                          '`nr`          int          DEFAULT 0 NOT NULL,' \
-                          '`description` varchar(255) DEFAULT \'ZzZzZz...\' NOT NULL,' \
-                          'PRIMARY KEY (`id`),' \
-                          'UNIQUE KEY `task_execution_steps_id_uindex` (`id`),' \
-                          'UNIQUE INDEX `task_execution_steps_task_id_nr_uindex` (`task_id`, `nr`))' \
-                          'ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci'
-        self.mysql.raw_query(new_table_query)
-
-        new_table_query = 'CREATE TABLE IF NOT EXISTS `task_variables`(' \
-                          'id INT NOT NULL AUTO_INCREMENT,' \
-                          'task_id INT NOT NULL,' \
-                          'name VARCHAR(150) NOT NULL,' \
-                          'description VARCHAR(255) NULL,' \
-                          'type ENUM(\'INT\', \'TINYINT\', \'BIGINT\', \'FLOAT\', \'DOUBLE\', \'DATETIME\', \'TIMESTAMP\', \'TIME\', \'VARCHAR\', \'BLOB\', \'TEXT\', \'LONGBLOB\') NOT NULL,' \
-                          'value VARCHAR(600) NULL,' \
-                          'temp_value VARCHAR(600) NULL,' \
-                          'PRIMARY KEY (`id`),' \
-                          'UNIQUE KEY `task_variables_id_uindex` (`id`),' \
-                          'UNIQUE INDEX `task_variables_name_value_uindex` (`task_id`, `name`, `value`), ' \
-                          'INDEX `task_variables_name_index` (`name`),' \
-                          'CONSTRAINT task_variables_task_scheduler_id_fk ' \
-                          'FOREIGN KEY (`task_id`) REFERENCES task_scheduler (`id`) ON DELETE CASCADE)' \
-                          'ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci'
-        self.mysql.raw_query(new_table_query)
-
-        # Add the variable 'email_errors_to' as default to the new added table if it doesn't exist for the current task
-        response = self.mysql.select('task_variables', 'temp_value',
-                                     f'WHERE name = \'email_errors_to\' AND task_id = {self.task_id}')
-        if len(response) == 0:
-            new_variables = f"-- INSERT INTO `task_variables` (`task_id`, `name`, `type`, `value`, `temp_value`) " \
-                            f"VALUES ({self.task_id}, 'email_errors_to', 'TEXT', 'example@brynq.com, example2@brynq.com', 'example@brynq.com, example2@brynq.com')"
-            self.mysql.raw_query(new_variables, insert=True)
+    def __get_caller_info(self):
+        stack = inspect.stack()
+        caller_frame = stack[2][0]
+        file_name = caller_frame.f_code.co_filename
+        line_number = caller_frame.f_lineno
+        function_name = stack[2][3]
+        return file_name, line_number, function_name
 
     def create_task_execution_steps(self, step_details: list):
         """
@@ -253,7 +170,8 @@ class TaskScheduler(BrynQ):
         if loglevel not in allowed_loglevels:
             raise Exception('You\'ve entered a not allowed loglevel. Choose one of: {}'.format(allowed_loglevels))
 
-        # Handling different data types and preparing extra payload information based on the data type
+        # For Elastic, we need to have the data in JSON format. Handling different data types and preparing extra payload information based on the data type
+        # If the data is just a series, count rows, columns and cells
         if isinstance(data, pd.Series):
             dataframe = pd.DataFrame(data).T
             extra_payload = {
@@ -263,6 +181,7 @@ class TaskScheduler(BrynQ):
             }
             if not full_extract:
                 extra_payload['payload'] = dataframe.to_json(orient='records')
+        # If the data is a list, count rows, columns and cells
         elif isinstance(data, dict):
             records = self.__count_keys(data)
             extra_payload = {
@@ -280,6 +199,7 @@ class TaskScheduler(BrynQ):
             }
             if not full_extract:
                 extra_payload['payload'] = data.to_json(orient='records')
+        # If the data is a response from an URL request, also store all the information about the URL request.
         elif isinstance(data, requests.Response):
             records = 1
             if data.request.body is not None:
@@ -312,41 +232,37 @@ class TaskScheduler(BrynQ):
         elif data is not None and full_extract is False:
             extra_payload['full_load'] = False
 
-        # Preparing the primary payload with log details
+        # Get the linenumber from where the logline is executed.
+        file_name, line_number, function_name = self.__get_caller_info()
+
+        # Put everything together in the payload for ElasticSearch and send it
         payload = {
-            'reload_id': self.run_id,
             'task_id': self.task_id,
-            'customer_id': os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_'),
+            'reload_id': self.run_id,
             'started_at': datetime.datetime.now().isoformat(),
-            'loglevel': loglevel,
+            'partner_id': self.partner_id,
+            'customer_id': self.customer_id,
+            'customer': os.getenv('BRYNQ_CUSTOMER_NAME').lower().replace(' ', '_'),
+            'file_name': file_name,
+            'function_name': function_name,
+            'line_number': line_number,
+            'task_loglevel': self.loglevel,
+            'line_loglevel': loglevel,
             'message': message
         }
         payload.update(extra_payload)
+        self.es.post_document(index_name=self.es_index, document=payload)
 
-        # Sending the payload to ElasticSearch
-        self.es.task_execution_log(payload)
-
-        # Get the linenumber from where the logline is executed. Get the stacktrace of this action, jump 1 file up and pick then the linenumber (second item)
-        linenumber = inspect.getouterframes(inspect.currentframe())[1][2]
-        # Write the logline to the database, depends on the chosen loglevel in the task
-        print('{} at line: {}'.format(message, linenumber))
+        # Write the logline to the MYSQL database, depends on the chosen loglevel in the task
+        print('{} at line: {}'.format(message, line_number))
         # Remove quotes from message since these break the query
         message = re.sub("[']", '', message)
-        timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
-        information = {
-            'reload_id': self.run_id,
-            'task_id': self.task_id,
-            'log_level': loglevel,
-            'line_number': linenumber,
-            'message': message,
-            'created_at': timestamp
-        }
         if self.loglevel == 'DEBUG':
             # Count the errors
             if loglevel == 'ERROR' or loglevel == 'CRITICAL':
                 self.error_count += 1
             return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message), insert=True)
+                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
             return self.mysql.update(table='task_execution_log',
                                      columns=['reload_id', 'task_id', 'log_level', 'created_at', 'line_number', 'message'],
                                      values=[self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message])
@@ -355,15 +271,15 @@ class TaskScheduler(BrynQ):
             if loglevel == 'ERROR' or loglevel == 'CRITICAL':
                 self.error_count += 1
             return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message), insert=True)
+                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
         elif self.loglevel == 'ERROR' and (loglevel == 'ERROR' or loglevel == 'CRITICAL'):
             self.error_count += 1
             return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message), insert=True)
+                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
         elif self.loglevel == 'CRITICAL' and loglevel == 'CRITICAL':
             self.error_count += 1
             return self.mysql.raw_query(
-                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), linenumber, message), insert=True)
+                "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, '{}', '{}', {}, '{}')".format(self.run_id, self.task_id, loglevel, datetime.datetime.now(), line_number, message), insert=True)
 
     def update_execution_step(self, step_number: int):
         """
@@ -385,27 +301,35 @@ class TaskScheduler(BrynQ):
         :param started_at: Give the time the task is started
         :return: nothing
         """
+        # Format error to a somewhat readable format
+        exc_type, exc_obj, exc_tb = sys.exc_info()
+        error = str(e)[:400].replace('\'', '').replace('\"', '') + ' | Line: {}'.format(exc_tb.tb_lineno)
 
-        # Preparing the primary payload with error details for upload to elastic
+        # Get the linenumber from where the logline is executed.
+        file_name, line_number, function_name = self.__get_caller_info()
+
+        # Preparing the primary payload with error details for upload to elastic and send it
         payload = {
-            'reload_id': self.run_id,
             'task_id': self.task_id,
-            'customer_id': os.getenv('BRYNQ_SUBDOMAIN').lower().replace(' ', '_'),
+            'reload_id': self.run_id,
             'started_at': datetime.datetime.now().isoformat(),
-            'loglevel': 'CRITICAL',
+            'partner_id': self.partner_id,
+            'customer_id': self.customer_id,
+            'customer': os.getenv('BRYNQ_CUSTOMER_NAME').lower().replace(' ', '_'),
+            'file_name': file_name,
+            'function_name': function_name,
+            'line_number': line_number,
+            'task_loglevel': self.loglevel,
+            'line_loglevel': 'CRITICAL',
             'message': str(e),
             'traceback': traceback.format_exc()
         }
+        self.es.post_document(index_name=self.es_index, document=payload)
 
-        # Sending the payload to ElasticSearch
-        self.es.task_execution_log(payload)
 
-        # Format error to a somewhat readable format
-        exc_type, exc_obj, exc_tb = sys.exc_info()
-        error = str(e)[:400].replace('\'', '').replace('\"', '') + ' | Line: {}'.format(exc_tb.tb_lineno)
         # Get scheduler task details for logging
         task_details = \
-            self.mysql.select('task_scheduler', 'docker_image, runfile_path', 'WHERE id = {}'.format(self.task_id))[0]
+            self.mysql.select('task_scheduler, data_interfaces', 'data_interfaces.docker_image, data_interfaces.runfile_path', 'WHERE task_scheduler.data_interface_id = data_interfaces.id AND task_scheduler.id = {}'.format(self.task_id))[0]
         taskname = task_details[0]
         customer = task_details[1].split('/')[-1].split('.')[0]
 
@@ -415,15 +339,6 @@ class TaskScheduler(BrynQ):
                               ['IDLE', datetime.datetime.now(), 'Failed', 0],
                               'WHERE `id` = {}'.format(self.task_id))
             # Log to database
-            timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
-            information = {
-                'reload_id': self.run_id,
-                'task_id': self.task_id,
-                'log_level': 'CRITICAL',
-                'line_number': exc_tb.tb_lineno,
-                'message': error,
-                'created_at': timestamp
-            }
             self.mysql.raw_query(
                 "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, 'CRITICAL', '{}', {}, '{}')".format(self.run_id,
                                                                                                                                                                         self.task_id,
@@ -448,15 +363,6 @@ class TaskScheduler(BrynQ):
 
             raise Exception(error)
         else:
-            timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
-            information = {
-                'reload_id': self.run_id,
-                'task_id': self.task_id,
-                'log_level': 'CRITICAL',
-                'line_number': exc_tb.tb_lineno,
-                'message': error,
-                'created_at': timestamp
-            }
             self.mysql.raw_query(
                 "INSERT INTO `task_execution_log` (reload_id, task_id, log_level, created_at, line_number, message) VALUES ({}, {}, 'CRITICAL', '{}', {}, '{}')".format(self.run_id,
                                                                                                                                                                         self.task_id,
@@ -537,11 +443,9 @@ class TaskScheduler(BrynQ):
                 for i in email_to:
                     email_list.append({'name': 'BrynQ User', 'mail': i.strip()})
                 # Set the content of the mail and all other stuff
-                task = self.mysql.select(table='task_scheduler', selection='title', filter=f'WHERE id = {self.task_id}')[0][
-                    0]
+                task = self.mysql.select(table='data_interfaces', selection='title', filter=f'WHERE id = {self.task_id}')[0][0]
                 finished_at = \
-                    self.mysql.select(table='task_scheduler', selection='last_reload', filter=f'WHERE id = {self.task_id}')[0][
-                        0]
+                    self.mysql.select(table='task_scheduler', selection='last_reload', filter=f'WHERE data_interface_id = {self.task_id}')[0][0]
                 if failed:
                     subject = f'Task \'{task}\' has failed'
                     content = f'Task \'{task}\' with task ID \'{self.task_id}\' failed during its last run and was stopped at {finished_at}. ' \

{brynq_sdk_task_scheduler-1.1.3 → brynq_sdk_task_scheduler-1.2.0}/brynq_sdk_task_scheduler.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: brynq-sdk-task-scheduler
-Version: 1.1.3
+Version: 1.2.0
 Summary: Code to execute tasks in BrynQ.com with the task scheduler
 Home-page: UNKNOWN
 Author: BrynQ

{brynq_sdk_task_scheduler-1.1.3 → brynq_sdk_task_scheduler-1.2.0}/setup.py

@@ -3,7 +3,7 @@ from setuptools import setup
 
 setup(
     name='brynq_sdk_task_scheduler',
-    version='1.1.3',
+    version='1.2.0',
     description='Code to execute tasks in BrynQ.com with the task scheduler',
     long_description='Code to execute tasks in the BrynQ.com platform with the task scheduler',
     author='BrynQ',