orbitkit 0.8.58__tar.gz → 0.8.59__tar.gz

{orbitkit-0.8.58/orbitkit.egg-info → orbitkit-0.8.59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.58
+Version: 0.8.59
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.59/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.59

{orbitkit-0.8.58 → orbitkit-0.8.59}/orbitkit/airflow_handler/file_flow_entry_process.py

@@ -151,7 +151,7 @@ class FilingOfficialProcessor:
 
         return file_flow_info
 
-    def send_task(self, file_flow_info, tags, is_important, priority, spider_name_source_type):
+    def send_task(self, file_flow_info, tags, is_important, priority, spider_name_source_type, queue_type):
         for step_str, records in file_flow_info.items():
             steps = step_str.split('@__@')
             start_stage = steps[0]
@@ -172,12 +172,14 @@ class FilingOfficialProcessor:
                                                                             tag=x_spider_name,
                                                                             priority=priority,
                                                                             source_type=spider_name_source_type[
-                                                                                x_spider_name])
+                                                                                x_spider_name],
+                                                                            queue_type=queue_type)
             else:
                 status, ids, message = self.file_handler.entry_point(records=records, start_stage=start_stage,
                                                                      target_stage=target_stage, tags=tags,tag=x_spider_name,
                                                                      priority=priority,
-                                                                     source_type=spider_name_source_type[x_spider_name])
+                                                                     source_type=spider_name_source_type[x_spider_name],
+                                                                     queue_type=queue_type)
             self.all_stat_count['file_flow'] += len(records)
             logger.info(f"{len(records)}--{start_stage}-{target_stage}-{x_spider_name} status: {status}, message: {message}")
 
@@ -213,7 +215,7 @@ class FilingOfficialProcessor:
 
     async def process_task_entry(self, source: str,
                            query: dict, tags: list[str], priority: str,
-                           is_important: bool = False, custom_step: Optional[list[str]] = None, important_level = None, db_name: str = None):
+                           is_important: bool = False, custom_step: Optional[list[str]] = None, important_level = None, db_name: str = None, queue_type: str = 'Default'):
 
         if not important_level or not isinstance(important_level, int):
             important_level = 0
@@ -251,7 +253,7 @@ class FilingOfficialProcessor:
                 self.all_stat_count['doc_error'] += len(doc_error_list)
                 self.all_stat_count['step_error'] += len(except_id_list)
                 self.all_stat_count['xbrl'] += len(xbrl_data)
-                self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type)
+                self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type, queue_type)
                 self.send_xbrl_data_to_mongo(xbrl_data)
                 self.update_doc_status_to_convert(collection, doc_error_list)
                 process_data.clear()
@@ -264,7 +266,7 @@ class FilingOfficialProcessor:
             self.all_stat_count['doc_error'] += len(doc_error_list)
             self.all_stat_count['step_error'] += len(except_id_list)
             self.all_stat_count['xbrl'] += len(xbrl_data)
-            self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type)
+            self.send_task(file_flow_info, tags, is_important, priority, spider_name_source_type, queue_type)
             self.send_xbrl_data_to_mongo(xbrl_data)
             self.update_doc_status_to_convert(collection, doc_error_list)
             process_data.clear()

{orbitkit-0.8.58 → orbitkit-0.8.59}/orbitkit/airflow_handler/file_handler_v2.py

@@ -141,7 +141,8 @@ class FileFlowHandleV2:
             'updated_at': now,
             'tags': params['tags'],
             'tag': params['tag'],
-            'important_level': record.get('important_level', 0)
+            'important_level': record.get('important_level', 0),
+            'queue': params['queue']
         }
 
         step = {
@@ -225,6 +226,7 @@ class FileFlowHandleV2:
             tag: str = None,
             priority: str = '1',
             source_type: Optional[str] = None,
+            queue_type: str = 'Default'
     ) -> Tuple[bool, Any, str]:
         """
             普通任务接口
@@ -252,7 +254,7 @@ class FileFlowHandleV2:
         """
         return self._file_flow_entry_point_internal(
             records, start_stage, target_stage, tags, priority, source_type,tag,
-            urgent=False, clean_exist_data=False
+            urgent=False, clean_exist_data=False, queue_type=queue_type
         )
 
     def entry_point_urgent(
@@ -264,6 +266,7 @@ class FileFlowHandleV2:
             tag: str = None,
             priority: str = '1',
             source_type: Optional[str] = None,
+            queue_type: str = 'Default'
     ) -> Tuple[bool, Any, str]:
         """
             加急任务接口
@@ -291,7 +294,7 @@ class FileFlowHandleV2:
         """
         return self._file_flow_entry_point_internal(
             records, start_stage, target_stage, tags, priority, source_type, tag,
-            urgent=True, clean_exist_data=True
+            urgent=True, clean_exist_data=True, queue_type=queue_type
         )
 
     def _file_flow_entry_point_internal(
@@ -304,7 +307,8 @@ class FileFlowHandleV2:
             source_type: Optional[str],
             tag: str,
             urgent: bool,
-            clean_exist_data: bool = False
+            clean_exist_data: bool = False,
+            queue_type: str = 'Default'
     ) -> Tuple[bool, Any, str]:
         """核心处理逻辑"""
         params = {
@@ -315,7 +319,8 @@ class FileFlowHandleV2:
             'current_stage': start_stage,
             'source_type': source_type,
             'tag': tag,
-            'urgent': urgent
+            'urgent': urgent,
+            'queue': queue_type
         }
 
         is_valid, msg = self._validate_params(params)

{orbitkit-0.8.58 → orbitkit-0.8.59/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.58
+Version: 0.8.59
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.58/orbitkit/VERSION

@@ -1 +0,0 @@
-0.8.58