orbitkit 0.8.38__tar.gz → 0.8.39__tar.gz

{orbitkit-0.8.38/orbitkit.egg-info → orbitkit-0.8.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: orbitkit
-Version: 0.8.38
+Version: 0.8.39
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.39/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.39

{orbitkit-0.8.38 → orbitkit-0.8.39}/orbitkit/airflow_handler/data_preprocessing.py

@@ -102,6 +102,15 @@ class DocumentProcessor:
         report_type_ids = doc.get('x_orbit_data', {}).get('report_type_id_list', [])
         return "extract" if report_type_ids == ['19999'] else target_stage
 
+    @staticmethod
+    def update_target_stage_for_reported_at(doc, target_stage):
+        date_str = doc.get('x_reported_at_utc_date', '1970-01-01')
+        try:
+            reported_date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
+        except ValueError:
+            reported_date = datetime.datetime(1970, 1, 1)
+        return "extract" if reported_date < datetime.datetime(2020, 1, 1) else target_stage
+
     @classmethod
     def create_record(cls, doc, start_stage):
         attachments = doc.get('x_attachments', []) if start_stage == 'convert' else doc.get('x_attachments_pdf', [])
@@ -141,7 +150,7 @@ class DocumentProcessor:
         }
 
     @classmethod
-    def process(cls, doc):
+    def process(cls, doc, check_doc):
         report_id = doc['_id']
         # 筛选文件
         doc = cls.stock_us_filter_by_is_primary(doc)
@@ -156,7 +165,11 @@ class DocumentProcessor:
             return cls.create_result_info("xbrl", "XBRL format cannot be processed.", xbrl_data)
 
         start_stage, target_stage, x_spider_name = cls.get_start_stage_target_stage(doc)
-        target_stage = cls.update_target_stage_for_report_type(doc, target_stage)
+
+        # 特殊条件下的数据不做embedding ('19999'类型和报告日期小于2020-01-01)
+        if target_stage == 'embedding' and check_doc:
+            target_stage = cls.update_target_stage_for_report_type(doc, target_stage)
+            target_stage = cls.update_target_stage_for_reported_at(doc, target_stage)
 
         if cls.DATA_PROCESS_STEPS.index(target_stage) < cls.DATA_PROCESS_STEPS.index(start_stage):
             return cls.create_result_info("step_error",

{orbitkit-0.8.38 → orbitkit-0.8.39}/orbitkit/airflow_handler/file_flow_entry_process.py

@@ -180,7 +180,7 @@ class FilingOfficialProcessor:
 
     def process_task_entry(self, source: Literal["filing_data", "filing_reports_astock_test0822", "reports_view"],
                            query: dict, tags: list[str], priority: str,
-                           is_important: bool = False):
+                           is_important: bool = False, check_doc: bool = True):
 
         if source == 'reports_view':
             collections = self.source_map[source]
@@ -199,7 +199,7 @@ class FilingOfficialProcessor:
                 self.all_stat_count['all'] += 1
                 for orbit_entity_id in doc['x_orbit_data']['perm_id_list']:
                     perm_id_set.add(orbit_entity_id)
-                process_data.append(self.data_processor.process(doc))
+                process_data.append(self.data_processor.process(doc, check_doc))
                 if len(process_data) >= self.max_batch_size:
                     file_flow_info, xbrl_data, except_id_list, doc_error_list = self.data_processor.split_data_by_spider_name_and_step(
                         process_data)

{orbitkit-0.8.38 → orbitkit-0.8.39/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: orbitkit
-Version: 0.8.38
+Version: 0.8.39
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.38/orbitkit/VERSION

@@ -1 +0,0 @@
-0.8.38