orbitkit 0.8.73__tar.gz → 0.8.74__tar.gz

{orbitkit-0.8.73/orbitkit.egg-info → orbitkit-0.8.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.73
+Version: 0.8.74
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.74/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.74

{orbitkit-0.8.73 → orbitkit-0.8.74}/orbitkit/airflow_handler/data_preprocessing.py

@@ -118,18 +118,19 @@ class DocumentProcessor:
     @staticmethod
     def update_target_stage_by_report_type(doc, target_stage):
         report_type_ids = doc.get('x_orbit_data', {}).get('report_type_id_list', [])
-        return "extract" if report_type_ids == ['19999'] else target_stage
-
-    @staticmethod
-    def update_target_stage_by_reported_at(doc, target_stage):
-        date_str = doc.get('x_reported_at_utc_date', '1970-01-01')
-        now = datetime.datetime.now()
-        one_year_ago = now - relativedelta(years=1)
-        try:
-            reported_date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
-        except ValueError:
-            reported_date = datetime.datetime(1970, 1, 1)
-        return "extract" if reported_date < one_year_ago else target_stage
+        allow_embedding_types = {'10002', '10085', '10076', '10122', '10311', '10178', '10075', '10090', '10050'}
+        return target_stage if any(rid in allow_embedding_types for rid in report_type_ids) else "extract"
+
+    # @staticmethod
+    # def update_target_stage_by_reported_at(doc, target_stage):
+    #     date_str = doc.get('x_reported_at_utc_date', '1970-01-01')
+    #     now = datetime.datetime.now()
+    #     one_year_ago = now - relativedelta(years=1)
+    #     try:
+    #         reported_date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
+    #     except ValueError:
+    #         reported_date = datetime.datetime(1970, 1, 1)
+    #     return "extract" if reported_date < one_year_ago else target_stage
 
     @staticmethod
     def update_target_stage_by_perm_match(doc, target_stage):
@@ -200,7 +201,7 @@ class DocumentProcessor:
         # 判断 特殊条件下的数据不做embedding ('19999'类型和报告日期小于2023-01-01)
         if target_stage == 'embedding' and not custom_process_step:
             target_stage = cls.update_target_stage_by_report_type(doc, target_stage)
-            target_stage = cls.update_target_stage_by_reported_at(doc, target_stage)
+            # target_stage = cls.update_target_stage_by_reported_at(doc, target_stage)
             target_stage = cls.update_target_stage_by_perm_match(doc, target_stage)
             # 特殊情况下只需要做embedding 但是这个数据被条件限制为只做到提取时状态异常
             if start_stage == 'embedding' and target_stage == 'extract':

{orbitkit-0.8.73 → orbitkit-0.8.74/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.73
+Version: 0.8.74
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.73/orbitkit/VERSION

@@ -1 +0,0 @@
-0.8.73