orbitkit 0.8.53__tar.gz → 0.8.56__tar.gz

{orbitkit-0.8.53/orbitkit.egg-info → orbitkit-0.8.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.53
+Version: 0.8.56
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.56/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.56

{orbitkit-0.8.53 → orbitkit-0.8.56}/orbitkit/airflow_handler/data_preprocessing.py

@@ -68,7 +68,9 @@ class DocumentProcessor:
         if not doc:
             return None
         suffixes = {cls.get_file_suffix(att['store_path']) for att in doc.get('x_attachments', [])}
-        return doc if suffixes.issubset(cls.ALL_ALLOWED_SUFFIXES) else None
+        if suffixes.issubset(cls.ALL_ALLOWED_SUFFIXES) or doc['x_status_list']['status_convert']['status'] == "convert_done":
+            return doc
+        return None
 
     @classmethod
     def xbrl_type_check(cls, doc):
@@ -126,6 +128,11 @@ class DocumentProcessor:
             reported_date = datetime.datetime(1970, 1, 1)
         return "extract" if reported_date < datetime.datetime(2023, 1, 1) else target_stage
 
+    @staticmethod
+    def update_target_stage_by_perm_match(doc, target_stage):
+        perm_match_status = doc['x_status_list']['status_perm']['status']
+        return target_stage if perm_match_status in {'perm_match_part', 'perm_match'} else "extract"
+
     @classmethod
     async def create_record(cls, doc, start_stage, important_level):
         attachments = doc.get('x_attachments', []) if start_stage == 'convert' else doc.get('x_attachments_pdf', [])
@@ -182,10 +189,11 @@ class DocumentProcessor:
 
         start_stage, target_stage, x_spider_name = cls.get_start_stage_target_stage(doc, custom_process_step)
 
-        # 判断 特殊条件下的数据不做embedding ('19999'类型和报告日期小于2020-01-01)
+        # 判断 特殊条件下的数据不做embedding ('19999'类型和报告日期小于2023-01-01)
         if target_stage == 'embedding' and not custom_process_step:
             target_stage = cls.update_target_stage_by_report_type(doc, target_stage)
             target_stage = cls.update_target_stage_by_reported_at(doc, target_stage)
+            target_stage = cls.update_target_stage_by_perm_match(doc, target_stage)
             # 特殊情况下只需要做embedding 但是这个数据被条件限制为只做到提取时状态异常
             if start_stage == 'embedding' and target_stage == 'extract':
                 start_stage = 'success'

{orbitkit-0.8.53 → orbitkit-0.8.56}/orbitkit/airflow_handler/file_flow_entry_process.py

@@ -112,14 +112,20 @@ class FilingOfficialProcessor:
     def update_doc_status_to_convert(self, collection, report_id_list):
         if len(report_id_list) == 0:
             return
-        collection.update_many({
-            '_id': {'$in': report_id_list}
-        }, {'$set': {
-            "x_status_list.status_convert.status": "convert_failed",
-            "x_status_list.status_convert.status_txt": "convert_txt_init",
-            "x_status_list.status_convert.status_meta": "meta_init",
-            "x_updated_date": datetime.now(tz=pytz.timezone('UTC')).strftime("%Y-%m-%dT%H:%M:%S%z"),
-        }})
+        collection.update_many(
+            {"_id": {"$in": report_id_list}},
+            {
+                "$set": {
+                    "x_status_list.status_convert.status": "convert_failed",
+                    "x_status_list.status_convert.status_txt": "convert_txt_init",
+                    "x_status_list.status_convert.status_meta": "meta_init",
+                    "x_updated_date": datetime.now(tz=pytz.timezone('UTC')).strftime("%Y-%m-%dT%H:%M:%S%z"),
+                },
+                "$unset": {
+                    "x_attachments_pdf": ""
+                }
+            }
+        )
         logger.info(f'Unable to convert {len(report_id_list)} document(s) due to unsupported file type.')
 
     def update_extends_fields(self, perm_id_list, file_flow_info):

{orbitkit-0.8.53 → orbitkit-0.8.56/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: orbitkit
-Version: 0.8.53
+Version: 0.8.56
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.53/orbitkit/VERSION

@@ -1 +0,0 @@
-0.8.53