orbitkit 0.8.38__tar.gz → 0.8.40__tar.gz

{orbitkit-0.8.38/orbitkit.egg-info → orbitkit-0.8.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: orbitkit
-Version: 0.8.38
+Version: 0.8.40
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.40/orbitkit/VERSION

@@ -0,0 +1 @@
+0.8.40

{orbitkit-0.8.38 → orbitkit-0.8.40}/orbitkit/airflow_handler/data_preprocessing.py

@@ -1,9 +1,13 @@
 import os
 import datetime
 from collections import defaultdict
+import asyncio
+from googletrans import Translator
 
 
 class DocumentProcessor:
+    _translator = Translator()
+
     AUDIO_SUFFIXES = [".mp3", ".wav", ".aac", ".wma", ".m4a"]
     VIDEO_SUFFIXES = [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".mpeg", ".3gp", ".hevc"]
     PDF_SUFFIXES = [".pdf"]
@@ -17,6 +21,15 @@ class DocumentProcessor:
     def get_file_suffix(file_path):
         return f".{file_path.split('.')[-1]}".lower()
 
+    @classmethod
+    async def translate(cls, text, dest='en'):
+        result = await cls._translator.translate(text, dest=dest)
+        return result.text
+
+    @classmethod
+    def translate_sync(cls, text, dest='en'):
+        return asyncio.run(cls.translate(text, dest))
+
     @staticmethod
     def create_xbrl_template():
         return {
@@ -102,6 +115,15 @@ class DocumentProcessor:
         report_type_ids = doc.get('x_orbit_data', {}).get('report_type_id_list', [])
         return "extract" if report_type_ids == ['19999'] else target_stage
 
+    @staticmethod
+    def update_target_stage_for_reported_at(doc, target_stage):
+        date_str = doc.get('x_reported_at_utc_date', '1970-01-01')
+        try:
+            reported_date = datetime.datetime.strptime(date_str, "%Y-%m-%d")
+        except ValueError:
+            reported_date = datetime.datetime(1970, 1, 1)
+        return "extract" if reported_date < datetime.datetime(2020, 1, 1) else target_stage
+
     @classmethod
     def create_record(cls, doc, start_stage):
         attachments = doc.get('x_attachments', []) if start_stage == 'convert' else doc.get('x_attachments_pdf', [])
@@ -115,7 +137,8 @@ class DocumentProcessor:
                 _, ext = os.path.splitext(att['store_path'])
                 if ext in cls.AUDIO_SUFFIXES or ext in cls.VIDEO_SUFFIXES:
                     add_extends = {
-                        "title": doc['x_orbit_data']['report_title'],
+                        "original_title": doc['x_orbit_data']['report_title'],
+                        "title": cls.translate_sync(doc['x_orbit_data']['report_title']),
                         "published": doc['x_reported_at_utc_date'],
                         "tickers": [],
                         "perm_id_list": doc['x_orbit_data']['perm_id_list'],
@@ -141,7 +164,7 @@ class DocumentProcessor:
         }
 
     @classmethod
-    def process(cls, doc):
+    def process(cls, doc, check_doc):
         report_id = doc['_id']
         # 筛选文件
         doc = cls.stock_us_filter_by_is_primary(doc)
@@ -156,7 +179,11 @@ class DocumentProcessor:
             return cls.create_result_info("xbrl", "XBRL format cannot be processed.", xbrl_data)
 
         start_stage, target_stage, x_spider_name = cls.get_start_stage_target_stage(doc)
-        target_stage = cls.update_target_stage_for_report_type(doc, target_stage)
+
+        # 特殊条件下的数据不做embedding ('19999'类型和报告日期小于2020-01-01)
+        if target_stage == 'embedding' and check_doc:
+            target_stage = cls.update_target_stage_for_report_type(doc, target_stage)
+            target_stage = cls.update_target_stage_for_reported_at(doc, target_stage)
 
         if cls.DATA_PROCESS_STEPS.index(target_stage) < cls.DATA_PROCESS_STEPS.index(start_stage):
             return cls.create_result_info("step_error",

{orbitkit-0.8.38 → orbitkit-0.8.40}/orbitkit/airflow_handler/file_flow_entry_process.py

@@ -180,7 +180,7 @@ class FilingOfficialProcessor:
 
     def process_task_entry(self, source: Literal["filing_data", "filing_reports_astock_test0822", "reports_view"],
                            query: dict, tags: list[str], priority: str,
-                           is_important: bool = False):
+                           is_important: bool = False, check_doc: bool = True):
 
         if source == 'reports_view':
             collections = self.source_map[source]
@@ -199,7 +199,7 @@ class FilingOfficialProcessor:
                 self.all_stat_count['all'] += 1
                 for orbit_entity_id in doc['x_orbit_data']['perm_id_list']:
                     perm_id_set.add(orbit_entity_id)
-                process_data.append(self.data_processor.process(doc))
+                process_data.append(self.data_processor.process(doc, check_doc))
                 if len(process_data) >= self.max_batch_size:
                     file_flow_info, xbrl_data, except_id_list, doc_error_list = self.data_processor.split_data_by_spider_name_and_step(
                         process_data)

{orbitkit-0.8.38 → orbitkit-0.8.40}/orbitkit/airflow_handler/file_handler_v2.py

@@ -5,7 +5,8 @@ from contextlib import contextmanager
 from typing import List, Dict, Tuple, Any, Optional, Union
 from sqlalchemy.dialects.postgresql import array
 from sqlalchemy.orm import sessionmaker, scoped_session
-from sqlalchemy import create_engine, MetaData, Table, select, Column, Integer, insert, and_, text, delete, update, desc
+from sqlalchemy import create_engine, MetaData, Table, select, Column, Integer, insert, and_, text, delete, update, \
+    desc, LABEL_STYLE_TABLENAME_PLUS_COL
 
 logger = logging.getLogger(__name__)
 
@@ -397,9 +398,9 @@ class FileFlowHandleV2:
             select(op_meta)
             .where(and_(*conditions)) if conditions else select(op_meta)
         )
-        meta_subquery = meta_subquery.order_by(desc(op_meta.c.priority)).limit(limit_size).subquery()
+        meta_subquery = meta_subquery.order_by(desc(op_meta.c.priority)).limit(limit_size).subquery(name="public_op_meta")
         join_stmt = meta_subquery.outerjoin(op_attachment, meta_subquery.c.id == op_attachment.c.meta_id)
-        stmt = select(meta_subquery, op_attachment).select_from(join_stmt)
+        stmt = (select(meta_subquery, op_attachment).select_from(join_stmt).set_label_style(LABEL_STYLE_TABLENAME_PLUS_COL))
 
         with self.session_scope() as session:
             result = session.execute(stmt).fetchall()
@@ -407,10 +408,10 @@ class FileFlowHandleV2:
             meta_map = {}
             for row in result:
                 row_dict = dict(row._mapping)
-                meta_id = row_dict['id']
+                meta_id = row_dict['public_op_meta_id']
 
-                meta_fields: Dict[str, Any] = {k: v for k, v in row_dict.items() if k in op_meta.c}
-                attachment_fields: Dict[str, Any] = {k: v for k, v in row_dict.items() if k in op_attachment.c}
+                meta_fields: Dict[str, Any] = {k.replace('public_op_meta_', ''): v for k, v in row_dict.items() if k.startswith('public_op_meta_')}
+                attachment_fields: Dict[str, Any] = {k.replace('public_op_attachment_', ''): v for k, v in row_dict.items() if k.startswith('public_op_attachment_')}
 
                 meta_fields['created_at'] = meta_fields['created_at'].isoformat()
                 meta_fields['updated_at'] = meta_fields['updated_at'].isoformat()

{orbitkit-0.8.38 → orbitkit-0.8.40/orbitkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: orbitkit
-Version: 0.8.38
+Version: 0.8.40
 Summary: This project is only for Orbit Tech internal use.
 Home-page: https://github.com/clown-0726/orbitkit
 Author: Lilu Cao

orbitkit-0.8.38/orbitkit/VERSION

@@ -1 +0,0 @@
-0.8.38