mns-scheduler 1.4.5.6__tar.gz → 1.4.8.5__tar.gz

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.8.5}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns_scheduler
-Version: 1.4.5.6
+Version: 1.4.8.5

mns_scheduler-1.4.8.5/mns_scheduler/company_info/em_stock_info/clean_em_us_hk_stock_info.py

@@ -0,0 +1,30 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_common.db.MongodbUtil import MongodbUtil
+from loguru import logger
+from datetime import datetime
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+
+mongodb_util = MongodbUtil('27017')
+
+
+def clean_us_stock_concept_industry():
+    em_us_stock_info_df = mongodb_util.find_all_data(extra_income_db_name.EM_US_STOCK_INFO)
+    return em_us_stock_info_df
+
+
+if __name__ == '__main__':
+    em_us_stock_info_test_df = clean_us_stock_concept_industry()
+    em_us_stock_info_test_industry_df = em_us_stock_info_test_df.loc[em_us_stock_info_test_df['industry'] != '-']
+
+    em_us_stock_info_test_no_industry_df = em_us_stock_info_test_df.loc[em_us_stock_info_test_df['industry'] == '-']
+    em_us_stock_info_test_no_industry_df = em_us_stock_info_test_no_industry_df.sort_values(by=['flow_mv'],
+                                                                                            ascending=False)
+
+    print(em_us_stock_info_test_industry_df)

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.8.5}/mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py

@@ -23,8 +23,8 @@ mongodb_util = MongodbUtil('27017')
 
 def sync_all_em_stock_info():
     logger.info("同步东方财富a,etf,kzz,us,hk,信息开始")
-    # 同步东方财富A股股票信息
-    sync_stock_info()
+    # # 同步东方财富A股股票信息
+    # sync_stock_info()
     # 同步东方财富A股可转债信息
     sync_kzz_info()
     # 同步东方财富A股ETF信息
@@ -34,7 +34,7 @@ def sync_all_em_stock_info():
     # 同步东方财富港股通信息
     sync_hk_ggt_stock_info()
     # 同步东方财富美股信息
-    sync_us_stock_info()
+    # sync_us_stock_info()
 
     logger.info("同步东方财富a,etf,kzz,us,hk,信息完成")
 
@@ -48,6 +48,7 @@ def sync_kzz_info():
         em_kzz_info = east_money_debt_api.get_kzz_real_time_quotes(30, 6)
         em_kzz_info['_id'] = em_kzz_info['symbol']
         em_kzz_info['sync_time'] = str_now_date
+        em_kzz_info = em_kzz_info.fillna(0)
         mongodb_util.save_mongo(em_kzz_info, extra_income_db_name.EM_KZZ_INFO)
     except BaseException as e:
         logger.error("同步东方财富可转债信息异常:{}", e)
@@ -62,6 +63,7 @@ def sync_etf_info():
         em_etf_info = east_money_etf_api.get_etf_real_time_quotes(30, 6)
         em_etf_info['_id'] = em_etf_info['symbol']
         em_etf_info['sync_time'] = str_now_date
+        em_etf_info = em_etf_info.fillna(0)
         mongodb_util.save_mongo(em_etf_info, extra_income_db_name.EM_ETF_INFO)
     except BaseException as e:
         logger.error("同步东方财富ETF信息异常:{}", e)
@@ -73,10 +75,11 @@ def sync_stock_info():
     str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
     logger.error("同步东方财富股票信息")
     try:
-        em_etf_info = east_money_stock_a_v2_api.get_stock_real_time_quotes(60)
-        em_etf_info['_id'] = em_etf_info['symbol']
-        em_etf_info['sync_time'] = str_now_date
-        mongodb_util.save_mongo(em_etf_info, extra_income_db_name.EM_A_STOCK_INFO)
+        em_stock_info = east_money_stock_a_v2_api.get_stock_real_time_quotes(60)
+        em_stock_info['_id'] = em_stock_info['symbol']
+        em_stock_info['sync_time'] = str_now_date
+        em_stock_info = em_stock_info.fillna(0)
+        mongodb_util.save_mongo(em_stock_info, extra_income_db_name.EM_A_STOCK_INFO)
     except BaseException as e:
         logger.error("同步东方财富ETF信息异常:{}", e)
 
@@ -93,6 +96,7 @@ def sync_hk_stock_info():
         em_hk_info = east_money_stock_hk_api.get_hk_real_time_quotes(30, em_cookie)
         em_hk_info['_id'] = em_hk_info['symbol']
         em_hk_info['sync_time'] = str_now_date
+        em_hk_info = em_hk_info.fillna(0)
         mongodb_util.save_mongo(em_hk_info, extra_income_db_name.EM_HK_STOCK_INFO)
     except BaseException as e:
         logger.error("同步东方财富ETF信息异常:{}", e)
@@ -109,6 +113,7 @@ def sync_hk_ggt_stock_info():
         em_hk_gtt_info = east_money_stock_hk_gtt_api.get_ggt_real_time_quotes(em_cookie, 30, 6)
         em_hk_gtt_info['_id'] = em_hk_gtt_info['symbol']
         em_hk_gtt_info['sync_time'] = str_now_date
+        em_hk_gtt_info = em_hk_gtt_info.fillna(0)
         mongodb_util.save_mongo(em_hk_gtt_info, extra_income_db_name.EM_HK_GGT_STOCK_INFO)
     except BaseException as e:
         logger.error("同步东方财富ETF信息异常:{}", e)
@@ -120,14 +125,16 @@ def sync_us_stock_info():
     str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
     logger.error("同步东方财富美股信息")
     em_cookie = cookie_info_service.get_em_cookie()
-    us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, None)
+    us_stock_info = east_money_stock_us_api.get_us_real_time_quotes(30, em_cookie)
     us_stock_info['_id'] = us_stock_info['symbol']
     us_stock_info['sync_time'] = str_now_date
-    mongodb_util.save_mongo(us_stock_info, extra_income_db_name.EM_US_STOCK_INFO)
+
+    us_stock_info = us_stock_info.fillna(0)
+    mongodb_util.save_mongo(us_stock_info, extra_income_db_name.US_STOCK_INFO_EM)
 
 
 if __name__ == '__main__':
-    sync_stock_info()
+    sync_all_em_stock_info()
     # em_cookie = cookie_info_service.get_em_cookie()
     # em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, None)
     # em_us_stock_info['_id'] = em_us_stock_info['symbol']

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.8.5}/mns_scheduler/concept/clean/ths_concept_clean_api.py

@@ -13,7 +13,7 @@ from loguru import logger
 import mns_common.component.company.company_common_service_api as company_common_service_api
 import mns_common.constant.db_name_constant as db_name_constant
 import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
-import mns_common.api.ths.concept.app.ths_concept_index_app as ths_concept_index_app
+import mns_common.utils.data_frame_util as data_frame_util
 
 mongodb_util = MongodbUtil('27017')
 
@@ -102,32 +102,75 @@ def update_null_name():
     ths_concept_list = ths_concept_list.sort_values(by=['_id'], ascending=False)
 
     for concept_one in ths_concept_list.itertuples():
-        concept_code = concept_one.symbol
-        name = concept_one.name
-        exist_url = concept_one.url
+        try:
+            concept_code = concept_one.symbol
+            name = concept_one.name
+            exist_url = concept_one.url
+
+            if bool(1 - is_valid_name(name)):
+                # 更新空字符串
+                update_one_ths_concept_name(concept_code)
+                time.sleep(10)
+
+            if exist_url == '' or pd.isna(exist_url):
+                url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
+                str_now_time = concept_one.str_day + " " + "00:00:00"
+                query_concept = {"symbol": concept_code}
+                new_values = {'$set': {"url": url, "str_now_time": str_now_time}}
+                mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
+        except BaseException as e:
+            logger.error("更新空字符串名字异常:{},{}", concept_code, str(e))
+
+
+# 更新一个概念 空字符串
+def update_one_ths_concept_name(concept_code):
+    concept_name = ths_concept_index_web.get_concept_name(concept_code)
+    query_concept = {"symbol": concept_code}
+    new_values = {'$set': {"name": concept_name}}
+    mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
+
+    new_values_detail = {'$set': {"concept_name": concept_name}}
 
-        if name == '':
-            concept_name = ths_concept_index_web.get_concept_name(concept_code)
-            query_concept = {"symbol": concept_code}
-            new_values = {'$set': {"name": concept_name}}
-            mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
+    query_concept_detail = {"concept_code": concept_code}
 
-            new_values_detail = {'$set': {"concept_name": concept_name}}
+    mongodb_util.update_many(query_concept_detail, new_values_detail, 'ths_stock_concept_detail')
 
-            query_concept_detail = {"concept_code": concept_code}
 
-            mongodb_util.update_many(query_concept_detail, new_values_detail, 'ths_stock_concept_detail')
-            time.sleep(10)
+# 更新ths 详情里面空名称的数据
+def query_ths_detail_null_name():
+    query = {"$or": [{"concept_name": ""},
+                     {"concept_name": " "},
+                     {"concept_name": {"$exists": False}}]}
+
+    ths_stock_concept_detail_list_df = mongodb_util.find_query_data(db_name_constant.THS_STOCK_CONCEPT_DETAIL, query)
+    if data_frame_util.is_empty(ths_stock_concept_detail_list_df):
+        return
+    null_concept_code_list = set(ths_stock_concept_detail_list_df['concept_code'])
+    for null_concept_code in null_concept_code_list:
+        try:
+            update_one_ths_concept_name(null_concept_code)
+        except BaseException as e:
+            logger.error("更新空字符串名字异常:{},{}", null_concept_code, str(e))
+
 
-        if exist_url == '' or pd.isna(exist_url):
-            url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
-            str_now_time = concept_one.str_day + " " + "00:00:00"
-            query_concept = {"symbol": concept_code}
-            new_values = {'$set': {"url": url, "str_now_time": str_now_time}}
-            mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
+def is_valid_name(name):
+    """
+    接口专用：判断name是否为有效非空值
+    :param name: 待判断的变量（字符串/NaN/None/任意类型）
+    :return: 布尔值 True=有效非空，False=空/空格/NaN/无效
+    """
+    # 1. 判断是否为 NaN（浮点型，且自身不等于自身）
+    if isinstance(name, float) and name != name:
+        return False
+    # 2. 判断是否为 None / 空字符串 / 纯空格字符串
+    if not name or not str(name).strip():
+        return False
+    # 3. 以上都不满足 → 有效非空值
+    return True
 
 
 if __name__ == '__main__':
+    query_ths_detail_null_name()
     # update_one_concept_relevance_industry(886095)
     update_null_name()
     logger.info("开始")

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.8.5}/mns_scheduler/db/script/sync/remote_data_sync_to_local.py

@@ -96,11 +96,10 @@ def sync_open_data():
 
 
 if __name__ == '__main__':
-    sync_zt_data('2025-12-21')
-    sync_zt_data('2025-12-23')
-    sync_zt_data('2025-12-24')
-    sync_zt_data('2025-12-25')
-    sync_zt_data('2025-12-26')
+    sync_zt_data('2026-01-15')
+    sync_zt_data('2026-01-14')
+    sync_zt_data('2026-01-13')
+    sync_zt_data('2026-01-12')
     remote_data()
     # sync_zt_data('2025-08-26')
     # sync_zt_data('2025-08-25')

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.8.5}/mns_scheduler/finance/sync_financial_report_service_task.py

@@ -17,7 +17,7 @@ import mns_scheduler.risk.financial_report_risk_check_api as financial_report_ri
 import mns_common.utils.data_frame_util as data_frame_util
 import mns_scheduler.finance.em.finance_common_api as finance_common_api
 import mns_scheduler.risk.compliance.undisclosed_annual_report_api as undisclosed_annual_report_api
-import mns_scheduler.finance.xue_qiu.sync_xue_qiu_fiance_data as sync_xue_qiu_fiance_data
+import mns_scheduler.finance.xue_qiu.sync_xue_qiu_finance_data as sync_xue_qiu_fiance_data
 
 mongodb_util = MongodbUtil('27017')
 
@@ -30,24 +30,25 @@ mongodb_util = MongodbUtil('27017')
 #    3季报:每年10月1日--10月31日
 #    4季报(年报) :每年1月1日--4月30日
 
-def sync_financial_report():
+def sync_financial_report(symbol_list):
     now_date = datetime.now()
     now_year = now_date.year
     now_month = now_date.month
     sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
 
     if now_month in [1, 2, 3, 4]:
-        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year - 1) + '年报', None)
+        sync_xue_qiu_fiance_data.sync_xue_qiu_each_period_report(str(now_year - 1) + '年报', symbol_list)
 
         # 年报
         period = 4
         period_time = str(now_year - 1) + "-12-31 00:00:00"
+        # 东财财报
         sync_em_profit_report(period_time, sync_time, period)
         sync_em_asset_liability_report(period_time, sync_time, period)
 
         # 一季报
         if now_month == 4:
-            sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '一季报', None)
+            sync_xue_qiu_fiance_data.sync_xue_qiu_each_period_report(str(now_year) + '一季报', symbol_list)
 
             period = 1
             period_time = str(now_year) + "-03-31 00:00:00"
@@ -58,7 +59,7 @@ def sync_financial_report():
 
     # 二季报
     elif now_month in [7, 8]:
-        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '中报', None)
+        sync_xue_qiu_fiance_data.sync_xue_qiu_each_period_report(str(now_year) + '中报', symbol_list)
 
         period = 2
         period_time = str(now_year) + "-06-30 00:00:00"
@@ -69,7 +70,7 @@ def sync_financial_report():
 
     # 三季报
     elif now_month == 10:
-        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '三季报', None)
+        sync_xue_qiu_fiance_data.sync_xue_qiu_each_period_report(str(now_year) + '三季报', symbol_list)
 
         period = 3
         period_time = str(now_year) + "-09-30 00:00:00"
@@ -78,8 +79,8 @@ def sync_financial_report():
 
     elif now_month == 5:
 
-        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year - 1) + '年报', None)
-        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '一季报', None)
+        sync_xue_qiu_fiance_data.sync_xue_qiu_each_period_report(str(now_year - 1) + '年报', symbol_list)
+        sync_xue_qiu_fiance_data.sync_xue_qiu_each_period_report(str(now_year) + '一季报', symbol_list)
 
         # 补偿年报和一季度,出不了报告的
         miss_period_04 = 4
@@ -94,7 +95,7 @@ def sync_financial_report():
         sync_em_asset_liability_report(period_time_01, sync_time, period_01)
 
     elif now_month == 9:
-        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '中报', None)
+        sync_xue_qiu_fiance_data.sync_xue_qiu_each_period_report(str(now_year) + '中报', symbol_list)
 
         # 补偿二季度
         period_02 = 2
@@ -103,14 +104,13 @@ def sync_financial_report():
         sync_em_asset_liability_report(period_time_02, sync_time, period_02)
 
     elif now_month in [11, 12]:
-        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '三季报', None)
+        sync_xue_qiu_fiance_data.sync_xue_qiu_each_period_report(str(now_year) + '三季报', symbol_list)
 
         # 补偿三季度
         period_03 = 3
         period_time_03 = str(now_year) + "-09-30 00:00:00"
         sync_em_profit_report(period_time_03, sync_time, period_03)
         sync_em_asset_liability_report(period_time_03, sync_time, period_03)
-
     # 未出报告check
     undisclosed_annual_report_api.un_disclosed_report_check(sync_time, now_year, period, period_time)
     # 新股或者未出报告的
@@ -122,8 +122,8 @@ def sync_em_asset_liability_report(period_time, sync_time, period):
     un_report_asset_df = finance_common_api.find_un_report_symbol(period_time,
                                                                   db_name_constant.EM_STOCK_ASSET_LIABILITY)
     for un_report_asset_one in un_report_asset_df.itertuples():
+        symbol = un_report_asset_one.symbol
         try:
-            symbol = un_report_asset_one.symbol
             new_asset_df = em_financial_asset_liability_sync_service_api.get_em_asset_liability_api(symbol)
             # 负债比
             new_asset_df['liability_ratio'] = round(
@@ -147,8 +147,9 @@ def sync_em_asset_liability_report(period_time, sync_time, period):
 def sync_em_profit_report(period_time, sync_time, period):
     un_report_profit_df = finance_common_api.find_un_report_symbol(period_time, db_name_constant.EM_STOCK_PROFIT)
     for un_report_profit_one in un_report_profit_df.itertuples():
+        symbol = un_report_profit_one.symbol
         try:
-            symbol = un_report_profit_one.symbol
+
             new_profit_df = em_financial_profit_sync_service_api.get_em_profit_api(symbol)
             if data_frame_util.is_empty(new_profit_df):
                 continue
@@ -168,8 +169,8 @@ def sync_miss_report(sync_time):
     query = {"total_operate_income": 0}
     un_report_company_info = mongodb_util.find_query_data(db_name_constant.COMPANY_INFO, query)
     for un_report_one in un_report_company_info.itertuples():
+        symbol = un_report_one.symbol
         try:
-            symbol = un_report_one.symbol
             new_profit_df = em_financial_profit_sync_service_api.get_em_profit_api(symbol)
             if data_frame_util.is_empty(new_profit_df):
                 continue

mns_scheduler-1.4.5.6/mns_scheduler/finance/xue_qiu/sync_xue_qiu_fiance_data.py → mns_scheduler-1.4.8.5/mns_scheduler/finance/xue_qiu/sync_xue_qiu_finance_data.py

@@ -25,8 +25,12 @@ mongodb_util_27017 = MongodbUtil('27017')
 # cash_flow 现金流量
 # balance 资产负债
 # 同步所有股票 报表
-def sync_all_stocks_report():
+def sync_all_stocks_report(symbol_list):
     em_a_stock_info_df = em_stock_info_api.get_a_stock_info()
+    # 代码列表不为空的时候
+    if len(symbol_list) > 0:
+        em_a_stock_info_df = em_a_stock_info_df.loc[em_a_stock_info_df['symbol'].isin(symbol_list)]
+
     em_a_stock_info_df = common_service_fun_api.add_pre_prefix(em_a_stock_info_df)
     # 或等效写法 df['A'].str[0:6]
 
@@ -35,7 +39,10 @@ def sync_all_stocks_report():
     xue_qiu_cookie = cookie_info_service.get_xue_qiu_cookie()
     report_type_list = ['income', 'balance', 'cash_flow']
     for stock_one in em_a_stock_info_df.itertuples():
-        fail_list = save_one_symbol_data(stock_one, report_type_list, xue_qiu_cookie, True, fail_list, '', False)
+        fail_list = save_one_symbol_data(stock_one, report_type_list,
+                                         xue_qiu_cookie, True,
+                                         fail_list, '',
+                                         False)
 
     handle_number = 0
     # 处理失败的
@@ -49,10 +56,11 @@ def sync_all_stocks_report():
 
 
 def save_one_symbol_data(stock_one, report_type_list, xue_qiu_cookie, save_tag, fail_list, report_name, check_exist):
+    symbol = stock_one.symbol
     try:
         symbol_prefix = stock_one.symbol_prefix
         name = stock_one.name
-        symbol = stock_one.symbol
+
         for report_type in report_type_list:
             if report_type == 'income':
                 col_name = extra_income_db_name.XUE_QIU_LRB_INCOME
@@ -60,6 +68,8 @@ def save_one_symbol_data(stock_one, report_type_list, xue_qiu_cookie, save_tag,
                 col_name = extra_income_db_name.XUE_QIU_ASSET_DEBT
             elif report_type == 'cash_flow':
                 col_name = extra_income_db_name.XUE_QIU_CASH_FLOW
+            else:
+                col_name = extra_income_db_name.XUE_QIU_LRB_INCOME
             if check_exist:
                 query_exist = {'symbol': symbol, 'report_name': report_name}
                 # 存在数据 不在同步
@@ -130,10 +140,10 @@ def save_one_symbol_data(stock_one, report_type_list, xue_qiu_cookie, save_tag,
     return fail_list
 
 
-def sync_xue_qiu_very_period_report(report_name, symbol):
+def sync_xue_qiu_each_period_report(report_name, symbol_list):
     em_a_stock_info_df = em_stock_info_api.get_a_stock_info()
-    if symbol is not None:
-        em_a_stock_info_df = em_a_stock_info_df.loc[em_a_stock_info_df['symbol'] == symbol]
+    if len(symbol_list) > 0:
+        em_a_stock_info_df = em_a_stock_info_df.loc[em_a_stock_info_df['symbol'].isin(symbol_list)]
     em_a_stock_info_df = common_service_fun_api.add_pre_prefix(em_a_stock_info_df)
     # 或等效写法 df['A'].str[0:6]
 
@@ -158,4 +168,4 @@ def sync_xue_qiu_very_period_report(report_name, symbol):
 
 
 if __name__ == '__main__':
-    sync_all_stocks_report()
+    sync_all_stocks_report([])

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.8.5}/mns_scheduler/k_line/clean/k_line_info_clean_task.py

@@ -116,6 +116,7 @@ def multi_threaded_k_line_sync(str_day):
     # 将list_date列中的所有NaN值设置为19890604
     real_time_quotes_now['list_date'].fillna(19890604, inplace=True)
     real_time_quotes_now['list_date'] = real_time_quotes_now['list_date'].replace(99990909, 19890604)
+    real_time_quotes_now['list_date'] = real_time_quotes_now['list_date'].replace(0, 19890604)
 
     # 将日期数值转换为日期时间格式
     real_time_quotes_now['list_date_01'] = pd.to_datetime(real_time_quotes_now['list_date'], format='%Y%m%d')

mns_scheduler-1.4.8.5/mns_scheduler/kpl/theme/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)