mns-scheduler 1.2.9.6__py3-none-any.whl → 1.4.2.5__py3-none-any.whl

mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py

@@ -2,68 +2,79 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
-import datetime
-import mns_common.component.proxies.proxy_common_api as proxy_common_api
+
 from mns_common.db.MongodbUtil import MongodbUtil
 from loguru import logger
-import mns_common.api.em.real_time.east_money_stock_a_v2_api as east_money_stock_a_v2_api
 import mns_common.api.em.real_time.east_money_debt_api as east_money_debt_api
 import mns_common.api.em.real_time.east_money_etf_api as east_money_etf_api
+import mns_common.api.em.real_time.east_money_stock_a_v2_api as east_money_stock_a_v2_api
+from datetime import datetime
+import mns_common.constant.extra_income_db_name as extra_income_db_name
 import mns_common.api.em.real_time.east_money_stock_hk_api as east_money_stock_hk_api
-import mns_common.api.em.real_time.east_money_stock_us_api as east_money_stock_us_api
-import mns_common.constant.db_name_constant as db_name_constant
 import mns_common.component.cookie.cookie_info_service as cookie_info_service
+import mns_common.api.em.real_time.east_money_stock_us_api as east_money_stock_us_api
 
 mongodb_util = MongodbUtil('27017')
 
 
 def sync_all_em_stock_info():
-    logger.info("同步东方财富a,etf,kzz,us,hk,信息开始")
-    now_date = datetime.datetime.now()
+    now_date = datetime.now()
     str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
-    proxy_ip = proxy_common_api.get_proxy_ip(str_now_date, 5)
-    proxies = {"https": proxy_ip}
+
+    logger.info("同步东方财富a,etf,kzz,us,hk,信息开始")
+
     try:
-        em_a_stock_info_df = east_money_stock_a_v2_api.get_all_real_time_quotes(proxies)
+        # 这里需要使用详情接口 获取全量数据
+        em_a_stock_info_df = east_money_stock_a_v2_api.get_stock_real_time_quotes(60)
+        em_a_stock_info_df['list_date'] = em_a_stock_info_df['list_date'].fillna(19890604)
         em_a_stock_info_df['_id'] = em_a_stock_info_df['symbol']
-        mongodb_util.save_mongo(em_a_stock_info_df, db_name_constant.EM_A_STOCK_INFO)
+        em_a_stock_info_df['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_a_stock_info_df, extra_income_db_name.EM_A_STOCK_INFO)
     except BaseException as e:
         logger.error("同步东方财富A股信息异常:{}", e)
-
     try:
-        em_etf_info = east_money_etf_api.get_etf_real_time_quotes(proxies)
-        em_etf_info['_id'] = em_etf_info['symbol']
-        mongodb_util.save_mongo(em_etf_info, db_name_constant.EM_ETF_INFO)
+        em_etf_info = east_money_etf_api.get_etf_real_time_quotes(30, 6)
+        em_etf_info['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_etf_info, extra_income_db_name.EM_ETF_INFO)
     except BaseException as e:
         logger.error("同步东方财富ETF信息异常:{}", e)
 
     try:
-        em_kzz_info = east_money_debt_api.get_debt_real_time_quotes(proxies)
-
+        em_kzz_info = east_money_debt_api.get_kzz_real_time_quotes(30, 6)
         em_kzz_info['_id'] = em_kzz_info['symbol']
-        mongodb_util.save_mongo(em_kzz_info, db_name_constant.EM_KZZ_INFO)
+        em_kzz_info['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_kzz_info, extra_income_db_name.EM_KZZ_INFO)
     except BaseException as e:
         logger.error("同步东方财富可转债信息异常:{}", e)
 
-    em_cookie = cookie_info_service.get_em_cookie()
-    try:
-        em_hk_stock_info = east_money_stock_hk_api.get_hk_real_time_quotes(em_cookie, proxies)
-        em_hk_stock_info['_id'] = em_hk_stock_info['symbol']
-        mongodb_util.save_mongo(em_hk_stock_info, db_name_constant.EM_HK_STOCK_INFO)
-    except BaseException as e:
-        logger.error("同步东方财富港股信息异常:{}", e)
-
-    try:
-        em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, proxies)
-        em_us_stock_info['_id'] = em_us_stock_info['symbol']
-        mongodb_util.save_mongo(em_us_stock_info, db_name_constant.EM_US_STOCK_INFO)
-    except BaseException as e:
-        logger.error("同步东方财富美股信息异常:{}", e)
+    # em_cookie = cookie_info_service.get_em_cookie()
+    # try:
+    #     em_hk_stock_info = east_money_stock_hk_api.get_hk_real_time_quotes(em_cookie, proxies)
+    #     em_hk_stock_info['_id'] = em_hk_stock_info['symbol']
+    #     mongodb_util.save_mongo(em_hk_stock_info, extra_income_db_name.EM_HK_STOCK_INFO)
+    # except BaseException as e:
+    #     logger.error("同步东方财富港股信息异常:{}", e)
+    #
+    # try:
+    #     em_cookie = cookie_info_service.get_em_cookie()
+    #     em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, proxies)
+    #     em_us_stock_info['_id'] = em_us_stock_info['symbol']
+    #     mongodb_util.save_mongo(em_us_stock_info, extra_income_db_name.EM_US_STOCK_INFO)
+    #     em_us_etf_info = em_us_stock_info.loc[(em_us_stock_info['amount'] != 0) & (em_us_stock_info['total_mv'] == 0)]
+    #     em_us_etf_info = em_us_etf_info.sort_values(by=['amount'], ascending=False)
+    #     mongodb_util.save_mongo(em_us_etf_info, extra_income_db_name.EM_US_ETF_INFO)
+    #
+    # except BaseException as e:
+    #     logger.error("同步东方财富美股信息异常:{}", e)
     logger.info("同步东方财富a,etf,kzz,us,hk,信息完成")
 
 
 if __name__ == '__main__':
     sync_all_em_stock_info()
+    # em_cookie = cookie_info_service.get_em_cookie()
+    # em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, None)
+    # em_us_stock_info['_id'] = em_us_stock_info['symbol']
+    # mongodb_util.save_mongo(em_us_stock_info, db_name_constant.EM_US_STOCK_INFO)

mns_scheduler/company_info/remark/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/company_info/remark/company_remark_info_sync.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.api.ths.concept.web.ths_company_info_web as ths_company_info_web

mns_scheduler/concept/clean/kpl_concept_clean_api.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil

mns_scheduler/concept/clean/ths_concept_clean_api.py

@@ -7,7 +7,7 @@ project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil
 import pandas as pd
-import mns_common.utils.data_frame_util as data_frame_util
+import mns_common.api.ths.concept.web.ths_concept_index_web as ths_concept_index_web
 import time
 from loguru import logger
 import mns_common.component.company.company_common_service_api as company_common_service_api
@@ -107,10 +107,7 @@ def update_null_name():
         exist_url = concept_one.url
 
         if name == '':
-            concept_name_df = ths_concept_index_app.get_new_concept_from_app_search(concept_code)
-            if data_frame_util.is_empty(concept_name_df):
-                continue
-            concept_name = list(concept_name_df['concept_name'])[0]
+            concept_name = ths_concept_index_web.get_concept_name(concept_code)
             query_concept = {"symbol": concept_code}
             new_values = {'$set': {"name": concept_name}}
             mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
@@ -131,7 +128,7 @@ def update_null_name():
 
 
 if __name__ == '__main__':
-    update_one_concept_relevance_industry(886095)
+    # update_one_concept_relevance_industry(886095)
     update_null_name()
     logger.info("开始")
     update_ths_concept_info()

mns_scheduler/db/script/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/col_move_script.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import pymongo

mns_scheduler/db/script/db_move/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/db_move/col_move_one_service.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from loguru import logger

mns_scheduler/db/script/sync/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/sync/remote_data_sync_to_local.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.utils.data_frame_util as data_frame_util
@@ -15,12 +15,15 @@ remote_mongodb_util = RemoteMongodbUtil('27017')
 local_mongodb_util = LocalMongodbUtil('27017')
 
 col_list = [
+    'em_a_stock_info',
+    'em_etf_info',
+    'em_kzz_info',
+    'em_hk_stock_info',
     'company_info',
     'company_remark_info',
     'company_holding_info',
     'industry_concept_remark',
     'trade_date_list',
-    'company_info',
     'de_list_stock',
     'kpl_best_choose_index',
     'kpl_best_choose_index_detail',
@@ -31,9 +34,11 @@ col_list = [
     'self_choose_stock',
     'stock_account_info',
     'ths_concept_list',
-    'stock_zt_pool',
+    'stock_zt_pool_five',
     'ths_stock_concept_detail',
-    'stock_high_chg_pool'
+    'stock_high_chg_pool',
+    'today_new_concept_list',
+    'ths_stock_concept_detail_app'
 ]
 
 
@@ -53,8 +58,22 @@ def remote_data():
             logger.error("同步失败:{},{}", e, col)
 
 
+def sync_zt_data(str_day):
+    col = 'stock_zt_pool'
+    try:
+        query = {'str_day': str_day}
+        col_df = remote_mongodb_util.find_query_data(col, query)
+        if data_frame_util.is_not_empty(col_df):
+            col_df.replace([np.inf, -np.inf], 0, inplace=True)
+            local_mongodb_util.save_mongo(col_df, col)
+
+        logger.info("同步集合完成:{}", col)
+    except BaseException as e:
+        logger.error("同步失败:{},{}", e, col)
+
+
 def sync_open_data():
-    query = {"$and": [{'trade_date': {"$gte": "2022-03-16"}}, {'trade_date': {"$lte": "2025-01-01"}}]}
+    query = {"$and": [{'trade_date': {"$gte": "2025-03-21"}}, {'trade_date': {"$lte": "2025-04-02"}}]}
     trade_date_list_df = remote_mongodb_util.find_query_data('trade_date_list', query)
     trade_date_list_df = trade_date_list_df.sort_values(by=['trade_date'], ascending=False)
     for trade_date_one in trade_date_list_df.itertuples():
@@ -62,14 +81,25 @@ def sync_open_data():
             trade_date = trade_date_one.trade_date
             query_01 = {"str_day": trade_date}
             realtime_quotes_now_open_df = remote_mongodb_util.find_query_data('realtime_quotes_now_open', query_01)
-            # del realtime_quotes_now_open_df['ths_concept_list']
+            if 'ths_concept_list' in realtime_quotes_now_open_df.columns:
+                del realtime_quotes_now_open_df['ths_concept_list']
             local_mongodb_util.insert_mongo(realtime_quotes_now_open_df, 'realtime_quotes_now_open')
-            logger.error("同步到:{}", trade_date)
+            logger.info("同步到:{}", trade_date)
         except BaseException as e:
             logger.error("同步异常:{}", e)
     return trade_date_list_df
 
 
 if __name__ == '__main__':
-    call_auction_data_df = local_mongodb_util.find_all_data('call_auction_data')
-    remote_mongodb_util.insert_mongo(call_auction_data_df, 'call_auction_data_01')
+    sync_zt_data('2025-09-17')
+    # sync_zt_data('2025-09-02')
+    remote_data()
+    # sync_zt_data('2025-08-26')
+    # sync_zt_data('2025-08-25')
+    # sync_zt_data('2025-08-26')
+    # remote_data()
+    # sync_zt_data('2025-07-23')
+    # sync_zt_data('2025-07-24')
+    # sync_zt_data('2025-07-25')
+    # sync_open_data()
+    # remote_data()

mns_scheduler/db/script/sync/sync_hui_ce_test_data.py

@@ -2,22 +2,30 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
-import mns_common.utils.data_frame_util as data_frame_util
 from mns_scheduler.db.script.sync.remote_mongo_util import RemoteMongodbUtil
 from mns_scheduler.db.script.sync.local_mongo_util import LocalMongodbUtil
 from loguru import logger
-import numpy as np
 
 remote_mongodb_util = RemoteMongodbUtil('27017')
 local_mongodb_util = LocalMongodbUtil('27017')
 
 
+def create_index(db_name):
+    local_mongodb_util.create_index(db_name, [("symbol", 1)])
+
+    local_mongodb_util.create_index(db_name, [("str_now_date", 1)])
+
+    local_mongodb_util.create_index(db_name, [("symbol", 1), ("number", 1)])
+
+
 def sync_real_time_data(str_day, min_number, max_number):
     db_name = 'realtime_quotes_now_' + str_day
+    # 创建索引
+    create_index(db_name)
     while min_number <= max_number:
         query = {'number': min_number}
         data_df = remote_mongodb_util.find_query_data(db_name, query)
@@ -31,39 +39,42 @@ def sync_k_line(str_day):
     db_name = 'k_line_info'
     data_df = remote_mongodb_util.find_query_data(db_name, query)
     try:
-        local_mongodb_util.save_mongo(data_df, db_name)
+        local_mongodb_util.insert_mongo(data_df, db_name)
     except BaseException as e:
+        logger.error("出现异常:{}", e)
         pass
 
 
 if __name__ == '__main__':
-
-    str_day_01 = '2025-02-18'
+    str_day_01 = '2025-09-10'
     sync_k_line(str_day_01)
+
+    # sync_real_time_data(str_day_01, 1, 2500)
+
     # sync_k_line(str_day_01)
     # sync_k_line('2024-11-13')
     # sync_k_line('2024-11-12')
     # sync_k_line('2024-11-15') 232
-    sync_real_time_data(str_day_01, 1, 2000)
+
     # sync_real_time_data('2024-11-15', 10, 1010)
 
-    sync_k_line('2024-12-20')
-    sync_k_line('2024-12-19')
-    sync_k_line('2024-12-18')
-    sync_k_line('2024-12-17')
-    sync_k_line('2024-12-16')
-
-    sync_k_line('2024-12-13')
-    sync_k_line('2024-12-12')
-    sync_k_line('2024-12-11')
-    sync_k_line('2024-12-10')
-    sync_k_line('2024-12-09')
-
-    sync_k_line('2024-12-06')
-    sync_k_line('2024-12-05')
-    sync_k_line('2024-12-04')
-    sync_k_line('2024-12-03')
-    sync_k_line('2024-12-02')
+    # sync_k_line('2024-12-20')
+    # sync_k_line('2024-12-19')
+    # sync_k_line('2024-12-18')
+    # sync_k_line('2024-12-17')
+    # sync_k_line('2024-12-16')
+    #
+    # sync_k_line('2024-12-13')
+    # sync_k_line('2024-12-12')
+    # sync_k_line('2024-12-11')
+    # sync_k_line('2024-12-10')
+    # sync_k_line('2024-12-09')
+    #
+    # sync_k_line('2024-12-06')
+    # sync_k_line('2024-12-05')
+    # sync_k_line('2024-12-04')
+    # sync_k_line('2024-12-03')
+    # sync_k_line('2024-12-02')
 
     # sync_real_time_data('2024-12-20', 1000, 2000)
     # sync_real_time_data('2024-11-13', 10, 1010)

mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
@@ -37,33 +37,33 @@ def sync_k_line(str_day):
 
 
 if __name__ == '__main__':
-
-    str_day_01 = '2025-02-18'
-    # sync_k_line(str_day_01)
+    sync_k_line('2025-08-27')
+    # sync_k_line('2025-07-24')
+    # sync_k_line('2025-07-25')
     # sync_k_line(str_day_01)
     # sync_k_line('2024-11-13')
     # sync_k_line('2024-11-12')
     # sync_k_line('2024-11-15') 232
-    sync_real_time_data(str_day_01, 436, 2000)
+    # sync_real_time_data(str_day_01, 1, 2000)
     # sync_real_time_data('2024-11-15', 10, 1010)
-
-    sync_k_line('2024-12-20')
-    sync_k_line('2024-12-19')
-    sync_k_line('2024-12-18')
-    sync_k_line('2024-12-17')
-    sync_k_line('2024-12-16')
-
-    sync_k_line('2024-12-13')
-    sync_k_line('2024-12-12')
-    sync_k_line('2024-12-11')
-    sync_k_line('2024-12-10')
-    sync_k_line('2024-12-09')
-
-    sync_k_line('2024-12-06')
-    sync_k_line('2024-12-05')
-    sync_k_line('2024-12-04')
-    sync_k_line('2024-12-03')
-    sync_k_line('2024-12-02')
+    #
+    # sync_k_line('2024-12-20')
+    # sync_k_line('2024-12-19')
+    # sync_k_line('2024-12-18')
+    # sync_k_line('2024-12-17')
+    # sync_k_line('2024-12-16')
+    #
+    # sync_k_line('2024-12-13')
+    # sync_k_line('2024-12-12')
+    # sync_k_line('2024-12-11')
+    # sync_k_line('2024-12-10')
+    # sync_k_line('2024-12-09')
+    #
+    # sync_k_line('2024-12-06')
+    # sync_k_line('2024-12-05')
+    # sync_k_line('2024-12-04')
+    # sync_k_line('2024-12-03')
+    # sync_k_line('2024-12-02')
 
     # sync_real_time_data('2024-12-20', 1000, 2000)
     # sync_real_time_data('2024-11-13', 10, 1010)

mns_scheduler/db/script/update/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/update/update_col_field.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil

mns_scheduler/finance/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/finance/{em_financial_asset_liability_sync_service_api.py → em/em_financial_asset_liability_sync_service_api.py}

@@ -2,12 +2,12 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import akshare as ak
 from mns_common.db.MongodbUtil import MongodbUtil
-import mns_scheduler.finance.finance_common_api as finance_common_api
+import mns_scheduler.finance.em.finance_common_api as finance_common_api
 import mns_common.constant.db_name_constant as db_name_constant
 
 mongodb_util = MongodbUtil('27017')

mns_scheduler/finance/{em_financial_profit_sync_service_api.py → em/em_financial_profit_sync_service_api.py}

@@ -1,8 +1,10 @@
 import sys
 import os
 
+import pandas as pd
+
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
@@ -12,7 +14,7 @@ import mns_common.utils.data_frame_util as data_frame_util
 
 mongodb_util = MongodbUtil('27017')
 import mns_common.constant.db_name_constant as db_name_constant
-import mns_scheduler.finance.finance_common_api as finance_common_api
+import mns_scheduler.finance.em.finance_common_api as finance_common_api
 from loguru import logger
 
 
@@ -293,7 +295,7 @@ def get_em_profit_api(symbol):
     else:
         new_profit_df = stock_profit_sheet_by_report_em
     if data_frame_util.is_empty(new_profit_df):
-        return None
+        return pd.DataFrame()
     new_profit_df = new_profit_df.fillna(0)
     return new_profit_df
 
@@ -341,7 +343,7 @@ def check_columns(profit_df):
 import mns_common.component.em.em_stock_info_api as em_stock_info_api
 
 if __name__ == '__main__':
-    get_em_profit_api('002193')
+    get_em_profit_api('688302')
     em_df = em_stock_info_api.get_a_stock_info()
     for em_one in em_df.itertuples():
         try:

mns_scheduler/finance/{finance_common_api.py → em/finance_common_api.py}

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)