mns-scheduler 1.2.6.4__py3-none-any.whl → 1.4.2.5__py3-none-any.whl

mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py

@@ -0,0 +1,80 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_common.db.MongodbUtil import MongodbUtil
+from loguru import logger
+import mns_common.api.em.real_time.east_money_debt_api as east_money_debt_api
+import mns_common.api.em.real_time.east_money_etf_api as east_money_etf_api
+import mns_common.api.em.real_time.east_money_stock_a_v2_api as east_money_stock_a_v2_api
+from datetime import datetime
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+import mns_common.api.em.real_time.east_money_stock_hk_api as east_money_stock_hk_api
+import mns_common.component.cookie.cookie_info_service as cookie_info_service
+import mns_common.api.em.real_time.east_money_stock_us_api as east_money_stock_us_api
+
+mongodb_util = MongodbUtil('27017')
+
+
+def sync_all_em_stock_info():
+    now_date = datetime.now()
+    str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
+
+    logger.info("同步东方财富a,etf,kzz,us,hk,信息开始")
+
+    try:
+        # 这里需要使用详情接口 获取全量数据
+        em_a_stock_info_df = east_money_stock_a_v2_api.get_stock_real_time_quotes(60)
+        em_a_stock_info_df['list_date'] = em_a_stock_info_df['list_date'].fillna(19890604)
+        em_a_stock_info_df['_id'] = em_a_stock_info_df['symbol']
+        em_a_stock_info_df['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_a_stock_info_df, extra_income_db_name.EM_A_STOCK_INFO)
+    except BaseException as e:
+        logger.error("同步东方财富A股信息异常:{}", e)
+    try:
+        em_etf_info = east_money_etf_api.get_etf_real_time_quotes(30, 6)
+        em_etf_info['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_etf_info, extra_income_db_name.EM_ETF_INFO)
+    except BaseException as e:
+        logger.error("同步东方财富ETF信息异常:{}", e)
+
+    try:
+        em_kzz_info = east_money_debt_api.get_kzz_real_time_quotes(30, 6)
+        em_kzz_info['_id'] = em_kzz_info['symbol']
+        em_kzz_info['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_kzz_info, extra_income_db_name.EM_KZZ_INFO)
+    except BaseException as e:
+        logger.error("同步东方财富可转债信息异常:{}", e)
+
+    # em_cookie = cookie_info_service.get_em_cookie()
+    # try:
+    #     em_hk_stock_info = east_money_stock_hk_api.get_hk_real_time_quotes(em_cookie, proxies)
+    #     em_hk_stock_info['_id'] = em_hk_stock_info['symbol']
+    #     mongodb_util.save_mongo(em_hk_stock_info, extra_income_db_name.EM_HK_STOCK_INFO)
+    # except BaseException as e:
+    #     logger.error("同步东方财富港股信息异常:{}", e)
+    #
+    # try:
+    #     em_cookie = cookie_info_service.get_em_cookie()
+    #     em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, proxies)
+    #     em_us_stock_info['_id'] = em_us_stock_info['symbol']
+    #     mongodb_util.save_mongo(em_us_stock_info, extra_income_db_name.EM_US_STOCK_INFO)
+    #     em_us_etf_info = em_us_stock_info.loc[(em_us_stock_info['amount'] != 0) & (em_us_stock_info['total_mv'] == 0)]
+    #     em_us_etf_info = em_us_etf_info.sort_values(by=['amount'], ascending=False)
+    #     mongodb_util.save_mongo(em_us_etf_info, extra_income_db_name.EM_US_ETF_INFO)
+    #
+    # except BaseException as e:
+    #     logger.error("同步东方财富美股信息异常:{}", e)
+    logger.info("同步东方财富a,etf,kzz,us,hk,信息完成")
+
+
+if __name__ == '__main__':
+    sync_all_em_stock_info()
+    # em_cookie = cookie_info_service.get_em_cookie()
+    # em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, None)
+    # em_us_stock_info['_id'] = em_us_stock_info['symbol']
+    # mongodb_util.save_mongo(em_us_stock_info, db_name_constant.EM_US_STOCK_INFO)

mns_scheduler/company_info/remark/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/company_info/remark/company_remark_info_sync.py

@@ -2,11 +2,11 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.api.ths.concept.web.ths_company_info_web as ths_company_info_web
-import mns_common.api.em.east_money_stock_v2_api as east_money_stock_v2_api
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
 from mns_common.db.MongodbUtil import MongodbUtil
 import mns_common.utils.data_frame_util as data_frame_util
 import mns_common.constant.db_name_constant as db_name_constant
@@ -18,7 +18,7 @@ mongodb_util = MongodbUtil('27017')
 
 
 def sync_company_remark_info():
-    east_money_stock_info = east_money_stock_v2_api.get_all_real_time_quotes()
+    east_money_stock_info = em_stock_info_api.get_a_stock_info()
     de_listed_stock_list = company_common_service_api.get_de_list_company()
     east_money_stock_info = east_money_stock_info.loc[~(
         east_money_stock_info['symbol'].isin(de_listed_stock_list))]

mns_scheduler/concept/clean/kpl_concept_clean_api.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil

mns_scheduler/concept/clean/ths_concept_clean_api.py

@@ -7,7 +7,7 @@ project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil
 import pandas as pd
-import mns_common.utils.data_frame_util as data_frame_util
+import mns_common.api.ths.concept.web.ths_concept_index_web as ths_concept_index_web
 import time
 from loguru import logger
 import mns_common.component.company.company_common_service_api as company_common_service_api
@@ -107,10 +107,7 @@ def update_null_name():
         exist_url = concept_one.url
 
         if name == '':
-            concept_name_df = ths_concept_index_app.get_new_concept_from_app_search(concept_code)
-            if data_frame_util.is_empty(concept_name_df):
-                continue
-            concept_name = list(concept_name_df['concept_name'])[0]
+            concept_name = ths_concept_index_web.get_concept_name(concept_code)
             query_concept = {"symbol": concept_code}
             new_values = {'$set': {"name": concept_name}}
             mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
@@ -131,7 +128,7 @@ def update_null_name():
 
 
 if __name__ == '__main__':
-    update_one_concept_relevance_industry(886095)
+    # update_one_concept_relevance_industry(886095)
     update_null_name()
     logger.info("开始")
     update_ths_concept_info()

mns_scheduler/concept/ths/detaill/ths_concept_detail_api.py

@@ -9,7 +9,7 @@ import mns_common.component.concept.ths_concept_common_service_api as ths_concep
 import mns_common.component.common_service_fun_api as common_service_fun_api
 import mns_common.api.ths.concept.app.ths_concept_detail_app as ths_concept_detail_app
 import mns_common.api.ths.wen_cai.ths_wen_cai_api as ths_wen_cai_api
-import mns_common.api.em.east_money_stock_api as east_money_stock_api
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
 import mns_common.component.company.company_common_service_new_api as company_common_service_new_api
 import mns_common.api.ths.concept.web.ths_concept_detail_web as ths_concept_detail_web
 import math
@@ -24,7 +24,7 @@ import mns_common.constant.redis_msg_constant as redis_msg_constant
 
 
 def get_ths_concept_detail(concept_code, concept_name):
-    real_time_quotes_all_stocks = east_money_stock_api.get_real_time_quotes_all_stocks()
+    real_time_quotes_all_stocks = em_stock_info_api.get_a_stock_info()
     # 1  通过入选理获取概念组成股票详情
     ths_concept_detail_by_explain = get_ths_concept_detail_by_explain(concept_code, real_time_quotes_all_stocks)
     # 2 app 分享链接获取概念组成详情

mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py

@@ -1,7 +1,7 @@
 import os
 import sys
 import mns_common.api.ths.concept.web.ths_concept_detail_web as ths_concept_detail_web
-import mns_common.api.em.east_money_stock_api as east_money_stock_api
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
 import mns_common.component.common_service_fun_api as common_service_fun_api
 from mns_common.db.MongodbUtil import MongodbUtil
 from loguru import logger
@@ -157,7 +157,7 @@ def update_symbol_new_concept(symbol_df, page_number):
 
 def sync_symbol_all_concept(symbol):
     create_index()
-    real_time_quotes_now = east_money_stock_api.get_real_time_quotes_all_stocks()
+    real_time_quotes_now = em_stock_info_api.get_a_stock_info()
     real_time_quotes_now = common_service_fun_api.classify_symbol(real_time_quotes_now)
     if symbol is not None:
         real_time_quotes_now = real_time_quotes_now.loc[real_time_quotes_now['symbol'] == symbol]

mns_scheduler/db/script/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/col_move_script.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import pymongo

mns_scheduler/db/script/db_move/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler/db/script/db_move/col_move_one_service.py

@@ -0,0 +1,34 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+from loguru import logger
+
+
+def db_export(db, col):
+    cmd = 'F:/mongo/bin/mongodump.exe --host ' + db + ' -d patience -c ' + col + ' -o D:/back'
+    os.system(cmd)
+    logger.info("export finished:{}", col)
+
+
+def db_import(db, col):
+    cmd = 'F:/mongo/bin/mongorestore.exe --host ' + db + ' -d patience -c ' + col + ' D:/back/patience/' + col + '.bson'
+    os.system(cmd)
+
+    path = 'D:\\back\\patience\\' + col + '.bson'
+    cmd_del = 'del /F /S /Q ' + path
+    os.system(cmd_del)
+
+    logger.info("import finished:{}", col)
+
+
+def handle_one_col(col_name):
+    db_export('127.0.0.1:27017', col_name)
+    db_import('127.0.0.1:27019', col_name)
+
+
+if __name__ == '__main__':
+    handle_one_col('one_minute_k_line_bfq_h')

mns_scheduler/db/script/sync/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/sync/remote_data_sync_to_local.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.utils.data_frame_util as data_frame_util
@@ -15,12 +15,15 @@ remote_mongodb_util = RemoteMongodbUtil('27017')
 local_mongodb_util = LocalMongodbUtil('27017')
 
 col_list = [
+    'em_a_stock_info',
+    'em_etf_info',
+    'em_kzz_info',
+    'em_hk_stock_info',
     'company_info',
     'company_remark_info',
     'company_holding_info',
     'industry_concept_remark',
     'trade_date_list',
-    'company_info',
     'de_list_stock',
     'kpl_best_choose_index',
     'kpl_best_choose_index_detail',
@@ -31,8 +34,11 @@ col_list = [
     'self_choose_stock',
     'stock_account_info',
     'ths_concept_list',
-    'stock_zt_pool',
-    'ths_stock_concept_detail'
+    'stock_zt_pool_five',
+    'ths_stock_concept_detail',
+    'stock_high_chg_pool',
+    'today_new_concept_list',
+    'ths_stock_concept_detail_app'
 ]
 
 
@@ -52,8 +58,22 @@ def remote_data():
             logger.error("同步失败:{},{}", e, col)
 
 
+def sync_zt_data(str_day):
+    col = 'stock_zt_pool'
+    try:
+        query = {'str_day': str_day}
+        col_df = remote_mongodb_util.find_query_data(col, query)
+        if data_frame_util.is_not_empty(col_df):
+            col_df.replace([np.inf, -np.inf], 0, inplace=True)
+            local_mongodb_util.save_mongo(col_df, col)
+
+        logger.info("同步集合完成:{}", col)
+    except BaseException as e:
+        logger.error("同步失败:{},{}", e, col)
+
+
 def sync_open_data():
-    query = {"$and": [{'trade_date': {"$gte": "2022-03-16"}}, {'trade_date': {"$lte": "2025-01-01"}}]}
+    query = {"$and": [{'trade_date': {"$gte": "2025-03-21"}}, {'trade_date': {"$lte": "2025-04-02"}}]}
     trade_date_list_df = remote_mongodb_util.find_query_data('trade_date_list', query)
     trade_date_list_df = trade_date_list_df.sort_values(by=['trade_date'], ascending=False)
     for trade_date_one in trade_date_list_df.itertuples():
@@ -61,13 +81,25 @@ def sync_open_data():
             trade_date = trade_date_one.trade_date
             query_01 = {"str_day": trade_date}
             realtime_quotes_now_open_df = remote_mongodb_util.find_query_data('realtime_quotes_now_open', query_01)
-            # del realtime_quotes_now_open_df['ths_concept_list']
+            if 'ths_concept_list' in realtime_quotes_now_open_df.columns:
+                del realtime_quotes_now_open_df['ths_concept_list']
             local_mongodb_util.insert_mongo(realtime_quotes_now_open_df, 'realtime_quotes_now_open')
-            logger.error("同步到:{}", trade_date)
+            logger.info("同步到:{}", trade_date)
         except BaseException as e:
             logger.error("同步异常:{}", e)
     return trade_date_list_df
 
 
 if __name__ == '__main__':
-    sync_open_data()
+    sync_zt_data('2025-09-17')
+    # sync_zt_data('2025-09-02')
+    remote_data()
+    # sync_zt_data('2025-08-26')
+    # sync_zt_data('2025-08-25')
+    # sync_zt_data('2025-08-26')
+    # remote_data()
+    # sync_zt_data('2025-07-23')
+    # sync_zt_data('2025-07-24')
+    # sync_zt_data('2025-07-25')
+    # sync_open_data()
+    # remote_data()

mns_scheduler/db/script/sync/sync_hui_ce_test_data.py

@@ -2,22 +2,30 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
-import mns_common.utils.data_frame_util as data_frame_util
 from mns_scheduler.db.script.sync.remote_mongo_util import RemoteMongodbUtil
 from mns_scheduler.db.script.sync.local_mongo_util import LocalMongodbUtil
 from loguru import logger
-import numpy as np
 
 remote_mongodb_util = RemoteMongodbUtil('27017')
 local_mongodb_util = LocalMongodbUtil('27017')
 
 
+def create_index(db_name):
+    local_mongodb_util.create_index(db_name, [("symbol", 1)])
+
+    local_mongodb_util.create_index(db_name, [("str_now_date", 1)])
+
+    local_mongodb_util.create_index(db_name, [("symbol", 1), ("number", 1)])
+
+
 def sync_real_time_data(str_day, min_number, max_number):
     db_name = 'realtime_quotes_now_' + str_day
+    # 创建索引
+    create_index(db_name)
     while min_number <= max_number:
         query = {'number': min_number}
         data_df = remote_mongodb_util.find_query_data(db_name, query)
@@ -31,37 +39,42 @@ def sync_k_line(str_day):
     db_name = 'k_line_info'
     data_df = remote_mongodb_util.find_query_data(db_name, query)
     try:
-        local_mongodb_util.save_mongo(data_df, db_name)
+        local_mongodb_util.insert_mongo(data_df, db_name)
     except BaseException as e:
+        logger.error("出现异常:{}", e)
         pass
 
 
 if __name__ == '__main__':
-    str_day_01 = '2024-12-13'
+    str_day_01 = '2025-09-10'
+    sync_k_line(str_day_01)
+
+    # sync_real_time_data(str_day_01, 1, 2500)
+
     # sync_k_line(str_day_01)
     # sync_k_line('2024-11-13')
     # sync_k_line('2024-11-12')
     # sync_k_line('2024-11-15') 232
-    sync_real_time_data(str_day_01, 1409, 7010)
+
     # sync_real_time_data('2024-11-15', 10, 1010)
 
-    sync_k_line('2024-12-20')
-    sync_k_line('2024-12-19')
-    sync_k_line('2024-12-18')
-    sync_k_line('2024-12-17')
-    sync_k_line('2024-12-16')
-
-    sync_k_line('2024-12-13')
-    sync_k_line('2024-12-12')
-    sync_k_line('2024-12-11')
-    sync_k_line('2024-12-10')
-    sync_k_line('2024-12-09')
-
-    sync_k_line('2024-12-06')
-    sync_k_line('2024-12-05')
-    sync_k_line('2024-12-04')
-    sync_k_line('2024-12-03')
-    sync_k_line('2024-12-02')
+    # sync_k_line('2024-12-20')
+    # sync_k_line('2024-12-19')
+    # sync_k_line('2024-12-18')
+    # sync_k_line('2024-12-17')
+    # sync_k_line('2024-12-16')
+    #
+    # sync_k_line('2024-12-13')
+    # sync_k_line('2024-12-12')
+    # sync_k_line('2024-12-11')
+    # sync_k_line('2024-12-10')
+    # sync_k_line('2024-12-09')
+    #
+    # sync_k_line('2024-12-06')
+    # sync_k_line('2024-12-05')
+    # sync_k_line('2024-12-04')
+    # sync_k_line('2024-12-03')
+    # sync_k_line('2024-12-02')
 
     # sync_real_time_data('2024-12-20', 1000, 2000)
     # sync_real_time_data('2024-11-13', 10, 1010)

mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py

@@ -0,0 +1,69 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+import mns_common.utils.data_frame_util as data_frame_util
+from mns_scheduler.db.script.sync.remote_mongo_util import RemoteMongodbUtil
+from mns_scheduler.db.script.sync.local_mongo_util import LocalMongodbUtil
+from loguru import logger
+import numpy as np
+
+remote_mongodb_util = RemoteMongodbUtil('27017')
+local_mongodb_util = LocalMongodbUtil('27017')
+
+
+def sync_real_time_data(str_day, min_number, max_number):
+    db_name = 'realtime_quotes_now_' + str_day
+    while min_number <= max_number:
+        query = {'number': min_number}
+        data_df = remote_mongodb_util.find_query_data(db_name, query)
+        local_mongodb_util.insert_mongo(data_df, db_name)
+        min_number = min_number + 1
+        logger.info(min_number)
+
+
+def sync_k_line(str_day):
+    query = {"str_day": str_day}
+    db_name = 'k_line_info'
+    data_df = remote_mongodb_util.find_query_data(db_name, query)
+    try:
+        local_mongodb_util.save_mongo(data_df, db_name)
+    except BaseException as e:
+        pass
+
+
+if __name__ == '__main__':
+    sync_k_line('2025-08-27')
+    # sync_k_line('2025-07-24')
+    # sync_k_line('2025-07-25')
+    # sync_k_line(str_day_01)
+    # sync_k_line('2024-11-13')
+    # sync_k_line('2024-11-12')
+    # sync_k_line('2024-11-15') 232
+    # sync_real_time_data(str_day_01, 1, 2000)
+    # sync_real_time_data('2024-11-15', 10, 1010)
+    #
+    # sync_k_line('2024-12-20')
+    # sync_k_line('2024-12-19')
+    # sync_k_line('2024-12-18')
+    # sync_k_line('2024-12-17')
+    # sync_k_line('2024-12-16')
+    #
+    # sync_k_line('2024-12-13')
+    # sync_k_line('2024-12-12')
+    # sync_k_line('2024-12-11')
+    # sync_k_line('2024-12-10')
+    # sync_k_line('2024-12-09')
+    #
+    # sync_k_line('2024-12-06')
+    # sync_k_line('2024-12-05')
+    # sync_k_line('2024-12-04')
+    # sync_k_line('2024-12-03')
+    # sync_k_line('2024-12-02')
+
+    # sync_real_time_data('2024-12-20', 1000, 2000)
+    # sync_real_time_data('2024-11-13', 10, 1010)

mns_scheduler/db/script/update/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/update/update_col_field.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil

mns_scheduler/finance/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/finance/{em_financial_asset_liability_sync_service_api.py → em/em_financial_asset_liability_sync_service_api.py}

@@ -2,12 +2,12 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import akshare as ak
 from mns_common.db.MongodbUtil import MongodbUtil
-import mns_scheduler.finance.finance_common_api as finance_common_api
+import mns_scheduler.finance.em.finance_common_api as finance_common_api
 import mns_common.constant.db_name_constant as db_name_constant
 
 mongodb_util = MongodbUtil('27017')