mns-scheduler 1.1.8.4__py3-none-any.whl → 1.4.3.2__py3-none-any.whl

mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py

@@ -0,0 +1,80 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_common.db.MongodbUtil import MongodbUtil
+from loguru import logger
+import mns_common.api.em.real_time.east_money_debt_api as east_money_debt_api
+import mns_common.api.em.real_time.east_money_etf_api as east_money_etf_api
+import mns_common.api.em.real_time.east_money_stock_a_v2_api as east_money_stock_a_v2_api
+from datetime import datetime
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+import mns_common.api.em.real_time.east_money_stock_hk_api as east_money_stock_hk_api
+import mns_common.component.cookie.cookie_info_service as cookie_info_service
+import mns_common.api.em.real_time.east_money_stock_us_api as east_money_stock_us_api
+
+mongodb_util = MongodbUtil('27017')
+
+
+def sync_all_em_stock_info():
+    now_date = datetime.now()
+    str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
+
+    logger.info("同步东方财富a,etf,kzz,us,hk,信息开始")
+
+    try:
+        # 这里需要使用详情接口 获取全量数据
+        em_a_stock_info_df = east_money_stock_a_v2_api.get_stock_real_time_quotes(60)
+        em_a_stock_info_df['list_date'] = em_a_stock_info_df['list_date'].fillna(19890604)
+        em_a_stock_info_df['_id'] = em_a_stock_info_df['symbol']
+        em_a_stock_info_df['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_a_stock_info_df, extra_income_db_name.EM_A_STOCK_INFO)
+    except BaseException as e:
+        logger.error("同步东方财富A股信息异常:{}", e)
+    try:
+        em_etf_info = east_money_etf_api.get_etf_real_time_quotes(30, 6)
+        em_etf_info['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_etf_info, extra_income_db_name.EM_ETF_INFO)
+    except BaseException as e:
+        logger.error("同步东方财富ETF信息异常:{}", e)
+
+    try:
+        em_kzz_info = east_money_debt_api.get_kzz_real_time_quotes(30, 6)
+        em_kzz_info['_id'] = em_kzz_info['symbol']
+        em_kzz_info['sync_time'] = str_now_date
+        mongodb_util.save_mongo(em_kzz_info, extra_income_db_name.EM_KZZ_INFO)
+    except BaseException as e:
+        logger.error("同步东方财富可转债信息异常:{}", e)
+
+    # em_cookie = cookie_info_service.get_em_cookie()
+    # try:
+    #     em_hk_stock_info = east_money_stock_hk_api.get_hk_real_time_quotes(em_cookie, proxies)
+    #     em_hk_stock_info['_id'] = em_hk_stock_info['symbol']
+    #     mongodb_util.save_mongo(em_hk_stock_info, extra_income_db_name.EM_HK_STOCK_INFO)
+    # except BaseException as e:
+    #     logger.error("同步东方财富港股信息异常:{}", e)
+    #
+    # try:
+    #     em_cookie = cookie_info_service.get_em_cookie()
+    #     em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, proxies)
+    #     em_us_stock_info['_id'] = em_us_stock_info['symbol']
+    #     mongodb_util.save_mongo(em_us_stock_info, extra_income_db_name.EM_US_STOCK_INFO)
+    #     em_us_etf_info = em_us_stock_info.loc[(em_us_stock_info['amount'] != 0) & (em_us_stock_info['total_mv'] == 0)]
+    #     em_us_etf_info = em_us_etf_info.sort_values(by=['amount'], ascending=False)
+    #     mongodb_util.save_mongo(em_us_etf_info, extra_income_db_name.EM_US_ETF_INFO)
+    #
+    # except BaseException as e:
+    #     logger.error("同步东方财富美股信息异常:{}", e)
+    logger.info("同步东方财富a,etf,kzz,us,hk,信息完成")
+
+
+if __name__ == '__main__':
+    sync_all_em_stock_info()
+    # em_cookie = cookie_info_service.get_em_cookie()
+    # em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, None)
+    # em_us_stock_info['_id'] = em_us_stock_info['symbol']
+    # mongodb_util.save_mongo(em_us_stock_info, db_name_constant.EM_US_STOCK_INFO)

mns_scheduler/company_info/remark/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/company_info/remark/company_remark_info_sync.py

@@ -2,11 +2,11 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.api.ths.concept.web.ths_company_info_web as ths_company_info_web
-import mns_common.api.em.east_money_stock_v2_api as east_money_stock_v2_api
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
 from mns_common.db.MongodbUtil import MongodbUtil
 import mns_common.utils.data_frame_util as data_frame_util
 import mns_common.constant.db_name_constant as db_name_constant
@@ -18,7 +18,7 @@ mongodb_util = MongodbUtil('27017')
 
 
 def sync_company_remark_info():
-    east_money_stock_info = east_money_stock_v2_api.get_all_real_time_quotes()
+    east_money_stock_info = em_stock_info_api.get_a_stock_info()
     de_listed_stock_list = company_common_service_api.get_de_list_company()
     east_money_stock_info = east_money_stock_info.loc[~(
         east_money_stock_info['symbol'].isin(de_listed_stock_list))]

mns_scheduler/concept/clean/kpl_concept_clean_api.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil

mns_scheduler/concept/clean/ths_concept_clean_api.py

@@ -7,12 +7,13 @@ project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil
 import pandas as pd
-import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_common.api.ths.concept.web.ths_concept_index_web as ths_concept_index_web
 import time
 from loguru import logger
 import mns_common.component.company.company_common_service_api as company_common_service_api
 import mns_common.constant.db_name_constant as db_name_constant
 import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
+import mns_common.api.ths.concept.app.ths_concept_index_app as ths_concept_index_app
 
 mongodb_util = MongodbUtil('27017')
 
@@ -76,9 +77,22 @@ def update_one_concept_relevance_industry(concept_code):
     ths_stock_concept_detail_df['first_relevance_industry'] = first_relevance_industry
     ths_stock_concept_detail_df['second_relevance_industry'] = second_relevance_industry
 
-    mongodb_util.save_mongo(ths_concept_list_one_df, db_name_constant.THS_CONCEPT_LIST)
+    new_values_list = {"$set": {"first_relevance_industry": first_relevance_industry,
+                                "second_relevance_industry": second_relevance_industry,
+                                "first_relevance_industry_number": first_relevance_industry_number,
+                                "second_relevance_industry_number": second_relevance_industry_number,
+                                "concept_count": concept_count
+                                }}
 
-    mongodb_util.save_mongo(ths_stock_concept_detail_df, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
+    update_query = {"symbol": concept_code}
+    mongodb_util.update_one_query(update_query, new_values_list, db_name_constant.THS_CONCEPT_LIST)
+
+    new_values_detail = {"$set": {"first_relevance_industry": first_relevance_industry,
+                                  "second_relevance_industry": second_relevance_industry
+
+                                  }}
+    update_query_detail = {"concept_code": concept_code}
+    mongodb_util.update_one_query(update_query_detail, new_values_detail, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
 
 
 # 更新空名字
@@ -93,7 +107,7 @@ def update_null_name():
         exist_url = concept_one.url
 
         if name == '':
-            concept_name = ths_concept_detail_api.get_ths_concept_detail(concept_code, None)
+            concept_name = ths_concept_index_web.get_concept_name(concept_code)
             query_concept = {"symbol": concept_code}
             new_values = {'$set': {"name": concept_name}}
             mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
@@ -114,6 +128,8 @@ def update_null_name():
 
 
 if __name__ == '__main__':
+    # update_one_concept_relevance_industry(886095)
+    update_null_name()
     logger.info("开始")
     update_ths_concept_info()
     logger.info("结束")

mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py

@@ -38,15 +38,6 @@ def push_msg_to_we_chat_and_redis(concept_code, concept_name, url):
     ths_concept_clean_api.update_ths_concept_info()
 
 
-# 保存新概念信息到数据库
-def save_ths_concept_list(concept_code, concept_name, str_day, str_now_time):
-    url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
-    ths_concept_list = pd.DataFrame([
-        [concept_code, concept_code, concept_name, str_day, url, str_now_time, True, True],
-    ], columns=['_id', 'symbol', 'name', 'str_day', 'url', 'str_now_time', 'success', 'valid'])
-    mongodb_util.save_mongo(ths_concept_list, 'ths_concept_list')
-
-
 # 保存新概念详细信息到数据库
 def save_ths_concept_detail(new_concept_symbol_df,
                             concept_name, str_day,
@@ -135,8 +126,6 @@ def save_ths_concept_detail(new_concept_symbol_df,
     ]]
     query_detail = {"concept_code": int(concept_code)}
     exist_concept_detail = mongodb_util.find_query_data(db_name_constant.THS_STOCK_CONCEPT_DETAIL, query_detail)
-    # 更新入选理由
-    update_long_short(new_concept_symbol_df)
 
     if exist_concept_detail is None or exist_concept_detail.shape[0] == 0:
         new_concept_symbol_df['grade'] = 1
@@ -169,26 +158,42 @@ def save_ths_concept_detail(new_concept_symbol_df,
 
 
 # 更新入选理由
-def update_long_short(new_concept_symbol_df):
+def update_long_short(new_concept_symbol_df, exist_concept_detail):
     if data_frame_util.is_empty(new_concept_symbol_df):
         return None
     for new_concept_one in new_concept_symbol_df.itertuples():
         try:
             query = {'symbol': new_concept_one.symbol, 'concept_code': new_concept_one.concept_code}
-            new_values = {"$set":{'long': new_concept_one.long, "short": new_concept_one.short}}
+            new_values = {"$set": {'long': new_concept_one.long, "short": new_concept_one.short}}
             mongodb_util.update_many(query, new_values, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
             mongodb_util.update_many(query, new_values, db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP)
         except BaseException as e:
             logger.error("更新入选理由异常:{}", e)
-
-        # 更新公司表信息 todo 清空cache 公司表中  common_service_fun_api.py  get_company_info_industry
+    # 更新已经被删除概念的股票
+    # update_delete_concept_symbol(list(new_concept_symbol_df['concept_code'])[0], new_concept_symbol_df,
+    #                              exist_concept_detail)
+    # 更新公司表信息 todo 清空cache 公司表中  common_service_fun_api.py  get_company_info_industry
+
+
+## 更新已经被删除这个概念的股票
+# def update_delete_concept_symbol(concept_code, new_concept_symbol_df, exist_concept_detail):
+#     delete_concept_symbol_df = exist_concept_detail.loc[
+#         ~(exist_concept_detail['symbol'].isin(list(new_concept_symbol_df['symbol'])))]
+#
+#     if data_frame_util.is_not_empty(delete_concept_symbol_df):
+#         new_values = {"$set": {"grade": 0}}
+#         query = {'symbol': {"$in": list(delete_concept_symbol_df['symbol'])}, 'concept_code': concept_code}
+#         mongodb_util.update_many(query, new_values, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
+#         mongodb_util.update_many(query, new_values, db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP)
+#
+#     exist_concept_detail = exist_concept_detail.loc[exist_concept_detail]
 
 
 def update_company_info(new_concept_symbol_df):
     if new_concept_symbol_df.shape[0] > 0:
         symbol_list = list(new_concept_symbol_df['symbol'])
         company_info_sync_api.sync_company_base_info(symbol_list)
-        company_info_clean_api.fix_company_industry(None)
+        company_info_clean_api.clean_company_info(None)
         # 公司缓存信息清除
         company_common_service_api.company_info_industry_cache_clear()
 

mns_scheduler/concept/ths/common/ths_concept_update_common_api.py

@@ -31,7 +31,10 @@ def update_ths_concept_choose_reason(ths_symbol_all_concepts, symbol):
                                      {'symbol': symbol, "concept_code": int(concept_one.cid)}]}
                     short = concept_one.short
                     long = concept_one.long
-                    new_values = {"$set": {"short": short, "long": long}}
+                    if data_frame_util.is_string_not_empty(long):
+                        new_values = {"$set": {"short": short, "long": long}}
+                    else:
+                        new_values = {"$set": {"grade": 0}}
                     mongodb_util.update_many(query, new_values, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
         except BaseException as e:
             logger.error("更新ths概念入选理由异常{},{},{}", symbol, concept_one.title, e)

mns_scheduler/concept/ths/detaill/ths_concept_detail_api.py

@@ -9,7 +9,7 @@ import mns_common.component.concept.ths_concept_common_service_api as ths_concep
 import mns_common.component.common_service_fun_api as common_service_fun_api
 import mns_common.api.ths.concept.app.ths_concept_detail_app as ths_concept_detail_app
 import mns_common.api.ths.wen_cai.ths_wen_cai_api as ths_wen_cai_api
-import mns_common.api.em.east_money_stock_api as east_money_stock_api
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
 import mns_common.component.company.company_common_service_new_api as company_common_service_new_api
 import mns_common.api.ths.concept.web.ths_concept_detail_web as ths_concept_detail_web
 import math
@@ -24,7 +24,7 @@ import mns_common.constant.redis_msg_constant as redis_msg_constant
 
 
 def get_ths_concept_detail(concept_code, concept_name):
-    real_time_quotes_all_stocks = east_money_stock_api.get_real_time_quotes_all_stocks()
+    real_time_quotes_all_stocks = em_stock_info_api.get_a_stock_info()
     # 1  通过入选理获取概念组成股票详情
     ths_concept_detail_by_explain = get_ths_concept_detail_by_explain(concept_code, real_time_quotes_all_stocks)
     # 2 app 分享链接获取概念组成详情
@@ -37,10 +37,10 @@ def get_ths_concept_detail(concept_code, concept_name):
             | (ths_concept_list['web_concept_code'] == int(concept_code))]
         if data_frame_util.is_not_empty(ths_concept_one_df):
             concept_name = list(ths_concept_one_df['name'])[0]
-    if data_frame_util.is_string_not_empty(concept_name) and concept_name is not None:
-        concept_detail_by_wen_cai_df = get_ths_concept_detail_by_wen_cai(concept_name, real_time_quotes_all_stocks)
-    else:
-        concept_detail_by_wen_cai_df = None
+    # if data_frame_util.is_string_not_empty(concept_name) and concept_name is not None:
+    #     concept_detail_by_wen_cai_df = get_ths_concept_detail_by_wen_cai(concept_name, real_time_quotes_all_stocks)
+    # else:
+    concept_detail_by_wen_cai_df = None
     # 4 通过web端
     ths_concept_detail_from_web_df = get_ths_concept_detail_from_web(concept_code, real_time_quotes_all_stocks)
     # 判断是否都为空
@@ -218,7 +218,7 @@ def sync_ths_concept_detail_to_db(concept_code, concept_name):
 
 
 if __name__ == '__main__':
-    sync_ths_concept_detail_to_db('886084', '光纤概念')
+    sync_ths_concept_detail_to_db('886090', '智谱AI')
     # real_time_quotes_all_stocks_df = east_money_stock_api.get_real_time_quotes_all_stocks()
     # get_ths_concept_detail_by_explain('886084', real_time_quotes_all_stocks_df)
     # get_ths_concept_detail_from_web('886084', real_time_quotes_all_stocks_df)

mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_new_index_api.py

@@ -39,7 +39,7 @@ def sync_ths_concept_new_index_from_app():
     # 当前最大概念代码
     max_concept_code = int(ths_concept_sync_common_api.get_max_concept_code())
     # 最大概念代码上限
-    max_concept_code_limit =max_concept_code + 2
+    max_concept_code_limit = max_concept_code + 2
     # 同花顺概念列表
     ths_concept_list_exist = ths_concept_common_service_api.get_all_ths_concept()
     # 同步向上3次
@@ -74,11 +74,14 @@ def sync_ths_concept_new_index_from_app():
                     'str_now_time': str_now_time,
                     'web_concept_code': concept_code,
                     'web_concept_url': url,
-                    'valid': True
+                    'valid': True,
+                    'grade': 1,
+                    'remark': ''
                 }
                 diff_one_df = pd.DataFrame(new_concept_one, index=[1])
+                # 保存新增概念信息
                 mongodb_util.save_mongo(diff_one_df, db_name_constant.THS_CONCEPT_LIST)
-                # 新增概念信息处理
+                # 新增概念信息处理 推送到微信
                 ths_concept_sync_common_api.push_msg_to_we_chat_and_redis(concept_code, concept_name,
                                                                           url)
                 #    同步概念详情到db
@@ -134,7 +137,9 @@ def sync_ths_concept_new_index_from_detail():
                     'str_now_time': str_now_time,
                     'web_concept_code': concept_code,
                     'web_concept_url': url,
-                    'valid': True
+                    'valid': True,
+                    'grade': 1,
+                    'remark': ''
                 }
                 diff_one_df = pd.DataFrame(new_concept_one, index=[1])
                 mongodb_util.save_mongo(diff_one_df, db_name_constant.THS_CONCEPT_LIST)

mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py

@@ -1,7 +1,7 @@
 import os
 import sys
 import mns_common.api.ths.concept.web.ths_concept_detail_web as ths_concept_detail_web
-import mns_common.api.em.east_money_stock_api as east_money_stock_api
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
 import mns_common.component.common_service_fun_api as common_service_fun_api
 from mns_common.db.MongodbUtil import MongodbUtil
 from loguru import logger
@@ -157,7 +157,7 @@ def update_symbol_new_concept(symbol_df, page_number):
 
 def sync_symbol_all_concept(symbol):
     create_index()
-    real_time_quotes_now = east_money_stock_api.get_real_time_quotes_all_stocks()
+    real_time_quotes_now = em_stock_info_api.get_a_stock_info()
     real_time_quotes_now = common_service_fun_api.classify_symbol(real_time_quotes_now)
     if symbol is not None:
         real_time_quotes_now = real_time_quotes_now.loc[real_time_quotes_now['symbol'] == symbol]
@@ -183,4 +183,4 @@ def sync_symbol_all_concept(symbol):
 
 
 if __name__ == '__main__':
-    sync_symbol_all_concept(None)
+    sync_symbol_all_concept('831175')

mns_scheduler/db/col_move_service.py

@@ -70,7 +70,7 @@ def sync_col_move(str_day):
         try:
             db_export('127.0.0.1:27017', str_day)
             db_import('127.0.0.1:27019', str_day)
-            # delete_exist_data(str_day)
+            delete_exist_data(str_day)
         except BaseException as e:
             logger.error("备份数据出现错误:{}", e)
 
@@ -115,5 +115,5 @@ if __name__ == '__main__':
 
     trade_date_list = mongodb_util.find_query_data('trade_date_list', query_trade)
     for trade_one in trade_date_list.itertuples():
-        trade_date = trade_one.trade_date
-        sync_col_move(trade_date)
+        trade_date_move = trade_one.trade_date
+        sync_col_move(trade_date_move)

mns_scheduler/db/script/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/col_move_script.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import pymongo

mns_scheduler/db/script/db_move/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler/db/script/db_move/col_move_one_service.py

@@ -0,0 +1,34 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+from loguru import logger
+
+
+def db_export(db, col):
+    cmd = 'F:/mongo/bin/mongodump.exe --host ' + db + ' -d patience -c ' + col + ' -o D:/back'
+    os.system(cmd)
+    logger.info("export finished:{}", col)
+
+
+def db_import(db, col):
+    cmd = 'F:/mongo/bin/mongorestore.exe --host ' + db + ' -d patience -c ' + col + ' D:/back/patience/' + col + '.bson'
+    os.system(cmd)
+
+    path = 'D:\\back\\patience\\' + col + '.bson'
+    cmd_del = 'del /F /S /Q ' + path
+    os.system(cmd_del)
+
+    logger.info("import finished:{}", col)
+
+
+def handle_one_col(col_name):
+    db_export('127.0.0.1:27017', col_name)
+    db_import('127.0.0.1:27019', col_name)
+
+
+if __name__ == '__main__':
+    handle_one_col('one_minute_k_line_bfq_h')

mns_scheduler/db/script/sync/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/sync/remote_data_sync_to_local.py

@@ -2,23 +2,28 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.utils.data_frame_util as data_frame_util
 from mns_scheduler.db.script.sync.remote_mongo_util import RemoteMongodbUtil
 from mns_scheduler.db.script.sync.local_mongo_util import LocalMongodbUtil
 from loguru import logger
+import numpy as np
 
 remote_mongodb_util = RemoteMongodbUtil('27017')
 local_mongodb_util = LocalMongodbUtil('27017')
 
 col_list = [
+    'em_a_stock_info',
+    'em_etf_info',
+    'em_kzz_info',
+    'em_hk_stock_info',
+    'company_info',
     'company_remark_info',
     'company_holding_info',
     'industry_concept_remark',
     'trade_date_list',
-    'company_info',
     'de_list_stock',
     'kpl_best_choose_index',
     'kpl_best_choose_index_detail',
@@ -29,8 +34,11 @@ col_list = [
     'self_choose_stock',
     'stock_account_info',
     'ths_concept_list',
-    'stock_zt_pool',
-    'ths_stock_concept_detail'
+    'stock_zt_pool_five',
+    'ths_stock_concept_detail',
+    'stock_high_chg_pool',
+    'today_new_concept_list',
+    'ths_stock_concept_detail_app'
 ]
 
 
@@ -41,6 +49,8 @@ def remote_data():
             if data_frame_util.is_not_empty(col_df):
                 result = local_mongodb_util.remove_all_data(col)
                 if result.acknowledged:
+                    col_df.replace([np.inf, -np.inf], 0, inplace=True)
+
                     local_mongodb_util.save_mongo(col_df, col)
 
                 logger.info("同步集合完成:{}", col)
@@ -48,5 +58,48 @@ def remote_data():
             logger.error("同步失败:{},{}", e, col)
 
 
+def sync_zt_data(str_day):
+    col = 'stock_zt_pool'
+    try:
+        query = {'str_day': str_day}
+        col_df = remote_mongodb_util.find_query_data(col, query)
+        if data_frame_util.is_not_empty(col_df):
+            col_df.replace([np.inf, -np.inf], 0, inplace=True)
+            local_mongodb_util.save_mongo(col_df, col)
+
+        logger.info("同步集合完成:{}", col)
+    except BaseException as e:
+        logger.error("同步失败:{},{}", e, col)
+
+
+def sync_open_data():
+    query = {"$and": [{'trade_date': {"$gte": "2025-03-21"}}, {'trade_date': {"$lte": "2025-04-02"}}]}
+    trade_date_list_df = remote_mongodb_util.find_query_data('trade_date_list', query)
+    trade_date_list_df = trade_date_list_df.sort_values(by=['trade_date'], ascending=False)
+    for trade_date_one in trade_date_list_df.itertuples():
+        try:
+            trade_date = trade_date_one.trade_date
+            query_01 = {"str_day": trade_date}
+            realtime_quotes_now_open_df = remote_mongodb_util.find_query_data('realtime_quotes_now_open', query_01)
+            if 'ths_concept_list' in realtime_quotes_now_open_df.columns:
+                del realtime_quotes_now_open_df['ths_concept_list']
+            local_mongodb_util.insert_mongo(realtime_quotes_now_open_df, 'realtime_quotes_now_open')
+            logger.info("同步到:{}", trade_date)
+        except BaseException as e:
+            logger.error("同步异常:{}", e)
+    return trade_date_list_df
+
+
 if __name__ == '__main__':
+    sync_zt_data('2025-09-17')
+    # sync_zt_data('2025-09-02')
     remote_data()
+    # sync_zt_data('2025-08-26')
+    # sync_zt_data('2025-08-25')
+    # sync_zt_data('2025-08-26')
+    # remote_data()
+    # sync_zt_data('2025-07-23')
+    # sync_zt_data('2025-07-24')
+    # sync_zt_data('2025-07-25')
+    # sync_open_data()
+    # remote_data()

mns_scheduler/db/script/sync/sync_hui_ce_test_data.py

@@ -0,0 +1,80 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_scheduler.db.script.sync.remote_mongo_util import RemoteMongodbUtil
+from mns_scheduler.db.script.sync.local_mongo_util import LocalMongodbUtil
+from loguru import logger
+
+remote_mongodb_util = RemoteMongodbUtil('27017')
+local_mongodb_util = LocalMongodbUtil('27017')
+
+
+def create_index(db_name):
+    local_mongodb_util.create_index(db_name, [("symbol", 1)])
+
+    local_mongodb_util.create_index(db_name, [("str_now_date", 1)])
+
+    local_mongodb_util.create_index(db_name, [("symbol", 1), ("number", 1)])
+
+
+def sync_real_time_data(str_day, min_number, max_number):
+    db_name = 'realtime_quotes_now_' + str_day
+    # 创建索引
+    create_index(db_name)
+    while min_number <= max_number:
+        query = {'number': min_number}
+        data_df = remote_mongodb_util.find_query_data(db_name, query)
+        local_mongodb_util.insert_mongo(data_df, db_name)
+        min_number = min_number + 1
+        logger.info(min_number)
+
+
+def sync_k_line(str_day):
+    query = {"str_day": str_day}
+    db_name = 'k_line_info'
+    data_df = remote_mongodb_util.find_query_data(db_name, query)
+    try:
+        local_mongodb_util.insert_mongo(data_df, db_name)
+    except BaseException as e:
+        logger.error("出现异常:{}", e)
+        pass
+
+
+if __name__ == '__main__':
+    str_day_01 = '2025-09-10'
+    sync_k_line(str_day_01)
+
+    # sync_real_time_data(str_day_01, 1, 2500)
+
+    # sync_k_line(str_day_01)
+    # sync_k_line('2024-11-13')
+    # sync_k_line('2024-11-12')
+    # sync_k_line('2024-11-15') 232
+
+    # sync_real_time_data('2024-11-15', 10, 1010)
+
+    # sync_k_line('2024-12-20')
+    # sync_k_line('2024-12-19')
+    # sync_k_line('2024-12-18')
+    # sync_k_line('2024-12-17')
+    # sync_k_line('2024-12-16')
+    #
+    # sync_k_line('2024-12-13')
+    # sync_k_line('2024-12-12')
+    # sync_k_line('2024-12-11')
+    # sync_k_line('2024-12-10')
+    # sync_k_line('2024-12-09')
+    #
+    # sync_k_line('2024-12-06')
+    # sync_k_line('2024-12-05')
+    # sync_k_line('2024-12-04')
+    # sync_k_line('2024-12-03')
+    # sync_k_line('2024-12-02')
+
+    # sync_real_time_data('2024-12-20', 1000, 2000)
+    # sync_real_time_data('2024-11-13', 10, 1010)