mns-scheduler 1.0.8.7__py3-none-any.whl → 1.4.3.2__py3-none-any.whl

mns_scheduler/concept/ths/detaill/ths_concept_detail_api.py

@@ -0,0 +1,226 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
+import mns_common.component.common_service_fun_api as common_service_fun_api
+import mns_common.api.ths.concept.app.ths_concept_detail_app as ths_concept_detail_app
+import mns_common.api.ths.wen_cai.ths_wen_cai_api as ths_wen_cai_api
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
+import mns_common.component.company.company_common_service_new_api as company_common_service_new_api
+import mns_common.api.ths.concept.web.ths_concept_detail_web as ths_concept_detail_web
+import math
+from loguru import logger
+import pandas as pd
+import mns_common.utils.data_frame_util as data_frame_util
+import time
+from datetime import datetime
+import mns_scheduler.concept.ths.common.ths_concept_sync_common_api as ths_concept_sync_common_api
+import mns_common.component.redis_msg.redis_msg_publish_service as redis_msg_publish_service
+import mns_common.constant.redis_msg_constant as redis_msg_constant
+
+
+def get_ths_concept_detail(concept_code, concept_name):
+    real_time_quotes_all_stocks = em_stock_info_api.get_a_stock_info()
+    # 1  通过入选理获取概念组成股票详情
+    ths_concept_detail_by_explain = get_ths_concept_detail_by_explain(concept_code, real_time_quotes_all_stocks)
+    # 2 app 分享链接获取概念组成详情
+    ths_concept_detail_from_app = get_ths_concept_detail_from_app(concept_code, real_time_quotes_all_stocks)
+    # 3 通过问财
+    if concept_name is None or data_frame_util.is_string_empty(concept_name):
+        ths_concept_list = ths_concept_common_service_api.get_all_ths_concept()
+        ths_concept_one_df = ths_concept_list.loc[
+            (ths_concept_list['symbol'] == int(concept_code))
+            | (ths_concept_list['web_concept_code'] == int(concept_code))]
+        if data_frame_util.is_not_empty(ths_concept_one_df):
+            concept_name = list(ths_concept_one_df['name'])[0]
+    # if data_frame_util.is_string_not_empty(concept_name) and concept_name is not None:
+    #     concept_detail_by_wen_cai_df = get_ths_concept_detail_by_wen_cai(concept_name, real_time_quotes_all_stocks)
+    # else:
+    concept_detail_by_wen_cai_df = None
+    # 4 通过web端
+    ths_concept_detail_from_web_df = get_ths_concept_detail_from_web(concept_code, real_time_quotes_all_stocks)
+    # 判断是否都为空
+    all_are_none = all(
+        df is None for df in [ths_concept_detail_by_explain, ths_concept_detail_from_app,
+                              concept_detail_by_wen_cai_df, ths_concept_detail_from_web_df])
+    if all_are_none:
+        return None
+    result_df = pd.concat([ths_concept_detail_by_explain, ths_concept_detail_from_app,
+                           concept_detail_by_wen_cai_df, ths_concept_detail_from_web_df])
+
+    result_df.drop_duplicates('symbol', keep='last', inplace=True)
+    if data_frame_util.is_not_empty(ths_concept_detail_by_explain):
+        ths_concept_detail_by_explain_new = ths_concept_detail_by_explain.loc[
+            ths_concept_detail_by_explain['symbol'].isin(result_df['symbol'])]
+
+        not_in_ths_concept_detail_by_explain = result_df.loc[~(
+            result_df['symbol'].isin(ths_concept_detail_by_explain['symbol']))]
+        result_df = pd.concat([ths_concept_detail_by_explain_new, not_in_ths_concept_detail_by_explain])
+    result_df['concept_code'] = concept_code
+    result_df['change'] = 0
+    result_df['index'] = 0
+    return result_df
+
+
+# 通过入选理获取概念组成股票详情
+def get_ths_concept_detail_by_explain(concept_code, real_time_quotes_all_stocks):
+    page_size = 800
+
+    de_list_symbols = company_common_service_new_api.get_de_list_company()
+    real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
+        ~(real_time_quotes_all_stocks['symbol'].isin(de_list_symbols))]
+
+    max_page_number = math.ceil(real_time_quotes_all_stocks.shape[0] / page_size)
+    page = 0
+
+    result_df = None
+
+    while page < max_page_number:
+        try:
+
+            begin_index = page * page_size
+            end_index = (page + 1) * page_size
+            page_df = real_time_quotes_all_stocks.iloc[begin_index: end_index]
+            code_list = ','.join(page_df['symbol'].astype(str))
+            all_ths_concept_detail = ths_concept_detail_app.get_concept_explain(concept_code, code_list)
+            if data_frame_util.is_empty(result_df) and data_frame_util.is_not_empty(all_ths_concept_detail):
+                result_df = all_ths_concept_detail
+            elif data_frame_util.is_not_empty(result_df) and data_frame_util.is_not_empty(all_ths_concept_detail):
+                result_df = pd.concat([all_ths_concept_detail, result_df])
+        except BaseException as e:
+            logger.error("通过ths概念入选理由列表获取详情异常:{},{}", concept_code, e)
+        page = page + 1
+    if data_frame_util.is_not_empty(result_df):
+        result_df = result_df[result_df['explain'].astype(str).str.len() > 0]
+    if data_frame_util.is_not_empty(result_df):
+        result_df = result_df.rename(columns={
+            "stockCode": "symbol",
+            "explain": "long"})
+        result_df = merge_data_common_fun(result_df, real_time_quotes_all_stocks)
+        return result_df
+    else:
+        return None
+
+
+# 通过分享链接获取
+def get_ths_concept_detail_from_app(concept_code, real_time_quotes_all_stocks):
+    ths_concept_detail_from_app = ths_concept_detail_app.get_ths_concept_detail_by_app(concept_code)
+    if data_frame_util.is_not_empty(ths_concept_detail_from_app):
+        ths_concept_detail_from_app['long'] = ''
+        ths_concept_detail_from_app = ths_concept_detail_from_app[[
+            'symbol',
+            'long'
+        ]]
+        ths_concept_detail_from_app = merge_data_common_fun(ths_concept_detail_from_app,
+                                                            real_time_quotes_all_stocks)
+        return ths_concept_detail_from_app
+
+    else:
+        return None
+
+
+# 3 通过问财
+def get_ths_concept_detail_by_wen_cai(concept_name, real_time_quotes_all_stocks):
+    concept_detail_by_wen_cai_df = ths_wen_cai_api.get_concept_detail_by_wen_cai(concept_name)
+    if data_frame_util.is_not_empty(concept_detail_by_wen_cai_df):
+        concept_detail_by_wen_cai_df = concept_detail_by_wen_cai_df[[
+            'symbol',
+            'explain'
+        ]]
+
+        concept_detail_by_wen_cai_df = concept_detail_by_wen_cai_df.rename(columns={
+            "explain": "long"})
+        concept_detail_by_wen_cai_df = merge_data_common_fun(concept_detail_by_wen_cai_df,
+                                                             real_time_quotes_all_stocks)
+
+    return concept_detail_by_wen_cai_df
+
+
+# 通过web端 最近不可用
+def get_ths_concept_detail_from_web(concept_code, real_time_quotes_all_stocks):
+    ths_concept_detail_from_web_df = ths_concept_detail_web.stock_board_cons_ths(concept_code)
+    if data_frame_util.is_not_empty(ths_concept_detail_from_web_df):
+        ths_concept_detail_from_web_df = ths_concept_detail_from_web_df[[
+            'symbol'
+        ]]
+        ths_concept_detail_from_web_df['long'] = ''
+
+        ths_concept_detail_from_web_df = merge_data_common_fun(ths_concept_detail_from_web_df,
+                                                               real_time_quotes_all_stocks)
+
+    return ths_concept_detail_from_web_df
+
+
+def merge_data_common_fun(result_df, real_time_quotes_all_stocks):
+    real_time_quotes_ths_detail = real_time_quotes_all_stocks.loc[
+        real_time_quotes_all_stocks['symbol'].isin(result_df['symbol'])]
+    real_time_quotes_ths_detail = common_service_fun_api.total_mv_classification(real_time_quotes_ths_detail)
+    real_time_quotes_ths_detail = real_time_quotes_ths_detail[[
+        "symbol",
+        "name",
+        "now_price",
+        "chg",
+        "exchange",
+        "amount",
+        "flow_mv_sp",
+        "total_mv_sp"]]
+
+    query_field = {
+        "concept_name": 1,
+        "concept_code": 1,
+        "company_type": 1,
+        "concept_create_day": 1,
+        "first_relevance_industry": 1,
+        "second_relevance_industry": 1,
+        "industry": 1
+    }
+    query_field_key = str(query_field)
+    query_key = str({})
+    company_df = company_common_service_new_api.get_company_info_by_field(query_key, query_field_key)
+    company_ths_df = company_df.loc[company_df['_id'].isin(result_df['symbol'])]
+
+    result_df['short'] = result_df['long']
+    result_df = result_df.set_index(['symbol'], drop=True)
+    company_ths_df = company_ths_df.set_index(['_id'], drop=True)
+    real_time_quotes_ths_detail = real_time_quotes_ths_detail.set_index(['symbol'], drop=False)
+    result_df = pd.merge(real_time_quotes_ths_detail, result_df,
+                         how='outer',
+                         left_index=True, right_index=True)
+    result_df = pd.merge(result_df, company_ths_df,
+                         how='outer',
+                         left_index=True, right_index=True)
+    return result_df
+
+
+# 同步概念详情到db
+def sync_ths_concept_detail_to_db(concept_code, concept_name):
+    new_concept_symbol_detail_df = get_ths_concept_detail(concept_code, concept_name)
+    if data_frame_util.is_empty(new_concept_symbol_detail_df):
+        time.sleep(10)
+        return None
+    now_date = datetime.now()
+    str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
+    time.sleep(1)
+    str_day = now_date.strftime('%Y-%m-%d')
+    new_concept_symbol_detail_df.loc[:, 'way'] = 'index_sync'
+    ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_detail_df,
+                                                        concept_name,
+                                                        str_day,
+                                                        str_now_time, concept_code)
+
+    # 项目之前推送消息
+    redis_msg_publish_service.send_redis_msg(redis_msg_constant.THS_CONCEPT_MSG_TOPIC,
+                                             redis_msg_constant.THS_NEW_CONCEPT_ADD_MSG)
+
+
+if __name__ == '__main__':
+    sync_ths_concept_detail_to_db('886090', '智谱AI')
+    # real_time_quotes_all_stocks_df = east_money_stock_api.get_real_time_quotes_all_stocks()
+    # get_ths_concept_detail_by_explain('886084', real_time_quotes_all_stocks_df)
+    # get_ths_concept_detail_from_web('886084', real_time_quotes_all_stocks_df)
+    # get_ths_concept_detail('886084', '光纤概念')
+    # get_ths_concept_detail_by_wen_cai('光纤概念', real_time_quotes_all_stocks_df)

mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_new_index_api.py

@@ -0,0 +1,169 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import mns_common.api.ths.concept.app.ths_concept_index_app as ths_concept_index_app
+import mns_scheduler.concept.ths.common.ths_concept_sync_common_api as ths_concept_sync_common_api
+import mns_common.utils.data_frame_util as data_frame_util
+from loguru import logger
+import mns_common.api.ths.concept.web.ths_concept_index_web as ths_concept_index_web
+from mns_common.db.MongodbUtil import MongodbUtil
+from datetime import datetime
+import mns_scheduler.concept.clean.ths_concept_clean_api as ths_concept_clean_api
+import pandas as pd
+import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_common.constant.db_name_constant as db_name_constant
+import mns_common.component.redis_msg.redis_msg_publish_service as redis_msg_publish_service
+import mns_common.constant.redis_msg_constant as redis_msg_constant
+
+mongodb_util = MongodbUtil('27017')
+import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
+
+
+def sync_ths_concept_new_index():
+    # 同步ths新概念 通过app搜索
+    sync_ths_concept_new_index_from_app()
+    # 通过ths详情搜索
+    sync_ths_concept_new_index_from_detail()
+
+
+'''
+同步ths新概念 通过app搜索代码
+'''
+
+
+def sync_ths_concept_new_index_from_app():
+    # 当前最大概念代码
+    max_concept_code = int(ths_concept_sync_common_api.get_max_concept_code())
+    # 最大概念代码上限
+    max_concept_code_limit = max_concept_code + 2
+    # 同花顺概念列表
+    ths_concept_list_exist = ths_concept_common_service_api.get_all_ths_concept()
+    # 同步向上3次
+    while max_concept_code <= max_concept_code_limit:
+        try:
+            max_concept_code = max_concept_code + 1
+            concept_new_index_df = ths_concept_index_app.get_new_concept_from_app_search(max_concept_code)
+            if data_frame_util.is_empty(concept_new_index_df):
+                continue
+            concept_name = list(concept_new_index_df['concept_name'])[0]
+
+            if data_frame_util.is_string_empty(concept_name):
+                concept_name = ths_concept_index_web.get_concept_name(max_concept_code)
+
+            exist_concept_df_one = ths_concept_list_exist.loc[
+                (ths_concept_list_exist['symbol'] == max_concept_code)
+                | (ths_concept_list_exist['web_concept_code'] == max_concept_code)]
+            now_date = datetime.now()
+            str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
+            str_day = now_date.strftime('%Y-%m-%d')
+            if data_frame_util.is_empty(exist_concept_df_one):
+                concept_code = max_concept_code
+
+                url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
+                new_concept_one = {
+                    '_id': int(concept_code),
+                    'symbol': int(concept_code),
+                    'name': concept_name,
+                    'url': url,
+                    'str_day': str_day,
+                    'success': True,
+                    'str_now_time': str_now_time,
+                    'web_concept_code': concept_code,
+                    'web_concept_url': url,
+                    'valid': True,
+                    'grade': 1,
+                    'remark': ''
+                }
+                diff_one_df = pd.DataFrame(new_concept_one, index=[1])
+                # 保存新增概念信息
+                mongodb_util.save_mongo(diff_one_df, db_name_constant.THS_CONCEPT_LIST)
+                # 新增概念信息处理 推送到微信
+                ths_concept_sync_common_api.push_msg_to_we_chat_and_redis(concept_code, concept_name,
+                                                                          url)
+                #    同步概念详情到db
+                ths_concept_detail_api.sync_ths_concept_detail_to_db(concept_code, concept_name)
+                # 更新ths概念统计信息
+                ths_concept_clean_api.update_ths_concept_info()
+
+                logger.info("新增同花顺新概念:{}", concept_name)
+
+        except BaseException as e:
+            logger.error("同步新增概念代码:{},信息异常:{}", max_concept_code, e)
+
+
+'''
+同步新概念 by ths detail 通过详情判断
+'''
+
+
+def sync_ths_concept_new_index_from_detail():
+    # 当前最大概念代码
+    max_concept_code = ths_concept_sync_common_api.get_max_concept_code()
+    # 最大概念代码上线
+    max_concept_code_limit = max_concept_code + 2
+    # 同花顺概念列表
+    ths_concept_list_exist = mongodb_util.find_all_data(db_name_constant.THS_CONCEPT_LIST)
+    # 同步向上3次
+    while max_concept_code <= max_concept_code_limit:
+        try:
+            max_concept_code = max_concept_code + 1
+            concept_code = max_concept_code
+            concept_name = ths_concept_index_web.get_concept_name(concept_code)
+            new_concept_symbol_detail_df = ths_concept_detail_api.get_ths_concept_detail(concept_code, concept_name)
+            if data_frame_util.is_empty(new_concept_symbol_detail_df):
+                continue
+
+            exist_concept_df_one = ths_concept_list_exist.loc[
+                (ths_concept_list_exist['symbol'] == concept_code)
+                | (ths_concept_list_exist['web_concept_code'] == concept_code)]
+            now_date = datetime.now()
+            str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
+            str_day = now_date.strftime('%Y-%m-%d')
+            if data_frame_util.is_empty(exist_concept_df_one):
+                concept_code = max_concept_code
+
+                url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
+                new_concept_one = {
+                    '_id': int(concept_code),
+                    'symbol': int(concept_code),
+                    'name': concept_name,
+                    'url': url,
+                    'str_day': str_day,
+                    'success': True,
+                    'str_now_time': str_now_time,
+                    'web_concept_code': concept_code,
+                    'web_concept_url': url,
+                    'valid': True,
+                    'grade': 1,
+                    'remark': ''
+                }
+                diff_one_df = pd.DataFrame(new_concept_one, index=[1])
+                mongodb_util.save_mongo(diff_one_df, db_name_constant.THS_CONCEPT_LIST)
+                # 新增概念信息处理
+                ths_concept_sync_common_api.push_msg_to_we_chat_and_redis(concept_code, concept_name,
+                                                                          url)
+
+                new_concept_symbol_detail_df.loc[:, 'way'] = 'index_sync'
+                ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_detail_df,
+                                                                    concept_name,
+                                                                    str_day,
+                                                                    str_now_time,
+                                                                    concept_code)
+
+                logger.info("新增同花顺新概念:{}", concept_name)
+
+        except BaseException as e:
+            logger.error("同步新增概念代码:{},信息异常:{}", max_concept_code, e)
+    # 更新ths概念统计信息
+    ths_concept_clean_api.update_ths_concept_info()
+    # 项目之间推送消息
+    redis_msg_publish_service.send_redis_msg(redis_msg_constant.THS_CONCEPT_MSG_TOPIC,
+                                             redis_msg_constant.THS_NEW_CONCEPT_ADD_MSG)
+
+
+if __name__ == '__main__':
+    sync_ths_concept_new_index()

mns_scheduler/concept/ths/update_concept_info/sync_one_concept_all_symbols_api.py

@@ -5,43 +5,31 @@ file_path = os.path.abspath(__file__)
 end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
-
-import time
 import datetime
-from mns_common.db.MongodbUtil import MongodbUtil
-import mns_scheduler.concept.ths.common.ths_concept_sync_common_api as ths_concept_sync_common_api
-
-mongodb_util = MongodbUtil('27017')
+import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
+from loguru import logger
 
 
 # 通过概念指数同步所有概念下的股票组成
 
 # 同步概念下所有股票组成
-def update_concept_all_symbol_detail():
-    query = {"symbol": {'$exists': True}}
-    new_concept_list = mongodb_util.find_query_data("ths_concept_list", query)
+def update_concept_all_detail_info():
+    new_concept_list = ths_concept_common_service_api.get_all_ths_concept()
     new_concept_list = new_concept_list.sort_values(by=['symbol'], ascending=False)
-    if new_concept_list.shape[0] > 0:
-        for one_concept in new_concept_list.itertuples():
-            now_date_time = datetime.datetime.now()
-            str_day = now_date_time.strftime('%Y-%m-%d')
-            str_now_time = now_date_time.strftime('%Y-%m-%d %H:%M:%S')
-            new_concept_symbol_df = ths_concept_sync_common_api.get_concept_detail_info_web(one_concept.symbol)
 
-            if new_concept_symbol_df is None or new_concept_symbol_df.shape[0] == 0:
-                time.sleep(10)
-                continue
-            time.sleep(1)
-            new_concept_symbol_df.loc[:, 'way'] = 'index_sync'
-            ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_df, one_concept.name,
-                                                                    str_day, str_now_time, one_concept.symbol)
+    for one_concept in new_concept_list.itertuples():
+        try:
+            ths_concept_detail_api.sync_ths_concept_detail_to_db(one_concept.symbol, one_concept.name)
+        except BaseException as e:
+            logger.error("同步概念下所有股票组成异常:{},{}，｛｝", one_concept.symbol, one_concept.name, e)
 
 
 if __name__ == '__main__':
     now_date = datetime.datetime.now()
     begin_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
     print('同步开始:' + begin_date)
-    update_concept_all_symbol_detail()
+    update_concept_all_detail_info()
     now_date = datetime.datetime.now()
     end_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
     print('同步结束:' + end_date)

mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py

@@ -1,8 +1,7 @@
 import os
 import sys
-import pandas as pd
-import mns_common.api.ths.ths_stock_api as ths_stock_api
-import mns_common.api.em.east_money_stock_api as east_money_stock_api
+import mns_common.api.ths.concept.web.ths_concept_detail_web as ths_concept_detail_web
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
 import mns_common.component.common_service_fun_api as common_service_fun_api
 from mns_common.db.MongodbUtil import MongodbUtil
 from loguru import logger
@@ -13,6 +12,9 @@ import mns_scheduler.concept.ths.common.ths_concept_sync_common_api as ths_conce
 import threading
 import mns_scheduler.concept.ths.common.ths_concept_update_common_api as ths_concept_update_common_api
 import mns_common.utils.date_handle_util as date_handle_util
+import mns_common.constant.db_name_constant as db_name_constant
+import mns_common.component.redis_msg.redis_msg_publish_service as redis_msg_publish_service
+import mns_common.constant.redis_msg_constant as redis_msg_constant
 
 file_path = os.path.abspath(__file__)
 end = file_path.index('mns') + 17
@@ -21,16 +23,16 @@ sys.path.append(project_path)
 mongodb_util = MongodbUtil('27017')
 # 分页大小
 MAX_PAGE_NUMBER = 2500
-# 获取单只股票新增概念
 
-db_name = 'ths_stock_concept_detail_app'
+
+# 获取单只股票新增概念
 
 
 def create_index():
-    mongodb_util.create_index(db_name, [("symbol", 1)])
-    mongodb_util.create_index(db_name, [("str_day", 1)])
-    mongodb_util.create_index(db_name, [("title", 1)])
-    mongodb_util.create_index(db_name, [("str_now_date", 1)])
+    mongodb_util.create_index(db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP, [("symbol", 1)])
+    mongodb_util.create_index(db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP, [("str_day", 1)])
+    mongodb_util.create_index(db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP, [("title", 1)])
+    mongodb_util.create_index(db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP, [("str_now_date", 1)])
 
 
 # 同步新概念到详情表中
@@ -52,6 +54,9 @@ def sync_new_concept_to_ths_detail(symbol_add_new_concept_df, str_day, str_now_t
 
         except BaseException as e:
             logger.error("转换同花顺概念异常:{},{}", new_concept_one, e)
+    # 项目之间推送消息
+    redis_msg_publish_service.send_redis_msg(redis_msg_constant.THS_CONCEPT_MSG_TOPIC,
+                                             redis_msg_constant.THS_NEW_CONCEPT_ADD_MSG)
 
 
 # 保存数据到对比
@@ -60,7 +65,7 @@ def save_data_to_db(ths_concept_df):
     if data_frame_util.is_empty(ths_concept_df):
         return
     json_data = ths_concept_df.to_dict(orient='records')
-    mongodb_util.save_mongo_json(json_data, db_name)
+    mongodb_util.save_mongo_json(json_data, db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP)
 
 
 # 对比数据库和接口概念详情的差值
@@ -86,9 +91,8 @@ def choose_type_new_concept(ths_concept_df):
 def update_symbol_new_concept(symbol_df, page_number):
     for stock_one in symbol_df.itertuples():
         try:
-            ths_concept_json = ths_stock_api.get_symbol_add_new_concept(stock_one.symbol)
-            logger.info("同步symbol概念信息:{}", stock_one.symbol)
-            symbol_ths_concept_all_df = pd.DataFrame(ths_concept_json)
+            symbol_ths_concept_all_df = ths_concept_detail_web.get_one_symbol_all_ths_concepts(stock_one.symbol)
+            logger.info("同步代码{},所有概念信息", stock_one.symbol)
             if data_frame_util.is_empty(symbol_ths_concept_all_df):
                 continue
 
@@ -153,7 +157,7 @@ def update_symbol_new_concept(symbol_df, page_number):
 
 def sync_symbol_all_concept(symbol):
     create_index()
-    real_time_quotes_now = east_money_stock_api.get_real_time_quotes_all_stocks()
+    real_time_quotes_now = em_stock_info_api.get_a_stock_info()
     real_time_quotes_now = common_service_fun_api.classify_symbol(real_time_quotes_now)
     if symbol is not None:
         real_time_quotes_now = real_time_quotes_now.loc[real_time_quotes_now['symbol'] == symbol]
@@ -179,4 +183,4 @@ def sync_symbol_all_concept(symbol):
 
 
 if __name__ == '__main__':
-    sync_symbol_all_concept(None)
+    sync_symbol_all_concept('831175')

mns_scheduler/db/col_move_service.py

@@ -93,7 +93,9 @@ def delete_exist_data(str_day):
 
 
 def db_export_col(db, col):
-    cmd = 'D:/MongoDB/tools/bin/mongodump.exe --host ' + db + ' -d patience -c ' + col + ' -o G:/backup'
+    cmd = 'F:/mongo/bin/mongodump.exe --host ' + db + ' -d patience -c ' + col + ' -o H:/back'
+    os.system(cmd)
+    logger.info("export finished:{}")
     os.system(cmd)
 
     logger.info("export finished:{}", col)
@@ -101,7 +103,7 @@ def db_export_col(db, col):
 
 # D:\software\mongodb-tools\mongodb-database-tools-windows-x86_64-100.5.2\bin
 def db_import_col(db, col):
-    cmd = 'D:/software/mongodb-tools/mongodb-database-tools-windows-x86_64-100.5.2/bin/mongorestore.exe --host ' + db + ' -d patience -c ' + col + ' D:/back/patience/' + col + '.bson'
+    cmd = 'F:/mongo/bin/mongodump.exe --host ' + db + ' -d patience -c ' + col + ' H:/back/patience/' + col + '.bson'
     os.system(cmd)
 
     logger.info("import finished:{}", col)
@@ -109,8 +111,9 @@ def db_import_col(db, col):
 
 # -u "username" -p "password"
 if __name__ == '__main__':
-    sync_col_move('2024-05-28')
-    sync_col_move('2024-05-30')
-    sync_col_move('2024-05-31')
-    sync_col_move('2024-06-03')
-    sync_col_move('2024-06-04')
+    query_trade = {"$and": [{"trade_date": {"$gte": "2024-09-24"}}, {"trade_date": {"$lte": "2024-10-22"}}]}
+
+    trade_date_list = mongodb_util.find_query_data('trade_date_list', query_trade)
+    for trade_one in trade_date_list.itertuples():
+        trade_date_move = trade_one.trade_date
+        sync_col_move(trade_date_move)

mns_scheduler/db/db_status.py

@@ -21,4 +21,4 @@ def db_status_check():
         ip = ip_util.get_host_ip()
         push_msg_api.push_msg_to_wechat("数据库挂了", "地址:" + ip + "-" + "数据库挂了")
         logger.error("出现异常:{}", e)
-        return
+

mns_scheduler/db/script/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler/db/script/col_move_script.py

@@ -0,0 +1,45 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import pymongo
+from mns_common.db.MongodbUtil import MongodbUtil
+
+mongodb_util = MongodbUtil('27017')
+
+client_win = pymongo.MongoClient("mongodb://192.168.1.6:" + '27017' + "/patience")
+from loguru import logger
+
+
+def insert_mongo(df, col):
+    db = client_win.patience
+
+    if df is None or len(df) == 0:
+        return
+    collection = db[col]
+    # 格式转换
+    try:
+        df = df.drop_duplicates()
+        # df = df.T.drop_duplicates().T
+        records = df.to_dict('records')
+        collection.insert_many(records)
+    except BaseException as e:
+        logger.error("插入数据异常:{}", e)
+
+
+def col_move():
+    col = 'realtime_quotes_now_2024-09-23'
+    number = 8
+    while number < 3312:
+        query = {'number': number}
+        realtime_quotes_now_df = mongodb_util.find_query_data(col, query)
+        insert_mongo(realtime_quotes_now_df, col)
+        number = number + 1
+        print(number)
+
+
+if __name__ == '__main__':
+    col_move()

mns_scheduler/db/script/db_move/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)