mns-scheduler 1.1.3.6__py3-none-any.whl → 1.1.3.9__py3-none-any.whl

mns_scheduler/common/redis/redis_msg_push_service.py

@@ -0,0 +1,18 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import redis
+import mns_common.constant.redis_msg_constant as redis_msg_constant
+
+# 连接到Redis服务器
+r = redis.Redis(host='localhost', port=6379, db=0)
+
+
+# 发送消息
+def send_redis_msg(message):
+    # 将消息推送到队列中
+    r.publish(redis_msg_constant.THS_NEW_CONCEPT_ADD, message)

mns_scheduler/company_info/clean/company_info_clean_api.py

@@ -11,7 +11,7 @@ from datetime import datetime
 from loguru import logger
 import mns_scheduler.company_info.constant.company_constant_data as company_constant_data
 import mns_common.constant.db_name_constant as db_name_constant
-import mns_scheduler.concept.common.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
 import mns_scheduler.company_info.base.sync_company_base_info_api as company_info_sync_api
 
 mongodb_util = MongodbUtil('27017')

mns_scheduler/concept/clean/ths_concept_clean_api.py

@@ -7,11 +7,12 @@ project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil
 import pandas as pd
-import mns_scheduler.concept.common.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
 import time
 from loguru import logger
 import mns_common.component.company.company_common_service_api as company_common_service_api
 import mns_common.constant.db_name_constant as db_name_constant
+import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
 
 mongodb_util = MongodbUtil('27017')
 
@@ -21,59 +22,64 @@ def update_ths_concept_info():
     ths_concept_list = mongodb_util.find_all_data(db_name_constant.THS_CONCEPT_LIST)
     for ths_concept_one in ths_concept_list.itertuples():
         try:
-            query = {'concept_code': ths_concept_one.symbol}
-            ths_stock_concept_detail_df = (mongodb_util
-                                           .find_query_data(db_name_constant.THS_STOCK_CONCEPT_DETAIL, query))
-            concept_count = ths_stock_concept_detail_df.shape[0]
-            ths_concept_list_one_df = ths_concept_list.loc[ths_concept_list['symbol'] == ths_concept_one.symbol]
-            ths_concept_list_one_df['concept_count'] = concept_count
-
-            ths_stock_concept_detail_df = ths_stock_concept_detail_df.reset_index(drop=True)
-            if 'industry' in ths_stock_concept_detail_df.columns:
-                del ths_stock_concept_detail_df['industry']
-
-            company_info_df = company_common_service_api.get_company_info_industry()
-            company_info_df = company_info_df[['_id', 'industry']]
-            company_info_df = company_info_df.loc[
-                company_info_df['_id'].isin(list(ths_stock_concept_detail_df['symbol']))]
-            company_info_df = company_info_df.set_index(['_id'], drop=True)
-
-            ths_stock_concept_detail_df = ths_stock_concept_detail_df.set_index(['symbol'], drop=False)
-            ths_stock_concept_detail_df = pd.merge(ths_stock_concept_detail_df, company_info_df,
-                                                   how='outer',
-                                                   left_index=True, right_index=True)
-            ths_stock_concept_detail_df.dropna(subset=['industry'], axis=0, inplace=True)
-
-            grouped = ths_stock_concept_detail_df.groupby('industry')
-            result_list = grouped.size()
-            ths_concept_group = pd.DataFrame(result_list, columns=['number'])
-            ths_concept_group['industry'] = ths_concept_group.index
-            ths_concept_group = ths_concept_group.sort_values(by=['number'], ascending=False)
-            if ths_concept_group.shape[0] >= 2:
-                first_relevance_industry = list(ths_concept_group.iloc[0:1]['industry'])[0]
-                first_relevance_industry_number = list(ths_concept_group.iloc[0:1]['number'])[0]
-                second_relevance_industry = list(ths_concept_group.iloc[1:2]['industry'])[0]
-                second_relevance_industry_number = list(ths_concept_group.iloc[1:2]['number'])[0]
-            else:
-                first_relevance_industry = list(ths_concept_group.iloc[0:1]['industry'])[0]
-                first_relevance_industry_number = list(ths_concept_group.iloc[0:1]['number'])[0]
-                second_relevance_industry = '无'
-                second_relevance_industry_number = 0
-            ths_concept_list_one_df['first_relevance_industry'] = first_relevance_industry
-            ths_concept_list_one_df['second_relevance_industry'] = second_relevance_industry
-            ths_concept_list_one_df['first_relevance_industry_number'] = first_relevance_industry_number
-            ths_concept_list_one_df['second_relevance_industry_number'] = second_relevance_industry_number
-
-            ths_stock_concept_detail_df['first_relevance_industry'] = first_relevance_industry
-            ths_stock_concept_detail_df['second_relevance_industry'] = second_relevance_industry
-
-            mongodb_util.save_mongo(ths_concept_list_one_df, db_name_constant.THS_CONCEPT_LIST)
-            mongodb_util.save_mongo(ths_stock_concept_detail_df, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
-
+            update_one_concept_relevance_industry(ths_concept_one.symbol)
         except BaseException as e:
             logger.error("更新概念信息异常:{},{}", e, ths_concept_one.name)
 
 
+# 更新一个概念行业相关
+def update_one_concept_relevance_industry(concept_code):
+    query = {'concept_code': concept_code}
+    ths_stock_concept_detail_df = (mongodb_util
+                                   .find_query_data(db_name_constant.THS_STOCK_CONCEPT_DETAIL, query))
+    concept_count = ths_stock_concept_detail_df.shape[0]
+    ths_concept_list = ths_concept_common_service_api.get_all_ths_concept()
+    ths_concept_list_one_df = ths_concept_list.loc[ths_concept_list['symbol'] == concept_code]
+    ths_concept_list_one_df['concept_count'] = concept_count
+
+    ths_stock_concept_detail_df = ths_stock_concept_detail_df.reset_index(drop=True)
+    if 'industry' in ths_stock_concept_detail_df.columns:
+        del ths_stock_concept_detail_df['industry']
+
+    company_info_df = company_common_service_api.get_company_info_industry()
+    company_info_df = company_info_df[['_id', 'industry']]
+    company_info_df = company_info_df.loc[
+        company_info_df['_id'].isin(list(ths_stock_concept_detail_df['symbol']))]
+    company_info_df = company_info_df.set_index(['_id'], drop=True)
+
+    ths_stock_concept_detail_df = ths_stock_concept_detail_df.set_index(['symbol'], drop=False)
+    ths_stock_concept_detail_df = pd.merge(ths_stock_concept_detail_df, company_info_df,
+                                           how='outer',
+                                           left_index=True, right_index=True)
+    ths_stock_concept_detail_df.dropna(subset=['industry'], axis=0, inplace=True)
+
+    grouped = ths_stock_concept_detail_df.groupby('industry')
+    result_list = grouped.size()
+    ths_concept_group = pd.DataFrame(result_list, columns=['number'])
+    ths_concept_group['industry'] = ths_concept_group.index
+    ths_concept_group = ths_concept_group.sort_values(by=['number'], ascending=False)
+    if ths_concept_group.shape[0] >= 2:
+        first_relevance_industry = list(ths_concept_group.iloc[0:1]['industry'])[0]
+        first_relevance_industry_number = list(ths_concept_group.iloc[0:1]['number'])[0]
+        second_relevance_industry = list(ths_concept_group.iloc[1:2]['industry'])[0]
+        second_relevance_industry_number = list(ths_concept_group.iloc[1:2]['number'])[0]
+    else:
+        first_relevance_industry = list(ths_concept_group.iloc[0:1]['industry'])[0]
+        first_relevance_industry_number = list(ths_concept_group.iloc[0:1]['number'])[0]
+        second_relevance_industry = '无'
+        second_relevance_industry_number = 0
+    ths_concept_list_one_df['first_relevance_industry'] = first_relevance_industry
+    ths_concept_list_one_df['second_relevance_industry'] = second_relevance_industry
+    ths_concept_list_one_df['first_relevance_industry_number'] = first_relevance_industry_number
+    ths_concept_list_one_df['second_relevance_industry_number'] = second_relevance_industry_number
+
+    ths_stock_concept_detail_df['first_relevance_industry'] = first_relevance_industry
+    ths_stock_concept_detail_df['second_relevance_industry'] = second_relevance_industry
+
+    mongodb_util.save_mongo(ths_concept_list_one_df, db_name_constant.THS_CONCEPT_LIST)
+    mongodb_util.save_mongo(ths_stock_concept_detail_df, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
+
+
 # 更新空名字
 def update_null_name():
     query = {"_id": {'$gte': 886025}}

mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py

@@ -13,32 +13,26 @@ mongodb_util = MongodbUtil('27017')
 import mns_common.api.msg.push_msg_api as push_msg_api
 import mns_scheduler.company_info.base.sync_company_base_info_api as company_info_sync_api
 import mns_scheduler.company_info.clean.company_info_clean_api as company_info_clean_api
-import mns_scheduler.concept.common.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
 import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
 import mns_common.component.company.company_common_service_new_api as company_common_service_new_api
 import mns_common.utils.data_frame_util as data_frame_util
 import mns_common.constant.db_name_constant as db_name_constant
+import mns_scheduler.common.redis.redis_msg_push_service as redis_msg_push_service
+import mns_scheduler.concept.clean.ths_concept_clean_api as ths_concept_clean_api
 
-max_concept_code = 886110
 
-order_fields = [
-    "index",
-    "symbol",
-    "name",
-    "now_price",
-    "chg",
-    "change",
-    "exchange",
-    "amount",
-    "concept_code",
-]
-
-
-# 推送到手机
-def push_msg_to_we_chat_web(concept_code, concept_name, url):
+# 推送消息
+def push_msg_to_we_chat_and_redis(concept_code, concept_name, url):
     msg = "概念代码:" + str(concept_code) + "," + "概念名称:" + concept_name + "," + "url:   " + url
     title = "新增同花顺概念:" + str(concept_code) + "-" + concept_name
+    # 推送到微信
     push_msg_api.push_msg_to_wechat(title, msg)
+    # 项目之前推送消息
+    redis_msg_push_service.send_redis_msg("新增同花顺概念")
+
+    # 更新ths概念信息
+    ths_concept_clean_api.update_ths_concept_info()
 
 
 # 保存新概念信息到数据库
@@ -50,18 +44,6 @@ def save_ths_concept_list(concept_code, concept_name, str_day, str_now_time):
     mongodb_util.save_mongo(ths_concept_list, 'ths_concept_list')
 
 
-# 获取最大概念代码
-def get_max_concept_code():
-    query = {"symbol": {'$ne': 'null'}, "success": True}
-    ths_concept_max = mongodb_util.descend_query(query, 'ths_concept_list', 'symbol', 1)
-    if ths_concept_max.shape[0] == 0:
-        concept_code = 885284
-    else:
-        concept_code = list(ths_concept_max['symbol'])[0]
-
-    return concept_code
-
-
 # 保存新概念详细信息到数据库
 def save_ths_concept_detail(new_concept_symbol_df,
                             concept_name, str_day,
@@ -179,6 +161,18 @@ def update_company_info(new_concept_symbol_df):
         company_common_service_api.company_info_industry_cache_clear()
 
 
+# 获取最大概念代码
+def get_max_concept_code():
+    query = {"symbol": {'$ne': 'null'}, "success": True}
+    ths_concept_max = mongodb_util.descend_query(query, 'ths_concept_list', 'symbol', 1)
+    if ths_concept_max.shape[0] == 0:
+        concept_code = 885284
+    else:
+        concept_code = list(ths_concept_max['symbol'])[0]
+
+    return concept_code
+
+
 def get_concept_detail_info_web(concept_code):
     new_concept_symbol_list = ths_concept_detail_api.get_ths_concept_detail(concept_code, None)
     if new_concept_symbol_list is None or new_concept_symbol_list.shape[0] == 0:

mns_scheduler/concept/{common → ths}/detaill/ths_concept_detail_api.py

@@ -16,7 +16,10 @@ import math
 from loguru import logger
 import pandas as pd
 import mns_common.utils.data_frame_util as data_frame_util
-
+import time
+from datetime import datetime
+import mns_scheduler.concept.ths.common.ths_concept_sync_common_api as ths_concept_sync_common_api
+import mns_scheduler.common.redis.redis_msg_push_service as redis_msg_push_service
 
 def get_ths_concept_detail(concept_code, concept_name):
     real_time_quotes_all_stocks = east_money_stock_api.get_real_time_quotes_all_stocks()
@@ -55,6 +58,9 @@ def get_ths_concept_detail(concept_code, concept_name):
         not_in_ths_concept_detail_by_explain = result_df.loc[~(
             result_df['symbol'].isin(ths_concept_detail_by_explain['symbol']))]
         result_df = pd.concat([ths_concept_detail_by_explain_new, not_in_ths_concept_detail_by_explain])
+    result_df['concept_code'] = concept_code
+    result_df['change'] = 0
+    result_df['index'] = 0
     return result_df
 
 
@@ -188,10 +194,30 @@ def merge_data_common_fun(result_df, real_time_quotes_all_stocks):
     return result_df
 
 
+# 同步概念详情到db
+def sync_ths_concept_detail_to_db(concept_code, concept_name):
+    new_concept_symbol_detail_df = get_ths_concept_detail(concept_code, concept_name)
+    if data_frame_util.is_empty(new_concept_symbol_detail_df):
+        time.sleep(10)
+        return None
+    now_date = datetime.now()
+    str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
+    time.sleep(1)
+    str_day = now_date.strftime('%Y-%m-%d')
+    new_concept_symbol_detail_df.loc[:, 'way'] = 'index_sync'
+    ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_detail_df,
+                                                        concept_name,
+                                                        str_day,
+                                                        str_now_time, concept_code)
+
+    # 项目之前推送消息
+    redis_msg_push_service.send_redis_msg("新增同花顺概念")
+
+
 if __name__ == '__main__':
-    get_ths_concept_detail('886080', '财税数字化')
-    real_time_quotes_all_stocks_df = east_money_stock_api.get_real_time_quotes_all_stocks()
-    get_ths_concept_detail_by_explain('886078', real_time_quotes_all_stocks_df)
-    get_ths_concept_detail_from_web('886078', real_time_quotes_all_stocks_df)
-    get_ths_concept_detail('886073', '铜缆高速连接')
-    get_ths_concept_detail_by_wen_cai('PCB概念', real_time_quotes_all_stocks_df)
+    sync_ths_concept_detail_to_db('886084', '光纤概念')
+    # real_time_quotes_all_stocks_df = east_money_stock_api.get_real_time_quotes_all_stocks()
+    # get_ths_concept_detail_by_explain('886084', real_time_quotes_all_stocks_df)
+    # get_ths_concept_detail_from_web('886084', real_time_quotes_all_stocks_df)
+    # get_ths_concept_detail('886084', '光纤概念')
+    # get_ths_concept_detail_by_wen_cai('光纤概念', real_time_quotes_all_stocks_df)

mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_new_index_api.py

@@ -14,10 +14,12 @@ from mns_common.db.MongodbUtil import MongodbUtil
 from datetime import datetime
 import mns_scheduler.concept.clean.ths_concept_clean_api as ths_concept_clean_api
 import pandas as pd
-import mns_scheduler.concept.common.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
 import mns_common.constant.db_name_constant as db_name_constant
+import mns_scheduler.common.redis.redis_msg_push_service as redis_msg_push_service
 
 mongodb_util = MongodbUtil('27017')
+import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
 
 
 def sync_ths_concept_new_index():
@@ -28,17 +30,17 @@ def sync_ths_concept_new_index():
 
 
 '''
-同步ths新概念 通过app搜索
+同步ths新概念 通过app搜索代码
 '''
 
 
 def sync_ths_concept_new_index_from_app():
     # 当前最大概念代码
     max_concept_code = ths_concept_sync_common_api.get_max_concept_code()
-    # 最大概念代码上线
+    # 最大概念代码上限
     max_concept_code_limit = max_concept_code + 2
     # 同花顺概念列表
-    ths_concept_list_exist = mongodb_util.find_all_data(db_name_constant.THS_CONCEPT_LIST)
+    ths_concept_list_exist = ths_concept_common_service_api.get_all_ths_concept()
     # 同步向上3次
     while max_concept_code <= max_concept_code_limit:
         try:
@@ -76,16 +78,10 @@ def sync_ths_concept_new_index_from_app():
                 diff_one_df = pd.DataFrame(new_concept_one, index=[1])
                 mongodb_util.save_mongo(diff_one_df, db_name_constant.THS_CONCEPT_LIST)
                 # 新增概念信息处理
-                handle_new_concept_msg(concept_code, concept_name, url)
-
-                new_concept_symbol_detail_df = ths_concept_detail_api.get_ths_concept_detail(concept_code, concept_name)
-                if data_frame_util.is_empty(new_concept_symbol_detail_df):
-                    continue
-                new_concept_symbol_detail_df.loc[:, 'way'] = 'index_sync'
-                ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_detail_df,
-                                                                    concept_name,
-                                                                    str_day,
-                                                                    str_now_time, concept_code)
+                ths_concept_sync_common_api.push_msg_to_we_chat_and_redis(concept_code, concept_name,
+                                                                          url)
+                #    同步概念详情到db
+                ths_concept_detail_api.sync_ths_concept_detail_to_db(concept_code, concept_name)
                 # 更新ths概念统计信息
                 ths_concept_clean_api.update_ths_concept_info()
 
@@ -96,7 +92,7 @@ def sync_ths_concept_new_index_from_app():
 
 
 '''
-同步新概念 by ths detail
+同步新概念 by ths detail 通过详情判断
 '''
 
 
@@ -142,7 +138,8 @@ def sync_ths_concept_new_index_from_detail():
                 diff_one_df = pd.DataFrame(new_concept_one, index=[1])
                 mongodb_util.save_mongo(diff_one_df, db_name_constant.THS_CONCEPT_LIST)
                 # 新增概念信息处理
-                handle_new_concept_msg(concept_code, concept_name, url)
+                ths_concept_sync_common_api.push_msg_to_we_chat_and_redis(concept_code, concept_name,
+                                                                          url)
 
                 new_concept_symbol_detail_df.loc[:, 'way'] = 'index_sync'
                 ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_detail_df,
@@ -150,21 +147,16 @@ def sync_ths_concept_new_index_from_detail():
                                                                     str_day,
                                                                     str_now_time,
                                                                     concept_code)
-                # 更新ths概念统计信息
-                ths_concept_clean_api.update_ths_concept_info()
 
                 logger.info("新增同花顺新概念:{}", concept_name)
 
         except BaseException as e:
             logger.error("同步新增概念代码:{},信息异常:{}", max_concept_code, e)
-
-
-# 新增消息处理
-def handle_new_concept_msg(concept_code, concept_name, url):
-    # 推送新概念信息到微信
-    ths_concept_sync_common_api.push_msg_to_we_chat_web(concept_code, concept_name,
-                                                        url)
+    # 更新ths概念统计信息
+    ths_concept_clean_api.update_ths_concept_info()
+    # 项目之前推送消息
+    redis_msg_push_service.send_redis_msg("新增同花顺概念")
 
 
 if __name__ == '__main__':
-    sync_ths_concept_new_index_from_detail()
+    sync_ths_concept_new_index()

mns_scheduler/concept/ths/update_concept_info/sync_one_concept_all_symbols_api.py

@@ -5,36 +5,24 @@ file_path = os.path.abspath(__file__)
 end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
-
-import time
 import datetime
-from mns_common.db.MongodbUtil import MongodbUtil
-import mns_scheduler.concept.ths.common.ths_concept_sync_common_api as ths_concept_sync_common_api
-
-mongodb_util = MongodbUtil('27017')
+import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
+from loguru import logger
 
 
 # 通过概念指数同步所有概念下的股票组成
 
 # 同步概念下所有股票组成
 def update_concept_all_detail_info():
-    query = {"symbol": {'$exists': True}}
-    new_concept_list = mongodb_util.find_query_data("ths_concept_list", query)
+    new_concept_list = ths_concept_common_service_api.get_all_ths_concept()
     new_concept_list = new_concept_list.sort_values(by=['symbol'], ascending=False)
-    if new_concept_list.shape[0] > 0:
-        for one_concept in new_concept_list.itertuples():
-            now_date_time = datetime.datetime.now()
-            str_day = now_date_time.strftime('%Y-%m-%d')
-            str_now_time = now_date_time.strftime('%Y-%m-%d %H:%M:%S')
-            new_concept_symbol_df = ths_concept_sync_common_api.get_concept_detail_info_web(one_concept.symbol)
 
-            if new_concept_symbol_df is None or new_concept_symbol_df.shape[0] == 0:
-                time.sleep(10)
-                continue
-            time.sleep(1)
-            new_concept_symbol_df.loc[:, 'way'] = 'index_sync'
-            ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_df, one_concept.name,
-                                                                str_day, str_now_time, one_concept.symbol)
+    for one_concept in new_concept_list.itertuples():
+        try:
+            ths_concept_detail_api.sync_ths_concept_detail_to_db(one_concept.symbol, one_concept.name)
+        except BaseException as e:
+            logger.error("同步概念下所有股票组成异常:{},{}，｛｝", one_concept.symbol, one_concept.name, e)
 
 
 if __name__ == '__main__':

mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py

@@ -1,6 +1,5 @@
 import os
 import sys
-import pandas as pd
 import mns_common.api.ths.concept.web.ths_concept_detail_web as ths_concept_detail_web
 import mns_common.api.em.east_money_stock_api as east_money_stock_api
 import mns_common.component.common_service_fun_api as common_service_fun_api
@@ -13,6 +12,8 @@ import mns_scheduler.concept.ths.common.ths_concept_sync_common_api as ths_conce
 import threading
 import mns_scheduler.concept.ths.common.ths_concept_update_common_api as ths_concept_update_common_api
 import mns_common.utils.date_handle_util as date_handle_util
+import mns_common.constant.db_name_constant as db_name_constant
+import mns_scheduler.common.redis.redis_msg_push_service as redis_msg_push_service
 
 file_path = os.path.abspath(__file__)
 end = file_path.index('mns') + 17
@@ -21,16 +22,16 @@ sys.path.append(project_path)
 mongodb_util = MongodbUtil('27017')
 # 分页大小
 MAX_PAGE_NUMBER = 2500
-# 获取单只股票新增概念
 
-db_name = 'ths_stock_concept_detail_app'
+
+# 获取单只股票新增概念
 
 
 def create_index():
-    mongodb_util.create_index(db_name, [("symbol", 1)])
-    mongodb_util.create_index(db_name, [("str_day", 1)])
-    mongodb_util.create_index(db_name, [("title", 1)])
-    mongodb_util.create_index(db_name, [("str_now_date", 1)])
+    mongodb_util.create_index(db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP, [("symbol", 1)])
+    mongodb_util.create_index(db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP, [("str_day", 1)])
+    mongodb_util.create_index(db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP, [("title", 1)])
+    mongodb_util.create_index(db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP, [("str_now_date", 1)])
 
 
 # 同步新概念到详情表中
@@ -52,6 +53,8 @@ def sync_new_concept_to_ths_detail(symbol_add_new_concept_df, str_day, str_now_t
 
         except BaseException as e:
             logger.error("转换同花顺概念异常:{},{}", new_concept_one, e)
+    # 项目之前推送消息
+    redis_msg_push_service.send_redis_msg("新增同花顺概念")
 
 
 # 保存数据到对比
@@ -60,7 +63,7 @@ def save_data_to_db(ths_concept_df):
     if data_frame_util.is_empty(ths_concept_df):
         return
     json_data = ths_concept_df.to_dict(orient='records')
-    mongodb_util.save_mongo_json(json_data, db_name)
+    mongodb_util.save_mongo_json(json_data, db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP)
 
 
 # 对比数据库和接口概念详情的差值

mns_scheduler/risk/self/wei_pan_stock_api.py

@@ -6,7 +6,7 @@ end = file_path.index('mns') + 16
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
-import mns_scheduler.concept.common.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
 import mns_common.component.self_choose.black_list_service_api as black_list_service_api
 from datetime import datetime
 from loguru import logger

mns_scheduler/zz_task/data_sync_task.py

@@ -437,7 +437,7 @@ blockingScheduler.add_job(update_best_choose_plate_relation, 'cron', hour='09,18
 blockingScheduler.add_job(sync_kpl_best_his_quotes, 'cron', hour='18,22', minute='25')
 
 # 更新开盘啦指数关系
-blockingScheduler.add_job(sync_position, 'cron', hour='0,08', minute='10')
+blockingScheduler.add_job(sync_position, 'cron', hour='0,09', minute='10')
 
 # 同步高风险股票
 blockingScheduler.add_job(sync_high_risk_stocks, 'cron', hour='0,09,12,16', minute='20')

{mns_scheduler-1.1.3.6.dist-info → mns_scheduler-1.1.3.9.dist-info}/METADATA

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.1.3.6
+Version: 1.1.3.9
 

{mns_scheduler-1.1.3.6.dist-info → mns_scheduler-1.1.3.9.dist-info}/RECORD

@@ -1,23 +1,14 @@
 mns_scheduler/__init__.py,sha256=Tyvi_iQlv3jz59EdH67Mycnt9CSixcWPQoJwu55bOq0,165
-mns_scheduler/backup/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_scheduler/backup/data/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_scheduler/backup/data/collection_move.py,sha256=WZN6-312jBbWrNqXCSk-0yTQ60ajUUWAqr2exVfotKU,2005
-mns_scheduler/backup/data/sync_data_to_local.py,sha256=o_kEcBoG8-CjLS7f_cXUBMo_z5MHn2z1WYnaobNDW90,1737
-mns_scheduler/backup/data/sync_remote_by_str_day.py,sha256=uEi-8j5gqRvWoJwIJGAr1Dc7UCmpXAh7q1IMS-Z88Fw,1220
-mns_scheduler/backup/em/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/backup/em/em_new_concept_his_sync.py,sha256=Yqbuz9VE9433o0Z6BvDo5adhj3lK4vmN66XNQBIdC1k,4992
-mns_scheduler/backup/em/em_new_concept_sync_common_api.py,sha256=KnNCf0KXy31GFwF1-wbbkfDM-H7-cEtcVoCwl4QmoVs,7083
-mns_scheduler/backup/em/em_new_concept_sync_web.py,sha256=wVS0yLyPYzpcxfWAuUqmqbYkjw9wjpo48CvLSZFckzw,2334
-mns_scheduler/backup/ths/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mns_scheduler/backup/ths/sync_ths_concept_by_ak_api.py,sha256=8QBiRKc0Q3ez-kBwvz7G8HrFDD_ANd8smOpQtGyvKIs,5371
-mns_scheduler/backup/ths/sync_ths_new_concept_by_web_api.py,sha256=dLd2V67AFnE9xf5H8YGyaJ0j7z8FhtTemFVGOuOeHJ4,4087
 mns_scheduler/big_deal/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/big_deal/ths_big_deal_sync.py,sha256=aMFj-_pLprh4vGjSSzmr_tlYoPA0L4Lm0SkLRkQwIiw,4564
+mns_scheduler/common/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
+mns_scheduler/common/redis/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
+mns_scheduler/common/redis/redis_msg_push_service.py,sha256=LypT8Sl91xGcO1bF9o3utThk_SUtotuD-UEfgFjCww8,476
 mns_scheduler/company_info/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/company_info/base/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/company_info/base/sync_company_base_info_api.py,sha256=NRuwc5oJF9Ngt932OrnG76oP22OmkjbGgcWKgWplafg,19106
 mns_scheduler/company_info/clean/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_scheduler/company_info/clean/company_info_clean_api.py,sha256=Pn2cRt3oRH-VRkHMZdsp-nJTFWrD5VSVRoH_uI_8-j0,4861
+mns_scheduler/company_info/clean/company_info_clean_api.py,sha256=n3kxsqihPjKJvGDkAjMYqB_U2EPemOi1Yjez5yOYJeo,4858
 mns_scheduler/company_info/constant/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/company_info/constant/company_constant_data.py,sha256=XIM81fA4PNalVJNBYXiMddkcW012fFtofoPGYy2VtD0,16001
 mns_scheduler/company_info/de_list_stock/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
@@ -27,19 +18,18 @@ mns_scheduler/company_info/remark/company_remark_info_sync.py,sha256=hzQ8uBK4-od
 mns_scheduler/concept/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/concept/clean/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/concept/clean/kpl_concept_clean_api.py,sha256=xxIIgrXLI6xLf10t4unJa7aMB_QeKeki1HJVeXsntkY,4512
-mns_scheduler/concept/clean/ths_concept_clean_api.py,sha256=42sbd8VqJeOKxJuxLVsU9YFK_YUSePTGrK2AWbabhA0,5964
-mns_scheduler/concept/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mns_scheduler/concept/common/detaill/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mns_scheduler/concept/common/detaill/ths_concept_detail_api.py,sha256=Au_l7SVcp4kRaR2vo1FP3BmYHeRI9fkOk_sTwzpRYw4,9435
+mns_scheduler/concept/clean/ths_concept_clean_api.py,sha256=wxdT5Eb3awHW9HuFlgSe8-C6sC6TNaDYozr8VvNdYe0,5953
 mns_scheduler/concept/ths/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/concept/ths/common/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py,sha256=CDo5d2dA36a99bS4vxg2ZRdSc-l_NwPSzmkxjAWLyFY,8146
+mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py,sha256=xreSNZDdooI0do9RQLTKqrQ85JgbNRhQzaNCI9SgKns,8328
 mns_scheduler/concept/ths/common/ths_concept_update_common_api.py,sha256=4BQT3A9t-nDIyCpILgRZF7ZOgK1oabp-gJl5nyWvKWc,4418
+mns_scheduler/concept/ths/detaill/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mns_scheduler/concept/ths/detaill/ths_concept_detail_api.py,sha256=OdtLnF4St15-4vYck_5qj4k5KMSTFbAaAbtWP5kvqLk,10694
 mns_scheduler/concept/ths/sync_new_index/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_new_index_api.py,sha256=9ol2cvxSO2Ot4Z5fmlo9tUPFe6WwsuVX_H9ogtv3QrI,7841
+mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_new_index_api.py,sha256=4vzKeQ2WfDOZlADgPSWsFHcqAIQAoRtUpWmsNCrJRnw,7533
 mns_scheduler/concept/ths/update_concept_info/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/concept/ths/update_concept_info/sync_one_concept_all_symbols_api.py,sha256=wlsr70rcS-uAH6b8NXPbA-wG37saeIQ0FIg8vSSTXQA,1920
-mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py,sha256=HVqDSBruORebtiliKps2swJo_GNg_8mqT0r9cZX6sj8,8839
+mns_scheduler/concept/ths/update_concept_info/sync_one_concept_all_symbols_api.py,sha256=wwuLfjj9AnFcHP-oQPC5AhpwgZ8IsPiNUh-Z6swcngA,1380
+mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py,sha256=83I6UDL9IDUrIozXyuLcFjilQFEk3tAWWxwn1SBcMgk,9217
 mns_scheduler/db/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/db/col_move_service.py,sha256=-d3JUCjWyaGKBhyUSkqwpCgZtHkIy1JOasgpxmH74zg,4057
 mns_scheduler/db/db_status.py,sha256=e5eW5ZSm5J7tHvmxxhFmFdbZb2_oB_SAcdcFqc4KDmw,733
@@ -102,7 +92,7 @@ mns_scheduler/risk/financial/stock_equity_mortgage_check_api.py,sha256=SQ7dieSCO
 mns_scheduler/risk/major_violations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mns_scheduler/risk/major_violations/register_and_investigate_stock_sync_api.py,sha256=_xPQxT6OS7J7_PtA7hbfCmMS7VzpZTWoZUOXzwoesUs,5672
 mns_scheduler/risk/self/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mns_scheduler/risk/self/wei_pan_stock_api.py,sha256=osU3XrPZXDzPdtVCq601ZlSTZUqIhGxoXsowWgPAM4w,1902
+mns_scheduler/risk/self/wei_pan_stock_api.py,sha256=5ebrjFqK49j_yEMffFBCumMeBzq_rmtsA4--lL5mhZs,1899
 mns_scheduler/risk/test/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/risk/test/fix_blask_list.py,sha256=jjLhhIWH1L4CgNp2jQ_HAHbC3wVgeX1TzhqC--_D2MI,1249
 mns_scheduler/risk/transactions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -137,8 +127,8 @@ mns_scheduler/zt/zt_pool/__init__.py,sha256=Tyvi_iQlv3jz59EdH67Mycnt9CSixcWPQoJw
 mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py,sha256=x3-NCugHoY5IuX2s1UOUMl8DZsi9nrtMbEGzxWqJj3c,7642
 mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py,sha256=LJVGqJyLn04oC-Xp59RVE1IOz7lx6ao78bw9l67sKBM,10382
 mns_scheduler/zz_task/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/zz_task/data_sync_task.py,sha256=EexJz4F5WtyZ5N7qG3U6IzHHQwnvN-EqQEUxcTyRRyk,19069
-mns_scheduler-1.1.3.6.dist-info/METADATA,sha256=opxAG3nIDFb9cCSSwfu74O337LqTVNM3majHT4m9AP8,64
-mns_scheduler-1.1.3.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-mns_scheduler-1.1.3.6.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
-mns_scheduler-1.1.3.6.dist-info/RECORD,,
+mns_scheduler/zz_task/data_sync_task.py,sha256=KI7I5R4RFZ67KSzXJiObnIrsKecvvvMQpCFNMX1qHJQ,19069
+mns_scheduler-1.1.3.9.dist-info/METADATA,sha256=B7k2ga2WhdPCcL4LZ8i8OMtzSTdNhLdpyl0RsdXC9oc,64
+mns_scheduler-1.1.3.9.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+mns_scheduler-1.1.3.9.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
+mns_scheduler-1.1.3.9.dist-info/RECORD,,

mns_scheduler/backup/data/collection_move.py

@@ -1,53 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-from loguru import logger
-
-db_name_list = [
-    # 'company_info', 'company_info_base', 'company_info_his', 'company_info_hk', 'de_list_stock',
-    # 'em_stock_asset_liability',
-    # 'em_stock_profit',
-    # 'k_line_info',
-    'kpl_best_choose_daily', 'kpl_best_choose_his', 'kpl_best_choose_index',
-    'kpl_best_choose_index_detail',
-    'kpl_his_quotes', 'recent_hot_stocks', 'self_black_stock', 'self_choose_plate', 'self_choose_stock', 'sse_info_uid',
-    'stock_gdfx_free_top_10', 'stock_high_chg_pool', 'stock_interactive_question', 'stock_qfq_daily',
-    'stock_qfq_monthly', 'stock_qfq_weekly', 'stock_zt_pool', 'stock_zt_pool_five', 'stock_zb_pool', 'sw_industry',
-    'ths_concept_list', 'ths_stock_concept_detail', 'ths_stock_concept_detail_app', 'ths_zt_pool',
-    'today_new_concept_list',
-    'trade_date_list', 'realtime_quotes_now_zt_new', 'realtime_quotes_now_open', 'realtime_quotes_now_zt_new_kc_open']
-
-
-def db_export(db, col):
-    cmd = 'F:/mongo/bin/mongodump.exe --host ' + db + ' -d patience -c ' + col + ' -o D:/back'
-    os.system(cmd)
-    logger.info("export finished:{}", col)
-
-
-def db_import(db, col):
-    cmd = 'F:/mongo/bin/mongorestore.exe --host ' + db + ' -d patience -c ' + col + ' D:/back/patience/' + col + '.bson'
-    os.system(cmd)
-
-    path = 'D:\\back\\patience\\' + col + '.bson'
-    cmd_del = 'del /F /S /Q ' + path
-    os.system(cmd_del)
-
-    logger.info("import finished:{}", col)
-
-
-def collection_to_local():
-    for db_name in db_name_list:
-        try:
-            db_export('127.0.0.1:27017', db_name)
-            db_import('192.168.1.6:27017', db_name)
-            logger.info("同步到集合:{}", db_name)
-        except BaseException as e:
-            logger.error("出现异常:{}", e)
-
-
-if __name__ == '__main__':
-    collection_to_local()

mns_scheduler/backup/data/sync_data_to_local.py

@@ -1,55 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-from mns_common.db.MongodbUtil import MongodbUtil
-from loguru import logger
-
-mongodb_util = MongodbUtil('27017')
-
-from mns_common.db.MongodbUtilLocal import MongodbUtil as MongodbUtilLocal
-
-mongodbUtilLocal = MongodbUtilLocal('27017')
-
-db_name_list = [
-    # 'company_info', 'company_info_base', 'company_info_his', 'company_info_hk', 'de_list_stock',
-    # 'em_stock_asset_liability',
-    # 'em_stock_profit',
-    # 'k_line_info',
-    # 'kpl_best_choose_daily',
-    # 'kpl_best_choose_his',
-    # 'kpl_best_choose_index',
-    # 'kpl_best_choose_index_detail',
-    # 'kpl_his_quotes',
-    # 'stock_qfq_daily',
-    # 'stock_qfq_monthly',
-    # 'stock_qfq_weekly',
-    # 'stock_gdfx_free_top_10',
-    # 'recent_hot_stocks',
-    # 'self_black_stock',
-    # 'self_choose_plate', 'self_choose_stock', 'sse_info_uid',
-    # 'stock_high_chg_pool', 'stock_interactive_question',
-    # 'stock_zt_pool', 'stock_zt_pool_five', 'stock_zb_pool', 'sw_industry',
-    # 'ths_concept_list', 'ths_stock_concept_detail','ths_zt_pool',
-    # 'today_new_concept_list',
-    # 'trade_date_list',
-    # 'ths_stock_concept_detail_app',
-    "stock_zt_pool"
-]
-
-
-def sync_collection_to_local():
-    for db_name in db_name_list:
-        try:
-            df_data = mongodb_util.find_all_data(db_name)
-            mongodbUtilLocal.save_mongo(df_data, db_name)
-            logger.info("同步到集合:{}", db_name)
-        except BaseException as e:
-            logger.error("出现异常:{}", e)
-
-
-if __name__ == '__main__':
-    sync_collection_to_local()

mns_scheduler/backup/data/sync_remote_by_str_day.py

@@ -1,33 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-from mns_common.db.MongodbUtil import MongodbUtil
-from loguru import logger
-
-mongodb_util = MongodbUtil('27017')
-from mns_common.db.MongodbUtilLocal import MongodbUtil as MongodbUtilLocal
-
-mongodbUtilLocal = MongodbUtilLocal('27017')
-
-
-def sync_remote_open_data(str_day):
-    query_remote = {"str_day": str_day}
-    realtime_quotes_now_open_df = mongodb_util.find_query_data('realtime_quotes_now_open', query_remote)
-    mongodbUtilLocal.insert_mongo(realtime_quotes_now_open_df, 'realtime_quotes_now_open')
-
-
-if __name__ == '__main__':
-    query = {'$and': [{"_id": {"$gte": "2022-03-16"}},
-                      {"_id": {"$lte": "2024-06-28"}}]}
-    trade_date_list = mongodb_util.find_query_data('trade_date_list', query)
-    for trade_one in trade_date_list.itertuples():
-        try:
-            sync_str_day = trade_one.trade_date
-            sync_remote_open_data(sync_str_day)
-            logger.info('str_day:{}', sync_str_day)
-        except BaseException as e:
-            logger.error("同步远程数据异常:{}", e)

mns_scheduler/backup/em/__init__.py

@@ -1,7 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 17
-project_path = file_path[0:end]
-sys.path.append(project_path)

mns_scheduler/backup/em/em_new_concept_his_sync.py

@@ -1,99 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 17
-project_path = file_path[0:end]
-sys.path.append(project_path)
-
-import akshare as ak
-from loguru import logger
-from datetime import datetime
-import mns_common.utils.date_handle_util as date_handle_util
-import mns_common.db.MongodbUtil as MongodbUtil
-import mns_scheduler.backup.em.em_new_concept_sync_common_api as em_new_concept_sync_common_api
-import mns_common.api.em.em_concept_index_api as em_concept_index_api
-
-mongodb_util = MongodbUtil('27017')
-
-
-# 同步概念k线
-def sync_concept_k_line(symbol, period, start_date, end_date, adjust):
-    try:
-        stock_board_concept_hist_em_df = ak.stock_board_concept_hist_em(
-            symbol=symbol, period=period, start_date=start_date, end_date=end_date, adjust=adjust
-        )
-        stock_board_concept_hist_em_df = stock_board_concept_hist_em_df.rename(columns={"日期": "date",
-                                                                                        "开盘": "open",
-                                                                                        "收盘": "close",
-                                                                                        "最高": "high",
-                                                                                        "最低": "low",
-                                                                                        "涨跌幅": "chg",
-                                                                                        "涨跌额": "change",
-                                                                                        "成交量": "volume",
-                                                                                        "成交额": "amount",
-                                                                                        "振幅": "pct_chg",
-                                                                                        "换手率": "exchange"
-                                                                                        })
-
-        return stock_board_concept_hist_em_df
-    except BaseException as e:
-        logger.error("同步东方财富概念信息异常:{}", e)
-        return None
-
-
-# 同步所有历史概念数据
-def sync_his_em_concept():
-    now_date = datetime.now()
-    str_day = now_date.strftime('%Y-%m-%d')
-
-    all_concept_list = em_concept_index_api.sync_all_concept()
-    for stock_one in all_concept_list.itertuples():
-        try:
-            stock_one_df_copy = all_concept_list.loc[all_concept_list['concept_code'] == stock_one.concept_code]
-            stock_one_df = stock_one_df_copy.copy()
-
-            stock_board_concept_hist_em_df = sync_concept_k_line(
-                symbol=stock_one.concept_name, period="daily", start_date="19980729",
-                end_date=date_handle_util.no_slash_date(str_day), adjust="qfq"
-            )
-            stock_board_concept_hist_em_df = stock_board_concept_hist_em_df.sort_values(by=['date'], ascending=True)
-            stock_board_concept_hist_em_df_first_day = stock_board_concept_hist_em_df.iloc[0:1]
-
-            list_date = list(stock_board_concept_hist_em_df_first_day['date'])[0]
-            str_now_time = list_date + " " + "09:10:00"
-            stock_one_df.loc[:, 'str_now_time'] = str_now_time
-
-            stock_one_df.loc[:, 'str_day'] = list_date
-
-            stock_one_df['_id'] = stock_one_df['concept_code']
-            stock_one_df['url'] = "http://quote.eastmoney.com/center/boardlist.html#boards-" + stock_one.concept_code
-            stock_one_df['success'] = True
-
-            mongodb_util.save_mongo(stock_one_df, 'em_concept_list')
-            #
-            concept_detail_list = em_new_concept_sync_common_api.sync_concept_detail(stock_one.concept_code)
-
-            concept_detail_list.loc[:, 'concept_code'] = stock_one.concept_code
-            concept_detail_list.loc[:, 'concept_name'] = stock_one.concept_name
-            concept_detail_list.loc[:, 'str_day'] = list_date
-
-            concept_detail_list.loc[:, 'str_now_time'] = str_now_time
-
-            concept_detail_list['_id'] = concept_detail_list['symbol'] + '-' + concept_detail_list['concept_code']
-
-            mongodb_util.insert_mongo(concept_detail_list, 'em_stock_concept_detail')
-        except BaseException as e:
-            logger.error("发生异常:{}", e)
-            continue
-
-
-if __name__ == '__main__':
-    # all_concept = sync_all_concept()
-    # all_concept = all_concept.loc[~(all_concept['list_day'] == '-')]
-    # all_concept = all_concept.sort_values(by=['list_day'], ascending=False)
-    sync_his_em_concept()
-    # stock_board_concept_hist_em_df = sync_concept_k_line(
-    #     symbol="车联网", period="daily", start_date="20220101", end_date="20221128", adjust=""
-    # )
-    # sync_concept_detail('MLOps概念')

mns_scheduler/backup/em/em_new_concept_sync_common_api.py

@@ -1,139 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 17
-project_path = file_path[0:end]
-sys.path.append(project_path)
-
-from mns_common.db.MongodbUtil import MongodbUtil
-import mns_common.api.msg.push_msg_api as push_msg_api
-import mns_common.api.em.em_concept_index_api as em_concept_index_api
-
-
-import pandas as pd
-import mns_common.utils.data_frame_util as data_frame_util
-from loguru import logger
-mongodb_util = MongodbUtil('27017')
-
-max_concept_code = 1200
-
-
-# 获取最大概念代码
-def get_max_concept_code():
-    query = {"concept_code": {'$ne': 'null'}, "success": True}
-    ths_concept_max = mongodb_util.descend_query(query, 'em_concept_list', 'concept_code', 1)
-    if ths_concept_max.shape[0] == 0:
-        concept_code = 'BK1134'
-    else:
-        concept_code = list(ths_concept_max['concept_code'])[0]
-    concept_code = concept_code.replace('BK', '')
-    return concept_code
-
-
-# 获取概念名称
-def get_concept_name(concept_code):
-    concept_df = em_concept_index_api.sync_all_concept()
-    concept_df_one = concept_df.loc[concept_df['concept_code'] == concept_code]
-    if data_frame_util.is_empty(concept_df_one):
-        return ""
-    else:
-        return concept_df_one
-
-
-# 获取概念详情
-def sync_concept_detail(concept_code):
-    try:
-        stock_board_concept_cons_em_df = em_concept_index_api.stock_board_concept_cons_em(concept_code)
-        if data_frame_util.is_empty(stock_board_concept_cons_em_df):
-            return None
-        stock_board_concept_cons_em_df = stock_board_concept_cons_em_df.rename(columns={"序号": "index",
-                                                                                        "代码": "symbol",
-                                                                                        "名称": "name",
-                                                                                        "最新价": "now_price",
-                                                                                        "涨跌幅": "chg",
-                                                                                        "涨跌额": "change",
-                                                                                        "成交量": "volume",
-                                                                                        "成交额": "amount",
-                                                                                        "振幅": "pct_chg",
-                                                                                        "最高": "high",
-                                                                                        "最低": "low",
-                                                                                        "今开": "open",
-                                                                                        "昨收": "last_price",
-                                                                                        "换手率": "exchange",
-                                                                                        "市盈率-动态": 'pe',
-                                                                                        "市净率": 'pb'
-                                                                                        })
-        return stock_board_concept_cons_em_df
-    except BaseException as e:
-        logger.error("同步东方财富概念信息异常:{}", e)
-        return None
-
-
-# web端获取新概念消息推送
-def push_msg_to_we_chat_web(concept_code, concept_name):
-    url = 'http://quote.eastmoney.com/center/boardlist.html#boards-' + str(concept_code)
-    msg = "概念代码:" + str(concept_code) + "," + "概念名称:" + concept_name + "," + "url:   " + url
-    title = "新增东财概念:" + str(concept_code) + "-" + concept_name
-    push_msg_api.push_msg_to_wechat(title, msg)
-
-
-# 保存新概念信息到数据库
-def save_em_concept_list(concept_code, concept_name, str_day, str_now_time, concept_df_one):
-    url = 'https://quote.eastmoney.com/center/boardlist.html#boards-' + str(concept_code)
-    if data_frame_util.is_empty(concept_df_one):
-        em_concept_list = pd.DataFrame([
-            [concept_code, concept_code, concept_name, str_day, url, str_now_time, True],
-        ], columns=['_id', 'concept_code', 'concept_name', 'str_day', 'url', 'str_now_time', 'success'])
-
-        mongodb_util.save_mongo(em_concept_list, 'em_concept_list')
-    else:
-        concept_df_one['_id'] = concept_df_one['concept_code']
-        concept_df_one.loc[:, 'str_now_time'] = str_now_time
-        concept_df_one.loc[:, 'str_day'] = str_day
-        concept_df_one.loc[:, 'url'] = url
-        concept_df_one.loc[:, 'success'] = True
-        mongodb_util.save_mongo(concept_df_one, 'em_concept_list')
-
-
-# 保存新概念详细信息到数据库
-def save_ths_concept_detail(new_concept_symbol_df, concept_name, str_day, str_now_time, concept_code):
-    new_concept_symbol_df['_id'] = new_concept_symbol_df['symbol'] + '-' + concept_code
-
-    new_concept_symbol_df['concept_code'] = concept_code
-    new_concept_symbol_df['concept_name'] = concept_name
-    new_concept_symbol_df['str_day'] = str_day
-    new_concept_symbol_df['str_now_time'] = str_now_time
-
-    new_concept_symbol_list = list(new_concept_symbol_df['symbol'])
-
-    query_company_info = {'symbol': {'$in': new_concept_symbol_list}}
-    query_field = {"first_industry": 1, "first_industry": 1, "industry": 1, "company_type": 1, "flow_mv_sp": 1,
-                   "total_mv_sp": 1}
-    company_info = mongodb_util.find_query_data_choose_field('company_info', query_company_info, query_field)
-
-    company_info = company_info.set_index(['_id'], drop=True)
-    new_concept_symbol_df = new_concept_symbol_df.set_index(['symbol'], drop=False)
-
-    new_concept_symbol_df = pd.merge(new_concept_symbol_df, company_info, how='outer',
-                                     left_index=True, right_index=True)
-    query = {'concept_code': concept_code}
-    exist_concept_detail = mongodb_util.find_query_data('em_stock_concept_detail', query)
-    if data_frame_util.is_empty(exist_concept_detail):
-        mongodb_util.save_mongo(new_concept_symbol_df, 'em_stock_concept_detail')
-    else:
-        exist_concept_detail_symbol_list = list(exist_concept_detail['symbol'])
-        new_concept_symbol_df = new_concept_symbol_df.loc[~(
-            new_concept_symbol_df['symbol'].isin(exist_concept_detail_symbol_list))]
-        if new_concept_symbol_df.shape[0] > 0:
-            mongodb_util.save_mongo(new_concept_symbol_df, 'em_stock_concept_detail')
-    # update_company_info(new_concept_symbol_df, concept_code, concept_name, str_day)
-    # # 公司缓存信息清除
-    # common_service_fun_api.company_info_industry_cache_clear()
-
-
-if __name__ == '__main__':
-    get_concept_name('BK1134')
-
-    sync_concept_detail('BK1055')
-    get_max_concept_code()

mns_scheduler/backup/em/em_new_concept_sync_web.py

@@ -1,55 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 17
-project_path = file_path[0:end]
-sys.path.append(project_path)
-
-import datetime
-import time
-from loguru import logger
-import mns_scheduler.backup.em.em_new_concept_sync_common_api as em_new_concept_sync_common_api
-
-
-# 同步东财新概念
-def sync_new_concept_data():
-    concept_code = em_new_concept_sync_common_api.get_max_concept_code()
-    # 装换为数字
-    concept_code = int(concept_code)
-
-    concept_code = concept_code + 1
-
-    while concept_code < em_new_concept_sync_common_api.max_concept_code:
-        try:
-            now_date = datetime.datetime.now()
-            str_day = now_date.strftime('%Y-%m-%d')
-            str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
-            str_concept_code = 'BK' + str(concept_code)
-            new_concept_symbol_df = em_new_concept_sync_common_api.sync_concept_detail(str_concept_code)
-            if new_concept_symbol_df is None or new_concept_symbol_df.shape[0] == 0:
-                concept_code = concept_code + 1
-                time.sleep(1)
-                continue
-            time.sleep(2)
-            concept_df_one = em_new_concept_sync_common_api.get_concept_name(str_concept_code)
-            # 获取概念名称
-            concept_name = list(concept_df_one['concept_name'])[0]
-
-            # 推送新概念信息到微信
-            em_new_concept_sync_common_api.push_msg_to_we_chat_web(str_concept_code, concept_name)
-            # 保存新概念信息到数据库
-            em_new_concept_sync_common_api.save_em_concept_list(str_concept_code, concept_name, str_day, str_now_time,
-                                                                concept_df_one)
-            # 保存新概念详细信息到数据库
-            em_new_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_df, concept_name, str_day,
-                                                                   str_now_time, str_concept_code)
-
-            concept_code = concept_code + 1
-        except BaseException as e:
-            logger.error("同步新概念异常:{},concept_code:{}", e, concept_code)
-            concept_code = concept_code + 1
-
-
-if __name__ == '__main__':
-    sync_new_concept_data()

mns_scheduler/backup/ths/sync_ths_concept_by_ak_api.py

@@ -1,103 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 17
-project_path = file_path[0:end]
-sys.path.append(project_path)
-from loguru import logger
-
-from mns_common.db.MongodbUtil import MongodbUtil
-import mns_common.utils.data_frame_util as data_frame_util
-import pandas as pd
-from datetime import datetime
-import akshare as ak
-import mns_scheduler.concept.ths.common.ths_concept_sync_common_api as ths_concept_sync_common_api
-
-import mns_scheduler.concept.clean.ths_concept_clean_api as ths_concept_clean_api
-import mns_scheduler.concept.common.detaill.ths_concept_detail_api as ths_concept_detail_api
-mongodb_util = MongodbUtil('27017')
-
-
-# 与概念接口对比 找出缺失数据和名称不一样的数据
-
-def sync_new_ths_concept_by_ak_api():
-    stock_board_concept_name_ths_df = ak.stock_board_concept_name_ths()
-    stock_board_concept_name_ths_df = stock_board_concept_name_ths_df.rename(columns={"日期": "str_day",
-                                                                                      "概念名称": "concept_name",
-                                                                                      "成分股数量": "numbers",
-                                                                                      "网址": "url",
-                                                                                      "代码": "concept_code",
-                                                                                      })
-
-    stock_board_concept_name_ths_df['str_day'].fillna(datetime(1970, 1, 1), inplace=True)
-    stock_board_concept_name_ths_df.concept_name = stock_board_concept_name_ths_df.concept_name.str.replace('（', '(')
-    stock_board_concept_name_ths_df.concept_name = stock_board_concept_name_ths_df.concept_name.str.replace('）', ')')
-
-    stock_board_concept_name_ths_df['str_day'] = stock_board_concept_name_ths_df['str_day'].apply(
-        lambda x: datetime.strftime(x, '%Y-%m-%d'))
-
-    # 同花顺概念列表
-    ths_concept_list_exist = mongodb_util.find_all_data('ths_concept_list')
-
-    for concept_one in stock_board_concept_name_ths_df.itertuples():
-        try:
-            exist_concept_df_one = ths_concept_list_exist.loc[
-                (ths_concept_list_exist['name'] == concept_one.concept_name)
-                | (ths_concept_list_exist['web_concept_code'] == concept_one.concept_code)]
-            now_date = datetime.now()
-            str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
-            str_day = concept_one.str_day
-            if data_frame_util.is_empty(exist_concept_df_one):
-                concept_code = concept_one.concept_code
-
-                diff_one = {
-                    '_id': int(concept_code),
-                    'symbol': int(concept_code),
-                    'name': concept_one.concept_name,
-                    'url': concept_one.url,
-                    'str_day': str_day,
-                    'success': True,
-                    'str_now_time': str_now_time,
-                    'web_concept_code': int(concept_one.concept_code),
-                    'web_concept_url': concept_one.url,
-                    'valid': True
-                }
-                diff_one_df = pd.DataFrame(diff_one, index=[1])
-                mongodb_util.save_mongo(diff_one_df, 'ths_concept_list')
-                url = 'http://q.10jqka.com.cn/gn/detail/code/' + str(concept_one.concept_code)
-                # 新增概念信息处理
-                handle_new_concept_msg(concept_code, concept_one.concept_name, url)
-                new_concept_symbol_df = get_concept_detail_info_web(concept_code)
-                if new_concept_symbol_df is None or new_concept_symbol_df.shape[0] == 0:
-                    return None
-                new_concept_symbol_df.loc[:, 'way'] = 'index_sync'
-                ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_df, concept_one.concept_name,
-                                                                    str_day,
-                                                                    str_now_time, concept_code)
-                logger.info("新增同花顺新概念:{}", concept_one.concept_name)
-        except BaseException as e:
-            logger.error("同步概念:{},信息异常:{}", concept_one.concept_name, e)
-
-
-def handle_new_concept_msg(concept_code, concept_name, url):
-    # 推送新概念信息到微信
-    ths_concept_sync_common_api.push_msg_to_we_chat_web(concept_code, concept_name,
-                                                        url)
-    # 更新ths概念信息
-    ths_concept_clean_api.update_ths_concept_info()
-
-
-# 获取web端 新增概念详情
-def get_concept_detail_info_web(concept_code):
-    new_concept_symbol_list = ths_concept_detail_api.get_ths_concept_detail(concept_code, None)
-    if new_concept_symbol_list is None or new_concept_symbol_list.shape[0] == 0:
-        return None
-    new_concept_symbol_list = new_concept_symbol_list[ths_concept_sync_common_api.order_fields]
-    new_concept_symbol_list['_id'] = str(concept_code) + '-' + new_concept_symbol_list['symbol']
-    return new_concept_symbol_list
-
-
-if __name__ == '__main__':
-    # get_concept_detail_info_web(886068)
-    sync_new_ths_concept_by_ak_api()

mns_scheduler/backup/ths/sync_ths_new_concept_by_web_api.py

@@ -1,88 +0,0 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 17
-project_path = file_path[0:end]
-sys.path.append(project_path)
-import mns_scheduler.concept.common.detaill.ths_concept_detail_api as ths_concept_detail_api
-import mns_common.api.ths.concept.web.ths_concept_index_web as ths_concept_index_web
-import datetime
-import time
-import mns_common.api.msg.push_msg_api as push_msg_api
-from loguru import logger
-import mns_scheduler.concept.ths.common.ths_concept_sync_common_api as ths_concept_sync_common_api
-import mns_scheduler.concept.clean.ths_concept_clean_api as ths_concept_clean_api
-from mns_common.db.MongodbUtil import MongodbUtil
-
-mongodb_util = MongodbUtil('27017')
-
-
-# 获取web端 新增概念详情
-def get_concept_detail_info_web(concept_code):
-    new_concept_symbol_list = ths_concept_detail_api.get_ths_concept_detail(concept_code, None)
-    if new_concept_symbol_list is None or new_concept_symbol_list.shape[0] == 0:
-        return None
-    new_concept_symbol_list = new_concept_symbol_list[ths_concept_sync_common_api.order_fields]
-    new_concept_symbol_list['_id'] = str(concept_code) + '-' + new_concept_symbol_list['symbol']
-    return new_concept_symbol_list
-
-
-# web端获取新概念消息推送
-def push_msg_to_we_chat_web(concept_code, concept_name):
-    url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
-    msg = "概念代码:" + str(concept_code) + "," + "概念名称:" + concept_name + "," + "url:   " + url
-    title = "新增同花顺概念:" + str(concept_code) + "-" + concept_name
-    push_msg_api.push_msg_to_wechat(title, msg)
-
-
-#     # 同步同花顺新增概念指数通过web端接口爬取
-def sync_new_concept_data_by_web():
-    concept_code = ths_concept_sync_common_api.get_max_concept_code()
-    concept_code = concept_code + 1
-
-    while concept_code < ths_concept_sync_common_api.max_concept_code:
-        try:
-            now_date = datetime.datetime.now()
-            str_day = now_date.strftime('%Y-%m-%d')
-            str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
-            new_concept_symbol_df = get_concept_detail_info_web(concept_code)
-            if new_concept_symbol_df is None or new_concept_symbol_df.shape[0] == 0:
-                concept_code = concept_code + 1
-                time.sleep(1)
-                continue
-            time.sleep(2)
-            concept_name = ths_concept_index_web.get_concept_name(concept_code)
-            concept_name = concept_name.replace('（', '(')
-            concept_name = concept_name.replace('）', ')')
-
-            url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
-            # 推送新概念信息到微信
-            handle_new_concept_msg(concept_code, concept_name, url)
-            # 保存新概念信息到概念列表
-            ths_concept_sync_common_api.save_ths_concept_list(concept_code, concept_name, str_day, str_now_time)
-            # 保存新概念详细信息到数据库
-            new_concept_symbol_df.loc[:, 'way'] = 'index_sync'
-            ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_df, concept_name, str_day,
-                                                                str_now_time, concept_code)
-
-            concept_code = concept_code + 1
-        except BaseException as e:
-            logger.error("同步新概念异常:{},concept_code:{}", e, concept_code)
-            concept_code = concept_code + 1
-
-
-def handle_new_concept_msg(concept_code, concept_name, url):
-    # 推送新概念信息到微信
-    ths_concept_sync_common_api.push_msg_to_we_chat_web(concept_code, concept_name,
-                                                        url)
-    # 更新ths概念信息
-    ths_concept_clean_api.update_ths_concept_info()
-
-
-if __name__ == '__main__':
-    # code = 886025
-    # push_msg_to_we_chat_web(code, name)
-    # get_concept_detail_info_web(886026)
-    # get_concept_detail_info_web(886035)
-    sync_new_concept_data_by_web()