mns-scheduler 1.0.8.7__py3-none-any.whl → 1.4.3.2__py3-none-any.whl

mns_scheduler/concept/clean/ths_concept_clean_api.py

@@ -7,11 +7,13 @@ project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil
 import pandas as pd
-import mns_common.api.ths.ths_stock_api as ths_stock_api
+import mns_common.api.ths.concept.web.ths_concept_index_web as ths_concept_index_web
 import time
 from loguru import logger
 import mns_common.component.company.company_common_service_api as company_common_service_api
 import mns_common.constant.db_name_constant as db_name_constant
+import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
+import mns_common.api.ths.concept.app.ths_concept_index_app as ths_concept_index_app
 
 mongodb_util = MongodbUtil('27017')
 
@@ -21,59 +23,78 @@ def update_ths_concept_info():
     ths_concept_list = mongodb_util.find_all_data(db_name_constant.THS_CONCEPT_LIST)
     for ths_concept_one in ths_concept_list.itertuples():
         try:
-            query = {'concept_code': ths_concept_one.symbol}
-            ths_stock_concept_detail_df = (mongodb_util
-                                           .find_query_data(db_name_constant.THS_STOCK_CONCEPT_DETAIL, query))
-            concept_count = ths_stock_concept_detail_df.shape[0]
-            ths_concept_list_one_df = ths_concept_list.loc[ths_concept_list['symbol'] == ths_concept_one.symbol]
-            ths_concept_list_one_df['concept_count'] = concept_count
-
-            ths_stock_concept_detail_df = ths_stock_concept_detail_df.reset_index(drop=True)
-            if 'industry' in ths_stock_concept_detail_df.columns:
-                del ths_stock_concept_detail_df['industry']
-
-            company_info_df = company_common_service_api.get_company_info_industry()
-            company_info_df = company_info_df[['_id', 'industry']]
-            company_info_df = company_info_df.loc[
-                company_info_df['_id'].isin(list(ths_stock_concept_detail_df['symbol']))]
-            company_info_df = company_info_df.set_index(['_id'], drop=True)
-
-            ths_stock_concept_detail_df = ths_stock_concept_detail_df.set_index(['symbol'], drop=False)
-            ths_stock_concept_detail_df = pd.merge(ths_stock_concept_detail_df, company_info_df,
-                                                   how='outer',
-                                                   left_index=True, right_index=True)
-            ths_stock_concept_detail_df.dropna(subset=['industry'], axis=0, inplace=True)
-
-            grouped = ths_stock_concept_detail_df.groupby('industry')
-            result_list = grouped.size()
-            ths_concept_group = pd.DataFrame(result_list, columns=['number'])
-            ths_concept_group['industry'] = ths_concept_group.index
-            ths_concept_group = ths_concept_group.sort_values(by=['number'], ascending=False)
-            if ths_concept_group.shape[0] >= 2:
-                first_relevance_industry = list(ths_concept_group.iloc[0:1]['industry'])[0]
-                first_relevance_industry_number = list(ths_concept_group.iloc[0:1]['number'])[0]
-                second_relevance_industry = list(ths_concept_group.iloc[1:2]['industry'])[0]
-                second_relevance_industry_number = list(ths_concept_group.iloc[1:2]['number'])[0]
-            else:
-                first_relevance_industry = list(ths_concept_group.iloc[0:1]['industry'])[0]
-                first_relevance_industry_number = list(ths_concept_group.iloc[0:1]['number'])[0]
-                second_relevance_industry = '无'
-                second_relevance_industry_number = 0
-            ths_concept_list_one_df['first_relevance_industry'] = first_relevance_industry
-            ths_concept_list_one_df['second_relevance_industry'] = second_relevance_industry
-            ths_concept_list_one_df['first_relevance_industry_number'] = first_relevance_industry_number
-            ths_concept_list_one_df['second_relevance_industry_number'] = second_relevance_industry_number
-
-            ths_stock_concept_detail_df['first_relevance_industry'] = first_relevance_industry
-            ths_stock_concept_detail_df['second_relevance_industry'] = second_relevance_industry
-
-            mongodb_util.save_mongo(ths_concept_list_one_df, db_name_constant.THS_CONCEPT_LIST)
-            mongodb_util.save_mongo(ths_stock_concept_detail_df, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
-
+            update_one_concept_relevance_industry(ths_concept_one.symbol)
         except BaseException as e:
             logger.error("更新概念信息异常:{},{}", e, ths_concept_one.name)
 
 
+# 更新一个概念行业相关
+def update_one_concept_relevance_industry(concept_code):
+    query = {'concept_code': concept_code}
+    ths_stock_concept_detail_df = (mongodb_util
+                                   .find_query_data(db_name_constant.THS_STOCK_CONCEPT_DETAIL, query))
+    concept_count = ths_stock_concept_detail_df.shape[0]
+    ths_concept_list = ths_concept_common_service_api.get_all_ths_concept()
+    ths_concept_list_one_df = ths_concept_list.loc[ths_concept_list['symbol'] == concept_code]
+    ths_concept_list_one_df['concept_count'] = concept_count
+
+    ths_stock_concept_detail_df = ths_stock_concept_detail_df.reset_index(drop=True)
+    if 'industry' in ths_stock_concept_detail_df.columns:
+        del ths_stock_concept_detail_df['industry']
+
+    company_info_df = company_common_service_api.get_company_info_industry()
+    company_info_df = company_info_df[['_id', 'industry']]
+    company_info_df = company_info_df.loc[
+        company_info_df['_id'].isin(list(ths_stock_concept_detail_df['symbol']))]
+    company_info_df = company_info_df.set_index(['_id'], drop=True)
+
+    ths_stock_concept_detail_df = ths_stock_concept_detail_df.set_index(['symbol'], drop=False)
+    ths_stock_concept_detail_df = pd.merge(ths_stock_concept_detail_df, company_info_df,
+                                           how='outer',
+                                           left_index=True, right_index=True)
+    ths_stock_concept_detail_df.dropna(subset=['industry'], axis=0, inplace=True)
+
+    grouped = ths_stock_concept_detail_df.groupby('industry')
+    result_list = grouped.size()
+    ths_concept_group = pd.DataFrame(result_list, columns=['number'])
+    ths_concept_group['industry'] = ths_concept_group.index
+    ths_concept_group = ths_concept_group.sort_values(by=['number'], ascending=False)
+    if ths_concept_group.shape[0] >= 2:
+        first_relevance_industry = list(ths_concept_group.iloc[0:1]['industry'])[0]
+        first_relevance_industry_number = list(ths_concept_group.iloc[0:1]['number'])[0]
+        second_relevance_industry = list(ths_concept_group.iloc[1:2]['industry'])[0]
+        second_relevance_industry_number = list(ths_concept_group.iloc[1:2]['number'])[0]
+    else:
+        first_relevance_industry = list(ths_concept_group.iloc[0:1]['industry'])[0]
+        first_relevance_industry_number = list(ths_concept_group.iloc[0:1]['number'])[0]
+        second_relevance_industry = '无'
+        second_relevance_industry_number = 0
+    ths_concept_list_one_df['first_relevance_industry'] = first_relevance_industry
+    ths_concept_list_one_df['second_relevance_industry'] = second_relevance_industry
+    ths_concept_list_one_df['first_relevance_industry_number'] = first_relevance_industry_number
+    ths_concept_list_one_df['second_relevance_industry_number'] = second_relevance_industry_number
+
+    ths_stock_concept_detail_df['first_relevance_industry'] = first_relevance_industry
+    ths_stock_concept_detail_df['second_relevance_industry'] = second_relevance_industry
+
+    new_values_list = {"$set": {"first_relevance_industry": first_relevance_industry,
+                                "second_relevance_industry": second_relevance_industry,
+                                "first_relevance_industry_number": first_relevance_industry_number,
+                                "second_relevance_industry_number": second_relevance_industry_number,
+                                "concept_count": concept_count
+                                }}
+
+    update_query = {"symbol": concept_code}
+    mongodb_util.update_one_query(update_query, new_values_list, db_name_constant.THS_CONCEPT_LIST)
+
+    new_values_detail = {"$set": {"first_relevance_industry": first_relevance_industry,
+                                  "second_relevance_industry": second_relevance_industry
+
+                                  }}
+    update_query_detail = {"concept_code": concept_code}
+    mongodb_util.update_one_query(update_query_detail, new_values_detail, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
+
+
 # 更新空名字
 def update_null_name():
     query = {"_id": {'$gte': 886025}}
@@ -86,7 +107,7 @@ def update_null_name():
         exist_url = concept_one.url
 
         if name == '':
-            concept_name = ths_stock_api.get_concept_name(concept_code)
+            concept_name = ths_concept_index_web.get_concept_name(concept_code)
             query_concept = {"symbol": concept_code}
             new_values = {'$set': {"name": concept_name}}
             mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
@@ -107,6 +128,8 @@ def update_null_name():
 
 
 if __name__ == '__main__':
+    # update_one_concept_relevance_industry(886095)
+    update_null_name()
     logger.info("开始")
     update_ths_concept_info()
     logger.info("结束")

mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py

@@ -2,7 +2,6 @@ import sys
 import os
 
 import pandas as pd
-import mns_common.api.ths.ths_stock_api as ths_stock_api
 from mns_common.db.MongodbUtil import MongodbUtil
 import mns_common.component.company.company_common_service_api as company_common_service_api
 
@@ -12,49 +11,31 @@ project_path = file_path[0:end]
 sys.path.append(project_path)
 mongodb_util = MongodbUtil('27017')
 import mns_common.api.msg.push_msg_api as push_msg_api
-import mns_scheduler.company_info.company_info_sync_api as company_info_sync_api
-
-max_concept_code = 886110
-
-order_fields = [
-    "index",
-    "symbol",
-    "name",
-    "now_price",
-    "chg",
-    "change",
-    "exchange",
-    "amount",
-    "concept_code",
-]
-
-
-# 推送到手机
-def push_msg_to_we_chat_web(concept_code, concept_name, url):
+import mns_scheduler.company_info.base.sync_company_base_info_api as company_info_sync_api
+import mns_scheduler.company_info.clean.company_info_clean_api as company_info_clean_api
+import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
+import mns_common.component.company.company_common_service_new_api as company_common_service_new_api
+import mns_common.utils.data_frame_util as data_frame_util
+import mns_common.constant.db_name_constant as db_name_constant
+import mns_scheduler.concept.clean.ths_concept_clean_api as ths_concept_clean_api
+import mns_common.constant.redis_msg_constant as redis_msg_constant
+import mns_common.component.redis_msg.redis_msg_publish_service as redis_msg_publish_service
+from loguru import logger
+
+
+# 推送消息
+def push_msg_to_we_chat_and_redis(concept_code, concept_name, url):
     msg = "概念代码:" + str(concept_code) + "," + "概念名称:" + concept_name + "," + "url:   " + url
     title = "新增同花顺概念:" + str(concept_code) + "-" + concept_name
+    # 推送到微信
     push_msg_api.push_msg_to_wechat(title, msg)
+    # 项目之前推送消息
+    redis_msg_publish_service.send_redis_msg(redis_msg_constant.THS_CONCEPT_MSG_TOPIC,
+                                             redis_msg_constant.THS_NEW_CONCEPT_ADD_MSG)
 
-
-# 保存新概念信息到数据库
-def save_ths_concept_list(concept_code, concept_name, str_day, str_now_time):
-    url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
-    ths_concept_list = pd.DataFrame([
-        [concept_code, concept_code, concept_name, str_day, url, str_now_time, True, True],
-    ], columns=['_id', 'symbol', 'name', 'str_day', 'url', 'str_now_time', 'success', 'valid'])
-    mongodb_util.save_mongo(ths_concept_list, 'ths_concept_list')
-
-
-# 获取最大概念代码
-def get_max_concept_code():
-    query = {"symbol": {'$ne': 'null'}, "success": True}
-    ths_concept_max = mongodb_util.descend_query(query, 'ths_concept_list', 'symbol', 1)
-    if ths_concept_max.shape[0] == 0:
-        concept_code = 885284
-    else:
-        concept_code = list(ths_concept_max['symbol'])[0]
-
-    return concept_code
+    # 更新ths概念信息
+    ths_concept_clean_api.update_ths_concept_info()
 
 
 # 保存新概念详细信息到数据库
@@ -69,12 +50,13 @@ def save_ths_concept_detail(new_concept_symbol_df,
 
     new_concept_symbol_df['concept_name'] = new_concept_symbol_df['concept_name'].replace(" ", "")
 
-    query_ths_concept = {'symbol': concept_code}
-    ths_concept_list = mongodb_util.find_query_data('ths_concept_list', query_ths_concept)
-    if ths_concept_list.shape[0] == 0:
+    all_ths_concept_df = ths_concept_common_service_api.get_all_ths_concept()
+    ths_concept_one_df = all_ths_concept_df.loc[all_ths_concept_df['symbol'] == int(concept_code)]
+
+    if data_frame_util.is_empty(ths_concept_one_df):
         concept_create_day = str_day
     else:
-        concept_create_day = list(ths_concept_list['str_day'])[0]
+        concept_create_day = list(ths_concept_one_df['str_day'])[0]
 
     new_concept_symbol_df['str_day'] = str_day
     new_concept_symbol_df['str_now_time'] = str_now_time
@@ -83,24 +65,42 @@ def save_ths_concept_detail(new_concept_symbol_df,
     new_concept_symbol_list = list(new_concept_symbol_df['symbol'])
 
     query_company_info = {'symbol': {'$in': new_concept_symbol_list}}
+    query_company_info_key = str(query_company_info)
     query_field = {"first_industry": 1, "first_industry": 1, "industry": 1,
                    "company_type": 1, "flow_mv_sp": 1,
                    "total_mv_sp": 1}
-    company_info = mongodb_util.find_query_data_choose_field('company_info',
-                                                             query_company_info, query_field)
+    query_field_key = str(query_field)
+    company_info = company_common_service_new_api.get_company_info_by_field(query_company_info_key, query_field_key)
+
+    if 'industry' in new_concept_symbol_df.columns:
+        del new_concept_symbol_df['industry']
+    if 'company_type' in new_concept_symbol_df.columns:
+        del new_concept_symbol_df['company_type']
+    if 'flow_mv_sp' in new_concept_symbol_df.columns:
+        del new_concept_symbol_df['flow_mv_sp']
+    if 'total_mv_sp' in new_concept_symbol_df.columns:
+        del new_concept_symbol_df['total_mv_sp']
 
     company_info = company_info.set_index(['_id'], drop=True)
     new_concept_symbol_df = new_concept_symbol_df.set_index(['symbol'], drop=False)
 
     new_concept_symbol_df = pd.merge(new_concept_symbol_df, company_info, how='outer',
                                      left_index=True, right_index=True)
+
+    if 'index' not in company_info.columns:
+        new_concept_symbol_df['index'] = 0
+
+    if 'change' not in company_info.columns:
+        new_concept_symbol_df['change'] = 0
+
     new_concept_symbol_df['concept_name'] = new_concept_symbol_df['concept_name'].replace(" ", "")
-    query = {'concept_code': concept_code}
 
     if bool(1 - ('way' in new_concept_symbol_df.columns)):
         new_concept_symbol_df['way'] = 'symbol_sync'
     if "long" not in new_concept_symbol_df.columns:
         new_concept_symbol_df['long'] = ''
+    if "short" not in new_concept_symbol_df.columns:
+        new_concept_symbol_df['short'] = new_concept_symbol_df['long']
     new_concept_symbol_df = new_concept_symbol_df[[
         "_id",
         "index",
@@ -121,41 +121,97 @@ def save_ths_concept_detail(new_concept_symbol_df,
         "company_type",
         "concept_create_day",
         "way",
-        "long"
+        "long",
+        'short'
     ]]
+    query_detail = {"concept_code": int(concept_code)}
+    exist_concept_detail = mongodb_util.find_query_data(db_name_constant.THS_STOCK_CONCEPT_DETAIL, query_detail)
 
-    exist_concept_detail = mongodb_util.find_query_data('ths_stock_concept_detail', query)
     if exist_concept_detail is None or exist_concept_detail.shape[0] == 0:
-        mongodb_util.save_mongo(new_concept_symbol_df, 'ths_stock_concept_detail')
+        new_concept_symbol_df['grade'] = 1
+        # 详细标识
+        new_concept_symbol_df['remark'] = ''
+        # 简单标识
+        new_concept_symbol_df['remark_flag'] = ''
+        mongodb_util.save_mongo(new_concept_symbol_df, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
         # 保存到当日新增概念列表
         new_concept_symbol_df['concept_type'] = 'ths'
-        mongodb_util.save_mongo(new_concept_symbol_df, 'today_new_concept_list')
+        mongodb_util.save_mongo(new_concept_symbol_df, db_name_constant.TODAY_NEW_CONCEPT_LIST)
     else:
         exist_concept_detail_symbol_list = list(exist_concept_detail['symbol'])
         new_concept_symbol_df = new_concept_symbol_df.loc[~(
             new_concept_symbol_df['symbol'].isin(exist_concept_detail_symbol_list))]
         if new_concept_symbol_df.shape[0] > 0:
-            mongodb_util.save_mongo(new_concept_symbol_df, 'ths_stock_concept_detail')
+            new_concept_symbol_df['grade'] = 1
+            # 详细标识
+            new_concept_symbol_df['remark'] = ''
+            # 简单标识
+            new_concept_symbol_df['remark_flag'] = ''
+            mongodb_util.save_mongo(new_concept_symbol_df, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
             # 保存到当日新增概念列表
             new_concept_symbol_df['concept_type'] = 'ths'
-            mongodb_util.save_mongo(new_concept_symbol_df, 'today_new_concept_list')
+            mongodb_util.save_mongo(new_concept_symbol_df, db_name_constant.TODAY_NEW_CONCEPT_LIST)
+
     update_company_info(new_concept_symbol_df)
     # 公司缓存信息清除
     company_common_service_api.company_info_industry_cache_clear()
 
 
-# 更新公司表信息 todo 清空cache 公司表中  common_service_fun_api.py  get_company_info_industry
+# 更新入选理由
+def update_long_short(new_concept_symbol_df, exist_concept_detail):
+    if data_frame_util.is_empty(new_concept_symbol_df):
+        return None
+    for new_concept_one in new_concept_symbol_df.itertuples():
+        try:
+            query = {'symbol': new_concept_one.symbol, 'concept_code': new_concept_one.concept_code}
+            new_values = {"$set": {'long': new_concept_one.long, "short": new_concept_one.short}}
+            mongodb_util.update_many(query, new_values, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
+            mongodb_util.update_many(query, new_values, db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP)
+        except BaseException as e:
+            logger.error("更新入选理由异常:{}", e)
+    # 更新已经被删除概念的股票
+    # update_delete_concept_symbol(list(new_concept_symbol_df['concept_code'])[0], new_concept_symbol_df,
+    #                              exist_concept_detail)
+    # 更新公司表信息 todo 清空cache 公司表中  common_service_fun_api.py  get_company_info_industry
+
+
+## 更新已经被删除这个概念的股票
+# def update_delete_concept_symbol(concept_code, new_concept_symbol_df, exist_concept_detail):
+#     delete_concept_symbol_df = exist_concept_detail.loc[
+#         ~(exist_concept_detail['symbol'].isin(list(new_concept_symbol_df['symbol'])))]
+#
+#     if data_frame_util.is_not_empty(delete_concept_symbol_df):
+#         new_values = {"$set": {"grade": 0}}
+#         query = {'symbol': {"$in": list(delete_concept_symbol_df['symbol'])}, 'concept_code': concept_code}
+#         mongodb_util.update_many(query, new_values, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
+#         mongodb_util.update_many(query, new_values, db_name_constant.THS_STOCK_CONCEPT_DETAIL_APP)
+#
+#     exist_concept_detail = exist_concept_detail.loc[exist_concept_detail]
+
+
 def update_company_info(new_concept_symbol_df):
     if new_concept_symbol_df.shape[0] > 0:
         symbol_list = list(new_concept_symbol_df['symbol'])
         company_info_sync_api.sync_company_base_info(symbol_list)
-        company_info_sync_api.fix_company_industry(None)
+        company_info_clean_api.clean_company_info(None)
         # 公司缓存信息清除
         company_common_service_api.company_info_industry_cache_clear()
 
 
+# 获取最大概念代码
+def get_max_concept_code():
+    query = {"symbol": {'$ne': 'null'}, "success": True}
+    ths_concept_max = mongodb_util.descend_query(query, 'ths_concept_list', 'symbol', 1)
+    if ths_concept_max.shape[0] == 0:
+        concept_code = 885284
+    else:
+        concept_code = list(ths_concept_max['symbol'])[0]
+
+    return concept_code
+
+
 def get_concept_detail_info_web(concept_code):
-    new_concept_symbol_list = ths_stock_api.ths_stock_concept(concept_code)
+    new_concept_symbol_list = ths_concept_detail_api.get_ths_concept_detail(concept_code, None)
     if new_concept_symbol_list is None or new_concept_symbol_list.shape[0] == 0:
         return None
     new_concept_symbol_list['_id'] = str(concept_code) + '-' + new_concept_symbol_list['symbol']

mns_scheduler/concept/ths/common/ths_concept_update_common_api.py

@@ -13,9 +13,7 @@ import mns_common.utils.data_frame_util as data_frame_util
 import mns_common.constant.db_name_constant as db_name_constant
 import mns_common.component.trade_date.trade_date_common_service_api as trade_date_common_service_api
 from datetime import datetime
-import mns_common.api.ths.ths_stock_api as ths_stock_api
-import pandas as pd
-
+import mns_common.api.ths.concept.web.ths_concept_detail_web as ths_concept_detail_web
 mongodb_util = MongodbUtil('27017')
 
 
@@ -33,7 +31,10 @@ def update_ths_concept_choose_reason(ths_symbol_all_concepts, symbol):
                                      {'symbol': symbol, "concept_code": int(concept_one.cid)}]}
                     short = concept_one.short
                     long = concept_one.long
-                    new_values = {"$set": {"short": short, "long": long}}
+                    if data_frame_util.is_string_not_empty(long):
+                        new_values = {"$set": {"short": short, "long": long}}
+                    else:
+                        new_values = {"$set": {"grade": 0}}
                     mongodb_util.update_many(query, new_values, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
         except BaseException as e:
             logger.error("更新ths概念入选理由异常{},{},{}", symbol, concept_one.title, e)
@@ -69,8 +70,7 @@ def update_null_reason(nan_reason_df):
                 continue
             web_concept_code = list(ths_concept_one_df['web_concept_code'])[0]
 
-            ths_concept_json = ths_stock_api.get_symbol_add_new_concept(nan_one.symbol)
-            symbol_ths_concept_all_df = pd.DataFrame(ths_concept_json)
+            symbol_ths_concept_all_df = ths_concept_detail_web.get_one_symbol_all_ths_concepts(nan_one.symbol)
 
             symbol_ths_concept_one_df = symbol_ths_concept_all_df[
                 symbol_ths_concept_all_df['cid'] == web_concept_code]