mns-scheduler 1.1.0.1__tar.gz → 1.1.0.3__tar.gz

{mns-scheduler-1.1.0.1 → mns-scheduler-1.1.0.3}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.1.0.1
+Version: 1.1.0.3

{mns-scheduler-1.1.0.1 → mns-scheduler-1.1.0.3}/mns_scheduler/concept/ths/common/ths_concept_update_common_api.py

@@ -14,8 +14,6 @@ import mns_common.constant.db_name_constant as db_name_constant
 import mns_common.component.trade_date.trade_date_common_service_api as trade_date_common_service_api
 from datetime import datetime
 import mns_common.api.ths.concept.web.ths_concept_detail_web as ths_concept_detail_web
-import pandas as pd
-
 mongodb_util = MongodbUtil('27017')
 
 

mns-scheduler-1.1.0.3/mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_new_index_api.py

@@ -0,0 +1,170 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import mns_common.api.ths.concept.app.ths_concept_index_app as ths_concept_index_app
+import mns_scheduler.concept.ths.common.ths_concept_sync_common_api as ths_concept_sync_common_api
+import mns_common.utils.data_frame_util as data_frame_util
+from loguru import logger
+import mns_common.api.ths.concept.web.ths_concept_index_web as ths_concept_index_web
+from mns_common.db.MongodbUtil import MongodbUtil
+from datetime import datetime
+import mns_scheduler.concept.clean.ths_concept_clean_api as ths_concept_clean_api
+import pandas as pd
+import mns_scheduler.concept.common.detaill.ths_concept_detail_api as ths_concept_detail_api
+import mns_common.constant.db_name_constant as db_name_constant
+
+mongodb_util = MongodbUtil('27017')
+
+
+def sync_ths_concept_new_index():
+    # 同步ths新概念 通过app搜索
+    sync_ths_concept_new_index_from_app()
+    # 通过ths详情搜索
+    sync_ths_concept_new_index_from_detail()
+
+
+'''
+同步ths新概念 通过app搜索
+'''
+
+
+def sync_ths_concept_new_index_from_app():
+    # 当前最大概念代码
+    max_concept_code = ths_concept_sync_common_api.get_max_concept_code()
+    # 最大概念代码上线
+    max_concept_code_limit = max_concept_code + 2
+    # 同花顺概念列表
+    ths_concept_list_exist = mongodb_util.find_all_data(db_name_constant.THS_CONCEPT_LIST)
+    # 同步向上3次
+    while max_concept_code <= max_concept_code_limit:
+        try:
+            max_concept_code = max_concept_code + 1
+            concept_new_index_df = ths_concept_index_app.get_new_concept_from_app_search(max_concept_code)
+            if data_frame_util.is_empty(concept_new_index_df):
+                continue
+            concept_name = list(concept_new_index_df['concept_name'])[0]
+
+            if data_frame_util.is_string_empty(concept_name):
+                concept_name = ths_concept_index_web.get_concept_name(max_concept_code)
+
+            exist_concept_df_one = ths_concept_list_exist.loc[
+                (ths_concept_list_exist['symbol'] == max_concept_code)
+                | (ths_concept_list_exist['web_concept_code'] == max_concept_code)]
+            now_date = datetime.now()
+            str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
+            str_day = now_date.strftime('%Y-%m-%d')
+            if data_frame_util.is_empty(exist_concept_df_one):
+                concept_code = max_concept_code
+
+                url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
+                new_concept_one = {
+                    '_id': int(concept_code),
+                    'symbol': int(concept_code),
+                    'name': concept_name,
+                    'url': url,
+                    'str_day': str_day,
+                    'success': True,
+                    'str_now_time': str_now_time,
+                    'web_concept_code': concept_code,
+                    'web_concept_url': url,
+                    'valid': True
+                }
+                diff_one_df = pd.DataFrame(new_concept_one, index=[1])
+                mongodb_util.save_mongo(diff_one_df, db_name_constant.THS_CONCEPT_LIST)
+                # 新增概念信息处理
+                handle_new_concept_msg(concept_code, concept_name, url)
+
+                new_concept_symbol_detail_df = ths_concept_detail_api.get_ths_concept_detail(concept_code, concept_name)
+                if data_frame_util.is_empty(new_concept_symbol_detail_df):
+                    continue
+                new_concept_symbol_detail_df.loc[:, 'way'] = 'index_sync'
+                ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_detail_df,
+                                                                    concept_name,
+                                                                    str_day,
+                                                                    str_now_time, concept_code)
+                # 更新ths概念统计信息
+                ths_concept_clean_api.update_ths_concept_info()
+
+                logger.info("新增同花顺新概念:{}", concept_name)
+
+        except BaseException as e:
+            logger.error("同步新增概念代码:{},信息异常:{}", max_concept_code, e)
+
+
+'''
+同步新概念 by ths detail
+'''
+
+
+def sync_ths_concept_new_index_from_detail():
+    # 当前最大概念代码
+    max_concept_code = ths_concept_sync_common_api.get_max_concept_code()
+    # 最大概念代码上线
+    max_concept_code_limit = max_concept_code + 2
+    # 同花顺概念列表
+    ths_concept_list_exist = mongodb_util.find_all_data(db_name_constant.THS_CONCEPT_LIST)
+    # 同步向上3次
+    while max_concept_code <= max_concept_code_limit:
+        try:
+            max_concept_code = max_concept_code + 1
+            concept_code = max_concept_code
+            concept_name = ths_concept_index_web.get_concept_name(concept_code)
+            new_concept_symbol_detail_df = ths_concept_detail_api.get_ths_concept_detail(concept_code, concept_name)
+            if data_frame_util.is_empty(new_concept_symbol_detail_df):
+                continue
+
+            exist_concept_df_one = ths_concept_list_exist.loc[
+                (ths_concept_list_exist['symbol'] == concept_code)
+                | (ths_concept_list_exist['web_concept_code'] == concept_code)]
+            now_date = datetime.now()
+            str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
+            str_day = now_date.strftime('%Y-%m-%d')
+            if data_frame_util.is_empty(exist_concept_df_one):
+                concept_code = max_concept_code
+
+                url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
+                new_concept_one = {
+                    '_id': int(concept_code),
+                    'symbol': int(concept_code),
+                    'name': concept_name,
+                    'url': url,
+                    'str_day': str_day,
+                    'success': True,
+                    'str_now_time': str_now_time,
+                    'web_concept_code': concept_code,
+                    'web_concept_url': url,
+                    'valid': True
+                }
+                diff_one_df = pd.DataFrame(new_concept_one, index=[1])
+                mongodb_util.save_mongo(diff_one_df, db_name_constant.THS_CONCEPT_LIST)
+                # 新增概念信息处理
+                handle_new_concept_msg(concept_code, concept_name, url)
+
+                new_concept_symbol_detail_df.loc[:, 'way'] = 'index_sync'
+                ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_detail_df,
+                                                                    concept_name,
+                                                                    str_day,
+                                                                    str_now_time,
+                                                                    concept_code)
+                # 更新ths概念统计信息
+                ths_concept_clean_api.update_ths_concept_info()
+
+                logger.info("新增同花顺新概念:{}", concept_name)
+
+        except BaseException as e:
+            logger.error("同步新增概念代码:{},信息异常:{}", max_concept_code, e)
+
+
+# 新增消息处理
+def handle_new_concept_msg(concept_code, concept_name, url):
+    # 推送新概念信息到微信
+    ths_concept_sync_common_api.push_msg_to_we_chat_web(concept_code, concept_name,
+                                                        url)
+
+
+if __name__ == '__main__':
+    sync_ths_concept_new_index_from_detail()

{mns-scheduler-1.1.0.1 → mns-scheduler-1.1.0.3}/mns_scheduler/concept/ths/update_concept_info/sync_one_concept_all_symbols_api.py

@@ -17,7 +17,7 @@ mongodb_util = MongodbUtil('27017')
 # 通过概念指数同步所有概念下的股票组成
 
 # 同步概念下所有股票组成
-def update_concept_all_symbol_detail():
+def update_concept_all_detail_info():
     query = {"symbol": {'$exists': True}}
     new_concept_list = mongodb_util.find_query_data("ths_concept_list", query)
     new_concept_list = new_concept_list.sort_values(by=['symbol'], ascending=False)
@@ -34,7 +34,7 @@ def update_concept_all_symbol_detail():
             time.sleep(1)
             new_concept_symbol_df.loc[:, 'way'] = 'index_sync'
             ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_df, one_concept.name,
-                                                                    str_day, str_now_time, one_concept.symbol)
+                                                                str_day, str_now_time, one_concept.symbol)
 
 
 if __name__ == '__main__':

{mns-scheduler-1.1.0.1 → mns-scheduler-1.1.0.3}/mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py

@@ -86,8 +86,8 @@ def choose_type_new_concept(ths_concept_df):
 def update_symbol_new_concept(symbol_df, page_number):
     for stock_one in symbol_df.itertuples():
         try:
-            symbol_ths_concept_all_df = ths_concept_detail_web.get_symbol_add_new_concept(stock_one.symbol)
-            logger.info("同步symbol概念信息:{}", stock_one.symbol)
+            symbol_ths_concept_all_df = ths_concept_detail_web.get_one_symbol_all_ths_concepts(stock_one.symbol)
+            logger.info("同步代码{},所有概念信息", stock_one.symbol)
             if data_frame_util.is_empty(symbol_ths_concept_all_df):
                 continue
 
@@ -178,4 +178,4 @@ def sync_symbol_all_concept(symbol):
 
 
 if __name__ == '__main__':
-    sync_symbol_all_concept('300085')
+    sync_symbol_all_concept(None)

{mns-scheduler-1.1.0.1 → mns-scheduler-1.1.0.3}/mns_scheduler/zz_task/data_sync_task.py

@@ -31,8 +31,6 @@ import \
     mns_scheduler.concept.ths.update_concept_info.sync_one_concept_all_symbols_api as sync_one_concept_all_symbols_api
 import \
     mns_scheduler.concept.ths.update_concept_info.sync_one_symbol_all_concepts_api as sync_one_symbol_all_concepts_api
-import mns_scheduler.concept.ths.sync_new_index.sync_ths_new_concept_by_web_api as sync_ths_new_concept_by_web_api
-import mns_scheduler.concept.ths.sync_new_index.sync_ths_concept_by_ak_api as sync_ths_concept_by_ak_api
 import mns_scheduler.kpl.selection.total.sync_kpl_best_total_sync_api as sync_kpl_best_total_sync_api
 import mns_scheduler.company_info.company_info_sync_api as company_info_sync_api
 import mns_scheduler.trade.auto_ipo_buy_api as auto_ipo_buy_api
@@ -47,6 +45,7 @@ import mns_scheduler.zt.high_chg.sync_high_chg_pool_service as sync_high_chg_poo
 import mns_scheduler.zt.high_chg.sync_high_chg_real_time_quotes_service as sync_high_chg_real_time_quotes_service
 import mns_scheduler.risk.self.wei_pan_stock_api as wei_pan_stock_api
 import mns_scheduler.risk.transactions.transactions_check_api as transactions_check_api
+import mns_scheduler.concept.ths.sync_new_index.sync_ths_concept_new_index_api as sync_ths_concept_new_index_api
 
 
 # 同步交易日期任务完成
@@ -235,27 +234,17 @@ def concept_info_clean():
 
 
 # 同步概念下所有股票组成 by 概念指数
-def update_one_concept_all_symbol_detail():
+def update_concept_all_detail_info():
     logger.info('同步概念下所有股票组成')
-    sync_one_concept_all_symbols_api.update_concept_all_symbol_detail()
-    update_ths_concept_choose_null_reason()
+    sync_one_concept_all_symbols_api.update_concept_all_detail_info()
+    ths_concept_update_common_api.update_ths_concept_choose_null_reason()
 
 
 # 同步单只股票下所有概念 by 股票代码
 def update_one_symbol_all_concepts():
     logger.info('同步单只股票所有概念组成')
     sync_one_symbol_all_concepts_api.sync_symbol_all_concept(None)
-    update_ths_concept_choose_null_reason()
-
-
-# 同步同花顺新增概念指数通过web端接口爬取 8开头
-def sync_new_concept_data_by_web():
-    sync_ths_new_concept_by_web_api.sync_new_concept_data_by_web()
-
-
-# 同步同花顺新增概念指数通过ak_share接口爬取 3开头
-def sync_new_ths_concept_by_ak_api():
-    sync_ths_concept_by_ak_api.sync_new_ths_concept_by_ak_api()
+    ths_concept_update_common_api.update_ths_concept_choose_null_reason()
 
 
 # 同步开盘啦精选指数
@@ -270,20 +259,22 @@ def update_best_choose_plate_relation():
     sync_kpl_best_total_sync_api.update_best_choose_plate_relation()
 
 
-# 同步新概念通过web端
-def sync_new_concept_data_web():
-    # 同步同花顺新增概念指数通过web端接口爬取
-    sync_new_concept_data_by_web()
-    # 同步同花顺新增概念指数通过ak_share接口爬取
-    sync_new_ths_concept_by_ak_api()
-    update_ths_concept_choose_null_reason()
-    logger.info("同步新概念web端数据任务完成")
+# 同步ths新概念
+def sync_new_concept_index():
+    sync_ths_concept_new_index_api.sync_ths_concept_new_index()
+    logger.info("同步ths新概念任务完成")
 
-
-# 更新空的入选概念
-def update_ths_concept_choose_null_reason():
-    logger.info("更新空的入选概念")
     ths_concept_update_common_api.update_ths_concept_choose_null_reason()
+    logger.info("更新空的入选概念任务完成")
+
+
+# 同步ths新概念 轮训任务
+def sync_new_concept_index_task():
+    now_date = datetime.now()
+    hour = now_date.hour
+    if hour != 9:
+        sync_ths_concept_new_index_api.sync_ths_concept_new_index()
+        logger.info("同步ths新概念任务完成")
 
 
 # 清洗公司基本信息
@@ -387,16 +378,16 @@ blockingScheduler.add_job(sync_stock_gdfx_free_top_10_one_day, 'cron', hour='08,
 blockingScheduler.add_job(concept_info_clean, 'cron', hour='9,12,20', minute='24')
 
 # 更新概念指数下所有股票组成 by 概念代码
-blockingScheduler.add_job(update_one_concept_all_symbol_detail, 'cron', hour='08,18,12', minute='30')
+blockingScheduler.add_job(update_concept_all_detail_info, 'cron', hour='08,18,12', minute='30')
 
 # 同步单只股票下所有概念 by 股票代码
 blockingScheduler.add_job(update_one_symbol_all_concepts, 'cron', hour='09,18,12', minute='15')
 
 # 开盘前同步同花顺新概念指数
-blockingScheduler.add_job(sync_new_concept_data_web, 'cron', hour='09,22', minute='00,10,20,28')
+blockingScheduler.add_job(sync_new_concept_index, 'cron', hour='09,22', minute='01,10,20,28,41,58')
 
 # 同步同花顺新增概念指数(定时轮训,暂时10分钟)
-# blockingScheduler.add_job(sync_new_concept_data_web, 'interval', minutes=10, max_instances=4)
+blockingScheduler.add_job(sync_new_concept_index_task, 'interval', minutes=10, max_instances=4)
 
 # 同步开盘啦新增精选概念(定时轮训,暂时五分钟)
 blockingScheduler.add_job(sync_all_kpl_plate_info, 'interval', minutes=5, max_instances=4)

{mns-scheduler-1.1.0.1 → mns-scheduler-1.1.0.3}/mns_scheduler.egg-info/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.1.0.1
+Version: 1.1.0.3

{mns-scheduler-1.1.0.1 → mns-scheduler-1.1.0.3}/mns_scheduler.egg-info/SOURCES.txt

@@ -10,6 +10,9 @@ mns_scheduler/backup/em/__init__.py
 mns_scheduler/backup/em/em_new_concept_his_sync.py
 mns_scheduler/backup/em/em_new_concept_sync_common_api.py
 mns_scheduler/backup/em/em_new_concept_sync_web.py
+mns_scheduler/backup/ths/__init__.py
+mns_scheduler/backup/ths/sync_ths_concept_by_ak_api.py
+mns_scheduler/backup/ths/sync_ths_new_concept_by_web_api.py
 mns_scheduler/big_deal/__init__.py
 mns_scheduler/big_deal/ths_big_deal_sync.py
 mns_scheduler/company_info/__init__.py
@@ -28,8 +31,7 @@ mns_scheduler/concept/ths/common/__init__.py
 mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py
 mns_scheduler/concept/ths/common/ths_concept_update_common_api.py
 mns_scheduler/concept/ths/sync_new_index/__init__.py
-mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_by_ak_api.py
-mns_scheduler/concept/ths/sync_new_index/sync_ths_new_concept_by_web_api.py
+mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_new_index_api.py
 mns_scheduler/concept/ths/update_concept_info/__init__.py
 mns_scheduler/concept/ths/update_concept_info/sync_one_concept_all_symbols_api.py
 mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py

{mns-scheduler-1.1.0.1 → mns-scheduler-1.1.0.3}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='mns-scheduler',
-    version='1.1.0.1',
+    version='1.1.0.3',
     packages=find_packages(),
     install_requires=[],  # 如果有依赖项，可以在这里列出
 )