mns-scheduler 1.0.1.0__py3-none-any.whl → 1.0.1.2__py3-none-any.whl

mns_scheduler/concept/ths/symbol/sync_symbol_all_concepts_api.py

@@ -18,7 +18,7 @@ project_path = file_path[0:end]
 sys.path.append(project_path)
 mongodb_util = MongodbUtil('27017')
 # 分页大小
-MAX_PAGE_NUMBER = 2500
+MAX_PAGE_NUMBER = 12500
 # 获取单只股票新增概念
 
 db_name = 'ths_stock_concept_detail_app'
@@ -81,7 +81,7 @@ def choose_type_new_concept(ths_concept_df):
 
 
 # 同步新概念 多线程实现
-def update_symbol_new_update(symbol_df, page_number):
+def update_symbol_new_concept(symbol_df, page_number):
     for stock_one in symbol_df.itertuples():
         try:
             ths_concept_json = ths_stock_api.get_symbol_add_new_concept(stock_one.symbol)
@@ -159,7 +159,7 @@ def sync_symbol_all_concept(symbol):
         end_count = (page + 1) * MAX_PAGE_NUMBER
         begin_count = page * MAX_PAGE_NUMBER
         page_df = real_time_quotes_now.loc[begin_count:end_count]
-        thread = threading.Thread(target=update_symbol_new_update, args=(page_df, page_number))
+        thread = threading.Thread(target=update_symbol_new_concept, args=(page_df, page_number))
         threads.append(thread)
         thread.start()
 

mns_scheduler/concept/ths/web/sync_ths_concept_by_ak_api.py

@@ -31,21 +31,30 @@ def sync_new_ths_concept_by_ak_api():
 
     stock_board_concept_name_ths_df['str_day'].fillna(datetime(1970, 1, 1), inplace=True)
 
+    stock_board_concept_name_ths_df.concept_name = stock_board_concept_name_ths_df.concept_name.str.replace(' ', '')
+    stock_board_concept_name_ths_df.concept_name = stock_board_concept_name_ths_df.concept_name.str.replace('（', '(')
+    stock_board_concept_name_ths_df.concept_name = stock_board_concept_name_ths_df.concept_name.str.replace('）', ')')
+
     stock_board_concept_name_ths_df['str_day'] = stock_board_concept_name_ths_df['str_day'].apply(
         lambda x: datetime.strftime(x, '%Y-%m-%d'))
 
     # 同花顺概念列表
-    ths_concept_list = mongodb_util.find_all_data('ths_concept_list')
+    ths_concept_list_exist = mongodb_util.find_all_data('ths_concept_list')
 
     for concept_one in stock_board_concept_name_ths_df.itertuples():
         try:
-            exist_concept_df_one = ths_concept_list.loc[ths_concept_list['name'] == concept_one.concept_name]
-            str_now_time = concept_one.str_day + " 00:00:00"
+            exist_concept_df_one = ths_concept_list_exist.loc[
+                (ths_concept_list_exist['name'] == concept_one.concept_name)
+                | (ths_concept_list_exist['web_concept_code'] == concept_one.concept_code)]
+            now_date = datetime.now()
+            str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
             str_day = concept_one.str_day
             if data_frame_util.is_empty(exist_concept_df_one):
+                concept_code = ths_concept_sync_common_api.get_max_concept_code()
+
                 diff_one = {
-                    '_id': int(concept_one.concept_code),
-                    'symbol': int(concept_one.concept_code),
+                    '_id': int(concept_code),
+                    'symbol': int(concept_code),
                     'name': concept_one.concept_name,
                     'url': concept_one.url,
                     'str_day': str_day,
@@ -58,26 +67,16 @@ def sync_new_ths_concept_by_ak_api():
                 mongodb_util.save_mongo(diff_one_df, 'ths_concept_list')
                 url = 'http://q.10jqka.com.cn/gn/detail/code/' + str(concept_one.concept_code)
                 # 推送新概念信息到微信
-                ths_concept_sync_common_api.push_msg_to_we_chat_web(concept_one.concept_code, concept_one.concept_name,
+                ths_concept_sync_common_api.push_msg_to_we_chat_web(concept_code, concept_one.concept_name,
                                                                     url)
-                new_concept_symbol_df = get_concept_detail_info_web(concept_one.concept_code)
+                new_concept_symbol_df = get_concept_detail_info_web(concept_code)
                 if new_concept_symbol_df is None or new_concept_symbol_df.shape[0] == 0:
                     return None
                 new_concept_symbol_df.loc[:, 'way'] = 'index_sync'
                 ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_df, concept_one.concept_name,
                                                                     str_day,
-                                                                    str_now_time, concept_one.concept_code)
-
+                                                                    str_now_time, concept_code)
                 logger.info("新增同花顺新概念:{}", concept_one.concept_name)
-            else:
-                # 更新概念关联关系
-                web_concept_code = concept_one.concept_code
-                web_concept_url = concept_one.url
-                concept_code = list(exist_concept_df_one['symbol'])[0]
-
-                update_query = {'_id': concept_code}
-                new_values = {"$set": {"web_concept_code": int(web_concept_code), 'web_concept_url': web_concept_url}}
-                mongodb_util.update_many(update_query, new_values, 'ths_concept_list')
         except BaseException as e:
             logger.error("同步概念:{},信息异常:{}", concept_one.concept_name, e)
 
@@ -93,4 +92,5 @@ def get_concept_detail_info_web(concept_code):
 
 
 if __name__ == '__main__':
+    get_concept_detail_info_web(886068)
     sync_new_ths_concept_by_ak_api()

mns_scheduler/concept/ths/web/sync_ths_new_concept_by_web_api.py

@@ -53,6 +53,10 @@ def sync_new_concept_data_by_web():
                 continue
             time.sleep(2)
             concept_name = ths_stock_api.get_concept_name(concept_code)
+            concept_name = concept_name.replace(' ',                                                '')
+            concept_name = concept_name.replace('（', '(')
+            concept_name = concept_name.replace('）', ')')
+
             url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
             # 推送新概念信息到微信
             ths_concept_sync_common_api.push_msg_to_we_chat_web(concept_code, concept_name, url)

mns_scheduler/zz_task/data_sync_task.py

@@ -224,12 +224,12 @@ def sync_symbol_all_concept():
     logger.info('同步单只股票所有概念组成')
 
 
-# 同步同花顺新增概念指数通过web端接口爬取
+# 同步同花顺新增概念指数通过web端接口爬取 8开头
 def sync_new_concept_data_by_web():
     sync_ths_new_concept_by_web_api.sync_new_concept_data_by_web()
 
 
-# 同步同花顺新增概念指数通过ak_share接口爬取
+# 同步同花顺新增概念指数通过ak_share接口爬取 3开头
 def sync_new_ths_concept_by_ak_api():
     sync_ths_concept_by_ak_api.sync_new_ths_concept_by_ak_api()
 
@@ -310,7 +310,7 @@ blockingScheduler.add_job(sync_today_trade_k_line_info, 'cron', hour='08', minut
 # 同步十大流通股东信息
 blockingScheduler.add_job(sync_stock_gdfx_free_top_10_one_day, 'cron', hour='08,22', minute='23')
 
-# 同步概念下所有股票组成
+# 更新概念指数下所有股票组成 by 概念代码
 blockingScheduler.add_job(update_concept_all_symbol_detail, 'cron', hour='08,18,12', minute='30')
 
 # 同步单只股票下所有概念 by 股票代码
@@ -319,7 +319,10 @@ blockingScheduler.add_job(sync_symbol_all_concept, 'cron', hour='09,18,12', minu
 # 开盘前同步同花顺新概念指数
 blockingScheduler.add_job(sync_new_concept_data_web, 'cron', hour='09', minute='05,10,15,20,25')
 
-# 同步开盘啦精选概念(定时轮训,暂时五分钟)
+# 同步同花顺新增概念指数(定时轮训,暂时10分钟)
+blockingScheduler.add_job(sync_new_concept_data_web, 'interval', minutes=10, max_instances=4)
+
+# 同步开盘啦新增精选概念(定时轮训,暂时五分钟)
 blockingScheduler.add_job(sync_all_kpl_plate_info, 'interval', minutes=5, max_instances=4)
 
 # 同步公司基本信息

{mns_scheduler-1.0.1.0.dist-info → mns_scheduler-1.0.1.2.dist-info}/METADATA

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.0.1.0
+Version: 1.0.1.2
 

{mns_scheduler-1.0.1.0.dist-info → mns_scheduler-1.0.1.2.dist-info}/RECORD

@@ -18,10 +18,10 @@ mns_scheduler/concept/ths/common/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiN
 mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py,sha256=f3g8mx1Ii5QrPVStsnXa68hrsGlvhbVc9v4gOdS9SWk,10349
 mns_scheduler/concept/ths/symbol/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/concept/ths/symbol/sync_concept_all_symbols_api.py,sha256=ipRtiihpHlbxuces7YpHfvBtCDxgzrVXjcCKZiFBPaw,1928
-mns_scheduler/concept/ths/symbol/sync_symbol_all_concepts_api.py,sha256=-FyzLcuP8H-_G6IyH1rLb1078apS4HtaMsksyX1D37I,8261
+mns_scheduler/concept/ths/symbol/sync_symbol_all_concepts_api.py,sha256=9-cp2ZmZJ7ZQwQUJNVBY99DthA-7gtwvDRQxIQ-QF4Q,8264
 mns_scheduler/concept/ths/web/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/concept/ths/web/sync_ths_concept_by_ak_api.py,sha256=z5zQUWp4QJFprCdpw8ky16MvncdW5rNUZyYMOWfKOzk,5017
-mns_scheduler/concept/ths/web/sync_ths_new_concept_by_web_api.py,sha256=U5e-TJtNTuM5ux2QB6mbndyITMsV79GYv2IoAXN_7gg,3481
+mns_scheduler/concept/ths/web/sync_ths_concept_by_ak_api.py,sha256=FIJ552806hDba0l1I_w_AMGhRXUUhWV0RiRMRqn74dM,5106
+mns_scheduler/concept/ths/web/sync_ths_new_concept_by_web_api.py,sha256=u2TRPWDk7B9mocuCp23f964arZKFD_TMg4dHYnIkMuk,3710
 mns_scheduler/concept/ths/wen_cai/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/concept/ths/wen_cai/wen_cai_concept_sync.py,sha256=boKtgOIXIsRny5n1keZm4luDFCiooudbPDsEFIGImIw,1747
 mns_scheduler/db/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
@@ -57,9 +57,9 @@ mns_scheduler/zt/today_high_chg_pool_sync_api.py,sha256=G6FXMH7HnvhyMFoE-RszggkB
 mns_scheduler/zt/zt_five_boards_sync_api.py,sha256=HfjPHKD99fU9c37kSenEX2_qNvFAjQGgy8ERuacSxwk,10916
 mns_scheduler/zt/zt_pool_sync_api.py,sha256=AhWYrpHzO3KOKecyGqQFvopYd2KFmf1ykRiv2R9Pjo4,7490
 mns_scheduler/zz_task/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/zz_task/data_sync_task.py,sha256=KI3yGAUj72rJqoJDr1jqCmwGf93OynjlFrOvq8is4zs,13412
+mns_scheduler/zz_task/data_sync_task.py,sha256=jWwmUZ47ZZ6zdVriqiftkV22b8iyfYtlsGbX8lIFDTo,13619
 mns_scheduler/zz_task/sync_realtime_quotes_task.py,sha256=DN3bq2XCDZC-PHlbD2NTog48bR44EruIEc2QVGKg7Tk,932
-mns_scheduler-1.0.1.0.dist-info/METADATA,sha256=fjHbjlpwfqoTzSi3TK8ZHrb-331m5PRHAvKQTxkr5Vk,64
-mns_scheduler-1.0.1.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-mns_scheduler-1.0.1.0.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
-mns_scheduler-1.0.1.0.dist-info/RECORD,,
+mns_scheduler-1.0.1.2.dist-info/METADATA,sha256=OTFm540p3k0VA-apegBT2hZn6mHGxyE0oZ6eRnfN9p4,64
+mns_scheduler-1.0.1.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+mns_scheduler-1.0.1.2.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
+mns_scheduler-1.0.1.2.dist-info/RECORD,,