mns-scheduler 1.0.1.1__tar.gz → 1.0.1.3__tar.gz

{mns-scheduler-1.0.1.1 → mns-scheduler-1.0.1.3}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.0.1.1
+Version: 1.0.1.3

{mns-scheduler-1.0.1.1 → mns-scheduler-1.0.1.3}/mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py

@@ -40,9 +40,8 @@ def push_msg_to_we_chat_web(concept_code, concept_name, url):
 def save_ths_concept_list(concept_code, concept_name, str_day, str_now_time):
     url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
     ths_concept_list = pd.DataFrame([
-        [concept_code, concept_code, concept_name, str_day, url, str_now_time, True],
-    ], columns=['_id', 'symbol', 'name', 'str_day', 'url', 'str_now_time', 'success'])
-    ths_concept_list['name'] = ths_concept_list['name'].replace(" ", "")
+        [concept_code, concept_code, concept_name, str_day, url, str_now_time, True, True],
+    ], columns=['_id', 'symbol', 'name', 'str_day', 'url', 'str_now_time', 'success', 'valid'])
     mongodb_util.save_mongo(ths_concept_list, 'ths_concept_list')
 
 

{mns-scheduler-1.0.1.1 → mns-scheduler-1.0.1.3}/mns_scheduler/concept/ths/symbol/sync_symbol_all_concepts_api.py

@@ -18,7 +18,7 @@ project_path = file_path[0:end]
 sys.path.append(project_path)
 mongodb_util = MongodbUtil('27017')
 # 分页大小
-MAX_PAGE_NUMBER = 2500
+MAX_PAGE_NUMBER = 12500
 # 获取单只股票新增概念
 
 db_name = 'ths_stock_concept_detail_app'
@@ -81,7 +81,7 @@ def choose_type_new_concept(ths_concept_df):
 
 
 # 同步新概念 多线程实现
-def update_symbol_new_update(symbol_df, page_number):
+def update_symbol_new_concept(symbol_df, page_number):
     for stock_one in symbol_df.itertuples():
         try:
             ths_concept_json = ths_stock_api.get_symbol_add_new_concept(stock_one.symbol)
@@ -159,7 +159,7 @@ def sync_symbol_all_concept(symbol):
         end_count = (page + 1) * MAX_PAGE_NUMBER
         begin_count = page * MAX_PAGE_NUMBER
         page_df = real_time_quotes_now.loc[begin_count:end_count]
-        thread = threading.Thread(target=update_symbol_new_update, args=(page_df, page_number))
+        thread = threading.Thread(target=update_symbol_new_concept, args=(page_df, page_number))
         threads.append(thread)
         thread.start()
 

{mns-scheduler-1.0.1.1 → mns-scheduler-1.0.1.3}/mns_scheduler/concept/ths/web/sync_ths_concept_by_ak_api.py

@@ -30,54 +30,52 @@ def sync_new_ths_concept_by_ak_api():
                                                                                       })
 
     stock_board_concept_name_ths_df['str_day'].fillna(datetime(1970, 1, 1), inplace=True)
+    stock_board_concept_name_ths_df.concept_name = stock_board_concept_name_ths_df.concept_name.str.replace('（', '(')
+    stock_board_concept_name_ths_df.concept_name = stock_board_concept_name_ths_df.concept_name.str.replace('）', ')')
 
     stock_board_concept_name_ths_df['str_day'] = stock_board_concept_name_ths_df['str_day'].apply(
         lambda x: datetime.strftime(x, '%Y-%m-%d'))
 
     # 同花顺概念列表
-    ths_concept_list = mongodb_util.find_all_data('ths_concept_list')
+    ths_concept_list_exist = mongodb_util.find_all_data('ths_concept_list')
 
     for concept_one in stock_board_concept_name_ths_df.itertuples():
         try:
-            exist_concept_df_one = ths_concept_list.loc[ths_concept_list['name'] == concept_one.concept_name]
-            str_now_time = concept_one.str_day + " 00:00:00"
+            exist_concept_df_one = ths_concept_list_exist.loc[
+                (ths_concept_list_exist['name'] == concept_one.concept_name)
+                | (ths_concept_list_exist['web_concept_code'] == concept_one.concept_code)]
+            now_date = datetime.now()
+            str_now_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
             str_day = concept_one.str_day
             if data_frame_util.is_empty(exist_concept_df_one):
+                concept_code = concept_one.concept_code
+
                 diff_one = {
-                    '_id': int(concept_one.concept_code),
-                    'symbol': int(concept_one.concept_code),
+                    '_id': int(concept_code),
+                    'symbol': int(concept_code),
                     'name': concept_one.concept_name,
                     'url': concept_one.url,
                     'str_day': str_day,
                     'success': True,
                     'str_now_time': str_now_time,
                     'web_concept_code': concept_one.concept_code,
-                    'web_concept_url': concept_one.url
+                    'web_concept_url': concept_one.url,
+                    'valid': True
                 }
                 diff_one_df = pd.DataFrame(diff_one, index=[1])
                 mongodb_util.save_mongo(diff_one_df, 'ths_concept_list')
                 url = 'http://q.10jqka.com.cn/gn/detail/code/' + str(concept_one.concept_code)
                 # 推送新概念信息到微信
-                ths_concept_sync_common_api.push_msg_to_we_chat_web(concept_one.concept_code, concept_one.concept_name,
+                ths_concept_sync_common_api.push_msg_to_we_chat_web(concept_code, concept_one.concept_name,
                                                                     url)
-                new_concept_symbol_df = get_concept_detail_info_web(concept_one.concept_code)
+                new_concept_symbol_df = get_concept_detail_info_web(concept_code)
                 if new_concept_symbol_df is None or new_concept_symbol_df.shape[0] == 0:
                     return None
                 new_concept_symbol_df.loc[:, 'way'] = 'index_sync'
                 ths_concept_sync_common_api.save_ths_concept_detail(new_concept_symbol_df, concept_one.concept_name,
                                                                     str_day,
-                                                                    str_now_time, concept_one.concept_code)
-
+                                                                    str_now_time, concept_code)
                 logger.info("新增同花顺新概念:{}", concept_one.concept_name)
-            else:
-                # 更新概念关联关系
-                web_concept_code = concept_one.concept_code
-                web_concept_url = concept_one.url
-                concept_code = list(exist_concept_df_one['symbol'])[0]
-
-                update_query = {'_id': concept_code}
-                new_values = {"$set": {"web_concept_code": int(web_concept_code), 'web_concept_url': web_concept_url}}
-                mongodb_util.update_many(update_query, new_values, 'ths_concept_list')
         except BaseException as e:
             logger.error("同步概念:{},信息异常:{}", concept_one.concept_name, e)
 
@@ -93,4 +91,5 @@ def get_concept_detail_info_web(concept_code):
 
 
 if __name__ == '__main__':
+    get_concept_detail_info_web(886068)
     sync_new_ths_concept_by_ak_api()

{mns-scheduler-1.0.1.1 → mns-scheduler-1.0.1.3}/mns_scheduler/concept/ths/web/sync_ths_new_concept_by_web_api.py

@@ -53,6 +53,9 @@ def sync_new_concept_data_by_web():
                 continue
             time.sleep(2)
             concept_name = ths_stock_api.get_concept_name(concept_code)
+            concept_name = concept_name.replace('（', '(')
+            concept_name = concept_name.replace('）', ')')
+
             url = 'http://q.10jqka.com.cn/thshy/detail/code/' + str(concept_code)
             # 推送新概念信息到微信
             ths_concept_sync_common_api.push_msg_to_we_chat_web(concept_code, concept_name, url)

{mns-scheduler-1.0.1.1 → mns-scheduler-1.0.1.3}/mns_scheduler.egg-info/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.0.1.1
+Version: 1.0.1.3

{mns-scheduler-1.0.1.1 → mns-scheduler-1.0.1.3}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='mns-scheduler',
-    version='1.0.1.1',
+    version='1.0.1.3',
     packages=find_packages(),
     install_requires=[],  # 如果有依赖项，可以在这里列出
 )