mns-scheduler 1.0.5.6__py3-none-any.whl → 1.0.5.9__py3-none-any.whl

mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py

@@ -18,11 +18,16 @@ from functools import lru_cache
 import pandas as pd
 import requests
 from bs4 import BeautifulSoup
+
+from mns_common.db.MongodbUtil import MongodbUtil
+import mns_common.constant.db_name_constant as db_name_constant
+
+mongodb_util = MongodbUtil('27017')
 from tqdm import tqdm
 
 
 @lru_cache()
-def _fetch_stock_uid() -> dict:
+def sync_stock_uid() -> pd.DataFrame:
     """
     上证e互动-代码ID映射
     https://sns.sseinfo.com/list/company.do
@@ -38,7 +43,7 @@ def _fetch_stock_uid() -> dict:
     }
     uid_list = list()
     code_list = list()
-    for page in tqdm(range(1, 73), leave=False):
+    for page in tqdm(range(1, 74), leave=False):
         data.update({"page": page})
         r = requests.post(url, data=data)
         data_json = r.json()
@@ -53,8 +58,16 @@ def _fetch_stock_uid() -> dict:
                 for item in soup.find_all("a", attrs={"rel": "tag"})
             ]
         )
-    code_uid_map = dict(zip(code_list, uid_list))
-    return code_uid_map
+    code_uid_df = pd.DataFrame()
+    code_uid_df['symbol'] = code_list
+    code_uid_df['uid'] = uid_list
+    code_uid_df['_id'] = uid_list
+    return code_uid_df
+
+
+@lru_cache()
+def get_stock_uid() -> pd.DataFrame:
+    return mongodb_util.find_all_data(db_name_constant.SSE_INFO_UID)
 
 
 def stock_sns_sse_info(symbol: str = "603119") -> pd.DataFrame:
@@ -66,13 +79,16 @@ def stock_sns_sse_info(symbol: str = "603119") -> pd.DataFrame:
     :return: 提问与回答
     :rtype: str
     """
-    code_uid_map = _fetch_stock_uid()
+
+    stock_uid_df = get_stock_uid()
+    stock_uid_df = stock_uid_df.loc[stock_uid_df['symbol'] == symbol]
+    uid = list(stock_uid_df['uid'])[0]
     url = "https://sns.sseinfo.com/ajax/userfeeds.do"
     params = {
         "typeCode": "company",
         "type": "11",
         "pageSize": "100",
-        "uid": code_uid_map[symbol],
+        "uid": uid,
         "page": "1",
     }
     big_df = pd.DataFrame()
@@ -148,5 +164,5 @@ def stock_sns_sse_info(symbol: str = "603119") -> pd.DataFrame:
 
 
 if __name__ == "__main__":
-    stock_sns_sse_info_df = stock_sns_sse_info(symbol="603119")
+    stock_sns_sse_info_df = stock_sns_sse_info(symbol="688787")
     print(stock_sns_sse_info_df)

mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py

@@ -33,6 +33,8 @@ def _fetch_org_id(symbol: str = "000001") -> str:
     return org_id
 
 
+# 获取股票所有问题
+
 def stock_irm_cninfo(symbol: str = "002594") -> pd.DataFrame:
     """
     互动易-提问

mns_scheduler/irm/stock_irm_cninfo_service.py

@@ -11,22 +11,25 @@ import pandas as pd
 import mns_common.api.em.east_money_stock_api as east_money_stock_api
 import mns_common.component.company.company_common_service_api as company_common_service_api
 from loguru import logger
-from datetime import datetime
-from mns_common.db.MongodbUtil import MongodbUtil
+from datetime import datetime, time
+
 import mns_common.utils.date_handle_util as date_handle_util
 
-mongodb_util = MongodbUtil('27017')
 import mns_common.constant.db_name_constant as db_name_constant
 import mns_common.component.common_service_fun_api as common_service_fun_api
 import mns_scheduler.irm.api.sh_stock_sns_sse_info_api as sh_stock_sns_sse_info_api
 import mns_scheduler.irm.api.sz_stock_sns_sse_info_api as sz_stock_sns_sse_info_api
 
+from mns_common.db.MongodbUtil import MongodbUtil
+
+mongodb_util = MongodbUtil('27017')
+
 
 # 获取股票提问    互动易-提问 深交所
 def get_stock_irm_cninfo_sz_api(symbol):
     try:
         # 获取一页
-        stock_irm_cninfo_df = sh_stock_sns_sse_info_api.stock_sns_sse_info(symbol)
+        stock_irm_cninfo_df = sz_stock_sns_sse_info_api.stock_irm_cninfo(symbol)
         # 获取全页
         # stock_irm_cninfo_df = ak.stock_irm_cninfo(symbol)
     except Exception as e:
@@ -67,7 +70,7 @@ def get_stock_irm_cninfo_sz_api(symbol):
 def get_stock_irm_cninfo_sh_api(symbol):
     try:
         # 获取一页
-        stock_sns_sse_info_df = sz_stock_sns_sse_info_api.stock_irm_cninfo(symbol)
+        stock_sns_sse_info_df = sh_stock_sns_sse_info_api.stock_sns_sse_info(symbol)
         # 获取全页
         # stock_sns_sse_info_df = ak.stock_sns_sseinfo(symbol)
     except Exception as e:
@@ -110,34 +113,53 @@ def replace_date_format(date_str):
     return date_str.replace('年', '-').replace('月', '-').replace('日', '').replace('昨天', last_day)
 
 
+def save_sh_stock_uid():
+    code_uid_df = sh_stock_sns_sse_info_api.sync_stock_uid()
+    mongodb_util.save_mongo(code_uid_df, db_name_constant.SSE_INFO_UID)
+
+
 # 同步所有互动问题
-def sync_all_interactive_questions():
+def sync_all_interactive_questions(symbol_list):
+    # 同步互动易映射
+    save_sh_stock_uid()
+
     real_time_quotes_all_stocks = east_money_stock_api.get_real_time_quotes_all_stocks()
     de_list_company_symbols = company_common_service_api.get_de_list_company()
     real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
         ~(real_time_quotes_all_stocks['symbol'].isin(de_list_company_symbols))]
     real_time_quotes_all_stocks = common_service_fun_api.classify_symbol(real_time_quotes_all_stocks)
+    real_time_quotes_all_stocks = real_time_quotes_all_stocks.sort_values(by=['chg'], ascending=False)
+    if symbol_list is not None:
+        real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
+            real_time_quotes_all_stocks['symbol'].isin(symbol_list)]
     for stock_one in real_time_quotes_all_stocks.itertuples():
         try:
             now_date = datetime.now()
             str_day = now_date.strftime('%Y-%m-%d')
             str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
             classification = stock_one.classification
+
             if classification in ['S', 'C']:
                 stock_irm_cninfo_df = get_stock_irm_cninfo_sz_api(stock_one.symbol)
+                time.sleep(1)
             elif classification in ['K', 'H']:
                 stock_irm_cninfo_df = get_stock_irm_cninfo_sh_api(stock_one.symbol)
+                time.sleep(1)
+
             else:
                 continue
             stock_irm_cninfo_df['sync_time'] = str_now_date
             stock_irm_cninfo_df['str_day'] = str_day
             stock_irm_cninfo_df.drop_duplicates('_id', keep='last', inplace=True)
+            stock_irm_cninfo_df.fillna("", inplace=True)
             mongodb_util.save_mongo(stock_irm_cninfo_df, db_name_constant.STOCK_INTERACTIVE_QUESTION)
-            logger.info("完成同步到:{}", stock_one.symbol)
+            logger.info("完成同步互动回答到:{}", stock_one.symbol)
         except Exception as e:
+            time.sleep(2)
             logger.error("同步互动问题出现异常:{},{}", stock_one.symbol, e)
 
 
 if __name__ == '__main__':
+    fail_symbol_list = ['000638', '002886', '688778', '688766', '688733', '688778', '688793', '688787']
     # get_stock_irm_cninfo_sh_api('603633')
-    sync_all_interactive_questions()
+    sync_all_interactive_questions(None)

mns_scheduler/zz_task/data_sync_task.py

@@ -326,7 +326,7 @@ def sync_new_high_risk_stocks():
 # 同步互动回答
 def sync_all_interactive_questions():
     logger.info('同步互动回答')
-    stock_irm_cninfo_service.sync_all_interactive_questions()
+    stock_irm_cninfo_service.sync_all_interactive_questions(None)
 
 
 # # 定义BlockingScheduler

{mns_scheduler-1.0.5.6.dist-info → mns_scheduler-1.0.5.9.dist-info}/METADATA

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.0.5.6
+Version: 1.0.5.9
 

{mns_scheduler-1.0.5.6.dist-info → mns_scheduler-1.0.5.9.dist-info}/RECORD

@@ -44,10 +44,10 @@ mns_scheduler/finance/test/fix_blask_list.py,sha256=sXmNcnfnjKHBylc27ysCCb9G4HYl
 mns_scheduler/hk/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/hk/hk_company_info_sync_service_api.py,sha256=JEfIl_up36b1UpDxfPhosP6_i2Lo17Ma_GHLytoMjWI,3661
 mns_scheduler/irm/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_scheduler/irm/stock_irm_cninfo_service.py,sha256=9_QFH5NvRYL_CYkcddiSnLnLKrKz1LweyjE0u7Pny_M,7110
+mns_scheduler/irm/stock_irm_cninfo_service.py,sha256=X0fMCUBP-eLYhZmhNU6yibUJ0bXuRhiQwiVuDda9BhU,7905
 mns_scheduler/irm/api/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py,sha256=Q7C5i_UOUmovaxkrVK5OqvsiY6FCsNDbYzlWjlhoDNA,4458
-mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py,sha256=C4-nk5vaACeOOCXSQMMwCxaZBjHc2PXmnGWZQdMruME,6169
+mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py,sha256=iXy_FukyyQ_ZcU1WcT_DSpZ8-aEtDrTEcyiiIlFzXho,4931
+mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py,sha256=6iZpdQ1pARchL3kV-oEVPDc9Ja0ciQBjBntVmxseeNE,6199
 mns_scheduler/k_line/__init__.py,sha256=ffZXFCLFdIwOsbxnw__u1MbQYh9yz7Bs8UMP6VF0X2M,161
 mns_scheduler/k_line/clean/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/k_line/clean/k_line_info_clean_impl.py,sha256=2xk7gzLX728PRTKUgx2sXoyPKDu-xGAvNpefgeBBd2w,22262
@@ -85,9 +85,9 @@ mns_scheduler/zt/today_high_chg_pool_sync_api.py,sha256=KfEp2Mu8dZKB8DBai4m0UpZw
 mns_scheduler/zt/zt_five_boards_sync_api.py,sha256=HfjPHKD99fU9c37kSenEX2_qNvFAjQGgy8ERuacSxwk,10916
 mns_scheduler/zt/zt_pool_sync_api.py,sha256=RrVAbU1u-HTqXF9BSwNlzIxMHrUgjNaLpDKFgXC1XuY,7604
 mns_scheduler/zz_task/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/zz_task/data_sync_task.py,sha256=CGRNqbP8AlU03qI74zq_usmy-1vx_Fc6E7dI77UWfMM,16629
+mns_scheduler/zz_task/data_sync_task.py,sha256=oC8J9dk1bk-WOskgFt0nnh5zGpCNdhkij4O9Gpi2qbU,16633
 mns_scheduler/zz_task/sync_realtime_quotes_task.py,sha256=DN3bq2XCDZC-PHlbD2NTog48bR44EruIEc2QVGKg7Tk,932
-mns_scheduler-1.0.5.6.dist-info/METADATA,sha256=oTtJgiXxHi49w487qTg5KxGgCsRXDawIrTWtCpkhzEI,64
-mns_scheduler-1.0.5.6.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-mns_scheduler-1.0.5.6.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
-mns_scheduler-1.0.5.6.dist-info/RECORD,,
+mns_scheduler-1.0.5.9.dist-info/METADATA,sha256=hQ7coztdl9cPKYVFPVRjiGkcAFjA-2w2AQSKdNoWrTQ,64
+mns_scheduler-1.0.5.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+mns_scheduler-1.0.5.9.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
+mns_scheduler-1.0.5.9.dist-info/RECORD,,