mns-scheduler 1.4.3.2__py3-none-any.whl → 1.4.5.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. mns_scheduler/{company_info/announce → auto_da_ban}/__init__.py +1 -1
  2. mns_scheduler/auto_da_ban/auto_da_ban_service.py +89 -0
  3. mns_scheduler/company_info/clean/company_info_clean_api.py +37 -16
  4. mns_scheduler/company_info/{base → common}/__init__.py +1 -1
  5. mns_scheduler/company_info/common/company_common_query_service.py +45 -0
  6. mns_scheduler/company_info/constant/company_constant_data.py +59 -49
  7. mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py +94 -40
  8. mns_scheduler/company_info/{remark → sync}/__init__.py +1 -1
  9. mns_scheduler/company_info/sync/company_info_set_service.py +208 -0
  10. mns_scheduler/company_info/sync/sync_company_info_task.py +203 -0
  11. mns_scheduler/company_info/task/__init__.py +7 -0
  12. mns_scheduler/company_info/{announce/company_announce_sync_service.py → task/company_announce_info_task.py} +25 -13
  13. mns_scheduler/company_info/task/company_base_info_task.py +64 -0
  14. mns_scheduler/company_info/{base/sync_company_product_area_industry.py → task/company_business_info_task.py} +33 -17
  15. mns_scheduler/company_info/task/company_hold_info_task.py +66 -0
  16. mns_scheduler/company_info/task/company_industry_info_task.py +167 -0
  17. mns_scheduler/company_info/task/company_total_task.py +69 -0
  18. mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py +3 -3
  19. mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py +1 -1
  20. mns_scheduler/db/script/sync/remote_data_sync_to_local.py +11 -3
  21. mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py +7 -57
  22. mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py +14 -25
  23. mns_scheduler/irm/stock_irm_cninfo_service.py +34 -26
  24. mns_scheduler/irm/stock_question_id_service.py +169 -0
  25. mns_scheduler/self_choose/ths_self_choose_service.py +53 -25
  26. mns_scheduler/trade/auto_login/trader_auto_service.py +2 -1
  27. mns_scheduler/trade/task/trader_task_service.py +13 -2
  28. mns_scheduler/trade/tfp/stock_tfp_info_sync.py +1 -1
  29. mns_scheduler/zb/stock_zb_pool_sync.py +1 -1
  30. mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py +80 -104
  31. mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py +1 -1
  32. mns_scheduler/zt/zt_pool/update_null_zt_reason_api.py +4 -1
  33. mns_scheduler/zz_task/compensation/compensate_task_one_day.py +1 -1
  34. mns_scheduler/zz_task/data_sync_task.py +45 -37
  35. {mns_scheduler-1.4.3.2.dist-info → mns_scheduler-1.4.5.7.dist-info}/METADATA +1 -1
  36. {mns_scheduler-1.4.3.2.dist-info → mns_scheduler-1.4.5.7.dist-info}/RECORD +38 -31
  37. mns_scheduler/company_info/base/sync_company_base_info_api.py +0 -531
  38. mns_scheduler/company_info/base/sync_company_hold_info_api.py +0 -37
  39. mns_scheduler/company_info/remark/company_remark_info_sync.py +0 -46
  40. {mns_scheduler-1.4.3.2.dist-info → mns_scheduler-1.4.5.7.dist-info}/WHEEL +0 -0
  41. {mns_scheduler-1.4.3.2.dist-info → mns_scheduler-1.4.5.7.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,66 @@
1
+ import sys
2
+ import os
3
+
4
+ file_path = os.path.abspath(__file__)
5
+ end = file_path.index('mns') + 16
6
+ project_path = file_path[0:end]
7
+ sys.path.append(project_path)
8
+ import mns_common.api.ths.company.ths_company_info_api as ths_company_info_api
9
+ import mns_common.component.cookie.cookie_info_service as cookie_info_service
10
+ import mns_common.utils.data_frame_util as data_frame_util
11
+ from mns_common.db.MongodbUtil import MongodbUtil
12
+ import mns_common.constant.db_name_constant as db_name_constant
13
+ from datetime import datetime
14
+ from loguru import logger
15
+ from mns_scheduler.company_info.common.company_common_query_service import get_company_info
16
+ import time
17
+
18
+ mongodb_util = MongodbUtil('27017')
19
+
20
+
21
+ # 同步公司控股子公司信息任务
22
+ def sync_one_company_hold_info(symbol):
23
+ try:
24
+ ths_cookie = cookie_info_service.get_ths_cookie()
25
+ company_hold_info_df = ths_company_info_api.get_company_hold_info(symbol, ths_cookie)
26
+
27
+ now_date = datetime.now()
28
+ sync_str_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
29
+
30
+ if data_frame_util.is_not_empty(company_hold_info_df):
31
+ query_exist = {'symbol': symbol}
32
+ exist_company_holding_info_df = mongodb_util.find_query_data(db_name_constant.COMPANY_HOLDING_INFO,
33
+ query_exist)
34
+ if data_frame_util.is_not_empty(exist_company_holding_info_df):
35
+ # 作废不在参股控股的子公司
36
+ invalid_symbol_df = exist_company_holding_info_df.loc[
37
+ ~exist_company_holding_info_df['symbol'].isin(list(company_hold_info_df['symbol']))]
38
+ invalid_symbol_df['valid'] = False
39
+
40
+ company_hold_info_df['valid'] = True
41
+ company_hold_info_df['_id'] = company_hold_info_df['symbol'] + '_' + company_hold_info_df[
42
+ 'holding_company']
43
+ company_hold_info_df['sync_str_date'] = sync_str_date
44
+ mongodb_util.save_mongo(company_hold_info_df, db_name_constant.COMPANY_HOLDING_INFO)
45
+ else:
46
+ logger.warning("同步控股子公司为空:{}", symbol)
47
+
48
+ except BaseException as e:
49
+ logger.error("同步公司控股子公司信息:{},{}", symbol, e)
50
+
51
+
52
+ def sync_all_company_hold_info_task(symbol_list):
53
+ all_company_info_df = get_company_info()
54
+ if len(symbol_list) > 0:
55
+ all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
56
+ for stock_one in all_company_info_df.itertuples():
57
+ try:
58
+ sync_one_company_hold_info(stock_one.symbol)
59
+ time.sleep(1)
60
+ except BaseException as e:
61
+ logger.error("同步控股子公司信息异常:{},{}", stock_one.symbol, e)
62
+ time.sleep(5)
63
+
64
+
65
+ if __name__ == '__main__':
66
+ sync_all_company_hold_info_task([])
@@ -0,0 +1,167 @@
1
+ import sys
2
+ import os
3
+
4
+ file_path = os.path.abspath(__file__)
5
+ end = file_path.index('mns') + 16
6
+ project_path = file_path[0:end]
7
+ sys.path.append(project_path)
8
+ import mns_common.api.ths.company.ths_company_info_web as ths_company_info_web
9
+ from mns_common.db.MongodbUtil import MongodbUtil
10
+ import mns_common.constant.db_name_constant as db_name_constant
11
+ from loguru import logger
12
+ from mns_scheduler.company_info.common.company_common_query_service import get_company_info
13
+ import time
14
+ import mns_common.utils.data_frame_util as data_frame_util
15
+ import mns_common.component.common_service_fun_api as common_service_fun_api
16
+
17
+ mongodb_util = MongodbUtil('27017')
18
+
19
+
20
+ def sync_company_industry_info_task(symbol_list):
21
+ all_company_info_df = get_company_info()
22
+ if len(symbol_list) > 0:
23
+ all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
24
+
25
+ all_company_info_df = common_service_fun_api.classify_symbol(all_company_info_df)
26
+ fail_list = []
27
+ for stock_one in all_company_info_df.itertuples():
28
+ try:
29
+ tag = sync_one_company_industry_info(stock_one.symbol, stock_one.classification)
30
+ if bool(1 - tag):
31
+ fail_list.append(stock_one.symbol)
32
+ time.sleep(0.5)
33
+ except BaseException as e:
34
+ time.sleep(1)
35
+ logger.error("同步公司行业信息发生异常:{},{}", stock_one.symbol, e)
36
+ fail_list.append(stock_one.symbol)
37
+ sync_number = 1
38
+ while len(fail_list) > 0 and sync_number < 10:
39
+ for symbol in fail_list:
40
+ try:
41
+ company_info_one_df = all_company_info_df.loc[all_company_info_df['symbol'] == symbol]
42
+ classification = list(company_info_one_df['classification'])[0]
43
+ tag = sync_one_company_industry_info(symbol, classification)
44
+ if tag and symbol in fail_list:
45
+ fail_list.remove(symbol)
46
+ time.sleep(2)
47
+ except BaseException as e:
48
+ time.sleep(3)
49
+ logger.error("同步公司行业信息发生异常:{},{}", symbol, e)
50
+ sync_number = sync_number + 1
51
+
52
+
53
+ def sync_one_company_industry_info(symbol, classification):
54
+ if classification in ['H', 'K']:
55
+ market_id = '17'
56
+ elif classification in ['S', 'C']:
57
+ market_id = '33'
58
+ elif classification in ['X']:
59
+ market_id = '151'
60
+
61
+ company_industry_info = ths_company_info_web.get_company_info_detail(symbol, market_id)
62
+ if data_frame_util.is_empty(company_industry_info):
63
+ return False
64
+
65
+ company_industry_info['first_industry_code'] = company_industry_info['hycode'].apply(
66
+ lambda x: x[1:3] + '0000')
67
+ company_industry_info['second_industry_code'] = company_industry_info['hy2code'].apply(
68
+ lambda x: x[1:5] + '00')
69
+ company_industry_info['third_industry_code'] = company_industry_info['hy3code'].apply(
70
+ lambda x: x[1:7])
71
+
72
+ company_industry_info['first_sw_industry'] = company_industry_info['hy']
73
+ company_industry_info['second_sw_industry'] = company_industry_info['hy2']
74
+ company_industry_info['third_sw_industry'] = company_industry_info['hy3']
75
+ del company_industry_info['hy']
76
+ del company_industry_info['hy2']
77
+ del company_industry_info['hy3']
78
+ del company_industry_info['hycode']
79
+ del company_industry_info['hy2code']
80
+ del company_industry_info['hy3code']
81
+
82
+ company_industry_info['_id'] = symbol
83
+ company_industry_info['symbol'] = symbol
84
+ mongodb_util.save_mongo(company_industry_info, db_name_constant.COMPANY_INDUSTRY_INFO)
85
+ # 保存股票申万行业原始信息
86
+ save_sw_data(company_industry_info)
87
+ return True
88
+
89
+
90
+ # 保存申万行业分类
91
+ def save_sw_data(company_industry_info):
92
+ first_sw_info = company_industry_info[[
93
+ 'first_sw_industry',
94
+ 'first_industry_code'
95
+ ]].copy()
96
+
97
+ first_sw_info.loc[:, "industry_code"] = first_sw_info['first_industry_code']
98
+ first_sw_info.loc[:, "_id"] = first_sw_info['first_industry_code']
99
+ first_sw_info.loc[:, "second_sw_industry"] = 0
100
+ first_sw_info.loc[:, "third_sw_industry"] = 0
101
+ first_sw_info.loc[:, "second_industry_code"] = 0
102
+ first_sw_info.loc[:, "third_industry_code"] = 0
103
+ first_sw_info = first_sw_info[[
104
+ "_id",
105
+ "industry_code",
106
+ 'first_industry_code',
107
+ 'first_sw_industry',
108
+ 'second_industry_code',
109
+ 'second_sw_industry',
110
+ 'third_industry_code',
111
+ 'third_sw_industry'
112
+ ]]
113
+ mongodb_util.save_mongo(first_sw_info, 'sw_industry')
114
+
115
+ second_sw_info = company_industry_info[[
116
+ 'first_industry_code',
117
+ 'first_sw_industry',
118
+ 'second_sw_industry',
119
+ 'second_industry_code',
120
+ ]].copy()
121
+
122
+ second_sw_info.loc[:, "industry_code"] = second_sw_info['second_industry_code']
123
+ second_sw_info.loc[:, "_id"] = second_sw_info['industry_code']
124
+
125
+ second_sw_info.loc[:, "third_sw_industry"] = 0
126
+ second_sw_info.loc[:, "third_sw_industry"] = 0
127
+ second_sw_info.loc[:, "third_industry_code"] = 0
128
+ second_sw_info = second_sw_info[[
129
+ "_id",
130
+ "industry_code",
131
+ 'first_industry_code',
132
+ 'first_sw_industry',
133
+ 'second_industry_code',
134
+ 'second_sw_industry',
135
+ 'third_industry_code',
136
+ 'third_sw_industry'
137
+ ]]
138
+ mongodb_util.save_mongo(second_sw_info, 'sw_industry')
139
+
140
+ third_sw_info = company_industry_info[[
141
+ 'first_industry_code',
142
+ 'first_sw_industry',
143
+ 'second_industry_code',
144
+ 'second_sw_industry',
145
+ 'third_industry_code',
146
+ 'third_sw_industry'
147
+ ]].copy()
148
+
149
+ third_sw_info.loc[:, "industry_code"] = third_sw_info['third_industry_code']
150
+
151
+ third_sw_info.loc[:, "_id"] = third_sw_info['industry_code']
152
+
153
+ third_sw_info = third_sw_info[[
154
+ "_id",
155
+ "industry_code",
156
+ 'first_industry_code',
157
+ 'first_sw_industry',
158
+ 'second_industry_code',
159
+ 'second_sw_industry',
160
+ 'third_industry_code',
161
+ 'third_sw_industry'
162
+ ]]
163
+ mongodb_util.save_mongo(third_sw_info, 'sw_industry')
164
+
165
+
166
+ if __name__ == '__main__':
167
+ sync_company_industry_info_task(['688795'])
@@ -0,0 +1,69 @@
1
+ import sys
2
+ import os
3
+
4
+ file_path = os.path.abspath(__file__)
5
+ end = file_path.index('mns') + 16
6
+ project_path = file_path[0:end]
7
+ sys.path.append(project_path)
8
+ import mns_scheduler.company_info.task.company_industry_info_task as company_industry_info_task
9
+ import mns_scheduler.company_info.task.company_business_info_task as company_business_info_task
10
+ import mns_scheduler.company_info.task.company_base_info_task as company_base_info_task
11
+ import mns_scheduler.company_info.task.company_hold_info_task as company_hold_info_task
12
+ import mns_scheduler.company_info.task.company_announce_info_task as company_announce_info_task
13
+ from mns_scheduler.company_info.common.company_common_query_service import get_company_info
14
+ from datetime import datetime
15
+ import mns_common.utils.data_frame_util as data_frame_util
16
+ from loguru import logger
17
+
18
+
19
+ def sync_all_company_task():
20
+ # 同步公司行业信息
21
+ company_industry_info_task.sync_company_industry_info_task([])
22
+ logger.info("[全量]同步公司[行业]信息完成")
23
+ # 同步公司业务信息
24
+ company_business_info_task.sync_company_business_task([])
25
+ logger.info("[全量]同步公司[业务]信息完成")
26
+ # 同步公司基本信息
27
+ company_base_info_task.sync_company_base_info_task([])
28
+ logger.info("[全量]同步公司[基本]信息完成")
29
+ # 同步公司控股信息
30
+ company_hold_info_task.sync_all_company_hold_info_task([])
31
+ logger.info("[全量]同步公司[控股]信息完成")
32
+ # 同步公司公告信息
33
+ company_announce_info_task.sync_company_announce_task([])
34
+ logger.info("[全量]同步公司[公告]信息完成")
35
+
36
+
37
+ def sync_new_stock_company_task():
38
+ all_company_info_df = get_company_info()
39
+
40
+ now_date = datetime.now()
41
+ str_day = now_date.strftime('%Y-%m-%d')
42
+ # 去掉横线并转换为整数
43
+ now_day = int(str_day.replace('-', ''))
44
+
45
+ new_company_info_df = all_company_info_df.loc[
46
+ (all_company_info_df['list_date'] == 19890604) | (all_company_info_df['list_date'] >= now_day)]
47
+ if data_frame_util.is_empty(new_company_info_df):
48
+ return
49
+ symbol_list = list(new_company_info_df['symbol'])
50
+ # 同步公司行业信息
51
+ company_industry_info_task.sync_company_industry_info_task(symbol_list)
52
+ logger.info("[新股]同步公司[行业]信息完成")
53
+ # 同步公司业务信息
54
+ company_business_info_task.sync_company_business_task(symbol_list)
55
+ logger.info("[新股]同步公司[业务]信息完成")
56
+ # 同步公司基本信息
57
+ company_base_info_task.sync_company_base_info_task(symbol_list)
58
+ logger.info("[新股]同步公司[基本]信息完成")
59
+ # 同步公司控股信息
60
+ company_hold_info_task.sync_all_company_hold_info_task(symbol_list)
61
+ logger.info("[新股]同步公司[控股]信息完成")
62
+ # 同步公司公告信息
63
+ company_announce_info_task.sync_company_announce_task(symbol_list)
64
+ logger.info("[新股]同步公司[公告]信息完成")
65
+
66
+
67
+ if __name__ == '__main__':
68
+ sync_new_stock_company_task()
69
+ sync_all_company_task()
@@ -11,7 +11,7 @@ project_path = file_path[0:end]
11
11
  sys.path.append(project_path)
12
12
  mongodb_util = MongodbUtil('27017')
13
13
  import mns_common.api.msg.push_msg_api as push_msg_api
14
- import mns_scheduler.company_info.base.sync_company_base_info_api as company_info_sync_api
14
+ import mns_scheduler.company_info.sync.sync_company_info_task as sync_company_info_task
15
15
  import mns_scheduler.company_info.clean.company_info_clean_api as company_info_clean_api
16
16
  import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
17
17
  import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
@@ -192,8 +192,8 @@ def update_long_short(new_concept_symbol_df, exist_concept_detail):
192
192
  def update_company_info(new_concept_symbol_df):
193
193
  if new_concept_symbol_df.shape[0] > 0:
194
194
  symbol_list = list(new_concept_symbol_df['symbol'])
195
- company_info_sync_api.sync_company_base_info(symbol_list)
196
- company_info_clean_api.clean_company_info(None)
195
+ sync_company_info_task.sync_company_base_info(symbol_list)
196
+ company_info_clean_api.clean_company_info(symbol_list)
197
197
  # 公司缓存信息清除
198
198
  company_common_service_api.company_info_industry_cache_clear()
199
199
 
@@ -183,4 +183,4 @@ def sync_symbol_all_concept(symbol):
183
183
 
184
184
 
185
185
  if __name__ == '__main__':
186
- sync_symbol_all_concept('831175')
186
+ sync_symbol_all_concept(None)
@@ -15,6 +15,10 @@ remote_mongodb_util = RemoteMongodbUtil('27017')
15
15
  local_mongodb_util = LocalMongodbUtil('27017')
16
16
 
17
17
  col_list = [
18
+ 'zt_reason_analysis',
19
+ 'today_exclude_stocks',
20
+ 'today_self_choose_stock',
21
+ 'main_line_detail',
18
22
  'em_a_stock_info',
19
23
  'em_etf_info',
20
24
  'em_kzz_info',
@@ -38,7 +42,8 @@ col_list = [
38
42
  'ths_stock_concept_detail',
39
43
  'stock_high_chg_pool',
40
44
  'today_new_concept_list',
41
- 'ths_stock_concept_detail_app'
45
+ 'ths_stock_concept_detail_app',
46
+
42
47
  ]
43
48
 
44
49
 
@@ -91,8 +96,11 @@ def sync_open_data():
91
96
 
92
97
 
93
98
  if __name__ == '__main__':
94
- sync_zt_data('2025-09-17')
95
- # sync_zt_data('2025-09-02')
99
+ sync_zt_data('2025-12-21')
100
+ sync_zt_data('2025-12-23')
101
+ sync_zt_data('2025-12-24')
102
+ sync_zt_data('2025-12-25')
103
+ sync_zt_data('2025-12-26')
96
104
  remote_data()
97
105
  # sync_zt_data('2025-08-26')
98
106
  # sync_zt_data('2025-08-25')
@@ -13,64 +13,17 @@ Desc: 上证e互动-提问与回答
13
13
  https://sns.sseinfo.com/
14
14
  """
15
15
  import warnings
16
- from functools import lru_cache
17
-
18
16
  import pandas as pd
19
17
  import requests
20
18
  from bs4 import BeautifulSoup
21
-
22
19
  from mns_common.db.MongodbUtil import MongodbUtil
23
- import mns_common.constant.db_name_constant as db_name_constant
24
20
 
25
21
  mongodb_util = MongodbUtil('27017')
26
- from tqdm import tqdm
27
-
28
-
29
- @lru_cache()
30
- def sync_stock_uid() -> pd.DataFrame:
31
- """
32
- 上证e互动-代码ID映射
33
- https://sns.sseinfo.com/list/company.do
34
- :return: 代码ID映射
35
- :rtype: str
36
- """
37
- url = "https://sns.sseinfo.com/allcompany.do"
38
- data = {
39
- "code": "0",
40
- "order": "2",
41
- "areaId": "0",
42
- "page": "1",
43
- }
44
- uid_list = list()
45
- code_list = list()
46
- for page in tqdm(range(1, 74), leave=False):
47
- data.update({"page": page})
48
- r = requests.post(url, data=data)
49
- data_json = r.json()
50
- soup = BeautifulSoup(data_json["content"], "lxml")
51
- soup.find_all("a", attrs={"rel": "tag"})
52
- uid_list.extend(
53
- [item["uid"] for item in soup.find_all("a", attrs={"rel": "tag"})]
54
- )
55
- code_list.extend(
56
- [
57
- item.find("img")["src"].split("?")[0].split("/")[-1].split(".")[0]
58
- for item in soup.find_all("a", attrs={"rel": "tag"})
59
- ]
60
- )
61
- code_uid_df = pd.DataFrame()
62
- code_uid_df['symbol'] = code_list
63
- code_uid_df['uid'] = uid_list
64
- code_uid_df['_id'] = uid_list
65
- return code_uid_df
66
-
67
-
68
- @lru_cache()
69
- def get_stock_uid() -> pd.DataFrame:
70
- return mongodb_util.find_all_data(db_name_constant.SSE_INFO_UID)
71
22
 
72
23
 
73
- def stock_sns_sse_info(symbol: str = "603119") -> pd.DataFrame:
24
+ def stock_sns_sse_info(org_ask_id,
25
+ page,
26
+ page_size) -> pd.DataFrame:
74
27
  """
75
28
  上证e互动-提问与回答
76
29
  https://sns.sseinfo.com/company.do?uid=65
@@ -80,16 +33,13 @@ def stock_sns_sse_info(symbol: str = "603119") -> pd.DataFrame:
80
33
  :rtype: str
81
34
  """
82
35
 
83
- stock_uid_df = get_stock_uid()
84
- stock_uid_df = stock_uid_df.loc[stock_uid_df['symbol'] == symbol]
85
- uid = list(stock_uid_df['uid'])[0]
86
36
  url = "https://sns.sseinfo.com/ajax/userfeeds.do"
87
37
  params = {
88
38
  "typeCode": "company",
89
39
  "type": "11",
90
- "pageSize": "100",
91
- "uid": uid,
92
- "page": "1",
40
+ "pageSize": str(page_size),
41
+ "uid": org_ask_id,
42
+ "page": str(page),
93
43
  }
94
44
  big_df = pd.DataFrame()
95
45
  page = 1
@@ -164,5 +114,5 @@ def stock_sns_sse_info(symbol: str = "603119") -> pd.DataFrame:
164
114
 
165
115
 
166
116
  if __name__ == "__main__":
167
- stock_sns_sse_info_df = stock_sns_sse_info(symbol="688787")
117
+ stock_sns_sse_info_df = stock_sns_sse_info('308', '2', '100')
168
118
  print(stock_sns_sse_info_df)
@@ -15,27 +15,18 @@ https://irm.cninfo.com.cn/
15
15
 
16
16
  import pandas as pd
17
17
  import requests
18
+ import time
18
19
 
19
20
 
20
- def _fetch_org_id(symbol: str = "000001") -> str:
21
- """
22
- 股票-互动易-组织代码
23
- https://irm.cninfo.com.cn/
24
- :return: 组织代码
25
- :rtype: str
26
- """
27
- url = "https://irm.cninfo.com.cn/newircs/index/queryKeyboardInfo"
28
- params = {"_t": "1691144074"}
29
- data = {"keyWord": symbol}
30
- r = requests.post(url, params=params, data=data)
31
- data_json = r.json()
32
- org_id = data_json["data"][0]["secid"]
33
- return org_id
34
-
35
-
36
- # 获取股票所有问题
21
+ # 获取单个股票所有问题
22
+ # todo https://irm.cninfo.com.cn/ircs/company/companyDetail?stockcode=000002&orgId=gssz0000002
23
+ def stock_irm_cninfo(symbol,
24
+ org_ask_id,
25
+ page,
26
+ page_size) -> pd.DataFrame:
27
+ # 毫秒级时间戳字符串
28
+ current_ms_timestamp_str = str(int(time.time() * 1000))
37
29
 
38
- def stock_irm_cninfo(symbol: str = "002594") -> pd.DataFrame:
39
30
  """
40
31
  互动易-提问
41
32
  https://irm.cninfo.com.cn/ircs/question/questionDetail?questionId=1515236357817618432
@@ -46,19 +37,17 @@ def stock_irm_cninfo(symbol: str = "002594") -> pd.DataFrame:
46
37
  """
47
38
  url = "https://irm.cninfo.com.cn/newircs/company/question"
48
39
  params = {
49
- "_t": "1691142650",
40
+ "_t": current_ms_timestamp_str,
50
41
  "stockcode": symbol,
51
- "orgId": _fetch_org_id(symbol),
52
- "pageSize": "100",
53
- "pageNum": "1",
42
+ "orgId": org_ask_id,
43
+ "pageSize": page_size,
44
+ "pageNum": page,
54
45
  "keyWord": "",
55
46
  "startDay": "",
56
47
  "endDay": "",
57
48
  }
58
49
 
59
50
  big_df = pd.DataFrame()
60
- page = 1
61
- params.update({"pageNum": page})
62
51
  r = requests.post(url, params=params)
63
52
  data_json = r.json()
64
53
  temp_df = pd.DataFrame(data_json["rows"])
@@ -195,7 +184,7 @@ def stock_irm_ans_cninfo(symbol: str = "1513586704097333248") -> pd.DataFrame:
195
184
 
196
185
 
197
186
  if __name__ == "__main__":
198
- stock_irm_cninfo_df = stock_irm_cninfo(symbol="002594")
187
+ stock_irm_cninfo_df = stock_irm_cninfo(symbol="000002", org_ask_id='gssz0000002', page=2, pageSize=100)
199
188
  print(stock_irm_cninfo_df)
200
189
 
201
190
  stock_irm_ans_cninfo_df = stock_irm_ans_cninfo(symbol="1495108801386602496")
@@ -5,7 +5,7 @@ file_path = os.path.abspath(__file__)
5
5
  end = file_path.index('mns') + 17
6
6
  project_path = file_path[0:end]
7
7
  sys.path.append(project_path)
8
- import akshare as ak
8
+
9
9
  import mns_common.utils.data_frame_util as data_frame_util
10
10
  import pandas as pd
11
11
  import mns_common.component.em.em_stock_info_api as em_stock_info_api
@@ -14,22 +14,25 @@ from loguru import logger
14
14
  from datetime import datetime
15
15
  import time
16
16
  import mns_common.utils.date_handle_util as date_handle_util
17
-
18
17
  import mns_common.constant.db_name_constant as db_name_constant
19
18
  import mns_common.component.common_service_fun_api as common_service_fun_api
20
19
  import mns_scheduler.irm.api.sh_stock_sns_sse_info_api as sh_stock_sns_sse_info_api
21
20
  import mns_scheduler.irm.api.sz_stock_sns_sse_info_api as sz_stock_sns_sse_info_api
22
-
23
21
  from mns_common.db.MongodbUtil import MongodbUtil
22
+ import mns_scheduler.irm.stock_question_id_service as stock_question_id_service
24
23
 
25
24
  mongodb_util = MongodbUtil('27017')
26
25
 
27
26
 
28
- # 获取股票提问 互动易-提问 深交所
27
+ # 获取股票提问 互动易-提问 todo 深交所 可以拉取未回答的,但是最拉到最近三个月
29
28
  def get_stock_irm_cninfo_sz_api(symbol):
30
29
  try:
30
+ org_ask_id = stock_question_id_service.get_one_sz_symbol_org_id(symbol)
31
31
  # 获取一页
32
- stock_irm_cninfo_df = sz_stock_sns_sse_info_api.stock_irm_cninfo(symbol)
32
+ stock_irm_cninfo_df = sz_stock_sns_sse_info_api.stock_irm_cninfo(symbol,
33
+ org_ask_id,
34
+ 1,
35
+ 100)
33
36
  # 获取全页
34
37
  # stock_irm_cninfo_df = ak.stock_irm_cninfo(symbol)
35
38
  except Exception as e:
@@ -66,11 +69,15 @@ def get_stock_irm_cninfo_sz_api(symbol):
66
69
  return stock_irm_cninfo_df
67
70
 
68
71
 
69
- # 获取股票提问 互动易-提问 上交所
72
+ # 获取股票提问 互动易-提问 todo 上交所
70
73
  def get_stock_irm_cninfo_sh_api(symbol):
71
74
  try:
72
75
  # 获取一页
73
- stock_sns_sse_info_df = sh_stock_sns_sse_info_api.stock_sns_sse_info(symbol)
76
+ org_ask_id = stock_question_id_service.get_one_sh_symbol_org_id(symbol)
77
+
78
+ stock_sns_sse_info_df = sh_stock_sns_sse_info_api.stock_sns_sse_info(org_ask_id,
79
+ 1,
80
+ 100)
74
81
  # 获取全页
75
82
  # stock_sns_sse_info_df = ak.stock_sns_sseinfo(symbol)
76
83
  except Exception as e:
@@ -101,10 +108,21 @@ def get_stock_irm_cninfo_sh_api(symbol):
101
108
  "answer_time",
102
109
  "questioner",
103
110
  "source"]]
111
+ now_date = datetime.now()
112
+ str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
113
+ stock_sns_sse_info_df.loc[
114
+ stock_sns_sse_info_df['answer_time'].str.contains('小时前', na=False), 'answer_time'] = str_now_date
115
+ stock_sns_sse_info_df.loc[
116
+ stock_sns_sse_info_df['answer_time'].str.contains('分钟前', na=False), 'answer_time'] = str_now_date
117
+ stock_sns_sse_info_df.loc[
118
+ stock_sns_sse_info_df['question_time'].str.contains('小时前', na=False), 'question_time'] = str_now_date
119
+ stock_sns_sse_info_df.loc[
120
+ stock_sns_sse_info_df['question_time'].str.contains('分钟前', na=False), 'answer_time'] = str_now_date
104
121
 
105
122
  return stock_sns_sse_info_df
106
123
 
107
124
 
125
+ # 格式化 时间
108
126
  def replace_date_format(date_str):
109
127
  now_date = datetime.now()
110
128
  str_day = now_date.strftime('%Y-%m-%d')
@@ -113,26 +131,20 @@ def replace_date_format(date_str):
113
131
  return date_str.replace('年', '-').replace('月', '-').replace('日', '').replace('昨天', last_day)
114
132
 
115
133
 
116
- def save_sh_stock_uid():
117
- code_uid_df = sh_stock_sns_sse_info_api.sync_stock_uid()
118
- mongodb_util.save_mongo(code_uid_df, db_name_constant.SSE_INFO_UID)
119
-
120
-
121
134
  # 同步所有互动问题
122
135
  def sync_symbols_interactive_questions(symbol_list):
123
- # 同步互动易映射
124
- if len(symbol_list) == 0:
125
- save_sh_stock_uid()
126
136
  real_time_quotes_all_stocks = em_stock_info_api.get_a_stock_info()
127
137
  de_list_company_symbols = company_common_service_api.get_de_list_company()
128
138
  real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
129
139
  ~(real_time_quotes_all_stocks['symbol'].isin(de_list_company_symbols))]
130
140
  real_time_quotes_all_stocks = common_service_fun_api.classify_symbol(real_time_quotes_all_stocks)
131
- real_time_quotes_all_stocks = real_time_quotes_all_stocks.sort_values(by=['chg'], ascending=False)
141
+ real_time_quotes_all_stocks = real_time_quotes_all_stocks.sort_values(by=['amount'], ascending=False)
132
142
  fail_symbol_list = []
133
143
  if len(symbol_list) != 0:
134
144
  real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
135
145
  real_time_quotes_all_stocks['symbol'].isin(symbol_list)]
146
+
147
+ real_time_quotes_all_stocks = real_time_quotes_all_stocks.reset_index(drop=True)
136
148
  for stock_one in real_time_quotes_all_stocks.itertuples():
137
149
  try:
138
150
  now_date = datetime.now()
@@ -155,14 +167,7 @@ def sync_symbols_interactive_questions(symbol_list):
155
167
  stock_irm_cninfo_df['str_day'] = str_day
156
168
  stock_irm_cninfo_df.drop_duplicates('_id', keep='last', inplace=True)
157
169
  stock_irm_cninfo_df.fillna("", inplace=True)
158
- stock_irm_cninfo_df.loc[
159
- stock_irm_cninfo_df['answer_time'].str.contains('小时前', na=False), 'answer_time'] = str_now_date
160
- stock_irm_cninfo_df.loc[
161
- stock_irm_cninfo_df['answer_time'].str.contains('分钟前', na=False), 'answer_time'] = str_now_date
162
- stock_irm_cninfo_df.loc[
163
- stock_irm_cninfo_df['question_time'].str.contains('小时前', na=False), 'question_time'] = str_now_date
164
- stock_irm_cninfo_df.loc[
165
- stock_irm_cninfo_df['question_time'].str.contains('分钟前', na=False), 'answer_time'] = str_now_date
170
+
166
171
  stock_irm_cninfo_df['valid'] = True
167
172
  stock_irm_cninfo_df['answer_content'].fillna('')
168
173
  # 保存新增数据
@@ -174,6 +179,7 @@ def sync_symbols_interactive_questions(symbol_list):
174
179
  logger.error("同步互动问题出现异常:{},{}", stock_one.symbol, e)
175
180
 
176
181
 
182
+ # 保存新增回答数据
177
183
  def save_new_data(stock_irm_cninfo_df):
178
184
  if data_frame_util.is_empty(stock_irm_cninfo_df):
179
185
  return None
@@ -193,8 +199,10 @@ def save_new_data(stock_irm_cninfo_df):
193
199
 
194
200
 
195
201
  if __name__ == '__main__':
202
+ get_stock_irm_cninfo_sz_api('000002')
196
203
  sync_symbols_interactive_questions([])
197
- get_stock_irm_cninfo_sh_api('688778')
198
- fail_symbol_list_01 = ['000638', '002886', '688778', '688766', '688733', '688778', '688793', '688787']
204
+ # sync_symbols_interactive_questions([])
205
+ # get_stock_irm_cninfo_sh_api('688778')
206
+ # fail_symbol_list_01 = ['000638', '002886', '688778', '688766', '688733', '688778', '688793', '688787']
199
207
  # get_stock_irm_cninfo_sh_api('603633')
200
208
  # sync_symbols_interactive_questions(None)