mns-scheduler 1.4.3.3__py3-none-any.whl → 1.4.4.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mns-scheduler might be problematic. Click here for more details.
- mns_scheduler/{company_info/announce → auto_da_ban}/__init__.py +1 -1
- mns_scheduler/auto_da_ban/auto_da_ban_service.py +87 -0
- mns_scheduler/company_info/clean/company_info_clean_api.py +37 -16
- mns_scheduler/company_info/{base → common}/__init__.py +1 -1
- mns_scheduler/company_info/common/company_common_query_service.py +45 -0
- mns_scheduler/company_info/constant/company_constant_data.py +60 -49
- mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py +1 -0
- mns_scheduler/company_info/{remark → sync}/__init__.py +1 -1
- mns_scheduler/company_info/sync/company_info_set_service.py +208 -0
- mns_scheduler/company_info/sync/sync_company_info_task.py +203 -0
- mns_scheduler/company_info/task/__init__.py +7 -0
- mns_scheduler/company_info/{announce/company_announce_sync_service.py → task/company_announce_info_task.py} +25 -13
- mns_scheduler/company_info/task/company_base_info_task.py +64 -0
- mns_scheduler/company_info/{base/sync_company_product_area_industry.py → task/company_business_info_task.py} +33 -17
- mns_scheduler/company_info/task/company_hold_info_task.py +66 -0
- mns_scheduler/company_info/task/company_industry_info_task.py +167 -0
- mns_scheduler/company_info/task/company_total_task.py +69 -0
- mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py +3 -3
- mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py +1 -1
- mns_scheduler/db/script/sync/remote_data_sync_to_local.py +7 -2
- mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py +7 -57
- mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py +14 -25
- mns_scheduler/irm/stock_irm_cninfo_service.py +32 -26
- mns_scheduler/irm/stock_question_id_service.py +159 -0
- mns_scheduler/trade/auto_login/trader_auto_service.py +2 -1
- mns_scheduler/trade/task/trader_task_service.py +13 -2
- mns_scheduler/zb/stock_zb_pool_sync.py +1 -1
- mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py +217 -126
- mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py +1 -1
- mns_scheduler/zt/zt_pool/update_null_zt_reason_api.py +4 -1
- mns_scheduler/zz_task/compensation/compensate_task_one_day.py +1 -1
- mns_scheduler/zz_task/data_sync_task.py +30 -36
- {mns_scheduler-1.4.3.3.dist-info → mns_scheduler-1.4.4.5.dist-info}/METADATA +1 -1
- {mns_scheduler-1.4.3.3.dist-info → mns_scheduler-1.4.4.5.dist-info}/RECORD +36 -29
- mns_scheduler/company_info/base/sync_company_base_info_api.py +0 -531
- mns_scheduler/company_info/base/sync_company_hold_info_api.py +0 -37
- mns_scheduler/company_info/remark/company_remark_info_sync.py +0 -46
- {mns_scheduler-1.4.3.3.dist-info → mns_scheduler-1.4.4.5.dist-info}/WHEEL +0 -0
- {mns_scheduler-1.4.3.3.dist-info → mns_scheduler-1.4.4.5.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
import mns_common.api.ths.company.ths_company_info_web as ths_company_info_web
|
|
9
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
10
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
11
|
+
from loguru import logger
|
|
12
|
+
from mns_scheduler.company_info.common.company_common_query_service import get_company_info
|
|
13
|
+
import time
|
|
14
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
15
|
+
import mns_common.component.common_service_fun_api as common_service_fun_api
|
|
16
|
+
|
|
17
|
+
mongodb_util = MongodbUtil('27017')
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def sync_company_industry_info_task(symbol_list):
|
|
21
|
+
all_company_info_df = get_company_info()
|
|
22
|
+
if len(symbol_list) > 0:
|
|
23
|
+
all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
|
|
24
|
+
|
|
25
|
+
all_company_info_df = common_service_fun_api.classify_symbol(all_company_info_df)
|
|
26
|
+
fail_list = []
|
|
27
|
+
for stock_one in all_company_info_df.itertuples():
|
|
28
|
+
try:
|
|
29
|
+
tag = sync_one_company_industry_info(stock_one.symbol, stock_one.classification)
|
|
30
|
+
if bool(1 - tag):
|
|
31
|
+
fail_list.append(stock_one.symbol)
|
|
32
|
+
time.sleep(0.5)
|
|
33
|
+
except BaseException as e:
|
|
34
|
+
time.sleep(1)
|
|
35
|
+
logger.error("同步公司行业信息发生异常:{},{}", stock_one.symbol, e)
|
|
36
|
+
fail_list.append(stock_one.symbol)
|
|
37
|
+
sync_number = 1
|
|
38
|
+
while len(fail_list) > 0 and sync_number < 10:
|
|
39
|
+
for symbol in fail_list:
|
|
40
|
+
try:
|
|
41
|
+
company_info_one_df = all_company_info_df.loc[all_company_info_df['symbol'] == symbol]
|
|
42
|
+
classification = list(company_info_one_df['classification'])[0]
|
|
43
|
+
tag = sync_one_company_industry_info(symbol, classification)
|
|
44
|
+
if tag and symbol in fail_list:
|
|
45
|
+
fail_list.remove(symbol)
|
|
46
|
+
time.sleep(2)
|
|
47
|
+
except BaseException as e:
|
|
48
|
+
time.sleep(3)
|
|
49
|
+
logger.error("同步公司行业信息发生异常:{},{}", symbol, e)
|
|
50
|
+
sync_number = sync_number + 1
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def sync_one_company_industry_info(symbol, classification):
|
|
54
|
+
if classification in ['H', 'K']:
|
|
55
|
+
market_id = '17'
|
|
56
|
+
elif classification in ['S', 'C']:
|
|
57
|
+
market_id = '33'
|
|
58
|
+
elif classification in ['X']:
|
|
59
|
+
market_id = '151'
|
|
60
|
+
|
|
61
|
+
company_industry_info = ths_company_info_web.get_company_info_detail(symbol, market_id)
|
|
62
|
+
if data_frame_util.is_empty(company_industry_info):
|
|
63
|
+
return False
|
|
64
|
+
|
|
65
|
+
company_industry_info['first_industry_code'] = company_industry_info['hycode'].apply(
|
|
66
|
+
lambda x: x[1:3] + '0000')
|
|
67
|
+
company_industry_info['second_industry_code'] = company_industry_info['hy2code'].apply(
|
|
68
|
+
lambda x: x[1:5] + '00')
|
|
69
|
+
company_industry_info['third_industry_code'] = company_industry_info['hy3code'].apply(
|
|
70
|
+
lambda x: x[1:7])
|
|
71
|
+
|
|
72
|
+
company_industry_info['first_sw_industry'] = company_industry_info['hy']
|
|
73
|
+
company_industry_info['second_sw_industry'] = company_industry_info['hy2']
|
|
74
|
+
company_industry_info['third_sw_industry'] = company_industry_info['hy3']
|
|
75
|
+
del company_industry_info['hy']
|
|
76
|
+
del company_industry_info['hy2']
|
|
77
|
+
del company_industry_info['hy3']
|
|
78
|
+
del company_industry_info['hycode']
|
|
79
|
+
del company_industry_info['hy2code']
|
|
80
|
+
del company_industry_info['hy3code']
|
|
81
|
+
|
|
82
|
+
company_industry_info['_id'] = symbol
|
|
83
|
+
company_industry_info['symbol'] = symbol
|
|
84
|
+
mongodb_util.save_mongo(company_industry_info, db_name_constant.COMPANY_INDUSTRY_INFO)
|
|
85
|
+
# 保存股票申万行业原始信息
|
|
86
|
+
save_sw_data(company_industry_info)
|
|
87
|
+
return True
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# 保存申万行业分类
|
|
91
|
+
def save_sw_data(company_industry_info):
|
|
92
|
+
first_sw_info = company_industry_info[[
|
|
93
|
+
'first_sw_industry',
|
|
94
|
+
'first_industry_code'
|
|
95
|
+
]].copy()
|
|
96
|
+
|
|
97
|
+
first_sw_info.loc[:, "industry_code"] = first_sw_info['first_industry_code']
|
|
98
|
+
first_sw_info.loc[:, "_id"] = first_sw_info['first_industry_code']
|
|
99
|
+
first_sw_info.loc[:, "second_sw_industry"] = 0
|
|
100
|
+
first_sw_info.loc[:, "third_sw_industry"] = 0
|
|
101
|
+
first_sw_info.loc[:, "second_industry_code"] = 0
|
|
102
|
+
first_sw_info.loc[:, "third_industry_code"] = 0
|
|
103
|
+
first_sw_info = first_sw_info[[
|
|
104
|
+
"_id",
|
|
105
|
+
"industry_code",
|
|
106
|
+
'first_industry_code',
|
|
107
|
+
'first_sw_industry',
|
|
108
|
+
'second_industry_code',
|
|
109
|
+
'second_sw_industry',
|
|
110
|
+
'third_industry_code',
|
|
111
|
+
'third_sw_industry'
|
|
112
|
+
]]
|
|
113
|
+
mongodb_util.save_mongo(first_sw_info, 'sw_industry')
|
|
114
|
+
|
|
115
|
+
second_sw_info = company_industry_info[[
|
|
116
|
+
'first_industry_code',
|
|
117
|
+
'first_sw_industry',
|
|
118
|
+
'second_sw_industry',
|
|
119
|
+
'second_industry_code',
|
|
120
|
+
]].copy()
|
|
121
|
+
|
|
122
|
+
second_sw_info.loc[:, "industry_code"] = second_sw_info['second_industry_code']
|
|
123
|
+
second_sw_info.loc[:, "_id"] = second_sw_info['industry_code']
|
|
124
|
+
|
|
125
|
+
second_sw_info.loc[:, "third_sw_industry"] = 0
|
|
126
|
+
second_sw_info.loc[:, "third_sw_industry"] = 0
|
|
127
|
+
second_sw_info.loc[:, "third_industry_code"] = 0
|
|
128
|
+
second_sw_info = second_sw_info[[
|
|
129
|
+
"_id",
|
|
130
|
+
"industry_code",
|
|
131
|
+
'first_industry_code',
|
|
132
|
+
'first_sw_industry',
|
|
133
|
+
'second_industry_code',
|
|
134
|
+
'second_sw_industry',
|
|
135
|
+
'third_industry_code',
|
|
136
|
+
'third_sw_industry'
|
|
137
|
+
]]
|
|
138
|
+
mongodb_util.save_mongo(second_sw_info, 'sw_industry')
|
|
139
|
+
|
|
140
|
+
third_sw_info = company_industry_info[[
|
|
141
|
+
'first_industry_code',
|
|
142
|
+
'first_sw_industry',
|
|
143
|
+
'second_industry_code',
|
|
144
|
+
'second_sw_industry',
|
|
145
|
+
'third_industry_code',
|
|
146
|
+
'third_sw_industry'
|
|
147
|
+
]].copy()
|
|
148
|
+
|
|
149
|
+
third_sw_info.loc[:, "industry_code"] = third_sw_info['third_industry_code']
|
|
150
|
+
|
|
151
|
+
third_sw_info.loc[:, "_id"] = third_sw_info['industry_code']
|
|
152
|
+
|
|
153
|
+
third_sw_info = third_sw_info[[
|
|
154
|
+
"_id",
|
|
155
|
+
"industry_code",
|
|
156
|
+
'first_industry_code',
|
|
157
|
+
'first_sw_industry',
|
|
158
|
+
'second_industry_code',
|
|
159
|
+
'second_sw_industry',
|
|
160
|
+
'third_industry_code',
|
|
161
|
+
'third_sw_industry'
|
|
162
|
+
]]
|
|
163
|
+
mongodb_util.save_mongo(third_sw_info, 'sw_industry')
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
if __name__ == '__main__':
|
|
167
|
+
sync_company_industry_info_task(['688795'])
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
import mns_scheduler.company_info.task.company_industry_info_task as company_industry_info_task
|
|
9
|
+
import mns_scheduler.company_info.task.company_business_info_task as company_business_info_task
|
|
10
|
+
import mns_scheduler.company_info.task.company_base_info_task as company_base_info_task
|
|
11
|
+
import mns_scheduler.company_info.task.company_hold_info_task as company_hold_info_task
|
|
12
|
+
import mns_scheduler.company_info.task.company_announce_info_task as company_announce_info_task
|
|
13
|
+
from mns_scheduler.company_info.common.company_common_query_service import get_company_info
|
|
14
|
+
from datetime import datetime
|
|
15
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
16
|
+
from loguru import logger
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def sync_all_company_task():
|
|
20
|
+
# 同步公司行业信息
|
|
21
|
+
company_industry_info_task.sync_company_industry_info_task([])
|
|
22
|
+
logger.info("[全量]同步公司[行业]信息完成")
|
|
23
|
+
# 同步公司业务信息
|
|
24
|
+
company_business_info_task.sync_company_business_task([])
|
|
25
|
+
logger.info("[全量]同步公司[业务]信息完成")
|
|
26
|
+
# 同步公司基本信息
|
|
27
|
+
company_base_info_task.sync_company_base_info_task([])
|
|
28
|
+
logger.info("[全量]同步公司[基本]信息完成")
|
|
29
|
+
# 同步公司控股信息
|
|
30
|
+
company_hold_info_task.sync_all_company_hold_info_task([])
|
|
31
|
+
logger.info("[全量]同步公司[控股]信息完成")
|
|
32
|
+
# 同步公司公告信息
|
|
33
|
+
company_announce_info_task.sync_company_announce_task([])
|
|
34
|
+
logger.info("[全量]同步公司[公告]信息完成")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def sync_new_stock_company_task():
|
|
38
|
+
all_company_info_df = get_company_info()
|
|
39
|
+
|
|
40
|
+
now_date = datetime.now()
|
|
41
|
+
str_day = now_date.strftime('%Y-%m-%d')
|
|
42
|
+
# 去掉横线并转换为整数
|
|
43
|
+
now_day = int(str_day.replace('-', ''))
|
|
44
|
+
|
|
45
|
+
new_company_info_df = all_company_info_df.loc[
|
|
46
|
+
(all_company_info_df['list_date'] == 19890604) | (all_company_info_df['list_date'] >= now_day)]
|
|
47
|
+
if data_frame_util.is_empty(new_company_info_df):
|
|
48
|
+
return
|
|
49
|
+
symbol_list = list(new_company_info_df['symbol'])
|
|
50
|
+
# 同步公司行业信息
|
|
51
|
+
company_industry_info_task.sync_company_industry_info_task(symbol_list)
|
|
52
|
+
logger.info("[新股]同步公司[行业]信息完成")
|
|
53
|
+
# 同步公司业务信息
|
|
54
|
+
company_business_info_task.sync_company_business_task(symbol_list)
|
|
55
|
+
logger.info("[新股]同步公司[业务]信息完成")
|
|
56
|
+
# 同步公司基本信息
|
|
57
|
+
company_base_info_task.sync_company_base_info_task(symbol_list)
|
|
58
|
+
logger.info("[新股]同步公司[基本]信息完成")
|
|
59
|
+
# 同步公司控股信息
|
|
60
|
+
company_hold_info_task.sync_all_company_hold_info_task(symbol_list)
|
|
61
|
+
logger.info("[新股]同步公司[控股]信息完成")
|
|
62
|
+
# 同步公司公告信息
|
|
63
|
+
company_announce_info_task.sync_company_announce_task(symbol_list)
|
|
64
|
+
logger.info("[新股]同步公司[公告]信息完成")
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if __name__ == '__main__':
|
|
68
|
+
sync_new_stock_company_task()
|
|
69
|
+
sync_all_company_task()
|
|
@@ -11,7 +11,7 @@ project_path = file_path[0:end]
|
|
|
11
11
|
sys.path.append(project_path)
|
|
12
12
|
mongodb_util = MongodbUtil('27017')
|
|
13
13
|
import mns_common.api.msg.push_msg_api as push_msg_api
|
|
14
|
-
import mns_scheduler.company_info.
|
|
14
|
+
import mns_scheduler.company_info.sync.sync_company_info_task as sync_company_info_task
|
|
15
15
|
import mns_scheduler.company_info.clean.company_info_clean_api as company_info_clean_api
|
|
16
16
|
import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
|
|
17
17
|
import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
|
|
@@ -192,8 +192,8 @@ def update_long_short(new_concept_symbol_df, exist_concept_detail):
|
|
|
192
192
|
def update_company_info(new_concept_symbol_df):
|
|
193
193
|
if new_concept_symbol_df.shape[0] > 0:
|
|
194
194
|
symbol_list = list(new_concept_symbol_df['symbol'])
|
|
195
|
-
|
|
196
|
-
company_info_clean_api.clean_company_info(
|
|
195
|
+
sync_company_info_task.sync_company_base_info(symbol_list)
|
|
196
|
+
company_info_clean_api.clean_company_info(symbol_list)
|
|
197
197
|
# 公司缓存信息清除
|
|
198
198
|
company_common_service_api.company_info_industry_cache_clear()
|
|
199
199
|
|
|
@@ -15,6 +15,10 @@ remote_mongodb_util = RemoteMongodbUtil('27017')
|
|
|
15
15
|
local_mongodb_util = LocalMongodbUtil('27017')
|
|
16
16
|
|
|
17
17
|
col_list = [
|
|
18
|
+
'zt_reason_analysis',
|
|
19
|
+
'today_exclude_stocks',
|
|
20
|
+
'today_self_choose_stock',
|
|
21
|
+
'main_line_detail',
|
|
18
22
|
'em_a_stock_info',
|
|
19
23
|
'em_etf_info',
|
|
20
24
|
'em_kzz_info',
|
|
@@ -38,7 +42,8 @@ col_list = [
|
|
|
38
42
|
'ths_stock_concept_detail',
|
|
39
43
|
'stock_high_chg_pool',
|
|
40
44
|
'today_new_concept_list',
|
|
41
|
-
'ths_stock_concept_detail_app'
|
|
45
|
+
'ths_stock_concept_detail_app',
|
|
46
|
+
|
|
42
47
|
]
|
|
43
48
|
|
|
44
49
|
|
|
@@ -91,7 +96,7 @@ def sync_open_data():
|
|
|
91
96
|
|
|
92
97
|
|
|
93
98
|
if __name__ == '__main__':
|
|
94
|
-
sync_zt_data('2025-09-17')
|
|
99
|
+
# sync_zt_data('2025-09-17')
|
|
95
100
|
# sync_zt_data('2025-09-02')
|
|
96
101
|
remote_data()
|
|
97
102
|
# sync_zt_data('2025-08-26')
|
|
@@ -13,64 +13,17 @@ Desc: 上证e互动-提问与回答
|
|
|
13
13
|
https://sns.sseinfo.com/
|
|
14
14
|
"""
|
|
15
15
|
import warnings
|
|
16
|
-
from functools import lru_cache
|
|
17
|
-
|
|
18
16
|
import pandas as pd
|
|
19
17
|
import requests
|
|
20
18
|
from bs4 import BeautifulSoup
|
|
21
|
-
|
|
22
19
|
from mns_common.db.MongodbUtil import MongodbUtil
|
|
23
|
-
import mns_common.constant.db_name_constant as db_name_constant
|
|
24
20
|
|
|
25
21
|
mongodb_util = MongodbUtil('27017')
|
|
26
|
-
from tqdm import tqdm
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
@lru_cache()
|
|
30
|
-
def sync_stock_uid() -> pd.DataFrame:
|
|
31
|
-
"""
|
|
32
|
-
上证e互动-代码ID映射
|
|
33
|
-
https://sns.sseinfo.com/list/company.do
|
|
34
|
-
:return: 代码ID映射
|
|
35
|
-
:rtype: str
|
|
36
|
-
"""
|
|
37
|
-
url = "https://sns.sseinfo.com/allcompany.do"
|
|
38
|
-
data = {
|
|
39
|
-
"code": "0",
|
|
40
|
-
"order": "2",
|
|
41
|
-
"areaId": "0",
|
|
42
|
-
"page": "1",
|
|
43
|
-
}
|
|
44
|
-
uid_list = list()
|
|
45
|
-
code_list = list()
|
|
46
|
-
for page in tqdm(range(1, 74), leave=False):
|
|
47
|
-
data.update({"page": page})
|
|
48
|
-
r = requests.post(url, data=data)
|
|
49
|
-
data_json = r.json()
|
|
50
|
-
soup = BeautifulSoup(data_json["content"], "lxml")
|
|
51
|
-
soup.find_all("a", attrs={"rel": "tag"})
|
|
52
|
-
uid_list.extend(
|
|
53
|
-
[item["uid"] for item in soup.find_all("a", attrs={"rel": "tag"})]
|
|
54
|
-
)
|
|
55
|
-
code_list.extend(
|
|
56
|
-
[
|
|
57
|
-
item.find("img")["src"].split("?")[0].split("/")[-1].split(".")[0]
|
|
58
|
-
for item in soup.find_all("a", attrs={"rel": "tag"})
|
|
59
|
-
]
|
|
60
|
-
)
|
|
61
|
-
code_uid_df = pd.DataFrame()
|
|
62
|
-
code_uid_df['symbol'] = code_list
|
|
63
|
-
code_uid_df['uid'] = uid_list
|
|
64
|
-
code_uid_df['_id'] = uid_list
|
|
65
|
-
return code_uid_df
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
@lru_cache()
|
|
69
|
-
def get_stock_uid() -> pd.DataFrame:
|
|
70
|
-
return mongodb_util.find_all_data(db_name_constant.SSE_INFO_UID)
|
|
71
22
|
|
|
72
23
|
|
|
73
|
-
def stock_sns_sse_info(
|
|
24
|
+
def stock_sns_sse_info(org_ask_id,
|
|
25
|
+
page,
|
|
26
|
+
page_size) -> pd.DataFrame:
|
|
74
27
|
"""
|
|
75
28
|
上证e互动-提问与回答
|
|
76
29
|
https://sns.sseinfo.com/company.do?uid=65
|
|
@@ -80,16 +33,13 @@ def stock_sns_sse_info(symbol: str = "603119") -> pd.DataFrame:
|
|
|
80
33
|
:rtype: str
|
|
81
34
|
"""
|
|
82
35
|
|
|
83
|
-
stock_uid_df = get_stock_uid()
|
|
84
|
-
stock_uid_df = stock_uid_df.loc[stock_uid_df['symbol'] == symbol]
|
|
85
|
-
uid = list(stock_uid_df['uid'])[0]
|
|
86
36
|
url = "https://sns.sseinfo.com/ajax/userfeeds.do"
|
|
87
37
|
params = {
|
|
88
38
|
"typeCode": "company",
|
|
89
39
|
"type": "11",
|
|
90
|
-
"pageSize":
|
|
91
|
-
"uid":
|
|
92
|
-
"page":
|
|
40
|
+
"pageSize": str(page_size),
|
|
41
|
+
"uid": org_ask_id,
|
|
42
|
+
"page": str(page),
|
|
93
43
|
}
|
|
94
44
|
big_df = pd.DataFrame()
|
|
95
45
|
page = 1
|
|
@@ -164,5 +114,5 @@ def stock_sns_sse_info(symbol: str = "603119") -> pd.DataFrame:
|
|
|
164
114
|
|
|
165
115
|
|
|
166
116
|
if __name__ == "__main__":
|
|
167
|
-
stock_sns_sse_info_df = stock_sns_sse_info(
|
|
117
|
+
stock_sns_sse_info_df = stock_sns_sse_info('308', '2', '100')
|
|
168
118
|
print(stock_sns_sse_info_df)
|
|
@@ -15,27 +15,18 @@ https://irm.cninfo.com.cn/
|
|
|
15
15
|
|
|
16
16
|
import pandas as pd
|
|
17
17
|
import requests
|
|
18
|
+
import time
|
|
18
19
|
|
|
19
20
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
params = {"_t": "1691144074"}
|
|
29
|
-
data = {"keyWord": symbol}
|
|
30
|
-
r = requests.post(url, params=params, data=data)
|
|
31
|
-
data_json = r.json()
|
|
32
|
-
org_id = data_json["data"][0]["secid"]
|
|
33
|
-
return org_id
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# 获取股票所有问题
|
|
21
|
+
# 获取单个股票所有问题
|
|
22
|
+
# todo https://irm.cninfo.com.cn/ircs/company/companyDetail?stockcode=000002&orgId=gssz0000002
|
|
23
|
+
def stock_irm_cninfo(symbol,
|
|
24
|
+
org_ask_id,
|
|
25
|
+
page,
|
|
26
|
+
page_size) -> pd.DataFrame:
|
|
27
|
+
# 毫秒级时间戳字符串
|
|
28
|
+
current_ms_timestamp_str = str(int(time.time() * 1000))
|
|
37
29
|
|
|
38
|
-
def stock_irm_cninfo(symbol: str = "002594") -> pd.DataFrame:
|
|
39
30
|
"""
|
|
40
31
|
互动易-提问
|
|
41
32
|
https://irm.cninfo.com.cn/ircs/question/questionDetail?questionId=1515236357817618432
|
|
@@ -46,19 +37,17 @@ def stock_irm_cninfo(symbol: str = "002594") -> pd.DataFrame:
|
|
|
46
37
|
"""
|
|
47
38
|
url = "https://irm.cninfo.com.cn/newircs/company/question"
|
|
48
39
|
params = {
|
|
49
|
-
"_t":
|
|
40
|
+
"_t": current_ms_timestamp_str,
|
|
50
41
|
"stockcode": symbol,
|
|
51
|
-
"orgId":
|
|
52
|
-
"pageSize":
|
|
53
|
-
"pageNum":
|
|
42
|
+
"orgId": org_ask_id,
|
|
43
|
+
"pageSize": page_size,
|
|
44
|
+
"pageNum": page,
|
|
54
45
|
"keyWord": "",
|
|
55
46
|
"startDay": "",
|
|
56
47
|
"endDay": "",
|
|
57
48
|
}
|
|
58
49
|
|
|
59
50
|
big_df = pd.DataFrame()
|
|
60
|
-
page = 1
|
|
61
|
-
params.update({"pageNum": page})
|
|
62
51
|
r = requests.post(url, params=params)
|
|
63
52
|
data_json = r.json()
|
|
64
53
|
temp_df = pd.DataFrame(data_json["rows"])
|
|
@@ -195,7 +184,7 @@ def stock_irm_ans_cninfo(symbol: str = "1513586704097333248") -> pd.DataFrame:
|
|
|
195
184
|
|
|
196
185
|
|
|
197
186
|
if __name__ == "__main__":
|
|
198
|
-
stock_irm_cninfo_df = stock_irm_cninfo(symbol="
|
|
187
|
+
stock_irm_cninfo_df = stock_irm_cninfo(symbol="000002", org_ask_id='gssz0000002', page=2, pageSize=100)
|
|
199
188
|
print(stock_irm_cninfo_df)
|
|
200
189
|
|
|
201
190
|
stock_irm_ans_cninfo_df = stock_irm_ans_cninfo(symbol="1495108801386602496")
|
|
@@ -5,7 +5,7 @@ file_path = os.path.abspath(__file__)
|
|
|
5
5
|
end = file_path.index('mns') + 17
|
|
6
6
|
project_path = file_path[0:end]
|
|
7
7
|
sys.path.append(project_path)
|
|
8
|
-
|
|
8
|
+
|
|
9
9
|
import mns_common.utils.data_frame_util as data_frame_util
|
|
10
10
|
import pandas as pd
|
|
11
11
|
import mns_common.component.em.em_stock_info_api as em_stock_info_api
|
|
@@ -14,22 +14,25 @@ from loguru import logger
|
|
|
14
14
|
from datetime import datetime
|
|
15
15
|
import time
|
|
16
16
|
import mns_common.utils.date_handle_util as date_handle_util
|
|
17
|
-
|
|
18
17
|
import mns_common.constant.db_name_constant as db_name_constant
|
|
19
18
|
import mns_common.component.common_service_fun_api as common_service_fun_api
|
|
20
19
|
import mns_scheduler.irm.api.sh_stock_sns_sse_info_api as sh_stock_sns_sse_info_api
|
|
21
20
|
import mns_scheduler.irm.api.sz_stock_sns_sse_info_api as sz_stock_sns_sse_info_api
|
|
22
|
-
|
|
23
21
|
from mns_common.db.MongodbUtil import MongodbUtil
|
|
22
|
+
import mns_scheduler.irm.stock_question_id_service as stock_question_id_service
|
|
24
23
|
|
|
25
24
|
mongodb_util = MongodbUtil('27017')
|
|
26
25
|
|
|
27
26
|
|
|
28
|
-
# 获取股票提问 互动易-提问 深交所
|
|
27
|
+
# 获取股票提问 互动易-提问 todo 深交所 可以拉取未回答的,但是最拉到最近三个月
|
|
29
28
|
def get_stock_irm_cninfo_sz_api(symbol):
|
|
30
29
|
try:
|
|
30
|
+
org_ask_id = stock_question_id_service.get_one_sz_symbol_org_id(symbol)
|
|
31
31
|
# 获取一页
|
|
32
|
-
stock_irm_cninfo_df = sz_stock_sns_sse_info_api.stock_irm_cninfo(symbol
|
|
32
|
+
stock_irm_cninfo_df = sz_stock_sns_sse_info_api.stock_irm_cninfo(symbol,
|
|
33
|
+
org_ask_id,
|
|
34
|
+
1,
|
|
35
|
+
100)
|
|
33
36
|
# 获取全页
|
|
34
37
|
# stock_irm_cninfo_df = ak.stock_irm_cninfo(symbol)
|
|
35
38
|
except Exception as e:
|
|
@@ -66,11 +69,15 @@ def get_stock_irm_cninfo_sz_api(symbol):
|
|
|
66
69
|
return stock_irm_cninfo_df
|
|
67
70
|
|
|
68
71
|
|
|
69
|
-
# 获取股票提问 互动易-提问 上交所
|
|
72
|
+
# 获取股票提问 互动易-提问 todo 上交所
|
|
70
73
|
def get_stock_irm_cninfo_sh_api(symbol):
|
|
71
74
|
try:
|
|
72
75
|
# 获取一页
|
|
73
|
-
|
|
76
|
+
org_ask_id = stock_question_id_service.get_one_sh_symbol_org_id(symbol)
|
|
77
|
+
|
|
78
|
+
stock_sns_sse_info_df = sh_stock_sns_sse_info_api.stock_sns_sse_info(org_ask_id,
|
|
79
|
+
1,
|
|
80
|
+
100)
|
|
74
81
|
# 获取全页
|
|
75
82
|
# stock_sns_sse_info_df = ak.stock_sns_sseinfo(symbol)
|
|
76
83
|
except Exception as e:
|
|
@@ -101,10 +108,21 @@ def get_stock_irm_cninfo_sh_api(symbol):
|
|
|
101
108
|
"answer_time",
|
|
102
109
|
"questioner",
|
|
103
110
|
"source"]]
|
|
111
|
+
now_date = datetime.now()
|
|
112
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
113
|
+
stock_sns_sse_info_df.loc[
|
|
114
|
+
stock_sns_sse_info_df['answer_time'].str.contains('小时前', na=False), 'answer_time'] = str_now_date
|
|
115
|
+
stock_sns_sse_info_df.loc[
|
|
116
|
+
stock_sns_sse_info_df['answer_time'].str.contains('分钟前', na=False), 'answer_time'] = str_now_date
|
|
117
|
+
stock_sns_sse_info_df.loc[
|
|
118
|
+
stock_sns_sse_info_df['question_time'].str.contains('小时前', na=False), 'question_time'] = str_now_date
|
|
119
|
+
stock_sns_sse_info_df.loc[
|
|
120
|
+
stock_sns_sse_info_df['question_time'].str.contains('分钟前', na=False), 'answer_time'] = str_now_date
|
|
104
121
|
|
|
105
122
|
return stock_sns_sse_info_df
|
|
106
123
|
|
|
107
124
|
|
|
125
|
+
# 格式化 时间
|
|
108
126
|
def replace_date_format(date_str):
|
|
109
127
|
now_date = datetime.now()
|
|
110
128
|
str_day = now_date.strftime('%Y-%m-%d')
|
|
@@ -113,22 +131,14 @@ def replace_date_format(date_str):
|
|
|
113
131
|
return date_str.replace('年', '-').replace('月', '-').replace('日', '').replace('昨天', last_day)
|
|
114
132
|
|
|
115
133
|
|
|
116
|
-
def save_sh_stock_uid():
|
|
117
|
-
code_uid_df = sh_stock_sns_sse_info_api.sync_stock_uid()
|
|
118
|
-
mongodb_util.save_mongo(code_uid_df, db_name_constant.SSE_INFO_UID)
|
|
119
|
-
|
|
120
|
-
|
|
121
134
|
# 同步所有互动问题
|
|
122
135
|
def sync_symbols_interactive_questions(symbol_list):
|
|
123
|
-
# 同步互动易映射
|
|
124
|
-
if len(symbol_list) == 0:
|
|
125
|
-
save_sh_stock_uid()
|
|
126
136
|
real_time_quotes_all_stocks = em_stock_info_api.get_a_stock_info()
|
|
127
137
|
de_list_company_symbols = company_common_service_api.get_de_list_company()
|
|
128
138
|
real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
|
|
129
139
|
~(real_time_quotes_all_stocks['symbol'].isin(de_list_company_symbols))]
|
|
130
140
|
real_time_quotes_all_stocks = common_service_fun_api.classify_symbol(real_time_quotes_all_stocks)
|
|
131
|
-
real_time_quotes_all_stocks = real_time_quotes_all_stocks.sort_values(by=['
|
|
141
|
+
real_time_quotes_all_stocks = real_time_quotes_all_stocks.sort_values(by=['amount'], ascending=False)
|
|
132
142
|
fail_symbol_list = []
|
|
133
143
|
if len(symbol_list) != 0:
|
|
134
144
|
real_time_quotes_all_stocks = real_time_quotes_all_stocks.loc[
|
|
@@ -155,14 +165,7 @@ def sync_symbols_interactive_questions(symbol_list):
|
|
|
155
165
|
stock_irm_cninfo_df['str_day'] = str_day
|
|
156
166
|
stock_irm_cninfo_df.drop_duplicates('_id', keep='last', inplace=True)
|
|
157
167
|
stock_irm_cninfo_df.fillna("", inplace=True)
|
|
158
|
-
|
|
159
|
-
stock_irm_cninfo_df['answer_time'].str.contains('小时前', na=False), 'answer_time'] = str_now_date
|
|
160
|
-
stock_irm_cninfo_df.loc[
|
|
161
|
-
stock_irm_cninfo_df['answer_time'].str.contains('分钟前', na=False), 'answer_time'] = str_now_date
|
|
162
|
-
stock_irm_cninfo_df.loc[
|
|
163
|
-
stock_irm_cninfo_df['question_time'].str.contains('小时前', na=False), 'question_time'] = str_now_date
|
|
164
|
-
stock_irm_cninfo_df.loc[
|
|
165
|
-
stock_irm_cninfo_df['question_time'].str.contains('分钟前', na=False), 'answer_time'] = str_now_date
|
|
168
|
+
|
|
166
169
|
stock_irm_cninfo_df['valid'] = True
|
|
167
170
|
stock_irm_cninfo_df['answer_content'].fillna('')
|
|
168
171
|
# 保存新增数据
|
|
@@ -174,6 +177,7 @@ def sync_symbols_interactive_questions(symbol_list):
|
|
|
174
177
|
logger.error("同步互动问题出现异常:{},{}", stock_one.symbol, e)
|
|
175
178
|
|
|
176
179
|
|
|
180
|
+
# 保存新增回答数据
|
|
177
181
|
def save_new_data(stock_irm_cninfo_df):
|
|
178
182
|
if data_frame_util.is_empty(stock_irm_cninfo_df):
|
|
179
183
|
return None
|
|
@@ -193,8 +197,10 @@ def save_new_data(stock_irm_cninfo_df):
|
|
|
193
197
|
|
|
194
198
|
|
|
195
199
|
if __name__ == '__main__':
|
|
200
|
+
get_stock_irm_cninfo_sz_api('000002')
|
|
196
201
|
sync_symbols_interactive_questions([])
|
|
197
|
-
|
|
198
|
-
|
|
202
|
+
# sync_symbols_interactive_questions([])
|
|
203
|
+
# get_stock_irm_cninfo_sh_api('688778')
|
|
204
|
+
# fail_symbol_list_01 = ['000638', '002886', '688778', '688766', '688733', '688778', '688793', '688787']
|
|
199
205
|
# get_stock_irm_cninfo_sh_api('603633')
|
|
200
206
|
# sync_symbols_interactive_questions(None)
|