mns-common 1.3.9.2__py3-none-any.whl → 1.6.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mns-common might be problematic. Click here for more details.
- mns_common/__init__.py +1 -0
- mns_common/api/akshare/__init__.py +0 -1
- mns_common/api/akshare/k_line_api.py +20 -82
- mns_common/api/akshare/stock_bid_ask_api.py +21 -14
- mns_common/api/akshare/stock_zb_pool.py +2 -0
- mns_common/api/akshare/stock_zt_pool_api.py +1 -1
- mns_common/api/em/gd/east_money_stock_gdfx_free_top_10_api.py +62 -7
- mns_common/api/em/real_time/__init__.py +1 -1
- mns_common/api/em/real_time/east_money_debt_api.py +168 -71
- mns_common/api/em/real_time/east_money_etf_api.py +165 -27
- mns_common/api/em/real_time/east_money_stock_a_api.py +37 -38
- mns_common/api/em/real_time/east_money_stock_a_v2_api.py +97 -53
- mns_common/api/em/real_time/east_money_stock_common_api.py +174 -0
- mns_common/api/em/real_time/east_money_stock_hk_api.py +252 -271
- mns_common/api/em/real_time/east_money_stock_hk_gtt_api.py +291 -0
- mns_common/api/em/real_time/east_money_stock_multi_thread_api_v3.py +154 -0
- mns_common/api/em/real_time/east_money_stock_us_api.py +210 -82
- mns_common/api/em/real_time/real_time_quotes_repeat_api.py +195 -0
- mns_common/api/foreign_exchange/foreign_exchange_api.py +38 -0
- mns_common/api/k_line/stock_k_line_data_api.py +11 -1
- mns_common/api/kpl/common/__init__.py +3 -2
- mns_common/api/kpl/common/kpl_common_api.py +35 -0
- mns_common/api/kpl/symbol/symbol_his_quotes_api.py +1 -1
- mns_common/api/kpl/theme/kpl_theme_api.py +69 -0
- mns_common/api/kpl/yidong/__init__.py +7 -0
- mns_common/api/kpl/yidong/stock_bid_yi_dong_api.py +52 -0
- mns_common/api/proxies/liu_guan_proxy_api.py +55 -5
- mns_common/api/ths/company/company_product_area_industry_index_query.py +46 -0
- mns_common/api/ths/company/ths_company_info_api.py +2 -1
- mns_common/api/ths/company/ths_company_info_web.py +159 -0
- mns_common/api/ths/concept/app/ths_concept_index_app.py +3 -1
- mns_common/api/ths/wen_cai/ths_wen_cai_api.py +1 -1
- mns_common/api/ths/zt/ths_stock_zt_pool_api.py +20 -1
- mns_common/api/ths/zt/ths_stock_zt_pool_v2_api.py +105 -29
- mns_common/api/ths/zt/ths_stock_zt_reason_web_api.py +100 -0
- mns_common/api/us/ths_us_company_info_api.py +131 -0
- mns_common/api/xueqiu/xue_qiu_k_line_api.py +31 -23
- mns_common/component/common_service_fun_api.py +28 -8
- mns_common/component/company/company_common_service_new_api.py +2 -0
- mns_common/component/cookie/cookie_enum.py +16 -0
- mns_common/component/cookie/cookie_info_service.py +18 -8
- mns_common/component/data/data_init_api.py +13 -8
- mns_common/component/deal/deal_service_api.py +70 -8
- mns_common/component/deal/deal_service_v2_api.py +167 -0
- mns_common/component/em/em_stock_info_api.py +12 -3
- mns_common/component/exception/ExceptionMonitor.py +86 -0
- mns_common/component/exception/__init__.py +7 -0
- mns_common/component/main_line/__init__.py +7 -0
- mns_common/component/main_line/main_line_zt_reason_service.py +257 -0
- mns_common/component/proxies/proxy_common_api.py +199 -31
- mns_common/component/tfp/stock_tfp_api.py +82 -12
- mns_common/component/us/__init__.py +7 -0
- mns_common/component/us/us_stock_etf_info_api.py +130 -0
- mns_common/constant/db_name_constant.py +75 -26
- mns_common/constant/extra_income_db_name.py +97 -11
- mns_common/constant/strategy_classify.py +72 -0
- mns_common/db/MongodbUtil.py +3 -0
- mns_common/db/MongodbUtilLocal.py +3 -0
- {mns_common-1.3.9.2.dist-info → mns_common-1.6.1.4.dist-info}/METADATA +1 -1
- {mns_common-1.3.9.2.dist-info → mns_common-1.6.1.4.dist-info}/RECORD +64 -47
- mns_common/api/ths/concept/web/ths_company_info_web.py +0 -163
- mns_common/component/qmt/qmt_buy_service.py +0 -172
- mns_common/component/task/real_time_data_sync_check.py +0 -97
- /mns_common/{component/qmt → api/foreign_exchange}/__init__.py +0 -0
- /mns_common/{component/task → api/kpl/theme}/__init__.py +0 -0
- {mns_common-1.3.9.2.dist-info → mns_common-1.6.1.4.dist-info}/WHEEL +0 -0
- {mns_common-1.3.9.2.dist-info → mns_common-1.6.1.4.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
file_path = os.path.abspath(__file__)
|
|
5
|
+
end = file_path.index('mns') + 16
|
|
6
|
+
project_path = file_path[0:end]
|
|
7
|
+
sys.path.append(project_path)
|
|
8
|
+
import mns_common.constant.db_name_constant as db_name_constant
|
|
9
|
+
from mns_common.db.MongodbUtil import MongodbUtil
|
|
10
|
+
import mns_common.api.ths.zt.ths_stock_zt_pool_api as ths_stock_zt_pool_api
|
|
11
|
+
from mns_common.utils.async_fun import async_fun
|
|
12
|
+
import mns_common.component.trade_date.trade_date_common_service_api as trade_date_common_service_api
|
|
13
|
+
import mns_common.api.ths.zt.ths_stock_zt_reason_web_api as ths_stock_zt_reason_web_api
|
|
14
|
+
|
|
15
|
+
mongodb_util = MongodbUtil('27017')
|
|
16
|
+
import mns_common.utils.data_frame_util as data_frame_util
|
|
17
|
+
import time
|
|
18
|
+
from loguru import logger
|
|
19
|
+
import pandas as pd
|
|
20
|
+
from datetime import datetime
|
|
21
|
+
import mns_common.component.cookie.cookie_info_service as cookie_info_service
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# 添加主线和涨停分析临时数据
|
|
25
|
+
def merge_main_line_info(str_day, data_df):
|
|
26
|
+
# 保证数据完整性
|
|
27
|
+
if 'main_line' not in data_df.columns:
|
|
28
|
+
data_df['main_line'] = ''
|
|
29
|
+
else:
|
|
30
|
+
data_df.fillna({'main_line': ''}, inplace=True)
|
|
31
|
+
|
|
32
|
+
if 'sub_main_line' not in data_df.columns:
|
|
33
|
+
data_df['sub_main_line'] = ''
|
|
34
|
+
else:
|
|
35
|
+
data_df.fillna({'sub_main_line': ''}, inplace=True)
|
|
36
|
+
|
|
37
|
+
if 'zt_analysis' not in data_df.columns:
|
|
38
|
+
data_df['zt_analysis'] = ''
|
|
39
|
+
|
|
40
|
+
else:
|
|
41
|
+
data_df.fillna({'zt_analysis': ''}, inplace=True)
|
|
42
|
+
|
|
43
|
+
if 'zt_reason' not in data_df.columns:
|
|
44
|
+
data_df['zt_reason'] = ''
|
|
45
|
+
else:
|
|
46
|
+
data_df.fillna({'zt_reason': ''}, inplace=True)
|
|
47
|
+
|
|
48
|
+
if 'main_line_choose_source' not in data_df.columns:
|
|
49
|
+
data_df['main_line_choose_source'] = 'now_zt'
|
|
50
|
+
else:
|
|
51
|
+
data_df.fillna({'main_line_choose_source': 'now_zt'}, inplace=True)
|
|
52
|
+
|
|
53
|
+
if 'main_line_grade' not in data_df.columns:
|
|
54
|
+
data_df['main_line_grade'] = 1
|
|
55
|
+
else:
|
|
56
|
+
data_df.fillna({'main_line_grade': 1}, inplace=True)
|
|
57
|
+
|
|
58
|
+
query_zt_now = {'symbol': {"$in": list(data_df['symbol'])}, 'str_day': str_day}
|
|
59
|
+
# merge 主线 涨停详情
|
|
60
|
+
main_line_detail_df = mongodb_util.find_query_data(db_name_constant.MAIN_LINE_DETAIL, query_zt_now)
|
|
61
|
+
if data_frame_util.is_not_empty(main_line_detail_df):
|
|
62
|
+
symbol_mapping_zt_reason_now = dict(
|
|
63
|
+
zip(main_line_detail_df['symbol'], main_line_detail_df['zt_reason']))
|
|
64
|
+
|
|
65
|
+
symbol_mapping_zt_analysis_now = dict(
|
|
66
|
+
zip(main_line_detail_df['symbol'], main_line_detail_df['zt_analysis']))
|
|
67
|
+
|
|
68
|
+
symbol_mapping_main_line_now = dict(
|
|
69
|
+
zip(main_line_detail_df['symbol'], main_line_detail_df['main_line']))
|
|
70
|
+
|
|
71
|
+
symbol_mapping_sub_main_line_now = dict(
|
|
72
|
+
zip(main_line_detail_df['symbol'], main_line_detail_df['sub_main_line']))
|
|
73
|
+
|
|
74
|
+
symbol_mapping_main_line_choose_source = dict(
|
|
75
|
+
zip(main_line_detail_df['symbol'], main_line_detail_df['main_line_choose_source']))
|
|
76
|
+
|
|
77
|
+
symbol_mapping_main_line_grade = dict(
|
|
78
|
+
zip(main_line_detail_df['symbol'], main_line_detail_df['main_line_grade']))
|
|
79
|
+
|
|
80
|
+
data_df['main_line_grade'] = data_df['symbol'].map(
|
|
81
|
+
symbol_mapping_main_line_grade).fillna(
|
|
82
|
+
data_df['main_line_grade'])
|
|
83
|
+
|
|
84
|
+
data_df['main_line'] = data_df['symbol'].map(
|
|
85
|
+
symbol_mapping_main_line_now).fillna(
|
|
86
|
+
data_df['main_line'])
|
|
87
|
+
|
|
88
|
+
data_df['sub_main_line'] = data_df['symbol'].map(
|
|
89
|
+
symbol_mapping_sub_main_line_now).fillna(
|
|
90
|
+
data_df['sub_main_line'])
|
|
91
|
+
data_df['zt_reason'] = data_df['symbol'].map(
|
|
92
|
+
symbol_mapping_zt_reason_now).fillna(
|
|
93
|
+
data_df['zt_reason'])
|
|
94
|
+
data_df['zt_analysis'] = data_df['symbol'].map(
|
|
95
|
+
symbol_mapping_zt_analysis_now).fillna(
|
|
96
|
+
data_df['zt_analysis'])
|
|
97
|
+
|
|
98
|
+
data_df['main_line_choose_source'] = data_df['symbol'].map(
|
|
99
|
+
symbol_mapping_main_line_choose_source).fillna(
|
|
100
|
+
data_df['main_line_choose_source'])
|
|
101
|
+
|
|
102
|
+
return data_df
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
# merge涨停分析 原因
|
|
106
|
+
def merge_zt_reason_info(str_day, data_df):
|
|
107
|
+
if 'zt_analysis' not in data_df.columns:
|
|
108
|
+
data_df['zt_analysis'] = ''
|
|
109
|
+
|
|
110
|
+
else:
|
|
111
|
+
data_df.fillna({'zt_analysis': ''}, inplace=True)
|
|
112
|
+
|
|
113
|
+
if 'zt_reason' not in data_df.columns:
|
|
114
|
+
data_df['zt_reason'] = ''
|
|
115
|
+
else:
|
|
116
|
+
data_df.fillna({'zt_reason': ''}, inplace=True)
|
|
117
|
+
|
|
118
|
+
query_zt_now = {'symbol': {"$in": list(data_df['symbol'])}, 'str_day': str_day}
|
|
119
|
+
# merge 主线 涨停详情
|
|
120
|
+
zt_reason_analysis_df = mongodb_util.find_query_data(db_name_constant.ZT_REASON_ANALYSIS, query_zt_now)
|
|
121
|
+
if data_frame_util.is_not_empty(zt_reason_analysis_df):
|
|
122
|
+
symbol_mapping_zt_reason_now = dict(
|
|
123
|
+
zip(zt_reason_analysis_df['symbol'], zt_reason_analysis_df['zt_reason']))
|
|
124
|
+
|
|
125
|
+
symbol_mapping_zt_analysis_now = dict(
|
|
126
|
+
zip(zt_reason_analysis_df['symbol'], zt_reason_analysis_df['zt_analysis']))
|
|
127
|
+
|
|
128
|
+
data_df['zt_reason'] = data_df['symbol'].map(
|
|
129
|
+
symbol_mapping_zt_reason_now).fillna(
|
|
130
|
+
data_df['zt_reason'])
|
|
131
|
+
data_df['zt_analysis'] = data_df['symbol'].map(
|
|
132
|
+
symbol_mapping_zt_analysis_now).fillna(
|
|
133
|
+
data_df['zt_analysis'])
|
|
134
|
+
|
|
135
|
+
return data_df
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def update_zt_reason_analysis(symbol, str_day, name, need_update):
|
|
139
|
+
try:
|
|
140
|
+
key_id = symbol + "_" + str_day
|
|
141
|
+
query_zt = {"_id": key_id}
|
|
142
|
+
|
|
143
|
+
# 已经存在的数据
|
|
144
|
+
zt_reason_analysis_exists_df = mongodb_util.find_query_data(db_name_constant.ZT_REASON_ANALYSIS,
|
|
145
|
+
query_zt)
|
|
146
|
+
|
|
147
|
+
if data_frame_util.is_not_empty(zt_reason_analysis_exists_df):
|
|
148
|
+
zt_analysis = list(zt_reason_analysis_exists_df['zt_analysis'])[0]
|
|
149
|
+
zt_reason = list(zt_reason_analysis_exists_df['zt_reason'])[0]
|
|
150
|
+
# 需要更新数据
|
|
151
|
+
if data_frame_util.is_string_empty(zt_analysis) or data_frame_util.is_string_empty(
|
|
152
|
+
zt_reason) or need_update:
|
|
153
|
+
|
|
154
|
+
# web端更新数据
|
|
155
|
+
try:
|
|
156
|
+
web_zt_result_dict = ths_stock_zt_reason_web_api.get_ths_web_zt_reason_info(symbol,
|
|
157
|
+
cookie_info_service.get_ths_cookie())
|
|
158
|
+
zt_analysis = web_zt_result_dict['zt_analysis']
|
|
159
|
+
zt_reason = web_zt_result_dict['zt_reason']
|
|
160
|
+
time.sleep(1)
|
|
161
|
+
except BaseException as e:
|
|
162
|
+
time.sleep(1)
|
|
163
|
+
zt_analysis = ''
|
|
164
|
+
zt_reason = ''
|
|
165
|
+
|
|
166
|
+
# 问财更新数据
|
|
167
|
+
if data_frame_util.is_string_empty(zt_analysis) or data_frame_util.is_string_empty(
|
|
168
|
+
zt_reason):
|
|
169
|
+
try:
|
|
170
|
+
zt_result_dict = ths_stock_zt_pool_api.zt_analyse_reason(symbol)
|
|
171
|
+
zt_analysis = zt_result_dict['zt_analyse_detail']
|
|
172
|
+
zt_reason = zt_result_dict['zt_reason']
|
|
173
|
+
time.sleep(1)
|
|
174
|
+
except BaseException as e:
|
|
175
|
+
time.sleep(1)
|
|
176
|
+
zt_analysis = ''
|
|
177
|
+
zt_reason = ''
|
|
178
|
+
|
|
179
|
+
zt_reason_analysis_exists_df['zt_analysis'] = zt_analysis
|
|
180
|
+
zt_reason_analysis_exists_df['zt_reason'] = zt_reason
|
|
181
|
+
now_date = datetime.now()
|
|
182
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
183
|
+
zt_reason_analysis_exists_df['update_time'] = str_now_date
|
|
184
|
+
mongodb_util.save_mongo(zt_reason_analysis_exists_df, db_name_constant.ZT_REASON_ANALYSIS)
|
|
185
|
+
else:
|
|
186
|
+
# 不存在临时主线数据
|
|
187
|
+
try:
|
|
188
|
+
zt_result_dict = ths_stock_zt_pool_api.zt_analyse_reason(symbol)
|
|
189
|
+
zt_analysis = zt_result_dict['zt_analyse_detail']
|
|
190
|
+
zt_reason = zt_result_dict['zt_reason']
|
|
191
|
+
time.sleep(1)
|
|
192
|
+
except BaseException as e:
|
|
193
|
+
time.sleep(1)
|
|
194
|
+
zt_analysis = ''
|
|
195
|
+
zt_reason = ''
|
|
196
|
+
now_date = datetime.now()
|
|
197
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
198
|
+
reason_dict = {'_id': key_id,
|
|
199
|
+
'symbol': symbol,
|
|
200
|
+
'name': name,
|
|
201
|
+
'zt_analysis': zt_analysis,
|
|
202
|
+
'zt_reason': zt_reason,
|
|
203
|
+
'str_day': str_day,
|
|
204
|
+
'update_time': str_now_date,
|
|
205
|
+
}
|
|
206
|
+
reason_df = pd.DataFrame(reason_dict, index=[1])
|
|
207
|
+
mongodb_util.save_mongo(reason_df, db_name_constant.ZT_REASON_ANALYSIS)
|
|
208
|
+
except BaseException as e:
|
|
209
|
+
logger.error("添加涨停原因详情异常:{},{}", e, name)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def update_symbol_list_zt_reason_analysis(data_df, need_update):
|
|
213
|
+
for stock_one in data_df.itertuples():
|
|
214
|
+
str_day = stock_one.str_day
|
|
215
|
+
symbol = stock_one.symbol
|
|
216
|
+
name = stock_one.name
|
|
217
|
+
update_zt_reason_analysis(symbol, str_day, name, need_update)
|
|
218
|
+
if need_update:
|
|
219
|
+
time.sleep(5)
|
|
220
|
+
else:
|
|
221
|
+
time.sleep(1)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
# 保存连板股票主线
|
|
225
|
+
@async_fun
|
|
226
|
+
def save_last_trade_day_main_line(str_day, stock_em_zt_pool_df_data):
|
|
227
|
+
last_trade_day = trade_date_common_service_api.get_last_trade_day(str_day)
|
|
228
|
+
stock_em_zt_pool_connected_df = stock_em_zt_pool_df_data.loc[
|
|
229
|
+
stock_em_zt_pool_df_data['connected_boards_numbers'] > 1]
|
|
230
|
+
if data_frame_util.is_empty(stock_em_zt_pool_connected_df):
|
|
231
|
+
return
|
|
232
|
+
else:
|
|
233
|
+
query = {'str_day': last_trade_day, 'symbol': {"$in": list(stock_em_zt_pool_connected_df['symbol'])}}
|
|
234
|
+
last_trade_day_main_line_detail_df = mongodb_util.find_query_data(db_name_constant.MAIN_LINE_DETAIL, query)
|
|
235
|
+
if data_frame_util.is_empty(last_trade_day_main_line_detail_df):
|
|
236
|
+
return
|
|
237
|
+
else:
|
|
238
|
+
last_trade_day_main_line_detail_df['_id'] = last_trade_day_main_line_detail_df['symbol'] + '_' + str_day
|
|
239
|
+
last_trade_day_main_line_detail_df['str_day'] = str_day
|
|
240
|
+
last_trade_day_main_line_detail_df['connected_boards_numbers'] = last_trade_day_main_line_detail_df[
|
|
241
|
+
'connected_boards_numbers'] + 1
|
|
242
|
+
now_date = datetime.now()
|
|
243
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
244
|
+
last_trade_day_main_line_detail_df['update_time'] = str_now_date
|
|
245
|
+
today_exist_main_line_df = mongodb_util.find_query_data(db_name_constant.MAIN_LINE_DETAIL,
|
|
246
|
+
{'str_day': str_day, 'symbol': {"$in": list(
|
|
247
|
+
last_trade_day_main_line_detail_df['symbol'])}})
|
|
248
|
+
if data_frame_util.is_not_empty(today_exist_main_line_df):
|
|
249
|
+
today_new_main_line_df = last_trade_day_main_line_detail_df.loc[~
|
|
250
|
+
last_trade_day_main_line_detail_df['symbol'].isin(list(today_exist_main_line_df['symbol']))]
|
|
251
|
+
else:
|
|
252
|
+
today_new_main_line_df = last_trade_day_main_line_detail_df.copy()
|
|
253
|
+
mongodb_util.save_mongo(today_new_main_line_df, db_name_constant.MAIN_LINE_DETAIL)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
if __name__ == '__main__':
|
|
257
|
+
update_zt_reason_analysis('600362', '2025-12-26', '江西铜业', True)
|
|
@@ -7,78 +7,246 @@ project_path = file_path[0:end]
|
|
|
7
7
|
sys.path.append(project_path)
|
|
8
8
|
|
|
9
9
|
import mns_common.api.proxies.liu_guan_proxy_api as liu_guan_proxy_api
|
|
10
|
-
from functools import lru_cache
|
|
11
10
|
import pandas as pd
|
|
12
11
|
import mns_common.utils.data_frame_util as data_frame_util
|
|
13
12
|
from mns_common.db.MongodbUtil import MongodbUtil
|
|
14
13
|
import mns_common.constant.db_name_constant as db_name_constant
|
|
15
14
|
import datetime
|
|
15
|
+
import time
|
|
16
|
+
from loguru import logger
|
|
17
|
+
from functools import lru_cache
|
|
18
|
+
import mns_common.api.em.real_time.east_money_stock_a_api as east_money_stock_a_api
|
|
19
|
+
import threading
|
|
16
20
|
|
|
17
21
|
mongodb_util = MongodbUtil('27017')
|
|
18
22
|
|
|
23
|
+
IP_POOL = 'ip_pool'
|
|
24
|
+
ONE_IP = 'one_ip'
|
|
25
|
+
query_one = {'ip_type': ONE_IP}
|
|
26
|
+
query_pool = {'ip_type': IP_POOL}
|
|
27
|
+
|
|
19
28
|
|
|
20
|
-
@lru_cache(maxsize=None)
|
|
21
29
|
def query_liu_guan_proxy_ip():
|
|
22
|
-
ip_proxy_pool = mongodb_util.
|
|
30
|
+
ip_proxy_pool = mongodb_util.find_query_data(db_name_constant.IP_PROXY_POOL, query_one)
|
|
23
31
|
return ip_proxy_pool
|
|
24
32
|
|
|
25
33
|
|
|
26
|
-
def
|
|
27
|
-
|
|
28
|
-
mongodb_util.remove_data({}, db_name_constant.IP_PROXY_POOL)
|
|
34
|
+
def remove_one_proxy_ip():
|
|
35
|
+
mongodb_util.remove_data(query_one, db_name_constant.IP_PROXY_POOL)
|
|
29
36
|
|
|
30
37
|
|
|
31
|
-
def check_valid(
|
|
38
|
+
def check_valid(ip_proxy_pool):
|
|
32
39
|
effect_time = list(ip_proxy_pool['effect_time'])[0]
|
|
40
|
+
now_date = datetime.datetime.now()
|
|
41
|
+
str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
33
42
|
if effect_time > str_now_date:
|
|
34
43
|
return True
|
|
35
44
|
else:
|
|
36
|
-
|
|
45
|
+
remove_one_proxy_ip()
|
|
37
46
|
return False
|
|
38
47
|
|
|
39
48
|
|
|
49
|
+
@lru_cache(maxsize=None)
|
|
50
|
+
def get_account_cache():
|
|
51
|
+
query = {"type": "liu_guan_proxy"}
|
|
52
|
+
return mongodb_util.find_query_data(db_name_constant.STOCK_ACCOUNT_INFO, query)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def generate_proxy_ip_api(minutes):
|
|
56
|
+
try_numer = 3
|
|
57
|
+
while try_numer > 0:
|
|
58
|
+
try:
|
|
59
|
+
stock_account_info = get_account_cache()
|
|
60
|
+
order_id = list(stock_account_info['password'])[0]
|
|
61
|
+
secret = list(stock_account_info['account'])[0]
|
|
62
|
+
# 获取10分钟动态ip
|
|
63
|
+
liu_guan_ip = liu_guan_proxy_api.get_proxy_api(order_id, secret, str(60 * minutes))
|
|
64
|
+
try_numer = try_numer
|
|
65
|
+
logger.info("生成新的ip:{}", liu_guan_ip)
|
|
66
|
+
return liu_guan_ip
|
|
67
|
+
except BaseException as e:
|
|
68
|
+
logger.error("获取ip失败:{}", str(e))
|
|
69
|
+
time.sleep(1)
|
|
70
|
+
continue
|
|
71
|
+
|
|
72
|
+
|
|
40
73
|
def generate_proxy_ip(minutes):
|
|
41
|
-
ip_proxy_pool =
|
|
74
|
+
ip_proxy_pool = query_liu_guan_proxy_ip()
|
|
42
75
|
if data_frame_util.is_not_empty(ip_proxy_pool):
|
|
43
76
|
return list(ip_proxy_pool['ip'])[0]
|
|
44
77
|
else:
|
|
45
|
-
|
|
46
|
-
stock_account_info = mongodb_util.find_query_data(db_name_constant.STOCK_ACCOUNT_INFO, query)
|
|
47
|
-
order_id = list(stock_account_info['password'])[0]
|
|
48
|
-
secret = list(stock_account_info['account'])[0]
|
|
49
|
-
|
|
78
|
+
remove_one_proxy_ip()
|
|
50
79
|
now_date = datetime.datetime.now()
|
|
51
|
-
|
|
80
|
+
# 加上分钟
|
|
81
|
+
time_to_add = datetime.timedelta(minutes=minutes)
|
|
52
82
|
new_date = now_date + time_to_add
|
|
53
83
|
str_now_date = new_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
84
|
+
try:
|
|
85
|
+
ip_proxy = generate_proxy_ip_api(minutes)
|
|
54
86
|
|
|
55
|
-
|
|
56
|
-
|
|
87
|
+
result_dict = {"_id": ip_proxy,
|
|
88
|
+
'ip_type': ONE_IP,
|
|
89
|
+
'effect_time': str_now_date,
|
|
90
|
+
'ip': ip_proxy}
|
|
91
|
+
result_df = pd.DataFrame(result_dict, index=[1])
|
|
57
92
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
result_df = pd.DataFrame(result_dict, index=[1])
|
|
93
|
+
mongodb_util.insert_mongo(result_df, db_name_constant.IP_PROXY_POOL)
|
|
94
|
+
except BaseException as e:
|
|
95
|
+
logger.error("获取ip失败:{}", str(e))
|
|
62
96
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
return ip
|
|
97
|
+
return ip_proxy
|
|
66
98
|
|
|
67
99
|
|
|
68
|
-
def get_proxy_ip(
|
|
100
|
+
def get_proxy_ip(minutes):
|
|
69
101
|
ip_proxy_pool = query_liu_guan_proxy_ip()
|
|
70
102
|
if data_frame_util.is_empty(ip_proxy_pool):
|
|
71
103
|
return generate_proxy_ip(minutes)
|
|
72
104
|
else:
|
|
73
|
-
if check_valid(
|
|
105
|
+
if check_valid(ip_proxy_pool):
|
|
74
106
|
return list(ip_proxy_pool['ip'])[0]
|
|
75
107
|
else:
|
|
76
108
|
return generate_proxy_ip(minutes)
|
|
77
109
|
|
|
78
110
|
|
|
79
|
-
|
|
80
|
-
|
|
111
|
+
#
|
|
112
|
+
# def check_proxy(proxy_ip):
|
|
113
|
+
# try:
|
|
114
|
+
# # 两秒超时
|
|
115
|
+
# test_df = call_with_timeout(get_em_real_time_data, proxy_ip, timeout=2)
|
|
116
|
+
# if data_frame_util.is_not_empty(test_df):
|
|
117
|
+
# logger.info("可用代理ip:{}", proxy_ip)
|
|
118
|
+
# return True
|
|
119
|
+
# else:
|
|
120
|
+
# return False
|
|
121
|
+
# except Exception as e:
|
|
122
|
+
# logger.error("代理ip不可用:{},{}", proxy_ip, e)
|
|
123
|
+
# return False
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# 查询ip池子
|
|
127
|
+
def query_liu_guan_proxy_ip_pool():
|
|
128
|
+
ip_proxy_pool = mongodb_util.find_query_data(db_name_constant.IP_PROXY_POOL, query_pool)
|
|
129
|
+
return ip_proxy_pool
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def remove_proxy_ip_pool():
|
|
133
|
+
mongodb_util.remove_data(query_pool, db_name_constant.IP_PROXY_POOL)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def generate_proxy_ip_pool_api(minutes, ip_num):
|
|
137
|
+
stock_account_info = get_account_cache()
|
|
138
|
+
order_id = list(stock_account_info['password'])[0]
|
|
139
|
+
secret = list(stock_account_info['account'])[0]
|
|
140
|
+
# 获取10分钟动态ip
|
|
141
|
+
ip_pool = liu_guan_proxy_api.get_proxy_pool_api(order_id, secret, str(60 * minutes), ip_num)
|
|
142
|
+
return ip_pool
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def get_proxy_ip_pool(minutes, seconds, ip_num):
|
|
146
|
+
ip_proxy_pool = query_liu_guan_proxy_ip_pool()
|
|
147
|
+
if data_frame_util.is_empty(ip_proxy_pool):
|
|
148
|
+
return generate_proxy_ip_pool(minutes, seconds, ip_num)
|
|
149
|
+
else:
|
|
150
|
+
if check_valid(ip_proxy_pool):
|
|
151
|
+
ip_pool = list(ip_proxy_pool['ip_pool'])[0]
|
|
152
|
+
effect_time = list(ip_proxy_pool['effect_time'])[0]
|
|
153
|
+
result = {'ip_pool': ip_pool,
|
|
154
|
+
'effect_time': effect_time}
|
|
155
|
+
return result
|
|
156
|
+
else:
|
|
157
|
+
# 已经失效 移除ip pool
|
|
158
|
+
remove_proxy_ip_pool()
|
|
159
|
+
# 重新生成
|
|
160
|
+
return generate_proxy_ip_pool(minutes, seconds, ip_num)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
# seconds 有效秒数,minutes 需要减1
|
|
164
|
+
def generate_proxy_ip_pool(minutes, seconds, ip_num):
|
|
165
|
+
ip_proxy_pool = query_liu_guan_proxy_ip_pool()
|
|
166
|
+
if data_frame_util.is_not_empty(ip_proxy_pool):
|
|
167
|
+
ip_pool = list(ip_proxy_pool['ip_pool'])[0]
|
|
168
|
+
effect_time = list(ip_proxy_pool['effect_time'])[0]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
else:
|
|
172
|
+
remove_proxy_ip_pool()
|
|
81
173
|
now_date = datetime.datetime.now()
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
174
|
+
# 加上分钟 少10秒
|
|
175
|
+
time_to_add = datetime.timedelta(minutes=minutes - 1, seconds=seconds)
|
|
176
|
+
new_date = now_date + time_to_add
|
|
177
|
+
effect_time = new_date.strftime('%Y-%m-%d %H:%M:%S')
|
|
178
|
+
ip_pool = generate_proxy_ip_pool_api(minutes, ip_num)
|
|
179
|
+
result_dict = {
|
|
180
|
+
"_id": [IP_POOL],
|
|
181
|
+
'ip_type': [IP_POOL],
|
|
182
|
+
'effect_time': [effect_time],
|
|
183
|
+
'ip_pool': [ip_pool] # 每个字段都包装成列表
|
|
184
|
+
}
|
|
185
|
+
result_df = pd.DataFrame(result_dict)
|
|
186
|
+
|
|
187
|
+
mongodb_util.insert_mongo(result_df, db_name_constant.IP_PROXY_POOL)
|
|
188
|
+
result = {'ip_pool': ip_pool,
|
|
189
|
+
'effect_time': effect_time}
|
|
190
|
+
return result
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def get_em_real_time_data(proxy_ip):
|
|
194
|
+
proxies = {
|
|
195
|
+
"http": proxy_ip,
|
|
196
|
+
"https": proxy_ip
|
|
197
|
+
}
|
|
198
|
+
return east_money_stock_a_api.get_stock_page_data(1, proxies, 20, 10)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
# 定义一个带超时的函数调用
|
|
202
|
+
def call_with_timeout(func, *args, timeout=2, **kwargs):
|
|
203
|
+
# 用于存储函数执行结果
|
|
204
|
+
result = None
|
|
205
|
+
exception = None
|
|
206
|
+
|
|
207
|
+
# 定义一个线程目标函数
|
|
208
|
+
def target():
|
|
209
|
+
nonlocal result, exception
|
|
210
|
+
try:
|
|
211
|
+
result = func(*args, **kwargs)
|
|
212
|
+
except Exception as e:
|
|
213
|
+
exception = e
|
|
214
|
+
|
|
215
|
+
# 创建线程并启动
|
|
216
|
+
thread = threading.Thread(target=target)
|
|
217
|
+
thread.start()
|
|
218
|
+
|
|
219
|
+
# 等待线程完成,最多等待 timeout 秒
|
|
220
|
+
thread.join(timeout)
|
|
221
|
+
|
|
222
|
+
# 如果线程仍然存活,说明函数超时了
|
|
223
|
+
if thread.is_alive():
|
|
224
|
+
raise TimeoutError(f"Function exceeded timeout of {timeout} seconds")
|
|
225
|
+
|
|
226
|
+
# 如果函数抛出了异常,重新抛出
|
|
227
|
+
if exception is not None:
|
|
228
|
+
raise exception
|
|
229
|
+
return result
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
@lru_cache(maxsize=None)
|
|
233
|
+
def query_province_and_city_info():
|
|
234
|
+
return mongodb_util.find_all_data(db_name_constant.IP_PROXY_CITY_PROVINCE)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def import_province_and_city():
|
|
238
|
+
# 设置文件夹路径
|
|
239
|
+
folder_path = r'E:\province-and-city.xlsx'
|
|
240
|
+
df = pd.read_excel(folder_path)
|
|
241
|
+
df['_id'] = df['cid']
|
|
242
|
+
|
|
243
|
+
mongodb_util.save_mongo(df, db_name_constant.IP_PROXY_CITY_PROVINCE)
|
|
244
|
+
return df
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
if __name__ == "__main__":
|
|
248
|
+
stock_account_info_test = get_account_cache()
|
|
249
|
+
order_id_test = list(stock_account_info_test['password'])[0]
|
|
250
|
+
secret_test = list(stock_account_info_test['account'])[0]
|
|
251
|
+
# 获取10分钟动态ip
|
|
252
|
+
ip = liu_guan_proxy_api.get_proxy_api(order_id_test, secret_test, str(60 * 1))
|
|
@@ -5,7 +5,6 @@ file_path = os.path.abspath(__file__)
|
|
|
5
5
|
end = file_path.index('mns') + 16
|
|
6
6
|
project_path = file_path[0:end]
|
|
7
7
|
sys.path.append(project_path)
|
|
8
|
-
import akshare as ak
|
|
9
8
|
import mns_common.utils.date_handle_util as date_handle_util
|
|
10
9
|
from loguru import logger
|
|
11
10
|
import mns_common.utils.data_frame_util as data_frame_util
|
|
@@ -15,20 +14,87 @@ import mns_common.constant.db_name_constant as db_name_constant
|
|
|
15
14
|
|
|
16
15
|
mongodb_util = MongodbUtil('27017')
|
|
17
16
|
|
|
17
|
+
"""
|
|
18
|
+
Date: 2024/4/29 15:00
|
|
19
|
+
Desc: 东方财富网-数据中心-特色数据-停复牌信息
|
|
20
|
+
https://data.eastmoney.com/tfpxx/
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import pandas as pd
|
|
24
|
+
import requests
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def stock_tfp_em(date: str = "20240426") -> pd.DataFrame:
|
|
28
|
+
"""
|
|
29
|
+
东方财富网-数据中心-特色数据-停复牌信息
|
|
30
|
+
https://data.eastmoney.com/tfpxx/
|
|
31
|
+
:param date: specific date as "2020-03-19"
|
|
32
|
+
:type date: str
|
|
33
|
+
:return: 停复牌信息表
|
|
34
|
+
:rtype: pandas.DataFrame
|
|
35
|
+
"""
|
|
36
|
+
url = "https://datacenter-web.eastmoney.com/api/data/v1/get"
|
|
37
|
+
params = {
|
|
38
|
+
"sortColumns": "SUSPEND_START_DATE",
|
|
39
|
+
"sortTypes": "-1",
|
|
40
|
+
"pageSize": "500",
|
|
41
|
+
"pageNumber": "1",
|
|
42
|
+
"reportName": "RPT_CUSTOM_SUSPEND_DATA_INTERFACE",
|
|
43
|
+
"columns": "ALL",
|
|
44
|
+
"source": "WEB",
|
|
45
|
+
"client": "WEB",
|
|
46
|
+
"filter": f"""(MARKET="全部")(DATETIME='{"-".join([date[:4], date[4:6], date[6:]])}')""",
|
|
47
|
+
}
|
|
48
|
+
r = requests.get(url, params=params)
|
|
49
|
+
data_json = r.json()
|
|
50
|
+
total_page = data_json["result"]["pages"]
|
|
51
|
+
big_df = pd.DataFrame()
|
|
52
|
+
for page in range(1, total_page + 1):
|
|
53
|
+
params.update({"pageNumber": page})
|
|
54
|
+
r = requests.get(url, params=params)
|
|
55
|
+
data_json = r.json()
|
|
56
|
+
temp_df = pd.DataFrame(data_json["result"]["data"])
|
|
57
|
+
big_df = pd.concat(objs=[big_df, temp_df], ignore_index=True)
|
|
58
|
+
|
|
59
|
+
big_df.reset_index(inplace=True)
|
|
60
|
+
|
|
61
|
+
big_df["SUSPEND_START_TIME"] = pd.to_datetime(big_df["SUSPEND_START_TIME"], errors="coerce").dt.date
|
|
62
|
+
big_df["SUSPEND_END_TIME"] = pd.to_datetime(
|
|
63
|
+
big_df["SUSPEND_END_TIME"], errors="coerce"
|
|
64
|
+
).dt.date
|
|
65
|
+
|
|
66
|
+
big_df["SUSPEND_START_DATE"] = pd.to_datetime(
|
|
67
|
+
big_df["SUSPEND_START_DATE"], errors="coerce"
|
|
68
|
+
).dt.date
|
|
69
|
+
big_df["PREDICT_RESUME_DATE"] = pd.to_datetime(
|
|
70
|
+
big_df["PREDICT_RESUME_DATE"], errors="coerce"
|
|
71
|
+
).dt.date
|
|
72
|
+
|
|
73
|
+
big_df = big_df[['index', 'SECURITY_CODE', 'SECURITY_NAME_ABBR', 'SUSPEND_START_TIME',
|
|
74
|
+
'SUSPEND_END_TIME', 'SUSPEND_EXPIRE', 'SUSPEND_REASON', 'TRADE_MARKET',
|
|
75
|
+
'SUSPEND_START_DATE',
|
|
76
|
+
'PREDICT_RESUME_DATE'
|
|
77
|
+
]]
|
|
78
|
+
|
|
79
|
+
return big_df
|
|
80
|
+
|
|
18
81
|
|
|
19
82
|
def get_stock_tfp_by_day(str_day):
|
|
20
|
-
stock_tfp_em_df =
|
|
83
|
+
stock_tfp_em_df = stock_tfp_em(date_handle_util.no_slash_date(str_day))
|
|
21
84
|
stock_tfp_em_df = stock_tfp_em_df.rename(
|
|
22
|
-
columns={'
|
|
23
|
-
'
|
|
24
|
-
'
|
|
25
|
-
'
|
|
26
|
-
'
|
|
27
|
-
|
|
28
|
-
'
|
|
29
|
-
'
|
|
30
|
-
|
|
31
|
-
'
|
|
85
|
+
columns={'index': 'index',
|
|
86
|
+
'SECURITY_CODE': 'symbol',
|
|
87
|
+
'SECURITY_NAME_ABBR': 'name',
|
|
88
|
+
'SUSPEND_START_TIME': 'sus_begin_time',
|
|
89
|
+
'SUSPEND_END_TIME': 'sus_end_time',
|
|
90
|
+
|
|
91
|
+
'SUSPEND_START_DATE': 'sus_begin_date',
|
|
92
|
+
'PREDICT_RESUME_DATE': 'resume_time',
|
|
93
|
+
|
|
94
|
+
'SUSPEND_EXPIRE': 'sus_period',
|
|
95
|
+
'SUSPEND_REASON': 'sus_reason',
|
|
96
|
+
'TRADE_MARKET': 'market',
|
|
97
|
+
|
|
32
98
|
})
|
|
33
99
|
return stock_tfp_em_df
|
|
34
100
|
|
|
@@ -60,3 +126,7 @@ def get_stock_tfp_symbol_from_db(str_day):
|
|
|
60
126
|
except BaseException as e:
|
|
61
127
|
logger.error("获取停牌信息异常:{}", e)
|
|
62
128
|
return ['666666']
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
if __name__ == '__main__':
|
|
132
|
+
get_stock_tfp_symbol_list_by_day('2025-06-21')
|