mns-common 1.3.3.5__py3-none-any.whl → 1.5.7.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. mns_common/api/akshare/__init__.py +0 -1
  2. mns_common/api/akshare/k_line_api.py +20 -82
  3. mns_common/api/akshare/stock_bid_ask_api.py +22 -97
  4. mns_common/api/akshare/stock_zb_pool.py +2 -0
  5. mns_common/api/akshare/stock_zt_pool_api.py +1 -1
  6. mns_common/api/em/gd/__init__.py +7 -0
  7. mns_common/api/em/{east_money_stock_gdfx_free_top_10_api.py → gd/east_money_stock_gdfx_free_top_10_api.py} +64 -9
  8. mns_common/api/em/real_time/__init__.py +7 -0
  9. mns_common/api/em/{east_money_debt_api.py → real_time/east_money_debt_api.py} +154 -69
  10. mns_common/api/em/{east_money_etf_api.py → real_time/east_money_etf_api.py} +149 -27
  11. mns_common/api/em/real_time/east_money_stock_a_api.py +301 -0
  12. mns_common/api/em/real_time/east_money_stock_a_v2_api.py +340 -0
  13. mns_common/api/em/real_time/east_money_stock_common_api.py +174 -0
  14. mns_common/api/em/real_time/east_money_stock_hk_api.py +288 -0
  15. mns_common/api/em/real_time/east_money_stock_hk_gtt_api.py +260 -0
  16. mns_common/api/em/real_time/east_money_stock_multi_thread_api_v3.py +154 -0
  17. mns_common/api/em/{east_money_stock_us_api.py → real_time/east_money_stock_us_api.py} +149 -72
  18. mns_common/api/em/real_time/real_time_quotes_repeat_api.py +195 -0
  19. mns_common/api/k_line/stock_k_line_data_api.py +11 -1
  20. mns_common/api/k_line/stock_minute_data_api.py +1 -0
  21. mns_common/api/kpl/common/kpl_common_api.py +35 -0
  22. mns_common/api/kpl/symbol/symbol_his_quotes_api.py +1 -1
  23. mns_common/api/proxies/__init__.py +7 -0
  24. mns_common/api/proxies/liu_guan_proxy_api.py +115 -0
  25. mns_common/api/ths/company/company_product_area_industry_index_query.py +46 -0
  26. mns_common/api/ths/company/ths_company_info_api.py +13 -9
  27. mns_common/api/ths/company/ths_company_info_web.py +159 -0
  28. mns_common/api/ths/concept/app/ths_concept_index_app.py +3 -1
  29. mns_common/api/ths/wen_cai/ths_wen_cai_api.py +10 -7
  30. mns_common/api/ths/zt/ths_stock_zt_pool_api.py +21 -4
  31. mns_common/api/ths/zt/ths_stock_zt_pool_v2_api.py +111 -40
  32. mns_common/api/xueqiu/__init__.py +7 -0
  33. mns_common/api/xueqiu/xue_qiu_k_line_api.py +83 -0
  34. mns_common/component/__init__.py +1 -1
  35. mns_common/component/classify/symbol_classify_api.py +7 -7
  36. mns_common/component/common_service_fun_api.py +66 -6
  37. mns_common/component/company/company_common_service_api.py +21 -1
  38. mns_common/component/company/company_common_service_new_api.py +4 -1
  39. mns_common/component/cookie/cookie_enum.py +15 -0
  40. mns_common/component/cookie/cookie_info_service.py +9 -4
  41. mns_common/component/data/data_init_api.py +13 -8
  42. mns_common/component/deal/deal_service_api.py +70 -8
  43. mns_common/component/deal/deal_service_v2_api.py +167 -0
  44. mns_common/component/em/__init__.py +7 -0
  45. mns_common/component/em/em_real_time_quotes_api.py +56 -0
  46. mns_common/component/em/em_stock_info_api.py +48 -0
  47. mns_common/component/exception/ExceptionMonitor.py +86 -0
  48. mns_common/component/exception/__init__.py +7 -0
  49. mns_common/component/k_line/common/k_line_common_service_api.py +4 -0
  50. mns_common/component/main_line/__init__.py +7 -0
  51. mns_common/component/main_line/main_line_zt_reason_service.py +237 -0
  52. mns_common/component/proxies/__init__.py +7 -0
  53. mns_common/component/proxies/proxy_common_api.py +252 -0
  54. mns_common/component/self_choose/__init__.py +13 -0
  55. mns_common/component/tfp/stock_tfp_api.py +82 -12
  56. mns_common/component/us/__init__.py +7 -0
  57. mns_common/component/us/us_stock_etf_info_api.py +125 -0
  58. mns_common/constant/__init__.py +1 -0
  59. mns_common/constant/db_name_constant.py +65 -34
  60. mns_common/constant/extra_income_db_name.py +154 -0
  61. mns_common/constant/strategy_classify.py +72 -0
  62. mns_common/db/MongodbUtil.py +2 -1
  63. mns_common/db/MongodbUtilLocal.py +1 -0
  64. mns_common/db/v2/MongodbUtilV2.py +0 -4
  65. mns_common-1.5.7.2.dist-info/METADATA +4 -0
  66. {mns_common-1.3.3.5.dist-info → mns_common-1.5.7.2.dist-info}/RECORD +70 -45
  67. {mns_common-1.3.3.5.dist-info → mns_common-1.5.7.2.dist-info}/WHEEL +1 -1
  68. mns_common/api/em/east_money_stock_api.py +0 -222
  69. mns_common/api/em/east_money_stock_hk_api.py +0 -318
  70. mns_common/api/em/east_money_stock_v2_api.py +0 -219
  71. mns_common/api/ths/concept/web/ths_company_info_web.py +0 -163
  72. mns_common/component/qmt/qmt_buy_service.py +0 -172
  73. mns_common-1.3.3.5.dist-info/METADATA +0 -4
  74. /mns_common/{component/qmt → api/em/concept}/__init__.py +0 -0
  75. /mns_common/api/em/{em_concept_index_api.py → concept/em_concept_index_api.py} +0 -0
  76. {mns_common-1.3.3.5.dist-info → mns_common-1.5.7.2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,301 @@
1
+ import sys
2
+ import os
3
+ import time
4
+
5
+ file_path = os.path.abspath(__file__)
6
+ end = file_path.index('mns') + 16
7
+ project_path = file_path[0:end]
8
+ sys.path.append(project_path)
9
+ from mns_common.db.MongodbUtil import MongodbUtil
10
+ import requests
11
+ import json
12
+ import pandas as pd
13
+ from concurrent.futures import ThreadPoolExecutor
14
+ import datetime
15
+ from loguru import logger
16
+ import mns_common.utils.data_frame_util as data_frame_util
17
+
18
+ mongodb_util = MongodbUtil('27017')
19
+ fields = ("f352,f2,f3,f5,f6,f8,f10,f11,f22,f12,f14,f15,f16,f17,"
20
+ "f18,f20,f21,f26,f33,f34,f35,f62,f66,f69,f72,f100,f184,f211,f212"),
21
+ fs = "m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048"
22
+
23
+ # 最大返回条数
24
+ max_number = 5800
25
+ # 最小返回条数
26
+ min_number = 5600
27
+ # 分页条数
28
+ PAGE_SIZE = 100
29
+
30
+
31
+ def get_stock_page_data(pn, proxies, page_size, time_out):
32
+ """
33
+ 获取单页股票数据
34
+ """
35
+ # 获取当前日期和时间
36
+ current_time = datetime.datetime.now()
37
+
38
+ # 将当前时间转换为时间戳(以毫秒为单位)
39
+ current_timestamp_ms = int(current_time.timestamp() * 1000)
40
+
41
+ url = "https://33.push2.eastmoney.com/api/qt/clist/get"
42
+ params = {
43
+ "cb": "jQuery1124046660442520420653_" + str(current_timestamp_ms),
44
+ "pn": str(pn),
45
+ "pz": str(page_size), # 每页最大200条
46
+ "po": "0",
47
+ "np": "3",
48
+ "ut": "bd1d9ddb04089700cf9c27f6f7426281",
49
+ "fltt": "2",
50
+ "invt": "2",
51
+ "wbp2u": "|0|0|0|web",
52
+ "fid": "f12",
53
+ "fs": fs,
54
+ "fields": fields,
55
+ "_": current_timestamp_ms
56
+ }
57
+ try:
58
+ if proxies is None:
59
+ r = requests.get(url, params, timeout=time_out)
60
+ else:
61
+ r = requests.get(url, params, proxies=proxies, timeout=time_out)
62
+
63
+ data_text = r.text
64
+
65
+
66
+ begin_index = data_text.index('[')
67
+ end_index = data_text.index(']')
68
+ data_json = data_text[begin_index:end_index + 1]
69
+ data_json = json.loads(data_json)
70
+ if data_json is None:
71
+ return pd.DataFrame()
72
+ else:
73
+ result_df = pd.DataFrame(data_json)
74
+ result_df['page_number'] = pn
75
+ return result_df
76
+ except Exception as e:
77
+ # logger.error("获取第{}页股票列表异常:{}", pn, str(e))
78
+ return pd.DataFrame()
79
+
80
+
81
+ def all_stock_ticker_data_new(proxies, time_out) -> pd.DataFrame:
82
+ """
83
+ 使用多线程获取所有股票数据
84
+ """
85
+
86
+ per_page = PAGE_SIZE
87
+ total_pages = (max_number + per_page - 1) // per_page # 向上取整
88
+
89
+ # 创建线程池
90
+ with ThreadPoolExecutor(max_workers=10) as executor:
91
+ # 提交任务,获取每页数据
92
+ futures = [executor.submit(get_stock_page_data, pn, proxies, PAGE_SIZE, time_out)
93
+ for pn in range(1, total_pages + 1)]
94
+
95
+ # 收集结果
96
+ results = []
97
+ for future in futures:
98
+ result = future.result()
99
+ if not result.empty:
100
+ results.append(result)
101
+
102
+ # 合并所有页面的数据
103
+ if results:
104
+ return pd.concat(results, ignore_index=True)
105
+ else:
106
+ return pd.DataFrame()
107
+
108
+
109
+ def get_real_time_quotes_all_stocks(proxies, time_out):
110
+ page_df = all_stock_ticker_data_new(proxies, time_out)
111
+ page_df = rename_real_time_quotes_df(page_df)
112
+ page_df.drop_duplicates('symbol', keep='last', inplace=True)
113
+ return page_df
114
+
115
+
116
+ # 获取所有股票实时行情数据 f33,委比
117
+ def rename_real_time_quotes_df(temp_df):
118
+ temp_df = temp_df.rename(columns={
119
+ "f2": "now_price",
120
+ "f3": "chg",
121
+ "f5": "volume",
122
+ "f6": "amount",
123
+ "f8": "exchange",
124
+ "f10": "quantity_ratio",
125
+ "f22": "up_speed",
126
+ "f11": "up_speed_05",
127
+ "f12": "symbol",
128
+ "f14": "name",
129
+ "f15": "high",
130
+ "f16": "low",
131
+ "f17": "open",
132
+ "f18": "yesterday_price",
133
+ "f20": "total_mv",
134
+ "f21": "flow_mv",
135
+ "f26": "list_date",
136
+ "f33": "wei_bi",
137
+ "f34": "outer_disk",
138
+ "f35": "inner_disk",
139
+ "f62": "today_main_net_inflow",
140
+ "f66": "super_large_order_net_inflow",
141
+ "f69": "super_large_order_net_inflow_ratio",
142
+ "f72": "large_order_net_inflow",
143
+ # "f78": "medium_order_net_inflow",
144
+ # "f84": "small_order_net_inflow",
145
+ "f100": "industry",
146
+ # "f103": "concept",
147
+ "f184": "today_main_net_inflow_ratio",
148
+ "f352": "average_price",
149
+ "f211": "buy_1_num",
150
+ "f212": "sell_1_num"
151
+ })
152
+ if data_frame_util.is_empty(temp_df):
153
+ return pd.DataFrame()
154
+ else:
155
+ temp_df.loc[temp_df['buy_1_num'] == '-', 'buy_1_num'] = 0
156
+ temp_df.loc[temp_df['sell_1_num'] == '-', 'sell_1_num'] = 0
157
+ temp_df.loc[temp_df['up_speed_05'] == '-', 'up_speed_05'] = 0
158
+ temp_df.loc[temp_df['up_speed'] == '-', 'up_speed'] = 0
159
+ temp_df.loc[temp_df['average_price'] == '-', 'average_price'] = 0
160
+ temp_df.loc[temp_df['wei_bi'] == '-', 'wei_bi'] = 0
161
+ temp_df.loc[temp_df['yesterday_price'] == '-', 'yesterday_price'] = 0
162
+ temp_df.loc[temp_df['now_price'] == '-', 'now_price'] = 0
163
+ temp_df.loc[temp_df['chg'] == '-', 'chg'] = 0
164
+ temp_df.loc[temp_df['volume'] == '-', 'volume'] = 0
165
+ temp_df.loc[temp_df['amount'] == '-', 'amount'] = 0
166
+ temp_df.loc[temp_df['exchange'] == '-', 'exchange'] = 0
167
+ temp_df.loc[temp_df['quantity_ratio'] == '-', 'quantity_ratio'] = 0
168
+ temp_df.loc[temp_df['high'] == '-', 'high'] = 0
169
+ temp_df.loc[temp_df['low'] == '-', 'low'] = 0
170
+ temp_df.loc[temp_df['open'] == '-', 'open'] = 0
171
+ temp_df.loc[temp_df['total_mv'] == '-', 'total_mv'] = 0
172
+ temp_df.loc[temp_df['flow_mv'] == '-', 'flow_mv'] = 0
173
+ temp_df.loc[temp_df['inner_disk'] == '-', 'inner_disk'] = 0
174
+ temp_df.loc[temp_df['outer_disk'] == '-', 'outer_disk'] = 0
175
+ temp_df.loc[temp_df['today_main_net_inflow_ratio'] == '-', 'today_main_net_inflow_ratio'] = 0
176
+ temp_df.loc[temp_df['today_main_net_inflow'] == '-', 'today_main_net_inflow'] = 0
177
+ temp_df.loc[temp_df['super_large_order_net_inflow'] == '-', 'super_large_order_net_inflow'] = 0
178
+ temp_df.loc[temp_df['super_large_order_net_inflow_ratio'] == '-', 'super_large_order_net_inflow_ratio'] = 0
179
+ temp_df.loc[temp_df['large_order_net_inflow'] == '-', 'large_order_net_inflow'] = 0
180
+ # temp_df.loc[temp_df['medium_order_net_inflow'] == '-', 'medium_order_net_inflow'] = 0
181
+ # temp_df.loc[temp_df['small_order_net_inflow'] == '-', 'small_order_net_inflow'] = 0
182
+
183
+ temp_df["list_date"] = pd.to_numeric(temp_df["list_date"], errors="coerce")
184
+ temp_df["wei_bi"] = pd.to_numeric(temp_df["wei_bi"], errors="coerce")
185
+ temp_df["average_price"] = pd.to_numeric(temp_df["average_price"], errors="coerce")
186
+ temp_df["yesterday_price"] = pd.to_numeric(temp_df["yesterday_price"], errors="coerce")
187
+ temp_df["now_price"] = pd.to_numeric(temp_df["now_price"], errors="coerce")
188
+ temp_df["chg"] = pd.to_numeric(temp_df["chg"], errors="coerce")
189
+ temp_df["volume"] = pd.to_numeric(temp_df["volume"], errors="coerce")
190
+ temp_df["amount"] = pd.to_numeric(temp_df["amount"], errors="coerce")
191
+ temp_df["exchange"] = pd.to_numeric(temp_df["exchange"], errors="coerce")
192
+ temp_df["quantity_ratio"] = pd.to_numeric(temp_df["quantity_ratio"], errors="coerce")
193
+ temp_df["high"] = pd.to_numeric(temp_df["high"], errors="coerce")
194
+ temp_df["low"] = pd.to_numeric(temp_df["low"], errors="coerce")
195
+ temp_df["open"] = pd.to_numeric(temp_df["open"], errors="coerce")
196
+ temp_df["total_mv"] = pd.to_numeric(temp_df["total_mv"], errors="coerce")
197
+ temp_df["flow_mv"] = pd.to_numeric(temp_df["flow_mv"], errors="coerce")
198
+ temp_df["outer_disk"] = pd.to_numeric(temp_df["outer_disk"], errors="coerce")
199
+ temp_df["inner_disk"] = pd.to_numeric(temp_df["inner_disk"], errors="coerce")
200
+ temp_df["today_main_net_inflow"] = pd.to_numeric(temp_df["today_main_net_inflow"], errors="coerce")
201
+ temp_df["super_large_order_net_inflow"] = pd.to_numeric(temp_df["super_large_order_net_inflow"],
202
+ errors="coerce")
203
+ temp_df["super_large_order_net_inflow_ratio"] = pd.to_numeric(temp_df["super_large_order_net_inflow_ratio"],
204
+ errors="coerce")
205
+ temp_df["large_order_net_inflow"] = pd.to_numeric(temp_df["large_order_net_inflow"],
206
+ errors="coerce")
207
+ # temp_df["medium_order_net_inflow"] = pd.to_numeric(temp_df["medium_order_net_inflow"],
208
+ # errors="coerce")
209
+ # temp_df["small_order_net_inflow"] = pd.to_numeric(temp_df["small_order_net_inflow"], errors="coerce")
210
+
211
+ # 大单比例
212
+ temp_df['large_order_net_inflow_ratio'] = round((temp_df['large_order_net_inflow'] / temp_df['amount']) * 100,
213
+ 2)
214
+
215
+ # 外盘是内盘倍数
216
+ temp_df['disk_ratio'] = round((temp_df['outer_disk'] - temp_df['inner_disk']) / temp_df['inner_disk'], 2)
217
+ # 只有外盘没有内盘
218
+ temp_df.loc[temp_df["inner_disk"] == 0, ['disk_ratio']] = 1688
219
+ temp_df = temp_df.sort_values(by=['chg'], ascending=False)
220
+ return temp_df
221
+
222
+
223
+ # 北向/南向资金状况 北向已经停止
224
+ def get_sum_north_south_net_buy_amt():
225
+ # 设置请求头部信息
226
+ headers = {
227
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
228
+ }
229
+
230
+ # 设置请求URL
231
+ url = 'http://push2.eastmoney.com/api/qt/kamt/get?fields1=f1,f2,f3,f4&fields2=f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61,f62,f63,f64,f65,f66,f67,f68,f69,f70&ut=b2884a393a59ad640022ce1e1e78431c&deviceid=0&cb=jsonp_1622790712837&_=1622790712926'
232
+
233
+ # 发送HTTP请求
234
+ response = requests.get(url, headers=headers, params={"type": "json"})
235
+
236
+ # 解析JSON数据
237
+ data = json.loads(response.text.lstrip('jsonp_1622790712837(').rstrip(');'))
238
+
239
+ # 处理数据
240
+
241
+ # 单位(万元)
242
+ # dayNetAmtIn 资金净流入
243
+ # dayAmtRemain 当日资金余额
244
+ # dayAmtThreshold 当日资金限额
245
+ # monthNetAmtIn 当月净流入
246
+ # yearNetAmtIn 年度净流入
247
+ # allNetAmtIn 总净流入
248
+ # buyAmt 当日买入金额
249
+ # sellAmt 当日卖出金额
250
+ # buySellAmt 当日买入卖出总金额
251
+ # netBuyAmt 成交净买额
252
+
253
+ # Hongkong to Shanghai
254
+ hk2sh = data['data']['hk2sh']
255
+ hk2sh_df = pd.DataFrame(hk2sh, index=[0])
256
+ # Hongkong to ShenZhen
257
+ hk2sz = data['data']['hk2sz']
258
+ hk2sz_df = pd.DataFrame(hk2sz, index=[0])
259
+
260
+ # Shanghai to Hongkong
261
+ sh2hk = data['data']['sh2hk']
262
+ sh2hk_df = pd.DataFrame(sh2hk, index=[0])
263
+
264
+ # ShenZhen to Hongkong
265
+ sz2hk = data['data']['sz2hk']
266
+ sz2hk_df = pd.DataFrame(sz2hk, index=[0])
267
+ # 北向总额
268
+ sum_north_netBuyAmt = hk2sh_df['netBuyAmt'] + hk2sz_df['netBuyAmt']
269
+
270
+ sum_south_netBuyAmt = sh2hk_df['netBuyAmt'] + sz2hk_df['netBuyAmt']
271
+
272
+ df = pd.DataFrame([[
273
+ list(hk2sh_df['netBuyAmt'])[0],
274
+ list(hk2sz_df['netBuyAmt'])[0],
275
+ list(sum_north_netBuyAmt)[0],
276
+ list(sh2hk_df['netBuyAmt'])[0],
277
+ list(sz2hk_df['netBuyAmt'])[0],
278
+ list(sum_south_netBuyAmt)[0]]],
279
+ columns=['sh_netBuyAmt', 'sz_netBuyAmt', 'sum_north_netBuyAmt',
280
+ 'sh_hk_netBuyAmt', 'sz_hk_netBuyAmt', 'sum_south_netBuyAmt'])
281
+
282
+ # 打印结果
283
+ return df
284
+
285
+
286
+ import mns_common.component.proxies.proxy_common_api as proxy_common_api
287
+
288
+ # 示例调用
289
+ if __name__ == "__main__":
290
+
291
+ while True:
292
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
293
+ proxy = {"https": proxy_ip}
294
+ logger.info(proxy_ip)
295
+ df = all_stock_ticker_data_new(proxy, 3)
296
+ logger.info("数据条数,{}", df.shape[0])
297
+ time.sleep(1)
298
+
299
+ else:
300
+ time.sleep(1)
301
+ logger.error("ip为空")
@@ -0,0 +1,340 @@
1
+ import sys
2
+ import os
3
+
4
+ file_path = os.path.abspath(__file__)
5
+ end = file_path.index('mns') + 16
6
+ project_path = file_path[0:end]
7
+ sys.path.append(project_path)
8
+
9
+ import requests
10
+
11
+ import mns_common.utils.data_frame_util as data_frame_util
12
+ import json
13
+ import datetime
14
+ from concurrent.futures import ThreadPoolExecutor
15
+ import mns_common.component.proxies.proxy_common_api as proxy_common_api
16
+ from loguru import logger
17
+ import concurrent.futures
18
+ import pandas as pd
19
+ import time
20
+ from concurrent.futures import ThreadPoolExecutor
21
+ from threading import Lock
22
+ import mns_common.api.em.real_time.real_time_quotes_repeat_api as real_time_quotes_repeat_api
23
+ import mns_common.api.em.real_time.east_money_stock_common_api as east_money_stock_common_api
24
+
25
+ #
26
+ # fields_02 = "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18,f19,f20,f21,f22,f23,f24,f25,f26,f27,f28,f29,f30,f31,f32,f33,f34,f35,f36,f37,f38,f39,f40,f41,f42,f43,f44,f45,f46,f47,f48,f49,f50,f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61,f62,f63,f64,f65,f66,f67,f68,f69,f70,f71,f72,f73,f74,f75,f76,f77,f78,f79,f80,f81,f82,f83,f84,f85,f86,f87,f88,f89,f90,f91,f92,f93,f94,f95,f96,f97,f98,f99,f100,f101,f102,f103,f104,f105,f106,f107,f108" \
27
+ # ",f109,f110,f111,f112,f113,f114,f115,f116,f117,f118,f119,f120,f121,f122,f123,f124,f125,f126,f127,f128,f129,f130,f131,f132,f133,f134,f135,f136,f137,f138,f139,f140,f141,f142,f143,f144,f145,f146,f147,f148,f149,f150,f151,f152,f153,f154,f155,f156,f157,f158,f159,f160,f161,f162,f163,f164,f165,f166,f167,f168,f169,f170,f171,f172,f173,f174,f175,f176,f177,f178,f179,f180,f181,f182,f183,f184,f185,f186,f187,f188,f189,f190,f191,f192,f193,f194,f195,f196,f197,f198,f199,f200" \
28
+ # ",f209,f210,f212,f213,f214,f215,f216,f217,f218,f219,f220,f221,f222,f223,f224,f225,f226,f227,f228,f229,f230,f231,f232,f233,f234,f235,f236,f237,f238,f239,f240,f241,f242,f243,f244,f245,f246,f247,f248,f249,f250,f251,f252,f253,f254,f255,f256,f257,f258,f259,f260,f261,f262,f263,f264,f265,f266,f267,f268,f269,f270,f271,f272,f273,f274,f275,f276,f277,f278,f279,f280,f281,f282,f283,f284,f285,f286,f287,f288,f289,f290,f291,f292,f293,f294,f295,f296,f297,f298,f299,f300" \
29
+ # ",f309,f310,f312,f313,f314,f315,f316,f317,f318,f319,f320,f321,f322,f323,f324,f325,f326,f327,f328,f329,f330,f331,f332,f333,f334,f335,f336,f337,f338,f339,f340,f341,f342,f343,f344,f345,f346,f347,f348,f349,f350,f351,f352,f353,f354,f355,f356,f357,f358,f359,f360,f361,f362,f363,f364,f365,f366,f367,f368,f369,f370,f371,f372,f373,f374,f375,f376,f377,f378,f379,f380,f381,f382,f383,f384,f385,f386,f387,f388,f389,f390,f391,f392,f393,f394,f395,f396,f397,f398,f399,f401"
30
+
31
+
32
+ fs = "m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048"
33
+
34
+ fields = ("f2,f3,f5,f6,f8,"
35
+ "f9,f10,f22,f12,f13,"
36
+ "f14,f15,f16,f17,f18,"
37
+ "f20,f21,f23,f26,f33,"
38
+ "f34,f35,f37,f38,f39,"
39
+ "f62,f64,f65,f67,f68,"
40
+ "f66,f69,f70,f71,f72,"
41
+ "f76,f77,f78,f82,f83,"
42
+ "f84,f102,f184,f100,f103,"
43
+ "f352,f191,f193,f24,f25")
44
+
45
+ # 分页条数
46
+ PAGE_SIZE = 100
47
+
48
+
49
+ def get_stock_page_data(pn, proxies, page_number, time_out):
50
+ """
51
+ 获取单页股票数据
52
+ """
53
+ # 获取当前日期和时间
54
+ current_time = datetime.datetime.now()
55
+
56
+ # 将当前时间转换为时间戳(以毫秒为单位)
57
+ current_timestamp_ms = int(current_time.timestamp() * 1000)
58
+
59
+ url = "https://13.push2.eastmoney.com/api/qt/clist/get"
60
+ params = {
61
+ "cb": "jQuery1124046660442520420653_" + str(current_timestamp_ms),
62
+ "pn": str(pn),
63
+ "pz": str(page_number), # 每页最大200条
64
+ "po": "1",
65
+ "np": "3",
66
+ "ut": "bd1d9ddb04089700cf9c27f6f7426281",
67
+ "fltt": "2",
68
+ "invt": "2",
69
+ "wbp2u": "|0|0|0|web",
70
+ "fid": "f3",
71
+ "fs": fs,
72
+ "fields": fields,
73
+ "_": current_timestamp_ms
74
+ }
75
+ try:
76
+ if proxies is None:
77
+ r = requests.get(url, params, timeout=time_out)
78
+ else:
79
+ r = requests.get(url, params, proxies=proxies, timeout=time_out)
80
+ data_text = r.text
81
+ begin_index = data_text.index('[')
82
+ end_index = data_text.index(']')
83
+ data_json = data_text[begin_index:end_index + 1]
84
+ data_json = json.loads(data_json)
85
+ if data_json is None:
86
+ return pd.DataFrame()
87
+ else:
88
+ return pd.DataFrame(data_json)
89
+ except Exception as e:
90
+ return pd.DataFrame()
91
+
92
+
93
+ def all_stock_ticker_data_new(initial_proxies, time_out, max_number) -> pd.DataFrame:
94
+ """
95
+ 使用多线程获取所有股票数据,失败页面会使用新IP重试,最多使用10个IP
96
+ """
97
+
98
+ total_pages = (max_number + PAGE_SIZE - 1) // PAGE_SIZE # 向上取整
99
+ all_pages = set(range(1, total_pages + 1)) # 所有需要获取的页码
100
+ success_pages = set() # 成功获取的页码
101
+ results = [] # 存储成功获取的数据
102
+ used_ip_count = 1 # 已使用IP计数器(初始IP算第一个)
103
+ MAX_IP_LIMIT = 10 # IP使用上限
104
+
105
+ # 循环处理直到所有页面成功或达到IP上限
106
+ while (all_pages - success_pages) and (used_ip_count < MAX_IP_LIMIT):
107
+ # 获取当前需要处理的失败页码
108
+ current_failed_pages = all_pages - success_pages
109
+ if used_ip_count > 1:
110
+ logger.info("当前需要处理的失败页码: {}, 已使用IP数量: {}/{}", current_failed_pages, used_ip_count,
111
+ MAX_IP_LIMIT)
112
+
113
+ # 首次使用初始代理,后续获取新代理
114
+ if len(success_pages) == 0:
115
+ proxies = initial_proxies
116
+ else:
117
+ # 每次重试前获取新代理并计数
118
+ # logger.info("获取新代理IP处理失败页面")
119
+ new_proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
120
+ proxies = {"https": new_proxy_ip}
121
+ # logger.info("新代理IP: {}, 已使用IP数量: {}/{}", new_proxy_ip, used_ip_count + 1, MAX_IP_LIMIT)
122
+ used_ip_count += 1 # 增加IP计数器
123
+
124
+ # 创建线程池处理当前失败的页码
125
+ with ThreadPoolExecutor(max_workers=10) as executor:
126
+ futures = {
127
+ executor.submit(get_stock_page_data, pn, proxies, PAGE_SIZE, time_out): pn
128
+ for pn in current_failed_pages
129
+ }
130
+
131
+ # 收集结果并记录成功页码
132
+ for future, pn in futures.items():
133
+ try:
134
+ result = future.result()
135
+ if not result.empty:
136
+ results.append(result)
137
+ success_pages.add(pn)
138
+ # else:
139
+ # logger.warning("页码 {} 未返回有效数据", pn)
140
+ except Exception as e:
141
+ continue
142
+ # logger.error("页码 {} 处理异常: {}", pn, str(e))
143
+
144
+ # 检查是否达到IP上限
145
+ if used_ip_count >= MAX_IP_LIMIT and (all_pages - success_pages):
146
+ remaining_pages = all_pages - success_pages
147
+ logger.warning("已达到最大IP使用限制({}个),剩余未获取页码: {}, 返回现有数据", MAX_IP_LIMIT, remaining_pages)
148
+
149
+ # 合并所有成功获取的数据
150
+ if results:
151
+ return pd.concat(results, ignore_index=True)
152
+ else:
153
+ return pd.DataFrame()
154
+
155
+
156
+ # 获取所有股票实时行情数据 f33,委比
157
+ def rename_real_time_quotes_df(temp_df):
158
+ temp_df = temp_df.rename(columns={
159
+ "f2": "now_price",
160
+ "f3": "chg",
161
+ "f5": "volume",
162
+ "f6": "amount",
163
+ "f8": "exchange",
164
+ "f9": "pe_ttm",
165
+ "f10": "quantity_ratio",
166
+ "f22": "up_speed",
167
+ "f12": "symbol",
168
+ "f13": "sz_sh",
169
+ "f14": "name",
170
+ "f15": "high",
171
+ "f16": "low",
172
+ "f17": "open",
173
+ "f18": "yesterday_price",
174
+ "f20": "total_mv",
175
+ "f21": "flow_mv",
176
+ "f23": "pb",
177
+ "f26": "list_date",
178
+ "f33": "wei_bi",
179
+ "f34": "outer_disk",
180
+ "f35": "inner_disk",
181
+ "f37": "ROE",
182
+ "f38": "total_share",
183
+ "f39": "flow_share",
184
+ "f62": "today_main_net_inflow",
185
+ "f64": "super_large_order_inflow",
186
+ "f65": "super_large_order_outflow",
187
+ "f67": "super_large_order_inflow_ratio",
188
+ "f68": "super_large_order_outflow_ratio",
189
+
190
+ "f66": "super_large_order_net_inflow",
191
+ "f69": "super_large_order_net_inflow_ratio",
192
+ "f70": "large_order_inflow",
193
+ "f71": "large_order_outflow",
194
+ "f72": "large_order_net_inflow",
195
+
196
+ "f76": "medium_order_inflow",
197
+ "f77": "medium_order_outflow",
198
+ "f78": "medium_order_net_inflow",
199
+ "f82": "small_order_inflow",
200
+ "f83": "small_order_outflow",
201
+
202
+ "f84": "small_order_net_inflow",
203
+ "f102": "area",
204
+ "f184": "today_main_net_inflow_ratio",
205
+ "f100": "industry",
206
+ "f103": "concept",
207
+
208
+ "f352": "average_price",
209
+ "f191": "hk_stock_code",
210
+ "f193": "hk_stock_name",
211
+ "f24": "sixty_day_chg",
212
+ "f25": "now_year_chg",
213
+ })
214
+ temp_df.loc[temp_df['sixty_day_chg'] == '-', 'total_share'] = 0
215
+
216
+ temp_df.loc[temp_df['now_year_chg'] == '-', 'now_year_chg'] = 0
217
+ temp_df.loc[temp_df['total_share'] == '-', 'total_share'] = 0
218
+ temp_df.loc[temp_df['flow_share'] == '-', 'flow_share'] = 0
219
+ temp_df.loc[temp_df['pe_ttm'] == '-', 'pe_ttm'] = 0
220
+ temp_df.loc[temp_df['up_speed'] == '-', 'up_speed'] = 0
221
+ temp_df.loc[temp_df['average_price'] == '-', 'average_price'] = 0
222
+ temp_df.loc[temp_df['wei_bi'] == '-', 'wei_bi'] = 0
223
+ temp_df.loc[temp_df['yesterday_price'] == '-', 'yesterday_price'] = 0
224
+ temp_df.loc[temp_df['now_price'] == '-', 'now_price'] = 0
225
+ temp_df.loc[temp_df['chg'] == '-', 'chg'] = 0
226
+ temp_df.loc[temp_df['volume'] == '-', 'volume'] = 0
227
+ temp_df.loc[temp_df['amount'] == '-', 'amount'] = 0
228
+ temp_df.loc[temp_df['exchange'] == '-', 'exchange'] = 0
229
+ temp_df.loc[temp_df['quantity_ratio'] == '-', 'quantity_ratio'] = 0
230
+ temp_df.loc[temp_df['high'] == '-', 'high'] = 0
231
+ temp_df.loc[temp_df['low'] == '-', 'low'] = 0
232
+ temp_df.loc[temp_df['open'] == '-', 'open'] = 0
233
+ temp_df.loc[temp_df['total_mv'] == '-', 'total_mv'] = 0
234
+ temp_df.loc[temp_df['flow_mv'] == '-', 'flow_mv'] = 0
235
+ temp_df.loc[temp_df['inner_disk'] == '-', 'inner_disk'] = 0
236
+ temp_df.loc[temp_df['outer_disk'] == '-', 'outer_disk'] = 0
237
+ temp_df.loc[temp_df['today_main_net_inflow_ratio'] == '-', 'today_main_net_inflow_ratio'] = 0
238
+ temp_df.loc[temp_df['today_main_net_inflow'] == '-', 'today_main_net_inflow'] = 0
239
+ temp_df.loc[temp_df['super_large_order_inflow'] == '-', 'super_large_order_inflow'] = 0
240
+ temp_df.loc[temp_df['super_large_order_outflow'] == '-', 'super_large_order_outflow'] = 0
241
+ temp_df.loc[temp_df['super_large_order_net_inflow'] == '-', 'super_large_order_net_inflow'] = 0
242
+ temp_df.loc[temp_df['super_large_order_inflow_ratio'] == '-', 'super_large_order_inflow_ratio'] = 0
243
+ temp_df.loc[temp_df['super_large_order_outflow_ratio'] == '-', 'super_large_order_outflow_ratio'] = 0
244
+ temp_df.loc[temp_df['super_large_order_net_inflow_ratio'] == '-', 'super_large_order_net_inflow_ratio'] = 0
245
+
246
+ temp_df.loc[temp_df['large_order_net_inflow'] == '-', 'large_order_net_inflow'] = 0
247
+ temp_df.loc[temp_df['large_order_inflow'] == '-', 'large_order_inflow'] = 0
248
+ temp_df.loc[temp_df['large_order_outflow'] == '-', 'large_order_outflow'] = 0
249
+
250
+ temp_df.loc[temp_df['medium_order_net_inflow'] == '-', 'medium_order_net_inflow'] = 0
251
+ temp_df.loc[temp_df['medium_order_outflow'] == '-', 'medium_order_outflow'] = 0
252
+ temp_df.loc[temp_df['medium_order_inflow'] == '-', 'medium_order_inflow'] = 0
253
+
254
+ temp_df.loc[temp_df['small_order_inflow'] == '-', 'small_order_inflow'] = 0
255
+ temp_df.loc[temp_df['small_order_outflow'] == '-', 'small_order_outflow'] = 0
256
+ temp_df.loc[temp_df['small_order_net_inflow'] == '-', 'small_order_net_inflow'] = 0
257
+
258
+ temp_df["list_date"] = pd.to_numeric(temp_df["list_date"], errors="coerce")
259
+ temp_df["wei_bi"] = pd.to_numeric(temp_df["wei_bi"], errors="coerce")
260
+ temp_df["average_price"] = pd.to_numeric(temp_df["average_price"], errors="coerce")
261
+ temp_df["yesterday_price"] = pd.to_numeric(temp_df["yesterday_price"], errors="coerce")
262
+ temp_df["now_price"] = pd.to_numeric(temp_df["now_price"], errors="coerce")
263
+ temp_df["chg"] = pd.to_numeric(temp_df["chg"], errors="coerce")
264
+ temp_df["volume"] = pd.to_numeric(temp_df["volume"], errors="coerce")
265
+ temp_df["amount"] = pd.to_numeric(temp_df["amount"], errors="coerce")
266
+ temp_df["exchange"] = pd.to_numeric(temp_df["exchange"], errors="coerce")
267
+ temp_df["quantity_ratio"] = pd.to_numeric(temp_df["quantity_ratio"], errors="coerce")
268
+ temp_df["high"] = pd.to_numeric(temp_df["high"], errors="coerce")
269
+ temp_df["low"] = pd.to_numeric(temp_df["low"], errors="coerce")
270
+ temp_df["open"] = pd.to_numeric(temp_df["open"], errors="coerce")
271
+ temp_df["total_mv"] = pd.to_numeric(temp_df["total_mv"], errors="coerce")
272
+ temp_df["flow_mv"] = pd.to_numeric(temp_df["flow_mv"], errors="coerce")
273
+ temp_df["outer_disk"] = pd.to_numeric(temp_df["outer_disk"], errors="coerce")
274
+ temp_df["inner_disk"] = pd.to_numeric(temp_df["inner_disk"], errors="coerce")
275
+ temp_df["today_main_net_inflow"] = pd.to_numeric(temp_df["today_main_net_inflow"], errors="coerce")
276
+ temp_df["super_large_order_net_inflow"] = pd.to_numeric(temp_df["super_large_order_net_inflow"],
277
+ errors="coerce")
278
+ temp_df["super_large_order_net_inflow_ratio"] = pd.to_numeric(temp_df["super_large_order_net_inflow_ratio"],
279
+ errors="coerce")
280
+ temp_df["large_order_net_inflow"] = pd.to_numeric(temp_df["large_order_net_inflow"],
281
+ errors="coerce")
282
+ temp_df["medium_order_net_inflow"] = pd.to_numeric(temp_df["medium_order_net_inflow"],
283
+ errors="coerce")
284
+
285
+ temp_df["small_order_net_inflow"] = pd.to_numeric(temp_df["small_order_net_inflow"], errors="coerce")
286
+
287
+ temp_df["pe_ttm"] = pd.to_numeric(temp_df["pe_ttm"], errors="coerce")
288
+ temp_df["total_share"] = pd.to_numeric(temp_df["total_share"], errors="coerce")
289
+ temp_df["flow_share"] = pd.to_numeric(temp_df["flow_share"], errors="coerce")
290
+
291
+ temp_df["super_large_order_inflow"] = pd.to_numeric(temp_df["super_large_order_inflow"], errors="coerce")
292
+ temp_df["super_large_order_outflow"] = pd.to_numeric(temp_df["super_large_order_outflow"], errors="coerce")
293
+
294
+ temp_df["super_large_order_inflow_ratio"] = pd.to_numeric(temp_df["super_large_order_inflow_ratio"],
295
+ errors="coerce")
296
+ temp_df["super_large_order_outflow_ratio"] = pd.to_numeric(temp_df["super_large_order_outflow_ratio"],
297
+ errors="coerce")
298
+
299
+ temp_df["super_large_order_net_inflow"] = pd.to_numeric(temp_df["super_large_order_net_inflow"], errors="coerce")
300
+ temp_df["super_large_order_net_inflow_ratio"] = pd.to_numeric(temp_df["super_large_order_net_inflow_ratio"],
301
+ errors="coerce")
302
+
303
+ temp_df["medium_order_inflow"] = pd.to_numeric(temp_df["medium_order_inflow"], errors="coerce")
304
+ temp_df["medium_order_outflow"] = pd.to_numeric(temp_df["medium_order_outflow"], errors="coerce")
305
+
306
+ temp_df["small_order_inflow"] = pd.to_numeric(temp_df["small_order_inflow"], errors="coerce")
307
+ temp_df["small_order_outflow"] = pd.to_numeric(temp_df["small_order_outflow"], errors="coerce")
308
+
309
+ outer_disk = temp_df['outer_disk']
310
+ inner_disk = temp_df['inner_disk']
311
+ disk_ratio = (outer_disk - inner_disk) / inner_disk
312
+ temp_df['disk_ratio'] = round(disk_ratio, 2)
313
+ return temp_df
314
+
315
+
316
+ def get_stock_real_time_quotes(time_out):
317
+ try_numer = 3
318
+ while try_numer > 0:
319
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
320
+ initial_proxies = {"https": proxy_ip,
321
+ "http": proxy_ip}
322
+
323
+ total_number = east_money_stock_common_api.get_stocks_num(1, initial_proxies, 20, time_out)
324
+ if total_number > 0:
325
+ break
326
+ try_numer = try_numer - 1
327
+ if total_number == 0:
328
+ return pd.DataFrame()
329
+
330
+ page_df = all_stock_ticker_data_new(initial_proxies, time_out, total_number)
331
+ page_df = rename_real_time_quotes_df(page_df)
332
+ page_df.drop_duplicates('symbol', keep='last', inplace=True)
333
+ return page_df
334
+
335
+
336
+ # 示例调用
337
+ if __name__ == "__main__":
338
+ df = get_stock_real_time_quotes(5)
339
+ zt_df = df.loc[df['wei_bi'] == 100]
340
+ logger.info("同步次数", zt_df.shape[0])