mns-common 1.5.1.8__py3-none-any.whl → 1.5.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mns-common might be problematic. Click here for more details.

@@ -0,0 +1,234 @@
1
+ import sys
2
+ import os
3
+
4
+ file_path = os.path.abspath(__file__)
5
+ end = file_path.index('mns') + 16
6
+ project_path = file_path[0:end]
7
+ sys.path.append(project_path)
8
+ from concurrent.futures import ThreadPoolExecutor
9
+ import pandas as pd
10
+ from loguru import logger
11
+ import requests
12
+ import time
13
+
14
+ # 最大返回条数
15
+ max_number = 12000
16
+ # 最小返回条数
17
+ min_number = 11000
18
+ # 分页条数
19
+ page_number = 100
20
+
21
+ fields = ("f352,f2,f3,f5,f6,f8,f10,f11,f13,f22,f12,f14,f15,f16,f17,f18,f20,f21,f26,"
22
+ "f33,f34,f35,f62,f66,f69,f72,f100,f184,f211,f212")
23
+
24
+
25
+ def us_real_time_quotes_page_df(cookie, pn, proxies):
26
+ try:
27
+ headers = {
28
+ 'Cookie': cookie
29
+ }
30
+
31
+ current_timestamp = str(int(round(time.time() * 1000, 0)))
32
+
33
+ url = "https://72.push2.eastmoney.com/api/qt/clist/get"
34
+ params = {
35
+ "pn": str(pn),
36
+ "pz": "50000",
37
+ "po": "1",
38
+ "np": "2",
39
+ "ut": "bd1d9ddb04089700cf9c27f6f7426281",
40
+ "fltt": "2",
41
+ "invt": "2",
42
+ "fid": "f12",
43
+ "fs": "m:105,m:106,m:107",
44
+ "fields": fields,
45
+ "_": str(current_timestamp),
46
+ }
47
+ if proxies is None:
48
+ r = requests.get(url, params=params, headers=headers)
49
+ else:
50
+ r = requests.get(url, params=params, headers=headers, proxies=proxies)
51
+ data_json = r.json()
52
+ if pn == 1:
53
+ try:
54
+ global max_number
55
+ max_number = int(data_json['data']['total'])
56
+ except Exception as e:
57
+ logger.error(f"获取第{pn}页美股列表异常: {e}")
58
+ return pd.DataFrame()
59
+
60
+ if not data_json["data"]["diff"]:
61
+ return pd.DataFrame()
62
+ temp_df = pd.DataFrame(data_json["data"]["diff"]).T
63
+
64
+ return temp_df
65
+ except Exception as e:
66
+ logger.error("获取美股实时行情异常:{}", e)
67
+ return pd.DataFrame()
68
+
69
+
70
+ def thread_pool_executor(cookie, proxies):
71
+ """
72
+ 使用多线程获取所有美股数据
73
+ """
74
+ # 计算总页数,假设总共有1000条数据,每页200条
75
+
76
+ per_page = page_number
77
+ total_pages = (max_number + per_page - 1) // per_page # 向上取整
78
+
79
+ # 创建线程池
80
+ with ThreadPoolExecutor(max_workers=3) as executor:
81
+ # 提交任务,获取每页数据
82
+ futures = [executor.submit(us_real_time_quotes_page_df, cookie, pn, proxies)
83
+ for pn in range(1, total_pages + 1)]
84
+
85
+ # 收集结果
86
+ results = []
87
+ for future in futures:
88
+ result = future.result()
89
+ if not result.empty:
90
+ results.append(result)
91
+
92
+ # 合并所有页面的数据
93
+ if results:
94
+ return pd.concat(results, ignore_index=True)
95
+ else:
96
+ return pd.DataFrame()
97
+
98
+
99
+ def get_us_stock_real_time_quotes(cookie, proxies):
100
+ # 获取第一页数据
101
+ page_one_df = us_real_time_quotes_page_df(cookie, 1, proxies)
102
+ # 数据接口正常返回5600以上的数量
103
+ if page_one_df.shape[0] > min_number:
104
+ page_one_df = rename_us_stock(page_one_df)
105
+ page_one_df.drop_duplicates('symbol', keep='last', inplace=True)
106
+ return page_one_df
107
+ else:
108
+ page_df = thread_pool_executor(cookie, proxies)
109
+ page_df = rename_us_stock(page_df)
110
+ page_df.drop_duplicates('symbol', keep='last', inplace=True)
111
+ return page_df
112
+
113
+
114
+ def rename_us_stock(temp_df):
115
+ temp_df = temp_df.rename(columns={
116
+
117
+ "f12": "symbol",
118
+ "f14": "name",
119
+ "f3": "chg",
120
+ "f2": "now_price",
121
+ "f5": "volume",
122
+ "f6": "amount",
123
+ "f8": "exchange",
124
+ "f10": "quantity_ratio",
125
+ "f22": "up_speed",
126
+ "f11": "up_speed_05",
127
+ "f13": "simple_symbol",
128
+ "f15": "high",
129
+ "f16": "low",
130
+ "f17": "open",
131
+ "f18": "yesterday_price",
132
+ "f20": "total_mv",
133
+ "f21": "flow_mv",
134
+ "f26": "list_date",
135
+ "f33": "wei_bi",
136
+ "f34": "outer_disk",
137
+ "f35": "inner_disk",
138
+ "f62": "today_main_net_inflow",
139
+ "f66": "super_large_order_net_inflow",
140
+ "f69": "super_large_order_net_inflow_ratio",
141
+ "f72": "large_order_net_inflow",
142
+ # "f78": "medium_order_net_inflow",
143
+ # "f84": "small_order_net_inflow",
144
+ "f100": "industry",
145
+ # "f103": "concept",
146
+ "f184": "today_main_net_inflow_ratio",
147
+ "f352": "average_price",
148
+ "f211": "buy_1_num",
149
+ "f212": "sell_1_num"
150
+ })
151
+ temp_df.loc[temp_df['buy_1_num'] == '-', 'buy_1_num'] = 0
152
+ temp_df.loc[temp_df['sell_1_num'] == '-', 'sell_1_num'] = 0
153
+ temp_df.loc[temp_df['up_speed_05'] == '-', 'up_speed_05'] = 0
154
+ temp_df.loc[temp_df['up_speed'] == '-', 'up_speed'] = 0
155
+ temp_df.loc[temp_df['average_price'] == '-', 'average_price'] = 0
156
+ temp_df.loc[temp_df['wei_bi'] == '-', 'wei_bi'] = 0
157
+ temp_df.loc[temp_df['yesterday_price'] == '-', 'yesterday_price'] = 0
158
+ temp_df.loc[temp_df['now_price'] == '-', 'now_price'] = 0
159
+ temp_df.loc[temp_df['chg'] == '-', 'chg'] = 0
160
+ temp_df.loc[temp_df['volume'] == '-', 'volume'] = 0
161
+ temp_df.loc[temp_df['amount'] == '-', 'amount'] = 0
162
+ temp_df.loc[temp_df['exchange'] == '-', 'exchange'] = 0
163
+ temp_df.loc[temp_df['quantity_ratio'] == '-', 'quantity_ratio'] = 0
164
+ temp_df.loc[temp_df['high'] == '-', 'high'] = 0
165
+ temp_df.loc[temp_df['low'] == '-', 'low'] = 0
166
+ temp_df.loc[temp_df['open'] == '-', 'open'] = 0
167
+ temp_df.loc[temp_df['total_mv'] == '-', 'total_mv'] = 0
168
+ temp_df.loc[temp_df['flow_mv'] == '-', 'flow_mv'] = 0
169
+ temp_df.loc[temp_df['inner_disk'] == '-', 'inner_disk'] = 0
170
+ temp_df.loc[temp_df['outer_disk'] == '-', 'outer_disk'] = 0
171
+ temp_df.loc[temp_df['today_main_net_inflow_ratio'] == '-', 'today_main_net_inflow_ratio'] = 0
172
+ temp_df.loc[temp_df['today_main_net_inflow'] == '-', 'today_main_net_inflow'] = 0
173
+ temp_df.loc[temp_df['super_large_order_net_inflow'] == '-', 'super_large_order_net_inflow'] = 0
174
+ temp_df.loc[temp_df['super_large_order_net_inflow_ratio'] == '-', 'super_large_order_net_inflow_ratio'] = 0
175
+ temp_df.loc[temp_df['large_order_net_inflow'] == '-', 'large_order_net_inflow'] = 0
176
+ # temp_df.loc[temp_df['medium_order_net_inflow'] == '-', 'medium_order_net_inflow'] = 0
177
+ # temp_df.loc[temp_df['small_order_net_inflow'] == '-', 'small_order_net_inflow'] = 0
178
+
179
+ temp_df["list_date"] = pd.to_numeric(temp_df["list_date"], errors="coerce")
180
+ temp_df["wei_bi"] = pd.to_numeric(temp_df["wei_bi"], errors="coerce")
181
+ temp_df["average_price"] = pd.to_numeric(temp_df["average_price"], errors="coerce")
182
+ temp_df["yesterday_price"] = pd.to_numeric(temp_df["yesterday_price"], errors="coerce")
183
+ temp_df["now_price"] = pd.to_numeric(temp_df["now_price"], errors="coerce")
184
+ temp_df["chg"] = pd.to_numeric(temp_df["chg"], errors="coerce")
185
+ temp_df["volume"] = pd.to_numeric(temp_df["volume"], errors="coerce")
186
+ temp_df["amount"] = pd.to_numeric(temp_df["amount"], errors="coerce")
187
+ temp_df["exchange"] = pd.to_numeric(temp_df["exchange"], errors="coerce")
188
+ temp_df["quantity_ratio"] = pd.to_numeric(temp_df["quantity_ratio"], errors="coerce")
189
+ temp_df["high"] = pd.to_numeric(temp_df["high"], errors="coerce")
190
+ temp_df["low"] = pd.to_numeric(temp_df["low"], errors="coerce")
191
+ temp_df["open"] = pd.to_numeric(temp_df["open"], errors="coerce")
192
+ temp_df["total_mv"] = pd.to_numeric(temp_df["total_mv"], errors="coerce")
193
+ temp_df["flow_mv"] = pd.to_numeric(temp_df["flow_mv"], errors="coerce")
194
+ temp_df["outer_disk"] = pd.to_numeric(temp_df["outer_disk"], errors="coerce")
195
+ temp_df["inner_disk"] = pd.to_numeric(temp_df["inner_disk"], errors="coerce")
196
+ temp_df["today_main_net_inflow"] = pd.to_numeric(temp_df["today_main_net_inflow"], errors="coerce")
197
+ temp_df["super_large_order_net_inflow"] = pd.to_numeric(temp_df["super_large_order_net_inflow"],
198
+ errors="coerce")
199
+ temp_df["super_large_order_net_inflow_ratio"] = pd.to_numeric(temp_df["super_large_order_net_inflow_ratio"],
200
+ errors="coerce")
201
+ temp_df["large_order_net_inflow"] = pd.to_numeric(temp_df["large_order_net_inflow"],
202
+ errors="coerce")
203
+ # temp_df["medium_order_net_inflow"] = pd.to_numeric(temp_df["medium_order_net_inflow"],
204
+ # errors="coerce")
205
+ # temp_df["small_order_net_inflow"] = pd.to_numeric(temp_df["small_order_net_inflow"], errors="coerce")
206
+
207
+ # 大单比例
208
+ temp_df['large_order_net_inflow_ratio'] = round((temp_df['large_order_net_inflow'] / temp_df['amount']) * 100,
209
+ 2)
210
+
211
+ # 外盘是内盘倍数
212
+ temp_df['disk_ratio'] = round((temp_df['outer_disk'] - temp_df['inner_disk']) / temp_df['inner_disk'], 2)
213
+ # 只有外盘没有内盘
214
+ temp_df.loc[temp_df["inner_disk"] == 0, ['disk_ratio']] = 1688
215
+ temp_df['disk_diff_amount'] = round(
216
+ (temp_df['outer_disk'] - temp_df['inner_disk']) * temp_df[
217
+ "average_price"],
218
+ 2)
219
+ return temp_df
220
+
221
+
222
+ if __name__ == '__main__':
223
+ cookie_test = 'qgqp_b_id=1e0d79428176ed54bef8434efdc0e8c3; mtp=1; ct=QVRY_s8Tiag1WfK2tSW2n03qpsX-PD8aH_rIjKVooawX8K33UVnpIofK088lD1lguWlE_OEIpQwn3PJWFPhHvSvyvYr4Zka3l4vxtZfH1Uikjtyy9z1H4Swo0rQzMKXncVzBXiOo5TjE-Dy9fcoG3ZF7UVdQ35jp_cFwzOlpK5Y; ut=FobyicMgeV51lVMr4ZJXvn-72bp0oeSOvtzifFY_U7kBFtR6og4Usd-VtBM5XBBvHq0lvd9xXkvpIqWro9EDKmv6cbKOQGyawUSMcKVP57isZCaM7lWQ6jWXajvTfvV4mIR-W_MZNK8VY0lL9W4qNMniJ6PBn_gkJsSAJCadmsyI9cxmjx--gR4m54pdF_nie_y4iWHys83cmWR2R7Bt1KKqB25OmkfCQTJJqIf7QsqangVGMUHwMC39Z9QhrfCFHKVNrlqS503O6b9GitQnXtvUdJhCmomu; pi=4253366368931142%3Bp4253366368931142%3B%E8%82%A1%E5%8F%8B9x56I87727%3BYNigLZRW%2FzMdGgVDOJbwReDWnTPHl51dB0gQLiwaCf1XY98mlJYx6eJbsoYr5Nie%2BX1L%2BzaMsec99KkX%2BT29Ds1arfST7sIBXxjUQ3dp11IPUnXy64PaBFRTHzMRWnCFJvvhc%2FAI41rXSGXolC8YMxI%2BvyPS%2BuErwgOVjC5vvsIiKeO7TLyKkhqqQJPX%2F7RWC5Sf3QLh%3Bdwjn4Xho10%2FKjqOgTWs%2FJF4%2FkdKzeuBwM8sz9aLvJovejAkCAyGMyGYA6AE67Xk2Ki7x8zdfBifF2DG%2Fvf2%2BXAYN8ZVISSEWTIXh32Z5MxEacK4JBTkqyiD93e1vFBOFQ82BqaiVmntUq0V6FrTUHGeh1gG5Sg%3D%3D; uidal=4253366368931142%e8%82%a1%e5%8f%8b9x56I87727; sid=170711377; vtpst=|; quote_lt=1; websitepoptg_api_time=1715777390466; emshistory=%5B%22%E8%BD%AC%E5%80%BA%E6%A0%87%22%2C%22%E8%BD%AC%E5%80%BA%E6%A0%87%E7%9A%84%22%5D; st_si=00364513876913; st_asi=delete; HAList=ty-116-00700-%u817E%u8BAF%u63A7%u80A1%2Cty-1-688695-%u4E2D%u521B%u80A1%u4EFD%2Cty-1-600849-%u4E0A%u836F%u8F6C%u6362%2Cty-1-603361-%u6D59%u6C5F%u56FD%u7965%2Cty-1-603555-ST%u8D35%u4EBA%2Cty-0-000627-%u5929%u8302%u96C6%u56E2%2Cty-0-002470-%u91D1%u6B63%u5927%2Cty-0-832876-%u6167%u4E3A%u667A%u80FD%2Cty-0-300059-%u4E1C%u65B9%u8D22%u5BCC%2Cty-107-CWB-%u53EF%u8F6C%u503AETF-SPDR; st_pvi=26930719093675; st_sp=2024-04-28%2017%3A27%3A05; st_inirUrl=https%3A%2F%2Fcn.bing.com%2F; st_sn=23; st_psi=20240517111108288-113200301321-2767127768'
224
+ while True:
225
+ us_df = get_us_stock_real_time_quotes(cookie_test, None)
226
+ us_df = us_df.loc[us_df['flow_mv'] != 0]
227
+ us_df = us_df.sort_values(by=['amount'], ascending=False)
228
+ us_stock_df = us_df[[
229
+ "symbol",
230
+ "name",
231
+ "chg",
232
+ "amount"
233
+ ]]
234
+ logger.info('test')
@@ -0,0 +1,359 @@
1
+ import requests
2
+
3
+ import mns_common.utils.data_frame_util as data_frame_util
4
+ import json
5
+ import datetime
6
+
7
+ import threading
8
+ from concurrent.futures import ThreadPoolExecutor
9
+ import mns_common.component.proxies.proxy_common_api as proxy_common_api
10
+ from loguru import logger
11
+ import concurrent.futures
12
+ import pandas as pd
13
+ import time
14
+ from concurrent.futures import ThreadPoolExecutor, as_completed
15
+ from threading import Lock
16
+
17
+ fields = ("f352,f2,f3,f5,f6,f8,f10,f11,f22,f12,f14,f15,f16,f17,"
18
+ "f18,f20,f21,f26,f33,f34,f35,f62,f66,f69,f72,f100,f184,f211,f212"),
19
+ fs = "m:0 t:6,m:0 t:80,m:1 t:2,m:1 t:23,m:0 t:81 s:2048"
20
+
21
+ # 分页条数
22
+ PAGE_SIZE = 100
23
+
24
+
25
+ def get_stock_page_data_time_out(pn, proxies, page_size, time_out):
26
+ """
27
+ 获取单页股票数据
28
+ """
29
+ # 获取当前日期和时间
30
+ current_time = datetime.datetime.now()
31
+
32
+ # 将当前时间转换为时间戳(以毫秒为单位)
33
+ current_timestamp_ms = int(current_time.timestamp() * 1000)
34
+
35
+ url = "https://33.push2.eastmoney.com/api/qt/clist/get"
36
+ params = {
37
+ "cb": "jQuery1124046660442520420653_" + str(current_timestamp_ms),
38
+ "pn": str(pn),
39
+ "pz": str(page_size), # 每页最大200条
40
+ "po": "0",
41
+ "np": "3",
42
+ "ut": "bd1d9ddb04089700cf9c27f6f7426281",
43
+ "fltt": "2",
44
+ "invt": "2",
45
+ "wbp2u": "|0|0|0|web",
46
+ "fid": "f12",
47
+ "fs": fs,
48
+ "fields": fields,
49
+ "_": current_timestamp_ms
50
+ }
51
+ try:
52
+ if proxies is None:
53
+ r = requests.get(url, params, timeout=time_out)
54
+ else:
55
+ r = requests.get(url, params, proxies=proxies, timeout=time_out)
56
+
57
+ data_text = r.text
58
+ if pn == 1:
59
+ try:
60
+ begin_index_total = data_text.index('"total":')
61
+
62
+ end_index_total = data_text.index('"diff"')
63
+ global max_number
64
+ max_number = int(data_text[begin_index_total + 8:end_index_total - 1])
65
+ except Exception as e:
66
+ logger.error("获取第{}页股票列表异常:{}", pn, str(e))
67
+ return pd.DataFrame()
68
+
69
+ begin_index = data_text.index('[')
70
+ end_index = data_text.index(']')
71
+ data_json = data_text[begin_index:end_index + 1]
72
+ data_json = json.loads(data_json)
73
+ if data_json is None:
74
+ return pd.DataFrame()
75
+ else:
76
+ result_df = pd.DataFrame(data_json)
77
+ result_df['page_number'] = pn
78
+ return result_df
79
+ except Exception as e:
80
+ logger.error("获取第{}页股票列表异常:{}", pn, str(e))
81
+ return pd.DataFrame()
82
+
83
+
84
+ def repeated_acquisition_ask(per_page, max_number, time_out, max_workers=5):
85
+ total_pages = (max_number + per_page - 1) // per_page # 向上取整
86
+ result_df = pd.DataFrame()
87
+ df_lock = Lock() # 线程安全的DataFrame合并锁
88
+
89
+ def fetch_pages(page_nums):
90
+ """单个线程处理一组页面,复用代理IP直到失效"""
91
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
92
+ proxies = {"https": proxy_ip, "http": proxy_ip}
93
+ thread_results = [] # 线程内临时存储结果
94
+
95
+ for page_num in page_nums:
96
+ while True: # 重试循环(复用当前IP)
97
+ try:
98
+ page_df = get_stock_page_data_time_out(
99
+ page_num, proxies, per_page, time_out
100
+ )
101
+ if data_frame_util.is_not_empty(page_df):
102
+ logger.info("线程{} 页面{}获取成功(IP复用中)",
103
+ threading.get_ident(), page_num)
104
+ thread_results.append(page_df)
105
+ break # 成功后继续用当前IP处理下一页
106
+ else:
107
+ logger.warning("页面数据为空:{},重试中...", page_num)
108
+ # 数据为空,更换IP
109
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
110
+ proxies = {"https": proxy_ip, "http": proxy_ip}
111
+ time.sleep(0.2)
112
+ except BaseException as e:
113
+ logger.error("线程{} 页面{}获取异常[{}],更换IP重试",
114
+ threading.get_ident(), page_num, str(e))
115
+ # 发生异常,更换IP
116
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
117
+ proxies = {"https": proxy_ip, "http": proxy_ip}
118
+ time.sleep(1)
119
+ return thread_results
120
+
121
+ # 页面分配:平均分配给每个线程
122
+ def split_pages(total, workers):
123
+ pages = list(range(1, total + 1))
124
+ avg = total // workers
125
+ remainder = total % workers
126
+ split = []
127
+ start = 0
128
+ for i in range(workers):
129
+ end = start + avg + (1 if i < remainder else 0)
130
+ split.append(pages[start:end])
131
+ start = end
132
+ return split
133
+
134
+ # 分配页面组
135
+ page_groups = split_pages(total_pages, max_workers)
136
+
137
+ # 多线程执行
138
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
139
+ futures = [executor.submit(fetch_pages, group) for group in page_groups]
140
+
141
+ # 合并结果
142
+ for future in as_completed(futures):
143
+ try:
144
+ thread_dfs = future.result()
145
+ if thread_dfs:
146
+ with df_lock:
147
+ result_df = pd.concat([result_df] + thread_dfs, ignore_index=True)
148
+ except Exception as e:
149
+ logger.error("线程结果处理失败:{}", str(e))
150
+
151
+ return result_df
152
+
153
+
154
+ def repeated_acquisition_ask_sync(time_out):
155
+ per_page = PAGE_SIZE
156
+ total_pages = (max_number + per_page - 1) // per_page # 向上取整
157
+ result_df = pd.DataFrame()
158
+ now_page = 1
159
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
160
+ while now_page <= total_pages:
161
+ proxies = {"https": proxy_ip,
162
+ "http": proxy_ip}
163
+ try:
164
+ page_df = get_stock_page_data_time_out(now_page, proxies, PAGE_SIZE, time_out)
165
+ if data_frame_util.is_not_empty(page_df):
166
+ result_df = pd.concat([page_df, result_df])
167
+ logger.info("获取页面数据成功:{}", now_page)
168
+ now_page = now_page + 1
169
+ else:
170
+ time.sleep(0.2)
171
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
172
+ logger.info("获取页面数据失败:{}", now_page)
173
+ except BaseException as e:
174
+ time.sleep(1)
175
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
176
+ # 示例调用
177
+ return result_df
178
+
179
+
180
+ def repeated_acquisition_ask_async(time_out, max_number, num_threads, pages_per_thread):
181
+ per_page = PAGE_SIZE
182
+ total_pages = (max_number + per_page - 1) // per_page # 向上取整
183
+ result_df = pd.DataFrame()
184
+
185
+ # 创建线程锁以确保线程安全
186
+ df_lock = Lock()
187
+
188
+ # 计算每个线程处理的页数范围
189
+ def process_page_range(start_page, end_page, thread_id):
190
+ nonlocal result_df
191
+ local_df = pd.DataFrame()
192
+ current_page = start_page
193
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
194
+
195
+ while current_page <= end_page and current_page <= total_pages:
196
+ proxies = {"https": proxy_ip, "http": proxy_ip}
197
+ try:
198
+ page_df = get_stock_page_data_time_out(current_page, proxies, PAGE_SIZE, time_out)
199
+ if data_frame_util.is_not_empty(page_df):
200
+ local_df = pd.concat([local_df, page_df])
201
+ logger.info("线程{}获取页面数据成功: {}", thread_id, current_page)
202
+ current_page += 1
203
+ else:
204
+ time.sleep(0.2)
205
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
206
+ logger.info("线程{}获取页面数据失败: {}", thread_id, current_page)
207
+ except BaseException as e:
208
+ time.sleep(1)
209
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
210
+ logger.error("线程{}处理页面{}时发生错误: {}", thread_id, current_page, e)
211
+
212
+ with df_lock:
213
+ result_df = pd.concat([result_df, local_df])
214
+ return len(local_df)
215
+
216
+ # 计算每个线程的页面范围
217
+ page_ranges = []
218
+ for i in range(num_threads):
219
+ start_page = i * pages_per_thread + 1
220
+ end_page = (i + 1) * pages_per_thread
221
+ if start_page > total_pages:
222
+ break
223
+ page_ranges.append((start_page, end_page, i + 1))
224
+
225
+ # 使用线程池执行任务
226
+ with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
227
+ # 提交所有任务
228
+ futures = [
229
+ executor.submit(process_page_range, start, end, tid)
230
+ for start, end, tid in page_ranges
231
+ ]
232
+
233
+ # 等待所有任务完成并获取结果
234
+ results = []
235
+ for future in concurrent.futures.as_completed(futures):
236
+ try:
237
+ result = future.result()
238
+ results.append(result)
239
+ except Exception as e:
240
+ logger.error("线程执行出错: {}", e)
241
+
242
+ return rename_real_time_quotes_df(result_df)
243
+
244
+
245
+ def rename_real_time_quotes_df(temp_df):
246
+ temp_df = temp_df.rename(columns={
247
+ "f2": "now_price",
248
+ "f3": "chg",
249
+ "f5": "volume",
250
+ "f6": "amount",
251
+ "f8": "exchange",
252
+ "f10": "quantity_ratio",
253
+ "f22": "up_speed",
254
+ "f11": "up_speed_05",
255
+ "f12": "symbol",
256
+ "f14": "name",
257
+ "f15": "high",
258
+ "f16": "low",
259
+ "f17": "open",
260
+ "f18": "yesterday_price",
261
+ "f20": "total_mv",
262
+ "f21": "flow_mv",
263
+ "f26": "list_date",
264
+ "f33": "wei_bi",
265
+ "f34": "outer_disk",
266
+ "f35": "inner_disk",
267
+ "f62": "today_main_net_inflow",
268
+ "f66": "super_large_order_net_inflow",
269
+ "f69": "super_large_order_net_inflow_ratio",
270
+ "f72": "large_order_net_inflow",
271
+ # "f78": "medium_order_net_inflow",
272
+ # "f84": "small_order_net_inflow",
273
+ "f100": "industry",
274
+ # "f103": "concept",
275
+ "f184": "today_main_net_inflow_ratio",
276
+ "f352": "average_price",
277
+ "f211": "buy_1_num",
278
+ "f212": "sell_1_num"
279
+ })
280
+ if data_frame_util.is_empty(temp_df):
281
+ return pd.DataFrame()
282
+ else:
283
+ temp_df.loc[temp_df['buy_1_num'] == '-', 'buy_1_num'] = 0
284
+ temp_df.loc[temp_df['sell_1_num'] == '-', 'sell_1_num'] = 0
285
+ temp_df.loc[temp_df['up_speed_05'] == '-', 'up_speed_05'] = 0
286
+ temp_df.loc[temp_df['up_speed'] == '-', 'up_speed'] = 0
287
+ temp_df.loc[temp_df['average_price'] == '-', 'average_price'] = 0
288
+ temp_df.loc[temp_df['wei_bi'] == '-', 'wei_bi'] = 0
289
+ temp_df.loc[temp_df['yesterday_price'] == '-', 'yesterday_price'] = 0
290
+ temp_df.loc[temp_df['now_price'] == '-', 'now_price'] = 0
291
+ temp_df.loc[temp_df['chg'] == '-', 'chg'] = 0
292
+ temp_df.loc[temp_df['volume'] == '-', 'volume'] = 0
293
+ temp_df.loc[temp_df['amount'] == '-', 'amount'] = 0
294
+ temp_df.loc[temp_df['exchange'] == '-', 'exchange'] = 0
295
+ temp_df.loc[temp_df['quantity_ratio'] == '-', 'quantity_ratio'] = 0
296
+ temp_df.loc[temp_df['high'] == '-', 'high'] = 0
297
+ temp_df.loc[temp_df['low'] == '-', 'low'] = 0
298
+ temp_df.loc[temp_df['open'] == '-', 'open'] = 0
299
+ temp_df.loc[temp_df['total_mv'] == '-', 'total_mv'] = 0
300
+ temp_df.loc[temp_df['flow_mv'] == '-', 'flow_mv'] = 0
301
+ temp_df.loc[temp_df['inner_disk'] == '-', 'inner_disk'] = 0
302
+ temp_df.loc[temp_df['outer_disk'] == '-', 'outer_disk'] = 0
303
+ temp_df.loc[temp_df['today_main_net_inflow_ratio'] == '-', 'today_main_net_inflow_ratio'] = 0
304
+ temp_df.loc[temp_df['today_main_net_inflow'] == '-', 'today_main_net_inflow'] = 0
305
+ temp_df.loc[temp_df['super_large_order_net_inflow'] == '-', 'super_large_order_net_inflow'] = 0
306
+ temp_df.loc[temp_df['super_large_order_net_inflow_ratio'] == '-', 'super_large_order_net_inflow_ratio'] = 0
307
+ temp_df.loc[temp_df['large_order_net_inflow'] == '-', 'large_order_net_inflow'] = 0
308
+ # temp_df.loc[temp_df['medium_order_net_inflow'] == '-', 'medium_order_net_inflow'] = 0
309
+ # temp_df.loc[temp_df['small_order_net_inflow'] == '-', 'small_order_net_inflow'] = 0
310
+
311
+ temp_df["list_date"] = pd.to_numeric(temp_df["list_date"], errors="coerce")
312
+ temp_df["wei_bi"] = pd.to_numeric(temp_df["wei_bi"], errors="coerce")
313
+ temp_df["average_price"] = pd.to_numeric(temp_df["average_price"], errors="coerce")
314
+ temp_df["yesterday_price"] = pd.to_numeric(temp_df["yesterday_price"], errors="coerce")
315
+ temp_df["now_price"] = pd.to_numeric(temp_df["now_price"], errors="coerce")
316
+ temp_df["chg"] = pd.to_numeric(temp_df["chg"], errors="coerce")
317
+ temp_df["volume"] = pd.to_numeric(temp_df["volume"], errors="coerce")
318
+ temp_df["amount"] = pd.to_numeric(temp_df["amount"], errors="coerce")
319
+ temp_df["exchange"] = pd.to_numeric(temp_df["exchange"], errors="coerce")
320
+ temp_df["quantity_ratio"] = pd.to_numeric(temp_df["quantity_ratio"], errors="coerce")
321
+ temp_df["high"] = pd.to_numeric(temp_df["high"], errors="coerce")
322
+ temp_df["low"] = pd.to_numeric(temp_df["low"], errors="coerce")
323
+ temp_df["open"] = pd.to_numeric(temp_df["open"], errors="coerce")
324
+ temp_df["total_mv"] = pd.to_numeric(temp_df["total_mv"], errors="coerce")
325
+ temp_df["flow_mv"] = pd.to_numeric(temp_df["flow_mv"], errors="coerce")
326
+ temp_df["outer_disk"] = pd.to_numeric(temp_df["outer_disk"], errors="coerce")
327
+ temp_df["inner_disk"] = pd.to_numeric(temp_df["inner_disk"], errors="coerce")
328
+ temp_df["today_main_net_inflow"] = pd.to_numeric(temp_df["today_main_net_inflow"], errors="coerce")
329
+ temp_df["super_large_order_net_inflow"] = pd.to_numeric(temp_df["super_large_order_net_inflow"],
330
+ errors="coerce")
331
+ temp_df["super_large_order_net_inflow_ratio"] = pd.to_numeric(temp_df["super_large_order_net_inflow_ratio"],
332
+ errors="coerce")
333
+ temp_df["large_order_net_inflow"] = pd.to_numeric(temp_df["large_order_net_inflow"],
334
+ errors="coerce")
335
+ # temp_df["medium_order_net_inflow"] = pd.to_numeric(temp_df["medium_order_net_inflow"],
336
+ # errors="coerce")
337
+ # temp_df["small_order_net_inflow"] = pd.to_numeric(temp_df["small_order_net_inflow"], errors="coerce")
338
+
339
+ # 大单比例
340
+ temp_df['large_order_net_inflow_ratio'] = round((temp_df['large_order_net_inflow'] / temp_df['amount']) * 100,
341
+ 2)
342
+
343
+ # 外盘是内盘倍数
344
+ temp_df['disk_ratio'] = round((temp_df['outer_disk'] - temp_df['inner_disk']) / temp_df['inner_disk'], 2)
345
+ # 只有外盘没有内盘
346
+ temp_df.loc[temp_df["inner_disk"] == 0, ['disk_ratio']] = 1688
347
+ temp_df = temp_df.sort_values(by=['chg'], ascending=False)
348
+ return temp_df
349
+
350
+
351
+ if __name__ == '__main__':
352
+
353
+ while True:
354
+ # proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
355
+ # proxies = {"https": proxy_ip,
356
+ # "http": proxy_ip}
357
+ time_out = 10 # Set the timeout value
358
+ result = repeated_acquisition_ask_async(time_out, 5800, 6, 10)
359
+ print(result)
@@ -1,4 +1,4 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mns-common
3
- Version: 1.5.1.8
3
+ Version: 1.5.2.0
4
4
 
@@ -1,6 +1,18 @@
1
1
  mns_common/__init__.py,sha256=2U9DiKslxsWwLLEcZKjS8UiQPN1QgALvnK3HiJNIZE0,163
2
2
  mns_common/api/__init__.py,sha256=2U9DiKslxsWwLLEcZKjS8UiQPN1QgALvnK3HiJNIZE0,163
3
3
  mns_common/api/em/__init__.py,sha256=2U9DiKslxsWwLLEcZKjS8UiQPN1QgALvnK3HiJNIZE0,163
4
+ mns_common/api/em/concept/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
5
+ mns_common/api/em/concept/em_concept_index_api.py,sha256=PP87ES8a_y0o3SKLzBsPrc7DCPI3MBCD-4SmoUUirl0,8285
6
+ mns_common/api/em/gd/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
7
+ mns_common/api/em/gd/east_money_stock_gdfx_free_top_10_api.py,sha256=I2-JjFjTjvOxjNAPzuJ1GK7ynvwJcZa_EbmRB2_pmm0,11813
8
+ mns_common/api/em/real_time/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
9
+ mns_common/api/em/real_time/east_money_debt_api.py,sha256=Ou97aq3-N7lge_Nj6FjkJo76agAhTLaz-Df-BK4dA0c,17483
10
+ mns_common/api/em/real_time/east_money_etf_api.py,sha256=wv1Ae3ydWDZkhlGgCKO8-z-PQj3qRSvoN75xNna4Qcs,19392
11
+ mns_common/api/em/real_time/east_money_stock_a_api.py,sha256=6xYcNJyMhFFL2eNxQ9c0TbQ53wgumNovVHisXgQX7bs,12737
12
+ mns_common/api/em/real_time/east_money_stock_a_v2_api.py,sha256=mL4uuL6sVsC2Vnl09826AUnzxePGAUhlZ7I5BBFw8Ks,14530
13
+ mns_common/api/em/real_time/east_money_stock_hk_api.py,sha256=G_GYF2JvKtYK0J61ySiUtkRKQGeCYz5-WACmucbZfzY,18015
14
+ mns_common/api/em/real_time/east_money_stock_us_api.py,sha256=RiTrdZDuDgTOtiMSD1Ba9aQAx4vghM66pEp_LicH3Ps,11632
15
+ mns_common/api/em/real_time/real_time_quotes_repeat_api.py,sha256=PKN2a_8fZpb7fA2Ns3FxVYiYlh34uxEFmmZu2MdqAu4,16008
4
16
  mns_common/api/hk/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
5
17
  mns_common/api/hk/ths_hk_company_info_api.py,sha256=Cxlbuccopa0G1s8o0uTnnyLn2QaxOvbDpJQJOj7J8a8,5360
6
18
  mns_common/api/k_line/__init__.py,sha256=itoGlqKhsx7EVXQoD1vchDKQ5GPB16vDjofTSuQtrXg,161
@@ -140,7 +152,7 @@ mns_common/utils/date_handle_util.py,sha256=XS-MyA8_7k35LOCFAYOHgVcVkMft_Kc4Wa9U
140
152
  mns_common/utils/db_util.py,sha256=hSmfNAN4vEeEaUva6_cicZEhb2jSnib-Gvk2reke1vc,2590
141
153
  mns_common/utils/file_util.py,sha256=egWu6PenGPRp_ixrNTHKarT4dAnOT6FETR82EHUZJnQ,1042
142
154
  mns_common/utils/ip_util.py,sha256=UTcYfz_uytB__6nlBf7T-izuI7hi4XdB6ET0sJgEel4,969
143
- mns_common-1.5.1.8.dist-info/METADATA,sha256=El6F612yVfiS14dd3hLUp27Xh47cFo-79NvGYlR88dQ,61
144
- mns_common-1.5.1.8.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
145
- mns_common-1.5.1.8.dist-info/top_level.txt,sha256=ZC58kAR-8Hvc6U2xhYNBNLAh3mb6sZazbdj5nZpvEkQ,11
146
- mns_common-1.5.1.8.dist-info/RECORD,,
155
+ mns_common-1.5.2.0.dist-info/METADATA,sha256=9O5_xcBozjOjz8fieymneWRrTLt4eb7mFIb8DUs_3C4,61
156
+ mns_common-1.5.2.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
157
+ mns_common-1.5.2.0.dist-info/top_level.txt,sha256=ZC58kAR-8Hvc6U2xhYNBNLAh3mb6sZazbdj5nZpvEkQ,11
158
+ mns_common-1.5.2.0.dist-info/RECORD,,