mns-common 1.4.1.8__py3-none-any.whl → 1.5.7.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. mns_common/api/akshare/__init__.py +0 -1
  2. mns_common/api/akshare/k_line_api.py +19 -2
  3. mns_common/api/akshare/stock_bid_ask_api.py +10 -3
  4. mns_common/api/akshare/stock_zb_pool.py +2 -0
  5. mns_common/api/akshare/stock_zt_pool_api.py +1 -1
  6. mns_common/api/em/gd/east_money_stock_gdfx_free_top_10_api.py +62 -7
  7. mns_common/api/em/real_time/__init__.py +1 -1
  8. mns_common/api/em/real_time/east_money_debt_api.py +140 -70
  9. mns_common/api/em/real_time/east_money_etf_api.py +138 -27
  10. mns_common/api/em/real_time/east_money_stock_a_api.py +24 -28
  11. mns_common/api/em/real_time/east_money_stock_a_v2_api.py +97 -53
  12. mns_common/api/em/real_time/east_money_stock_common_api.py +174 -0
  13. mns_common/api/em/real_time/east_money_stock_hk_api.py +223 -272
  14. mns_common/api/em/real_time/east_money_stock_hk_gtt_api.py +260 -0
  15. mns_common/api/em/real_time/east_money_stock_multi_thread_api_v3.py +154 -0
  16. mns_common/api/em/real_time/east_money_stock_us_api.py +146 -82
  17. mns_common/api/em/real_time/real_time_quotes_repeat_api.py +195 -0
  18. mns_common/api/k_line/stock_k_line_data_api.py +11 -1
  19. mns_common/api/kpl/common/kpl_common_api.py +35 -0
  20. mns_common/api/proxies/liu_guan_proxy_api.py +55 -5
  21. mns_common/api/ths/company/company_product_area_industry_index_query.py +46 -0
  22. mns_common/api/ths/company/ths_company_info_api.py +2 -1
  23. mns_common/api/ths/company/ths_company_info_web.py +159 -0
  24. mns_common/api/ths/concept/app/ths_concept_index_app.py +3 -1
  25. mns_common/api/ths/wen_cai/ths_wen_cai_api.py +1 -1
  26. mns_common/api/ths/zt/ths_stock_zt_pool_api.py +20 -1
  27. mns_common/api/ths/zt/ths_stock_zt_pool_v2_api.py +105 -29
  28. mns_common/api/xueqiu/xue_qiu_k_line_api.py +2 -2
  29. mns_common/component/common_service_fun_api.py +26 -6
  30. mns_common/component/data/data_init_api.py +13 -8
  31. mns_common/component/deal/deal_service_api.py +70 -8
  32. mns_common/component/deal/deal_service_v2_api.py +167 -0
  33. mns_common/component/em/em_stock_info_api.py +9 -3
  34. mns_common/component/main_line/main_line_zt_reason_service.py +237 -0
  35. mns_common/component/proxies/proxy_common_api.py +141 -45
  36. mns_common/component/us/us_stock_etf_info_api.py +125 -0
  37. mns_common/constant/db_name_constant.py +40 -16
  38. mns_common/constant/extra_income_db_name.py +79 -19
  39. mns_common/constant/strategy_classify.py +17 -2
  40. mns_common/db/MongodbUtil.py +3 -0
  41. mns_common/db/MongodbUtilLocal.py +3 -0
  42. {mns_common-1.4.1.8.dist-info → mns_common-1.5.7.2.dist-info}/METADATA +1 -1
  43. {mns_common-1.4.1.8.dist-info → mns_common-1.5.7.2.dist-info}/RECORD +47 -41
  44. mns_common/api/ths/concept/web/ths_company_info_web.py +0 -163
  45. mns_common/component/qmt/qmt_buy_service.py +0 -172
  46. mns_common/component/task/real_time_data_sync_check.py +0 -110
  47. /mns_common/component/{qmt → main_line}/__init__.py +0 -0
  48. /mns_common/component/{task → us}/__init__.py +0 -0
  49. {mns_common-1.4.1.8.dist-info → mns_common-1.5.7.2.dist-info}/WHEEL +0 -0
  50. {mns_common-1.4.1.8.dist-info → mns_common-1.5.7.2.dist-info}/top_level.txt +0 -0
@@ -5,4 +5,3 @@ file_path = os.path.abspath(__file__)
5
5
  end = file_path.index('mns') + 14
6
6
  project_path = file_path[0:end]
7
7
  sys.path.append(project_path)
8
-
@@ -18,6 +18,7 @@ def stock_zh_a_hist(
18
18
  start_date: str = "19700101",
19
19
  end_date: str = "22220101",
20
20
  adjust: str = "",
21
+ proxies: str = None
21
22
  ) -> pd.DataFrame:
22
23
  """
23
24
  东方财富网-行情首页-沪深京 A 股-每日行情
@@ -32,6 +33,9 @@ def stock_zh_a_hist(
32
33
  :type end_date: str
33
34
  :param adjust: choice of {"qfq": "前复权", "hfq": "后复权", "": "不复权"}
34
35
  :type adjust: str
36
+ :param proxies: 代理ip
37
+ :type proxies: str
38
+
35
39
  :return: 每日行情
36
40
  :rtype: pandas.DataFrame
37
41
  """
@@ -53,7 +57,12 @@ def stock_zh_a_hist(
53
57
  "end": "20500000",
54
58
  "_": now_time,
55
59
  }
56
- r = requests.get(url, params=params)
60
+
61
+ if proxies is None:
62
+ r = requests.get(url, params=params)
63
+ else:
64
+ r = requests.get(url, params=params, proxies=proxies)
65
+
57
66
  data_json = r.json()
58
67
  temp_df = pd.DataFrame([item.split(",") for item in data_json["data"]["klines"]])
59
68
  temp_df.columns = [
@@ -100,7 +109,15 @@ def stock_zh_a_hist(
100
109
  return temp_df
101
110
 
102
111
 
112
+ import mns_common.component.proxies.proxy_common_api as proxy_common_api
113
+
103
114
  if __name__ == '__main__':
104
115
  while True:
105
- df = stock_zh_a_hist()
116
+ proxy_ip = proxy_common_api.get_proxy_ip(5)
117
+ df = stock_zh_a_hist("0.000001",
118
+ 'daily',
119
+ "19700101",
120
+ "22220101",
121
+ "",
122
+ proxy_ip)
106
123
  print(df)
@@ -9,7 +9,7 @@ import pandas as pd
9
9
  import requests
10
10
 
11
11
 
12
- def stock_bid_ask_em(symbol: str = "000001") -> dict:
12
+ def stock_bid_ask_em(symbol: str = "000001") -> pd.DataFrame:
13
13
  """
14
14
  东方财富-行情报价
15
15
  https://quote.eastmoney.com/sz000001.html
@@ -22,8 +22,15 @@ def stock_bid_ask_em(symbol: str = "000001") -> dict:
22
22
  params = {
23
23
  "fltt": "2",
24
24
  "invt": "2",
25
- "fields": "f43,f51,f52,f191,f120,f121,f122,f174,f175,f59,f163,f43,f57,f58,f169,f170,f46,f44,f51,f168,f47,f164,f116,f60,f45,f52,f50,f48,f167,f117,f71,f161,f49,f530,f135,f136,f137,f138,f139,f141,f142,f144,f145,f147,f148,f140,f143,f146,f149,f55,f62,f162,f92,f173,f104,f105,f84,f85,f183,f184,f185,f186,f187,f188,f189,f190,f191,f192,f107,f111,f86,f177,f78,f110,f262,f263,f264,f267,f268,f255,f256,f257,f258,f127,f199,f128,f198,f259,f260,f261,f171,f277,f278,f279,f288,f152,f250,f251,f252,f253,f254,f269,f270,f271,f272,f273,f274,f275,f276,f265,f266,f289,f290,f286,f285,f292,f293,f294,f295",
26
- # "fields" :fields_02,
25
+ "fields": "f120,f121,f122,f174,f175,f59,f163,f43,f57,f58,f169,f170,f46,f44,f51,"
26
+ "f168,f47,f164,f116,f60,f45,f52,f50,f48,f167,f117,f71,f161,f49,f530,"
27
+ "f135,f136,f137,f138,f139,f141,f142,f144,f145,f147,f148,f140,f143,f146,"
28
+ "f149,f55,f62,f162,f92,f173,f104,f105,f84,f85,f183,f184,f185,f186,f187,"
29
+ "f188,f189,f190,f191,f192,f107,f111,f86,f177,f78,f110,f262,f263,f264,f267,"
30
+ "f268,f255,f256,f257,f258,f127,f199,f128,f198,f259,f260,f261,f171,f277,f278,"
31
+ "f279,f288,f152,f250,f251,f252,f253,f254,f269,f270,f271,f272,f273,f274,f275,"
32
+ "f276,f265,f266,f289,f290,f286,f285,f292,f293,f294,f295",
33
+
27
34
  "secid": symbol,
28
35
  }
29
36
  r = requests.get(url, params=params)
@@ -32,6 +32,8 @@ def stock_zb_pool_df(date):
32
32
  "首次封板时间": "first_closure_time",
33
33
  "炸板次数": "frying_plates_numbers",
34
34
  "炸板股统计": "statistics",
35
+ "涨停统计": "zt_statistics",
36
+ "涨停价": "zt_price",
35
37
  "振幅": "pct_chg",
36
38
  "所属行业": "industry"
37
39
  }, inplace=True)
@@ -43,5 +43,5 @@ def stock_em_zt_pool_df(date):
43
43
 
44
44
 
45
45
  if __name__ == '__main__':
46
- df = stock_em_zt_pool_df('2024-09-04')
46
+ df = stock_em_zt_pool_df('2025-10-24')
47
47
  print(df)
@@ -13,6 +13,7 @@ import mns_common.component.em.em_stock_info_api as em_stock_info_api
13
13
  from mns_common.db.MongodbUtil import MongodbUtil
14
14
  import mns_common.constant.db_name_constant as db_name_constant
15
15
  import mns_common.component.company.company_common_service_new_api as company_common_service_new_api
16
+ from datetime import datetime
16
17
 
17
18
  mongodb_util = MongodbUtil('27017')
18
19
 
@@ -38,6 +39,8 @@ def get_stock_gdfx_free_top_10_em_api(str_day, symbol):
38
39
  return None
39
40
  stock_gdfx_free_top_10_em_df = stock_gdfx_free_top_10_em_df.fillna(0)
40
41
  stock_gdfx_free_top_10_em_df.index = stock_gdfx_free_top_10_em_df.index.astype(str)
42
+ stock_gdfx_free_top_10_em_df.drop_duplicates('shareholder_name', keep='last', inplace=True)
43
+
41
44
  return stock_gdfx_free_top_10_em_df
42
45
 
43
46
 
@@ -58,6 +61,7 @@ def get_stock_gdfx_top_10_em_api(str_day, symbol):
58
61
  return None
59
62
  stock_gdfx_top_10_em_df = stock_gdfx_top_10_em_df.fillna(0)
60
63
  stock_gdfx_top_10_em_df.index = stock_gdfx_top_10_em_df.index.astype(str)
64
+ stock_gdfx_top_10_em_df.drop_duplicates('shareholder_name', keep='last', inplace=True)
61
65
  return stock_gdfx_top_10_em_df
62
66
 
63
67
 
@@ -159,10 +163,34 @@ def get_stock_gdfx_free_top_10_em(str_day, symbol):
159
163
  # 保存10大流通股东
160
164
  def sync_stock_gdfx_free_top_10(stock_gdfx_free_top_10_em_df, period, symbol, str_day):
161
165
  if stock_gdfx_free_top_10_em_df is not None and stock_gdfx_free_top_10_em_df.shape[0] > 0:
166
+ # 更新日期
162
167
  stock_gdfx_free_top_10_em_df['str_day'] = str_day
168
+
163
169
  stock_gdfx_free_top_10_em_df['symbol'] = symbol
164
- stock_gdfx_free_top_10_em_df['_id'] = symbol + '_' + stock_gdfx_free_top_10_em_df.index + '_' + period
170
+
171
+ stock_gdfx_free_top_10_em_df['shares_number_str'] = stock_gdfx_free_top_10_em_df['shares_number'].astype(str)
172
+
173
+ stock_gdfx_free_top_10_em_df[
174
+ '_id'] = symbol + '_' + period + '_' + stock_gdfx_free_top_10_em_df.shares_number_str
165
175
  stock_gdfx_free_top_10_em_df['period'] = period
176
+
177
+ query_exist = {'symbol': symbol, 'period': period}
178
+ exist_df = mongodb_util.find_query_data(db_name_constant.STOCK_GDFX_FREE_TOP_10, query_exist)
179
+ now_date = datetime.now()
180
+ str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
181
+ # 不存在的时候更新创建时间
182
+ if exist_df.shape[0] == 0:
183
+ stock_gdfx_free_top_10_em_df['create_day'] = str_day
184
+ stock_gdfx_free_top_10_em_df['sync_time'] = str_now_date
185
+ else:
186
+ if 'create_day' in exist_df.columns:
187
+ stock_gdfx_free_top_10_em_df['create_day'] = list(exist_df['create_day'])[0]
188
+ else:
189
+ stock_gdfx_free_top_10_em_df['create_day'] = str_day
190
+ if 'sync_time' in exist_df.columns:
191
+ stock_gdfx_free_top_10_em_df['sync_time'] = list(exist_df['sync_time'])[0]
192
+ else:
193
+ stock_gdfx_free_top_10_em_df['sync_time'] = str_now_date
166
194
  mongodb_util.save_mongo(stock_gdfx_free_top_10_em_df, db_name_constant.STOCK_GDFX_FREE_TOP_10)
167
195
 
168
196
 
@@ -171,8 +199,34 @@ def sync_stock_gdfx_top_10(stock_gdfx_top_10_em_df, period, symbol, str_day):
171
199
  if stock_gdfx_top_10_em_df is not None and stock_gdfx_top_10_em_df.shape[0] > 0:
172
200
  stock_gdfx_top_10_em_df['str_day'] = str_day
173
201
  stock_gdfx_top_10_em_df['symbol'] = symbol
174
- stock_gdfx_top_10_em_df['_id'] = symbol + '_' + stock_gdfx_top_10_em_df.index + '_' + period
202
+
203
+ stock_gdfx_top_10_em_df['shares_number_str'] = stock_gdfx_top_10_em_df['shares_number'].astype(str)
204
+
205
+ stock_gdfx_top_10_em_df['_id'] = symbol + '_' + period + '_' + stock_gdfx_top_10_em_df.shares_number_str
175
206
  stock_gdfx_top_10_em_df['period'] = period
207
+
208
+ query_exist = {'symbol': symbol, 'period': period}
209
+ exist_df = mongodb_util.find_query_data(db_name_constant.STOCK_GDFX_TOP_10, query_exist)
210
+ now_date = datetime.now()
211
+ str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
212
+
213
+ # 不存在的时候更新创建时间
214
+ if exist_df.shape[0] == 0:
215
+ stock_gdfx_top_10_em_df['create_day'] = str_day
216
+ stock_gdfx_top_10_em_df['sync_time'] = str_now_date
217
+ else:
218
+ if 'create_day' in exist_df.columns:
219
+ stock_gdfx_top_10_em_df['create_day'] = list(exist_df['create_day'])[0]
220
+
221
+
222
+ else:
223
+ stock_gdfx_top_10_em_df['create_day'] = str_day
224
+
225
+ if 'sync_time' in exist_df.columns:
226
+ stock_gdfx_top_10_em_df['sync_time'] = list(exist_df['sync_time'])[0]
227
+ else:
228
+ stock_gdfx_top_10_em_df['sync_time'] = str_now_date
229
+
176
230
  mongodb_util.save_mongo(stock_gdfx_top_10_em_df, db_name_constant.STOCK_GDFX_TOP_10)
177
231
 
178
232
 
@@ -184,14 +238,15 @@ def sync_stock_gdfx_free_top_10_one_day(str_day):
184
238
  for real_time_one in real_time_quotes.itertuples():
185
239
  try:
186
240
  get_stock_gdfx_free_top_10_em(str_day, real_time_one.symbol)
241
+ logger.info('同步股票前十大流通东:{},{}', real_time_one.symbol, real_time_one.name)
187
242
  except BaseException as e:
188
243
  logger.error('同步所有股票前十大流通股本异常:{},{}', real_time_one.symbol, e)
244
+ logger.info('同步所有股票股东列表完成:{}', str_day)
189
245
 
190
246
 
191
- from datetime import datetime
192
-
193
247
  if __name__ == '__main__':
194
- now_date = datetime.now()
195
- str_day_test = now_date.strftime('%Y-%m-%d')
248
+ get_stock_gdfx_free_top_10_em('20250930', '300697')
249
+ now_date_test = datetime.now()
250
+ str_day_test = now_date_test.strftime('%Y-%m-%d')
196
251
  logger.info('同步所有股票前十大流通股本')
197
- get_stock_gdfx_free_top_10_em(str_day_test, '833580')
252
+ sync_stock_gdfx_free_top_10_one_day(str_day_test)
@@ -4,4 +4,4 @@ import os
4
4
  file_path = os.path.abspath(__file__)
5
5
  end = file_path.index('mns') + 16
6
6
  project_path = file_path[0:end]
7
- sys.path.append(project_path)
7
+ sys.path.append(project_path)
@@ -1,42 +1,76 @@
1
1
  import sys
2
2
  import os
3
3
 
4
- import sys
5
- import os
6
-
7
4
  file_path = os.path.abspath(__file__)
8
5
  end = file_path.index('mns') + 14
9
6
  project_path = file_path[0:end]
10
7
  sys.path.append(project_path)
11
- from concurrent.futures import ThreadPoolExecutor
8
+
9
+ import json
10
+ import akshare as ak
12
11
  import pandas as pd
13
12
  from loguru import logger
14
- import json
15
13
  import requests
16
14
  import time
17
- import akshare as ak
18
15
  import numpy as np
16
+ import mns_common.component.proxies.proxy_common_api as proxy_common_api
17
+ from concurrent.futures import ThreadPoolExecutor
18
+ from threading import Lock
19
+ import concurrent.futures
20
+ import mns_common.utils.data_frame_util as data_frame_util
19
21
 
20
- # 最大返回条数
21
- max_number = 600
22
- # 最小返回条数
23
- min_number = 500
24
22
  # 分页条数
25
23
  page_number = 100
26
24
 
25
+ fields = ("f352,f2,f3,f5,f6,f8,f10,f11,f22,f12,f14,f15,f16,f17,f18,f20,f21,f26,f33,f34,f35,f62,f66,f69,f72,f184,"
26
+ "f211,f212,f232,f233,f234")
27
+
28
+
29
+ def get_kzz_count(pn, proxies, page_size, time_out):
30
+ current_timestamp = str(int(round(time.time() * 1000, 0)))
31
+ url = "https://push2.eastmoney.com/api/qt/clist/get"
32
+
33
+ params = {
34
+ "cb": "jQuery34103608466964799838_" + current_timestamp,
35
+ "pn": str(pn),
36
+ "np": 3,
37
+ "ut": "8a086bfc3570bdde64a6a1c585cccb35",
38
+ "fltt": 1,
39
+ "invt": 1,
40
+ "fs": "m:0+e:11,m:1+e:11,m:1+e:11+s:4194304,m:0+e:11+s:8388608",
41
+ "dpt": "zqsc.zpg",
42
+ "fields": fields,
43
+ "wbp2u": "|0|0|0|wap",
44
+ "fid": "f12",
45
+ "po": 1,
46
+ "pz": str(page_size),
47
+ "_": current_timestamp
48
+ }
49
+ try:
50
+ if proxies is None:
51
+ r = requests.get(url, params, timeout=time_out)
52
+ else:
53
+ r = requests.get(url, params, proxies=proxies, timeout=time_out)
54
+ data_text = r.text
55
+
56
+ begin_index_total = data_text.index('"total":')
57
+
58
+ end_index_total = data_text.index('"diff"')
59
+ max_number = int(data_text[begin_index_total + 8:end_index_total - 1])
60
+ return max_number
61
+
62
+
63
+ except Exception as e:
64
+ logger.error("获取可转债列表,实时行情异常:{}", e)
65
+ return 0
27
66
 
28
- # fields_02 = "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13,f14,f15,f16,f17,f18,f19,f20,f21,f22,f23,f24,f25,f26,f27,f28,f29,f30,f31,f32,f33,f34,f35,f36,f37,f38,f39,f40,f41,f42,f43,f44,f45,f46,f47,f48,f49,f50,f51,f52,f53,f54,f55,f56,f57,f58,f59,f60,f61,f62,f63,f64,f65,f66,f67,f68,f69,f70,f71,f72,f73,f74,f75,f76,f77,f78,f79,f80,f81,f82,f83,f84,f85,f86,f87,f88,f89,f90,f91,f92,f93,f94,f95,f96,f97,f98,f99,f100,f101,f102,f103,f104,f105,f106,f107,f108" \
29
- # ",f109,f110,f111,f112,f113,f114,f115,f116,f117,f118,f119,f120,f121,f122,f123,f124,f125,f126,f127,f128,f129,f130,f131,f132,f133,f134,f135,f136,f137,f138,f139,f140,f141,f142,f143,f144,f145,f146,f147,f148,f149,f150,f151,f152,f153,f154,f155,f156,f157,f158,f159,f160,f161,f162,f163,f164,f165,f166,f167,f168,f169,f170,f171,f172,f173,f174,f175,f176,f177,f178,f179,f180,f181,f182,f183,f184,f185,f186,f187,f188,f189,f190,f191,f192,f193,f194,f195,f196,f197,f198,f199,f200" \
30
- # ",f209,f210,f211,f212,f213,f214,f215,f216,f217,f218,f219,f220,f221,f222,f223,f224,f225,f226,f227,f228,f229,f230,f231,f232,f233,f234,f235,f236,f237,f238,f239,f240,f241,f242,f243,f244,f245,f246,f247,f248,f249,f250,f251,f252,f253,f254,f255,f256,f257,f258,f259,f260,f261,f262,f263,f264,f265,f266,f267,f268,f269,f270,f271,f272,f273,f274,f275,f276,f277,f278,f279,f280,f281,f282,f283,f284,f285,f286,f287,f288,f289,f290,f291,f292,f293,f294,f295,f296,f297,f298,f299,f300" \
31
- # ",f309,f310,f312,f313,f314,f315,f316,f317,f318,f319,f320,f321,f322,f323,f324,f325,f326,f327,f328,f329,f330,f331,f332,f333,f334,f335,f336,f337,f338,f339,f340,f341,f342,f343,f344,f345,f346,f347,f348,f349,f350,f351,f352,f353,f354,f355,f356,f357,f358,f359,f360,f361,f362,f363,f364,f365,f366,f367,f368,f369,f370,f371,f372,f373,f374,f375,f376,f377,f378,f379,f380,f381,f382,f383,f384,f385,f386,f387,f388,f389,f390,f391,f392,f393,f394,f395,f396,f397,f398,f399,f401"
32
- #
33
67
 
34
68
  #
35
69
  # url = https://push2.eastmoney.com/api/qt/clist/get?cb=jQuery34103608466964799838_1718163189869&pn=1&np=1&ut
36
70
  # =8a086bfc3570bdde64a6a1c585cccb35&fltt=1&invt=1&fs=m:0+e:11,m:1+e:11,m:1+e:11+s:4194304,
37
71
  # m:0+e:11+s:8388608&dpt=zqsc.zpg&fields=f1,f2,f3,f4,f5,f6,f8,f10,f12,f13,f14,f18,f22,f152,
38
72
  # f237&wbp2u=|0|0|0|wap&fid=f3&po=1&pz=2000&_=1718163189870
39
- def get_debt_page_data(fields, pn, proxies) -> pd.DataFrame:
73
+ def get_debt_page_data(pn, proxies, page_size, time_out) -> pd.DataFrame:
40
74
  current_timestamp = str(int(round(time.time() * 1000, 0)))
41
75
  url = "https://push2.eastmoney.com/api/qt/clist/get"
42
76
 
@@ -53,14 +87,14 @@ def get_debt_page_data(fields, pn, proxies) -> pd.DataFrame:
53
87
  "wbp2u": "|0|0|0|wap",
54
88
  "fid": "f12",
55
89
  "po": 1,
56
- "pz": 2000,
90
+ "pz": str(page_size),
57
91
  "_": current_timestamp
58
92
  }
59
93
  try:
60
94
  if proxies is None:
61
- r = requests.get(url, params)
95
+ r = requests.get(url, params, timeout=time_out)
62
96
  else:
63
- r = requests.get(url, params, proxies=proxies)
97
+ r = requests.get(url, params, proxies=proxies, timeout=time_out)
64
98
  data_text = r.text
65
99
 
66
100
  if pn == 1:
@@ -87,52 +121,6 @@ def get_debt_page_data(fields, pn, proxies) -> pd.DataFrame:
87
121
  return pd.DataFrame()
88
122
 
89
123
 
90
- def all_debt_ticker_data(fields, proxies) -> pd.DataFrame:
91
- """
92
- 使用多线程获取所有债券数据
93
- """
94
- # 计算总页数,假设总共有1000条数据,每页200条
95
-
96
- per_page = page_number
97
- total_pages = (max_number + per_page - 1) // per_page # 向上取整
98
-
99
- # 创建线程池
100
- with ThreadPoolExecutor(max_workers=3) as executor:
101
- # 提交任务,获取每页数据
102
- futures = [executor.submit(get_debt_page_data, fields, pn, proxies)
103
- for pn in range(1, total_pages + 1)]
104
-
105
- # 收集结果
106
- results = []
107
- for future in futures:
108
- result = future.result()
109
- if not result.empty:
110
- results.append(result)
111
-
112
- # 合并所有页面的数据
113
- if results:
114
- return pd.concat(results, ignore_index=True)
115
- else:
116
- return pd.DataFrame()
117
-
118
-
119
- def get_debt_real_time_quotes(proxies):
120
- fields = ("f352,f2,f3,f5,f6,f8,f10,f11,f22,f12,f14,f15,f16,f17,f18,f20,f21,f26,f33,f34,f35,f62,f66,f69,f72,f184,"
121
- "f211,f212,f232,f233,f234")
122
- # 获取第一页数据
123
- page_one_df = get_debt_page_data(fields, 1, proxies)
124
- # 数据接口正常返回5600以上的数量
125
- if page_one_df.shape[0] > min_number:
126
- page_one_df = rename_real_time_quotes_df(page_one_df)
127
- page_one_df.drop_duplicates('symbol', keep='last', inplace=True)
128
- return page_one_df
129
- else:
130
- page_df = all_debt_ticker_data(fields, proxies)
131
- page_df = rename_real_time_quotes_df(page_df)
132
- page_df.drop_duplicates('symbol', keep='last', inplace=True)
133
- return page_df
134
-
135
-
136
124
  def rename_real_time_quotes_df(temp_df):
137
125
  temp_df = temp_df.rename(columns={
138
126
  "f2": "now_price",
@@ -298,9 +286,91 @@ def get_kzz_bond_info():
298
286
  logger.error("获取可转债信息异常:{}", e)
299
287
 
300
288
 
289
+ def repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread):
290
+ per_page = page_number
291
+ total_pages = (max_number + per_page - 1) // per_page # 向上取整
292
+ result_df = pd.DataFrame()
293
+
294
+ # 创建线程锁以确保线程安全
295
+ df_lock = Lock()
296
+
297
+ # 计算每个线程处理的页数范围
298
+ def process_page_range(start_page, end_page, thread_id):
299
+ nonlocal result_df
300
+ local_df = pd.DataFrame()
301
+ current_page = start_page
302
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
303
+
304
+ while current_page <= end_page and current_page <= total_pages:
305
+ proxies = {"https": proxy_ip, "http": proxy_ip}
306
+ try:
307
+ page_df = get_debt_page_data(current_page, proxies, page_number, time_out)
308
+ if data_frame_util.is_not_empty(page_df):
309
+ local_df = pd.concat([local_df, page_df])
310
+ logger.info("线程{}获取页面数据成功: {}", thread_id, current_page)
311
+ current_page += 1
312
+ else:
313
+ time.sleep(0.2)
314
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
315
+ logger.info("线程{}获取页面数据失败: {}", thread_id, current_page)
316
+ except BaseException as e:
317
+ time.sleep(1)
318
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
319
+ logger.error("线程{}处理页面{}时发生错误: {}", thread_id, current_page, e)
320
+
321
+ with df_lock:
322
+ result_df = pd.concat([result_df, local_df])
323
+ return len(local_df)
324
+
325
+ # 计算每个线程的页面范围
326
+ page_ranges = []
327
+ for i in range(num_threads):
328
+ start_page = i * pages_per_thread + 1
329
+ end_page = (i + 1) * pages_per_thread
330
+ if start_page > total_pages:
331
+ break
332
+ page_ranges.append((start_page, end_page, i + 1))
333
+
334
+ # 使用线程池执行任务
335
+ with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
336
+ # 提交所有任务
337
+ futures = [
338
+ executor.submit(process_page_range, start, end, tid)
339
+ for start, end, tid in page_ranges
340
+ ]
341
+
342
+ # 等待所有任务完成并获取结果
343
+ results = []
344
+ for future in concurrent.futures.as_completed(futures):
345
+ try:
346
+ result = future.result()
347
+ results.append(result)
348
+ except Exception as e:
349
+ logger.error("线程执行出错: {}", e)
350
+
351
+ return rename_real_time_quotes_df(result_df)
352
+
353
+
354
+ def get_kzz_real_time_quotes(time_out, pages_per_thread):
355
+ try_numer = 3
356
+ while try_numer > 0:
357
+ proxy_ip = proxy_common_api.generate_proxy_ip_api(1)
358
+ proxies = {"https": proxy_ip,
359
+ "http": proxy_ip}
360
+
361
+ max_number = get_kzz_count(1, proxies, 20, time_out)
362
+ if max_number > 0:
363
+ break
364
+ try_numer = try_numer - 1
365
+ if max_number == 0:
366
+ return pd.DataFrame()
367
+
368
+ total_pages = (max_number + page_number - 1) // page_number # 向上取整
369
+
370
+ num_threads = int((total_pages / pages_per_thread) + 1)
371
+ return repeated_acquisition_ask_etf_async(time_out, max_number, num_threads, pages_per_thread)
372
+
373
+
301
374
  if __name__ == '__main__':
302
- info_df = get_kzz_bond_info()
303
- print(info_df)
304
- while True:
305
- df = get_debt_real_time_quotes(None)
306
- logger.info(df)
375
+ test_df = get_kzz_real_time_quotes(30, 6)
376
+ print(test_df)