FinMind 2.0.9__tar.gz → 2.0.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {finmind-2.0.9 → finmind-2.0.10}/FinMind/data/data_loader.py +33 -0
  2. {finmind-2.0.9 → finmind-2.0.10}/FinMind/data/finmind_api.py +17 -10
  3. {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/data.py +1 -0
  4. finmind-2.0.10/FinMind/utility/request.py +277 -0
  5. {finmind-2.0.9/finmind.egg-info → finmind-2.0.10}/PKG-INFO +1 -1
  6. {finmind-2.0.9 → finmind-2.0.10/finmind.egg-info}/PKG-INFO +1 -1
  7. finmind-2.0.9/FinMind/utility/request.py +0 -156
  8. {finmind-2.0.9 → finmind-2.0.10}/FinMind/__init__.py +0 -0
  9. {finmind-2.0.9 → finmind-2.0.10}/FinMind/_version.py +0 -0
  10. {finmind-2.0.9 → finmind-2.0.10}/FinMind/crawler/__init__.py +0 -0
  11. {finmind-2.0.9 → finmind-2.0.10}/FinMind/crawler/base.py +0 -0
  12. {finmind-2.0.9 → finmind-2.0.10}/FinMind/crawler/commodities.py +0 -0
  13. {finmind-2.0.9 → finmind-2.0.10}/FinMind/crawler/demo.py +0 -0
  14. {finmind-2.0.9 → finmind-2.0.10}/FinMind/crawler/government_bonds.py +0 -0
  15. {finmind-2.0.9 → finmind-2.0.10}/FinMind/data/__init__.py +0 -0
  16. {finmind-2.0.9 → finmind-2.0.10}/FinMind/data/data_subscriber.py +0 -0
  17. {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/__init__.py +0 -0
  18. {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/bias.py +0 -0
  19. {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/continue_holding.py +0 -0
  20. {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/institutional_investors_follower.py +0 -0
  21. {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/institutional_investors_over_buy.py +0 -0
  22. {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/kd.py +0 -0
  23. {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/kd_crossover.py +0 -0
  24. {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/ma_cross_orver.py +0 -0
  25. {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/short_sale_margin_purchase_ratio.py +0 -0
  26. {finmind-2.0.9 → finmind-2.0.10}/FinMind/plotting/__init__.py +0 -0
  27. {finmind-2.0.9 → finmind-2.0.10}/FinMind/plotting/bar.py +0 -0
  28. {finmind-2.0.9 → finmind-2.0.10}/FinMind/plotting/kline.py +0 -0
  29. {finmind-2.0.9 → finmind-2.0.10}/FinMind/plotting/line.py +0 -0
  30. {finmind-2.0.9 → finmind-2.0.10}/FinMind/plotting/pie.py +0 -0
  31. {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/__init__.py +0 -0
  32. {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/indicators.py +0 -0
  33. {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/info.py +0 -0
  34. {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/plot.py +0 -0
  35. {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/rule.py +0 -0
  36. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/__init__.py +0 -0
  37. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/base.py +0 -0
  38. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/bias.py +0 -0
  39. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/continue_holding.py +0 -0
  40. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/institutional_investors_follower.py +0 -0
  41. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/kd.py +0 -0
  42. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/kd_crossover.py +0 -0
  43. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/ma_crossover.py +0 -0
  44. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/macd_crossover.py +0 -0
  45. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/max_min_period_bias.py +0 -0
  46. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/naive_kd.py +0 -0
  47. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/short_sale_margin_purchase_ratio.py +0 -0
  48. {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/utils.py +0 -0
  49. {finmind-2.0.9 → finmind-2.0.10}/FinMind/templates/post.html +0 -0
  50. {finmind-2.0.9 → finmind-2.0.10}/FinMind/utility/__init__.py +0 -0
  51. {finmind-2.0.9 → finmind-2.0.10}/FinMind/utility/rule.py +0 -0
  52. {finmind-2.0.9 → finmind-2.0.10}/LICENSE +0 -0
  53. {finmind-2.0.9 → finmind-2.0.10}/README.md +0 -0
  54. {finmind-2.0.9 → finmind-2.0.10}/finmind.egg-info/SOURCES.txt +0 -0
  55. {finmind-2.0.9 → finmind-2.0.10}/finmind.egg-info/dependency_links.txt +0 -0
  56. {finmind-2.0.9 → finmind-2.0.10}/finmind.egg-info/requires.txt +0 -0
  57. {finmind-2.0.9 → finmind-2.0.10}/finmind.egg-info/top_level.txt +0 -0
  58. {finmind-2.0.9 → finmind-2.0.10}/pyproject.toml +0 -0
  59. {finmind-2.0.9 → finmind-2.0.10}/setup.cfg +0 -0
  60. {finmind-2.0.9 → finmind-2.0.10}/setup.py +0 -0
@@ -1217,6 +1217,38 @@ class DataLoader(FinMindApi):
1217
1217
  )
1218
1218
  return futures_daily
1219
1219
 
1220
+ def taiwan_futures_kbar(
1221
+ self,
1222
+ futures_id: str = "",
1223
+ date: str = "",
1224
+ timeout: int = None,
1225
+ ) -> pd.DataFrame:
1226
+ """get 期貨分K資料
1227
+ :param futures_id: 期貨代號("TX")
1228
+ :param date (str): 日期("2024-01-02")
1229
+ :param timeout (int): timeout seconds, default None
1230
+
1231
+ :return: 期貨分K TaiwanFuturesKBar
1232
+ :rtype pd.DataFrame
1233
+ :rtype column date (str): 日期
1234
+ :rtype column futures_id (str): 期貨代碼
1235
+ :rtype column contract_date (str): 到期月份
1236
+ :rtype column minute (str): 分鐘時間
1237
+ :rtype column open (float): 開盤價
1238
+ :rtype column high (float): 最高價
1239
+ :rtype column low (float): 最低價
1240
+ :rtype column close (float): 收盤價
1241
+ :rtype column volume (int): 成交量
1242
+ """
1243
+ futures_kbar = self.get_data(
1244
+ dataset=Dataset.TaiwanFuturesKBar,
1245
+ data_id=futures_id,
1246
+ start_date=date,
1247
+ end_date=date,
1248
+ timeout=timeout,
1249
+ )
1250
+ return futures_kbar
1251
+
1220
1252
  def taiwan_option_daily(
1221
1253
  self,
1222
1254
  option_id: str = "",
@@ -2088,6 +2120,7 @@ class DataLoader(FinMindApi):
2088
2120
  use_object: bool = False,
2089
2121
  ) -> pd.DataFrame:
2090
2122
  """get 台股分 K 資料表
2123
+ :param stock_id (str): 股票代碼, 加權指數請帶 "TAIEX"
2091
2124
  :param timeout (int): timeout seconds, default None
2092
2125
  :param use_object (bool): 是否透過 signed URL 下載整日 parquet 資料物件,
2093
2126
  設為 True 時忽略 stock_id, stock_id_list, use_async 參數, default False
@@ -7,7 +7,11 @@ import requests
7
7
  from loguru import logger
8
8
 
9
9
  from FinMind.schema.data import Dataset
10
- from FinMind.utility.request import async_request_get, request_get
10
+ from FinMind.utility.request import (
11
+ async_request_get,
12
+ download_storage_object,
13
+ request_get,
14
+ )
11
15
 
12
16
  logger.remove()
13
17
  logger.add(sys.stderr, level="INFO")
@@ -398,11 +402,19 @@ class FinMindApi:
398
402
  data = pd.DataFrame(self._extract_data(data))
399
403
  return data
400
404
 
401
- def get_object(self, dataset: str, date: str, timeout: int) -> pd.DataFrame:
405
+ def get_object(
406
+ self,
407
+ dataset: str,
408
+ date: str,
409
+ timeout: int,
410
+ max_retry_times: int = 10,
411
+ ) -> pd.DataFrame:
402
412
  """透過 signed URL 下載整日資料物件 (parquet 格式)
413
+ 傳輸中斷時會自動重試,並以 HTTP Range 從中斷處續傳
403
414
  :param dataset (str): 資料集名稱 (e.g. "TaiwanStockPriceTick")
404
415
  :param date (str): 資料日期 ("2025-01-06")
405
416
  :param timeout (int): request timeout 秒數
417
+ :param max_retry_times (int): 最多重試次數
406
418
 
407
419
  :return: 該日期所有資料
408
420
  :rtype: pd.DataFrame
@@ -412,16 +424,11 @@ class FinMindApi:
412
424
  "dataset": dataset,
413
425
  "date": date,
414
426
  }
415
- response = request_get(
427
+ content = download_storage_object(
416
428
  self.__session,
417
429
  url,
418
430
  params=params,
419
431
  timeout=timeout,
432
+ max_retry_times=max_retry_times,
420
433
  )
421
- status_code = response.status_code
422
- if status_code == 200:
423
- return pd.read_parquet(io.BytesIO(response.content))
424
- else:
425
- resp_json = response.json()
426
- logger.info(resp_json)
427
- return pd.DataFrame()
434
+ return pd.read_parquet(io.BytesIO(content))
@@ -30,6 +30,7 @@ class Dataset(str, Enum):
30
30
  TaiwanOptionTick = "TaiwanOptionTick"
31
31
  TaiwanStockPriceBidAsk = "TaiwanStockPriceBidAsk"
32
32
  TaiwanFuturesDaily = "TaiwanFuturesDaily"
33
+ TaiwanFuturesKBar = "TaiwanFuturesKBar"
33
34
  TaiwanOptionDaily = "TaiwanOptionDaily"
34
35
  TaiwanStockNews = "TaiwanStockNews"
35
36
  TaiwanStockTotalReturnIndex = "TaiwanStockTotalReturnIndex"
@@ -0,0 +1,277 @@
1
+ import asyncio
2
+ import concurrent.futures
3
+ import hashlib
4
+ import os
5
+ import ssl
6
+ import time
7
+ from typing import Dict, List, Union
8
+
9
+ import nest_asyncio
10
+ import requests
11
+ from loguru import logger
12
+ from tqdm import tqdm
13
+
14
+ nest_asyncio.apply()
15
+
16
+
17
+ def request_get(
18
+ session: requests.Session,
19
+ url: str,
20
+ params: Dict[str, Union[int, str, float]] = None,
21
+ timeout: int = 60,
22
+ max_retry_times: int = 10,
23
+ verbose: bool = False,
24
+ ):
25
+ """
26
+ 單次 request,支援 retry 與 log
27
+ """
28
+ response = None
29
+ for retry_times in range(1, max_retry_times + 1):
30
+ try:
31
+ response = session.get(
32
+ url,
33
+ verify=True,
34
+ params=params,
35
+ timeout=timeout,
36
+ )
37
+ if response.status_code == 504:
38
+ if verbose:
39
+ logger.warning(
40
+ f"status_code=504, retry {retry_times}/{max_retry_times}"
41
+ )
42
+ time.sleep(retry_times * 0.1)
43
+ else:
44
+ break
45
+ except (
46
+ requests.ConnectionError,
47
+ ssl.SSLError,
48
+ requests.exceptions.ReadTimeout,
49
+ requests.exceptions.ChunkedEncodingError,
50
+ ) as exc:
51
+ if verbose:
52
+ logger.warning(
53
+ f"Error: {exc}, retry {retry_times}/{max_retry_times}"
54
+ )
55
+ time.sleep(retry_times * 0.1)
56
+ except Exception as exc:
57
+ if verbose:
58
+ logger.warning(
59
+ f"Unexpected error: {exc}, retry {retry_times}/{max_retry_times}"
60
+ )
61
+ time.sleep(retry_times * 0.1)
62
+ if response and response.status_code != 200:
63
+ raise Exception(
64
+ f"Final response status: {response.status_code}, text: {response.text}"
65
+ )
66
+ return response
67
+
68
+
69
+ RETRYABLE_ERRORS = (
70
+ requests.ConnectionError,
71
+ ssl.SSLError,
72
+ requests.exceptions.ReadTimeout,
73
+ requests.exceptions.ChunkedEncodingError,
74
+ )
75
+ REDIRECT_STATUS = (301, 302, 303, 307, 308)
76
+
77
+
78
+ def download_storage_object(
79
+ session: requests.Session,
80
+ url: str,
81
+ params: Dict[str, Union[int, str, float]] = None,
82
+ timeout: int = 60,
83
+ max_retry_times: int = 10,
84
+ verbose: bool = False,
85
+ ) -> bytes:
86
+ """
87
+ 下載整日資料物件,傳輸中斷時以 Range + If-Match 續傳
88
+
89
+ storage_objects 回 307 導向有效期限約 3 分鐘的下載網址,
90
+ 所以每次(含重試)都重新取得網址;已收到的內容保留,
91
+ 只要求剩下的位元組,並以 If-Match 確認仍是同一版本,
92
+ 版本不同(412)才從頭下載。完成後檢查長度,ETag 不含 "-" 時再比對 MD5。
93
+ """
94
+ buffer = bytearray()
95
+ etag = None
96
+ total = None
97
+ for retry_times in range(1, max_retry_times + 1):
98
+ try:
99
+ response = session.get(
100
+ url,
101
+ verify=True,
102
+ params=params,
103
+ timeout=timeout,
104
+ allow_redirects=False,
105
+ )
106
+ except RETRYABLE_ERRORS as exc:
107
+ if verbose:
108
+ logger.warning(
109
+ f"Error: {exc}, retry {retry_times}/{max_retry_times}"
110
+ )
111
+ time.sleep(retry_times * 0.5)
112
+ continue
113
+ if response.status_code == 200:
114
+ return response.content
115
+ if response.status_code == 504:
116
+ time.sleep(retry_times * 0.5)
117
+ continue
118
+ if response.status_code not in REDIRECT_STATUS:
119
+ raise Exception(
120
+ f"Final response status: {response.status_code}, text: {response.text}"
121
+ )
122
+ object_url = response.headers["Location"]
123
+
124
+ headers = {}
125
+ if buffer and etag:
126
+ headers = {"Range": f"bytes={len(buffer)}-", "If-Match": etag}
127
+ try:
128
+ # 下載網址本身帶簽章,不能帶 session 的 Authorization header
129
+ with requests.get(
130
+ object_url, headers=headers, stream=True, timeout=timeout
131
+ ) as object_response:
132
+ status_code = object_response.status_code
133
+ if status_code == 200:
134
+ buffer = bytearray()
135
+ etag = object_response.headers.get("ETag")
136
+ content_length = object_response.headers.get(
137
+ "Content-Length"
138
+ )
139
+ total = int(content_length) if content_length else None
140
+ elif status_code == 412:
141
+ # 物件在兩次請求之間已更新,已收到的內容作廢
142
+ buffer, etag, total = bytearray(), None, None
143
+ continue
144
+ elif status_code != 206:
145
+ if verbose:
146
+ logger.warning(
147
+ f"status_code={status_code}, retry {retry_times}/{max_retry_times}"
148
+ )
149
+ time.sleep(retry_times * 0.5)
150
+ continue
151
+ for chunk in object_response.iter_content(chunk_size=1 << 20):
152
+ buffer.extend(chunk)
153
+ except RETRYABLE_ERRORS as exc:
154
+ if verbose:
155
+ logger.warning(
156
+ f"Download interrupted at {len(buffer)} bytes: {exc}, "
157
+ f"retry {retry_times}/{max_retry_times}"
158
+ )
159
+ time.sleep(retry_times * 0.5)
160
+ continue
161
+
162
+ if total is not None and len(buffer) < total:
163
+ continue
164
+ too_long = total is not None and len(buffer) > total
165
+ if too_long or not _match_etag(buffer, etag):
166
+ if verbose:
167
+ logger.warning(
168
+ f"Downloaded object failed verification, retry {retry_times}/{max_retry_times}"
169
+ )
170
+ buffer, etag, total = bytearray(), None, None
171
+ continue
172
+ return bytes(buffer)
173
+ raise Exception(
174
+ f"Download storage object failed after {max_retry_times} retries: "
175
+ f"received {len(buffer)} of {total} bytes"
176
+ )
177
+
178
+
179
+ def _match_etag(content: bytearray, etag: str) -> bool:
180
+ """ETag 不含 "-" 時等於整個物件的 MD5;分段上傳的 ETag 無法以 MD5 驗證"""
181
+ if not etag:
182
+ return True
183
+ etag = etag.strip('"')
184
+ if "-" in etag:
185
+ return True
186
+ return hashlib.md5(content).hexdigest() == etag
187
+
188
+
189
+ async def _loop_run_get(
190
+ executor: concurrent.futures.ThreadPoolExecutor,
191
+ loop: asyncio.AbstractEventLoop,
192
+ session: requests.Session,
193
+ url: str,
194
+ params: Dict[str, Union[str, int, float]],
195
+ timeout: int = 60,
196
+ max_retry_times: int = 10,
197
+ verbose: bool = False,
198
+ ):
199
+ """
200
+ 將同步 request 包成 async
201
+ """
202
+ return await loop.run_in_executor(
203
+ executor,
204
+ request_get,
205
+ session,
206
+ url,
207
+ params,
208
+ timeout,
209
+ max_retry_times,
210
+ verbose,
211
+ )
212
+
213
+
214
+ def async_request_get(
215
+ session: requests.Session,
216
+ url: str,
217
+ params_list: List[Dict[str, Union[str, int, float]]],
218
+ timeout: int = 60,
219
+ max_retry_times: int = 10,
220
+ verbose: bool = False,
221
+ auto_tune: bool = True,
222
+ max_concurrency: int = None,
223
+ batch_size: int = 10,
224
+ ):
225
+ """
226
+ 批量 async request,支援自動根據機器資源調整 max_concurrency / batch_size
227
+ """
228
+ # 自動調整
229
+ if auto_tune:
230
+ cpu_count = os.cpu_count() or 1
231
+ max_concurrency = max_concurrency or max(1, cpu_count * 2)
232
+ else:
233
+ max_concurrency = max_concurrency or 10
234
+
235
+ async def runner():
236
+ semaphore = asyncio.Semaphore(max_concurrency)
237
+ loop = asyncio.get_event_loop()
238
+ executor = concurrent.futures.ThreadPoolExecutor(
239
+ max_workers=max_concurrency
240
+ )
241
+
242
+ async def _limited_run(params):
243
+ async with semaphore:
244
+ resp = await _loop_run_get(
245
+ executor,
246
+ loop,
247
+ session,
248
+ url=url,
249
+ params=params,
250
+ timeout=timeout,
251
+ max_retry_times=max_retry_times,
252
+ verbose=verbose,
253
+ )
254
+ return resp
255
+
256
+ results = []
257
+ pbar = tqdm(total=len(params_list))
258
+
259
+ # 分 batch 建立 task
260
+ for i in range(0, len(params_list), batch_size):
261
+ batch = params_list[i : i + batch_size]
262
+ tasks = [asyncio.create_task(_limited_run(p)) for p in batch]
263
+
264
+ for coro in asyncio.as_completed(tasks):
265
+ try:
266
+ res = await coro
267
+ results.append(res)
268
+ except Exception as exc:
269
+ if verbose:
270
+ logger.error(f"Task failed: {exc}")
271
+ pbar.update(1)
272
+
273
+ pbar.close()
274
+ executor.shutdown()
275
+ return results
276
+
277
+ return asyncio.run(runner())
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: finmind
3
- Version: 2.0.9
3
+ Version: 2.0.10
4
4
  Summary: financial mining
5
5
  Author-email: linsam <finmind.tw@gmail.com>
6
6
  License: Apache License
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: finmind
3
- Version: 2.0.9
3
+ Version: 2.0.10
4
4
  Summary: financial mining
5
5
  Author-email: linsam <finmind.tw@gmail.com>
6
6
  License: Apache License
@@ -1,156 +0,0 @@
1
- import asyncio
2
- import concurrent.futures
3
- import os
4
- import ssl
5
- import time
6
- from typing import Dict, List, Union
7
-
8
- import nest_asyncio
9
- import requests
10
- from loguru import logger
11
- from tqdm import tqdm
12
-
13
- nest_asyncio.apply()
14
-
15
-
16
- def request_get(
17
- session: requests.Session,
18
- url: str,
19
- params: Dict[str, Union[int, str, float]] = None,
20
- timeout: int = 60,
21
- max_retry_times: int = 10,
22
- verbose: bool = False,
23
- ):
24
- """
25
- 單次 request,支援 retry 與 log
26
- """
27
- response = None
28
- for retry_times in range(1, max_retry_times + 1):
29
- try:
30
- response = session.get(
31
- url,
32
- verify=True,
33
- params=params,
34
- timeout=timeout,
35
- )
36
- if response.status_code == 504:
37
- if verbose:
38
- logger.warning(
39
- f"status_code=504, retry {retry_times}/{max_retry_times}"
40
- )
41
- time.sleep(retry_times * 0.1)
42
- else:
43
- break
44
- except (
45
- requests.ConnectionError,
46
- ssl.SSLError,
47
- requests.exceptions.ReadTimeout,
48
- requests.exceptions.ChunkedEncodingError,
49
- ) as exc:
50
- if verbose:
51
- logger.warning(
52
- f"Error: {exc}, retry {retry_times}/{max_retry_times}"
53
- )
54
- time.sleep(retry_times * 0.1)
55
- except Exception as exc:
56
- if verbose:
57
- logger.warning(
58
- f"Unexpected error: {exc}, retry {retry_times}/{max_retry_times}"
59
- )
60
- time.sleep(retry_times * 0.1)
61
- if response and response.status_code != 200:
62
- raise Exception(
63
- f"Final response status: {response.status_code}, text: {response.text}"
64
- )
65
- return response
66
-
67
-
68
- async def _loop_run_get(
69
- executor: concurrent.futures.ThreadPoolExecutor,
70
- loop: asyncio.AbstractEventLoop,
71
- session: requests.Session,
72
- url: str,
73
- params: Dict[str, Union[str, int, float]],
74
- timeout: int = 60,
75
- max_retry_times: int = 10,
76
- verbose: bool = False,
77
- ):
78
- """
79
- 將同步 request 包成 async
80
- """
81
- return await loop.run_in_executor(
82
- executor,
83
- request_get,
84
- session,
85
- url,
86
- params,
87
- timeout,
88
- max_retry_times,
89
- verbose,
90
- )
91
-
92
-
93
- def async_request_get(
94
- session: requests.Session,
95
- url: str,
96
- params_list: List[Dict[str, Union[str, int, float]]],
97
- timeout: int = 60,
98
- max_retry_times: int = 10,
99
- verbose: bool = False,
100
- auto_tune: bool = True,
101
- max_concurrency: int = None,
102
- batch_size: int = 10,
103
- ):
104
- """
105
- 批量 async request,支援自動根據機器資源調整 max_concurrency / batch_size
106
- """
107
- # 自動調整
108
- if auto_tune:
109
- cpu_count = os.cpu_count() or 1
110
- max_concurrency = max_concurrency or max(1, cpu_count * 2)
111
- else:
112
- max_concurrency = max_concurrency or 10
113
-
114
- async def runner():
115
- semaphore = asyncio.Semaphore(max_concurrency)
116
- loop = asyncio.get_event_loop()
117
- executor = concurrent.futures.ThreadPoolExecutor(
118
- max_workers=max_concurrency
119
- )
120
-
121
- async def _limited_run(params):
122
- async with semaphore:
123
- resp = await _loop_run_get(
124
- executor,
125
- loop,
126
- session,
127
- url=url,
128
- params=params,
129
- timeout=timeout,
130
- max_retry_times=max_retry_times,
131
- verbose=verbose,
132
- )
133
- return resp
134
-
135
- results = []
136
- pbar = tqdm(total=len(params_list))
137
-
138
- # 分 batch 建立 task
139
- for i in range(0, len(params_list), batch_size):
140
- batch = params_list[i : i + batch_size]
141
- tasks = [asyncio.create_task(_limited_run(p)) for p in batch]
142
-
143
- for coro in asyncio.as_completed(tasks):
144
- try:
145
- res = await coro
146
- results.append(res)
147
- except Exception as exc:
148
- if verbose:
149
- logger.error(f"Task failed: {exc}")
150
- pbar.update(1)
151
-
152
- pbar.close()
153
- executor.shutdown()
154
- return results
155
-
156
- return asyncio.run(runner())
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes