FinMind 2.0.9__tar.gz → 2.0.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/data/data_loader.py +33 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/data/finmind_api.py +17 -10
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/data.py +1 -0
- finmind-2.0.10/FinMind/utility/request.py +277 -0
- {finmind-2.0.9/finmind.egg-info → finmind-2.0.10}/PKG-INFO +1 -1
- {finmind-2.0.9 → finmind-2.0.10/finmind.egg-info}/PKG-INFO +1 -1
- finmind-2.0.9/FinMind/utility/request.py +0 -156
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/_version.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/crawler/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/crawler/base.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/crawler/commodities.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/crawler/demo.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/crawler/government_bonds.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/data/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/data/data_subscriber.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/bias.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/continue_holding.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/institutional_investors_follower.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/institutional_investors_over_buy.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/kd.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/kd_crossover.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/ma_cross_orver.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/indicators/short_sale_margin_purchase_ratio.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/plotting/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/plotting/bar.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/plotting/kline.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/plotting/line.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/plotting/pie.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/indicators.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/info.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/plot.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/schema/rule.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/base.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/bias.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/continue_holding.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/institutional_investors_follower.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/kd.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/kd_crossover.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/ma_crossover.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/macd_crossover.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/max_min_period_bias.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/naive_kd.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/short_sale_margin_purchase_ratio.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/strategies/utils.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/templates/post.html +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/utility/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/FinMind/utility/rule.py +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/LICENSE +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/README.md +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/finmind.egg-info/SOURCES.txt +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/finmind.egg-info/dependency_links.txt +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/finmind.egg-info/requires.txt +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/finmind.egg-info/top_level.txt +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/pyproject.toml +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/setup.cfg +0 -0
- {finmind-2.0.9 → finmind-2.0.10}/setup.py +0 -0
|
@@ -1217,6 +1217,38 @@ class DataLoader(FinMindApi):
|
|
|
1217
1217
|
)
|
|
1218
1218
|
return futures_daily
|
|
1219
1219
|
|
|
1220
|
+
def taiwan_futures_kbar(
|
|
1221
|
+
self,
|
|
1222
|
+
futures_id: str = "",
|
|
1223
|
+
date: str = "",
|
|
1224
|
+
timeout: int = None,
|
|
1225
|
+
) -> pd.DataFrame:
|
|
1226
|
+
"""get 期貨分K資料
|
|
1227
|
+
:param futures_id: 期貨代號("TX")
|
|
1228
|
+
:param date (str): 日期("2024-01-02")
|
|
1229
|
+
:param timeout (int): timeout seconds, default None
|
|
1230
|
+
|
|
1231
|
+
:return: 期貨分K TaiwanFuturesKBar
|
|
1232
|
+
:rtype pd.DataFrame
|
|
1233
|
+
:rtype column date (str): 日期
|
|
1234
|
+
:rtype column futures_id (str): 期貨代碼
|
|
1235
|
+
:rtype column contract_date (str): 到期月份
|
|
1236
|
+
:rtype column minute (str): 分鐘時間
|
|
1237
|
+
:rtype column open (float): 開盤價
|
|
1238
|
+
:rtype column high (float): 最高價
|
|
1239
|
+
:rtype column low (float): 最低價
|
|
1240
|
+
:rtype column close (float): 收盤價
|
|
1241
|
+
:rtype column volume (int): 成交量
|
|
1242
|
+
"""
|
|
1243
|
+
futures_kbar = self.get_data(
|
|
1244
|
+
dataset=Dataset.TaiwanFuturesKBar,
|
|
1245
|
+
data_id=futures_id,
|
|
1246
|
+
start_date=date,
|
|
1247
|
+
end_date=date,
|
|
1248
|
+
timeout=timeout,
|
|
1249
|
+
)
|
|
1250
|
+
return futures_kbar
|
|
1251
|
+
|
|
1220
1252
|
def taiwan_option_daily(
|
|
1221
1253
|
self,
|
|
1222
1254
|
option_id: str = "",
|
|
@@ -2088,6 +2120,7 @@ class DataLoader(FinMindApi):
|
|
|
2088
2120
|
use_object: bool = False,
|
|
2089
2121
|
) -> pd.DataFrame:
|
|
2090
2122
|
"""get 台股分 K 資料表
|
|
2123
|
+
:param stock_id (str): 股票代碼, 加權指數請帶 "TAIEX"
|
|
2091
2124
|
:param timeout (int): timeout seconds, default None
|
|
2092
2125
|
:param use_object (bool): 是否透過 signed URL 下載整日 parquet 資料物件,
|
|
2093
2126
|
設為 True 時忽略 stock_id, stock_id_list, use_async 參數, default False
|
|
@@ -7,7 +7,11 @@ import requests
|
|
|
7
7
|
from loguru import logger
|
|
8
8
|
|
|
9
9
|
from FinMind.schema.data import Dataset
|
|
10
|
-
from FinMind.utility.request import
|
|
10
|
+
from FinMind.utility.request import (
|
|
11
|
+
async_request_get,
|
|
12
|
+
download_storage_object,
|
|
13
|
+
request_get,
|
|
14
|
+
)
|
|
11
15
|
|
|
12
16
|
logger.remove()
|
|
13
17
|
logger.add(sys.stderr, level="INFO")
|
|
@@ -398,11 +402,19 @@ class FinMindApi:
|
|
|
398
402
|
data = pd.DataFrame(self._extract_data(data))
|
|
399
403
|
return data
|
|
400
404
|
|
|
401
|
-
def get_object(
|
|
405
|
+
def get_object(
|
|
406
|
+
self,
|
|
407
|
+
dataset: str,
|
|
408
|
+
date: str,
|
|
409
|
+
timeout: int,
|
|
410
|
+
max_retry_times: int = 10,
|
|
411
|
+
) -> pd.DataFrame:
|
|
402
412
|
"""透過 signed URL 下載整日資料物件 (parquet 格式)
|
|
413
|
+
傳輸中斷時會自動重試,並以 HTTP Range 從中斷處續傳
|
|
403
414
|
:param dataset (str): 資料集名稱 (e.g. "TaiwanStockPriceTick")
|
|
404
415
|
:param date (str): 資料日期 ("2025-01-06")
|
|
405
416
|
:param timeout (int): request timeout 秒數
|
|
417
|
+
:param max_retry_times (int): 最多重試次數
|
|
406
418
|
|
|
407
419
|
:return: 該日期所有資料
|
|
408
420
|
:rtype: pd.DataFrame
|
|
@@ -412,16 +424,11 @@ class FinMindApi:
|
|
|
412
424
|
"dataset": dataset,
|
|
413
425
|
"date": date,
|
|
414
426
|
}
|
|
415
|
-
|
|
427
|
+
content = download_storage_object(
|
|
416
428
|
self.__session,
|
|
417
429
|
url,
|
|
418
430
|
params=params,
|
|
419
431
|
timeout=timeout,
|
|
432
|
+
max_retry_times=max_retry_times,
|
|
420
433
|
)
|
|
421
|
-
|
|
422
|
-
if status_code == 200:
|
|
423
|
-
return pd.read_parquet(io.BytesIO(response.content))
|
|
424
|
-
else:
|
|
425
|
-
resp_json = response.json()
|
|
426
|
-
logger.info(resp_json)
|
|
427
|
-
return pd.DataFrame()
|
|
434
|
+
return pd.read_parquet(io.BytesIO(content))
|
|
@@ -30,6 +30,7 @@ class Dataset(str, Enum):
|
|
|
30
30
|
TaiwanOptionTick = "TaiwanOptionTick"
|
|
31
31
|
TaiwanStockPriceBidAsk = "TaiwanStockPriceBidAsk"
|
|
32
32
|
TaiwanFuturesDaily = "TaiwanFuturesDaily"
|
|
33
|
+
TaiwanFuturesKBar = "TaiwanFuturesKBar"
|
|
33
34
|
TaiwanOptionDaily = "TaiwanOptionDaily"
|
|
34
35
|
TaiwanStockNews = "TaiwanStockNews"
|
|
35
36
|
TaiwanStockTotalReturnIndex = "TaiwanStockTotalReturnIndex"
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import concurrent.futures
|
|
3
|
+
import hashlib
|
|
4
|
+
import os
|
|
5
|
+
import ssl
|
|
6
|
+
import time
|
|
7
|
+
from typing import Dict, List, Union
|
|
8
|
+
|
|
9
|
+
import nest_asyncio
|
|
10
|
+
import requests
|
|
11
|
+
from loguru import logger
|
|
12
|
+
from tqdm import tqdm
|
|
13
|
+
|
|
14
|
+
nest_asyncio.apply()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def request_get(
|
|
18
|
+
session: requests.Session,
|
|
19
|
+
url: str,
|
|
20
|
+
params: Dict[str, Union[int, str, float]] = None,
|
|
21
|
+
timeout: int = 60,
|
|
22
|
+
max_retry_times: int = 10,
|
|
23
|
+
verbose: bool = False,
|
|
24
|
+
):
|
|
25
|
+
"""
|
|
26
|
+
單次 request,支援 retry 與 log
|
|
27
|
+
"""
|
|
28
|
+
response = None
|
|
29
|
+
for retry_times in range(1, max_retry_times + 1):
|
|
30
|
+
try:
|
|
31
|
+
response = session.get(
|
|
32
|
+
url,
|
|
33
|
+
verify=True,
|
|
34
|
+
params=params,
|
|
35
|
+
timeout=timeout,
|
|
36
|
+
)
|
|
37
|
+
if response.status_code == 504:
|
|
38
|
+
if verbose:
|
|
39
|
+
logger.warning(
|
|
40
|
+
f"status_code=504, retry {retry_times}/{max_retry_times}"
|
|
41
|
+
)
|
|
42
|
+
time.sleep(retry_times * 0.1)
|
|
43
|
+
else:
|
|
44
|
+
break
|
|
45
|
+
except (
|
|
46
|
+
requests.ConnectionError,
|
|
47
|
+
ssl.SSLError,
|
|
48
|
+
requests.exceptions.ReadTimeout,
|
|
49
|
+
requests.exceptions.ChunkedEncodingError,
|
|
50
|
+
) as exc:
|
|
51
|
+
if verbose:
|
|
52
|
+
logger.warning(
|
|
53
|
+
f"Error: {exc}, retry {retry_times}/{max_retry_times}"
|
|
54
|
+
)
|
|
55
|
+
time.sleep(retry_times * 0.1)
|
|
56
|
+
except Exception as exc:
|
|
57
|
+
if verbose:
|
|
58
|
+
logger.warning(
|
|
59
|
+
f"Unexpected error: {exc}, retry {retry_times}/{max_retry_times}"
|
|
60
|
+
)
|
|
61
|
+
time.sleep(retry_times * 0.1)
|
|
62
|
+
if response and response.status_code != 200:
|
|
63
|
+
raise Exception(
|
|
64
|
+
f"Final response status: {response.status_code}, text: {response.text}"
|
|
65
|
+
)
|
|
66
|
+
return response
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
RETRYABLE_ERRORS = (
|
|
70
|
+
requests.ConnectionError,
|
|
71
|
+
ssl.SSLError,
|
|
72
|
+
requests.exceptions.ReadTimeout,
|
|
73
|
+
requests.exceptions.ChunkedEncodingError,
|
|
74
|
+
)
|
|
75
|
+
REDIRECT_STATUS = (301, 302, 303, 307, 308)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def download_storage_object(
|
|
79
|
+
session: requests.Session,
|
|
80
|
+
url: str,
|
|
81
|
+
params: Dict[str, Union[int, str, float]] = None,
|
|
82
|
+
timeout: int = 60,
|
|
83
|
+
max_retry_times: int = 10,
|
|
84
|
+
verbose: bool = False,
|
|
85
|
+
) -> bytes:
|
|
86
|
+
"""
|
|
87
|
+
下載整日資料物件,傳輸中斷時以 Range + If-Match 續傳
|
|
88
|
+
|
|
89
|
+
storage_objects 回 307 導向有效期限約 3 分鐘的下載網址,
|
|
90
|
+
所以每次(含重試)都重新取得網址;已收到的內容保留,
|
|
91
|
+
只要求剩下的位元組,並以 If-Match 確認仍是同一版本,
|
|
92
|
+
版本不同(412)才從頭下載。完成後檢查長度,ETag 不含 "-" 時再比對 MD5。
|
|
93
|
+
"""
|
|
94
|
+
buffer = bytearray()
|
|
95
|
+
etag = None
|
|
96
|
+
total = None
|
|
97
|
+
for retry_times in range(1, max_retry_times + 1):
|
|
98
|
+
try:
|
|
99
|
+
response = session.get(
|
|
100
|
+
url,
|
|
101
|
+
verify=True,
|
|
102
|
+
params=params,
|
|
103
|
+
timeout=timeout,
|
|
104
|
+
allow_redirects=False,
|
|
105
|
+
)
|
|
106
|
+
except RETRYABLE_ERRORS as exc:
|
|
107
|
+
if verbose:
|
|
108
|
+
logger.warning(
|
|
109
|
+
f"Error: {exc}, retry {retry_times}/{max_retry_times}"
|
|
110
|
+
)
|
|
111
|
+
time.sleep(retry_times * 0.5)
|
|
112
|
+
continue
|
|
113
|
+
if response.status_code == 200:
|
|
114
|
+
return response.content
|
|
115
|
+
if response.status_code == 504:
|
|
116
|
+
time.sleep(retry_times * 0.5)
|
|
117
|
+
continue
|
|
118
|
+
if response.status_code not in REDIRECT_STATUS:
|
|
119
|
+
raise Exception(
|
|
120
|
+
f"Final response status: {response.status_code}, text: {response.text}"
|
|
121
|
+
)
|
|
122
|
+
object_url = response.headers["Location"]
|
|
123
|
+
|
|
124
|
+
headers = {}
|
|
125
|
+
if buffer and etag:
|
|
126
|
+
headers = {"Range": f"bytes={len(buffer)}-", "If-Match": etag}
|
|
127
|
+
try:
|
|
128
|
+
# 下載網址本身帶簽章,不能帶 session 的 Authorization header
|
|
129
|
+
with requests.get(
|
|
130
|
+
object_url, headers=headers, stream=True, timeout=timeout
|
|
131
|
+
) as object_response:
|
|
132
|
+
status_code = object_response.status_code
|
|
133
|
+
if status_code == 200:
|
|
134
|
+
buffer = bytearray()
|
|
135
|
+
etag = object_response.headers.get("ETag")
|
|
136
|
+
content_length = object_response.headers.get(
|
|
137
|
+
"Content-Length"
|
|
138
|
+
)
|
|
139
|
+
total = int(content_length) if content_length else None
|
|
140
|
+
elif status_code == 412:
|
|
141
|
+
# 物件在兩次請求之間已更新,已收到的內容作廢
|
|
142
|
+
buffer, etag, total = bytearray(), None, None
|
|
143
|
+
continue
|
|
144
|
+
elif status_code != 206:
|
|
145
|
+
if verbose:
|
|
146
|
+
logger.warning(
|
|
147
|
+
f"status_code={status_code}, retry {retry_times}/{max_retry_times}"
|
|
148
|
+
)
|
|
149
|
+
time.sleep(retry_times * 0.5)
|
|
150
|
+
continue
|
|
151
|
+
for chunk in object_response.iter_content(chunk_size=1 << 20):
|
|
152
|
+
buffer.extend(chunk)
|
|
153
|
+
except RETRYABLE_ERRORS as exc:
|
|
154
|
+
if verbose:
|
|
155
|
+
logger.warning(
|
|
156
|
+
f"Download interrupted at {len(buffer)} bytes: {exc}, "
|
|
157
|
+
f"retry {retry_times}/{max_retry_times}"
|
|
158
|
+
)
|
|
159
|
+
time.sleep(retry_times * 0.5)
|
|
160
|
+
continue
|
|
161
|
+
|
|
162
|
+
if total is not None and len(buffer) < total:
|
|
163
|
+
continue
|
|
164
|
+
too_long = total is not None and len(buffer) > total
|
|
165
|
+
if too_long or not _match_etag(buffer, etag):
|
|
166
|
+
if verbose:
|
|
167
|
+
logger.warning(
|
|
168
|
+
f"Downloaded object failed verification, retry {retry_times}/{max_retry_times}"
|
|
169
|
+
)
|
|
170
|
+
buffer, etag, total = bytearray(), None, None
|
|
171
|
+
continue
|
|
172
|
+
return bytes(buffer)
|
|
173
|
+
raise Exception(
|
|
174
|
+
f"Download storage object failed after {max_retry_times} retries: "
|
|
175
|
+
f"received {len(buffer)} of {total} bytes"
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _match_etag(content: bytearray, etag: str) -> bool:
|
|
180
|
+
"""ETag 不含 "-" 時等於整個物件的 MD5;分段上傳的 ETag 無法以 MD5 驗證"""
|
|
181
|
+
if not etag:
|
|
182
|
+
return True
|
|
183
|
+
etag = etag.strip('"')
|
|
184
|
+
if "-" in etag:
|
|
185
|
+
return True
|
|
186
|
+
return hashlib.md5(content).hexdigest() == etag
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
async def _loop_run_get(
|
|
190
|
+
executor: concurrent.futures.ThreadPoolExecutor,
|
|
191
|
+
loop: asyncio.AbstractEventLoop,
|
|
192
|
+
session: requests.Session,
|
|
193
|
+
url: str,
|
|
194
|
+
params: Dict[str, Union[str, int, float]],
|
|
195
|
+
timeout: int = 60,
|
|
196
|
+
max_retry_times: int = 10,
|
|
197
|
+
verbose: bool = False,
|
|
198
|
+
):
|
|
199
|
+
"""
|
|
200
|
+
將同步 request 包成 async
|
|
201
|
+
"""
|
|
202
|
+
return await loop.run_in_executor(
|
|
203
|
+
executor,
|
|
204
|
+
request_get,
|
|
205
|
+
session,
|
|
206
|
+
url,
|
|
207
|
+
params,
|
|
208
|
+
timeout,
|
|
209
|
+
max_retry_times,
|
|
210
|
+
verbose,
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def async_request_get(
|
|
215
|
+
session: requests.Session,
|
|
216
|
+
url: str,
|
|
217
|
+
params_list: List[Dict[str, Union[str, int, float]]],
|
|
218
|
+
timeout: int = 60,
|
|
219
|
+
max_retry_times: int = 10,
|
|
220
|
+
verbose: bool = False,
|
|
221
|
+
auto_tune: bool = True,
|
|
222
|
+
max_concurrency: int = None,
|
|
223
|
+
batch_size: int = 10,
|
|
224
|
+
):
|
|
225
|
+
"""
|
|
226
|
+
批量 async request,支援自動根據機器資源調整 max_concurrency / batch_size
|
|
227
|
+
"""
|
|
228
|
+
# 自動調整
|
|
229
|
+
if auto_tune:
|
|
230
|
+
cpu_count = os.cpu_count() or 1
|
|
231
|
+
max_concurrency = max_concurrency or max(1, cpu_count * 2)
|
|
232
|
+
else:
|
|
233
|
+
max_concurrency = max_concurrency or 10
|
|
234
|
+
|
|
235
|
+
async def runner():
|
|
236
|
+
semaphore = asyncio.Semaphore(max_concurrency)
|
|
237
|
+
loop = asyncio.get_event_loop()
|
|
238
|
+
executor = concurrent.futures.ThreadPoolExecutor(
|
|
239
|
+
max_workers=max_concurrency
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
async def _limited_run(params):
|
|
243
|
+
async with semaphore:
|
|
244
|
+
resp = await _loop_run_get(
|
|
245
|
+
executor,
|
|
246
|
+
loop,
|
|
247
|
+
session,
|
|
248
|
+
url=url,
|
|
249
|
+
params=params,
|
|
250
|
+
timeout=timeout,
|
|
251
|
+
max_retry_times=max_retry_times,
|
|
252
|
+
verbose=verbose,
|
|
253
|
+
)
|
|
254
|
+
return resp
|
|
255
|
+
|
|
256
|
+
results = []
|
|
257
|
+
pbar = tqdm(total=len(params_list))
|
|
258
|
+
|
|
259
|
+
# 分 batch 建立 task
|
|
260
|
+
for i in range(0, len(params_list), batch_size):
|
|
261
|
+
batch = params_list[i : i + batch_size]
|
|
262
|
+
tasks = [asyncio.create_task(_limited_run(p)) for p in batch]
|
|
263
|
+
|
|
264
|
+
for coro in asyncio.as_completed(tasks):
|
|
265
|
+
try:
|
|
266
|
+
res = await coro
|
|
267
|
+
results.append(res)
|
|
268
|
+
except Exception as exc:
|
|
269
|
+
if verbose:
|
|
270
|
+
logger.error(f"Task failed: {exc}")
|
|
271
|
+
pbar.update(1)
|
|
272
|
+
|
|
273
|
+
pbar.close()
|
|
274
|
+
executor.shutdown()
|
|
275
|
+
return results
|
|
276
|
+
|
|
277
|
+
return asyncio.run(runner())
|
|
@@ -1,156 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import concurrent.futures
|
|
3
|
-
import os
|
|
4
|
-
import ssl
|
|
5
|
-
import time
|
|
6
|
-
from typing import Dict, List, Union
|
|
7
|
-
|
|
8
|
-
import nest_asyncio
|
|
9
|
-
import requests
|
|
10
|
-
from loguru import logger
|
|
11
|
-
from tqdm import tqdm
|
|
12
|
-
|
|
13
|
-
nest_asyncio.apply()
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def request_get(
|
|
17
|
-
session: requests.Session,
|
|
18
|
-
url: str,
|
|
19
|
-
params: Dict[str, Union[int, str, float]] = None,
|
|
20
|
-
timeout: int = 60,
|
|
21
|
-
max_retry_times: int = 10,
|
|
22
|
-
verbose: bool = False,
|
|
23
|
-
):
|
|
24
|
-
"""
|
|
25
|
-
單次 request,支援 retry 與 log
|
|
26
|
-
"""
|
|
27
|
-
response = None
|
|
28
|
-
for retry_times in range(1, max_retry_times + 1):
|
|
29
|
-
try:
|
|
30
|
-
response = session.get(
|
|
31
|
-
url,
|
|
32
|
-
verify=True,
|
|
33
|
-
params=params,
|
|
34
|
-
timeout=timeout,
|
|
35
|
-
)
|
|
36
|
-
if response.status_code == 504:
|
|
37
|
-
if verbose:
|
|
38
|
-
logger.warning(
|
|
39
|
-
f"status_code=504, retry {retry_times}/{max_retry_times}"
|
|
40
|
-
)
|
|
41
|
-
time.sleep(retry_times * 0.1)
|
|
42
|
-
else:
|
|
43
|
-
break
|
|
44
|
-
except (
|
|
45
|
-
requests.ConnectionError,
|
|
46
|
-
ssl.SSLError,
|
|
47
|
-
requests.exceptions.ReadTimeout,
|
|
48
|
-
requests.exceptions.ChunkedEncodingError,
|
|
49
|
-
) as exc:
|
|
50
|
-
if verbose:
|
|
51
|
-
logger.warning(
|
|
52
|
-
f"Error: {exc}, retry {retry_times}/{max_retry_times}"
|
|
53
|
-
)
|
|
54
|
-
time.sleep(retry_times * 0.1)
|
|
55
|
-
except Exception as exc:
|
|
56
|
-
if verbose:
|
|
57
|
-
logger.warning(
|
|
58
|
-
f"Unexpected error: {exc}, retry {retry_times}/{max_retry_times}"
|
|
59
|
-
)
|
|
60
|
-
time.sleep(retry_times * 0.1)
|
|
61
|
-
if response and response.status_code != 200:
|
|
62
|
-
raise Exception(
|
|
63
|
-
f"Final response status: {response.status_code}, text: {response.text}"
|
|
64
|
-
)
|
|
65
|
-
return response
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
async def _loop_run_get(
|
|
69
|
-
executor: concurrent.futures.ThreadPoolExecutor,
|
|
70
|
-
loop: asyncio.AbstractEventLoop,
|
|
71
|
-
session: requests.Session,
|
|
72
|
-
url: str,
|
|
73
|
-
params: Dict[str, Union[str, int, float]],
|
|
74
|
-
timeout: int = 60,
|
|
75
|
-
max_retry_times: int = 10,
|
|
76
|
-
verbose: bool = False,
|
|
77
|
-
):
|
|
78
|
-
"""
|
|
79
|
-
將同步 request 包成 async
|
|
80
|
-
"""
|
|
81
|
-
return await loop.run_in_executor(
|
|
82
|
-
executor,
|
|
83
|
-
request_get,
|
|
84
|
-
session,
|
|
85
|
-
url,
|
|
86
|
-
params,
|
|
87
|
-
timeout,
|
|
88
|
-
max_retry_times,
|
|
89
|
-
verbose,
|
|
90
|
-
)
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def async_request_get(
|
|
94
|
-
session: requests.Session,
|
|
95
|
-
url: str,
|
|
96
|
-
params_list: List[Dict[str, Union[str, int, float]]],
|
|
97
|
-
timeout: int = 60,
|
|
98
|
-
max_retry_times: int = 10,
|
|
99
|
-
verbose: bool = False,
|
|
100
|
-
auto_tune: bool = True,
|
|
101
|
-
max_concurrency: int = None,
|
|
102
|
-
batch_size: int = 10,
|
|
103
|
-
):
|
|
104
|
-
"""
|
|
105
|
-
批量 async request,支援自動根據機器資源調整 max_concurrency / batch_size
|
|
106
|
-
"""
|
|
107
|
-
# 自動調整
|
|
108
|
-
if auto_tune:
|
|
109
|
-
cpu_count = os.cpu_count() or 1
|
|
110
|
-
max_concurrency = max_concurrency or max(1, cpu_count * 2)
|
|
111
|
-
else:
|
|
112
|
-
max_concurrency = max_concurrency or 10
|
|
113
|
-
|
|
114
|
-
async def runner():
|
|
115
|
-
semaphore = asyncio.Semaphore(max_concurrency)
|
|
116
|
-
loop = asyncio.get_event_loop()
|
|
117
|
-
executor = concurrent.futures.ThreadPoolExecutor(
|
|
118
|
-
max_workers=max_concurrency
|
|
119
|
-
)
|
|
120
|
-
|
|
121
|
-
async def _limited_run(params):
|
|
122
|
-
async with semaphore:
|
|
123
|
-
resp = await _loop_run_get(
|
|
124
|
-
executor,
|
|
125
|
-
loop,
|
|
126
|
-
session,
|
|
127
|
-
url=url,
|
|
128
|
-
params=params,
|
|
129
|
-
timeout=timeout,
|
|
130
|
-
max_retry_times=max_retry_times,
|
|
131
|
-
verbose=verbose,
|
|
132
|
-
)
|
|
133
|
-
return resp
|
|
134
|
-
|
|
135
|
-
results = []
|
|
136
|
-
pbar = tqdm(total=len(params_list))
|
|
137
|
-
|
|
138
|
-
# 分 batch 建立 task
|
|
139
|
-
for i in range(0, len(params_list), batch_size):
|
|
140
|
-
batch = params_list[i : i + batch_size]
|
|
141
|
-
tasks = [asyncio.create_task(_limited_run(p)) for p in batch]
|
|
142
|
-
|
|
143
|
-
for coro in asyncio.as_completed(tasks):
|
|
144
|
-
try:
|
|
145
|
-
res = await coro
|
|
146
|
-
results.append(res)
|
|
147
|
-
except Exception as exc:
|
|
148
|
-
if verbose:
|
|
149
|
-
logger.error(f"Task failed: {exc}")
|
|
150
|
-
pbar.update(1)
|
|
151
|
-
|
|
152
|
-
pbar.close()
|
|
153
|
-
executor.shutdown()
|
|
154
|
-
return results
|
|
155
|
-
|
|
156
|
-
return asyncio.run(runner())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|