FinMind 2.0.9__tar.gz → 2.0.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/data/data_loader.py +103 -3
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/data/finmind_api.py +17 -10
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/data.py +2 -0
- finmind-2.0.11/FinMind/utility/request.py +277 -0
- {finmind-2.0.9/finmind.egg-info → finmind-2.0.11}/PKG-INFO +1 -1
- {finmind-2.0.9 → finmind-2.0.11/finmind.egg-info}/PKG-INFO +1 -1
- finmind-2.0.9/FinMind/utility/request.py +0 -156
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/_version.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/crawler/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/crawler/base.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/crawler/commodities.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/crawler/demo.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/crawler/government_bonds.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/data/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/data/data_subscriber.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/bias.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/continue_holding.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/institutional_investors_follower.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/institutional_investors_over_buy.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/kd.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/kd_crossover.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/ma_cross_orver.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/short_sale_margin_purchase_ratio.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/plotting/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/plotting/bar.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/plotting/kline.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/plotting/line.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/plotting/pie.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/indicators.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/info.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/plot.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/rule.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/base.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/bias.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/continue_holding.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/institutional_investors_follower.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/kd.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/kd_crossover.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/ma_crossover.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/macd_crossover.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/max_min_period_bias.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/naive_kd.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/short_sale_margin_purchase_ratio.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/utils.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/templates/post.html +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/utility/__init__.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/FinMind/utility/rule.py +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/LICENSE +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/README.md +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/finmind.egg-info/SOURCES.txt +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/finmind.egg-info/dependency_links.txt +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/finmind.egg-info/requires.txt +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/finmind.egg-info/top_level.txt +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/pyproject.toml +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/setup.cfg +0 -0
- {finmind-2.0.9 → finmind-2.0.11}/setup.py +0 -0
|
@@ -191,9 +191,9 @@ class DataLoader(FinMindApi):
|
|
|
191
191
|
date=date,
|
|
192
192
|
timeout=timeout,
|
|
193
193
|
)
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
194
|
+
self._require_data_id(
|
|
195
|
+
Dataset.TaiwanStockPriceTick, "stock_id", stock_id, stock_id_list
|
|
196
|
+
)
|
|
197
197
|
stock_tick = self.get_data(
|
|
198
198
|
dataset=Dataset.TaiwanStockPriceTick,
|
|
199
199
|
data_id=stock_id,
|
|
@@ -1217,6 +1217,50 @@ class DataLoader(FinMindApi):
|
|
|
1217
1217
|
)
|
|
1218
1218
|
return futures_daily
|
|
1219
1219
|
|
|
1220
|
+
def taiwan_futures_kbar(
|
|
1221
|
+
self,
|
|
1222
|
+
futures_id: str = "",
|
|
1223
|
+
date: str = "",
|
|
1224
|
+
timeout: int = None,
|
|
1225
|
+
use_object: bool = False,
|
|
1226
|
+
) -> pd.DataFrame:
|
|
1227
|
+
"""get 期貨分K資料
|
|
1228
|
+
:param futures_id: 期貨代號("TX")
|
|
1229
|
+
:param date (str): 日期("2024-01-02")
|
|
1230
|
+
:param timeout (int): timeout seconds, default None
|
|
1231
|
+
:param use_object (bool): 是否透過 signed URL 下載整日 parquet 資料物件,
|
|
1232
|
+
設為 True 時忽略 futures_id 參數, default False
|
|
1233
|
+
|
|
1234
|
+
:return: 期貨分K TaiwanFuturesKBar
|
|
1235
|
+
:rtype pd.DataFrame
|
|
1236
|
+
:rtype column date (str): 日期
|
|
1237
|
+
:rtype column futures_id (str): 期貨代碼
|
|
1238
|
+
:rtype column contract_date (str): 到期月份
|
|
1239
|
+
:rtype column minute (str): 分鐘時間
|
|
1240
|
+
:rtype column open (float): 開盤價
|
|
1241
|
+
:rtype column high (float): 最高價
|
|
1242
|
+
:rtype column low (float): 最低價
|
|
1243
|
+
:rtype column close (float): 收盤價
|
|
1244
|
+
:rtype column volume (int): 成交量
|
|
1245
|
+
"""
|
|
1246
|
+
if use_object:
|
|
1247
|
+
return self.get_object(
|
|
1248
|
+
dataset=Dataset.TaiwanFuturesKBar,
|
|
1249
|
+
date=date,
|
|
1250
|
+
timeout=timeout,
|
|
1251
|
+
)
|
|
1252
|
+
self._require_data_id(
|
|
1253
|
+
Dataset.TaiwanFuturesKBar, "futures_id", futures_id
|
|
1254
|
+
)
|
|
1255
|
+
futures_kbar = self.get_data(
|
|
1256
|
+
dataset=Dataset.TaiwanFuturesKBar,
|
|
1257
|
+
data_id=futures_id,
|
|
1258
|
+
start_date=date,
|
|
1259
|
+
end_date=date,
|
|
1260
|
+
timeout=timeout,
|
|
1261
|
+
)
|
|
1262
|
+
return futures_kbar
|
|
1263
|
+
|
|
1220
1264
|
def taiwan_option_daily(
|
|
1221
1265
|
self,
|
|
1222
1266
|
option_id: str = "",
|
|
@@ -1387,6 +1431,9 @@ class DataLoader(FinMindApi):
|
|
|
1387
1431
|
date=date,
|
|
1388
1432
|
timeout=timeout,
|
|
1389
1433
|
)
|
|
1434
|
+
self._require_data_id(
|
|
1435
|
+
Dataset.TaiwanFuturesTick, "futures_id", futures_id
|
|
1436
|
+
)
|
|
1390
1437
|
futures_tick = self.get_data(
|
|
1391
1438
|
dataset=Dataset.TaiwanFuturesTick,
|
|
1392
1439
|
data_id=futures_id,
|
|
@@ -1458,6 +1505,9 @@ class DataLoader(FinMindApi):
|
|
|
1458
1505
|
date=date,
|
|
1459
1506
|
timeout=timeout,
|
|
1460
1507
|
)
|
|
1508
|
+
self._require_data_id(
|
|
1509
|
+
Dataset.TaiwanOptionTick, "option_id", option_id, option_id_list
|
|
1510
|
+
)
|
|
1461
1511
|
option_tick = self.get_data(
|
|
1462
1512
|
dataset=Dataset.TaiwanOptionTick,
|
|
1463
1513
|
data_id=option_id,
|
|
@@ -2088,6 +2138,7 @@ class DataLoader(FinMindApi):
|
|
|
2088
2138
|
use_object: bool = False,
|
|
2089
2139
|
) -> pd.DataFrame:
|
|
2090
2140
|
"""get 台股分 K 資料表
|
|
2141
|
+
:param stock_id (str): 股票代碼, 加權指數請帶 "TAIEX"
|
|
2091
2142
|
:param timeout (int): timeout seconds, default None
|
|
2092
2143
|
:param use_object (bool): 是否透過 signed URL 下載整日 parquet 資料物件,
|
|
2093
2144
|
設為 True 時忽略 stock_id, stock_id_list, use_async 參數, default False
|
|
@@ -2109,6 +2160,9 @@ class DataLoader(FinMindApi):
|
|
|
2109
2160
|
date=date,
|
|
2110
2161
|
timeout=timeout,
|
|
2111
2162
|
)
|
|
2163
|
+
self._require_data_id(
|
|
2164
|
+
Dataset.TaiwanStockKBar, "stock_id", stock_id, stock_id_list
|
|
2165
|
+
)
|
|
2112
2166
|
taiwan_stock_bar = self.get_data(
|
|
2113
2167
|
dataset=Dataset.TaiwanStockKBar,
|
|
2114
2168
|
data_id=stock_id,
|
|
@@ -3093,6 +3147,35 @@ class DataLoader(FinMindApi):
|
|
|
3093
3147
|
)
|
|
3094
3148
|
return taiwan_stock_day_trading_borrowing_fee_rate
|
|
3095
3149
|
|
|
3150
|
+
def taiwan_stock_broker_daily_concentration(
|
|
3151
|
+
self,
|
|
3152
|
+
stock_id: str = "",
|
|
3153
|
+
start_date: str = "",
|
|
3154
|
+
end_date: str = "",
|
|
3155
|
+
timeout: int = None,
|
|
3156
|
+
) -> pd.DataFrame:
|
|
3157
|
+
"""get 每日個股主力集中度
|
|
3158
|
+
:param stock_id (str): 股票代號("2330")
|
|
3159
|
+
:param start_date (str): 開始日期("2024-01-01")
|
|
3160
|
+
:param end_date (str): 結束日期("2024-01-31")
|
|
3161
|
+
:param timeout (int): timeout seconds, default None
|
|
3162
|
+
|
|
3163
|
+
:return: 每日個股主力集中度 TaiwanStockBrokerDailyConcentration
|
|
3164
|
+
:rtype pd.DataFrame
|
|
3165
|
+
:rtype column date (str): 日期
|
|
3166
|
+
:rtype column stock_id (str): 股票代號
|
|
3167
|
+
:rtype column top_k (int): 前 K 大(目前固定為 15)
|
|
3168
|
+
:rtype column top_buy_volume (int): 前 top_k 大買超券商買超股數總和(股)
|
|
3169
|
+
:rtype column top_sell_volume (int): 前 top_k 大賣超券商賣超股數總和(股,正值)
|
|
3170
|
+
"""
|
|
3171
|
+
return self.get_data(
|
|
3172
|
+
dataset=Dataset.TaiwanStockBrokerDailyConcentration,
|
|
3173
|
+
data_id=stock_id,
|
|
3174
|
+
start_date=start_date,
|
|
3175
|
+
end_date=end_date,
|
|
3176
|
+
timeout=timeout,
|
|
3177
|
+
)
|
|
3178
|
+
|
|
3096
3179
|
def taiwan_stock_convertible_bond_monthly_analysis(
|
|
3097
3180
|
self,
|
|
3098
3181
|
cb_id: str = "",
|
|
@@ -3261,6 +3344,23 @@ class DataLoader(FinMindApi):
|
|
|
3261
3344
|
)
|
|
3262
3345
|
return data
|
|
3263
3346
|
|
|
3347
|
+
@staticmethod
|
|
3348
|
+
def _require_data_id(
|
|
3349
|
+
dataset: str,
|
|
3350
|
+
id_name: str,
|
|
3351
|
+
data_id: str,
|
|
3352
|
+
data_id_list: typing.List[str] = None,
|
|
3353
|
+
) -> None:
|
|
3354
|
+
"""逐筆 / 分K 資料必須指定代號, 不支援不帶代號一次查詢所有商品
|
|
3355
|
+
一次取得特定日期所有資料, 請用 use_object=True (限 sponsorpro)
|
|
3356
|
+
"""
|
|
3357
|
+
if not data_id and not data_id_list:
|
|
3358
|
+
dataset = getattr(dataset, "value", dataset)
|
|
3359
|
+
raise ValueError(
|
|
3360
|
+
f"{dataset} 必須指定 {id_name}; "
|
|
3361
|
+
f"一次取得特定日期所有資料, 請使用 use_object=True (限 sponsorpro)"
|
|
3362
|
+
)
|
|
3363
|
+
|
|
3264
3364
|
def _get_stock_id_list(
|
|
3265
3365
|
self, date: str, timeout: int = None
|
|
3266
3366
|
) -> typing.List[str]:
|
|
@@ -7,7 +7,11 @@ import requests
|
|
|
7
7
|
from loguru import logger
|
|
8
8
|
|
|
9
9
|
from FinMind.schema.data import Dataset
|
|
10
|
-
from FinMind.utility.request import
|
|
10
|
+
from FinMind.utility.request import (
|
|
11
|
+
async_request_get,
|
|
12
|
+
download_storage_object,
|
|
13
|
+
request_get,
|
|
14
|
+
)
|
|
11
15
|
|
|
12
16
|
logger.remove()
|
|
13
17
|
logger.add(sys.stderr, level="INFO")
|
|
@@ -398,11 +402,19 @@ class FinMindApi:
|
|
|
398
402
|
data = pd.DataFrame(self._extract_data(data))
|
|
399
403
|
return data
|
|
400
404
|
|
|
401
|
-
def get_object(
|
|
405
|
+
def get_object(
|
|
406
|
+
self,
|
|
407
|
+
dataset: str,
|
|
408
|
+
date: str,
|
|
409
|
+
timeout: int,
|
|
410
|
+
max_retry_times: int = 10,
|
|
411
|
+
) -> pd.DataFrame:
|
|
402
412
|
"""透過 signed URL 下載整日資料物件 (parquet 格式)
|
|
413
|
+
傳輸中斷時會自動重試,並以 HTTP Range 從中斷處續傳
|
|
403
414
|
:param dataset (str): 資料集名稱 (e.g. "TaiwanStockPriceTick")
|
|
404
415
|
:param date (str): 資料日期 ("2025-01-06")
|
|
405
416
|
:param timeout (int): request timeout 秒數
|
|
417
|
+
:param max_retry_times (int): 最多重試次數
|
|
406
418
|
|
|
407
419
|
:return: 該日期所有資料
|
|
408
420
|
:rtype: pd.DataFrame
|
|
@@ -412,16 +424,11 @@ class FinMindApi:
|
|
|
412
424
|
"dataset": dataset,
|
|
413
425
|
"date": date,
|
|
414
426
|
}
|
|
415
|
-
|
|
427
|
+
content = download_storage_object(
|
|
416
428
|
self.__session,
|
|
417
429
|
url,
|
|
418
430
|
params=params,
|
|
419
431
|
timeout=timeout,
|
|
432
|
+
max_retry_times=max_retry_times,
|
|
420
433
|
)
|
|
421
|
-
|
|
422
|
-
if status_code == 200:
|
|
423
|
-
return pd.read_parquet(io.BytesIO(response.content))
|
|
424
|
-
else:
|
|
425
|
-
resp_json = response.json()
|
|
426
|
-
logger.info(resp_json)
|
|
427
|
-
return pd.DataFrame()
|
|
434
|
+
return pd.read_parquet(io.BytesIO(content))
|
|
@@ -30,6 +30,7 @@ class Dataset(str, Enum):
|
|
|
30
30
|
TaiwanOptionTick = "TaiwanOptionTick"
|
|
31
31
|
TaiwanStockPriceBidAsk = "TaiwanStockPriceBidAsk"
|
|
32
32
|
TaiwanFuturesDaily = "TaiwanFuturesDaily"
|
|
33
|
+
TaiwanFuturesKBar = "TaiwanFuturesKBar"
|
|
33
34
|
TaiwanOptionDaily = "TaiwanOptionDaily"
|
|
34
35
|
TaiwanStockNews = "TaiwanStockNews"
|
|
35
36
|
TaiwanStockTotalReturnIndex = "TaiwanStockTotalReturnIndex"
|
|
@@ -151,6 +152,7 @@ class Dataset(str, Enum):
|
|
|
151
152
|
TaiwanAssetSwapFixedIncomeDaily = "TaiwanAssetSwapFixedIncomeDaily"
|
|
152
153
|
TaiwanAssetSwapOptionDaily = "TaiwanAssetSwapOptionDaily"
|
|
153
154
|
TaiwanStockMarginMaintenance = "TaiwanStockMarginMaintenance"
|
|
155
|
+
TaiwanStockBrokerDailyConcentration = "TaiwanStockBrokerDailyConcentration"
|
|
154
156
|
|
|
155
157
|
|
|
156
158
|
class Version(str, Enum):
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import concurrent.futures
|
|
3
|
+
import hashlib
|
|
4
|
+
import os
|
|
5
|
+
import ssl
|
|
6
|
+
import time
|
|
7
|
+
from typing import Dict, List, Union
|
|
8
|
+
|
|
9
|
+
import nest_asyncio
|
|
10
|
+
import requests
|
|
11
|
+
from loguru import logger
|
|
12
|
+
from tqdm import tqdm
|
|
13
|
+
|
|
14
|
+
nest_asyncio.apply()
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def request_get(
|
|
18
|
+
session: requests.Session,
|
|
19
|
+
url: str,
|
|
20
|
+
params: Dict[str, Union[int, str, float]] = None,
|
|
21
|
+
timeout: int = 60,
|
|
22
|
+
max_retry_times: int = 10,
|
|
23
|
+
verbose: bool = False,
|
|
24
|
+
):
|
|
25
|
+
"""
|
|
26
|
+
單次 request,支援 retry 與 log
|
|
27
|
+
"""
|
|
28
|
+
response = None
|
|
29
|
+
for retry_times in range(1, max_retry_times + 1):
|
|
30
|
+
try:
|
|
31
|
+
response = session.get(
|
|
32
|
+
url,
|
|
33
|
+
verify=True,
|
|
34
|
+
params=params,
|
|
35
|
+
timeout=timeout,
|
|
36
|
+
)
|
|
37
|
+
if response.status_code == 504:
|
|
38
|
+
if verbose:
|
|
39
|
+
logger.warning(
|
|
40
|
+
f"status_code=504, retry {retry_times}/{max_retry_times}"
|
|
41
|
+
)
|
|
42
|
+
time.sleep(retry_times * 0.1)
|
|
43
|
+
else:
|
|
44
|
+
break
|
|
45
|
+
except (
|
|
46
|
+
requests.ConnectionError,
|
|
47
|
+
ssl.SSLError,
|
|
48
|
+
requests.exceptions.ReadTimeout,
|
|
49
|
+
requests.exceptions.ChunkedEncodingError,
|
|
50
|
+
) as exc:
|
|
51
|
+
if verbose:
|
|
52
|
+
logger.warning(
|
|
53
|
+
f"Error: {exc}, retry {retry_times}/{max_retry_times}"
|
|
54
|
+
)
|
|
55
|
+
time.sleep(retry_times * 0.1)
|
|
56
|
+
except Exception as exc:
|
|
57
|
+
if verbose:
|
|
58
|
+
logger.warning(
|
|
59
|
+
f"Unexpected error: {exc}, retry {retry_times}/{max_retry_times}"
|
|
60
|
+
)
|
|
61
|
+
time.sleep(retry_times * 0.1)
|
|
62
|
+
if response and response.status_code != 200:
|
|
63
|
+
raise Exception(
|
|
64
|
+
f"Final response status: {response.status_code}, text: {response.text}"
|
|
65
|
+
)
|
|
66
|
+
return response
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
RETRYABLE_ERRORS = (
|
|
70
|
+
requests.ConnectionError,
|
|
71
|
+
ssl.SSLError,
|
|
72
|
+
requests.exceptions.ReadTimeout,
|
|
73
|
+
requests.exceptions.ChunkedEncodingError,
|
|
74
|
+
)
|
|
75
|
+
REDIRECT_STATUS = (301, 302, 303, 307, 308)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def download_storage_object(
|
|
79
|
+
session: requests.Session,
|
|
80
|
+
url: str,
|
|
81
|
+
params: Dict[str, Union[int, str, float]] = None,
|
|
82
|
+
timeout: int = 60,
|
|
83
|
+
max_retry_times: int = 10,
|
|
84
|
+
verbose: bool = False,
|
|
85
|
+
) -> bytes:
|
|
86
|
+
"""
|
|
87
|
+
下載整日資料物件,傳輸中斷時以 Range + If-Match 續傳
|
|
88
|
+
|
|
89
|
+
storage_objects 回 307 導向有效期限約 3 分鐘的下載網址,
|
|
90
|
+
所以每次(含重試)都重新取得網址;已收到的內容保留,
|
|
91
|
+
只要求剩下的位元組,並以 If-Match 確認仍是同一版本,
|
|
92
|
+
版本不同(412)才從頭下載。完成後檢查長度,ETag 不含 "-" 時再比對 MD5。
|
|
93
|
+
"""
|
|
94
|
+
buffer = bytearray()
|
|
95
|
+
etag = None
|
|
96
|
+
total = None
|
|
97
|
+
for retry_times in range(1, max_retry_times + 1):
|
|
98
|
+
try:
|
|
99
|
+
response = session.get(
|
|
100
|
+
url,
|
|
101
|
+
verify=True,
|
|
102
|
+
params=params,
|
|
103
|
+
timeout=timeout,
|
|
104
|
+
allow_redirects=False,
|
|
105
|
+
)
|
|
106
|
+
except RETRYABLE_ERRORS as exc:
|
|
107
|
+
if verbose:
|
|
108
|
+
logger.warning(
|
|
109
|
+
f"Error: {exc}, retry {retry_times}/{max_retry_times}"
|
|
110
|
+
)
|
|
111
|
+
time.sleep(retry_times * 0.5)
|
|
112
|
+
continue
|
|
113
|
+
if response.status_code == 200:
|
|
114
|
+
return response.content
|
|
115
|
+
if response.status_code == 504:
|
|
116
|
+
time.sleep(retry_times * 0.5)
|
|
117
|
+
continue
|
|
118
|
+
if response.status_code not in REDIRECT_STATUS:
|
|
119
|
+
raise Exception(
|
|
120
|
+
f"Final response status: {response.status_code}, text: {response.text}"
|
|
121
|
+
)
|
|
122
|
+
object_url = response.headers["Location"]
|
|
123
|
+
|
|
124
|
+
headers = {}
|
|
125
|
+
if buffer and etag:
|
|
126
|
+
headers = {"Range": f"bytes={len(buffer)}-", "If-Match": etag}
|
|
127
|
+
try:
|
|
128
|
+
# 下載網址本身帶簽章,不能帶 session 的 Authorization header
|
|
129
|
+
with requests.get(
|
|
130
|
+
object_url, headers=headers, stream=True, timeout=timeout
|
|
131
|
+
) as object_response:
|
|
132
|
+
status_code = object_response.status_code
|
|
133
|
+
if status_code == 200:
|
|
134
|
+
buffer = bytearray()
|
|
135
|
+
etag = object_response.headers.get("ETag")
|
|
136
|
+
content_length = object_response.headers.get(
|
|
137
|
+
"Content-Length"
|
|
138
|
+
)
|
|
139
|
+
total = int(content_length) if content_length else None
|
|
140
|
+
elif status_code == 412:
|
|
141
|
+
# 物件在兩次請求之間已更新,已收到的內容作廢
|
|
142
|
+
buffer, etag, total = bytearray(), None, None
|
|
143
|
+
continue
|
|
144
|
+
elif status_code != 206:
|
|
145
|
+
if verbose:
|
|
146
|
+
logger.warning(
|
|
147
|
+
f"status_code={status_code}, retry {retry_times}/{max_retry_times}"
|
|
148
|
+
)
|
|
149
|
+
time.sleep(retry_times * 0.5)
|
|
150
|
+
continue
|
|
151
|
+
for chunk in object_response.iter_content(chunk_size=1 << 20):
|
|
152
|
+
buffer.extend(chunk)
|
|
153
|
+
except RETRYABLE_ERRORS as exc:
|
|
154
|
+
if verbose:
|
|
155
|
+
logger.warning(
|
|
156
|
+
f"Download interrupted at {len(buffer)} bytes: {exc}, "
|
|
157
|
+
f"retry {retry_times}/{max_retry_times}"
|
|
158
|
+
)
|
|
159
|
+
time.sleep(retry_times * 0.5)
|
|
160
|
+
continue
|
|
161
|
+
|
|
162
|
+
if total is not None and len(buffer) < total:
|
|
163
|
+
continue
|
|
164
|
+
too_long = total is not None and len(buffer) > total
|
|
165
|
+
if too_long or not _match_etag(buffer, etag):
|
|
166
|
+
if verbose:
|
|
167
|
+
logger.warning(
|
|
168
|
+
f"Downloaded object failed verification, retry {retry_times}/{max_retry_times}"
|
|
169
|
+
)
|
|
170
|
+
buffer, etag, total = bytearray(), None, None
|
|
171
|
+
continue
|
|
172
|
+
return bytes(buffer)
|
|
173
|
+
raise Exception(
|
|
174
|
+
f"Download storage object failed after {max_retry_times} retries: "
|
|
175
|
+
f"received {len(buffer)} of {total} bytes"
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _match_etag(content: bytearray, etag: str) -> bool:
|
|
180
|
+
"""ETag 不含 "-" 時等於整個物件的 MD5;分段上傳的 ETag 無法以 MD5 驗證"""
|
|
181
|
+
if not etag:
|
|
182
|
+
return True
|
|
183
|
+
etag = etag.strip('"')
|
|
184
|
+
if "-" in etag:
|
|
185
|
+
return True
|
|
186
|
+
return hashlib.md5(content).hexdigest() == etag
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
async def _loop_run_get(
|
|
190
|
+
executor: concurrent.futures.ThreadPoolExecutor,
|
|
191
|
+
loop: asyncio.AbstractEventLoop,
|
|
192
|
+
session: requests.Session,
|
|
193
|
+
url: str,
|
|
194
|
+
params: Dict[str, Union[str, int, float]],
|
|
195
|
+
timeout: int = 60,
|
|
196
|
+
max_retry_times: int = 10,
|
|
197
|
+
verbose: bool = False,
|
|
198
|
+
):
|
|
199
|
+
"""
|
|
200
|
+
將同步 request 包成 async
|
|
201
|
+
"""
|
|
202
|
+
return await loop.run_in_executor(
|
|
203
|
+
executor,
|
|
204
|
+
request_get,
|
|
205
|
+
session,
|
|
206
|
+
url,
|
|
207
|
+
params,
|
|
208
|
+
timeout,
|
|
209
|
+
max_retry_times,
|
|
210
|
+
verbose,
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def async_request_get(
|
|
215
|
+
session: requests.Session,
|
|
216
|
+
url: str,
|
|
217
|
+
params_list: List[Dict[str, Union[str, int, float]]],
|
|
218
|
+
timeout: int = 60,
|
|
219
|
+
max_retry_times: int = 10,
|
|
220
|
+
verbose: bool = False,
|
|
221
|
+
auto_tune: bool = True,
|
|
222
|
+
max_concurrency: int = None,
|
|
223
|
+
batch_size: int = 10,
|
|
224
|
+
):
|
|
225
|
+
"""
|
|
226
|
+
批量 async request,支援自動根據機器資源調整 max_concurrency / batch_size
|
|
227
|
+
"""
|
|
228
|
+
# 自動調整
|
|
229
|
+
if auto_tune:
|
|
230
|
+
cpu_count = os.cpu_count() or 1
|
|
231
|
+
max_concurrency = max_concurrency or max(1, cpu_count * 2)
|
|
232
|
+
else:
|
|
233
|
+
max_concurrency = max_concurrency or 10
|
|
234
|
+
|
|
235
|
+
async def runner():
|
|
236
|
+
semaphore = asyncio.Semaphore(max_concurrency)
|
|
237
|
+
loop = asyncio.get_event_loop()
|
|
238
|
+
executor = concurrent.futures.ThreadPoolExecutor(
|
|
239
|
+
max_workers=max_concurrency
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
async def _limited_run(params):
|
|
243
|
+
async with semaphore:
|
|
244
|
+
resp = await _loop_run_get(
|
|
245
|
+
executor,
|
|
246
|
+
loop,
|
|
247
|
+
session,
|
|
248
|
+
url=url,
|
|
249
|
+
params=params,
|
|
250
|
+
timeout=timeout,
|
|
251
|
+
max_retry_times=max_retry_times,
|
|
252
|
+
verbose=verbose,
|
|
253
|
+
)
|
|
254
|
+
return resp
|
|
255
|
+
|
|
256
|
+
results = []
|
|
257
|
+
pbar = tqdm(total=len(params_list))
|
|
258
|
+
|
|
259
|
+
# 分 batch 建立 task
|
|
260
|
+
for i in range(0, len(params_list), batch_size):
|
|
261
|
+
batch = params_list[i : i + batch_size]
|
|
262
|
+
tasks = [asyncio.create_task(_limited_run(p)) for p in batch]
|
|
263
|
+
|
|
264
|
+
for coro in asyncio.as_completed(tasks):
|
|
265
|
+
try:
|
|
266
|
+
res = await coro
|
|
267
|
+
results.append(res)
|
|
268
|
+
except Exception as exc:
|
|
269
|
+
if verbose:
|
|
270
|
+
logger.error(f"Task failed: {exc}")
|
|
271
|
+
pbar.update(1)
|
|
272
|
+
|
|
273
|
+
pbar.close()
|
|
274
|
+
executor.shutdown()
|
|
275
|
+
return results
|
|
276
|
+
|
|
277
|
+
return asyncio.run(runner())
|
|
@@ -1,156 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import concurrent.futures
|
|
3
|
-
import os
|
|
4
|
-
import ssl
|
|
5
|
-
import time
|
|
6
|
-
from typing import Dict, List, Union
|
|
7
|
-
|
|
8
|
-
import nest_asyncio
|
|
9
|
-
import requests
|
|
10
|
-
from loguru import logger
|
|
11
|
-
from tqdm import tqdm
|
|
12
|
-
|
|
13
|
-
nest_asyncio.apply()
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def request_get(
|
|
17
|
-
session: requests.Session,
|
|
18
|
-
url: str,
|
|
19
|
-
params: Dict[str, Union[int, str, float]] = None,
|
|
20
|
-
timeout: int = 60,
|
|
21
|
-
max_retry_times: int = 10,
|
|
22
|
-
verbose: bool = False,
|
|
23
|
-
):
|
|
24
|
-
"""
|
|
25
|
-
單次 request,支援 retry 與 log
|
|
26
|
-
"""
|
|
27
|
-
response = None
|
|
28
|
-
for retry_times in range(1, max_retry_times + 1):
|
|
29
|
-
try:
|
|
30
|
-
response = session.get(
|
|
31
|
-
url,
|
|
32
|
-
verify=True,
|
|
33
|
-
params=params,
|
|
34
|
-
timeout=timeout,
|
|
35
|
-
)
|
|
36
|
-
if response.status_code == 504:
|
|
37
|
-
if verbose:
|
|
38
|
-
logger.warning(
|
|
39
|
-
f"status_code=504, retry {retry_times}/{max_retry_times}"
|
|
40
|
-
)
|
|
41
|
-
time.sleep(retry_times * 0.1)
|
|
42
|
-
else:
|
|
43
|
-
break
|
|
44
|
-
except (
|
|
45
|
-
requests.ConnectionError,
|
|
46
|
-
ssl.SSLError,
|
|
47
|
-
requests.exceptions.ReadTimeout,
|
|
48
|
-
requests.exceptions.ChunkedEncodingError,
|
|
49
|
-
) as exc:
|
|
50
|
-
if verbose:
|
|
51
|
-
logger.warning(
|
|
52
|
-
f"Error: {exc}, retry {retry_times}/{max_retry_times}"
|
|
53
|
-
)
|
|
54
|
-
time.sleep(retry_times * 0.1)
|
|
55
|
-
except Exception as exc:
|
|
56
|
-
if verbose:
|
|
57
|
-
logger.warning(
|
|
58
|
-
f"Unexpected error: {exc}, retry {retry_times}/{max_retry_times}"
|
|
59
|
-
)
|
|
60
|
-
time.sleep(retry_times * 0.1)
|
|
61
|
-
if response and response.status_code != 200:
|
|
62
|
-
raise Exception(
|
|
63
|
-
f"Final response status: {response.status_code}, text: {response.text}"
|
|
64
|
-
)
|
|
65
|
-
return response
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
async def _loop_run_get(
|
|
69
|
-
executor: concurrent.futures.ThreadPoolExecutor,
|
|
70
|
-
loop: asyncio.AbstractEventLoop,
|
|
71
|
-
session: requests.Session,
|
|
72
|
-
url: str,
|
|
73
|
-
params: Dict[str, Union[str, int, float]],
|
|
74
|
-
timeout: int = 60,
|
|
75
|
-
max_retry_times: int = 10,
|
|
76
|
-
verbose: bool = False,
|
|
77
|
-
):
|
|
78
|
-
"""
|
|
79
|
-
將同步 request 包成 async
|
|
80
|
-
"""
|
|
81
|
-
return await loop.run_in_executor(
|
|
82
|
-
executor,
|
|
83
|
-
request_get,
|
|
84
|
-
session,
|
|
85
|
-
url,
|
|
86
|
-
params,
|
|
87
|
-
timeout,
|
|
88
|
-
max_retry_times,
|
|
89
|
-
verbose,
|
|
90
|
-
)
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def async_request_get(
|
|
94
|
-
session: requests.Session,
|
|
95
|
-
url: str,
|
|
96
|
-
params_list: List[Dict[str, Union[str, int, float]]],
|
|
97
|
-
timeout: int = 60,
|
|
98
|
-
max_retry_times: int = 10,
|
|
99
|
-
verbose: bool = False,
|
|
100
|
-
auto_tune: bool = True,
|
|
101
|
-
max_concurrency: int = None,
|
|
102
|
-
batch_size: int = 10,
|
|
103
|
-
):
|
|
104
|
-
"""
|
|
105
|
-
批量 async request,支援自動根據機器資源調整 max_concurrency / batch_size
|
|
106
|
-
"""
|
|
107
|
-
# 自動調整
|
|
108
|
-
if auto_tune:
|
|
109
|
-
cpu_count = os.cpu_count() or 1
|
|
110
|
-
max_concurrency = max_concurrency or max(1, cpu_count * 2)
|
|
111
|
-
else:
|
|
112
|
-
max_concurrency = max_concurrency or 10
|
|
113
|
-
|
|
114
|
-
async def runner():
|
|
115
|
-
semaphore = asyncio.Semaphore(max_concurrency)
|
|
116
|
-
loop = asyncio.get_event_loop()
|
|
117
|
-
executor = concurrent.futures.ThreadPoolExecutor(
|
|
118
|
-
max_workers=max_concurrency
|
|
119
|
-
)
|
|
120
|
-
|
|
121
|
-
async def _limited_run(params):
|
|
122
|
-
async with semaphore:
|
|
123
|
-
resp = await _loop_run_get(
|
|
124
|
-
executor,
|
|
125
|
-
loop,
|
|
126
|
-
session,
|
|
127
|
-
url=url,
|
|
128
|
-
params=params,
|
|
129
|
-
timeout=timeout,
|
|
130
|
-
max_retry_times=max_retry_times,
|
|
131
|
-
verbose=verbose,
|
|
132
|
-
)
|
|
133
|
-
return resp
|
|
134
|
-
|
|
135
|
-
results = []
|
|
136
|
-
pbar = tqdm(total=len(params_list))
|
|
137
|
-
|
|
138
|
-
# 分 batch 建立 task
|
|
139
|
-
for i in range(0, len(params_list), batch_size):
|
|
140
|
-
batch = params_list[i : i + batch_size]
|
|
141
|
-
tasks = [asyncio.create_task(_limited_run(p)) for p in batch]
|
|
142
|
-
|
|
143
|
-
for coro in asyncio.as_completed(tasks):
|
|
144
|
-
try:
|
|
145
|
-
res = await coro
|
|
146
|
-
results.append(res)
|
|
147
|
-
except Exception as exc:
|
|
148
|
-
if verbose:
|
|
149
|
-
logger.error(f"Task failed: {exc}")
|
|
150
|
-
pbar.update(1)
|
|
151
|
-
|
|
152
|
-
pbar.close()
|
|
153
|
-
executor.shutdown()
|
|
154
|
-
return results
|
|
155
|
-
|
|
156
|
-
return asyncio.run(runner())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|