FinMind 2.0.9__tar.gz → 2.0.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. {finmind-2.0.9 → finmind-2.0.11}/FinMind/data/data_loader.py +103 -3
  2. {finmind-2.0.9 → finmind-2.0.11}/FinMind/data/finmind_api.py +17 -10
  3. {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/data.py +2 -0
  4. finmind-2.0.11/FinMind/utility/request.py +277 -0
  5. {finmind-2.0.9/finmind.egg-info → finmind-2.0.11}/PKG-INFO +1 -1
  6. {finmind-2.0.9 → finmind-2.0.11/finmind.egg-info}/PKG-INFO +1 -1
  7. finmind-2.0.9/FinMind/utility/request.py +0 -156
  8. {finmind-2.0.9 → finmind-2.0.11}/FinMind/__init__.py +0 -0
  9. {finmind-2.0.9 → finmind-2.0.11}/FinMind/_version.py +0 -0
  10. {finmind-2.0.9 → finmind-2.0.11}/FinMind/crawler/__init__.py +0 -0
  11. {finmind-2.0.9 → finmind-2.0.11}/FinMind/crawler/base.py +0 -0
  12. {finmind-2.0.9 → finmind-2.0.11}/FinMind/crawler/commodities.py +0 -0
  13. {finmind-2.0.9 → finmind-2.0.11}/FinMind/crawler/demo.py +0 -0
  14. {finmind-2.0.9 → finmind-2.0.11}/FinMind/crawler/government_bonds.py +0 -0
  15. {finmind-2.0.9 → finmind-2.0.11}/FinMind/data/__init__.py +0 -0
  16. {finmind-2.0.9 → finmind-2.0.11}/FinMind/data/data_subscriber.py +0 -0
  17. {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/__init__.py +0 -0
  18. {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/bias.py +0 -0
  19. {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/continue_holding.py +0 -0
  20. {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/institutional_investors_follower.py +0 -0
  21. {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/institutional_investors_over_buy.py +0 -0
  22. {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/kd.py +0 -0
  23. {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/kd_crossover.py +0 -0
  24. {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/ma_cross_orver.py +0 -0
  25. {finmind-2.0.9 → finmind-2.0.11}/FinMind/indicators/short_sale_margin_purchase_ratio.py +0 -0
  26. {finmind-2.0.9 → finmind-2.0.11}/FinMind/plotting/__init__.py +0 -0
  27. {finmind-2.0.9 → finmind-2.0.11}/FinMind/plotting/bar.py +0 -0
  28. {finmind-2.0.9 → finmind-2.0.11}/FinMind/plotting/kline.py +0 -0
  29. {finmind-2.0.9 → finmind-2.0.11}/FinMind/plotting/line.py +0 -0
  30. {finmind-2.0.9 → finmind-2.0.11}/FinMind/plotting/pie.py +0 -0
  31. {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/__init__.py +0 -0
  32. {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/indicators.py +0 -0
  33. {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/info.py +0 -0
  34. {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/plot.py +0 -0
  35. {finmind-2.0.9 → finmind-2.0.11}/FinMind/schema/rule.py +0 -0
  36. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/__init__.py +0 -0
  37. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/base.py +0 -0
  38. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/bias.py +0 -0
  39. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/continue_holding.py +0 -0
  40. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/institutional_investors_follower.py +0 -0
  41. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/kd.py +0 -0
  42. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/kd_crossover.py +0 -0
  43. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/ma_crossover.py +0 -0
  44. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/macd_crossover.py +0 -0
  45. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/max_min_period_bias.py +0 -0
  46. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/naive_kd.py +0 -0
  47. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/short_sale_margin_purchase_ratio.py +0 -0
  48. {finmind-2.0.9 → finmind-2.0.11}/FinMind/strategies/utils.py +0 -0
  49. {finmind-2.0.9 → finmind-2.0.11}/FinMind/templates/post.html +0 -0
  50. {finmind-2.0.9 → finmind-2.0.11}/FinMind/utility/__init__.py +0 -0
  51. {finmind-2.0.9 → finmind-2.0.11}/FinMind/utility/rule.py +0 -0
  52. {finmind-2.0.9 → finmind-2.0.11}/LICENSE +0 -0
  53. {finmind-2.0.9 → finmind-2.0.11}/README.md +0 -0
  54. {finmind-2.0.9 → finmind-2.0.11}/finmind.egg-info/SOURCES.txt +0 -0
  55. {finmind-2.0.9 → finmind-2.0.11}/finmind.egg-info/dependency_links.txt +0 -0
  56. {finmind-2.0.9 → finmind-2.0.11}/finmind.egg-info/requires.txt +0 -0
  57. {finmind-2.0.9 → finmind-2.0.11}/finmind.egg-info/top_level.txt +0 -0
  58. {finmind-2.0.9 → finmind-2.0.11}/pyproject.toml +0 -0
  59. {finmind-2.0.9 → finmind-2.0.11}/setup.cfg +0 -0
  60. {finmind-2.0.9 → finmind-2.0.11}/setup.py +0 -0
@@ -191,9 +191,9 @@ class DataLoader(FinMindApi):
191
191
  date=date,
192
192
  timeout=timeout,
193
193
  )
194
- if not stock_id and not stock_id_list:
195
- stock_id_list = self._get_stock_id_list(date, timeout)
196
-
194
+ self._require_data_id(
195
+ Dataset.TaiwanStockPriceTick, "stock_id", stock_id, stock_id_list
196
+ )
197
197
  stock_tick = self.get_data(
198
198
  dataset=Dataset.TaiwanStockPriceTick,
199
199
  data_id=stock_id,
@@ -1217,6 +1217,50 @@ class DataLoader(FinMindApi):
1217
1217
  )
1218
1218
  return futures_daily
1219
1219
 
1220
+ def taiwan_futures_kbar(
1221
+ self,
1222
+ futures_id: str = "",
1223
+ date: str = "",
1224
+ timeout: int = None,
1225
+ use_object: bool = False,
1226
+ ) -> pd.DataFrame:
1227
+ """get 期貨分K資料
1228
+ :param futures_id: 期貨代號("TX")
1229
+ :param date (str): 日期("2024-01-02")
1230
+ :param timeout (int): timeout seconds, default None
1231
+ :param use_object (bool): 是否透過 signed URL 下載整日 parquet 資料物件,
1232
+ 設為 True 時忽略 futures_id 參數, default False
1233
+
1234
+ :return: 期貨分K TaiwanFuturesKBar
1235
+ :rtype pd.DataFrame
1236
+ :rtype column date (str): 日期
1237
+ :rtype column futures_id (str): 期貨代碼
1238
+ :rtype column contract_date (str): 到期月份
1239
+ :rtype column minute (str): 分鐘時間
1240
+ :rtype column open (float): 開盤價
1241
+ :rtype column high (float): 最高價
1242
+ :rtype column low (float): 最低價
1243
+ :rtype column close (float): 收盤價
1244
+ :rtype column volume (int): 成交量
1245
+ """
1246
+ if use_object:
1247
+ return self.get_object(
1248
+ dataset=Dataset.TaiwanFuturesKBar,
1249
+ date=date,
1250
+ timeout=timeout,
1251
+ )
1252
+ self._require_data_id(
1253
+ Dataset.TaiwanFuturesKBar, "futures_id", futures_id
1254
+ )
1255
+ futures_kbar = self.get_data(
1256
+ dataset=Dataset.TaiwanFuturesKBar,
1257
+ data_id=futures_id,
1258
+ start_date=date,
1259
+ end_date=date,
1260
+ timeout=timeout,
1261
+ )
1262
+ return futures_kbar
1263
+
1220
1264
  def taiwan_option_daily(
1221
1265
  self,
1222
1266
  option_id: str = "",
@@ -1387,6 +1431,9 @@ class DataLoader(FinMindApi):
1387
1431
  date=date,
1388
1432
  timeout=timeout,
1389
1433
  )
1434
+ self._require_data_id(
1435
+ Dataset.TaiwanFuturesTick, "futures_id", futures_id
1436
+ )
1390
1437
  futures_tick = self.get_data(
1391
1438
  dataset=Dataset.TaiwanFuturesTick,
1392
1439
  data_id=futures_id,
@@ -1458,6 +1505,9 @@ class DataLoader(FinMindApi):
1458
1505
  date=date,
1459
1506
  timeout=timeout,
1460
1507
  )
1508
+ self._require_data_id(
1509
+ Dataset.TaiwanOptionTick, "option_id", option_id, option_id_list
1510
+ )
1461
1511
  option_tick = self.get_data(
1462
1512
  dataset=Dataset.TaiwanOptionTick,
1463
1513
  data_id=option_id,
@@ -2088,6 +2138,7 @@ class DataLoader(FinMindApi):
2088
2138
  use_object: bool = False,
2089
2139
  ) -> pd.DataFrame:
2090
2140
  """get 台股分 K 資料表
2141
+ :param stock_id (str): 股票代碼, 加權指數請帶 "TAIEX"
2091
2142
  :param timeout (int): timeout seconds, default None
2092
2143
  :param use_object (bool): 是否透過 signed URL 下載整日 parquet 資料物件,
2093
2144
  設為 True 時忽略 stock_id, stock_id_list, use_async 參數, default False
@@ -2109,6 +2160,9 @@ class DataLoader(FinMindApi):
2109
2160
  date=date,
2110
2161
  timeout=timeout,
2111
2162
  )
2163
+ self._require_data_id(
2164
+ Dataset.TaiwanStockKBar, "stock_id", stock_id, stock_id_list
2165
+ )
2112
2166
  taiwan_stock_bar = self.get_data(
2113
2167
  dataset=Dataset.TaiwanStockKBar,
2114
2168
  data_id=stock_id,
@@ -3093,6 +3147,35 @@ class DataLoader(FinMindApi):
3093
3147
  )
3094
3148
  return taiwan_stock_day_trading_borrowing_fee_rate
3095
3149
 
3150
+ def taiwan_stock_broker_daily_concentration(
3151
+ self,
3152
+ stock_id: str = "",
3153
+ start_date: str = "",
3154
+ end_date: str = "",
3155
+ timeout: int = None,
3156
+ ) -> pd.DataFrame:
3157
+ """get 每日個股主力集中度
3158
+ :param stock_id (str): 股票代號("2330")
3159
+ :param start_date (str): 開始日期("2024-01-01")
3160
+ :param end_date (str): 結束日期("2024-01-31")
3161
+ :param timeout (int): timeout seconds, default None
3162
+
3163
+ :return: 每日個股主力集中度 TaiwanStockBrokerDailyConcentration
3164
+ :rtype pd.DataFrame
3165
+ :rtype column date (str): 日期
3166
+ :rtype column stock_id (str): 股票代號
3167
+ :rtype column top_k (int): 前 K 大(目前固定為 15)
3168
+ :rtype column top_buy_volume (int): 前 top_k 大買超券商買超股數總和(股)
3169
+ :rtype column top_sell_volume (int): 前 top_k 大賣超券商賣超股數總和(股,正值)
3170
+ """
3171
+ return self.get_data(
3172
+ dataset=Dataset.TaiwanStockBrokerDailyConcentration,
3173
+ data_id=stock_id,
3174
+ start_date=start_date,
3175
+ end_date=end_date,
3176
+ timeout=timeout,
3177
+ )
3178
+
3096
3179
  def taiwan_stock_convertible_bond_monthly_analysis(
3097
3180
  self,
3098
3181
  cb_id: str = "",
@@ -3261,6 +3344,23 @@ class DataLoader(FinMindApi):
3261
3344
  )
3262
3345
  return data
3263
3346
 
3347
+ @staticmethod
3348
+ def _require_data_id(
3349
+ dataset: str,
3350
+ id_name: str,
3351
+ data_id: str,
3352
+ data_id_list: typing.List[str] = None,
3353
+ ) -> None:
3354
+ """逐筆 / 分K 資料必須指定代號, 不支援不帶代號一次查詢所有商品
3355
+ 一次取得特定日期所有資料, 請用 use_object=True (限 sponsorpro)
3356
+ """
3357
+ if not data_id and not data_id_list:
3358
+ dataset = getattr(dataset, "value", dataset)
3359
+ raise ValueError(
3360
+ f"{dataset} 必須指定 {id_name}; "
3361
+ f"一次取得特定日期所有資料, 請使用 use_object=True (限 sponsorpro)"
3362
+ )
3363
+
3264
3364
  def _get_stock_id_list(
3265
3365
  self, date: str, timeout: int = None
3266
3366
  ) -> typing.List[str]:
@@ -7,7 +7,11 @@ import requests
7
7
  from loguru import logger
8
8
 
9
9
  from FinMind.schema.data import Dataset
10
- from FinMind.utility.request import async_request_get, request_get
10
+ from FinMind.utility.request import (
11
+ async_request_get,
12
+ download_storage_object,
13
+ request_get,
14
+ )
11
15
 
12
16
  logger.remove()
13
17
  logger.add(sys.stderr, level="INFO")
@@ -398,11 +402,19 @@ class FinMindApi:
398
402
  data = pd.DataFrame(self._extract_data(data))
399
403
  return data
400
404
 
401
- def get_object(self, dataset: str, date: str, timeout: int) -> pd.DataFrame:
405
+ def get_object(
406
+ self,
407
+ dataset: str,
408
+ date: str,
409
+ timeout: int,
410
+ max_retry_times: int = 10,
411
+ ) -> pd.DataFrame:
402
412
  """透過 signed URL 下載整日資料物件 (parquet 格式)
413
+ 傳輸中斷時會自動重試,並以 HTTP Range 從中斷處續傳
403
414
  :param dataset (str): 資料集名稱 (e.g. "TaiwanStockPriceTick")
404
415
  :param date (str): 資料日期 ("2025-01-06")
405
416
  :param timeout (int): request timeout 秒數
417
+ :param max_retry_times (int): 最多重試次數
406
418
 
407
419
  :return: 該日期所有資料
408
420
  :rtype: pd.DataFrame
@@ -412,16 +424,11 @@ class FinMindApi:
412
424
  "dataset": dataset,
413
425
  "date": date,
414
426
  }
415
- response = request_get(
427
+ content = download_storage_object(
416
428
  self.__session,
417
429
  url,
418
430
  params=params,
419
431
  timeout=timeout,
432
+ max_retry_times=max_retry_times,
420
433
  )
421
- status_code = response.status_code
422
- if status_code == 200:
423
- return pd.read_parquet(io.BytesIO(response.content))
424
- else:
425
- resp_json = response.json()
426
- logger.info(resp_json)
427
- return pd.DataFrame()
434
+ return pd.read_parquet(io.BytesIO(content))
@@ -30,6 +30,7 @@ class Dataset(str, Enum):
30
30
  TaiwanOptionTick = "TaiwanOptionTick"
31
31
  TaiwanStockPriceBidAsk = "TaiwanStockPriceBidAsk"
32
32
  TaiwanFuturesDaily = "TaiwanFuturesDaily"
33
+ TaiwanFuturesKBar = "TaiwanFuturesKBar"
33
34
  TaiwanOptionDaily = "TaiwanOptionDaily"
34
35
  TaiwanStockNews = "TaiwanStockNews"
35
36
  TaiwanStockTotalReturnIndex = "TaiwanStockTotalReturnIndex"
@@ -151,6 +152,7 @@ class Dataset(str, Enum):
151
152
  TaiwanAssetSwapFixedIncomeDaily = "TaiwanAssetSwapFixedIncomeDaily"
152
153
  TaiwanAssetSwapOptionDaily = "TaiwanAssetSwapOptionDaily"
153
154
  TaiwanStockMarginMaintenance = "TaiwanStockMarginMaintenance"
155
+ TaiwanStockBrokerDailyConcentration = "TaiwanStockBrokerDailyConcentration"
154
156
 
155
157
 
156
158
  class Version(str, Enum):
@@ -0,0 +1,277 @@
1
+ import asyncio
2
+ import concurrent.futures
3
+ import hashlib
4
+ import os
5
+ import ssl
6
+ import time
7
+ from typing import Dict, List, Union
8
+
9
+ import nest_asyncio
10
+ import requests
11
+ from loguru import logger
12
+ from tqdm import tqdm
13
+
14
+ nest_asyncio.apply()
15
+
16
+
17
+ def request_get(
18
+ session: requests.Session,
19
+ url: str,
20
+ params: Dict[str, Union[int, str, float]] = None,
21
+ timeout: int = 60,
22
+ max_retry_times: int = 10,
23
+ verbose: bool = False,
24
+ ):
25
+ """
26
+ 單次 request,支援 retry 與 log
27
+ """
28
+ response = None
29
+ for retry_times in range(1, max_retry_times + 1):
30
+ try:
31
+ response = session.get(
32
+ url,
33
+ verify=True,
34
+ params=params,
35
+ timeout=timeout,
36
+ )
37
+ if response.status_code == 504:
38
+ if verbose:
39
+ logger.warning(
40
+ f"status_code=504, retry {retry_times}/{max_retry_times}"
41
+ )
42
+ time.sleep(retry_times * 0.1)
43
+ else:
44
+ break
45
+ except (
46
+ requests.ConnectionError,
47
+ ssl.SSLError,
48
+ requests.exceptions.ReadTimeout,
49
+ requests.exceptions.ChunkedEncodingError,
50
+ ) as exc:
51
+ if verbose:
52
+ logger.warning(
53
+ f"Error: {exc}, retry {retry_times}/{max_retry_times}"
54
+ )
55
+ time.sleep(retry_times * 0.1)
56
+ except Exception as exc:
57
+ if verbose:
58
+ logger.warning(
59
+ f"Unexpected error: {exc}, retry {retry_times}/{max_retry_times}"
60
+ )
61
+ time.sleep(retry_times * 0.1)
62
+ if response and response.status_code != 200:
63
+ raise Exception(
64
+ f"Final response status: {response.status_code}, text: {response.text}"
65
+ )
66
+ return response
67
+
68
+
69
+ RETRYABLE_ERRORS = (
70
+ requests.ConnectionError,
71
+ ssl.SSLError,
72
+ requests.exceptions.ReadTimeout,
73
+ requests.exceptions.ChunkedEncodingError,
74
+ )
75
+ REDIRECT_STATUS = (301, 302, 303, 307, 308)
76
+
77
+
78
+ def download_storage_object(
79
+ session: requests.Session,
80
+ url: str,
81
+ params: Dict[str, Union[int, str, float]] = None,
82
+ timeout: int = 60,
83
+ max_retry_times: int = 10,
84
+ verbose: bool = False,
85
+ ) -> bytes:
86
+ """
87
+ 下載整日資料物件,傳輸中斷時以 Range + If-Match 續傳
88
+
89
+ storage_objects 回 307 導向有效期限約 3 分鐘的下載網址,
90
+ 所以每次(含重試)都重新取得網址;已收到的內容保留,
91
+ 只要求剩下的位元組,並以 If-Match 確認仍是同一版本,
92
+ 版本不同(412)才從頭下載。完成後檢查長度,ETag 不含 "-" 時再比對 MD5。
93
+ """
94
+ buffer = bytearray()
95
+ etag = None
96
+ total = None
97
+ for retry_times in range(1, max_retry_times + 1):
98
+ try:
99
+ response = session.get(
100
+ url,
101
+ verify=True,
102
+ params=params,
103
+ timeout=timeout,
104
+ allow_redirects=False,
105
+ )
106
+ except RETRYABLE_ERRORS as exc:
107
+ if verbose:
108
+ logger.warning(
109
+ f"Error: {exc}, retry {retry_times}/{max_retry_times}"
110
+ )
111
+ time.sleep(retry_times * 0.5)
112
+ continue
113
+ if response.status_code == 200:
114
+ return response.content
115
+ if response.status_code == 504:
116
+ time.sleep(retry_times * 0.5)
117
+ continue
118
+ if response.status_code not in REDIRECT_STATUS:
119
+ raise Exception(
120
+ f"Final response status: {response.status_code}, text: {response.text}"
121
+ )
122
+ object_url = response.headers["Location"]
123
+
124
+ headers = {}
125
+ if buffer and etag:
126
+ headers = {"Range": f"bytes={len(buffer)}-", "If-Match": etag}
127
+ try:
128
+ # 下載網址本身帶簽章,不能帶 session 的 Authorization header
129
+ with requests.get(
130
+ object_url, headers=headers, stream=True, timeout=timeout
131
+ ) as object_response:
132
+ status_code = object_response.status_code
133
+ if status_code == 200:
134
+ buffer = bytearray()
135
+ etag = object_response.headers.get("ETag")
136
+ content_length = object_response.headers.get(
137
+ "Content-Length"
138
+ )
139
+ total = int(content_length) if content_length else None
140
+ elif status_code == 412:
141
+ # 物件在兩次請求之間已更新,已收到的內容作廢
142
+ buffer, etag, total = bytearray(), None, None
143
+ continue
144
+ elif status_code != 206:
145
+ if verbose:
146
+ logger.warning(
147
+ f"status_code={status_code}, retry {retry_times}/{max_retry_times}"
148
+ )
149
+ time.sleep(retry_times * 0.5)
150
+ continue
151
+ for chunk in object_response.iter_content(chunk_size=1 << 20):
152
+ buffer.extend(chunk)
153
+ except RETRYABLE_ERRORS as exc:
154
+ if verbose:
155
+ logger.warning(
156
+ f"Download interrupted at {len(buffer)} bytes: {exc}, "
157
+ f"retry {retry_times}/{max_retry_times}"
158
+ )
159
+ time.sleep(retry_times * 0.5)
160
+ continue
161
+
162
+ if total is not None and len(buffer) < total:
163
+ continue
164
+ too_long = total is not None and len(buffer) > total
165
+ if too_long or not _match_etag(buffer, etag):
166
+ if verbose:
167
+ logger.warning(
168
+ f"Downloaded object failed verification, retry {retry_times}/{max_retry_times}"
169
+ )
170
+ buffer, etag, total = bytearray(), None, None
171
+ continue
172
+ return bytes(buffer)
173
+ raise Exception(
174
+ f"Download storage object failed after {max_retry_times} retries: "
175
+ f"received {len(buffer)} of {total} bytes"
176
+ )
177
+
178
+
179
+ def _match_etag(content: bytearray, etag: str) -> bool:
180
+ """ETag 不含 "-" 時等於整個物件的 MD5;分段上傳的 ETag 無法以 MD5 驗證"""
181
+ if not etag:
182
+ return True
183
+ etag = etag.strip('"')
184
+ if "-" in etag:
185
+ return True
186
+ return hashlib.md5(content).hexdigest() == etag
187
+
188
+
189
+ async def _loop_run_get(
190
+ executor: concurrent.futures.ThreadPoolExecutor,
191
+ loop: asyncio.AbstractEventLoop,
192
+ session: requests.Session,
193
+ url: str,
194
+ params: Dict[str, Union[str, int, float]],
195
+ timeout: int = 60,
196
+ max_retry_times: int = 10,
197
+ verbose: bool = False,
198
+ ):
199
+ """
200
+ 將同步 request 包成 async
201
+ """
202
+ return await loop.run_in_executor(
203
+ executor,
204
+ request_get,
205
+ session,
206
+ url,
207
+ params,
208
+ timeout,
209
+ max_retry_times,
210
+ verbose,
211
+ )
212
+
213
+
214
+ def async_request_get(
215
+ session: requests.Session,
216
+ url: str,
217
+ params_list: List[Dict[str, Union[str, int, float]]],
218
+ timeout: int = 60,
219
+ max_retry_times: int = 10,
220
+ verbose: bool = False,
221
+ auto_tune: bool = True,
222
+ max_concurrency: int = None,
223
+ batch_size: int = 10,
224
+ ):
225
+ """
226
+ 批量 async request,支援自動根據機器資源調整 max_concurrency / batch_size
227
+ """
228
+ # 自動調整
229
+ if auto_tune:
230
+ cpu_count = os.cpu_count() or 1
231
+ max_concurrency = max_concurrency or max(1, cpu_count * 2)
232
+ else:
233
+ max_concurrency = max_concurrency or 10
234
+
235
+ async def runner():
236
+ semaphore = asyncio.Semaphore(max_concurrency)
237
+ loop = asyncio.get_event_loop()
238
+ executor = concurrent.futures.ThreadPoolExecutor(
239
+ max_workers=max_concurrency
240
+ )
241
+
242
+ async def _limited_run(params):
243
+ async with semaphore:
244
+ resp = await _loop_run_get(
245
+ executor,
246
+ loop,
247
+ session,
248
+ url=url,
249
+ params=params,
250
+ timeout=timeout,
251
+ max_retry_times=max_retry_times,
252
+ verbose=verbose,
253
+ )
254
+ return resp
255
+
256
+ results = []
257
+ pbar = tqdm(total=len(params_list))
258
+
259
+ # 分 batch 建立 task
260
+ for i in range(0, len(params_list), batch_size):
261
+ batch = params_list[i : i + batch_size]
262
+ tasks = [asyncio.create_task(_limited_run(p)) for p in batch]
263
+
264
+ for coro in asyncio.as_completed(tasks):
265
+ try:
266
+ res = await coro
267
+ results.append(res)
268
+ except Exception as exc:
269
+ if verbose:
270
+ logger.error(f"Task failed: {exc}")
271
+ pbar.update(1)
272
+
273
+ pbar.close()
274
+ executor.shutdown()
275
+ return results
276
+
277
+ return asyncio.run(runner())
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: finmind
3
- Version: 2.0.9
3
+ Version: 2.0.11
4
4
  Summary: financial mining
5
5
  Author-email: linsam <finmind.tw@gmail.com>
6
6
  License: Apache License
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: finmind
3
- Version: 2.0.9
3
+ Version: 2.0.11
4
4
  Summary: financial mining
5
5
  Author-email: linsam <finmind.tw@gmail.com>
6
6
  License: Apache License
@@ -1,156 +0,0 @@
1
- import asyncio
2
- import concurrent.futures
3
- import os
4
- import ssl
5
- import time
6
- from typing import Dict, List, Union
7
-
8
- import nest_asyncio
9
- import requests
10
- from loguru import logger
11
- from tqdm import tqdm
12
-
13
- nest_asyncio.apply()
14
-
15
-
16
- def request_get(
17
- session: requests.Session,
18
- url: str,
19
- params: Dict[str, Union[int, str, float]] = None,
20
- timeout: int = 60,
21
- max_retry_times: int = 10,
22
- verbose: bool = False,
23
- ):
24
- """
25
- 單次 request,支援 retry 與 log
26
- """
27
- response = None
28
- for retry_times in range(1, max_retry_times + 1):
29
- try:
30
- response = session.get(
31
- url,
32
- verify=True,
33
- params=params,
34
- timeout=timeout,
35
- )
36
- if response.status_code == 504:
37
- if verbose:
38
- logger.warning(
39
- f"status_code=504, retry {retry_times}/{max_retry_times}"
40
- )
41
- time.sleep(retry_times * 0.1)
42
- else:
43
- break
44
- except (
45
- requests.ConnectionError,
46
- ssl.SSLError,
47
- requests.exceptions.ReadTimeout,
48
- requests.exceptions.ChunkedEncodingError,
49
- ) as exc:
50
- if verbose:
51
- logger.warning(
52
- f"Error: {exc}, retry {retry_times}/{max_retry_times}"
53
- )
54
- time.sleep(retry_times * 0.1)
55
- except Exception as exc:
56
- if verbose:
57
- logger.warning(
58
- f"Unexpected error: {exc}, retry {retry_times}/{max_retry_times}"
59
- )
60
- time.sleep(retry_times * 0.1)
61
- if response and response.status_code != 200:
62
- raise Exception(
63
- f"Final response status: {response.status_code}, text: {response.text}"
64
- )
65
- return response
66
-
67
-
68
- async def _loop_run_get(
69
- executor: concurrent.futures.ThreadPoolExecutor,
70
- loop: asyncio.AbstractEventLoop,
71
- session: requests.Session,
72
- url: str,
73
- params: Dict[str, Union[str, int, float]],
74
- timeout: int = 60,
75
- max_retry_times: int = 10,
76
- verbose: bool = False,
77
- ):
78
- """
79
- 將同步 request 包成 async
80
- """
81
- return await loop.run_in_executor(
82
- executor,
83
- request_get,
84
- session,
85
- url,
86
- params,
87
- timeout,
88
- max_retry_times,
89
- verbose,
90
- )
91
-
92
-
93
- def async_request_get(
94
- session: requests.Session,
95
- url: str,
96
- params_list: List[Dict[str, Union[str, int, float]]],
97
- timeout: int = 60,
98
- max_retry_times: int = 10,
99
- verbose: bool = False,
100
- auto_tune: bool = True,
101
- max_concurrency: int = None,
102
- batch_size: int = 10,
103
- ):
104
- """
105
- 批量 async request,支援自動根據機器資源調整 max_concurrency / batch_size
106
- """
107
- # 自動調整
108
- if auto_tune:
109
- cpu_count = os.cpu_count() or 1
110
- max_concurrency = max_concurrency or max(1, cpu_count * 2)
111
- else:
112
- max_concurrency = max_concurrency or 10
113
-
114
- async def runner():
115
- semaphore = asyncio.Semaphore(max_concurrency)
116
- loop = asyncio.get_event_loop()
117
- executor = concurrent.futures.ThreadPoolExecutor(
118
- max_workers=max_concurrency
119
- )
120
-
121
- async def _limited_run(params):
122
- async with semaphore:
123
- resp = await _loop_run_get(
124
- executor,
125
- loop,
126
- session,
127
- url=url,
128
- params=params,
129
- timeout=timeout,
130
- max_retry_times=max_retry_times,
131
- verbose=verbose,
132
- )
133
- return resp
134
-
135
- results = []
136
- pbar = tqdm(total=len(params_list))
137
-
138
- # 分 batch 建立 task
139
- for i in range(0, len(params_list), batch_size):
140
- batch = params_list[i : i + batch_size]
141
- tasks = [asyncio.create_task(_limited_run(p)) for p in batch]
142
-
143
- for coro in asyncio.as_completed(tasks):
144
- try:
145
- res = await coro
146
- results.append(res)
147
- except Exception as exc:
148
- if verbose:
149
- logger.error(f"Task failed: {exc}")
150
- pbar.update(1)
151
-
152
- pbar.close()
153
- executor.shutdown()
154
- return results
155
-
156
- return asyncio.run(runner())
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes