bbstrader 0.2.92__py3-none-any.whl → 0.2.94__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of bbstrader might be problematic. Click here for more details.

Files changed (36) hide show
  1. bbstrader/__ini__.py +20 -20
  2. bbstrader/__main__.py +50 -50
  3. bbstrader/btengine/__init__.py +54 -54
  4. bbstrader/btengine/data.py +11 -9
  5. bbstrader/btengine/scripts.py +157 -157
  6. bbstrader/compat.py +19 -19
  7. bbstrader/config.py +137 -137
  8. bbstrader/core/data.py +22 -22
  9. bbstrader/core/utils.py +146 -146
  10. bbstrader/metatrader/__init__.py +6 -6
  11. bbstrader/metatrader/account.py +1516 -1516
  12. bbstrader/metatrader/copier.py +750 -735
  13. bbstrader/metatrader/rates.py +584 -584
  14. bbstrader/metatrader/risk.py +749 -748
  15. bbstrader/metatrader/scripts.py +81 -81
  16. bbstrader/metatrader/trade.py +1836 -1826
  17. bbstrader/metatrader/utils.py +645 -645
  18. bbstrader/models/__init__.py +10 -10
  19. bbstrader/models/factors.py +312 -312
  20. bbstrader/models/ml.py +1272 -1265
  21. bbstrader/models/optimization.py +182 -182
  22. bbstrader/models/portfolio.py +223 -223
  23. bbstrader/models/risk.py +398 -398
  24. bbstrader/trading/__init__.py +11 -11
  25. bbstrader/trading/execution.py +846 -842
  26. bbstrader/trading/script.py +155 -155
  27. bbstrader/trading/scripts.py +69 -69
  28. bbstrader/trading/strategies.py +860 -860
  29. bbstrader/tseries.py +1842 -1842
  30. {bbstrader-0.2.92.dist-info → bbstrader-0.2.94.dist-info}/LICENSE +21 -21
  31. {bbstrader-0.2.92.dist-info → bbstrader-0.2.94.dist-info}/METADATA +188 -187
  32. bbstrader-0.2.94.dist-info/RECORD +44 -0
  33. {bbstrader-0.2.92.dist-info → bbstrader-0.2.94.dist-info}/WHEEL +1 -1
  34. bbstrader-0.2.92.dist-info/RECORD +0 -44
  35. {bbstrader-0.2.92.dist-info → bbstrader-0.2.94.dist-info}/entry_points.txt +0 -0
  36. {bbstrader-0.2.92.dist-info → bbstrader-0.2.94.dist-info}/top_level.txt +0 -0
@@ -1,584 +1,584 @@
1
- from datetime import datetime
2
- from typing import Optional, Union
3
-
4
- import pandas as pd
5
- from exchange_calendars import get_calendar, get_calendar_names
6
- from pandas.tseries.holiday import USFederalHolidayCalendar
7
- from pandas.tseries.offsets import CustomBusinessDay
8
-
9
- from bbstrader.metatrader.account import AMG_EXCHANGES, Account, check_mt5_connection
10
- from bbstrader.metatrader.utils import TIMEFRAMES, TimeFrame, raise_mt5_error
11
-
12
- try:
13
- import MetaTrader5 as Mt5
14
- except ImportError:
15
- import bbstrader.compat # noqa: F401
16
-
17
-
18
- __all__ = [
19
- "Rates",
20
- "download_historical_data",
21
- "get_data_from_pos",
22
- "get_data_from_date",
23
- ]
24
-
25
- MAX_BARS = 10_000_000
26
-
27
- IDX_CALENDARS = {
28
- "CAD": "XTSE",
29
- "AUD": "XASX",
30
- "GBP": "XLON",
31
- "HKD": "XSHG",
32
- "ZAR": "XJSE",
33
- "CHF": "XSWX",
34
- "NOK": "XOSL",
35
- "EUR": "XETR",
36
- "SGD": "XSES",
37
- "USD": "us_futures",
38
- "JPY": "us_futures",
39
- }
40
-
41
- COMD_CALENDARS = {
42
- "Energies": "us_futures",
43
- "Metals": "us_futures",
44
- "Agricultures": "CBOT",
45
- "Bonds": {"USD": "CBOT", "EUR": "EUREX"},
46
- }
47
-
48
- CALENDARS = {
49
- "FX": "us_futures",
50
- "STK": AMG_EXCHANGES,
51
- "ETF": AMG_EXCHANGES,
52
- "IDX": IDX_CALENDARS,
53
- "COMD": COMD_CALENDARS,
54
- "CRYPTO": "24/7",
55
- "FUT": None,
56
- }
57
-
58
- SESSION_TIMEFRAMES = [
59
- Mt5.TIMEFRAME_D1,
60
- Mt5.TIMEFRAME_W1,
61
- Mt5.TIMEFRAME_H12,
62
- Mt5.TIMEFRAME_MN1,
63
- ]
64
-
65
-
66
- class Rates(object):
67
- """
68
- Provides methods to retrieve historical financial data from MetaTrader 5.
69
-
70
- This class encapsulates interactions with the MetaTrader 5 (MT5) terminal
71
- to fetch historical price data for a given symbol and timeframe. It offers
72
- flexibility in retrieving data either by specifying a starting position
73
- and count of bars or by providing a specific date range.
74
-
75
- Notes:
76
- 1. Befor using this class, ensure that the `Max bars in chart` in you terminal
77
- is set to a value that is greater than the number of bars you want to retrieve
78
- or just set it to Unlimited.
79
- In your MT5 terminal, go to `Tools` -> `Options` -> `Charts` -> `Max bars in chart`.
80
-
81
- 2. The `open, high, low, close, adjclose, returns,
82
- volume` properties returns data in Broker's timezone by default.
83
-
84
- See `bbstrader.metatrader.account.check_mt5_connection()` for more details on how to connect to MT5 terminal.
85
-
86
- Example:
87
- >>> rates = Rates("EURUSD", "1h")
88
- >>> df = rates.get_historical_data(
89
- ... date_from=datetime(2023, 1, 1),
90
- ... date_to=datetime(2023, 1, 10),
91
- ... )
92
- >>> print(df.head())
93
- """
94
-
95
- def __init__(
96
- self,
97
- symbol: str,
98
- timeframe: TimeFrame = "D1",
99
- start_pos: Union[int, str] = 0,
100
- count: Optional[int] = MAX_BARS,
101
- session_duration: Optional[float] = None,
102
- **kwargs,
103
- ):
104
- """
105
- Initializes a new Rates instance.
106
-
107
- Args:
108
- symbol (str): Financial instrument symbol (e.g., "EURUSD").
109
- timeframe (str): Timeframe string (e.g., "D1", "1h", "5m").
110
- start_pos (int, | str): Starting index (int) or date (str) for data retrieval.
111
- count (int, optional): Number of bars to retrieve default is
112
- the maximum bars availble in the MT5 terminal.
113
- session_duration (float): Number of trading hours per day.
114
-
115
- Raises:
116
- ValueError: If the provided timeframe is invalid.
117
-
118
- Notes:
119
- If `start_pos` is an str, it must be in 'YYYY-MM-DD' format.
120
- For `session_duration` check your broker symbols details
121
- """
122
- self.symbol = symbol
123
- self.time_frame = self._validate_time_frame(timeframe)
124
- self.sd = session_duration
125
- self.start_pos = self._get_start_pos(start_pos, timeframe)
126
- self.count = count
127
- self._mt5_initialized(**kwargs)
128
- self.__account = Account(**kwargs)
129
- self.__data = self.get_rates_from_pos()
130
-
131
- def _mt5_initialized(self, **kwargs):
132
- check_mt5_connection(**kwargs)
133
-
134
- def _get_start_pos(self, index, time_frame):
135
- if isinstance(index, int):
136
- start_pos = index
137
- elif isinstance(index, str):
138
- assert self.sd is not None, ValueError(
139
- "Please provide the session_duration in hour"
140
- )
141
- start_pos = self._get_pos_index(index, time_frame, self.sd)
142
- return start_pos
143
-
144
- def _get_pos_index(self, start_date, time_frame, sd):
145
- # Create a custom business day calendar
146
- us_business_day = CustomBusinessDay(calendar=USFederalHolidayCalendar())
147
-
148
- start_date = pd.to_datetime(start_date)
149
- end_date = pd.to_datetime(datetime.now())
150
-
151
- # Generate a range of business days
152
- trading_days = pd.date_range(
153
- start=start_date, end=end_date, freq=us_business_day
154
- )
155
-
156
- # Calculate the number of trading days
157
- trading_days = len(trading_days)
158
- td = trading_days
159
- time_frame_mapping = {}
160
- for minutes in [
161
- 1,
162
- 2,
163
- 3,
164
- 4,
165
- 5,
166
- 6,
167
- 10,
168
- 12,
169
- 15,
170
- 20,
171
- 30,
172
- 60,
173
- 120,
174
- 180,
175
- 240,
176
- 360,
177
- 480,
178
- 720,
179
- ]:
180
- key = f"{minutes // 60}h" if minutes >= 60 else f"{minutes}m"
181
- time_frame_mapping[key] = int(td * (60 / minutes) * sd)
182
- time_frame_mapping["D1"] = int(td)
183
-
184
- if time_frame not in time_frame_mapping:
185
- pv = list(time_frame_mapping.keys())
186
- raise ValueError(f"Unsupported time frame, Possible Values are {pv}")
187
-
188
- index = time_frame_mapping.get(time_frame, 0) - 1
189
- return max(index, 0)
190
-
191
- def _validate_time_frame(self, time_frame: str) -> int:
192
- """Validates and returns the MT5 timeframe code."""
193
- if time_frame not in TIMEFRAMES:
194
- raise ValueError(
195
- f"Unsupported time frame '{time_frame}'. "
196
- f"Possible values are: {list(TIMEFRAMES.keys())}"
197
- )
198
- return TIMEFRAMES[time_frame]
199
-
200
- def _fetch_data(
201
- self,
202
- start: Union[int, datetime, pd.Timestamp],
203
- count: Union[int, datetime, pd.Timestamp],
204
- lower_colnames=False,
205
- utc=False,
206
- ) -> Union[pd.DataFrame, None]:
207
- """Fetches data from MT5 and returns a DataFrame or None."""
208
- try:
209
- if isinstance(start, int) and isinstance(count, int):
210
- rates = Mt5.copy_rates_from_pos(
211
- self.symbol, self.time_frame, start, count
212
- )
213
- elif isinstance(start, (datetime, pd.Timestamp)) and isinstance(count, int):
214
- rates = Mt5.copy_rates_from(self.symbol, self.time_frame, start, count)
215
- elif isinstance(start, (datetime, pd.Timestamp)) and isinstance(
216
- count, (datetime, pd.Timestamp)
217
- ):
218
- rates = Mt5.copy_rates_range(self.symbol, self.time_frame, start, count)
219
- if rates is None:
220
- return None
221
-
222
- df = pd.DataFrame(rates)
223
- return self._format_dataframe(df, lower_colnames=lower_colnames, utc=utc)
224
- except Exception as e:
225
- raise_mt5_error(e)
226
-
227
- def _format_dataframe(
228
- self, df: pd.DataFrame, lower_colnames=False, utc=False
229
- ) -> pd.DataFrame:
230
- """Formats the raw MT5 data into a standardized DataFrame."""
231
- df = df.copy()
232
- df = df[["time", "open", "high", "low", "close", "tick_volume"]]
233
- df.columns = ["Date", "Open", "High", "Low", "Close", "Volume"]
234
- df["Adj Close"] = df["Close"]
235
- df = df[["Date", "Open", "High", "Low", "Close", "Adj Close", "Volume"]]
236
- df["Date"] = pd.to_datetime(df["Date"], unit="s", utc=utc)
237
- df.set_index("Date", inplace=True)
238
- if lower_colnames:
239
- df.columns = df.columns.str.lower().str.replace(" ", "_")
240
- df.index.name = df.index.name.lower().replace(" ", "_")
241
- return df
242
-
243
- def _filter_data(
244
- self, df: pd.DataFrame, date_from=None, date_to=None, fill_na=False
245
- ) -> pd.DataFrame:
246
- df = df.copy()
247
- symbol_type = self.__account.get_symbol_type(self.symbol)
248
- currencies = self.__account.get_currency_rates(self.symbol)
249
- s_info = self.__account.get_symbol_info(self.symbol)
250
- if symbol_type in CALENDARS:
251
- if symbol_type == "STK" or symbol_type == "ETF":
252
- for exchange in CALENDARS[symbol_type]:
253
- if exchange in get_calendar_names():
254
- symbols = self.__account.get_stocks_from_exchange(
255
- exchange_code=exchange
256
- )
257
- if self.symbol in symbols:
258
- calendar = get_calendar(exchange, side="right")
259
- break
260
- elif symbol_type == "IDX":
261
- calendar = get_calendar(
262
- CALENDARS[symbol_type][currencies["mc"]], side="right"
263
- )
264
- elif symbol_type == "COMD":
265
- for commodity in CALENDARS[symbol_type]:
266
- if commodity in s_info.path:
267
- calendar = get_calendar(
268
- CALENDARS[symbol_type][commodity], side="right"
269
- )
270
- elif symbol_type == "FUT":
271
- if "Index" in s_info.path:
272
- calendar = get_calendar(
273
- CALENDARS["IDX"][currencies["mc"]], side="right"
274
- )
275
- else:
276
- for commodity, cal in COMD_CALENDARS.items():
277
- if self.symbol in self.__account.get_future_symbols(
278
- category=commodity
279
- ):
280
- if commodity == "Bonds":
281
- calendar = get_calendar(
282
- cal[currencies["mc"]], side="right"
283
- )
284
- else:
285
- calendar = get_calendar(cal, side="right")
286
- else:
287
- calendar = get_calendar(CALENDARS[symbol_type], side="right")
288
- date_from = date_from or df.index[0]
289
- date_to = date_to or df.index[-1]
290
- if self.time_frame in SESSION_TIMEFRAMES:
291
- valid_sessions = calendar.sessions_in_range(date_from, date_to)
292
- else:
293
- valid_sessions = calendar.minutes_in_range(date_from, date_to)
294
- if self.time_frame in [Mt5.TIMEFRAME_M1, Mt5.TIMEFRAME_D1]:
295
- # save the index name of the dataframe
296
- index_name = df.index.name
297
- if fill_na:
298
- if isinstance(fill_na, bool):
299
- method = "nearest"
300
- if isinstance(fill_na, str):
301
- method = fill_na
302
- df = df.reindex(valid_sessions, method=method)
303
- else:
304
- df.reindex(valid_sessions, method=None)
305
- df.index = df.index.rename(index_name)
306
- else:
307
- df = df[df.index.isin(valid_sessions)]
308
- return df
309
-
310
- def _check_filter(self, filter, utc):
311
- if filter and self.time_frame not in SESSION_TIMEFRAMES and not utc:
312
- utc = True
313
- elif filter and self.time_frame in SESSION_TIMEFRAMES and utc:
314
- utc = False
315
- return utc
316
-
317
- def get_rates_from_pos(
318
- self, filter=False, fill_na=False, lower_colnames=False, utc=False
319
- ) -> Union[pd.DataFrame, None]:
320
- """
321
- Retrieves historical data starting from a specific position.
322
-
323
- Uses the `start_pos` and `count` attributes specified during
324
- initialization to fetch data.
325
-
326
- Args:
327
- filter : See `Rates.get_historical_data` for more details.
328
- fill_na : See `Rates.get_historical_data` for more details.
329
- lower_colnames : If True, the column names will be converted to lowercase.
330
- utc (bool, optional): If True, the data will be in UTC timezone.
331
- Defaults to False.
332
-
333
- Returns:
334
- Union[pd.DataFrame, None]: A DataFrame containing historical
335
- data if successful, otherwise None.
336
-
337
- Raises:
338
- ValueError: If `start_pos` or `count` is not provided during
339
- initialization.
340
-
341
- Notes:
342
- The Datetime for this method is in Broker's timezone.
343
- """
344
- if self.start_pos is None or self.count is None:
345
- raise ValueError(
346
- "Both 'start_pos' and 'count' must be provided "
347
- "when calling 'get_rates_from_pos'."
348
- )
349
- utc = self._check_filter(filter, utc)
350
- df = self._fetch_data(
351
- self.start_pos, self.count, lower_colnames=lower_colnames, utc=utc
352
- )
353
- if df is None:
354
- return None
355
- if filter:
356
- return self._filter_data(df, fill_na=fill_na)
357
- return df
358
-
359
- def get_rates_from(
360
- self,
361
- date_from: datetime | pd.Timestamp,
362
- count: int = MAX_BARS,
363
- filter=False,
364
- fill_na=False,
365
- lower_colnames=False,
366
- utc=False,
367
- ) -> Union[pd.DataFrame, None]:
368
- """
369
- Retrieves historical data within a specified date range.
370
-
371
- Args:
372
- date_from : Starting date for data retrieval.
373
- The data will be retrieved from this date going to the past.
374
-
375
- count : Number of bars to retrieve.
376
-
377
- filter : See `Rates.get_historical_data` for more details.
378
- fill_na : See `Rates.get_historical_data` for more details.
379
- lower_colnames : If True, the column names will be converted to lowercase.
380
- utc (bool, optional): If True, the data will be in UTC timezone.
381
- Defaults to False.
382
-
383
- Returns:
384
- Union[pd.DataFrame, None]: A DataFrame containing historical
385
- data if successful, otherwise None.
386
- """
387
- utc = self._check_filter(filter, utc)
388
- df = self._fetch_data(date_from, count, lower_colnames=lower_colnames, utc=utc)
389
- if df is None:
390
- return None
391
- if filter:
392
- return self._filter_data(df, fill_na=fill_na)
393
- return df
394
-
395
- @property
396
- def open(self):
397
- return self.__data["Open"]
398
-
399
- @property
400
- def high(self):
401
- return self.__data["High"]
402
-
403
- @property
404
- def low(self):
405
- return self.__data["Low"]
406
-
407
- @property
408
- def close(self):
409
- return self.__data["Close"]
410
-
411
- @property
412
- def adjclose(self):
413
- return self.__data["Adj Close"]
414
-
415
- @property
416
- def returns(self):
417
- """
418
- Fractional change between the current and a prior element.
419
-
420
- Computes the fractional change from the immediately previous row by default.
421
- This is useful in comparing the fraction of change in a time series of elements.
422
-
423
- Note
424
- ----
425
- It calculates fractional change (also known as `per unit change or relative change`)
426
- and `not percentage change`. If you need the percentage change, multiply these values by 100.
427
- """
428
- data = self.__data.copy()
429
- data["Returns"] = data["Adj Close"].pct_change()
430
- data = data.dropna()
431
- return data["Returns"]
432
-
433
- @property
434
- def volume(self):
435
- return self.__data["Volume"]
436
-
437
- def get_historical_data(
438
- self,
439
- date_from: datetime | pd.Timestamp,
440
- date_to: datetime | pd.Timestamp = pd.Timestamp.now(),
441
- utc: bool = False,
442
- filter: Optional[bool] = False,
443
- fill_na: Optional[bool | str] = False,
444
- lower_colnames: Optional[bool] = True,
445
- save_csv: Optional[bool] = False,
446
- ) -> Union[pd.DataFrame, None]:
447
- """
448
- Retrieves historical data within a specified date range.
449
-
450
- Args:
451
- date_from : Starting date for data retrieval.
452
-
453
- date_to : Ending date for data retrieval.
454
- Defaults to the current time.
455
-
456
- utc : If True, the data will be in UTC timezone.
457
- Defaults to False.
458
-
459
- filter : If True, the data will be filtered based
460
- on the trading sessions for the symbol.
461
- This is use when we want to use the data for backtesting using Zipline.
462
-
463
- fill_na : If True, the data will be filled with the nearest value.
464
- This is use only when `filter` is True and time frame is "1m" or "D1",
465
- this is because we use ``calendar.minutes_in_range`` or ``calendar.sessions_in_range``
466
- where calendar is the ``ExchangeCalendar`` from `exchange_calendars` package.
467
- So, for "1m" or "D1" time frame, the data will be filled with the nearest value
468
- because the data from MT5 will have approximately the same number of rows as the
469
- number of trading days or minute in the exchange calendar, so we can fill the missing
470
- data with the nearest value.
471
-
472
- But for other time frames, the data will be reindexed with the exchange calendar
473
- because the data from MT5 will have more rows than the number of trading days or minute
474
- in the exchange calendar. So we only take the data that is in the range of the exchange
475
- calendar sessions or minutes.
476
-
477
- lower_colnames : If True, the column names will be converted to lowercase.
478
-
479
- save_csv : File path to save the data as a CSV.
480
- If None, the data won't be saved.
481
-
482
- Returns:
483
- Union[pd.DataFrame, None]: A DataFrame containing historical data
484
- if successful, otherwise None.
485
-
486
- Raises:
487
- ValueError: If the starting date is greater than the ending date.
488
-
489
- Notes:
490
- The `filter` for this method can be use only for Admira Markets Group (AMG) symbols.
491
- The Datetime for this method is in Local timezone by default.
492
- All STK symbols are filtered based on the the exchange calendar.
493
- All FX symbols are filtered based on the ``us_futures`` calendar.
494
- All IDX symbols are filtered based on the exchange calendar of margin currency.
495
- All COMD symbols are filtered based on the exchange calendar of the commodity.
496
- """
497
- utc = self._check_filter(filter, utc)
498
- df = self._fetch_data(
499
- date_from, date_to, lower_colnames=lower_colnames, utc=utc
500
- )
501
- if df is None:
502
- return None
503
- if filter:
504
- df = self._filter_data(
505
- df, date_from=date_from, date_to=date_to, fill_na=fill_na
506
- )
507
- if save_csv:
508
- df.to_csv(f"{self.symbol}.csv")
509
- return df
510
-
511
-
512
- def download_historical_data(
513
- symbol,
514
- timeframe,
515
- date_from,
516
- date_to=pd.Timestamp.now(),
517
- lower_colnames=True,
518
- utc=False,
519
- filter=False,
520
- fill_na=False,
521
- save_csv=False,
522
- **kwargs,
523
- ):
524
- """Download historical data from MetaTrader 5 terminal.
525
- See `Rates.get_historical_data` for more details.
526
- """
527
- rates = Rates(symbol, timeframe, **kwargs)
528
- data = rates.get_historical_data(
529
- date_from=date_from,
530
- date_to=date_to,
531
- save_csv=save_csv,
532
- utc=utc,
533
- filter=filter,
534
- lower_colnames=lower_colnames,
535
- )
536
- return data
537
-
538
-
539
- def get_data_from_pos(
540
- symbol,
541
- timeframe,
542
- start_pos=0,
543
- fill_na=False,
544
- count=MAX_BARS,
545
- lower_colnames=False,
546
- utc=False,
547
- filter=False,
548
- session_duration=23.0,
549
- **kwargs,
550
- ):
551
- """Get historical data from a specific position.
552
- See `Rates.get_rates_from_pos` for more details.
553
- """
554
- rates = Rates(symbol, timeframe, start_pos, count, session_duration, **kwargs)
555
- data = rates.get_rates_from_pos(
556
- filter=filter, fill_na=fill_na, lower_colnames=lower_colnames, utc=utc
557
- )
558
- return data
559
-
560
-
561
- def get_data_from_date(
562
- symbol,
563
- timeframe,
564
- date_from,
565
- count=MAX_BARS,
566
- fill_na=False,
567
- lower_colnames=False,
568
- utc=False,
569
- filter=False,
570
- **kwargs,
571
- ):
572
- """Get historical data from a specific date.
573
- See `Rates.get_rates_from` for more details.
574
- """
575
- rates = Rates(symbol, timeframe, **kwargs)
576
- data = rates.get_rates_from(
577
- date_from,
578
- count,
579
- filter=filter,
580
- fill_na=fill_na,
581
- lower_colnames=lower_colnames,
582
- utc=utc,
583
- )
584
- return data
1
+ from datetime import datetime
2
+ from typing import Optional, Union
3
+
4
+ import pandas as pd
5
+ from exchange_calendars import get_calendar, get_calendar_names
6
+ from pandas.tseries.holiday import USFederalHolidayCalendar
7
+ from pandas.tseries.offsets import CustomBusinessDay
8
+
9
+ from bbstrader.metatrader.account import AMG_EXCHANGES, Account, check_mt5_connection
10
+ from bbstrader.metatrader.utils import TIMEFRAMES, TimeFrame, raise_mt5_error
11
+
12
+ try:
13
+ import MetaTrader5 as Mt5
14
+ except ImportError:
15
+ import bbstrader.compat # noqa: F401
16
+
17
+
18
+ __all__ = [
19
+ "Rates",
20
+ "download_historical_data",
21
+ "get_data_from_pos",
22
+ "get_data_from_date",
23
+ ]
24
+
25
+ MAX_BARS = 10_000_000
26
+
27
+ IDX_CALENDARS = {
28
+ "CAD": "XTSE",
29
+ "AUD": "XASX",
30
+ "GBP": "XLON",
31
+ "HKD": "XSHG",
32
+ "ZAR": "XJSE",
33
+ "CHF": "XSWX",
34
+ "NOK": "XOSL",
35
+ "EUR": "XETR",
36
+ "SGD": "XSES",
37
+ "USD": "us_futures",
38
+ "JPY": "us_futures",
39
+ }
40
+
41
+ COMD_CALENDARS = {
42
+ "Energies": "us_futures",
43
+ "Metals": "us_futures",
44
+ "Agricultures": "CBOT",
45
+ "Bonds": {"USD": "CBOT", "EUR": "EUREX"},
46
+ }
47
+
48
+ CALENDARS = {
49
+ "FX": "us_futures",
50
+ "STK": AMG_EXCHANGES,
51
+ "ETF": AMG_EXCHANGES,
52
+ "IDX": IDX_CALENDARS,
53
+ "COMD": COMD_CALENDARS,
54
+ "CRYPTO": "24/7",
55
+ "FUT": None,
56
+ }
57
+
58
+ SESSION_TIMEFRAMES = [
59
+ Mt5.TIMEFRAME_D1,
60
+ Mt5.TIMEFRAME_W1,
61
+ Mt5.TIMEFRAME_H12,
62
+ Mt5.TIMEFRAME_MN1,
63
+ ]
64
+
65
+
66
+ class Rates(object):
67
+ """
68
+ Provides methods to retrieve historical financial data from MetaTrader 5.
69
+
70
+ This class encapsulates interactions with the MetaTrader 5 (MT5) terminal
71
+ to fetch historical price data for a given symbol and timeframe. It offers
72
+ flexibility in retrieving data either by specifying a starting position
73
+ and count of bars or by providing a specific date range.
74
+
75
+ Notes:
76
+ 1. Befor using this class, ensure that the `Max bars in chart` in you terminal
77
+ is set to a value that is greater than the number of bars you want to retrieve
78
+ or just set it to Unlimited.
79
+ In your MT5 terminal, go to `Tools` -> `Options` -> `Charts` -> `Max bars in chart`.
80
+
81
+ 2. The `open, high, low, close, adjclose, returns,
82
+ volume` properties returns data in Broker's timezone by default.
83
+
84
+ See `bbstrader.metatrader.account.check_mt5_connection()` for more details on how to connect to MT5 terminal.
85
+
86
+ Example:
87
+ >>> rates = Rates("EURUSD", "1h")
88
+ >>> df = rates.get_historical_data(
89
+ ... date_from=datetime(2023, 1, 1),
90
+ ... date_to=datetime(2023, 1, 10),
91
+ ... )
92
+ >>> print(df.head())
93
+ """
94
+
95
+ def __init__(
96
+ self,
97
+ symbol: str,
98
+ timeframe: TimeFrame = "D1",
99
+ start_pos: Union[int, str] = 0,
100
+ count: Optional[int] = MAX_BARS,
101
+ session_duration: Optional[float] = None,
102
+ **kwargs,
103
+ ):
104
+ """
105
+ Initializes a new Rates instance.
106
+
107
+ Args:
108
+ symbol (str): Financial instrument symbol (e.g., "EURUSD").
109
+ timeframe (str): Timeframe string (e.g., "D1", "1h", "5m").
110
+ start_pos (int, | str): Starting index (int) or date (str) for data retrieval.
111
+ count (int, optional): Number of bars to retrieve default is
112
+ the maximum bars availble in the MT5 terminal.
113
+ session_duration (float): Number of trading hours per day.
114
+
115
+ Raises:
116
+ ValueError: If the provided timeframe is invalid.
117
+
118
+ Notes:
119
+ If `start_pos` is an str, it must be in 'YYYY-MM-DD' format.
120
+ For `session_duration` check your broker symbols details
121
+ """
122
+ self.symbol = symbol
123
+ self.time_frame = self._validate_time_frame(timeframe)
124
+ self.sd = session_duration
125
+ self.start_pos = self._get_start_pos(start_pos, timeframe)
126
+ self.count = count
127
+ self._mt5_initialized(**kwargs)
128
+ self.__account = Account(**kwargs)
129
+ self.__data = self.get_rates_from_pos()
130
+
131
+ def _mt5_initialized(self, **kwargs):
132
+ check_mt5_connection(**kwargs)
133
+
134
+ def _get_start_pos(self, index, time_frame):
135
+ if isinstance(index, int):
136
+ start_pos = index
137
+ elif isinstance(index, str):
138
+ assert self.sd is not None, ValueError(
139
+ "Please provide the session_duration in hour"
140
+ )
141
+ start_pos = self._get_pos_index(index, time_frame, self.sd)
142
+ return start_pos
143
+
144
+ def _get_pos_index(self, start_date, time_frame, sd):
145
+ # Create a custom business day calendar
146
+ us_business_day = CustomBusinessDay(calendar=USFederalHolidayCalendar())
147
+
148
+ start_date = pd.to_datetime(start_date)
149
+ end_date = pd.to_datetime(datetime.now())
150
+
151
+ # Generate a range of business days
152
+ trading_days = pd.date_range(
153
+ start=start_date, end=end_date, freq=us_business_day
154
+ )
155
+
156
+ # Calculate the number of trading days
157
+ trading_days = len(trading_days)
158
+ td = trading_days
159
+ time_frame_mapping = {}
160
+ for minutes in [
161
+ 1,
162
+ 2,
163
+ 3,
164
+ 4,
165
+ 5,
166
+ 6,
167
+ 10,
168
+ 12,
169
+ 15,
170
+ 20,
171
+ 30,
172
+ 60,
173
+ 120,
174
+ 180,
175
+ 240,
176
+ 360,
177
+ 480,
178
+ 720,
179
+ ]:
180
+ key = f"{minutes // 60}h" if minutes >= 60 else f"{minutes}m"
181
+ time_frame_mapping[key] = int(td * (60 / minutes) * sd)
182
+ time_frame_mapping["D1"] = int(td)
183
+
184
+ if time_frame not in time_frame_mapping:
185
+ pv = list(time_frame_mapping.keys())
186
+ raise ValueError(f"Unsupported time frame, Possible Values are {pv}")
187
+
188
+ index = time_frame_mapping.get(time_frame, 0) - 1
189
+ return max(index, 0)
190
+
191
+ def _validate_time_frame(self, time_frame: str) -> int:
192
+ """Validates and returns the MT5 timeframe code."""
193
+ if time_frame not in TIMEFRAMES:
194
+ raise ValueError(
195
+ f"Unsupported time frame '{time_frame}'. "
196
+ f"Possible values are: {list(TIMEFRAMES.keys())}"
197
+ )
198
+ return TIMEFRAMES[time_frame]
199
+
200
+ def _fetch_data(
201
+ self,
202
+ start: Union[int, datetime, pd.Timestamp],
203
+ count: Union[int, datetime, pd.Timestamp],
204
+ lower_colnames=False,
205
+ utc=False,
206
+ ) -> Union[pd.DataFrame, None]:
207
+ """Fetches data from MT5 and returns a DataFrame or None."""
208
+ try:
209
+ if isinstance(start, int) and isinstance(count, int):
210
+ rates = Mt5.copy_rates_from_pos(
211
+ self.symbol, self.time_frame, start, count
212
+ )
213
+ elif isinstance(start, (datetime, pd.Timestamp)) and isinstance(count, int):
214
+ rates = Mt5.copy_rates_from(self.symbol, self.time_frame, start, count)
215
+ elif isinstance(start, (datetime, pd.Timestamp)) and isinstance(
216
+ count, (datetime, pd.Timestamp)
217
+ ):
218
+ rates = Mt5.copy_rates_range(self.symbol, self.time_frame, start, count)
219
+ if rates is None:
220
+ return None
221
+
222
+ df = pd.DataFrame(rates)
223
+ return self._format_dataframe(df, lower_colnames=lower_colnames, utc=utc)
224
+ except Exception as e:
225
+ raise_mt5_error(e)
226
+
227
+ def _format_dataframe(
228
+ self, df: pd.DataFrame, lower_colnames=False, utc=False
229
+ ) -> pd.DataFrame:
230
+ """Formats the raw MT5 data into a standardized DataFrame."""
231
+ df = df.copy()
232
+ df = df[["time", "open", "high", "low", "close", "tick_volume"]]
233
+ df.columns = ["Date", "Open", "High", "Low", "Close", "Volume"]
234
+ df["Adj Close"] = df["Close"]
235
+ df = df[["Date", "Open", "High", "Low", "Close", "Adj Close", "Volume"]]
236
+ df["Date"] = pd.to_datetime(df["Date"], unit="s", utc=utc)
237
+ df.set_index("Date", inplace=True)
238
+ if lower_colnames:
239
+ df.columns = df.columns.str.lower().str.replace(" ", "_")
240
+ df.index.name = df.index.name.lower().replace(" ", "_")
241
+ return df
242
+
243
+ def _filter_data(
244
+ self, df: pd.DataFrame, date_from=None, date_to=None, fill_na=False
245
+ ) -> pd.DataFrame:
246
+ df = df.copy()
247
+ symbol_type = self.__account.get_symbol_type(self.symbol)
248
+ currencies = self.__account.get_currency_rates(self.symbol)
249
+ s_info = self.__account.get_symbol_info(self.symbol)
250
+ if symbol_type in CALENDARS:
251
+ if symbol_type == "STK" or symbol_type == "ETF":
252
+ for exchange in CALENDARS[symbol_type]:
253
+ if exchange in get_calendar_names():
254
+ symbols = self.__account.get_stocks_from_exchange(
255
+ exchange_code=exchange
256
+ )
257
+ if self.symbol in symbols:
258
+ calendar = get_calendar(exchange, side="right")
259
+ break
260
+ elif symbol_type == "IDX":
261
+ calendar = get_calendar(
262
+ CALENDARS[symbol_type][currencies["mc"]], side="right"
263
+ )
264
+ elif symbol_type == "COMD":
265
+ for commodity in CALENDARS[symbol_type]:
266
+ if commodity in s_info.path:
267
+ calendar = get_calendar(
268
+ CALENDARS[symbol_type][commodity], side="right"
269
+ )
270
+ elif symbol_type == "FUT":
271
+ if "Index" in s_info.path:
272
+ calendar = get_calendar(
273
+ CALENDARS["IDX"][currencies["mc"]], side="right"
274
+ )
275
+ else:
276
+ for commodity, cal in COMD_CALENDARS.items():
277
+ if self.symbol in self.__account.get_future_symbols(
278
+ category=commodity
279
+ ):
280
+ if commodity == "Bonds":
281
+ calendar = get_calendar(
282
+ cal[currencies["mc"]], side="right"
283
+ )
284
+ else:
285
+ calendar = get_calendar(cal, side="right")
286
+ else:
287
+ calendar = get_calendar(CALENDARS[symbol_type], side="right")
288
+ date_from = date_from or df.index[0]
289
+ date_to = date_to or df.index[-1]
290
+ if self.time_frame in SESSION_TIMEFRAMES:
291
+ valid_sessions = calendar.sessions_in_range(date_from, date_to)
292
+ else:
293
+ valid_sessions = calendar.minutes_in_range(date_from, date_to)
294
+ if self.time_frame in [Mt5.TIMEFRAME_M1, Mt5.TIMEFRAME_D1]:
295
+ # save the index name of the dataframe
296
+ index_name = df.index.name
297
+ if fill_na:
298
+ if isinstance(fill_na, bool):
299
+ method = "nearest"
300
+ if isinstance(fill_na, str):
301
+ method = fill_na
302
+ df = df.reindex(valid_sessions, method=method)
303
+ else:
304
+ df.reindex(valid_sessions, method=None)
305
+ df.index = df.index.rename(index_name)
306
+ else:
307
+ df = df[df.index.isin(valid_sessions)]
308
+ return df
309
+
310
+ def _check_filter(self, filter, utc):
311
+ if filter and self.time_frame not in SESSION_TIMEFRAMES and not utc:
312
+ utc = True
313
+ elif filter and self.time_frame in SESSION_TIMEFRAMES and utc:
314
+ utc = False
315
+ return utc
316
+
317
+ def get_rates_from_pos(
318
+ self, filter=False, fill_na=False, lower_colnames=False, utc=False
319
+ ) -> Union[pd.DataFrame, None]:
320
+ """
321
+ Retrieves historical data starting from a specific position.
322
+
323
+ Uses the `start_pos` and `count` attributes specified during
324
+ initialization to fetch data.
325
+
326
+ Args:
327
+ filter : See `Rates.get_historical_data` for more details.
328
+ fill_na : See `Rates.get_historical_data` for more details.
329
+ lower_colnames : If True, the column names will be converted to lowercase.
330
+ utc (bool, optional): If True, the data will be in UTC timezone.
331
+ Defaults to False.
332
+
333
+ Returns:
334
+ Union[pd.DataFrame, None]: A DataFrame containing historical
335
+ data if successful, otherwise None.
336
+
337
+ Raises:
338
+ ValueError: If `start_pos` or `count` is not provided during
339
+ initialization.
340
+
341
+ Notes:
342
+ The Datetime for this method is in Broker's timezone.
343
+ """
344
+ if self.start_pos is None or self.count is None:
345
+ raise ValueError(
346
+ "Both 'start_pos' and 'count' must be provided "
347
+ "when calling 'get_rates_from_pos'."
348
+ )
349
+ utc = self._check_filter(filter, utc)
350
+ df = self._fetch_data(
351
+ self.start_pos, self.count, lower_colnames=lower_colnames, utc=utc
352
+ )
353
+ if df is None:
354
+ return None
355
+ if filter:
356
+ return self._filter_data(df, fill_na=fill_na)
357
+ return df
358
+
359
+ def get_rates_from(
360
+ self,
361
+ date_from: datetime | pd.Timestamp,
362
+ count: int = MAX_BARS,
363
+ filter=False,
364
+ fill_na=False,
365
+ lower_colnames=False,
366
+ utc=False,
367
+ ) -> Union[pd.DataFrame, None]:
368
+ """
369
+ Retrieves historical data within a specified date range.
370
+
371
+ Args:
372
+ date_from : Starting date for data retrieval.
373
+ The data will be retrieved from this date going to the past.
374
+
375
+ count : Number of bars to retrieve.
376
+
377
+ filter : See `Rates.get_historical_data` for more details.
378
+ fill_na : See `Rates.get_historical_data` for more details.
379
+ lower_colnames : If True, the column names will be converted to lowercase.
380
+ utc (bool, optional): If True, the data will be in UTC timezone.
381
+ Defaults to False.
382
+
383
+ Returns:
384
+ Union[pd.DataFrame, None]: A DataFrame containing historical
385
+ data if successful, otherwise None.
386
+ """
387
+ utc = self._check_filter(filter, utc)
388
+ df = self._fetch_data(date_from, count, lower_colnames=lower_colnames, utc=utc)
389
+ if df is None:
390
+ return None
391
+ if filter:
392
+ return self._filter_data(df, fill_na=fill_na)
393
+ return df
394
+
395
+ @property
396
+ def open(self):
397
+ return self.__data["Open"]
398
+
399
+ @property
400
+ def high(self):
401
+ return self.__data["High"]
402
+
403
+ @property
404
+ def low(self):
405
+ return self.__data["Low"]
406
+
407
+ @property
408
+ def close(self):
409
+ return self.__data["Close"]
410
+
411
+ @property
412
+ def adjclose(self):
413
+ return self.__data["Adj Close"]
414
+
415
+ @property
416
+ def returns(self):
417
+ """
418
+ Fractional change between the current and a prior element.
419
+
420
+ Computes the fractional change from the immediately previous row by default.
421
+ This is useful in comparing the fraction of change in a time series of elements.
422
+
423
+ Note
424
+ ----
425
+ It calculates fractional change (also known as `per unit change or relative change`)
426
+ and `not percentage change`. If you need the percentage change, multiply these values by 100.
427
+ """
428
+ data = self.__data.copy()
429
+ data["Returns"] = data["Adj Close"].pct_change()
430
+ data = data.dropna()
431
+ return data["Returns"]
432
+
433
+ @property
434
+ def volume(self):
435
+ return self.__data["Volume"]
436
+
437
+ def get_historical_data(
438
+ self,
439
+ date_from: datetime | pd.Timestamp,
440
+ date_to: datetime | pd.Timestamp = pd.Timestamp.now(),
441
+ utc: bool = False,
442
+ filter: Optional[bool] = False,
443
+ fill_na: Optional[bool | str] = False,
444
+ lower_colnames: Optional[bool] = True,
445
+ save_csv: Optional[bool] = False,
446
+ ) -> Union[pd.DataFrame, None]:
447
+ """
448
+ Retrieves historical data within a specified date range.
449
+
450
+ Args:
451
+ date_from : Starting date for data retrieval.
452
+
453
+ date_to : Ending date for data retrieval.
454
+ Defaults to the current time.
455
+
456
+ utc : If True, the data will be in UTC timezone.
457
+ Defaults to False.
458
+
459
+ filter : If True, the data will be filtered based
460
+ on the trading sessions for the symbol.
461
+ This is use when we want to use the data for backtesting using Zipline.
462
+
463
+ fill_na : If True, the data will be filled with the nearest value.
464
+ This is use only when `filter` is True and time frame is "1m" or "D1",
465
+ this is because we use ``calendar.minutes_in_range`` or ``calendar.sessions_in_range``
466
+ where calendar is the ``ExchangeCalendar`` from `exchange_calendars` package.
467
+ So, for "1m" or "D1" time frame, the data will be filled with the nearest value
468
+ because the data from MT5 will have approximately the same number of rows as the
469
+ number of trading days or minute in the exchange calendar, so we can fill the missing
470
+ data with the nearest value.
471
+
472
+ But for other time frames, the data will be reindexed with the exchange calendar
473
+ because the data from MT5 will have more rows than the number of trading days or minute
474
+ in the exchange calendar. So we only take the data that is in the range of the exchange
475
+ calendar sessions or minutes.
476
+
477
+ lower_colnames : If True, the column names will be converted to lowercase.
478
+
479
+ save_csv : File path to save the data as a CSV.
480
+ If None, the data won't be saved.
481
+
482
+ Returns:
483
+ Union[pd.DataFrame, None]: A DataFrame containing historical data
484
+ if successful, otherwise None.
485
+
486
+ Raises:
487
+ ValueError: If the starting date is greater than the ending date.
488
+
489
+ Notes:
490
+ The `filter` for this method can be use only for Admira Markets Group (AMG) symbols.
491
+ The Datetime for this method is in Local timezone by default.
492
+ All STK symbols are filtered based on the the exchange calendar.
493
+ All FX symbols are filtered based on the ``us_futures`` calendar.
494
+ All IDX symbols are filtered based on the exchange calendar of margin currency.
495
+ All COMD symbols are filtered based on the exchange calendar of the commodity.
496
+ """
497
+ utc = self._check_filter(filter, utc)
498
+ df = self._fetch_data(
499
+ date_from, date_to, lower_colnames=lower_colnames, utc=utc
500
+ )
501
+ if df is None:
502
+ return None
503
+ if filter:
504
+ df = self._filter_data(
505
+ df, date_from=date_from, date_to=date_to, fill_na=fill_na
506
+ )
507
+ if save_csv:
508
+ df.to_csv(f"{self.symbol}.csv")
509
+ return df
510
+
511
+
512
+ def download_historical_data(
513
+ symbol,
514
+ timeframe,
515
+ date_from,
516
+ date_to=pd.Timestamp.now(),
517
+ lower_colnames=True,
518
+ utc=False,
519
+ filter=False,
520
+ fill_na=False,
521
+ save_csv=False,
522
+ **kwargs,
523
+ ):
524
+ """Download historical data from MetaTrader 5 terminal.
525
+ See `Rates.get_historical_data` for more details.
526
+ """
527
+ rates = Rates(symbol, timeframe, **kwargs)
528
+ data = rates.get_historical_data(
529
+ date_from=date_from,
530
+ date_to=date_to,
531
+ save_csv=save_csv,
532
+ utc=utc,
533
+ filter=filter,
534
+ lower_colnames=lower_colnames,
535
+ )
536
+ return data
537
+
538
+
539
+ def get_data_from_pos(
540
+ symbol,
541
+ timeframe,
542
+ start_pos=0,
543
+ fill_na=False,
544
+ count=MAX_BARS,
545
+ lower_colnames=False,
546
+ utc=False,
547
+ filter=False,
548
+ session_duration=23.0,
549
+ **kwargs,
550
+ ):
551
+ """Get historical data from a specific position.
552
+ See `Rates.get_rates_from_pos` for more details.
553
+ """
554
+ rates = Rates(symbol, timeframe, start_pos, count, session_duration, **kwargs)
555
+ data = rates.get_rates_from_pos(
556
+ filter=filter, fill_na=fill_na, lower_colnames=lower_colnames, utc=utc
557
+ )
558
+ return data
559
+
560
+
561
+ def get_data_from_date(
562
+ symbol,
563
+ timeframe,
564
+ date_from,
565
+ count=MAX_BARS,
566
+ fill_na=False,
567
+ lower_colnames=False,
568
+ utc=False,
569
+ filter=False,
570
+ **kwargs,
571
+ ):
572
+ """Get historical data from a specific date.
573
+ See `Rates.get_rates_from` for more details.
574
+ """
575
+ rates = Rates(symbol, timeframe, **kwargs)
576
+ data = rates.get_rates_from(
577
+ date_from,
578
+ count,
579
+ filter=filter,
580
+ fill_na=fill_na,
581
+ lower_colnames=lower_colnames,
582
+ utc=utc,
583
+ )
584
+ return data