lumibot 4.0.22__py3-none-any.whl → 4.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of lumibot might be problematic. Click here for more details.

Files changed (164) hide show
  1. lumibot/__pycache__/__init__.cpython-312.pyc +0 -0
  2. lumibot/__pycache__/constants.cpython-312.pyc +0 -0
  3. lumibot/__pycache__/credentials.cpython-312.pyc +0 -0
  4. lumibot/backtesting/__init__.py +6 -5
  5. lumibot/backtesting/__pycache__/__init__.cpython-312.pyc +0 -0
  6. lumibot/backtesting/__pycache__/alpaca_backtesting.cpython-312.pyc +0 -0
  7. lumibot/backtesting/__pycache__/alpha_vantage_backtesting.cpython-312.pyc +0 -0
  8. lumibot/backtesting/__pycache__/backtesting_broker.cpython-312.pyc +0 -0
  9. lumibot/backtesting/__pycache__/ccxt_backtesting.cpython-312.pyc +0 -0
  10. lumibot/backtesting/__pycache__/databento_backtesting.cpython-312.pyc +0 -0
  11. lumibot/backtesting/__pycache__/interactive_brokers_rest_backtesting.cpython-312.pyc +0 -0
  12. lumibot/backtesting/__pycache__/pandas_backtesting.cpython-312.pyc +0 -0
  13. lumibot/backtesting/__pycache__/polygon_backtesting.cpython-312.pyc +0 -0
  14. lumibot/backtesting/__pycache__/thetadata_backtesting.cpython-312.pyc +0 -0
  15. lumibot/backtesting/__pycache__/yahoo_backtesting.cpython-312.pyc +0 -0
  16. lumibot/backtesting/backtesting_broker.py +209 -9
  17. lumibot/backtesting/databento_backtesting.py +141 -24
  18. lumibot/backtesting/thetadata_backtesting.py +63 -42
  19. lumibot/brokers/__pycache__/__init__.cpython-312.pyc +0 -0
  20. lumibot/brokers/__pycache__/alpaca.cpython-312.pyc +0 -0
  21. lumibot/brokers/__pycache__/bitunix.cpython-312.pyc +0 -0
  22. lumibot/brokers/__pycache__/broker.cpython-312.pyc +0 -0
  23. lumibot/brokers/__pycache__/ccxt.cpython-312.pyc +0 -0
  24. lumibot/brokers/__pycache__/example_broker.cpython-312.pyc +0 -0
  25. lumibot/brokers/__pycache__/interactive_brokers.cpython-312.pyc +0 -0
  26. lumibot/brokers/__pycache__/interactive_brokers_rest.cpython-312.pyc +0 -0
  27. lumibot/brokers/__pycache__/projectx.cpython-312.pyc +0 -0
  28. lumibot/brokers/__pycache__/schwab.cpython-312.pyc +0 -0
  29. lumibot/brokers/__pycache__/tradier.cpython-312.pyc +0 -0
  30. lumibot/brokers/__pycache__/tradovate.cpython-312.pyc +0 -0
  31. lumibot/brokers/alpaca.py +11 -1
  32. lumibot/brokers/tradeovate.py +475 -0
  33. lumibot/components/grok_news_helper.py +284 -0
  34. lumibot/components/options_helper.py +90 -34
  35. lumibot/credentials.py +3 -0
  36. lumibot/data_sources/__init__.py +2 -1
  37. lumibot/data_sources/__pycache__/__init__.cpython-312.pyc +0 -0
  38. lumibot/data_sources/__pycache__/alpaca_data.cpython-312.pyc +0 -0
  39. lumibot/data_sources/__pycache__/alpha_vantage_data.cpython-312.pyc +0 -0
  40. lumibot/data_sources/__pycache__/bitunix_data.cpython-312.pyc +0 -0
  41. lumibot/data_sources/__pycache__/ccxt_backtesting_data.cpython-312.pyc +0 -0
  42. lumibot/data_sources/__pycache__/ccxt_data.cpython-312.pyc +0 -0
  43. lumibot/data_sources/__pycache__/data_source.cpython-312.pyc +0 -0
  44. lumibot/data_sources/__pycache__/data_source_backtesting.cpython-312.pyc +0 -0
  45. lumibot/data_sources/__pycache__/databento_data_polars_backtesting.cpython-312.pyc +0 -0
  46. lumibot/data_sources/__pycache__/databento_data_polars_live.cpython-312.pyc +0 -0
  47. lumibot/data_sources/__pycache__/example_broker_data.cpython-312.pyc +0 -0
  48. lumibot/data_sources/__pycache__/exceptions.cpython-312.pyc +0 -0
  49. lumibot/data_sources/__pycache__/interactive_brokers_data.cpython-312.pyc +0 -0
  50. lumibot/data_sources/__pycache__/interactive_brokers_rest_data.cpython-312.pyc +0 -0
  51. lumibot/data_sources/__pycache__/pandas_data.cpython-312.pyc +0 -0
  52. lumibot/data_sources/__pycache__/polars_mixin.cpython-312.pyc +0 -0
  53. lumibot/data_sources/__pycache__/polygon_data_polars.cpython-312.pyc +0 -0
  54. lumibot/data_sources/__pycache__/projectx_data.cpython-312.pyc +0 -0
  55. lumibot/data_sources/__pycache__/schwab_data.cpython-312.pyc +0 -0
  56. lumibot/data_sources/__pycache__/tradier_data.cpython-312.pyc +0 -0
  57. lumibot/data_sources/__pycache__/tradovate_data.cpython-312.pyc +0 -0
  58. lumibot/data_sources/__pycache__/yahoo_data_polars.cpython-312.pyc +0 -0
  59. lumibot/data_sources/data_source_backtesting.py +3 -5
  60. lumibot/data_sources/databento_data.py +5 -5
  61. lumibot/data_sources/databento_data_polars_backtesting.py +636 -0
  62. lumibot/data_sources/databento_data_polars_live.py +793 -0
  63. lumibot/data_sources/pandas_data.py +6 -3
  64. lumibot/data_sources/polars_mixin.py +126 -21
  65. lumibot/data_sources/tradeovate_data.py +80 -0
  66. lumibot/data_sources/tradier_data.py +2 -1
  67. lumibot/entities/__pycache__/__init__.cpython-312.pyc +0 -0
  68. lumibot/entities/__pycache__/asset.cpython-312.pyc +0 -0
  69. lumibot/entities/__pycache__/bar.cpython-312.pyc +0 -0
  70. lumibot/entities/__pycache__/bars.cpython-312.pyc +0 -0
  71. lumibot/entities/__pycache__/chains.cpython-312.pyc +0 -0
  72. lumibot/entities/__pycache__/data.cpython-312.pyc +0 -0
  73. lumibot/entities/__pycache__/dataline.cpython-312.pyc +0 -0
  74. lumibot/entities/__pycache__/order.cpython-312.pyc +0 -0
  75. lumibot/entities/__pycache__/position.cpython-312.pyc +0 -0
  76. lumibot/entities/__pycache__/quote.cpython-312.pyc +0 -0
  77. lumibot/entities/__pycache__/trading_fee.cpython-312.pyc +0 -0
  78. lumibot/entities/asset.py +8 -0
  79. lumibot/entities/order.py +1 -1
  80. lumibot/entities/quote.py +14 -0
  81. lumibot/example_strategies/__pycache__/__init__.cpython-312.pyc +0 -0
  82. lumibot/example_strategies/__pycache__/test_broker_functions.cpython-312-pytest-8.4.1.pyc +0 -0
  83. lumibot/strategies/__pycache__/__init__.cpython-312.pyc +0 -0
  84. lumibot/strategies/__pycache__/_strategy.cpython-312.pyc +0 -0
  85. lumibot/strategies/__pycache__/strategy.cpython-312.pyc +0 -0
  86. lumibot/strategies/__pycache__/strategy_executor.cpython-312.pyc +0 -0
  87. lumibot/strategies/_strategy.py +95 -27
  88. lumibot/strategies/strategy.py +5 -6
  89. lumibot/strategies/strategy_executor.py +2 -2
  90. lumibot/tools/__pycache__/__init__.cpython-312.pyc +0 -0
  91. lumibot/tools/__pycache__/alpaca_helpers.cpython-312.pyc +0 -0
  92. lumibot/tools/__pycache__/bitunix_helpers.cpython-312.pyc +0 -0
  93. lumibot/tools/__pycache__/black_scholes.cpython-312.pyc +0 -0
  94. lumibot/tools/__pycache__/ccxt_data_store.cpython-312.pyc +0 -0
  95. lumibot/tools/__pycache__/databento_helper.cpython-312.pyc +0 -0
  96. lumibot/tools/__pycache__/databento_helper_polars.cpython-312.pyc +0 -0
  97. lumibot/tools/__pycache__/debugers.cpython-312.pyc +0 -0
  98. lumibot/tools/__pycache__/decorators.cpython-312.pyc +0 -0
  99. lumibot/tools/__pycache__/helpers.cpython-312.pyc +0 -0
  100. lumibot/tools/__pycache__/indicators.cpython-312.pyc +0 -0
  101. lumibot/tools/__pycache__/lumibot_logger.cpython-312.pyc +0 -0
  102. lumibot/tools/__pycache__/pandas.cpython-312.pyc +0 -0
  103. lumibot/tools/__pycache__/polygon_helper.cpython-312.pyc +0 -0
  104. lumibot/tools/__pycache__/polygon_helper_async.cpython-312.pyc +0 -0
  105. lumibot/tools/__pycache__/polygon_helper_polars_optimized.cpython-312.pyc +0 -0
  106. lumibot/tools/__pycache__/projectx_helpers.cpython-312.pyc +0 -0
  107. lumibot/tools/__pycache__/schwab_helper.cpython-312.pyc +0 -0
  108. lumibot/tools/__pycache__/thetadata_helper.cpython-312.pyc +0 -0
  109. lumibot/tools/__pycache__/types.cpython-312.pyc +0 -0
  110. lumibot/tools/__pycache__/yahoo_helper.cpython-312.pyc +0 -0
  111. lumibot/tools/__pycache__/yahoo_helper_polars_optimized.cpython-312.pyc +0 -0
  112. lumibot/tools/databento_helper.py +384 -133
  113. lumibot/tools/databento_helper_polars.py +218 -156
  114. lumibot/tools/databento_roll.py +216 -0
  115. lumibot/tools/lumibot_logger.py +32 -17
  116. lumibot/tools/polygon_helper.py +65 -0
  117. lumibot/tools/thetadata_helper.py +588 -70
  118. lumibot/traders/__pycache__/__init__.cpython-312.pyc +0 -0
  119. lumibot/traders/__pycache__/trader.cpython-312.pyc +0 -0
  120. lumibot/traders/trader.py +1 -1
  121. lumibot/trading_builtins/__pycache__/__init__.cpython-312.pyc +0 -0
  122. lumibot/trading_builtins/__pycache__/custom_stream.cpython-312.pyc +0 -0
  123. lumibot/trading_builtins/__pycache__/safe_list.cpython-312.pyc +0 -0
  124. {lumibot-4.0.22.dist-info → lumibot-4.1.0.dist-info}/METADATA +1 -2
  125. {lumibot-4.0.22.dist-info → lumibot-4.1.0.dist-info}/RECORD +164 -46
  126. tests/backtest/check_timing_offset.py +198 -0
  127. tests/backtest/check_volume_spike.py +112 -0
  128. tests/backtest/comprehensive_comparison.py +166 -0
  129. tests/backtest/debug_comparison.py +91 -0
  130. tests/backtest/diagnose_price_difference.py +97 -0
  131. tests/backtest/direct_api_comparison.py +203 -0
  132. tests/backtest/profile_thetadata_vs_polygon.py +255 -0
  133. tests/backtest/root_cause_analysis.py +109 -0
  134. tests/backtest/test_accuracy_verification.py +244 -0
  135. tests/backtest/test_daily_data_timestamp_comparison.py +801 -0
  136. tests/backtest/test_databento.py +57 -0
  137. tests/backtest/test_databento_comprehensive_trading.py +564 -0
  138. tests/backtest/test_debug_avg_fill_price.py +112 -0
  139. tests/backtest/test_dividends.py +8 -3
  140. tests/backtest/test_example_strategies.py +54 -47
  141. tests/backtest/test_futures_edge_cases.py +451 -0
  142. tests/backtest/test_futures_single_trade.py +270 -0
  143. tests/backtest/test_futures_ultra_simple.py +191 -0
  144. tests/backtest/test_index_data_verification.py +348 -0
  145. tests/backtest/test_polygon.py +45 -24
  146. tests/backtest/test_thetadata.py +246 -60
  147. tests/backtest/test_thetadata_comprehensive.py +729 -0
  148. tests/backtest/test_thetadata_vs_polygon.py +557 -0
  149. tests/backtest/test_yahoo.py +1 -2
  150. tests/conftest.py +20 -0
  151. tests/test_backtesting_data_source_env.py +249 -0
  152. tests/test_backtesting_quiet_logs_complete.py +10 -11
  153. tests/test_databento_helper.py +73 -86
  154. tests/test_databento_live.py +10 -10
  155. tests/test_databento_timezone_fixes.py +21 -4
  156. tests/test_get_historical_prices.py +6 -6
  157. tests/test_options_helper.py +162 -40
  158. tests/test_polygon_helper.py +21 -13
  159. tests/test_quiet_logs_requirements.py +5 -5
  160. tests/test_thetadata_helper.py +487 -171
  161. tests/test_yahoo_data.py +125 -0
  162. {lumibot-4.0.22.dist-info → lumibot-4.1.0.dist-info}/LICENSE +0 -0
  163. {lumibot-4.0.22.dist-info → lumibot-4.1.0.dist-info}/WHEEL +0 -0
  164. {lumibot-4.0.22.dist-info → lumibot-4.1.0.dist-info}/top_level.txt +0 -0
@@ -103,8 +103,8 @@ class PandasData(DataSourceBacktesting):
103
103
  df = pd.DataFrame(range(len(dt_index)), index=dt_index)
104
104
  df = df.sort_index()
105
105
 
106
- # Create a column for the date portion only
107
- df["dates"] = df.index.date
106
+ # Create a column for the date portion only (normalize to date, keeping as datetime64 type)
107
+ df["dates"] = df.index.normalize()
108
108
 
109
109
  # Merge with the trading calendar on the 'dates' column to get market open/close times.
110
110
  # Use a left join to keep all rows from the original index.
@@ -145,7 +145,8 @@ class PandasData(DataSourceBacktesting):
145
145
 
146
146
  else:
147
147
  pcal.columns = ["datetime"]
148
- pcal["date"] = pcal["datetime"].dt.date
148
+ # Normalize to date but keep as datetime64 type (not date objects)
149
+ pcal["date"] = pcal["datetime"].dt.normalize()
149
150
  result = pcal.groupby("date").agg(
150
151
  market_open=(
151
152
  "datetime",
@@ -290,6 +291,8 @@ class PandasData(DataSourceBacktesting):
290
291
  ask=ohlcv_bid_ask_dict.get('ask'),
291
292
  volume=ohlcv_bid_ask_dict.get('volume'),
292
293
  timestamp=dt,
294
+ bid_size=ohlcv_bid_ask_dict.get('bid_size'),
295
+ ask_size=ohlcv_bid_ask_dict.get('ask_size'),
293
296
  raw_data=ohlcv_bid_ask_dict
294
297
  )
295
298
  else:
@@ -72,17 +72,19 @@ class PolarsMixin:
72
72
 
73
73
  def _get_data_lazy(self, asset: Asset) -> Optional[pl.LazyFrame]:
74
74
  """Get lazy frame for asset.
75
-
75
+
76
76
  Parameters
77
77
  ----------
78
- asset : Asset
79
- The asset to get data for
80
-
78
+ asset : Asset or tuple
79
+ The asset to get data for (can be a tuple of (asset, quote))
80
+
81
81
  Returns
82
82
  -------
83
83
  Optional[pl.LazyFrame]
84
84
  The lazy frame or None if not found
85
85
  """
86
+ # CRITICAL FIX: Handle both Asset and (Asset, quote) tuple keys
87
+ # The data store uses tuple keys (asset, quote), so we need to look up by that key
86
88
  return self._data_store.get(asset)
87
89
 
88
90
  def _parse_source_symbol_bars_polars(
@@ -95,7 +97,7 @@ class PolarsMixin:
95
97
  return_polars: bool = False
96
98
  ) -> Bars:
97
99
  """Parse bars from polars DataFrame.
98
-
100
+
99
101
  Parameters
100
102
  ----------
101
103
  response : pl.DataFrame
@@ -108,7 +110,7 @@ class PolarsMixin:
108
110
  The quote asset for forex/crypto
109
111
  length : Optional[int]
110
112
  Limit the number of bars
111
-
113
+
112
114
  Returns
113
115
  -------
114
116
  Bars
@@ -121,6 +123,21 @@ class PolarsMixin:
121
123
  if length and len(response) > length:
122
124
  response = response.tail(length)
123
125
 
126
+ # Filter to only keep OHLCV + datetime columns (remove DataBento metadata like rtype, publisher_id, etc.)
127
+ # Required columns for strategies
128
+ required_cols = ['open', 'high', 'low', 'close', 'volume']
129
+ optional_cols = ['datetime', 'timestamp', 'date', 'time', 'dividend', 'stock_splits', 'symbol']
130
+
131
+ # Determine which columns to keep
132
+ keep_cols = []
133
+ for col in response.columns:
134
+ if col in required_cols or col in optional_cols:
135
+ keep_cols.append(col)
136
+
137
+ # Select only the relevant columns
138
+ if keep_cols:
139
+ response = response.select(keep_cols)
140
+
124
141
  # Create bars object
125
142
  bars = Bars(response, source, asset, raw=response, quote=quote, return_polars=return_polars)
126
143
  return bars
@@ -209,22 +226,45 @@ class PolarsMixin:
209
226
  self._last_price_cache[cache_key] = price
210
227
 
211
228
  def _convert_datetime_for_filtering(self, dt: Any) -> datetime:
212
- """Convert datetime to naive datetime for filtering.
213
-
229
+ """Convert datetime to naive UTC datetime for filtering.
230
+
231
+ CRITICAL FIX: Must convert to UTC BEFORE stripping timezone!
232
+ If we strip timezone from ET datetime, we lose 5 hours of data.
233
+
234
+ Example:
235
+ - Input: 2024-01-02 18:00:00-05:00 (ET)
236
+ - Convert to UTC: 2024-01-02 23:00:00+00:00
237
+ - Strip timezone: 2024-01-02 23:00:00 (naive UTC)
238
+
239
+ OLD BUGGY CODE:
240
+ - Input: 2024-01-02 18:00:00-05:00 (ET)
241
+ - Strip timezone: 2024-01-02 18:00:00 (naive, loses timezone!)
242
+ - Compare to cached data in naive UTC: WRONG by 5 hours!
243
+
214
244
  Parameters
215
245
  ----------
216
246
  dt : Any
217
247
  Datetime-like object
218
-
248
+
219
249
  Returns
220
250
  -------
221
251
  datetime
222
- Naive datetime object
252
+ Naive UTC datetime object
223
253
  """
224
- if hasattr(dt, 'tz_localize'):
225
- return dt.tz_localize(None)
254
+ from datetime import timezone
255
+
256
+ # First convert to UTC if timezone-aware
257
+ if hasattr(dt, 'tzinfo') and dt.tzinfo is not None:
258
+ # Convert to UTC
259
+ dt_utc = dt.astimezone(timezone.utc)
260
+ # Then strip timezone
261
+ return dt_utc.replace(tzinfo=None)
262
+ elif hasattr(dt, 'tz_localize'):
263
+ # Pandas Timestamp
264
+ return dt.tz_convert('UTC').tz_localize(None)
226
265
  elif hasattr(dt, 'replace'):
227
- return dt.replace(tzinfo=None)
266
+ # Already naive
267
+ return dt
228
268
  else:
229
269
  return dt
230
270
 
@@ -283,10 +323,11 @@ class PolarsMixin:
283
323
  lazy_data: pl.LazyFrame,
284
324
  end_filter: datetime,
285
325
  length: int,
286
- timestep: str = "minute"
326
+ timestep: str = "minute",
327
+ use_strict_less_than: bool = False
287
328
  ) -> Optional[pl.DataFrame]:
288
329
  """Filter data up to end_filter and return last length rows.
289
-
330
+
290
331
  Parameters
291
332
  ----------
292
333
  asset : Asset
@@ -299,15 +340,23 @@ class PolarsMixin:
299
340
  Number of rows to return
300
341
  timestep : str
301
342
  Timestep for caching strategy
302
-
343
+ use_strict_less_than : bool
344
+ If True, use < instead of <= for filtering (matches Pandas behavior without timeshift)
345
+
303
346
  Returns
304
347
  -------
305
348
  Optional[pl.DataFrame]
306
349
  Filtered dataframe or None
307
350
  """
351
+ # DEBUG
352
+ logger.debug(f"[POLARS FILTER] end_filter={end_filter}, tzinfo={end_filter.tzinfo if hasattr(end_filter, 'tzinfo') else 'N/A'}, length={length}")
353
+
308
354
  # Convert end_filter to naive
309
355
  end_filter_naive = self._convert_datetime_for_filtering(end_filter)
310
356
 
357
+ # DEBUG
358
+ logger.debug(f"[POLARS FILTER] end_filter_naive={end_filter_naive}")
359
+
311
360
  # For daily timestep, use caching
312
361
  if timestep == "day":
313
362
  current_date = end_filter.date() if hasattr(end_filter, 'date') else end_filter
@@ -335,11 +384,37 @@ class PolarsMixin:
335
384
  return None
336
385
 
337
386
  # Filter and collect
387
+ # CRITICAL FIX: Keep timezone info! Match the DataFrame's timezone
388
+ # Get the DataFrame column's timezone from schema
389
+ dt_dtype = schema[dt_col]
390
+
391
+ # Convert filter to match DataFrame's timezone
392
+ if hasattr(dt_dtype, 'time_zone') and dt_dtype.time_zone:
393
+ # DataFrame has timezone, convert filter to match
394
+ import pytz
395
+ df_tz = pytz.timezone(dt_dtype.time_zone)
396
+ end_filter_with_tz = pytz.utc.localize(end_filter_naive).astimezone(df_tz)
397
+ else:
398
+ # DataFrame is naive, use UTC
399
+ from datetime import timezone as tz
400
+ end_filter_with_tz = datetime.combine(
401
+ end_filter_naive.date(),
402
+ end_filter_naive.time(),
403
+ tzinfo=tz.utc
404
+ )
405
+
406
+ # CRITICAL FIX: Deduplicate before caching
407
+ # Use < or <= based on use_strict_less_than flag
408
+ if use_strict_less_than:
409
+ filter_expr = pl.col(dt_col) < end_filter_with_tz
410
+ else:
411
+ filter_expr = pl.col(dt_col) <= end_filter_with_tz
412
+
338
413
  result = (
339
414
  lazy_data
340
- .with_columns(pl.col(dt_col).cast(pl.Datetime("us")))
341
- .filter(pl.col(dt_col) <= end_filter_naive)
415
+ .filter(filter_expr)
342
416
  .sort(dt_col)
417
+ .unique(subset=[dt_col], keep='last', maintain_order=True)
343
418
  .tail(fetch_length)
344
419
  .collect()
345
420
  )
@@ -362,11 +437,41 @@ class PolarsMixin:
362
437
  logger.error("No datetime column found")
363
438
  return None
364
439
 
365
- return (
440
+ # CRITICAL FIX: Keep timezone info during filtering!
441
+ # Match the DataFrame's timezone to avoid comparison errors
442
+ # Get the DataFrame column's timezone from schema
443
+ dt_dtype = schema[dt_col]
444
+
445
+ # Convert filter to match DataFrame's timezone
446
+ if hasattr(dt_dtype, 'time_zone') and dt_dtype.time_zone:
447
+ # DataFrame has timezone, convert filter to match
448
+ import pytz
449
+ df_tz = pytz.timezone(dt_dtype.time_zone)
450
+ end_filter_with_tz = pytz.utc.localize(end_filter_naive).astimezone(df_tz)
451
+ else:
452
+ # DataFrame is naive, use UTC
453
+ from datetime import timezone as tz
454
+ end_filter_with_tz = datetime.combine(
455
+ end_filter_naive.date(),
456
+ end_filter_naive.time(),
457
+ tzinfo=tz.utc
458
+ )
459
+
460
+ # CRITICAL FIX: Deduplicate before returning
461
+ # Sometimes lazy operations can create duplicates
462
+ # Use < or <= based on use_strict_less_than flag
463
+ if use_strict_less_than:
464
+ filter_expr = pl.col(dt_col) < end_filter_with_tz
465
+ else:
466
+ filter_expr = pl.col(dt_col) <= end_filter_with_tz
467
+
468
+ result = (
366
469
  lazy_data
367
- .with_columns(pl.col(dt_col).cast(pl.Datetime("us")))
368
- .filter(pl.col(dt_col) <= end_filter_naive)
470
+ .filter(filter_expr)
369
471
  .sort(dt_col)
472
+ .unique(subset=[dt_col], keep='last', maintain_order=True)
370
473
  .tail(length)
371
474
  .collect()
372
475
  )
476
+
477
+ return result
@@ -0,0 +1,80 @@
1
+ import logging
2
+ from decimal import Decimal
3
+ from typing import Union
4
+
5
+ from termcolor import colored
6
+ from lumibot.entities import Asset, Bars
7
+ from lumibot.data_sources import DataSource
8
+
9
+ class TradeovateData(DataSource):
10
+ """
11
+ Data source that connects to the Tradovate Market Data API.
12
+ Note: Tradovate market data is delivered via WebSocket.
13
+ """
14
+ MIN_TIMESTEP = "minute"
15
+ SOURCE = "Tradeovate"
16
+
17
+ def __init__(self, config, trading_token=None, market_token=None):
18
+ super().__init__()
19
+ self.config = config
20
+ # Use the market data WebSocket URL from config or default.
21
+ self.ws_url = config.get("MD_WS_URL", "wss://md.tradovateapi.com/v1/websocket")
22
+ # REST endpoint for market data.
23
+ self.market_data_url = config.get("MD_URL", "https://md.tradovateapi.com/v1")
24
+ # Store tokens directly
25
+ self.trading_token = trading_token
26
+ self.market_token = market_token
27
+ # Trading API URL for contract lookup
28
+ self.trading_api_url = config.get("TRADING_API_URL", "https://demo.tradovateapi.com/v1")
29
+
30
+ def _get_headers(self, with_auth=True, with_content_type=False):
31
+ """
32
+ Create headers for API requests.
33
+
34
+ Parameters
35
+ ----------
36
+ with_auth : bool
37
+ Whether to include the Authorization header with the trading token
38
+ with_content_type : bool
39
+ Whether to include Content-Type header for JSON requests
40
+
41
+ Returns
42
+ -------
43
+ dict
44
+ Dictionary of headers for API requests
45
+ """
46
+ headers = {"Accept": "application/json"}
47
+ if with_auth and self.trading_token:
48
+ headers["Authorization"] = f"Bearer {self.trading_token}"
49
+ if with_content_type:
50
+ headers["Content-Type"] = "application/json"
51
+ return headers
52
+
53
+ def get_chains(self, asset: Asset, quote: Asset = None) -> dict:
54
+ logging.error(colored("Method 'get_chains' does not work with Tradovate.", "red"))
55
+ return {}
56
+
57
+ def get_historical_prices(
58
+ self, asset, length, timestep="", timeshift=None, quote=None, exchange=None, include_after_hours=True
59
+ ) -> Bars:
60
+ """
61
+ Retrieve historical chart data for the given asset via WebSocket using the md/getChart command.
62
+ This method sends a WebSocket request to retrieve 'length' bars of historical data.
63
+
64
+ Note: Tradovate provides historical chart data via WebSocket, not via a REST GET.
65
+ """
66
+
67
+ # Log that this method is not supported because Tradovate requires you to get a CME subscription which costs $440/month
68
+ logging.error(colored("Method 'get_historical_prices' is not implemented for Tradovate because it requires a CME subscription which costs $440/month.", "red"))
69
+ return None
70
+
71
+ def get_last_price(self, asset, quote=None, exchange=None) -> Union[float, Decimal, None]:
72
+ """
73
+ Retrieve the most recent price for the given asset via WebSocket.
74
+ This method first retrieves the contract ID for the asset's symbol, then subscribes
75
+ to market data using that contract ID.
76
+ """
77
+
78
+ # Log that this method is not supported because Tradovate requires you to get a CME subscription which costs $440/month
79
+ logging.error(colored("Method 'get_last_price' is not implemented for Tradovate because it requires a CME subscription which costs $440/month.", "red"))
80
+ return None
@@ -255,7 +255,8 @@ class TradierData(DataSource):
255
255
  days_needed = length
256
256
  else:
257
257
  # For minute bars, calculate additional days needed accounting for weekends/holidays
258
- minutes_per_day = 390 # ~6.5 hours of trading per day
258
+ # minutes_per_day = 390 # ~6.5 hours of trading per day
259
+ minutes_per_day = 24 * 60 / timestep_qty # Need to include premarket and after hours
259
260
  days_needed = (length // minutes_per_day) + 1
260
261
 
261
262
  start_date = date_n_trading_days_from_date(
lumibot/entities/asset.py CHANGED
@@ -249,6 +249,10 @@ class Asset:
249
249
  if asset_type == self.AssetType.OPTION:
250
250
  self.multiplier = 100
251
251
 
252
+ # Note: Futures multipliers should be fetched from data provider (e.g., DataBento)
253
+ # at the data source level, not hardcoded here. The Asset class accepts multiplier
254
+ # as a parameter if the data source provides it.
255
+
252
256
  # Make sure right is upper case
253
257
  if right is not None:
254
258
  self.right = right.upper()
@@ -707,6 +711,10 @@ class Asset:
707
711
  if reference_date is None:
708
712
  reference_date = datetime.now()
709
713
 
714
+ # import logging
715
+ # logger = logging.getLogger(__name__)
716
+ # logger.info(f"[CONTRACT RESOLUTION] symbol={self.symbol}, reference_date={reference_date}, month={reference_date.month}, day={reference_date.day}")
717
+
710
718
  current_month = reference_date.month
711
719
  current_year = reference_date.year
712
720
  current_day = reference_date.day
lumibot/entities/order.py CHANGED
@@ -1108,7 +1108,7 @@ class Order:
1108
1108
  if self.asset is None:
1109
1109
  logger.error(f"Cannot create position from order {self.identifier} - asset is None")
1110
1110
  return None
1111
-
1111
+
1112
1112
  position_qty = quantity
1113
1113
  if self.side == SELL:
1114
1114
  position_qty = -quantity
lumibot/entities/quote.py CHANGED
@@ -83,6 +83,20 @@ class Quote:
83
83
  return (self.bid + self.ask) / 2
84
84
  return self.price
85
85
 
86
+ def __getitem__(self, key):
87
+ """
88
+ Allow dictionary-style access to Quote attributes for backward compatibility.
89
+ Tries to get the attribute first, then falls back to raw_data if available.
90
+ """
91
+ # Try to get as an attribute first
92
+ if hasattr(self, key):
93
+ return getattr(self, key)
94
+ # Fall back to raw_data if it exists
95
+ elif self.raw_data and key in self.raw_data:
96
+ return self.raw_data[key]
97
+ else:
98
+ raise KeyError(f"'{key}' not found in Quote object or raw_data")
99
+
86
100
  def __str__(self):
87
101
  return (f"Quote(asset={self.asset}, price={self.price}, bid={self.bid}, ask={self.ask}, "
88
102
  f"volume={self.volume}, timestamp={self.timestamp})")
@@ -684,6 +684,7 @@ class _Strategy:
684
684
 
685
685
  positions = self.broker.get_tracked_positions(self._name)
686
686
  assets_original = [position.asset for position in positions]
687
+
687
688
  # Set the base currency for crypto valuations.
688
689
 
689
690
  prices = {}
@@ -752,8 +753,33 @@ class _Strategy:
752
753
  if isinstance(asset, tuple):
753
754
  multiplier = 1
754
755
  else:
755
- multiplier = asset.multiplier if asset.asset_type in ["option", "future"] else 1
756
- portfolio_value += float(quantity) * float(price) * multiplier
756
+ multiplier = asset.multiplier if asset.asset_type in ["option", "future", "cont_future"] else 1
757
+
758
+ # BACKTESTING ONLY: Special handling for futures portfolio value
759
+ # In backtesting, cash has margin deducted, so we need to add it back
760
+ # In live trading, brokers handle this internally
761
+ if (
762
+ self.is_backtesting
763
+ and not isinstance(asset, tuple)
764
+ and asset.asset_type in ["future", "cont_future"]
765
+ ):
766
+ # Import here to avoid circular dependency
767
+ from lumibot.backtesting.backtesting_broker import get_futures_margin_requirement
768
+
769
+ # Add margin tied up in position (was deducted from cash)
770
+ margin_per_contract = get_futures_margin_requirement(asset)
771
+ total_margin = margin_per_contract * abs(float(quantity))
772
+ portfolio_value += total_margin
773
+
774
+ # Add unrealized P&L = (current_price - entry_price) × quantity × multiplier
775
+ entry_price = position.avg_fill_price if (hasattr(position, 'avg_fill_price') and position.avg_fill_price) else price
776
+ unrealized_pnl = (float(price) - float(entry_price)) * float(quantity) * multiplier
777
+ portfolio_value += unrealized_pnl
778
+ else:
779
+ # All other cases (stocks, options, crypto, live trading)
780
+ position_value = float(quantity) * float(price) * multiplier
781
+ portfolio_value += position_value
782
+
757
783
  self._portfolio_value = portfolio_value
758
784
  return portfolio_value
759
785
 
@@ -1238,6 +1264,63 @@ class _Strategy:
1238
1264
  if show_indicators is None:
1239
1265
  show_indicators = SHOW_INDICATORS
1240
1266
 
1267
+ # Auto-select datasource from environment variable if None
1268
+ if datasource_class is None:
1269
+ from lumibot.credentials import BACKTESTING_DATA_SOURCE
1270
+ from lumibot.backtesting import (
1271
+ PolygonDataBacktesting,
1272
+ ThetaDataBacktesting,
1273
+ YahooDataBacktesting,
1274
+ AlpacaBacktesting,
1275
+ CcxtBacktesting,
1276
+ DataBentoDataBacktesting,
1277
+ )
1278
+
1279
+ datasource_map = {
1280
+ "polygon": PolygonDataBacktesting,
1281
+ "thetadata": ThetaDataBacktesting,
1282
+ "yahoo": YahooDataBacktesting,
1283
+ "alpaca": AlpacaBacktesting,
1284
+ "ccxt": CcxtBacktesting,
1285
+ "databento": DataBentoDataBacktesting,
1286
+ }
1287
+
1288
+ datasource_name = BACKTESTING_DATA_SOURCE.lower()
1289
+ if datasource_name not in datasource_map:
1290
+ raise ValueError(
1291
+ f"Unknown BACKTESTING_DATA_SOURCE: '{BACKTESTING_DATA_SOURCE}'. "
1292
+ f"Valid options: {list(datasource_map.keys())}"
1293
+ )
1294
+
1295
+ datasource_class = datasource_map[datasource_name]
1296
+ get_logger(__name__).info(colored(
1297
+ f"Auto-selected backtesting data source from BACKTESTING_DATA_SOURCE env var: {BACKTESTING_DATA_SOURCE}",
1298
+ "green"
1299
+ ))
1300
+
1301
+ # Make sure polygon_api_key is set if using PolygonDataBacktesting
1302
+ polygon_api_key = polygon_api_key if polygon_api_key is not None else POLYGON_API_KEY
1303
+ if datasource_class.__name__ == 'PolygonDataBacktesting' and polygon_api_key is None:
1304
+ raise ValueError(
1305
+ "Please set `POLYGON_API_KEY` to your API key from polygon.io as an environment variable if "
1306
+ "you are using PolygonDataBacktesting. If you don't have one, you can get a free API key "
1307
+ "from https://polygon.io/."
1308
+ )
1309
+
1310
+ # Make sure thetadata_username and thetadata_password are set if using ThetaDataBacktesting
1311
+ if thetadata_username is None or thetadata_password is None:
1312
+ # Try getting the Theta Data credentials from credentials
1313
+ thetadata_username = THETADATA_CONFIG.get('THETADATA_USERNAME')
1314
+ thetadata_password = THETADATA_CONFIG.get('THETADATA_PASSWORD')
1315
+
1316
+ # Check again if theta data username and pass are set (before checking dict)
1317
+ if datasource_class.__name__ == 'ThetaDataBacktesting' and (thetadata_username is None or thetadata_password is None):
1318
+ raise ValueError(
1319
+ "Please set `thetadata_username` and `thetadata_password` in the backtest() function if "
1320
+ "you are using ThetaDataBacktesting. If you don't have one, you can do registeration "
1321
+ "from https://www.thetadata.net/."
1322
+ )
1323
+
1241
1324
  # check if datasource_class is a class or a dictionary
1242
1325
  if isinstance(datasource_class, dict):
1243
1326
  optionsource_class = datasource_class["OPTION"]
@@ -1247,6 +1330,14 @@ class _Strategy:
1247
1330
  use_other_option_source = False
1248
1331
  else:
1249
1332
  use_other_option_source = True
1333
+
1334
+ # Check ThetaData credentials for optionsource_class after dict extraction
1335
+ if optionsource_class.__name__ == 'ThetaDataBacktesting' and (thetadata_username is None or thetadata_password is None):
1336
+ raise ValueError(
1337
+ "Please set `thetadata_username` and `thetadata_password` in the backtest() function if "
1338
+ "you are using ThetaDataBacktesting. If you don't have one, you can do registeration "
1339
+ "from https://www.thetadata.net/."
1340
+ )
1250
1341
  else:
1251
1342
  optionsource_class = None
1252
1343
  use_other_option_source = False
@@ -1277,29 +1368,6 @@ class _Strategy:
1277
1368
 
1278
1369
  self.verify_backtest_inputs(backtesting_start, backtesting_end)
1279
1370
 
1280
- # Make sure polygon_api_key is set if using PolygonDataBacktesting
1281
- polygon_api_key = polygon_api_key if polygon_api_key is not None else POLYGON_API_KEY
1282
- if datasource_class == PolygonDataBacktesting and polygon_api_key is None:
1283
- raise ValueError(
1284
- "Please set `POLYGON_API_KEY` to your API key from polygon.io as an environment variable if "
1285
- "you are using PolygonDataBacktesting. If you don't have one, you can get a free API key "
1286
- "from https://polygon.io/."
1287
- )
1288
-
1289
- # Make sure thetadata_username and thetadata_password are set if using ThetaDataBacktesting
1290
- if thetadata_username is None or thetadata_password is None:
1291
- # Try getting the Theta Data credentials from credentials
1292
- thetadata_username = THETADATA_CONFIG.get('THETADATA_USERNAME')
1293
- thetadata_password = THETADATA_CONFIG.get('THETADATA_PASSWORD')
1294
-
1295
- # Check again if theta data username and pass are set
1296
- if (thetadata_username is None or thetadata_password is None) and (datasource_class == ThetaDataBacktesting or optionsource_class == ThetaDataBacktesting):
1297
- raise ValueError(
1298
- "Please set `thetadata_username` and `thetadata_password` in the backtest() function if "
1299
- "you are using ThetaDataBacktesting. If you don't have one, you can do registeration "
1300
- "from https://www.thetadata.net/."
1301
- )
1302
-
1303
1371
  if not self.IS_BACKTESTABLE:
1304
1372
  get_logger(__name__).warning(f"Strategy {name + ' ' if name is not None else ''}cannot be " f"backtested at the moment")
1305
1373
  return None
@@ -1323,7 +1391,7 @@ class _Strategy:
1323
1391
 
1324
1392
  self._trader = trader_class(logfile=logfile, backtest=True, quiet_logs=quiet_logs)
1325
1393
 
1326
- if datasource_class == PolygonDataBacktesting:
1394
+ if datasource_class.__name__ == 'PolygonDataBacktesting':
1327
1395
  data_source = datasource_class(
1328
1396
  backtesting_start,
1329
1397
  backtesting_end,
@@ -1336,7 +1404,7 @@ class _Strategy:
1336
1404
  log_backtest_progress_to_file=LOG_BACKTEST_PROGRESS_TO_FILE,
1337
1405
  **kwargs,
1338
1406
  )
1339
- elif datasource_class == ThetaDataBacktesting or optionsource_class == ThetaDataBacktesting:
1407
+ elif datasource_class.__name__ == 'ThetaDataBacktesting' or (optionsource_class and optionsource_class.__name__ == 'ThetaDataBacktesting'):
1340
1408
  data_source = datasource_class(
1341
1409
  backtesting_start,
1342
1410
  backtesting_end,
@@ -374,11 +374,10 @@ class Strategy(_Strategy):
374
374
  # Send the message to Discord
375
375
  self.send_discord_message(message)
376
376
 
377
- # If we are backtesting and we don't want to save the logfile, don't log (they're not displayed in the console anyway)
378
- if not self.save_logfile and self.is_backtesting:
379
- return
380
-
381
- # Check if INFO level is enabled before logging
377
+ # Performance optimization: skip logging if INFO is not enabled
378
+ # This respects BACKTESTING_QUIET_LOGS via StrategyLoggerAdapter.isEnabledFor()
379
+ # When BACKTESTING_QUIET_LOGS=true (default), this returns False and saves CPU cycles
380
+ # When BACKTESTING_QUIET_LOGS=false, this returns True and logs are displayed
382
381
  if not self.logger.isEnabledFor(logging.INFO):
383
382
  return
384
383
 
@@ -4411,7 +4410,7 @@ class Strategy(_Strategy):
4411
4410
  save_logfile: bool = False,
4412
4411
  thetadata_username: str = None,
4413
4412
  thetadata_password: str = None,
4414
- use_quote_data: bool = False,
4413
+ use_quote_data: bool = True, # Changed to True for ThetaData options support
4415
4414
  show_progress_bar: bool = True,
4416
4415
  quiet_logs: bool = True,
4417
4416
  trader_class: Type[Trader] = Trader,