lumibot 4.0.22__py3-none-any.whl → 4.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of lumibot might be problematic. Click here for more details.
- lumibot/__pycache__/__init__.cpython-312.pyc +0 -0
- lumibot/__pycache__/constants.cpython-312.pyc +0 -0
- lumibot/__pycache__/credentials.cpython-312.pyc +0 -0
- lumibot/backtesting/__init__.py +6 -5
- lumibot/backtesting/__pycache__/__init__.cpython-312.pyc +0 -0
- lumibot/backtesting/__pycache__/alpaca_backtesting.cpython-312.pyc +0 -0
- lumibot/backtesting/__pycache__/alpha_vantage_backtesting.cpython-312.pyc +0 -0
- lumibot/backtesting/__pycache__/backtesting_broker.cpython-312.pyc +0 -0
- lumibot/backtesting/__pycache__/ccxt_backtesting.cpython-312.pyc +0 -0
- lumibot/backtesting/__pycache__/databento_backtesting.cpython-312.pyc +0 -0
- lumibot/backtesting/__pycache__/interactive_brokers_rest_backtesting.cpython-312.pyc +0 -0
- lumibot/backtesting/__pycache__/pandas_backtesting.cpython-312.pyc +0 -0
- lumibot/backtesting/__pycache__/polygon_backtesting.cpython-312.pyc +0 -0
- lumibot/backtesting/__pycache__/thetadata_backtesting.cpython-312.pyc +0 -0
- lumibot/backtesting/__pycache__/yahoo_backtesting.cpython-312.pyc +0 -0
- lumibot/backtesting/backtesting_broker.py +209 -9
- lumibot/backtesting/databento_backtesting.py +141 -24
- lumibot/backtesting/thetadata_backtesting.py +63 -42
- lumibot/brokers/__pycache__/__init__.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/alpaca.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/bitunix.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/broker.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/ccxt.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/example_broker.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/interactive_brokers.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/interactive_brokers_rest.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/projectx.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/schwab.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/tradier.cpython-312.pyc +0 -0
- lumibot/brokers/__pycache__/tradovate.cpython-312.pyc +0 -0
- lumibot/brokers/alpaca.py +11 -1
- lumibot/brokers/tradeovate.py +475 -0
- lumibot/components/grok_news_helper.py +284 -0
- lumibot/components/options_helper.py +90 -34
- lumibot/credentials.py +3 -0
- lumibot/data_sources/__init__.py +2 -1
- lumibot/data_sources/__pycache__/__init__.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/alpaca_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/alpha_vantage_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/bitunix_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/ccxt_backtesting_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/ccxt_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/data_source.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/data_source_backtesting.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/databento_data_polars_backtesting.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/databento_data_polars_live.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/example_broker_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/exceptions.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/interactive_brokers_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/interactive_brokers_rest_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/pandas_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/polars_mixin.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/polygon_data_polars.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/projectx_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/schwab_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/tradier_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/tradovate_data.cpython-312.pyc +0 -0
- lumibot/data_sources/__pycache__/yahoo_data_polars.cpython-312.pyc +0 -0
- lumibot/data_sources/data_source_backtesting.py +3 -5
- lumibot/data_sources/databento_data.py +5 -5
- lumibot/data_sources/databento_data_polars_backtesting.py +636 -0
- lumibot/data_sources/databento_data_polars_live.py +793 -0
- lumibot/data_sources/pandas_data.py +6 -3
- lumibot/data_sources/polars_mixin.py +126 -21
- lumibot/data_sources/tradeovate_data.py +80 -0
- lumibot/data_sources/tradier_data.py +2 -1
- lumibot/entities/__pycache__/__init__.cpython-312.pyc +0 -0
- lumibot/entities/__pycache__/asset.cpython-312.pyc +0 -0
- lumibot/entities/__pycache__/bar.cpython-312.pyc +0 -0
- lumibot/entities/__pycache__/bars.cpython-312.pyc +0 -0
- lumibot/entities/__pycache__/chains.cpython-312.pyc +0 -0
- lumibot/entities/__pycache__/data.cpython-312.pyc +0 -0
- lumibot/entities/__pycache__/dataline.cpython-312.pyc +0 -0
- lumibot/entities/__pycache__/order.cpython-312.pyc +0 -0
- lumibot/entities/__pycache__/position.cpython-312.pyc +0 -0
- lumibot/entities/__pycache__/quote.cpython-312.pyc +0 -0
- lumibot/entities/__pycache__/trading_fee.cpython-312.pyc +0 -0
- lumibot/entities/asset.py +8 -0
- lumibot/entities/order.py +1 -1
- lumibot/entities/quote.py +14 -0
- lumibot/example_strategies/__pycache__/__init__.cpython-312.pyc +0 -0
- lumibot/example_strategies/__pycache__/test_broker_functions.cpython-312-pytest-8.4.1.pyc +0 -0
- lumibot/strategies/__pycache__/__init__.cpython-312.pyc +0 -0
- lumibot/strategies/__pycache__/_strategy.cpython-312.pyc +0 -0
- lumibot/strategies/__pycache__/strategy.cpython-312.pyc +0 -0
- lumibot/strategies/__pycache__/strategy_executor.cpython-312.pyc +0 -0
- lumibot/strategies/_strategy.py +95 -27
- lumibot/strategies/strategy.py +5 -6
- lumibot/strategies/strategy_executor.py +2 -2
- lumibot/tools/__pycache__/__init__.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/alpaca_helpers.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/bitunix_helpers.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/black_scholes.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/ccxt_data_store.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/databento_helper.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/databento_helper_polars.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/debugers.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/decorators.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/helpers.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/indicators.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/lumibot_logger.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/pandas.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/polygon_helper.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/polygon_helper_async.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/polygon_helper_polars_optimized.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/projectx_helpers.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/schwab_helper.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/thetadata_helper.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/types.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/yahoo_helper.cpython-312.pyc +0 -0
- lumibot/tools/__pycache__/yahoo_helper_polars_optimized.cpython-312.pyc +0 -0
- lumibot/tools/databento_helper.py +384 -133
- lumibot/tools/databento_helper_polars.py +218 -156
- lumibot/tools/databento_roll.py +216 -0
- lumibot/tools/lumibot_logger.py +32 -17
- lumibot/tools/polygon_helper.py +65 -0
- lumibot/tools/thetadata_helper.py +588 -70
- lumibot/traders/__pycache__/__init__.cpython-312.pyc +0 -0
- lumibot/traders/__pycache__/trader.cpython-312.pyc +0 -0
- lumibot/traders/trader.py +1 -1
- lumibot/trading_builtins/__pycache__/__init__.cpython-312.pyc +0 -0
- lumibot/trading_builtins/__pycache__/custom_stream.cpython-312.pyc +0 -0
- lumibot/trading_builtins/__pycache__/safe_list.cpython-312.pyc +0 -0
- {lumibot-4.0.22.dist-info → lumibot-4.1.0.dist-info}/METADATA +1 -2
- {lumibot-4.0.22.dist-info → lumibot-4.1.0.dist-info}/RECORD +164 -46
- tests/backtest/check_timing_offset.py +198 -0
- tests/backtest/check_volume_spike.py +112 -0
- tests/backtest/comprehensive_comparison.py +166 -0
- tests/backtest/debug_comparison.py +91 -0
- tests/backtest/diagnose_price_difference.py +97 -0
- tests/backtest/direct_api_comparison.py +203 -0
- tests/backtest/profile_thetadata_vs_polygon.py +255 -0
- tests/backtest/root_cause_analysis.py +109 -0
- tests/backtest/test_accuracy_verification.py +244 -0
- tests/backtest/test_daily_data_timestamp_comparison.py +801 -0
- tests/backtest/test_databento.py +57 -0
- tests/backtest/test_databento_comprehensive_trading.py +564 -0
- tests/backtest/test_debug_avg_fill_price.py +112 -0
- tests/backtest/test_dividends.py +8 -3
- tests/backtest/test_example_strategies.py +54 -47
- tests/backtest/test_futures_edge_cases.py +451 -0
- tests/backtest/test_futures_single_trade.py +270 -0
- tests/backtest/test_futures_ultra_simple.py +191 -0
- tests/backtest/test_index_data_verification.py +348 -0
- tests/backtest/test_polygon.py +45 -24
- tests/backtest/test_thetadata.py +246 -60
- tests/backtest/test_thetadata_comprehensive.py +729 -0
- tests/backtest/test_thetadata_vs_polygon.py +557 -0
- tests/backtest/test_yahoo.py +1 -2
- tests/conftest.py +20 -0
- tests/test_backtesting_data_source_env.py +249 -0
- tests/test_backtesting_quiet_logs_complete.py +10 -11
- tests/test_databento_helper.py +73 -86
- tests/test_databento_live.py +10 -10
- tests/test_databento_timezone_fixes.py +21 -4
- tests/test_get_historical_prices.py +6 -6
- tests/test_options_helper.py +162 -40
- tests/test_polygon_helper.py +21 -13
- tests/test_quiet_logs_requirements.py +5 -5
- tests/test_thetadata_helper.py +487 -171
- tests/test_yahoo_data.py +125 -0
- {lumibot-4.0.22.dist-info → lumibot-4.1.0.dist-info}/LICENSE +0 -0
- {lumibot-4.0.22.dist-info → lumibot-4.1.0.dist-info}/WHEEL +0 -0
- {lumibot-4.0.22.dist-info → lumibot-4.1.0.dist-info}/top_level.txt +0 -0
|
@@ -103,8 +103,8 @@ class PandasData(DataSourceBacktesting):
|
|
|
103
103
|
df = pd.DataFrame(range(len(dt_index)), index=dt_index)
|
|
104
104
|
df = df.sort_index()
|
|
105
105
|
|
|
106
|
-
# Create a column for the date portion only
|
|
107
|
-
df["dates"] = df.index.
|
|
106
|
+
# Create a column for the date portion only (normalize to date, keeping as datetime64 type)
|
|
107
|
+
df["dates"] = df.index.normalize()
|
|
108
108
|
|
|
109
109
|
# Merge with the trading calendar on the 'dates' column to get market open/close times.
|
|
110
110
|
# Use a left join to keep all rows from the original index.
|
|
@@ -145,7 +145,8 @@ class PandasData(DataSourceBacktesting):
|
|
|
145
145
|
|
|
146
146
|
else:
|
|
147
147
|
pcal.columns = ["datetime"]
|
|
148
|
-
|
|
148
|
+
# Normalize to date but keep as datetime64 type (not date objects)
|
|
149
|
+
pcal["date"] = pcal["datetime"].dt.normalize()
|
|
149
150
|
result = pcal.groupby("date").agg(
|
|
150
151
|
market_open=(
|
|
151
152
|
"datetime",
|
|
@@ -290,6 +291,8 @@ class PandasData(DataSourceBacktesting):
|
|
|
290
291
|
ask=ohlcv_bid_ask_dict.get('ask'),
|
|
291
292
|
volume=ohlcv_bid_ask_dict.get('volume'),
|
|
292
293
|
timestamp=dt,
|
|
294
|
+
bid_size=ohlcv_bid_ask_dict.get('bid_size'),
|
|
295
|
+
ask_size=ohlcv_bid_ask_dict.get('ask_size'),
|
|
293
296
|
raw_data=ohlcv_bid_ask_dict
|
|
294
297
|
)
|
|
295
298
|
else:
|
|
@@ -72,17 +72,19 @@ class PolarsMixin:
|
|
|
72
72
|
|
|
73
73
|
def _get_data_lazy(self, asset: Asset) -> Optional[pl.LazyFrame]:
|
|
74
74
|
"""Get lazy frame for asset.
|
|
75
|
-
|
|
75
|
+
|
|
76
76
|
Parameters
|
|
77
77
|
----------
|
|
78
|
-
asset : Asset
|
|
79
|
-
The asset to get data for
|
|
80
|
-
|
|
78
|
+
asset : Asset or tuple
|
|
79
|
+
The asset to get data for (can be a tuple of (asset, quote))
|
|
80
|
+
|
|
81
81
|
Returns
|
|
82
82
|
-------
|
|
83
83
|
Optional[pl.LazyFrame]
|
|
84
84
|
The lazy frame or None if not found
|
|
85
85
|
"""
|
|
86
|
+
# CRITICAL FIX: Handle both Asset and (Asset, quote) tuple keys
|
|
87
|
+
# The data store uses tuple keys (asset, quote), so we need to look up by that key
|
|
86
88
|
return self._data_store.get(asset)
|
|
87
89
|
|
|
88
90
|
def _parse_source_symbol_bars_polars(
|
|
@@ -95,7 +97,7 @@ class PolarsMixin:
|
|
|
95
97
|
return_polars: bool = False
|
|
96
98
|
) -> Bars:
|
|
97
99
|
"""Parse bars from polars DataFrame.
|
|
98
|
-
|
|
100
|
+
|
|
99
101
|
Parameters
|
|
100
102
|
----------
|
|
101
103
|
response : pl.DataFrame
|
|
@@ -108,7 +110,7 @@ class PolarsMixin:
|
|
|
108
110
|
The quote asset for forex/crypto
|
|
109
111
|
length : Optional[int]
|
|
110
112
|
Limit the number of bars
|
|
111
|
-
|
|
113
|
+
|
|
112
114
|
Returns
|
|
113
115
|
-------
|
|
114
116
|
Bars
|
|
@@ -121,6 +123,21 @@ class PolarsMixin:
|
|
|
121
123
|
if length and len(response) > length:
|
|
122
124
|
response = response.tail(length)
|
|
123
125
|
|
|
126
|
+
# Filter to only keep OHLCV + datetime columns (remove DataBento metadata like rtype, publisher_id, etc.)
|
|
127
|
+
# Required columns for strategies
|
|
128
|
+
required_cols = ['open', 'high', 'low', 'close', 'volume']
|
|
129
|
+
optional_cols = ['datetime', 'timestamp', 'date', 'time', 'dividend', 'stock_splits', 'symbol']
|
|
130
|
+
|
|
131
|
+
# Determine which columns to keep
|
|
132
|
+
keep_cols = []
|
|
133
|
+
for col in response.columns:
|
|
134
|
+
if col in required_cols or col in optional_cols:
|
|
135
|
+
keep_cols.append(col)
|
|
136
|
+
|
|
137
|
+
# Select only the relevant columns
|
|
138
|
+
if keep_cols:
|
|
139
|
+
response = response.select(keep_cols)
|
|
140
|
+
|
|
124
141
|
# Create bars object
|
|
125
142
|
bars = Bars(response, source, asset, raw=response, quote=quote, return_polars=return_polars)
|
|
126
143
|
return bars
|
|
@@ -209,22 +226,45 @@ class PolarsMixin:
|
|
|
209
226
|
self._last_price_cache[cache_key] = price
|
|
210
227
|
|
|
211
228
|
def _convert_datetime_for_filtering(self, dt: Any) -> datetime:
|
|
212
|
-
"""Convert datetime to naive datetime for filtering.
|
|
213
|
-
|
|
229
|
+
"""Convert datetime to naive UTC datetime for filtering.
|
|
230
|
+
|
|
231
|
+
CRITICAL FIX: Must convert to UTC BEFORE stripping timezone!
|
|
232
|
+
If we strip timezone from ET datetime, we lose 5 hours of data.
|
|
233
|
+
|
|
234
|
+
Example:
|
|
235
|
+
- Input: 2024-01-02 18:00:00-05:00 (ET)
|
|
236
|
+
- Convert to UTC: 2024-01-02 23:00:00+00:00
|
|
237
|
+
- Strip timezone: 2024-01-02 23:00:00 (naive UTC)
|
|
238
|
+
|
|
239
|
+
OLD BUGGY CODE:
|
|
240
|
+
- Input: 2024-01-02 18:00:00-05:00 (ET)
|
|
241
|
+
- Strip timezone: 2024-01-02 18:00:00 (naive, loses timezone!)
|
|
242
|
+
- Compare to cached data in naive UTC: WRONG by 5 hours!
|
|
243
|
+
|
|
214
244
|
Parameters
|
|
215
245
|
----------
|
|
216
246
|
dt : Any
|
|
217
247
|
Datetime-like object
|
|
218
|
-
|
|
248
|
+
|
|
219
249
|
Returns
|
|
220
250
|
-------
|
|
221
251
|
datetime
|
|
222
|
-
Naive datetime object
|
|
252
|
+
Naive UTC datetime object
|
|
223
253
|
"""
|
|
224
|
-
|
|
225
|
-
|
|
254
|
+
from datetime import timezone
|
|
255
|
+
|
|
256
|
+
# First convert to UTC if timezone-aware
|
|
257
|
+
if hasattr(dt, 'tzinfo') and dt.tzinfo is not None:
|
|
258
|
+
# Convert to UTC
|
|
259
|
+
dt_utc = dt.astimezone(timezone.utc)
|
|
260
|
+
# Then strip timezone
|
|
261
|
+
return dt_utc.replace(tzinfo=None)
|
|
262
|
+
elif hasattr(dt, 'tz_localize'):
|
|
263
|
+
# Pandas Timestamp
|
|
264
|
+
return dt.tz_convert('UTC').tz_localize(None)
|
|
226
265
|
elif hasattr(dt, 'replace'):
|
|
227
|
-
|
|
266
|
+
# Already naive
|
|
267
|
+
return dt
|
|
228
268
|
else:
|
|
229
269
|
return dt
|
|
230
270
|
|
|
@@ -283,10 +323,11 @@ class PolarsMixin:
|
|
|
283
323
|
lazy_data: pl.LazyFrame,
|
|
284
324
|
end_filter: datetime,
|
|
285
325
|
length: int,
|
|
286
|
-
timestep: str = "minute"
|
|
326
|
+
timestep: str = "minute",
|
|
327
|
+
use_strict_less_than: bool = False
|
|
287
328
|
) -> Optional[pl.DataFrame]:
|
|
288
329
|
"""Filter data up to end_filter and return last length rows.
|
|
289
|
-
|
|
330
|
+
|
|
290
331
|
Parameters
|
|
291
332
|
----------
|
|
292
333
|
asset : Asset
|
|
@@ -299,15 +340,23 @@ class PolarsMixin:
|
|
|
299
340
|
Number of rows to return
|
|
300
341
|
timestep : str
|
|
301
342
|
Timestep for caching strategy
|
|
302
|
-
|
|
343
|
+
use_strict_less_than : bool
|
|
344
|
+
If True, use < instead of <= for filtering (matches Pandas behavior without timeshift)
|
|
345
|
+
|
|
303
346
|
Returns
|
|
304
347
|
-------
|
|
305
348
|
Optional[pl.DataFrame]
|
|
306
349
|
Filtered dataframe or None
|
|
307
350
|
"""
|
|
351
|
+
# DEBUG
|
|
352
|
+
logger.debug(f"[POLARS FILTER] end_filter={end_filter}, tzinfo={end_filter.tzinfo if hasattr(end_filter, 'tzinfo') else 'N/A'}, length={length}")
|
|
353
|
+
|
|
308
354
|
# Convert end_filter to naive
|
|
309
355
|
end_filter_naive = self._convert_datetime_for_filtering(end_filter)
|
|
310
356
|
|
|
357
|
+
# DEBUG
|
|
358
|
+
logger.debug(f"[POLARS FILTER] end_filter_naive={end_filter_naive}")
|
|
359
|
+
|
|
311
360
|
# For daily timestep, use caching
|
|
312
361
|
if timestep == "day":
|
|
313
362
|
current_date = end_filter.date() if hasattr(end_filter, 'date') else end_filter
|
|
@@ -335,11 +384,37 @@ class PolarsMixin:
|
|
|
335
384
|
return None
|
|
336
385
|
|
|
337
386
|
# Filter and collect
|
|
387
|
+
# CRITICAL FIX: Keep timezone info! Match the DataFrame's timezone
|
|
388
|
+
# Get the DataFrame column's timezone from schema
|
|
389
|
+
dt_dtype = schema[dt_col]
|
|
390
|
+
|
|
391
|
+
# Convert filter to match DataFrame's timezone
|
|
392
|
+
if hasattr(dt_dtype, 'time_zone') and dt_dtype.time_zone:
|
|
393
|
+
# DataFrame has timezone, convert filter to match
|
|
394
|
+
import pytz
|
|
395
|
+
df_tz = pytz.timezone(dt_dtype.time_zone)
|
|
396
|
+
end_filter_with_tz = pytz.utc.localize(end_filter_naive).astimezone(df_tz)
|
|
397
|
+
else:
|
|
398
|
+
# DataFrame is naive, use UTC
|
|
399
|
+
from datetime import timezone as tz
|
|
400
|
+
end_filter_with_tz = datetime.combine(
|
|
401
|
+
end_filter_naive.date(),
|
|
402
|
+
end_filter_naive.time(),
|
|
403
|
+
tzinfo=tz.utc
|
|
404
|
+
)
|
|
405
|
+
|
|
406
|
+
# CRITICAL FIX: Deduplicate before caching
|
|
407
|
+
# Use < or <= based on use_strict_less_than flag
|
|
408
|
+
if use_strict_less_than:
|
|
409
|
+
filter_expr = pl.col(dt_col) < end_filter_with_tz
|
|
410
|
+
else:
|
|
411
|
+
filter_expr = pl.col(dt_col) <= end_filter_with_tz
|
|
412
|
+
|
|
338
413
|
result = (
|
|
339
414
|
lazy_data
|
|
340
|
-
.
|
|
341
|
-
.filter(pl.col(dt_col) <= end_filter_naive)
|
|
415
|
+
.filter(filter_expr)
|
|
342
416
|
.sort(dt_col)
|
|
417
|
+
.unique(subset=[dt_col], keep='last', maintain_order=True)
|
|
343
418
|
.tail(fetch_length)
|
|
344
419
|
.collect()
|
|
345
420
|
)
|
|
@@ -362,11 +437,41 @@ class PolarsMixin:
|
|
|
362
437
|
logger.error("No datetime column found")
|
|
363
438
|
return None
|
|
364
439
|
|
|
365
|
-
|
|
440
|
+
# CRITICAL FIX: Keep timezone info during filtering!
|
|
441
|
+
# Match the DataFrame's timezone to avoid comparison errors
|
|
442
|
+
# Get the DataFrame column's timezone from schema
|
|
443
|
+
dt_dtype = schema[dt_col]
|
|
444
|
+
|
|
445
|
+
# Convert filter to match DataFrame's timezone
|
|
446
|
+
if hasattr(dt_dtype, 'time_zone') and dt_dtype.time_zone:
|
|
447
|
+
# DataFrame has timezone, convert filter to match
|
|
448
|
+
import pytz
|
|
449
|
+
df_tz = pytz.timezone(dt_dtype.time_zone)
|
|
450
|
+
end_filter_with_tz = pytz.utc.localize(end_filter_naive).astimezone(df_tz)
|
|
451
|
+
else:
|
|
452
|
+
# DataFrame is naive, use UTC
|
|
453
|
+
from datetime import timezone as tz
|
|
454
|
+
end_filter_with_tz = datetime.combine(
|
|
455
|
+
end_filter_naive.date(),
|
|
456
|
+
end_filter_naive.time(),
|
|
457
|
+
tzinfo=tz.utc
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
# CRITICAL FIX: Deduplicate before returning
|
|
461
|
+
# Sometimes lazy operations can create duplicates
|
|
462
|
+
# Use < or <= based on use_strict_less_than flag
|
|
463
|
+
if use_strict_less_than:
|
|
464
|
+
filter_expr = pl.col(dt_col) < end_filter_with_tz
|
|
465
|
+
else:
|
|
466
|
+
filter_expr = pl.col(dt_col) <= end_filter_with_tz
|
|
467
|
+
|
|
468
|
+
result = (
|
|
366
469
|
lazy_data
|
|
367
|
-
.
|
|
368
|
-
.filter(pl.col(dt_col) <= end_filter_naive)
|
|
470
|
+
.filter(filter_expr)
|
|
369
471
|
.sort(dt_col)
|
|
472
|
+
.unique(subset=[dt_col], keep='last', maintain_order=True)
|
|
370
473
|
.tail(length)
|
|
371
474
|
.collect()
|
|
372
475
|
)
|
|
476
|
+
|
|
477
|
+
return result
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from decimal import Decimal
|
|
3
|
+
from typing import Union
|
|
4
|
+
|
|
5
|
+
from termcolor import colored
|
|
6
|
+
from lumibot.entities import Asset, Bars
|
|
7
|
+
from lumibot.data_sources import DataSource
|
|
8
|
+
|
|
9
|
+
class TradeovateData(DataSource):
|
|
10
|
+
"""
|
|
11
|
+
Data source that connects to the Tradovate Market Data API.
|
|
12
|
+
Note: Tradovate market data is delivered via WebSocket.
|
|
13
|
+
"""
|
|
14
|
+
MIN_TIMESTEP = "minute"
|
|
15
|
+
SOURCE = "Tradeovate"
|
|
16
|
+
|
|
17
|
+
def __init__(self, config, trading_token=None, market_token=None):
|
|
18
|
+
super().__init__()
|
|
19
|
+
self.config = config
|
|
20
|
+
# Use the market data WebSocket URL from config or default.
|
|
21
|
+
self.ws_url = config.get("MD_WS_URL", "wss://md.tradovateapi.com/v1/websocket")
|
|
22
|
+
# REST endpoint for market data.
|
|
23
|
+
self.market_data_url = config.get("MD_URL", "https://md.tradovateapi.com/v1")
|
|
24
|
+
# Store tokens directly
|
|
25
|
+
self.trading_token = trading_token
|
|
26
|
+
self.market_token = market_token
|
|
27
|
+
# Trading API URL for contract lookup
|
|
28
|
+
self.trading_api_url = config.get("TRADING_API_URL", "https://demo.tradovateapi.com/v1")
|
|
29
|
+
|
|
30
|
+
def _get_headers(self, with_auth=True, with_content_type=False):
|
|
31
|
+
"""
|
|
32
|
+
Create headers for API requests.
|
|
33
|
+
|
|
34
|
+
Parameters
|
|
35
|
+
----------
|
|
36
|
+
with_auth : bool
|
|
37
|
+
Whether to include the Authorization header with the trading token
|
|
38
|
+
with_content_type : bool
|
|
39
|
+
Whether to include Content-Type header for JSON requests
|
|
40
|
+
|
|
41
|
+
Returns
|
|
42
|
+
-------
|
|
43
|
+
dict
|
|
44
|
+
Dictionary of headers for API requests
|
|
45
|
+
"""
|
|
46
|
+
headers = {"Accept": "application/json"}
|
|
47
|
+
if with_auth and self.trading_token:
|
|
48
|
+
headers["Authorization"] = f"Bearer {self.trading_token}"
|
|
49
|
+
if with_content_type:
|
|
50
|
+
headers["Content-Type"] = "application/json"
|
|
51
|
+
return headers
|
|
52
|
+
|
|
53
|
+
def get_chains(self, asset: Asset, quote: Asset = None) -> dict:
|
|
54
|
+
logging.error(colored("Method 'get_chains' does not work with Tradovate.", "red"))
|
|
55
|
+
return {}
|
|
56
|
+
|
|
57
|
+
def get_historical_prices(
|
|
58
|
+
self, asset, length, timestep="", timeshift=None, quote=None, exchange=None, include_after_hours=True
|
|
59
|
+
) -> Bars:
|
|
60
|
+
"""
|
|
61
|
+
Retrieve historical chart data for the given asset via WebSocket using the md/getChart command.
|
|
62
|
+
This method sends a WebSocket request to retrieve 'length' bars of historical data.
|
|
63
|
+
|
|
64
|
+
Note: Tradovate provides historical chart data via WebSocket, not via a REST GET.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
# Log that this method is not supported because Tradovate requires you to get a CME subscription which costs $440/month
|
|
68
|
+
logging.error(colored("Method 'get_historical_prices' is not implemented for Tradovate because it requires a CME subscription which costs $440/month.", "red"))
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
def get_last_price(self, asset, quote=None, exchange=None) -> Union[float, Decimal, None]:
|
|
72
|
+
"""
|
|
73
|
+
Retrieve the most recent price for the given asset via WebSocket.
|
|
74
|
+
This method first retrieves the contract ID for the asset's symbol, then subscribes
|
|
75
|
+
to market data using that contract ID.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
# Log that this method is not supported because Tradovate requires you to get a CME subscription which costs $440/month
|
|
79
|
+
logging.error(colored("Method 'get_last_price' is not implemented for Tradovate because it requires a CME subscription which costs $440/month.", "red"))
|
|
80
|
+
return None
|
|
@@ -255,7 +255,8 @@ class TradierData(DataSource):
|
|
|
255
255
|
days_needed = length
|
|
256
256
|
else:
|
|
257
257
|
# For minute bars, calculate additional days needed accounting for weekends/holidays
|
|
258
|
-
minutes_per_day = 390 # ~6.5 hours of trading per day
|
|
258
|
+
# minutes_per_day = 390 # ~6.5 hours of trading per day
|
|
259
|
+
minutes_per_day = 24 * 60 / timestep_qty # Need to include premarket and after hours
|
|
259
260
|
days_needed = (length // minutes_per_day) + 1
|
|
260
261
|
|
|
261
262
|
start_date = date_n_trading_days_from_date(
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
lumibot/entities/asset.py
CHANGED
|
@@ -249,6 +249,10 @@ class Asset:
|
|
|
249
249
|
if asset_type == self.AssetType.OPTION:
|
|
250
250
|
self.multiplier = 100
|
|
251
251
|
|
|
252
|
+
# Note: Futures multipliers should be fetched from data provider (e.g., DataBento)
|
|
253
|
+
# at the data source level, not hardcoded here. The Asset class accepts multiplier
|
|
254
|
+
# as a parameter if the data source provides it.
|
|
255
|
+
|
|
252
256
|
# Make sure right is upper case
|
|
253
257
|
if right is not None:
|
|
254
258
|
self.right = right.upper()
|
|
@@ -707,6 +711,10 @@ class Asset:
|
|
|
707
711
|
if reference_date is None:
|
|
708
712
|
reference_date = datetime.now()
|
|
709
713
|
|
|
714
|
+
# import logging
|
|
715
|
+
# logger = logging.getLogger(__name__)
|
|
716
|
+
# logger.info(f"[CONTRACT RESOLUTION] symbol={self.symbol}, reference_date={reference_date}, month={reference_date.month}, day={reference_date.day}")
|
|
717
|
+
|
|
710
718
|
current_month = reference_date.month
|
|
711
719
|
current_year = reference_date.year
|
|
712
720
|
current_day = reference_date.day
|
lumibot/entities/order.py
CHANGED
lumibot/entities/quote.py
CHANGED
|
@@ -83,6 +83,20 @@ class Quote:
|
|
|
83
83
|
return (self.bid + self.ask) / 2
|
|
84
84
|
return self.price
|
|
85
85
|
|
|
86
|
+
def __getitem__(self, key):
|
|
87
|
+
"""
|
|
88
|
+
Allow dictionary-style access to Quote attributes for backward compatibility.
|
|
89
|
+
Tries to get the attribute first, then falls back to raw_data if available.
|
|
90
|
+
"""
|
|
91
|
+
# Try to get as an attribute first
|
|
92
|
+
if hasattr(self, key):
|
|
93
|
+
return getattr(self, key)
|
|
94
|
+
# Fall back to raw_data if it exists
|
|
95
|
+
elif self.raw_data and key in self.raw_data:
|
|
96
|
+
return self.raw_data[key]
|
|
97
|
+
else:
|
|
98
|
+
raise KeyError(f"'{key}' not found in Quote object or raw_data")
|
|
99
|
+
|
|
86
100
|
def __str__(self):
|
|
87
101
|
return (f"Quote(asset={self.asset}, price={self.price}, bid={self.bid}, ask={self.ask}, "
|
|
88
102
|
f"volume={self.volume}, timestamp={self.timestamp})")
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
lumibot/strategies/_strategy.py
CHANGED
|
@@ -684,6 +684,7 @@ class _Strategy:
|
|
|
684
684
|
|
|
685
685
|
positions = self.broker.get_tracked_positions(self._name)
|
|
686
686
|
assets_original = [position.asset for position in positions]
|
|
687
|
+
|
|
687
688
|
# Set the base currency for crypto valuations.
|
|
688
689
|
|
|
689
690
|
prices = {}
|
|
@@ -752,8 +753,33 @@ class _Strategy:
|
|
|
752
753
|
if isinstance(asset, tuple):
|
|
753
754
|
multiplier = 1
|
|
754
755
|
else:
|
|
755
|
-
multiplier = asset.multiplier if asset.asset_type in ["option", "future"] else 1
|
|
756
|
-
|
|
756
|
+
multiplier = asset.multiplier if asset.asset_type in ["option", "future", "cont_future"] else 1
|
|
757
|
+
|
|
758
|
+
# BACKTESTING ONLY: Special handling for futures portfolio value
|
|
759
|
+
# In backtesting, cash has margin deducted, so we need to add it back
|
|
760
|
+
# In live trading, brokers handle this internally
|
|
761
|
+
if (
|
|
762
|
+
self.is_backtesting
|
|
763
|
+
and not isinstance(asset, tuple)
|
|
764
|
+
and asset.asset_type in ["future", "cont_future"]
|
|
765
|
+
):
|
|
766
|
+
# Import here to avoid circular dependency
|
|
767
|
+
from lumibot.backtesting.backtesting_broker import get_futures_margin_requirement
|
|
768
|
+
|
|
769
|
+
# Add margin tied up in position (was deducted from cash)
|
|
770
|
+
margin_per_contract = get_futures_margin_requirement(asset)
|
|
771
|
+
total_margin = margin_per_contract * abs(float(quantity))
|
|
772
|
+
portfolio_value += total_margin
|
|
773
|
+
|
|
774
|
+
# Add unrealized P&L = (current_price - entry_price) × quantity × multiplier
|
|
775
|
+
entry_price = position.avg_fill_price if (hasattr(position, 'avg_fill_price') and position.avg_fill_price) else price
|
|
776
|
+
unrealized_pnl = (float(price) - float(entry_price)) * float(quantity) * multiplier
|
|
777
|
+
portfolio_value += unrealized_pnl
|
|
778
|
+
else:
|
|
779
|
+
# All other cases (stocks, options, crypto, live trading)
|
|
780
|
+
position_value = float(quantity) * float(price) * multiplier
|
|
781
|
+
portfolio_value += position_value
|
|
782
|
+
|
|
757
783
|
self._portfolio_value = portfolio_value
|
|
758
784
|
return portfolio_value
|
|
759
785
|
|
|
@@ -1238,6 +1264,63 @@ class _Strategy:
|
|
|
1238
1264
|
if show_indicators is None:
|
|
1239
1265
|
show_indicators = SHOW_INDICATORS
|
|
1240
1266
|
|
|
1267
|
+
# Auto-select datasource from environment variable if None
|
|
1268
|
+
if datasource_class is None:
|
|
1269
|
+
from lumibot.credentials import BACKTESTING_DATA_SOURCE
|
|
1270
|
+
from lumibot.backtesting import (
|
|
1271
|
+
PolygonDataBacktesting,
|
|
1272
|
+
ThetaDataBacktesting,
|
|
1273
|
+
YahooDataBacktesting,
|
|
1274
|
+
AlpacaBacktesting,
|
|
1275
|
+
CcxtBacktesting,
|
|
1276
|
+
DataBentoDataBacktesting,
|
|
1277
|
+
)
|
|
1278
|
+
|
|
1279
|
+
datasource_map = {
|
|
1280
|
+
"polygon": PolygonDataBacktesting,
|
|
1281
|
+
"thetadata": ThetaDataBacktesting,
|
|
1282
|
+
"yahoo": YahooDataBacktesting,
|
|
1283
|
+
"alpaca": AlpacaBacktesting,
|
|
1284
|
+
"ccxt": CcxtBacktesting,
|
|
1285
|
+
"databento": DataBentoDataBacktesting,
|
|
1286
|
+
}
|
|
1287
|
+
|
|
1288
|
+
datasource_name = BACKTESTING_DATA_SOURCE.lower()
|
|
1289
|
+
if datasource_name not in datasource_map:
|
|
1290
|
+
raise ValueError(
|
|
1291
|
+
f"Unknown BACKTESTING_DATA_SOURCE: '{BACKTESTING_DATA_SOURCE}'. "
|
|
1292
|
+
f"Valid options: {list(datasource_map.keys())}"
|
|
1293
|
+
)
|
|
1294
|
+
|
|
1295
|
+
datasource_class = datasource_map[datasource_name]
|
|
1296
|
+
get_logger(__name__).info(colored(
|
|
1297
|
+
f"Auto-selected backtesting data source from BACKTESTING_DATA_SOURCE env var: {BACKTESTING_DATA_SOURCE}",
|
|
1298
|
+
"green"
|
|
1299
|
+
))
|
|
1300
|
+
|
|
1301
|
+
# Make sure polygon_api_key is set if using PolygonDataBacktesting
|
|
1302
|
+
polygon_api_key = polygon_api_key if polygon_api_key is not None else POLYGON_API_KEY
|
|
1303
|
+
if datasource_class.__name__ == 'PolygonDataBacktesting' and polygon_api_key is None:
|
|
1304
|
+
raise ValueError(
|
|
1305
|
+
"Please set `POLYGON_API_KEY` to your API key from polygon.io as an environment variable if "
|
|
1306
|
+
"you are using PolygonDataBacktesting. If you don't have one, you can get a free API key "
|
|
1307
|
+
"from https://polygon.io/."
|
|
1308
|
+
)
|
|
1309
|
+
|
|
1310
|
+
# Make sure thetadata_username and thetadata_password are set if using ThetaDataBacktesting
|
|
1311
|
+
if thetadata_username is None or thetadata_password is None:
|
|
1312
|
+
# Try getting the Theta Data credentials from credentials
|
|
1313
|
+
thetadata_username = THETADATA_CONFIG.get('THETADATA_USERNAME')
|
|
1314
|
+
thetadata_password = THETADATA_CONFIG.get('THETADATA_PASSWORD')
|
|
1315
|
+
|
|
1316
|
+
# Check again if theta data username and pass are set (before checking dict)
|
|
1317
|
+
if datasource_class.__name__ == 'ThetaDataBacktesting' and (thetadata_username is None or thetadata_password is None):
|
|
1318
|
+
raise ValueError(
|
|
1319
|
+
"Please set `thetadata_username` and `thetadata_password` in the backtest() function if "
|
|
1320
|
+
"you are using ThetaDataBacktesting. If you don't have one, you can do registeration "
|
|
1321
|
+
"from https://www.thetadata.net/."
|
|
1322
|
+
)
|
|
1323
|
+
|
|
1241
1324
|
# check if datasource_class is a class or a dictionary
|
|
1242
1325
|
if isinstance(datasource_class, dict):
|
|
1243
1326
|
optionsource_class = datasource_class["OPTION"]
|
|
@@ -1247,6 +1330,14 @@ class _Strategy:
|
|
|
1247
1330
|
use_other_option_source = False
|
|
1248
1331
|
else:
|
|
1249
1332
|
use_other_option_source = True
|
|
1333
|
+
|
|
1334
|
+
# Check ThetaData credentials for optionsource_class after dict extraction
|
|
1335
|
+
if optionsource_class.__name__ == 'ThetaDataBacktesting' and (thetadata_username is None or thetadata_password is None):
|
|
1336
|
+
raise ValueError(
|
|
1337
|
+
"Please set `thetadata_username` and `thetadata_password` in the backtest() function if "
|
|
1338
|
+
"you are using ThetaDataBacktesting. If you don't have one, you can do registeration "
|
|
1339
|
+
"from https://www.thetadata.net/."
|
|
1340
|
+
)
|
|
1250
1341
|
else:
|
|
1251
1342
|
optionsource_class = None
|
|
1252
1343
|
use_other_option_source = False
|
|
@@ -1277,29 +1368,6 @@ class _Strategy:
|
|
|
1277
1368
|
|
|
1278
1369
|
self.verify_backtest_inputs(backtesting_start, backtesting_end)
|
|
1279
1370
|
|
|
1280
|
-
# Make sure polygon_api_key is set if using PolygonDataBacktesting
|
|
1281
|
-
polygon_api_key = polygon_api_key if polygon_api_key is not None else POLYGON_API_KEY
|
|
1282
|
-
if datasource_class == PolygonDataBacktesting and polygon_api_key is None:
|
|
1283
|
-
raise ValueError(
|
|
1284
|
-
"Please set `POLYGON_API_KEY` to your API key from polygon.io as an environment variable if "
|
|
1285
|
-
"you are using PolygonDataBacktesting. If you don't have one, you can get a free API key "
|
|
1286
|
-
"from https://polygon.io/."
|
|
1287
|
-
)
|
|
1288
|
-
|
|
1289
|
-
# Make sure thetadata_username and thetadata_password are set if using ThetaDataBacktesting
|
|
1290
|
-
if thetadata_username is None or thetadata_password is None:
|
|
1291
|
-
# Try getting the Theta Data credentials from credentials
|
|
1292
|
-
thetadata_username = THETADATA_CONFIG.get('THETADATA_USERNAME')
|
|
1293
|
-
thetadata_password = THETADATA_CONFIG.get('THETADATA_PASSWORD')
|
|
1294
|
-
|
|
1295
|
-
# Check again if theta data username and pass are set
|
|
1296
|
-
if (thetadata_username is None or thetadata_password is None) and (datasource_class == ThetaDataBacktesting or optionsource_class == ThetaDataBacktesting):
|
|
1297
|
-
raise ValueError(
|
|
1298
|
-
"Please set `thetadata_username` and `thetadata_password` in the backtest() function if "
|
|
1299
|
-
"you are using ThetaDataBacktesting. If you don't have one, you can do registeration "
|
|
1300
|
-
"from https://www.thetadata.net/."
|
|
1301
|
-
)
|
|
1302
|
-
|
|
1303
1371
|
if not self.IS_BACKTESTABLE:
|
|
1304
1372
|
get_logger(__name__).warning(f"Strategy {name + ' ' if name is not None else ''}cannot be " f"backtested at the moment")
|
|
1305
1373
|
return None
|
|
@@ -1323,7 +1391,7 @@ class _Strategy:
|
|
|
1323
1391
|
|
|
1324
1392
|
self._trader = trader_class(logfile=logfile, backtest=True, quiet_logs=quiet_logs)
|
|
1325
1393
|
|
|
1326
|
-
if datasource_class == PolygonDataBacktesting:
|
|
1394
|
+
if datasource_class.__name__ == 'PolygonDataBacktesting':
|
|
1327
1395
|
data_source = datasource_class(
|
|
1328
1396
|
backtesting_start,
|
|
1329
1397
|
backtesting_end,
|
|
@@ -1336,7 +1404,7 @@ class _Strategy:
|
|
|
1336
1404
|
log_backtest_progress_to_file=LOG_BACKTEST_PROGRESS_TO_FILE,
|
|
1337
1405
|
**kwargs,
|
|
1338
1406
|
)
|
|
1339
|
-
elif datasource_class == ThetaDataBacktesting or optionsource_class == ThetaDataBacktesting:
|
|
1407
|
+
elif datasource_class.__name__ == 'ThetaDataBacktesting' or (optionsource_class and optionsource_class.__name__ == 'ThetaDataBacktesting'):
|
|
1340
1408
|
data_source = datasource_class(
|
|
1341
1409
|
backtesting_start,
|
|
1342
1410
|
backtesting_end,
|
lumibot/strategies/strategy.py
CHANGED
|
@@ -374,11 +374,10 @@ class Strategy(_Strategy):
|
|
|
374
374
|
# Send the message to Discord
|
|
375
375
|
self.send_discord_message(message)
|
|
376
376
|
|
|
377
|
-
#
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
# Check if INFO level is enabled before logging
|
|
377
|
+
# Performance optimization: skip logging if INFO is not enabled
|
|
378
|
+
# This respects BACKTESTING_QUIET_LOGS via StrategyLoggerAdapter.isEnabledFor()
|
|
379
|
+
# When BACKTESTING_QUIET_LOGS=true (default), this returns False and saves CPU cycles
|
|
380
|
+
# When BACKTESTING_QUIET_LOGS=false, this returns True and logs are displayed
|
|
382
381
|
if not self.logger.isEnabledFor(logging.INFO):
|
|
383
382
|
return
|
|
384
383
|
|
|
@@ -4411,7 +4410,7 @@ class Strategy(_Strategy):
|
|
|
4411
4410
|
save_logfile: bool = False,
|
|
4412
4411
|
thetadata_username: str = None,
|
|
4413
4412
|
thetadata_password: str = None,
|
|
4414
|
-
use_quote_data: bool =
|
|
4413
|
+
use_quote_data: bool = True, # Changed to True for ThetaData options support
|
|
4415
4414
|
show_progress_bar: bool = True,
|
|
4416
4415
|
quiet_logs: bool = True,
|
|
4417
4416
|
trader_class: Type[Trader] = Trader,
|