bbstrader 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of bbstrader might be problematic. Click here for more details.

bbstrader/__init__.py CHANGED
@@ -7,7 +7,7 @@ __author__ = "Bertin Balouki SIMYELI"
7
7
  __copyright__ = "2023-2025 Bertin Balouki SIMYELI"
8
8
  __email__ = "bertin@bbstrader.com"
9
9
  __license__ = "MIT"
10
- __version__ = "0.2.0"
10
+ __version__ = "0.3.2"
11
11
 
12
12
  from bbstrader import compat # noqa: F401
13
13
  from bbstrader import core # noqa: F401
bbstrader/__main__.py CHANGED
@@ -1,4 +1,5 @@
1
1
  import argparse
2
+ import multiprocessing
2
3
  import sys
3
4
  from enum import Enum
4
5
 
@@ -49,20 +50,25 @@ def main():
49
50
  if ("-h" in sys.argv or "--help" in sys.argv) and args.run is None:
50
51
  print(Fore.WHITE + USAGE_TEXT)
51
52
  sys.exit(0)
52
-
53
- match args.run:
54
- case Module.COPIER.value:
55
- copy_trades(unknown)
56
- case Module.BACKTEST.value:
57
- backtest(unknown)
58
- case Module.EXECUTION.value:
59
- execute_strategy(unknown)
60
- case Module.NEWS_FEED.value:
61
- send_news_feed(unknown)
62
- case _:
63
- print(Fore.RED + f"Unknown module: {args.run}")
64
- sys.exit(1)
53
+ try:
54
+ match args.run:
55
+ case Module.COPIER.value:
56
+ copy_trades(unknown)
57
+ case Module.BACKTEST.value:
58
+ backtest(unknown)
59
+ case Module.EXECUTION.value:
60
+ execute_strategy(unknown)
61
+ case Module.NEWS_FEED.value:
62
+ send_news_feed(unknown)
63
+ case _:
64
+ print(Fore.RED + f"Unknown module: {args.run}")
65
+ sys.exit(1)
66
+ except KeyboardInterrupt:
67
+ sys.exit(0)
68
+ except Exception:
69
+ sys.exit(1)
65
70
 
66
71
 
67
72
  if __name__ == "__main__":
73
+ multiprocessing.freeze_support()
68
74
  main()
@@ -248,14 +248,13 @@ def run_backtest(
248
248
 
249
249
  start_date (datetime): Start date of the backtest.
250
250
 
251
- data_handler (DataHandler): An instance of the `DataHandler` class, responsible for managing
251
+ data_handler (DataHandler): A subclass of the `DataHandler` class, responsible for managing
252
252
  and processing market data. Available options include `CSVDataHandler`,
253
- `MT5DataHandler`, and `YFDataHandler`. Ensure that the `DataHandler`
254
- instance is initialized before passing it to the function.
253
+ `MT5DataHandler`, and `YFDataHandler`.
255
254
 
256
255
  strategy (Strategy): The trading strategy to be employed during the backtest.
257
- The strategy must be an instance of `Strategy` and should include the following attributes:
258
- - `bars` (DataHandler): The `DataHandler` instance for the strategy.
256
+ The strategy must be a subclass of `Strategy` and should include the following attributes:
257
+ - `bars` (DataHandler): The `DataHandler` class for the strategy.
259
258
  - `events` (Queue): Queue instance for managing events.
260
259
  - `symbol_list` (List[str]): List of symbols to trade.
261
260
  - `mode` (str): 'live' or 'backtest'.
@@ -307,9 +306,9 @@ def run_backtest(
307
306
  >>> run_backtest(
308
307
  ... symbol_list=symbol_list,
309
308
  ... start_date=start,
310
- ... data_handler=MT5DataHandler(),
311
- ... strategy=StockIndexSTBOTrading(),
312
- ... exc_handler=MT5ExecutionHandler(),
309
+ ... data_handler=MT5DataHandler,
310
+ ... strategy=StockIndexSTBOTrading,
311
+ ... exc_handler=MT5ExecutionHandler,
313
312
  ... initial_capital=100000.0,
314
313
  ... heartbeat=0.0,
315
314
  ... **kwargs
@@ -98,7 +98,7 @@ class SimExecutionHandler(ExecutionHandler):
98
98
  self.events.put(fill_event)
99
99
  self.logger.info(
100
100
  f"{event.direction} ORDER FILLED: SYMBOL={event.symbol}, "
101
- f"QUANTITY={event.quantity}, PRICE @{event.price} EXCHANGE={fill_event.exchange}",
101
+ f"QUANTITY={event.quantity}, PRICE @{round(event.price, 5)} EXCHANGE={fill_event.exchange}",
102
102
  custom_time=fill_event.timeindex,
103
103
  )
104
104
 
@@ -264,7 +264,7 @@ class MT5ExecutionHandler(ExecutionHandler):
264
264
  self.events.put(fill_event)
265
265
  self.logger.info(
266
266
  f"{direction} ORDER FILLED: SYMBOL={symbol}, QUANTITY={quantity}, "
267
- f"PRICE @{price} EXCHANGE={fill_event.exchange}",
267
+ f"PRICE @{round(event.price, 5)} EXCHANGE={fill_event.exchange}",
268
268
  custom_time=fill_event.timeindex,
269
269
  )
270
270
 
@@ -12,17 +12,18 @@ from loguru import logger
12
12
  from bbstrader.btengine.data import DataHandler
13
13
  from bbstrader.btengine.event import Events, FillEvent, SignalEvent
14
14
  from bbstrader.config import BBSTRADER_DIR
15
- from bbstrader.metatrader.account import (
15
+ from bbstrader.metatrader import (
16
16
  Account,
17
17
  AdmiralMarktsGroup,
18
18
  PepperstoneGroupLimited,
19
+ TradeOrder,
20
+ Rates,
21
+ TradeSignal,
22
+ TradingMode,
23
+ SymbolType
19
24
  )
20
- from bbstrader.metatrader.utils import SymbolType
21
- from bbstrader.metatrader.rates import Rates
22
- from bbstrader.metatrader.trade import TradeSignal, TradingMode
23
25
  from bbstrader.models.optimization import optimized_weights
24
26
 
25
-
26
27
  __all__ = ["Strategy", "MT5Strategy"]
27
28
 
28
29
  logger.add(
@@ -71,8 +72,13 @@ class MT5Strategy(Strategy):
71
72
  calculate signals for the MetaTrader 5 trading platform. The signals
72
73
  are generated by the `MT5Strategy` object and sent to the the `Mt5ExecutionEngine`
73
74
  for live trading and `MT5BacktestEngine` objects for backtesting.
74
- """
75
75
 
76
+ # NOTE
77
+ It is recommanded that every strategy specfic method to be a private method
78
+ in order to avoid naming collusion.
79
+ """
80
+ tf: str
81
+ max_trades: Dict[str, int]
76
82
  def __init__(
77
83
  self,
78
84
  events: Queue = None,
@@ -107,9 +113,11 @@ class MT5Strategy(Strategy):
107
113
  self._initialize_portfolio()
108
114
  self.kwargs = kwargs
109
115
  self.periodes = 0
110
-
111
- @property
116
+
117
+ @property
112
118
  def account(self):
119
+ if self.mode != TradingMode.LIVE:
120
+ raise ValueError("account attribute is only allowed in Live mode")
113
121
  return Account(**self.kwargs)
114
122
 
115
123
  @property
@@ -127,7 +135,7 @@ class MT5Strategy(Strategy):
127
135
  @property
128
136
  def orders(self):
129
137
  if self.mode == TradingMode.LIVE:
130
- return self.account.get_orders()
138
+ return self.account.get_orders() or []
131
139
  return self._orders
132
140
 
133
141
  @property
@@ -139,7 +147,7 @@ class MT5Strategy(Strategy):
139
147
  @property
140
148
  def positions(self):
141
149
  if self.mode == TradingMode.LIVE:
142
- return self.account.get_positions()
150
+ return self.account.get_positions() or []
143
151
  return self._positions
144
152
 
145
153
  @property
@@ -345,7 +353,7 @@ class MT5Strategy(Strategy):
345
353
  log = True
346
354
  if log:
347
355
  self.logger.info(
348
- f"{signal} ORDER EXECUTED: SYMBOL={symbol}, QUANTITY={quantity}, PRICE @{price}",
356
+ f"{signal} ORDER EXECUTED: SYMBOL={symbol}, QUANTITY={quantity}, PRICE @{round(price, 5)}",
349
357
  custom_time=dtime,
350
358
  )
351
359
 
@@ -526,7 +534,7 @@ class MT5Strategy(Strategy):
526
534
  def logmsg(order, type, symbol, dtime):
527
535
  return self.logger.info(
528
536
  f"{type} ORDER EXECUTED: SYMBOL={symbol}, QUANTITY={order.quantity}, "
529
- f"PRICE @ {order.price}",
537
+ f"PRICE @ {round(order.price, 5)}",
530
538
  custom_time=dtime,
531
539
  )
532
540
 
@@ -614,7 +622,7 @@ class MT5Strategy(Strategy):
614
622
  )
615
623
 
616
624
  @staticmethod
617
- def calculate_pct_change(current_price, lh_price):
625
+ def calculate_pct_change(current_price, lh_price) -> float:
618
626
  return ((current_price - lh_price) / lh_price) * 100
619
627
 
620
628
  def get_asset_values(
@@ -649,8 +657,8 @@ class MT5Strategy(Strategy):
649
657
  In Live mode, the `bbstrader.metatrader.rates.Rates` class is used to get the historical data
650
658
  so the value_type must be 'returns', 'open', 'high', 'low', 'close', 'adjclose', 'volume'.
651
659
  """
652
- if mode not in ["backtest", "live"]:
653
- raise ValueError("Mode must be either backtest or live.")
660
+ if mode not in [TradingMode.BACKTEST, TradingMode.LIVE]:
661
+ raise ValueError("Mode must be an instance of TradingMode")
654
662
  asset_values = {}
655
663
  if mode == TradingMode.BACKTEST:
656
664
  if bars is None:
@@ -696,7 +704,7 @@ class MT5Strategy(Strategy):
696
704
  if period_count == 0 or period_count is None:
697
705
  return True
698
706
  return period_count % signal_inverval == 0
699
-
707
+
700
708
  @staticmethod
701
709
  def stop_time(time_zone: str, stop_time: str) -> bool:
702
710
  now = datetime.now(pytz.timezone(time_zone)).time()
@@ -760,6 +768,47 @@ class MT5Strategy(Strategy):
760
768
  )
761
769
  return prices
762
770
  return np.array([])
771
+
772
+ def get_active_orders(self, symbol: str, strategy_id: int, order_type: int = None) -> List[TradeOrder]:
773
+ """
774
+ Get the active orders for a given symbol and strategy.
775
+
776
+ Args:
777
+ symbol : The symbol for the trade.
778
+ strategy_id : The unique identifier for the strategy.
779
+ order_type : The type of order to filter by (optional):
780
+ "BUY_LIMIT": 2
781
+ "SELL_LIMIT": 3
782
+ "BUY_STOP": 4
783
+ "SELL_STOP": 5
784
+ "BUY_STOP_LIMIT": 6
785
+ "SELL_STOP_LIMIT": 7
786
+
787
+ Returns:
788
+ List[TradeOrder] : A list of active orders for the given symbol and strategy.
789
+ """
790
+ orders = [o for o in self.orders if o.symbol == symbol and o.magic == strategy_id]
791
+ if order_type is not None and len(orders) > 0:
792
+ orders = [o for o in orders if o.type == order_type]
793
+ return orders
794
+
795
+ def exit_positions(self, position, prices, asset, th: float = 0.01):
796
+ if len(prices) == 0:
797
+ return False
798
+ tick_info = self.account.get_tick_info(asset)
799
+ bid, ask = tick_info.bid, tick_info.ask
800
+ if len(prices) == 1:
801
+ price = prices[0]
802
+ elif len(prices) in range(2, self.max_trades[asset] + 1):
803
+ price = np.mean(prices)
804
+ if (
805
+ position == 0
806
+ and self.calculate_pct_change(ask, price) >= th
807
+ or position == 1
808
+ and abs(self.calculate_pct_change(bid, price)) >= th
809
+ ):
810
+ return True
811
+ return False
763
812
 
764
813
  @staticmethod
765
814
  def get_current_dt(time_zone: str = "US/Eastern") -> datetime:
@@ -796,7 +845,9 @@ class MT5Strategy(Strategy):
796
845
  return dt_to
797
846
 
798
847
  @staticmethod
799
- def get_mt5_equivalent(symbols, symbol_type: str | SymbolType = SymbolType.STOCKS, **kwargs) -> List[str]:
848
+ def get_mt5_equivalent(
849
+ symbols, symbol_type: str | SymbolType = SymbolType.STOCKS, **kwargs
850
+ ) -> List[str]:
800
851
  """
801
852
  Get the MetaTrader 5 equivalent symbols for the symbols in the list.
802
853
  This method is used to get the symbols that are available on the MetaTrader 5 platform.
bbstrader/config.py CHANGED
@@ -3,8 +3,8 @@ from pathlib import Path
3
3
  from typing import List
4
4
 
5
5
 
6
- TERMINAL = "\\terminal64.exe"
7
- BASE_FOLDER = "C:\\Program Files\\"
6
+ TERMINAL = "/terminal64.exe"
7
+ BASE_FOLDER = "C:/Program Files/"
8
8
 
9
9
  AMG_PATH = BASE_FOLDER + "Admirals Group MT5 Terminal" + TERMINAL
10
10
  PGL_PATH = BASE_FOLDER + "Pepperstone MetaTrader 5" + TERMINAL
bbstrader/core/data.py CHANGED
@@ -149,7 +149,7 @@ class FmpNews(object):
149
149
  try:
150
150
  articles = pd.read_csv("latest_fmp_articles.csv")
151
151
  articles = articles.to_dict(orient="records")
152
- if self._last_date(articles[0]["date"]) < end_date:
152
+ if self._last_date(articles[0]["date"]).hour < end_date.hour:
153
153
  articles = self.get_articles(**kwargs)
154
154
  else:
155
155
  return articles
@@ -161,7 +161,9 @@ class FmpNews(object):
161
161
  df.to_csv("latest_fmp_articles.csv", index=False)
162
162
  return articles
163
163
 
164
- def get_news(self, query, source="articles", articles=None, symbol=None, **kwargs):
164
+ def get_news(
165
+ self, query, source="articles", articles=None, symbol: str = None, **kwargs
166
+ ):
165
167
  """
166
168
  Retrieves relevant financial news based on the specified source.
167
169
 
@@ -183,6 +185,10 @@ class FmpNews(object):
183
185
  Returns an empty list if no relevant news is found.
184
186
  """
185
187
  query = _get_search_query(query)
188
+ if symbol is not None:
189
+ symbol = symbol.replace("-", "").split("=")[
190
+ 0
191
+ ] # if symbol is a yahoo finance ticker
186
192
  source_methods = {
187
193
  "articles": lambda: self.get_latest_articles(articles=articles, save=True),
188
194
  "releases": lambda: self.get_releases(symbol=symbol, **kwargs),
@@ -191,6 +197,8 @@ class FmpNews(object):
191
197
  "forex": lambda: self.get_forex_news(symbol=symbol, **kwargs),
192
198
  }
193
199
  news_source = source_methods.get(source, lambda: [])()
200
+ if source == "articles":
201
+ symbol = None # Articles do not require a symbol filter
194
202
  news = self.parse_news(news_source, symbol=symbol)
195
203
  return _filter_news(news, query)
196
204
 
@@ -203,7 +211,7 @@ class FinancialNews(object):
203
211
 
204
212
  """
205
213
 
206
- def _fetch_news(self, url, query, asset_type, n_news, headline_tag) -> List[str]:
214
+ def _fetch_news(self, url, query, n_news, headline_tag) -> List[str]:
207
215
  headers = {"User-Agent": "Mozilla/5.0"}
208
216
  try:
209
217
  response = requests.get(url, headers=headers)
@@ -224,32 +232,60 @@ class FinancialNews(object):
224
232
  ]
225
233
  return headlines[:n_news]
226
234
 
227
- def get_yahoo_finance_news(self, query, asset_type="stock", n_news=10):
235
+ def get_yahoo_finance_news(self, query: str, asset_type="stock", n_news=10):
228
236
  """
229
237
  Fetches recent Yahoo Finance news headlines for a given financial asset.
230
238
 
231
239
  Args:
232
240
  query (str): The asset symbol or name (e.g., "AAPL").
233
- asset_type (str, optional): The type of asset (e.g., "stock", "etf"). Defaults to "stock".
241
+ asset_type (str, optional): The type of asset (e.g., "stock", "etf"). Defaults to "stock",
242
+ supported types include:
243
+ - "stock": Stock symbols (e.g., AAPL, MSFT)
244
+ - "etf": Exchange-traded funds (e.g., SPY, QQQ)
245
+ - "future": Futures contracts (e.g., CL=F for crude oil)
246
+ - "forex": Forex pairs (e.g., EURUSD=X, USDJPY=X)
247
+ - "crypto": Cryptocurrency pairs (e.g., BTC-USD, ETH-USD)
248
+ - "index": Stock market indices (e.g., ^GSPC for S&P 500)
234
249
  n_news (int, optional): The number of news headlines to return. Defaults to 10.
235
250
 
251
+ Note:
252
+ For commotities and bonds, use the "Future" asset type.
253
+
236
254
  Returns:
237
255
  list[str]: A list of Yahoo Finance news headlines relevant to the query.
238
256
  """
257
+ if asset_type == "forex" or asset_type == "future":
258
+ assert (
259
+ "=" in query
260
+ ), "Forex query must contain '=' for currency pairs (e.g., EURUSD=X, CL=F)"
261
+ if asset_type == "crypto":
262
+ assert (
263
+ "-" in query
264
+ ), "Crypto query must contain '-' for crypto pairs (e.g., BTC-USD, ETH-USD)"
265
+ if asset_type == "index":
266
+ assert query.startswith(
267
+ "^"
268
+ ), "Index query must start with '^' (e.g., ^GSPC for S&P 500)"
239
269
  url = (
240
270
  f"https://finance.yahoo.com/quote/{query}/news"
241
- if asset_type in ["stock", "etf"]
271
+ if asset_type in ["stock", "etf", "index", "future", "forex"]
242
272
  else "https://finance.yahoo.com/news"
243
273
  )
244
- return self._fetch_news(url, query, asset_type, n_news, "h3")
274
+ return self._fetch_news(url, query, n_news, "h3")
245
275
 
246
- def get_google_finance_news(self, query, asset_type="stock", n_news=10):
276
+ def get_google_finance_news(self, query: str, asset_type="stock", n_news=10):
247
277
  """
248
278
  Fetches recent Google Finance news headlines for a given financial asset.
249
279
 
250
280
  Args:
251
281
  query (str): The asset symbol or name (e.g., "AAPL").
252
282
  asset_type (str, optional): The type of asset (e.g., "stock", "crypto"). Defaults to "stock".
283
+ Supported types include:
284
+ - "stock": Stock symbols (e.g., AAPL, MSFT)
285
+ - "etf": Exchange-traded funds (e.g., SPY, QQQ)
286
+ - "future": Futures contracts (e.g., CL=F or crude oil)
287
+ - "forex": Forex pairs (e.g., EURUSD, USDJPY)
288
+ - "crypto": Cryptocurrency pairs (e.g., BTCUSD, ETHUSD)
253
289
  n_news (int, optional): The number of news headlines to return. Defaults to 10.
254
290
 
255
291
  Returns:
@@ -258,20 +294,18 @@ class FinancialNews(object):
258
294
  search_terms = {
259
295
  "stock": f"{query} stock OR {query} shares OR {query} market",
260
296
  "etf": f"{query} ETF OR {query} fund OR {query} exchange-traded fund",
261
- "futures": f"{query} futures OR {query} price OR {query} market",
262
- "commodity": f"{query} price OR {query} futures OR {query} market",
297
+ "future": f"{query} futures OR {query} price OR {query} market",
263
298
  "forex": f"{query} forex OR {query} exchange rate OR {query} market",
264
299
  "crypto": f"{query} cryptocurrency OR {query} price OR {query} market",
265
- "bond": f"{query} bond OR {query} yield OR {query} interest rate",
266
300
  "index": f"{query} index OR {query} stock market OR {query} performance",
267
301
  }
268
302
  search_query = search_terms.get(asset_type, query)
269
303
  url = f"https://news.google.com/search?q={search_query.replace(' ', '+')}"
270
- return self._fetch_news(url, query, asset_type, n_news, "a")
304
+ return self._fetch_news(url, query, n_news, "a")
271
305
 
272
306
  def get_reddit_posts(
273
307
  self,
274
- symbol,
308
+ symbol: str,
275
309
  client_id=None,
276
310
  client_secret=None,
277
311
  user_agent=None,
@@ -294,8 +328,8 @@ class FinancialNews(object):
294
328
  - "stock": Searches in stock-related subreddits (e.g., wallstreetbets, stocks).
295
329
  - "forex": Searches in forex-related subreddits.
296
330
  - "commodities": Searches in commodity-related subreddits (e.g., gold, oil).
297
- - "etfs": Searches in ETF-related subreddits.
298
- - "futures": Searches in futures and options trading subreddits.
331
+ - "etf": Searches in ETF-related subreddits.
332
+ - "future": Searches in futures and options trading subreddits.
299
333
  - "crypto": Searches in cryptocurrency-related subreddits.
300
334
  - If an unrecognized asset class is provided, defaults to stock-related subreddits.
301
335
  n_posts (int, optional): The number of posts to return per subreddit. Defaults to 10.
@@ -317,15 +351,36 @@ class FinancialNews(object):
317
351
  """
318
352
 
319
353
  reddit = praw.Reddit(
320
- client_id=client_id, client_secret=client_secret, user_agent=user_agent
354
+ client_id=client_id,
355
+ client_secret=client_secret,
356
+ user_agent=user_agent,
357
+ check_for_updates=False,
358
+ comment_kind="t1",
359
+ message_kind="t4",
360
+ redditor_kind="t2",
361
+ submission_kind="t3",
362
+ subreddit_kind="t5",
363
+ trophy_kind="t6",
364
+ oauth_url="https://oauth.reddit.com",
365
+ reddit_url="https://www.reddit.com",
366
+ short_url="https://redd.it",
367
+ timeout=16,
368
+ ratelimit_seconds=5,
321
369
  )
322
-
370
+ assert reddit.read_only
323
371
  subreddit_mapping = {
324
372
  "stock": ["wallstreetbets", "stocks", "investing", "StockMarket"],
325
373
  "forex": ["Forex", "ForexTrading", "DayTrading"],
326
- "commodities": ["Commodities", "Gold", "Silverbugs", "oil"],
327
374
  "etfs": ["ETFs", "investing"],
328
- "futures": ["FuturesTrading", "OptionsTrading", "DayTrading"],
375
+ "futures": [
376
+ "FuturesTrading",
377
+ "OptionsTrading",
378
+ "DayTrading",
379
+ "Commodities",
380
+ "Gold",
381
+ "Silverbugs",
382
+ "oil",
383
+ ],
329
384
  "crypto": ["CryptoCurrency", "Bitcoin", "ethereum", "altcoin"],
330
385
  }
331
386
  try:
@@ -473,16 +528,23 @@ class FinancialNews(object):
473
528
  maximum = 100
474
529
  if limit > maximum:
475
530
  raise ValueError(f"Number of total news articles allowed is {maximum}")
476
-
477
- response = requests.get(
478
- "https://data-api.coindesk.com/news/v1/article/list",
479
- params={"lang": lang, "limit": limit},
480
- headers={"Content-type": "application/json; charset=UTF-8"},
481
- )
482
- json_response = response.json()
483
- articles = json_response["Data"]
484
- if len(articles) == 0:
531
+ try:
532
+ response = requests.get(
533
+ "https://data-api.coindesk.com/news/v1/article/list",
534
+ params={"lang": lang, "limit": limit},
535
+ headers={"Content-type": "application/json; charset=UTF-8"},
536
+ )
537
+ response.raise_for_status()
538
+ json_response = response.json()
539
+ except requests.exceptions.RequestException:
485
540
  return []
541
+ if (
542
+ response.status_code != 200
543
+ or "Data" not in json_response
544
+ or len(json_response["Data"]) == 0
545
+ ):
546
+ return []
547
+ articles = json_response["Data"]
486
548
  to_keep = [
487
549
  "PUBLISHED_ON",
488
550
  "TITLE",
@@ -495,10 +557,11 @@ class FinancialNews(object):
495
557
  ]
496
558
  filtered_articles = []
497
559
  for article in articles:
560
+ keys = article.keys()
498
561
  filtered_articles.append(
499
562
  {
500
563
  k.lower(): article[k]
501
- if k in article and k != "PUBLISHED_ON"
564
+ if k in keys and k != "PUBLISHED_ON"
502
565
  else datetime.fromtimestamp(article[k])
503
566
  for k in to_keep
504
567
  if article[k] is not None and "sponsored" not in str(article[k])