lumibot 4.0.23__py3-none-any.whl → 4.1.0__py3-none-any.whl

lumibot/components/grok_news_helper.py

@@ -0,0 +1,284 @@
+import json
+import datetime
+from openai import OpenAI
+
+class GrokNewsHelper:
+    """
+    GrokNewsHelper queries xAI's Grok API via the OpenAI-compatible client to fetch 
+    financial/news data in a structured JSON format.
+
+    How it Works:
+    -------------
+    1. Build a system prompt describing the desired JSON schema in detail.
+    2. Use xAI's OpenAI-compatible client to create a chat completion with:
+       - A system message (detailed instructions).
+       - A user message (the actual user query).
+    3. Parse the assistant's text content as JSON. 
+    4. Return a Python dictionary with fields:
+       {
+         "query": ...,
+         "timestamp_utc": ...,
+         "analysis_summary": ...,
+         "items": [...]
+       }
+
+    Example:
+    --------
+    from grok_news_helper import GrokNewsHelper
+
+    helper = GrokNewsHelper(api_key="YOUR_XAI_API_KEY")
+    result = helper.execute_query("What stocks are trending right now on Twitter?")
+    print(json.dumps(result, indent=2))
+    """
+
+    def __init__(self, api_key: str):
+        """
+        Initialize the GrokNewsHelper with your xAI API key. 
+        Internally creates an xAI OpenAI-compatible client.
+
+        Parameters
+        ----------
+        api_key : str
+            The xAI API key (required).
+        """
+        if not api_key:
+            # Try to get the API key from the environment if not provided
+            import os
+
+            # Might be called GROK_API_KEY or XAI_API_KEY
+            api_key = os.getenv("GROK_API_KEY")
+            if not api_key:
+                api_key = os.getenv("XAI_API_KEY")
+
+            if not api_key:
+                raise ValueError("API key is required for GrokNewsHelper. Get one from x.ai and set it as GROK_API_KEY or XAI_API_KEY in your secrets, environment variables, .env file or directly in the code.")
+        
+        self.api_key = api_key
+
+        # Create the xAI client (OpenAI-compatible) 
+        self.client = OpenAI(
+            api_key=self.api_key,
+            base_url="https://api.x.ai/v1",
+        )
+
+    def build_prompt(self, user_query: str) -> str:
+        """
+        Constructs a system prompt describing the required JSON schema in detail
+        and instructing the model to avoid hallucinations and remain factual.
+
+        The schema includes fields like 'symbol', 'confidence', 'magnitude', etc.
+        Now also includes an optional 'price_targets' object with float fields.
+
+        Parameters
+        ----------
+        user_query : str
+            The question or prompt about financial/news data.
+
+        Returns
+        -------
+        str
+            The system message content with instructions for Grok.
+        """
+        system_prompt = f"""\
+You are a financial news aggregator assistant with real-time access to Twitter, news feeds, 
+and other finance data. Remain factual and accurate in your response (avoid hallucinations).
+
+Your task is to return ONLY valid JSON, following this exact schema:
+
+JSON Schema:
+{{
+  "query": "<string, required - echo the user's query>",
+  "timestamp_utc": "<string, required - current UTC time in ISO 8601 format>",
+  "analysis_summary": "<string, required - short summary of the findings>",
+  "items": [
+    {{
+      "symbol": "<string, required - e.g. 'AAPL', 'BTC'. Always neds to be a valid symbol>",
+      "asset_type": "<string, required - 'stock', 'crypto', 'index', 'commodity', 'forex', 'none'>",
+      "headline": "<string, required - short note on why it's trending>",
+      "confidence": "<integer, 0-10, required - reliability score>",
+      "sentiment_score": "<integer, -10 to 10, required - negative is bearish, positive is bullish>",
+      "popularity_metric": "<integer >= 0, required - measure of mentions>",
+      "volume_of_messages": "<integer, optional - if known>",
+      "magnitude": "<integer, 0-10, required - overall impact level>",
+      "type_of_news": "<string, optional - e.g. 'earnings', 'ipo', 'macro', 'ceo_tweet'>",
+      "price_targets": {{
+        "low": "<float, optional>",
+        "high": "<float, optional>",
+        "average": "<float, optional>"
+      }},
+      "additional_info": {{
+        "sector": "<string, optional>",
+        "recent_events": "<string, optional>",
+        "notable_executive_actions": "<string, optional>",
+        "macro_support": "<string, optional>",
+        "related_tickers": ["<string>", "..."],
+        "external_links": ["<string>", "..."]
+      }}
+    }}
+  ]
+}}
+
+Instructions:
+1) Output MUST be valid JSON (no extra text or markdown).
+2) If no data found, 'items' can be empty.
+3) The 'magnitude' field is required (0-10).
+4) Provide minimal text in 'analysis_summary' (1-3 sentences).
+5) The field 'query' must repeat the user's question exactly.
+
+Now, the user's query is:
+\"{user_query}\"
+
+Return only valid JSON following the schema.
+"""
+        return system_prompt
+
+    def execute_query(self, user_query: str) -> dict:
+        """
+        Executes a query by creating a chat completion via the xAI OpenAI-compatible client.
+        We pass a system message (JSON schema instructions) + user message (user_query).
+
+        Steps:
+        ------
+        1) Build the system prompt with `build_prompt(user_query)`.
+        2) Call `client.chat.completions.create(...)` with:
+           - model = "grok-2-latest" (or any available model)
+           - messages (system + user).
+           - temperature=0 to reduce hallucinations.
+        3) Parse the assistant's output text as JSON.
+        4) Return a dictionary matching the schema. If parsing fails, return an error structure.
+
+        Parameters
+        ----------
+        user_query : str
+            The textual query about financial/news data (e.g. "Which stocks are trending now?").
+
+        Returns
+        -------
+        dict
+            A dictionary conforming to the described JSON schema:
+            {
+              "query": "...",
+              "timestamp_utc": "...",
+              "analysis_summary": "...",
+              "items": [...]
+            }
+        """
+        # 1) Build system prompt
+        system_msg = self.build_prompt(user_query)
+
+        # 2) Create the chat completion using the xAI client
+        try:
+            completion = self.client.chat.completions.create(
+                model="grok-2-latest",
+                messages=[
+                    {"role": "system", "content": system_msg},
+                    {"role": "user", "content": user_query}
+                ],
+                temperature=0,
+                stream=False
+            )
+        except Exception as e:
+            # If there's a network or API error, fallback with an error structure
+            return {
+                "query": user_query,
+                "timestamp_utc": datetime.datetime.now(datetime.UTC).isoformat()
+                    if hasattr(datetime, 'UTC') else datetime.datetime.utcnow().isoformat(),
+                "analysis_summary": f"Error calling xAI API: {str(e)}",
+                "items": []
+            }
+
+        # 3) Extract the assistant's text (the content of the first choice)
+        try:
+            assistant_text = completion.choices[0].message.content
+        except (AttributeError, IndexError, KeyError) as e:
+            return {
+                "query": user_query,
+                "timestamp_utc": datetime.datetime.now(datetime.UTC).isoformat()
+                    if hasattr(datetime, 'UTC') else datetime.datetime.utcnow().isoformat(),
+                "analysis_summary": f"Error: Unexpected response format from Grok API. {str(e)}",
+                "items": []
+            }
+
+        # 4) Parse the assistant's text as JSON
+        try:
+            data = json.loads(assistant_text)
+        except json.JSONDecodeError as e:
+            return {
+                "query": user_query,
+                "timestamp_utc": datetime.datetime.now(datetime.UTC).isoformat()
+                    if hasattr(datetime, 'UTC') else datetime.datetime.utcnow().isoformat(),
+                "analysis_summary": f"Error: LLM output was not valid JSON. {str(e)}",
+                "items": []
+            }
+
+        # 5) Post-process to ensure numeric fields are integers
+        self._post_process_data(data)
+
+        return data
+
+    def _post_process_data(self, data: dict) -> None:
+        """
+        Enforces that certain fields are integers:
+         - confidence
+         - sentiment_score
+         - popularity_metric
+         - magnitude
+         - volume_of_messages (if present)
+
+        Also handles optional price_targets as floats if present.
+
+        Modifies the 'data' dictionary in-place.
+
+        Parameters
+        ----------
+        data : dict
+            The parsed JSON dictionary from the assistant.
+        """
+        items = data.get("items", [])
+        for item in items:
+            for int_field in ("confidence", "sentiment_score", "popularity_metric", "magnitude"):
+                if int_field in item:
+                    try:
+                        item[int_field] = int(item[int_field])
+                    except (ValueError, TypeError):
+                        item[int_field] = 0
+
+            if "volume_of_messages" in item:
+                try:
+                    item["volume_of_messages"] = int(item["volume_of_messages"])
+                except (ValueError, TypeError):
+                    item["volume_of_messages"] = 0
+
+            # Optional price_targets handling
+            if "price_targets" in item and isinstance(item["price_targets"], dict):
+                for float_field in ("low", "high", "average"):
+                    if float_field in item["price_targets"]:
+                        try:
+                            item["price_targets"][float_field] = float(item["price_targets"][float_field])
+                        except (ValueError, TypeError):
+                            item["price_targets"][float_field] = None
+
+
+# ------------------------------------------------------------------------------
+# Example usage in a standalone script:
+# ------------------------------------------------------------------------------
+if __name__ == "__main__":
+    # Attempt to retrieve xAI API key from environment
+    import os
+    import dotenv
+
+    dotenv.load_dotenv()
+    xai_api_key = os.getenv("XAI_API_KEY")
+    if not xai_api_key or xai_api_key == "YOUR_XAI_API_KEY":
+        print("WARNING: No valid XAI_API_KEY found in environment!")
+        xai_api_key = "YOUR_XAI_API_KEY"  # fallback
+
+    nq_helper = GrokNewsHelper(api_key=xai_api_key)
+
+    # Example user query
+    # user_query = "What drugs from small biotech companies are expected to get FDA approvals soon?"
+    user_query = "what is the twitter account @Banana3Stocks recommending to buy or sell right now and at what price targets? he usually says things like 'see you at XX price' or 'XX price soon' or 'run to XX' or 'pivot is XX' or 'XX later this year' or 'ready for $XX'"
+    result = nq_helper.execute_query(user_query)
+
+    # Print the structured response
+    print(json.dumps(result, indent=2))

lumibot/components/options_helper.py

@@ -66,8 +66,8 @@ class OptionsHelper:
                                  expiry: date, put_or_call: str = "call") -> Optional[Asset]:
         """
         Find a valid option with the given expiry and strike.
-        If no option is available (e.g. due to holidays/weekends), increment the expiry day-by-day
-        (up to 10 times) until a valid option is found.
+        First tries the requested strike, then searches nearby strikes from the option chain.
+        If no strikes work for this expiry, tries the next expiry date.
 
         Parameters
         ----------
@@ -86,55 +86,111 @@ class OptionsHelper:
             The valid option asset or None if not found.
         """
         self.strategy.log_message(f"Finding next valid option for {underlying_asset.symbol} at strike {rounded_underlying_price} and expiry {expiry}", color="blue")
-        loop_counter = 0
-        while True:
+
+        expiry_attempts = 0
+        while expiry_attempts < 10:
+            # Check if this expiry was previously marked as invalid
             for record in self.non_existing_expiry_dates:
                 if (record["underlying_asset_symbol"] == underlying_asset.symbol and
                     record["expiry"] == expiry):
                     self.strategy.log_message(f"Expiry {expiry} previously invalid for {underlying_asset.symbol}; trying next day.", color="yellow")
                     expiry += timedelta(days=1)
+                    expiry_attempts += 1
                     continue
 
-            option = Asset(
-                underlying_asset.symbol,
-                asset_type="option",
-                expiration=expiry,
-                strike=rounded_underlying_price,
-                right=put_or_call,
-                underlying_asset=underlying_asset,
-            )
-
-            # First check if quote data exists (preferred)
+            # Try to get option chain to find available strikes
             try:
-                quote = self.strategy.get_quote(option)
-                has_valid_quote = quote and (quote.bid is not None or quote.ask is not None)
-                if has_valid_quote:
-                    self.strategy.log_message(f"Found valid quote for option {option.symbol} at expiry {expiry}", color="blue")
-                    return option
-            except Exception as e:
-                self.strategy.log_message(f"Error getting quote for {option.symbol}: {e}", color="yellow")
+                chains = self.strategy.get_chains(underlying_asset)
+                self.strategy.log_message(f"Got chains for {underlying_asset.symbol}: {bool(chains)}", color="cyan")
+
+                if chains:
+                    # Get available strikes for this expiry
+                    available_strikes = chains.strikes(expiry, put_or_call.upper())
+                    self.strategy.log_message(f"Available strikes for {expiry}: {len(available_strikes) if available_strikes else 0} strikes", color="cyan")
+
+                    if not available_strikes:
+                        self.strategy.log_message(f"No strikes available for {put_or_call.upper()} on {expiry}; trying next expiry.", color="yellow")
+                        self.non_existing_expiry_dates.append({
+                            "underlying_asset_symbol": underlying_asset.symbol,
+                            "expiry": expiry,
+                        })
+                        expiry += timedelta(days=1)
+                        expiry_attempts += 1
+                        continue
+
+                    # Find the closest strike to our target
+                    closest_strike = min(available_strikes, key=lambda x: abs(x - rounded_underlying_price))
+                    self.strategy.log_message(f"Target strike {rounded_underlying_price} -> Closest available strike: {closest_strike}", color="green")
+
+                    # Create option with the closest available strike
+                    option = Asset(
+                        underlying_asset.symbol,
+                        asset_type="option",
+                        expiration=expiry,
+                        strike=closest_strike,
+                        right=put_or_call,
+                        underlying_asset=underlying_asset,
+                    )
+
+                    # Verify this option has price data
+                    try:
+                        quote = self.strategy.get_quote(option)
+                        has_valid_quote = quote and (quote.bid is not None or quote.ask is not None)
+                        if has_valid_quote:
+                            self.strategy.log_message(f"Found valid option: {option.symbol} {option.right} {option.strike} exp {option.expiration}", color="green")
+                            return option
+                    except Exception as e:
+                        self.strategy.log_message(f"Error getting quote for {option.symbol}: {e}", color="yellow")
+
+                    # Fallback to last price
+                    try:
+                        price = self.strategy.get_last_price(option)
+                        if price is not None:
+                            self.strategy.log_message(f"Found valid option (via last price): {option.symbol} {option.right} {option.strike} exp {option.expiration}", color="green")
+                            return option
+                    except Exception as e:
+                        pass
+
+                    # If closest strike didn't work, this expiry might be invalid
+                    self.strategy.log_message(f"Could not get price data for strike {closest_strike} on {expiry}; trying next expiry.", color="yellow")
+                    self.non_existing_expiry_dates.append({
+                        "underlying_asset_symbol": underlying_asset.symbol,
+                        "expiry": expiry,
+                    })
+                    expiry += timedelta(days=1)
+                    expiry_attempts += 1
+                    continue
 
-            # Fallback to checking last price if no quote
-            try:
-                price = self.strategy.get_last_price(option)
-                self.strategy.log_message(f"Price for option {option.symbol} at expiry {expiry} is {price}", color="blue")
             except Exception as e:
-                self.strategy.log_message(f"Error getting price for {option.symbol}: {e}", color="red")
-                price = None
+                self.strategy.log_message(f"Error getting chains for {underlying_asset.symbol}: {e}; falling back to direct strike attempt.", color="yellow")
+                # Fallback: Try the exact strike requested (old behavior)
+                option = Asset(
+                    underlying_asset.symbol,
+                    asset_type="option",
+                    expiration=expiry,
+                    strike=rounded_underlying_price,
+                    right=put_or_call,
+                    underlying_asset=underlying_asset,
+                )
 
-            if price is not None:
-                return option
+                try:
+                    price = self.strategy.get_last_price(option)
+                    if price is not None:
+                        return option
+                except Exception:
+                    pass
 
-            self.strategy.log_message(f"No price found for option {option.symbol} at expiry {expiry}.", color="yellow")
+            # No valid option found for this expiry, try next day
+            self.strategy.log_message(f"No valid option found for expiry {expiry}; trying next expiry.", color="yellow")
             self.non_existing_expiry_dates.append({
                 "underlying_asset_symbol": underlying_asset.symbol,
                 "expiry": expiry,
             })
             expiry += timedelta(days=1)
-            loop_counter += 1
-            if loop_counter >= 10:
-                self.strategy.log_message("Exceeded maximum attempts to find a valid option.", color="red")
-                return None
+            expiry_attempts += 1
+
+        self.strategy.log_message("Exceeded maximum attempts to find a valid option.", color="red")
+        return None
 
     def get_strike_deltas(self, underlying_asset: Asset, expiry: date, strikes: List[float],
                           right: str, stop_greater_than: Optional[float] = None,

lumibot/credentials.py

@@ -78,6 +78,9 @@ BACKTESTING_END = None
 if backtesting_end:
     BACKTESTING_END = parser.parse(backtesting_end)
 
+# Get the backtesting data source
+BACKTESTING_DATA_SOURCE = os.environ.get("BACKTESTING_DATA_SOURCE", "ThetaData")
+
 # Check if we should hide trades
 hide_trades = os.environ.get("HIDE_TRADES")
 if not hide_trades or hide_trades.lower() == "false":

lumibot/data_sources/data_source_backtesting.py

@@ -68,11 +68,9 @@ class DataSourceBacktesting(DataSource, ABC):
         # catch it here and ignore it in this class. Child classes that need it should error check it themselves.
         self._config = config
 
-        # If false, we don't show the progress bar
-        # Also disable progress bar if BACKTESTING_QUIET_LOGS is enabled
-        import os
-        quiet_logs_enabled = os.environ.get("BACKTESTING_QUIET_LOGS", "").lower() == "true"
-        self._show_progress_bar = show_progress_bar and not quiet_logs_enabled
+        # Progress bar should always show when enabled, regardless of quiet_logs
+        # Quiet logs only affects INFO/WARNING/ERROR logging, not progress bar
+        self._show_progress_bar = show_progress_bar
 
         self._progress_csv_path = "logs/progress.csv"
         # Add initialization for the logging timer attribute