scythe-ttp 0.12.4__py3-none-any.whl → 0.14.0__py3-none-any.whl

scythe/core/headers.py

@@ -19,6 +19,51 @@ class HeaderExtractor:
     def __init__(self):
         self.logger = logging.getLogger("HeaderExtractor")
 
+    @staticmethod
+    def _normalize_url(url: str) -> str:
+        """Ensure the URL has a scheme so requests can handle it."""
+        if not isinstance(url, str):
+            return url
+        lower = url.lower().strip()
+        if lower.startswith("http://") or lower.startswith("https://"):
+            return url
+        return f"http://{url}"
+
+    @staticmethod
+    def _is_static_asset(url: str, headers: Optional[Dict[str, Any]] = None) -> bool:
+        """Heuristically determine if a URL/log entry is a static asset (css/js/image/font/etc.)."""
+        try:
+            if not isinstance(url, str):
+                return False
+            u = url.lower()
+            # Common static file extensions
+            static_exts = (
+                '.css', '.js', '.mjs', '.map', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico',
+                '.woff', '.woff2', '.ttf', '.otf', '.eot', '.webp', '.mp4', '.webm', '.mp3', '.wav'
+            )
+            if any(u.endswith(ext) for ext in static_exts):
+                return True
+            if '/static/' in u or '/assets/' in u:
+                return True
+            # Content-Type hint
+            if isinstance(headers, dict):
+                # case-insensitive lookup
+                ctype = None
+                for k, v in headers.items():
+                    if isinstance(k, str) and k.lower() == 'content-type':
+                        ctype = str(v).lower()
+                        break
+                if ctype and (ctype.startswith('text/css') or
+                              ctype.startswith('application/javascript') or
+                              ctype.startswith('text/javascript') or
+                              ctype.startswith('image/') or
+                              ctype.startswith('font/')):
+                    return True
+        except Exception:
+            # Be safe: if unsure, do not classify as static
+            return False
+        return False
+
     @staticmethod
     def enable_logging_for_driver(chrome_options: Options) -> None:
         """
@@ -50,13 +95,14 @@ class HeaderExtractor:
             Version string if header found, None otherwise
         """
         try:
-            self.logger.debug(f"Making {method} request to {url} for header extraction")
+            norm_url = self._normalize_url(url)
+            self.logger.debug(f"Making {method} request to {norm_url} for header extraction")
             
             # Use HEAD by default for efficiency, fallback to GET if needed
             if method.upper() == "HEAD":
-                response = requests.head(url, timeout=timeout, allow_redirects=True)
+                response = requests.head(norm_url, timeout=timeout, allow_redirects=True)
             else:
-                response = requests.get(url, timeout=timeout, allow_redirects=True)
+                response = requests.get(norm_url, timeout=timeout, allow_redirects=True)
             
             # Check if request was successful
             response.raise_for_status()
@@ -71,7 +117,8 @@ class HeaderExtractor:
                 return None
                 
         except requests.exceptions.RequestException as e:
-            self.logger.warning(f"Failed to make {method} request to {url}: {e}")
+            hint = " (tip: include http:// or https://)" if isinstance(url, str) and not url.lower().startswith(("http://","https://")) else ""
+            self.logger.warning(f"Failed to make {method} request to {url}: {e}{hint}")
             return None
         except Exception as e:
             self.logger.warning(f"Unexpected error during banner grab: {e}")
@@ -90,12 +137,13 @@ class HeaderExtractor:
             Dictionary of all response headers
         """
         try:
-            self.logger.debug(f"Making {method} request to {url} for all headers")
+            norm_url = self._normalize_url(url)
+            self.logger.debug(f"Making {method} request to {norm_url} for all headers")
             
             if method.upper() == "HEAD":
-                response = requests.head(url, timeout=timeout, allow_redirects=True)
+                response = requests.head(norm_url, timeout=timeout, allow_redirects=True)
             else:
-                response = requests.get(url, timeout=timeout, allow_redirects=True)
+                response = requests.get(norm_url, timeout=timeout, allow_redirects=True)
                 
             response.raise_for_status()
             
@@ -103,7 +151,8 @@ class HeaderExtractor:
             return {k: str(v) for k, v in response.headers.items()}
             
         except requests.exceptions.RequestException as e:
-            self.logger.warning(f"Failed to get headers from {url}: {e}")
+            hint = " (tip: include http:// or https://)" if isinstance(url, str) and not url.lower().startswith(("http://","https://")) else ""
+            self.logger.warning(f"Failed to get headers from {url}: {e}{hint}")
             return {}
         except Exception as e:
             self.logger.warning(f"Unexpected error getting headers: {e}")
@@ -179,7 +228,11 @@ class HeaderExtractor:
                 self.logger.debug(f"Successfully extracted version '{version}' via banner grab")
                 return version
             else:
-                self.logger.debug("Banner grab failed, falling back to Selenium performance logs")
+                self.logger.debug("Banner grab failed")
+        
+        # In API mode (no driver), do not fall back to Selenium to avoid noisy warnings
+        if driver is None:
+            return None
         
         # Fall back to Selenium performance logs
         self.logger.debug("Using Selenium performance logs method")
@@ -223,6 +276,13 @@ class HeaderExtractor:
                         if target_url and target_url not in response_url:
                             continue
 
+                        # Ignore static assets (css/js/images/fonts) to avoid false detections/noise
+                        try:
+                            if self._is_static_asset(response_url, headers):
+                                continue
+                        except Exception:
+                            pass
+
                         # Look for the version header (case-insensitive)
                         version = self._find_version_header(headers)
                         if version:

scythe/journeys/__init__.py

@@ -7,7 +7,7 @@ or other custom actions like navigation, form filling, etc.
 """
 
 from .base import Journey, Step, Action
-from .actions import NavigateAction, ClickAction, FillFormAction, WaitAction, TTPAction
+from .actions import NavigateAction, ClickAction, FillFormAction, WaitAction, TTPAction, ApiRequestAction
 from .executor import JourneyExecutor
 
 __all__ = [
@@ -19,5 +19,6 @@ __all__ = [
     'FillFormAction', 
     'WaitAction',
     'TTPAction',
+    'ApiRequestAction',
     'JourneyExecutor'
 ]

scythe/journeys/actions.py

@@ -1,10 +1,12 @@
 import time
+import logging
 from typing import Dict, Any, Optional
 from selenium.webdriver.remote.webdriver import WebDriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.common.exceptions import TimeoutException, NoSuchElementException
+import requests
 
 from .base import Action
 from ..core.ttp import TTP
@@ -643,4 +645,236 @@ class AssertAction(Action):
         by_method = selector_map.get(self.selector_type)
         if by_method is None:
             raise ValueError(f"Unsupported selector type: {self.selector_type}")
-        return by_method
+        return by_method
+
+class ApiRequestAction(Action):
+    """Action to perform a REST API request in Journey API mode.
+    
+    This action ignores the WebDriver and uses a requests.Session provided
+    in the journey context under the key 'requests_session'. It merges any
+    'auth_headers' from the context into the request headers.
+    Optionally, it can validate and parse the JSON response using a Pydantic
+    model class when provided.
+    """
+    def __init__(self,
+                 method: str,
+                 url: str,
+                 params: Optional[Dict[str, Any]] = None,
+                 body_json: Optional[Dict[str, Any]] = None,
+                 data: Optional[Dict[str, Any]] = None,
+                 headers: Optional[Dict[str, str]] = None,
+                 expected_status: Optional[int] = 200,
+                 timeout: float = 10.0,
+                 name: Optional[str] = None,
+                 description: Optional[str] = None,
+                 expected_result: bool = True,
+                 response_model: Optional[Any] = None,
+                 response_model_context_key: Optional[str] = None,
+                 fail_on_validation_error: bool = False):
+        self.method = method.upper()
+        self.url = url
+        self.params = params or {}
+        self.body_json = body_json
+        self.data = data
+        self.headers = headers or {}
+        self.expected_status = expected_status
+        self.timeout = timeout
+        self.response_model = response_model
+        self.response_model_context_key = response_model_context_key
+        self.fail_on_validation_error = fail_on_validation_error
+        name = name or f"API {self.method} {url}"
+        description = description or f"Perform {self.method} request to {url}"
+        super().__init__(name, description, expected_result)
+
+    def execute(self, driver: WebDriver, context: Dict[str, Any]) -> bool:
+        # Resolve session
+        session = context.get('requests_session')
+        if session is None:
+            session = requests.Session()
+            context['requests_session'] = session
+        
+        # Build headers: auth headers from context + action headers (action overrides)
+        final_headers = {}
+        auth_headers = context.get('auth_headers', {}) or {}
+        if auth_headers:
+            final_headers.update(auth_headers)
+        if self.headers:
+            final_headers.update(self.headers)
+        
+        # Simple masking for sensitive headers
+        def _mask_headers(headers: Dict[str, Any]) -> Dict[str, Any]:
+            masked = {}
+            for k, v in (headers or {}).items():
+                if k is None:
+                    continue
+                key_lower = str(k).lower()
+                if key_lower in {"authorization", "proxy-authorization", "cookie", "set-cookie", "x-api-key", "x-auth-token"}:
+                    masked[k] = "***"
+                else:
+                    masked[k] = v
+            return masked
+        
+        # Resolve URL: absolute or join with target_url from context
+        from urllib.parse import urljoin
+        from ..core.headers import HeaderExtractor
+        base_url = context.get('target_url') or ''
+        # Ensure base_url has a scheme so urljoin works with relative paths
+        if isinstance(base_url, str) and base_url and not base_url.lower().startswith(('http://', 'https://')):
+            base_url = HeaderExtractor._normalize_url(base_url)
+        if isinstance(self.url, str) and self.url.lower().startswith('http'):
+            resolved_url = self.url
+        else:
+            resolved_url = urljoin(base_url, self.url)
+        
+        # Store request details early
+        self.store_result('request_method', self.method)
+        self.store_result('url', resolved_url)
+        if self.params:
+            self.store_result('request_params', self.params)
+        if self.body_json is not None:
+            self.store_result('request_json', self.body_json)
+        if self.data is not None:
+            self.store_result('request_data', self.data)
+        self.store_result('request_headers', _mask_headers(final_headers))
+        
+        logger = logging.getLogger("Journey.ApiRequestAction")
+        # Honor any pending rate-limit resume time set by previous actions/steps
+        try:
+            resume_at = context.get('rate_limit_resume_at')
+            now = time.time()
+            if isinstance(resume_at, (int, float)) and resume_at > now:
+                wait_s = min(resume_at - now, 30)
+                if wait_s > 0:
+                    self.store_result('waited_ms_before_request', int(wait_s * 1000))
+                    try:
+                        logger.info(f"Delaying {wait_s:.2f}s due to prior rate limit (resume_at)")
+                    except Exception:
+                        pass
+                    time.sleep(wait_s)
+        except Exception:
+            pass
+
+        def _h(headers: Dict[str, Any], name: str):
+            lname = (name or '').lower()
+            for k, v in (headers or {}).items():
+                try:
+                    if isinstance(k, str) and k.lower() == lname:
+                        return v
+                except Exception:
+                    continue
+            return None
+
+        attempts = 2  # at most one retry on 429 for idempotent methods
+        last_exception = None
+        for attempt in range(attempts):
+            start_ts = time.time()
+            try:
+                response = session.request(
+                    self.method,
+                    resolved_url,
+                    params=self.params or None,
+                    json=self.body_json,
+                    data=self.data,
+                    headers=final_headers or None,
+                    timeout=self.timeout,
+                )
+                duration_ms = int((time.time() - start_ts) * 1000)
+                self.store_result('duration_ms', duration_ms)
+
+                # Store details
+                status_code = getattr(response, 'status_code', None)
+                self.store_result('status_code', status_code)
+                response_headers = dict(getattr(response, 'headers', {}) or {})
+                # Mask sensitive response headers
+                self.store_result('response_headers', _mask_headers(response_headers))
+                # Publish to context for downstream version extraction and rate-limit coordination
+                context['last_response_headers'] = response_headers
+                context['last_response_url'] = resolved_url
+
+                # Parse rate-limit headers
+                try:
+                    # Normalize numeric values
+                    remaining = _h(response_headers, 'X-RateLimit-Remaining')
+                    if remaining is None:
+                        remaining = _h(response_headers, 'X-Ratelimit-Remaining')
+                    reset = _h(response_headers, 'X-RateLimit-Reset')
+                    if reset is None:
+                        reset = _h(response_headers, 'X-Ratelimit-Reset')
+                    retry_after = _h(response_headers, 'Retry-After')
+
+                    # If explicit Retry-After or 429, set resume time and optionally retry
+                    if status_code == 429:
+                        wait_s = 0
+                        try:
+                            wait_s = int(str(retry_after).strip()) if retry_after is not None else 1
+                        except Exception:
+                            wait_s = 1
+                        wait_s = max(1, min(wait_s, 30))
+                        context['rate_limit_resume_at'] = time.time() + wait_s
+                        self.store_result('rate_limit_wait_s', wait_s)
+                        if attempt == 0 and self.method in {'GET', 'HEAD', 'OPTIONS'}:
+                            try:
+                                logger.info(f"Hit 429 Too Many Requests; backing off {wait_s}s and retrying once")
+                            except Exception:
+                                pass
+                            time.sleep(wait_s)
+                            continue  # retry once
+                    else:
+                        # If remaining == 0 and reset provided, set resume time
+                        try:
+                            if remaining is not None and str(remaining).strip() == '0' and reset is not None:
+                                wait_s2 = int(str(reset).strip())
+                                if wait_s2 > 0:
+                                    context['rate_limit_resume_at'] = time.time() + min(wait_s2, 30)
+                        except Exception:
+                            pass
+                except Exception:
+                    pass
+
+                # Try JSON, fallback to text
+                parsed_model = None
+                try:
+                    body = response.json()
+                    self.store_result('response_json', body)
+                    # If a response_model is provided, attempt validation/parsing
+                    if self.response_model is not None:
+                        try:
+                            # Pydantic v2 preferred: model_validate
+                            if hasattr(self.response_model, 'model_validate'):
+                                parsed_model = self.response_model.model_validate(body)
+                            else:
+                                # Pydantic v1 fallback
+                                parsed_model = self.response_model.parse_obj(body)
+                            self.store_result('response_model_instance', parsed_model)
+                            # Save into context for downstream actions
+                            key = self.response_model_context_key or 'last_response_model'
+                            context[key] = parsed_model
+                        except Exception as ve:
+                            self.store_result('response_validation_error', str(ve))
+                except Exception:
+                    text = getattr(response, 'text', '')
+                    # Limit stored text to keep logs light
+                    if text is not None and isinstance(text, str):
+                        self.store_result('response_text', text if len(text) <= 2000 else text[:2000])
+                    else:
+                        self.store_result('response_text', text)
+
+                # Determine success (status-based by default)
+                if self.expected_status is not None:
+                    http_ok = (getattr(response, 'status_code', None) == self.expected_status)
+                else:
+                    http_ok = bool(getattr(response, 'ok', False))
+
+                # Optionally fail on validation error
+                if self.response_model is not None and self.fail_on_validation_error and self.get_result('response_validation_error'):
+                    return False
+
+                return http_ok
+            except Exception as e:
+                last_exception = e
+                self.store_result('duration_ms', int((time.time() - start_ts) * 1000))
+                self.store_result('error', str(e))
+                break
+
+        # If we got here and had an exception or no return, fail
+        return False

scythe/journeys/base.py

@@ -102,7 +102,7 @@ class Step:
         """Add an action to this step."""
         self.actions.append(action)
     
-    def execute(self, driver: WebDriver, context: Dict[str, Any]) -> bool:
+    def execute(self, driver: WebDriver|None, context: Dict[str, Any]) -> bool:
         """
         Execute all actions in this step.
         
@@ -111,7 +111,7 @@ class Step:
             context: Shared context data
             
         Returns:
-            True if step succeeded, False otherwise
+            True if a step succeeded, False otherwise
         """
         logger = logging.getLogger(f"Journey.Step.{self.name}")
         logger.info(f"Executing step: {self.name}")
@@ -136,12 +136,14 @@ class Step:
                 result = action.execute(driver, context)
                 
                 # Store result
+                details = getattr(action, 'execution_data', {})
                 action_result = {
                     'action_name': action.name,
                     'action_description': action.description,
                     'expected': action.expected_result,
                     'actual': result,
-                    'timestamp': time.time()
+                    'timestamp': time.time(),
+                    'details': details.copy() if isinstance(details, dict) else {}
                 }
                 self.execution_results.append(action_result)
                 
@@ -156,6 +158,47 @@ class Step:
                 else:
                     if action.expected_result:
                         logger.error(f"✗ Action failed: {action.name}")
+                        # Emit diagnostic details when available (e.g., for API requests)
+                        try:
+                            ad = action_result.get('details', {}) or {}
+                            method = ad.get('request_method') or getattr(action, 'method', None)
+                            url = ad.get('url') or getattr(action, 'url', None)
+                            status = ad.get('status_code')
+                            dur = ad.get('duration_ms')
+                            if method or url or status is not None:
+                                parts = []
+                                if method:
+                                    parts.append(f"method={method}")
+                                if url:
+                                    parts.append(f"url={url}")
+                                if status is not None:
+                                    parts.append(f"status={status}")
+                                if dur is not None:
+                                    parts.append(f"duration_ms={dur}")
+                                logger.error("    Details: " + ", ".join(parts))
+                            req_headers = ad.get('request_headers')
+                            if req_headers:
+                                logger.error(f"    Request headers: {req_headers}")
+                            req_params = ad.get('request_params')
+                            if req_params:
+                                logger.error(f"    Request params: {req_params}")
+                            req_json = ad.get('request_json')
+                            if req_json is not None:
+                                logger.error(f"    Request JSON: {req_json}")
+                            req_data = ad.get('request_data')
+                            if req_data is not None:
+                                logger.error(f"    Request data: {req_data}")
+                            resp_headers = ad.get('response_headers')
+                            if resp_headers:
+                                logger.error(f"    Response headers: {resp_headers}")
+                            if 'response_json' in ad:
+                                logger.error(f"    Response JSON: {ad.get('response_json')}")
+                            elif 'response_text' in ad:
+                                logger.error(f"    Response text: {ad.get('response_text')}")
+                            if ad.get('error'):
+                                logger.error(f"    Error: {ad.get('error')}")
+                        except Exception:
+                            pass
                         failure_count += 1
                         if not self.continue_on_failure:
                             return False
@@ -165,6 +208,35 @@ class Step:
                 
             except Exception as e:
                 logger.error(f"Exception in action {action.name}: {str(e)}")
+                # Emit any available diagnostics even on exceptions
+                try:
+                    details = getattr(action, 'execution_data', {}) or {}
+                    if details:
+                        method = details.get('request_method') or getattr(action, 'method', None)
+                        url = details.get('url') or getattr(action, 'url', None)
+                        status = details.get('status_code')
+                        dur = details.get('duration_ms')
+                        parts = []
+                        if method:
+                            parts.append(f"method={method}")
+                        if url:
+                            parts.append(f"url={url}")
+                        if status is not None:
+                            parts.append(f"status={status}")
+                        if dur is not None:
+                            parts.append(f"duration_ms={dur}")
+                        if parts:
+                            logger.error("    Details: " + ", ".join(parts))
+                        if details.get('request_headers'):
+                            logger.error(f"    Request headers: {details.get('request_headers')}")
+                        if details.get('response_headers'):
+                            logger.error(f"    Response headers: {details.get('response_headers')}")
+                        if 'response_json' in details:
+                            logger.error(f"    Response JSON: {details.get('response_json')}")
+                        elif 'response_text' in details:
+                            logger.error(f"    Response text: {details.get('response_text')}")
+                except Exception:
+                    pass
                 failure_count += 1
                 if not self.continue_on_failure:
                     return False
@@ -260,7 +332,7 @@ class Journey:
             logger.error(f"Authentication failed: {str(e)}")
             return False
     
-    def execute(self, driver: WebDriver, target_url: str) -> Dict[str, Any]:
+    def execute(self, driver: WebDriver|None, target_url: str) -> Dict[str, Any]:
         """
         Execute the complete journey.
         
@@ -283,7 +355,9 @@ class Journey:
         start_time = time.time()
         
         # Set initial context
-        self.set_context('target_url', target_url)
+        # Normalize target_url to include scheme when missing (e.g., 'localhost:8080' -> 'http://localhost:8080')
+        normalized_target_url = HeaderExtractor._normalize_url(target_url) if isinstance(target_url, str) else target_url
+        self.set_context('target_url', normalized_target_url)
         self.set_context('journey_name', self.name)
         self.set_context('start_time', start_time)
         
@@ -311,12 +385,45 @@ class Journey:
             if self.requires_authentication():
                 auth_name = self.authentication.name if self.authentication else "Unknown"
                 logger.info(f"Authentication required: {auth_name}")
-                auth_success = self.authenticate(driver, target_url)
-                if not auth_success:
-                    logger.error("Authentication failed - aborting journey")
-                    results['errors'].append("Authentication failed")
-                    return results
-                logger.info("Authentication successful")
+                if driver is None:
+                    # API mode: use header-based authentication if available
+                    headers = {}
+                    try:
+                        if self.authentication and hasattr(self.authentication, 'get_auth_headers'):
+                            headers = self.authentication.get_auth_headers() or {}
+                    except Exception as e:
+                        logger.error(f"Failed to get authentication headers: {e}")
+                        headers = {}
+                    cookies = {}
+                    if headers:
+                        # Merge into existing context headers
+                        existing = self.get_context('auth_headers', {})
+                        merged = {**existing, **headers}
+                        self.set_context('auth_headers', merged)
+                        logger.info("Authentication headers prepared for API mode")
+                    # Try to merge cookies as well (hybrid auth)
+                    try:
+                        if self.authentication and hasattr(self.authentication, 'get_auth_cookies'):
+                            cookies = self.authentication.get_auth_cookies() or {}
+                    except Exception as e:
+                        logger.error(f"Failed to get authentication cookies: {e}")
+                        cookies = {}
+                    if cookies:
+                        existing_cookies = self.get_context('auth_cookies', {})
+                        merged_cookies = {**existing_cookies, **cookies}
+                        self.set_context('auth_cookies', merged_cookies)
+                        logger.info("Authentication cookies prepared for API mode")
+                    if not headers and not cookies:
+                        logger.error("Authentication required but no headers/cookies available in API mode")
+                        results['errors'].append("Authentication failed (no API auth data)")
+                        return results
+                else:
+                    auth_success = self.authenticate(driver, target_url)
+                    if not auth_success:
+                        logger.error("Authentication failed - aborting journey")
+                        results['errors'].append("Authentication failed")
+                        return results
+                    logger.info("Authentication successful")
             
             # Execute each step
             for i, step in enumerate(self.steps, 1):
@@ -342,7 +449,7 @@ class Journey:
                             results['actions_failed'] += 1
                     
                     # Extract target version header after step execution
-                    target_version = header_extractor.extract_target_version(driver, target_url)
+                    target_version = header_extractor.extract_target_version_hybrid(driver, target_url)
                     if target_version:
                         results['target_versions'].append(target_version)
                         logger.info(f"Target version detected: {target_version}")
@@ -358,6 +465,48 @@ class Journey:
                         'target_version': target_version
                     }
                     results['step_results'].append(step_result)
+
+                    # If the previous step exhausted the rate limit, pause before starting the next one
+                    try:
+                        # Prefer an explicit resume time set by actions
+                        resume_at = self.context.get('rate_limit_resume_at')
+                        now = time.time()
+                        if isinstance(resume_at, (int, float)) and resume_at > now:
+                            wait_s = min(resume_at - now, 30)
+                            if wait_s > 0:
+                                logger.info(f"Rate limit backoff in effect; waiting {wait_s:.2f}s before next step")
+                                time.sleep(wait_s)
+                        else:
+                            last_headers = (self.context.get('last_response_headers') or {})
+                            if isinstance(last_headers, dict) and last_headers:
+                                def _h(name: str):
+                                    name = (name or '').lower()
+                                    for k, v in last_headers.items():
+                                        if isinstance(k, str) and k.lower() == name:
+                                            return v
+                                    return None
+                                retry_after = _h('retry-after')
+                                if retry_after is not None:
+                                    try:
+                                        wait_s = int(str(retry_after).strip())
+                                        if wait_s > 0:
+                                            logger.info(f"Rate-limited by server (Retry-After={wait_s}s); pausing before next step")
+                                            time.sleep(min(wait_s, 30))
+                                    except Exception:
+                                        pass
+                                else:
+                                    remaining = _h('x-ratelimit-remaining')
+                                    reset = _h('x-ratelimit-reset')
+                                    if remaining is not None and str(remaining).strip() == '0' and reset is not None:
+                                        try:
+                                            wait_s = int(str(reset).strip())
+                                            if wait_s > 0:
+                                                logger.info(f"Rate limit reached (remaining=0). Waiting {wait_s}s for reset before next step")
+                                                time.sleep(min(wait_s, 30))
+                                        except Exception:
+                                            pass
+                    except Exception:
+                        pass
                     
                 except Exception as e:
                     logger.error(f"Exception in step {step.name}: {str(e)}")