scythe-ttp 0.13.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

scythe/core/headers.py

@@ -19,6 +19,51 @@ class HeaderExtractor:
     def __init__(self):
         self.logger = logging.getLogger("HeaderExtractor")
 
+    @staticmethod
+    def _normalize_url(url: str) -> str:
+        """Ensure the URL has a scheme so requests can handle it."""
+        if not isinstance(url, str):
+            return url
+        lower = url.lower().strip()
+        if lower.startswith("http://") or lower.startswith("https://"):
+            return url
+        return f"http://{url}"
+
+    @staticmethod
+    def _is_static_asset(url: str, headers: Optional[Dict[str, Any]] = None) -> bool:
+        """Heuristically determine if a URL/log entry is a static asset (css/js/image/font/etc.)."""
+        try:
+            if not isinstance(url, str):
+                return False
+            u = url.lower()
+            # Common static file extensions
+            static_exts = (
+                '.css', '.js', '.mjs', '.map', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico',
+                '.woff', '.woff2', '.ttf', '.otf', '.eot', '.webp', '.mp4', '.webm', '.mp3', '.wav'
+            )
+            if any(u.endswith(ext) for ext in static_exts):
+                return True
+            if '/static/' in u or '/assets/' in u:
+                return True
+            # Content-Type hint
+            if isinstance(headers, dict):
+                # case-insensitive lookup
+                ctype = None
+                for k, v in headers.items():
+                    if isinstance(k, str) and k.lower() == 'content-type':
+                        ctype = str(v).lower()
+                        break
+                if ctype and (ctype.startswith('text/css') or
+                              ctype.startswith('application/javascript') or
+                              ctype.startswith('text/javascript') or
+                              ctype.startswith('image/') or
+                              ctype.startswith('font/')):
+                    return True
+        except Exception:
+            # Be safe: if unsure, do not classify as static
+            return False
+        return False
+
     @staticmethod
     def enable_logging_for_driver(chrome_options: Options) -> None:
         """
@@ -50,13 +95,14 @@ class HeaderExtractor:
             Version string if header found, None otherwise
         """
         try:
-            self.logger.debug(f"Making {method} request to {url} for header extraction")
+            norm_url = self._normalize_url(url)
+            self.logger.debug(f"Making {method} request to {norm_url} for header extraction")
             
             # Use HEAD by default for efficiency, fallback to GET if needed
             if method.upper() == "HEAD":
-                response = requests.head(url, timeout=timeout, allow_redirects=True)
+                response = requests.head(norm_url, timeout=timeout, allow_redirects=True)
             else:
-                response = requests.get(url, timeout=timeout, allow_redirects=True)
+                response = requests.get(norm_url, timeout=timeout, allow_redirects=True)
             
             # Check if request was successful
             response.raise_for_status()
@@ -71,7 +117,8 @@ class HeaderExtractor:
                 return None
                 
         except requests.exceptions.RequestException as e:
-            self.logger.warning(f"Failed to make {method} request to {url}: {e}")
+            hint = " (tip: include http:// or https://)" if isinstance(url, str) and not url.lower().startswith(("http://","https://")) else ""
+            self.logger.warning(f"Failed to make {method} request to {url}: {e}{hint}")
             return None
         except Exception as e:
             self.logger.warning(f"Unexpected error during banner grab: {e}")
@@ -90,12 +137,13 @@ class HeaderExtractor:
             Dictionary of all response headers
         """
         try:
-            self.logger.debug(f"Making {method} request to {url} for all headers")
+            norm_url = self._normalize_url(url)
+            self.logger.debug(f"Making {method} request to {norm_url} for all headers")
             
             if method.upper() == "HEAD":
-                response = requests.head(url, timeout=timeout, allow_redirects=True)
+                response = requests.head(norm_url, timeout=timeout, allow_redirects=True)
             else:
-                response = requests.get(url, timeout=timeout, allow_redirects=True)
+                response = requests.get(norm_url, timeout=timeout, allow_redirects=True)
                 
             response.raise_for_status()
             
@@ -103,7 +151,8 @@ class HeaderExtractor:
             return {k: str(v) for k, v in response.headers.items()}
             
         except requests.exceptions.RequestException as e:
-            self.logger.warning(f"Failed to get headers from {url}: {e}")
+            hint = " (tip: include http:// or https://)" if isinstance(url, str) and not url.lower().startswith(("http://","https://")) else ""
+            self.logger.warning(f"Failed to get headers from {url}: {e}{hint}")
             return {}
         except Exception as e:
             self.logger.warning(f"Unexpected error getting headers: {e}")
@@ -179,7 +228,11 @@ class HeaderExtractor:
                 self.logger.debug(f"Successfully extracted version '{version}' via banner grab")
                 return version
             else:
-                self.logger.debug("Banner grab failed, falling back to Selenium performance logs")
+                self.logger.debug("Banner grab failed")
+        
+        # In API mode (no driver), do not fall back to Selenium to avoid noisy warnings
+        if driver is None:
+            return None
         
         # Fall back to Selenium performance logs
         self.logger.debug("Using Selenium performance logs method")
@@ -223,6 +276,13 @@ class HeaderExtractor:
                         if target_url and target_url not in response_url:
                             continue
 
+                        # Ignore static assets (css/js/images/fonts) to avoid false detections/noise
+                        try:
+                            if self._is_static_asset(response_url, headers):
+                                continue
+                        except Exception:
+                            pass
+
                         # Look for the version header (case-insensitive)
                         version = self._find_version_header(headers)
                         if version:

scythe/journeys/actions.py

@@ -1,4 +1,5 @@
 import time
+import logging
 from typing import Dict, Any, Optional
 from selenium.webdriver.remote.webdriver import WebDriver
 from selenium.webdriver.common.by import By
@@ -700,70 +701,180 @@ class ApiRequestAction(Action):
         if self.headers:
             final_headers.update(self.headers)
         
+        # Simple masking for sensitive headers
+        def _mask_headers(headers: Dict[str, Any]) -> Dict[str, Any]:
+            masked = {}
+            for k, v in (headers or {}).items():
+                if k is None:
+                    continue
+                key_lower = str(k).lower()
+                if key_lower in {"authorization", "proxy-authorization", "cookie", "set-cookie", "x-api-key", "x-auth-token"}:
+                    masked[k] = "***"
+                else:
+                    masked[k] = v
+            return masked
+        
         # Resolve URL: absolute or join with target_url from context
         from urllib.parse import urljoin
+        from ..core.headers import HeaderExtractor
         base_url = context.get('target_url') or ''
-        resolved_url = self.url if self.url.lower().startswith('http') else urljoin(base_url, self.url)
+        # Ensure base_url has a scheme so urljoin works with relative paths
+        if isinstance(base_url, str) and base_url and not base_url.lower().startswith(('http://', 'https://')):
+            base_url = HeaderExtractor._normalize_url(base_url)
+        if isinstance(self.url, str) and self.url.lower().startswith('http'):
+            resolved_url = self.url
+        else:
+            resolved_url = urljoin(base_url, self.url)
         
+        # Store request details early
+        self.store_result('request_method', self.method)
+        self.store_result('url', resolved_url)
+        if self.params:
+            self.store_result('request_params', self.params)
+        if self.body_json is not None:
+            self.store_result('request_json', self.body_json)
+        if self.data is not None:
+            self.store_result('request_data', self.data)
+        self.store_result('request_headers', _mask_headers(final_headers))
+        
+        logger = logging.getLogger("Journey.ApiRequestAction")
+        # Honor any pending rate-limit resume time set by previous actions/steps
         try:
-            response = session.request(
-                self.method,
-                resolved_url,
-                params=self.params or None,
-                json=self.body_json,
-                data=self.data,
-                headers=final_headers or None,
-                timeout=self.timeout,
-            )
-            
-            # Store details
-            self.store_result('url', resolved_url)
-            self.store_result('request_headers', final_headers)
-            self.store_result('status_code', getattr(response, 'status_code', None))
-            response_headers = dict(getattr(response, 'headers', {}) or {})
-            self.store_result('response_headers', response_headers)
-            # Publish to context for downstream version extraction
-            context['last_response_headers'] = response_headers
-            context['last_response_url'] = resolved_url
-            # Try JSON, fallback to text
-            body = None
-            parsed_model = None
-            validation_error = None
-            try:
-                body = response.json()
-                self.store_result('response_json', body)
-                # If a response_model is provided, attempt validation/parsing
-                if self.response_model is not None:
+            resume_at = context.get('rate_limit_resume_at')
+            now = time.time()
+            if isinstance(resume_at, (int, float)) and resume_at > now:
+                wait_s = min(resume_at - now, 30)
+                if wait_s > 0:
+                    self.store_result('waited_ms_before_request', int(wait_s * 1000))
                     try:
-                        # Pydantic v2 preferred: model_validate
-                        if hasattr(self.response_model, 'model_validate'):
-                            parsed_model = self.response_model.model_validate(body)
-                        else:
-                            # Pydantic v1 fallback
-                            parsed_model = self.response_model.parse_obj(body)
-                        self.store_result('response_model_instance', parsed_model)
-                        # Save into context for downstream actions
-                        key = self.response_model_context_key or 'last_response_model'
-                        context[key] = parsed_model
-                    except Exception as ve:
-                        validation_error = str(ve)
-                        self.store_result('response_validation_error', validation_error)
-            except Exception:
-                text = getattr(response, 'text', '')
-                # Limit stored text to keep logs light
-                self.store_result('response_text', text if text is None or len(text) <= 2000 else text[:2000])
-            
-            # Determine success (status-based by default)
-            if self.expected_status is not None:
-                http_ok = (getattr(response, 'status_code', None) == self.expected_status)
-            else:
-                http_ok = bool(getattr(response, 'ok', False))
-            
-            # Optionally fail on validation error
-            if self.response_model is not None and self.fail_on_validation_error and self.get_result('response_validation_error'):
-                return False if http_ok else False
-            
-            return http_ok
-        except Exception as e:
-            self.store_result('error', str(e))
-            return False
+                        logger.info(f"Delaying {wait_s:.2f}s due to prior rate limit (resume_at)")
+                    except Exception:
+                        pass
+                    time.sleep(wait_s)
+        except Exception:
+            pass
+
+        def _h(headers: Dict[str, Any], name: str):
+            lname = (name or '').lower()
+            for k, v in (headers or {}).items():
+                try:
+                    if isinstance(k, str) and k.lower() == lname:
+                        return v
+                except Exception:
+                    continue
+            return None
+
+        attempts = 2  # at most one retry on 429 for idempotent methods
+        last_exception = None
+        for attempt in range(attempts):
+            start_ts = time.time()
+            try:
+                response = session.request(
+                    self.method,
+                    resolved_url,
+                    params=self.params or None,
+                    json=self.body_json,
+                    data=self.data,
+                    headers=final_headers or None,
+                    timeout=self.timeout,
+                )
+                duration_ms = int((time.time() - start_ts) * 1000)
+                self.store_result('duration_ms', duration_ms)
+
+                # Store details
+                status_code = getattr(response, 'status_code', None)
+                self.store_result('status_code', status_code)
+                response_headers = dict(getattr(response, 'headers', {}) or {})
+                # Mask sensitive response headers
+                self.store_result('response_headers', _mask_headers(response_headers))
+                # Publish to context for downstream version extraction and rate-limit coordination
+                context['last_response_headers'] = response_headers
+                context['last_response_url'] = resolved_url
+
+                # Parse rate-limit headers
+                try:
+                    # Normalize numeric values
+                    remaining = _h(response_headers, 'X-RateLimit-Remaining')
+                    if remaining is None:
+                        remaining = _h(response_headers, 'X-Ratelimit-Remaining')
+                    reset = _h(response_headers, 'X-RateLimit-Reset')
+                    if reset is None:
+                        reset = _h(response_headers, 'X-Ratelimit-Reset')
+                    retry_after = _h(response_headers, 'Retry-After')
+
+                    # If explicit Retry-After or 429, set resume time and optionally retry
+                    if status_code == 429:
+                        wait_s = 0
+                        try:
+                            wait_s = int(str(retry_after).strip()) if retry_after is not None else 1
+                        except Exception:
+                            wait_s = 1
+                        wait_s = max(1, min(wait_s, 30))
+                        context['rate_limit_resume_at'] = time.time() + wait_s
+                        self.store_result('rate_limit_wait_s', wait_s)
+                        if attempt == 0 and self.method in {'GET', 'HEAD', 'OPTIONS'}:
+                            try:
+                                logger.info(f"Hit 429 Too Many Requests; backing off {wait_s}s and retrying once")
+                            except Exception:
+                                pass
+                            time.sleep(wait_s)
+                            continue  # retry once
+                    else:
+                        # If remaining == 0 and reset provided, set resume time
+                        try:
+                            if remaining is not None and str(remaining).strip() == '0' and reset is not None:
+                                wait_s2 = int(str(reset).strip())
+                                if wait_s2 > 0:
+                                    context['rate_limit_resume_at'] = time.time() + min(wait_s2, 30)
+                        except Exception:
+                            pass
+                except Exception:
+                    pass
+
+                # Try JSON, fallback to text
+                parsed_model = None
+                try:
+                    body = response.json()
+                    self.store_result('response_json', body)
+                    # If a response_model is provided, attempt validation/parsing
+                    if self.response_model is not None:
+                        try:
+                            # Pydantic v2 preferred: model_validate
+                            if hasattr(self.response_model, 'model_validate'):
+                                parsed_model = self.response_model.model_validate(body)
+                            else:
+                                # Pydantic v1 fallback
+                                parsed_model = self.response_model.parse_obj(body)
+                            self.store_result('response_model_instance', parsed_model)
+                            # Save into context for downstream actions
+                            key = self.response_model_context_key or 'last_response_model'
+                            context[key] = parsed_model
+                        except Exception as ve:
+                            self.store_result('response_validation_error', str(ve))
+                except Exception:
+                    text = getattr(response, 'text', '')
+                    # Limit stored text to keep logs light
+                    if text is not None and isinstance(text, str):
+                        self.store_result('response_text', text if len(text) <= 2000 else text[:2000])
+                    else:
+                        self.store_result('response_text', text)
+
+                # Determine success (status-based by default)
+                if self.expected_status is not None:
+                    http_ok = (getattr(response, 'status_code', None) == self.expected_status)
+                else:
+                    http_ok = bool(getattr(response, 'ok', False))
+
+                # Optionally fail on validation error
+                if self.response_model is not None and self.fail_on_validation_error and self.get_result('response_validation_error'):
+                    return False
+
+                return http_ok
+            except Exception as e:
+                last_exception = e
+                self.store_result('duration_ms', int((time.time() - start_ts) * 1000))
+                self.store_result('error', str(e))
+                break
+
+        # If we got here and had an exception or no return, fail
+        return False

scythe/journeys/base.py

@@ -102,7 +102,7 @@ class Step:
         """Add an action to this step."""
         self.actions.append(action)
     
-    def execute(self, driver: WebDriver, context: Dict[str, Any]) -> bool:
+    def execute(self, driver: WebDriver|None, context: Dict[str, Any]) -> bool:
         """
         Execute all actions in this step.
         
@@ -111,7 +111,7 @@ class Step:
             context: Shared context data
             
         Returns:
-            True if step succeeded, False otherwise
+            True if a step succeeded, False otherwise
         """
         logger = logging.getLogger(f"Journey.Step.{self.name}")
         logger.info(f"Executing step: {self.name}")
@@ -136,12 +136,14 @@ class Step:
                 result = action.execute(driver, context)
                 
                 # Store result
+                details = getattr(action, 'execution_data', {})
                 action_result = {
                     'action_name': action.name,
                     'action_description': action.description,
                     'expected': action.expected_result,
                     'actual': result,
-                    'timestamp': time.time()
+                    'timestamp': time.time(),
+                    'details': details.copy() if isinstance(details, dict) else {}
                 }
                 self.execution_results.append(action_result)
                 
@@ -156,6 +158,47 @@ class Step:
                 else:
                     if action.expected_result:
                         logger.error(f"✗ Action failed: {action.name}")
+                        # Emit diagnostic details when available (e.g., for API requests)
+                        try:
+                            ad = action_result.get('details', {}) or {}
+                            method = ad.get('request_method') or getattr(action, 'method', None)
+                            url = ad.get('url') or getattr(action, 'url', None)
+                            status = ad.get('status_code')
+                            dur = ad.get('duration_ms')
+                            if method or url or status is not None:
+                                parts = []
+                                if method:
+                                    parts.append(f"method={method}")
+                                if url:
+                                    parts.append(f"url={url}")
+                                if status is not None:
+                                    parts.append(f"status={status}")
+                                if dur is not None:
+                                    parts.append(f"duration_ms={dur}")
+                                logger.error("    Details: " + ", ".join(parts))
+                            req_headers = ad.get('request_headers')
+                            if req_headers:
+                                logger.error(f"    Request headers: {req_headers}")
+                            req_params = ad.get('request_params')
+                            if req_params:
+                                logger.error(f"    Request params: {req_params}")
+                            req_json = ad.get('request_json')
+                            if req_json is not None:
+                                logger.error(f"    Request JSON: {req_json}")
+                            req_data = ad.get('request_data')
+                            if req_data is not None:
+                                logger.error(f"    Request data: {req_data}")
+                            resp_headers = ad.get('response_headers')
+                            if resp_headers:
+                                logger.error(f"    Response headers: {resp_headers}")
+                            if 'response_json' in ad:
+                                logger.error(f"    Response JSON: {ad.get('response_json')}")
+                            elif 'response_text' in ad:
+                                logger.error(f"    Response text: {ad.get('response_text')}")
+                            if ad.get('error'):
+                                logger.error(f"    Error: {ad.get('error')}")
+                        except Exception:
+                            pass
                         failure_count += 1
                         if not self.continue_on_failure:
                             return False
@@ -165,6 +208,35 @@ class Step:
                 
             except Exception as e:
                 logger.error(f"Exception in action {action.name}: {str(e)}")
+                # Emit any available diagnostics even on exceptions
+                try:
+                    details = getattr(action, 'execution_data', {}) or {}
+                    if details:
+                        method = details.get('request_method') or getattr(action, 'method', None)
+                        url = details.get('url') or getattr(action, 'url', None)
+                        status = details.get('status_code')
+                        dur = details.get('duration_ms')
+                        parts = []
+                        if method:
+                            parts.append(f"method={method}")
+                        if url:
+                            parts.append(f"url={url}")
+                        if status is not None:
+                            parts.append(f"status={status}")
+                        if dur is not None:
+                            parts.append(f"duration_ms={dur}")
+                        if parts:
+                            logger.error("    Details: " + ", ".join(parts))
+                        if details.get('request_headers'):
+                            logger.error(f"    Request headers: {details.get('request_headers')}")
+                        if details.get('response_headers'):
+                            logger.error(f"    Response headers: {details.get('response_headers')}")
+                        if 'response_json' in details:
+                            logger.error(f"    Response JSON: {details.get('response_json')}")
+                        elif 'response_text' in details:
+                            logger.error(f"    Response text: {details.get('response_text')}")
+                except Exception:
+                    pass
                 failure_count += 1
                 if not self.continue_on_failure:
                     return False
@@ -260,7 +332,7 @@ class Journey:
             logger.error(f"Authentication failed: {str(e)}")
             return False
     
-    def execute(self, driver: WebDriver, target_url: str) -> Dict[str, Any]:
+    def execute(self, driver: WebDriver|None, target_url: str) -> Dict[str, Any]:
         """
         Execute the complete journey.
         
@@ -283,7 +355,9 @@ class Journey:
         start_time = time.time()
         
         # Set initial context
-        self.set_context('target_url', target_url)
+        # Normalize target_url to include scheme when missing (e.g., 'localhost:8080' -> 'http://localhost:8080')
+        normalized_target_url = HeaderExtractor._normalize_url(target_url) if isinstance(target_url, str) else target_url
+        self.set_context('target_url', normalized_target_url)
         self.set_context('journey_name', self.name)
         self.set_context('start_time', start_time)
         
@@ -391,6 +465,48 @@ class Journey:
                         'target_version': target_version
                     }
                     results['step_results'].append(step_result)
+
+                    # If the previous step exhausted the rate limit, pause before starting the next one
+                    try:
+                        # Prefer an explicit resume time set by actions
+                        resume_at = self.context.get('rate_limit_resume_at')
+                        now = time.time()
+                        if isinstance(resume_at, (int, float)) and resume_at > now:
+                            wait_s = min(resume_at - now, 30)
+                            if wait_s > 0:
+                                logger.info(f"Rate limit backoff in effect; waiting {wait_s:.2f}s before next step")
+                                time.sleep(wait_s)
+                        else:
+                            last_headers = (self.context.get('last_response_headers') or {})
+                            if isinstance(last_headers, dict) and last_headers:
+                                def _h(name: str):
+                                    name = (name or '').lower()
+                                    for k, v in last_headers.items():
+                                        if isinstance(k, str) and k.lower() == name:
+                                            return v
+                                    return None
+                                retry_after = _h('retry-after')
+                                if retry_after is not None:
+                                    try:
+                                        wait_s = int(str(retry_after).strip())
+                                        if wait_s > 0:
+                                            logger.info(f"Rate-limited by server (Retry-After={wait_s}s); pausing before next step")
+                                            time.sleep(min(wait_s, 30))
+                                    except Exception:
+                                        pass
+                                else:
+                                    remaining = _h('x-ratelimit-remaining')
+                                    reset = _h('x-ratelimit-reset')
+                                    if remaining is not None and str(remaining).strip() == '0' and reset is not None:
+                                        try:
+                                            wait_s = int(str(reset).strip())
+                                            if wait_s > 0:
+                                                logger.info(f"Rate limit reached (remaining=0). Waiting {wait_s}s for reset before next step")
+                                                time.sleep(min(wait_s, 30))
+                                        except Exception:
+                                            pass
+                    except Exception:
+                        pass
                     
                 except Exception as e:
                     logger.error(f"Exception in step {step.name}: {str(e)}")

scythe/journeys/executor.py

@@ -147,7 +147,7 @@ class JourneyExecutor:
                 self.journey.set_context('auth_headers', auth_headers)
                 self.journey.set_context('auth_cookies', auth_cookies)
                 
-                # Execute journey with a None driver (API actions ignore driver)
+                # Execute a journey with a None driver (API actions ignore a driver)
                 self.execution_results = self.journey.execute(None, self.target_url)
             else:
                 # UI mode (default)
@@ -352,6 +352,45 @@ class JourneyExecutor:
                 
                 self.logger.info(f"  {status} Step {i}: {step_name} - {result_text} ({expected_text}){version_info}")
                 self.logger.info(f"    Actions: {len([a for a in actions if a.get('actual', False)])}/{len(actions)} succeeded")
+                # Print diagnostic details only for unexpected outcomes
+                for a in actions:
+                    actual = a.get('actual', False)
+                    expected = a.get('expected', True)
+                    if actual != expected:
+                        prefix = "✗ Action failed" if expected else "✗ Action unexpectedly succeeded"
+                        self.logger.error(f"    {prefix}: {a.get('action_name')}")
+                        ad = a.get('details', {}) or {}
+                        method = ad.get('request_method')
+                        url = ad.get('url')
+                        status_code = ad.get('status_code')
+                        dur = ad.get('duration_ms')
+                        parts = []
+                        if method:
+                            parts.append(f"method={method}")
+                        if url:
+                            parts.append(f"url={url}")
+                        if status_code is not None:
+                            parts.append(f"status={status_code}")
+                        if dur is not None:
+                            parts.append(f"duration_ms={dur}")
+                        if parts:
+                            self.logger.error("      Details: " + ", ".join(parts))
+                        if ad.get('request_headers'):
+                            self.logger.error(f"      Request headers: {ad.get('request_headers')}")
+                        if ad.get('request_params'):
+                            self.logger.error(f"      Request params: {ad.get('request_params')}")
+                        if ad.get('request_json') is not None:
+                            self.logger.error(f"      Request JSON: {ad.get('request_json')}")
+                        if ad.get('request_data') is not None:
+                            self.logger.error(f"      Request data: {ad.get('request_data')}")
+                        if ad.get('response_headers'):
+                            self.logger.error(f"      Response headers: {ad.get('response_headers')}")
+                        if 'response_json' in ad:
+                            self.logger.error(f"      Response JSON: {ad.get('response_json')}")
+                        elif 'response_text' in ad:
+                            self.logger.error(f"      Response text: {ad.get('response_text')}")
+                        if ad.get('error'):
+                            self.logger.error(f"      Error: {ad.get('error')}")
         
         # Version summary
         target_versions = self.execution_results.get('target_versions', [])