stoobly-agent 1.10.0__py3-none-any.whl → 1.10.2__py3-none-any.whl

stoobly_agent/app/proxy/intercept_settings.py

@@ -38,9 +38,6 @@ class InterceptSettings:
     self.__lifecycle_hooks = None
     self.__initialize_lifecycle_hooks()
 
-    self.__response_fixtures = None
-    self.__initialize_response_fixtures()
-
     self._mock_rewrite_rules = None
     self._record_rewrite_rules = None
     self._replay_rewrite_rules = None
@@ -110,11 +107,12 @@ class InterceptSettings:
 
   @property
   def public_directory_path(self):
+    """Get raw public directory paths string from environment or headers."""
     if self.__headers and custom_headers.PUBLIC_DIRECTORY_PATH in self.__headers:
       return self.__headers[custom_headers.PUBLIC_DIRECTORY_PATH]
-
-    if os.environ.get(env_vars.AGENT_PUBLIC_DIRECTORY_PATH):
-      return os.environ[env_vars.AGENT_PUBLIC_DIRECTORY_PATH] 
+    elif os.environ.get(env_vars.AGENT_PUBLIC_DIRECTORY_PATH):
+      return os.environ[env_vars.AGENT_PUBLIC_DIRECTORY_PATH]
+    return None
 
   @property
   def remote_project_key(self):
@@ -130,16 +128,13 @@ class InterceptSettings:
 
   @property
   def response_fixtures_path(self):
+    """Returns comma-separated list of response fixtures paths, optionally with origin prefix."""
     if self.__headers and custom_headers.RESPONSE_FIXTURES_PATH in self.__headers:
       return self.__headers[custom_headers.RESPONSE_FIXTURES_PATH]
 
     if os.environ.get(env_vars.AGENT_RESPONSE_FIXTURES_PATH):
       return os.environ[env_vars.AGENT_RESPONSE_FIXTURES_PATH] 
 
-  @property
-  def response_fixtures(self):
-    return self.__response_fixtures or {}
-
   @property
   def parsed_remote_project_key(self):
     try: 
@@ -280,7 +275,7 @@ class InterceptSettings:
     if self.__headers and custom_headers.REQUEST_ORIGIN in self.__headers:
       return self.__headers[custom_headers.REQUEST_ORIGIN]
 
-    return request_origin.WEB
+    return request_origin.PROXY
 
   def for_response(self):
     self.__for_response = True
@@ -333,21 +328,6 @@ class InterceptSettings:
     except Exception as e:
         return Logger.instance().error(e)
 
-  def __initialize_response_fixtures(self):
-    fixtures_path = self.response_fixtures_path
-
-    if not fixtures_path:
-        return
-
-    if not os.path.exists(fixtures_path):
-        return Logger.instance().error(f"Response fixtures {fixtures_path} does not exist")
-
-    with open(fixtures_path, 'r') as stream:
-        try:
-            self.__response_fixtures = yaml.safe_load(stream)
-        except yaml.YAMLError as exc:
-            Logger.instance().error(exc)
-
   def __order(self, mode):
     if mode == intercept_mode.RECORD:
       return self.__data_rules.record_order

stoobly_agent/app/proxy/mock/eval_fixtures_service.py

@@ -2,25 +2,23 @@ import mimetypes
 import os
 import pdb
 import re
+import yaml
 
 from io import BytesIO
 from mitmproxy.http import Request as MitmproxyRequest
 from requests import Response
 from requests.structures import CaseInsensitiveDict
-from typing import Optional, Union
+from typing import List, Optional, Union
+from urllib.parse import urlparse
 
 from stoobly_agent.lib.logger import bcolors, Logger
 from stoobly_agent.config.constants.custom_headers import MOCK_FIXTURE_PATH
 
-from .types import Fixtures
+from .types import Fixtures, MockOptions, PublicDirectoryPath, ResponseFixturesPath
 
 LOG_ID = 'Fixture'
 
-class Options():
-  public_directory_path: str
-  response_fixtures: Fixtures
-
-def eval_fixtures(request: MitmproxyRequest, **options: Options) -> Union[Response, None]:
+def eval_fixtures(request: MitmproxyRequest, **options: MockOptions) -> Union[Response, None]:
   fixture_path = request.headers.get(MOCK_FIXTURE_PATH)
   headers = CaseInsensitiveDict()
   status_code = 200
@@ -30,23 +28,49 @@ def eval_fixtures(request: MitmproxyRequest, **options: Options) -> Union[Respon
       return
   else:
     response_fixtures = options.get('response_fixtures')
-    fixture: dict = __eval_response_fixtures(request, response_fixtures)
+    
+    # Try response fixtures in order of preference
+    fixture = None
 
-    if not fixture:
-      public_directory_path = options.get('public_directory_path')
+    if response_fixtures:
+      fixture = __find_fixture_for_request(request, response_fixtures, request.method)
 
-      if not public_directory_path:
+    if options.get('response_fixtures_path'):
+      fixture = __eval_response_fixtures_from_paths(request, options['response_fixtures_path'])
+    
+    if not fixture:
+      raw_paths = options.get('public_directory_path')
+      public_directory_paths = __parse_public_directory_paths(raw_paths)
+      
+      if not public_directory_paths:
         return
 
       request_path = 'index' if request.path == '/' else request.path
-      _fixture_path = os.path.join(public_directory_path, request_path.lstrip('/'))
-      if request.headers.get('accept'):
-        fixture_path = __guess_file_path(_fixture_path, request.headers['accept'])
-
+      # Extract origin from request URL (e.g., https://example.com/path -> https://example.com)
+      request_origin = __request_origin(request)
+      
+      # Try to find a matching file in the public directory paths
+      fixture_path = None
+      for dir_path_config in public_directory_paths:
+        # Check if origin matches (if origin is specified)
+        if dir_path_config.get('origin') and request_origin:
+          if not __origin_matches(dir_path_config['origin'], request_origin):
+            continue
+        
+        # Try to find the file in this directory
+        _fixture_path = os.path.join(dir_path_config['path'], request_path.lstrip('/'))
+        if request.headers.get('accept'):
+          fixture_path = __guess_file_path(_fixture_path, request.headers['accept'])
+        
+        if not fixture_path:
+          fixture_path = _fixture_path
+        
+        if os.path.isfile(fixture_path):
+          break
+        else:
+          fixture_path = None
+      
       if not fixture_path:
-        fixture_path = _fixture_path
-
-      if not os.path.isfile(fixture_path):
         return
     else:
       fixture_path = fixture.get('path')
@@ -81,6 +105,53 @@ def eval_fixtures(request: MitmproxyRequest, **options: Options) -> Union[Respon
 
     return response
 
+def __eval_response_fixtures_from_paths(request: MitmproxyRequest, fixtures_paths: str):
+  """Iterate through response fixtures paths and return the first matching fixture."""
+  if not fixtures_paths:
+    return None
+
+  # Parse multiple response fixtures paths with optional origin specification
+  parsed_paths = __parse_response_fixtures_paths(fixtures_paths)
+  
+  # Extract origin from request URL
+  request_origin = __request_origin(request)
+  method = request.method
+
+  # Iterate through each fixtures path
+  for path_config in parsed_paths:
+    fixtures_path = path_config['path']
+    origin = path_config.get('origin')
+    
+    # If origin is specified, check if it matches the request origin
+    if origin and request_origin:
+      if not __origin_matches(origin, request_origin):
+        continue  # Skip this file if origin doesn't match
+    elif origin and not request_origin:
+      continue  # Skip origin-specific files if request has no origin
+    
+    # Load and parse this specific fixtures file
+    if not os.path.exists(fixtures_path):
+      Logger.instance(LOG_ID).error(f"Response fixtures {fixtures_path} does not exist")
+      continue
+
+    try:
+      with open(fixtures_path, 'r') as stream:
+        fixtures = yaml.safe_load(stream) or {}
+        
+        # Try to find a matching fixture in this file
+        fixture = __find_fixture_for_request(request, fixtures, method)
+        if fixture:
+          # Convert fixture path to absolute path if it's a relative path and if 'path' exists in the fixture
+          if fixture.get('path') and not os.path.isabs(fixture['path']):
+            fixture['path'] = os.path.join(os.path.dirname(fixtures_path), fixture['path'])
+          return fixture  # Return immediately on first match
+          
+    except yaml.YAMLError as exc:
+      Logger.instance(LOG_ID).error(f"Error parsing {fixtures_path}: {exc}")
+      continue
+  
+  return None  # No matching fixture found in any file
+
 def __guess_content_type(file_path):
   file_extension = os.path.splitext(file_path)[1]
   if not file_extension:
@@ -107,18 +178,22 @@ def __guess_file_path(file_path, content_type):
     if os.path.isfile(_file_path):
       return _file_path
 
-def __eval_response_fixtures(request: MitmproxyRequest, response_fixtures: Fixtures):
-  if not isinstance(response_fixtures, dict):
-    return
-
-  method = request.method
-  routes = response_fixtures.get(method)
-
+def __find_fixture_for_request(request: MitmproxyRequest, fixtures: dict, method: str):
+  """Find a fixture for the given request in the provided fixtures."""
+  if not fixtures:
+    return None
+  
+  return __find_fixture_in_routes(fixtures, method, request.path)
+
+def __find_fixture_in_routes(fixtures: dict, method: str, request_path: str):
+  """Find a fixture for the given method and path in the provided fixtures."""
+  routes = fixtures.get(method)
+  
   if not isinstance(routes, dict):
-    return
+    return None
 
   for path_pattern in routes:
-    if not re.match(path_pattern, request.path):
+    if not re.match(path_pattern, request_path):
       continue
       
     fixture = routes[path_pattern]
@@ -129,6 +204,8 @@ def __eval_response_fixtures(request: MitmproxyRequest, response_fixtures: Fixtu
 
     if path:
       return fixture
+  
+  return None
 
 def __choose_highest_priority_content_type(accept_header: str) -> Optional[str]:
     if not accept_header:
@@ -156,6 +233,9 @@ def __choose_highest_priority_content_type(accept_header: str) -> Optional[str]:
     types.sort(key=lambda x: -x[1])
     return types[0][0] if types else None
 
+def __origin_matches(pattern: str, request_origin: str) -> bool:
+    return bool(re.match(pattern, request_origin))
+
 def __parse_accept_header(accept_header):
     types = []
     for item in accept_header.split(","):
@@ -171,3 +251,73 @@ def __parse_accept_header(accept_header):
 
     # Sort by quality factor in descending order
     return [content_type for content_type, _ in sorted(types, key=lambda x: x[1], reverse=True)]
+
+def __parse_origin_path_item(path_item: str):
+  """Parse a single path:origin item and return (path, origin) tuple."""
+  path_item = path_item.strip()
+  
+  # Check if this looks like a path:origin format
+  # Format: <FILE-PATH>:<ORIGIN> where ORIGIN is scheme:hostname:port
+  # We need to find the colon that separates path from origin
+  if '://' in path_item:
+    # Find the colon that separates path from origin
+    # Origin format: scheme:hostname:port (e.g., https://api.example.com:8080)
+    origin_start = path_item.rfind('://')
+    if origin_start > 0:
+      # Look for the colon before the scheme
+      colon_before_scheme = path_item.rfind(':', 0, origin_start)
+      if colon_before_scheme != -1:
+        # Format: path:origin
+        path = path_item[:colon_before_scheme]
+        origin = path_item[colon_before_scheme + 1:]
+        return (path.strip(), origin.strip())
+    
+    # No colon before scheme found, treat entire string as path
+    return (path_item, None)
+  else:
+    # Check for path:hostname:port format (without scheme)
+    colons = [i for i, char in enumerate(path_item) if char == ':']
+    
+    if len(colons) >= 1:
+      # Find the first colon that separates path from origin
+      first_colon_idx = colons[0]
+      potential_origin = path_item[first_colon_idx + 1:]
+      
+      # Check if this looks like a hostname:port format
+      # A valid hostname:port should have another colon for the port
+      if ':' in potential_origin:
+        # Format: path:hostname:port
+        path = path_item[:first_colon_idx]
+        origin = potential_origin
+        return (path.strip(), origin.strip())
+    
+    # No valid path:origin format found, treat entire string as path
+    return (path_item, None)
+
+def __parse_public_directory_paths(raw_paths: str) -> List[PublicDirectoryPath]:
+  """Parse public directory paths from comma-separated string."""
+  if not raw_paths:
+    return []
+  
+  paths = []
+  for path_item in raw_paths.split(','):
+    path, origin = __parse_origin_path_item(path_item)
+    paths.append(PublicDirectoryPath(origin=origin, path=path))
+  
+  return paths
+
+def __parse_response_fixtures_paths(raw_paths: str) -> List[ResponseFixturesPath]:
+  """Parse response fixtures paths from comma-separated string."""
+  if not raw_paths:
+    return []
+  
+  paths = []
+  for path_item in raw_paths.split(','):
+    path, origin = __parse_origin_path_item(path_item)
+    paths.append(ResponseFixturesPath(origin=origin, path=path))
+  
+  return paths
+
+def __request_origin(request: MitmproxyRequest) -> str:
+  parsed_url = urlparse(request.url)
+  return f"{parsed_url.scheme}://{parsed_url.hostname}" + (f":{parsed_url.port}" if parsed_url.port else "")

stoobly_agent/app/proxy/mock/types/__init__.py

@@ -1,4 +1,4 @@
-from typing import TypedDict
+from typing import TypedDict, Optional
 
 class Route(TypedDict):
   path: str
@@ -8,3 +8,24 @@ class Fixtures(TypedDict):
   GET: Route
   POST: Route
   PUT: Route
+
+class PublicDirectoryPath(TypedDict):
+  """Represents a public directory path with optional origin specification.
+  
+  Format: <FOLDER-PATH>:<ORIGIN> where ORIGIN is scheme:hostname:port
+  """
+  origin: Optional[str]  # Optional origin (e.g., "https://api.example.com:8080")
+  path: str              # File system path to the public directory
+
+class ResponseFixturesPath(TypedDict):
+  """Represents a response fixtures file path with optional origin specification.
+  
+  Format: <FILE-PATH>:<ORIGIN> where ORIGIN is scheme:hostname:port
+  """
+  origin: Optional[str]  # Optional origin (e.g., "https://api.example.com:8080")
+  path: str              # File system path to the response fixtures file
+
+class MockOptions(TypedDict):
+  """Options for mock service."""
+  public_directory_path: str
+  response_fixtures_path: str                        # Comma-separated list of paths, optionally with origin prefix

stoobly_agent/app/proxy/record/upload_request_service.py

@@ -63,10 +63,7 @@ def upload_request(
         scenario_key=scenario_key
     )
 
-    # If request_origin is WEB, then we are in proxy
-    # This means that we have access to Cache singleton and do not need send a request to update the status
-    sync = intercept_settings.request_origin == request_origin.WEB
-    res = __upload_request_with_body_params(request_model, body_params, sync)
+    res = __upload_request_with_body_params(request_model, body_params)
 
     if intercept_settings.settings.is_debug():
         file_path = __debug_request(flow.request, joined_request.build())
@@ -107,11 +104,11 @@ def upload_staged_request(
 
     return __upload_request_with_body_params(request_model, body_params)
 
-def __upload_request_with_body_params(request_model: RequestModel, body_params: dict, sync=True):
+def __upload_request_with_body_params(request_model: RequestModel, body_params: dict):
     request, status = request_model.create(**body_params)
 
     if status < 400:
-        publish_requests_modified(body_params['project_id'], sync=sync)
+        publish_requests_modified(body_params['project_id'])
 
         return request
 

stoobly_agent/app/proxy/replay/body_parser_service.py

@@ -69,13 +69,16 @@ def parse_json(content):
         return content
 
 def parse_multipart_form_data(content, content_type) -> Dict[bytes, bytes]:
-    headers = {'content-type': content_type}
-    decoded_multipart = multipart_decode(headers, content)
+    try:
+        headers = {'content-type': content_type}
+        decoded_multipart = multipart_decode(headers, content)
 
-    if not decoded_multipart:
+        if not decoded_multipart:
+            return content
+    
+        return MultiDict(decoded_multipart)
+    except:
         return content
- 
-    return MultiDict(decoded_multipart)
 
 def parse_www_form_urlencoded(content):
     try:

stoobly_agent/app/proxy/replay/multipart.py

@@ -3,7 +3,7 @@ import mimetypes
 import re
 import pdb
 
-from multipart import MultipartParser
+from multipart import MultipartParser, ParserError
 from urllib.parse import quote
 
 from mitmproxy.net.http import headers
@@ -51,29 +51,31 @@ def decode(hdrs, content):
         Takes a multipart boundary encoded string and returns list of (key, value) tuples.
     """
     if not isinstance(content, bytes) and not isinstance(content, str):
-        return
+        return content
 
     v = hdrs.get("content-type")
     if not v:
-        return
+        return content
 
     v = headers.parse_content_type(v)
     if not v:
-        return
+        return content
 
     try:
         boundary = v[2]["boundary"].encode("ascii")
     except (KeyError, UnicodeError):
-        return
+        return content
 
     r = []
-    parser = MultipartParser(io.BytesIO(content), boundary=boundary)
-    for part in parser.parts():
-        if part.content_type.lower() == 'application/octet-stream':
+
+    try:
+        parser = MultipartParser(io.BytesIO(content), boundary=boundary)
+
+        for part in parser.parts():
             r.append((part.name, part.raw))
-        else:
-            r.append((part.name, part.value))
 
-        # Free up resources after use
-        part.close()
-    return r
+            # Free up resources after use
+            part.close()
+        return r
+    except ParserError:
+        return content

stoobly_agent/app/proxy/replay/replay_request_service.py

@@ -27,11 +27,11 @@ class ReplayRequestOptions(TypedDict):
   after_replay: Union[Callable[[ReplayContext], Union[requests.Response, None]], None]
   project_key: Union[str, None]
   proxies: dict
-  public_directory_path: str
+  public_directory_path: str  # Comma-separated list of paths, optionally with origin prefix
   remote_project_key: str
   report_key: Union[str, None] 
   request_origin: Union[request_origin.CLI, None] 
-  response_fixtures_path: str
+  response_fixtures_path: str  # Comma-separated list of paths, optionally with origin prefix
   response_mode: Union[mode.RECORD, None]
   scenario_key: Union[str, None] 
   scheme: str

stoobly_agent/app/proxy/run.py

@@ -105,6 +105,9 @@ def __filter_options(options):
     if 'cert_passphrase' in options and not options['cert_passphrase']:
         del options['cert_passphrase']
 
+    if 'detached' in options:
+        del options['detached']
+
     if 'headless' in options:
         del options['headless']
 

stoobly_agent/app/proxy/test/context.py

@@ -166,10 +166,6 @@ class TestContext(TestContextABC):
   def response(self) -> TestContextResponse:
     return self.__response
 
-  @property
-  def response_fixtures(self):
-    return self.__intercept_settings.response_fixtures
-
   @property
   def response_fixtures_path(self):
     return self.__intercept_settings.response_fixtures_path

stoobly_agent/app/proxy/test/context_abc.py

@@ -149,11 +149,6 @@ class TestContextABC(abc.ABC):
   def response(self) -> TestContextResponse:
     pass
 
-  @property
-  @abc.abstractmethod
-  def response_fixtures(self):
-    pass
-
   @property
   @abc.abstractmethod
   def response_fixtures_path(self):

stoobly_agent/app/proxy/utils/publish_change_service.py

@@ -1,41 +1,35 @@
-import os
 import pdb
+import requests
 import threading
 
-from typing import TypedDict
-
 from stoobly_agent.app.settings import Settings
-from stoobly_agent.config.constants.statuses import REQUESTS_MODIFIED
+from stoobly_agent.config.constants.statuses import REQUESTS_MODIFIED, SETTINGS_MODIFIED
 from stoobly_agent.lib.api.agent_api import AgentApi
 from stoobly_agent.lib.cache import Cache
 from stoobly_agent.lib.logger import Logger
 
-class Options(TypedDict):
-  sync: bool
-
-# Announce that a new request has been created
-def publish_change(status: str, value, **options: Options):
-  if options.get('sync'):
-    return __publish_change_sync(status, value)
+LOG_ID = 'PublishChange'
 
+def publish_settings_modified(value):
+  return __publish_change_sync(SETTINGS_MODIFIED, value)
+ 
+def publish_requests_modified(value):
   settings: Settings = Settings.instance()
 
-  # If ui is not active, return
-  if  not settings.ui.active:
-    return False
+  # If not headless...
+  if settings.ui.active:
+    return __publish_change_sync(REQUESTS_MODIFIED, value)
 
   ui_url = settings.ui.url
-
   if not ui_url:
-    Logger.instance().warn('Settings.ui.url not configured')
     return False
-  else:
-    thread = threading.Thread(target=__put_status, args=(ui_url, status, value))
-    thread.start()
-    return True
 
-def publish_requests_modified(value, **options: Options):
-  return publish_change(REQUESTS_MODIFIED, value, **options)
+  return __publish_change_async(REQUESTS_MODIFIED, value, ui_url)
+
+def __publish_change_async(status, value, ui_url: str):
+  thread = threading.Thread(target=__put_status, args=(ui_url, status, value))
+  thread.start()
+  return True
 
 def __publish_change_sync(status: str, value):
   cache = Cache.instance()
@@ -44,4 +38,7 @@ def __publish_change_sync(status: str, value):
 
 def __put_status(ui_url, status, value):
     api: AgentApi = AgentApi(ui_url)
-    api.update_status(status, value)
+    try:
+      api.update_status(status, value)
+    except requests.exceptions.ConnectionError:
+      Logger.instance(LOG_ID).error(f"could not connect to {ui_url}")

stoobly_agent/app/settings/__init__.py

@@ -8,7 +8,7 @@ from watchdog.observers import Observer
 from watchdog.events import PatternMatchingEventHandler
 from yamale import *
 
-from stoobly_agent.config.constants import env_vars, statuses
+from stoobly_agent.config.constants import env_vars
 from stoobly_agent.config.data_dir import DataDir
 from stoobly_agent.config.source_dir import SourceDir
 from stoobly_agent.lib.logger import Logger
@@ -19,6 +19,7 @@ from .remote_settings import RemoteSettings
 from .ui_settings import UISettings
 
 LOG_ID = 'Settings'
+SETTINGS_YML = 'settings.yml'
 
 class Settings:
     _instances = None
@@ -102,7 +103,7 @@ class Settings:
         if self.__watching:
             return False
 
-        patterns = ['settings.yml']
+        patterns = [SETTINGS_YML]
         ignore_patterns = None
         ignore_directories = False
         case_sensitive = True
@@ -180,11 +181,12 @@ class Settings:
         if not contents:
             return
 
-        path = self.__settings_file_path
-        lock = FileLock(path + ".lock")  # lock file alongside the target
+        lock_file = f".{SETTINGS_YML}.lock"
+        lock_file_path = os.path.join(os.path.dirname(self.__settings_file_path), lock_file)
+        lock = FileLock(lock_file_path)  # lock file alongside the target
 
         with lock:
-            with open(path, 'w') as fp:
+            with open(self.__settings_file_path, 'w') as fp:
                 yaml.dump(contents, fp, allow_unicode=True)
 
     ### Helpers
@@ -215,13 +217,14 @@ class Settings:
 
     def __reload_settings(self, event):
         if not self.__load_lock:
-            from stoobly_agent.app.proxy.utils.publish_change_service import publish_change
+            from stoobly_agent.app.proxy.utils.publish_change_service import publish_settings_modified
 
             self.__load_lock = True
 
             Logger.instance(LOG_ID).debug('Reloading settings')
             self.__load_settings()
 
-            publish_change(statuses.SETTINGS_MODIFIED, self.__settings, sync=True)
+            if self.__ui_settings.active:
+                publish_settings_modified(self.__settings)
 
         self.__load_lock = False