stoobly-agent 1.10.1__py3-none-any.whl → 1.10.2__py3-none-any.whl

stoobly_agent/app/proxy/mock/eval_fixtures_service.py

@@ -2,25 +2,23 @@ import mimetypes
 import os
 import pdb
 import re
+import yaml
 
 from io import BytesIO
 from mitmproxy.http import Request as MitmproxyRequest
 from requests import Response
 from requests.structures import CaseInsensitiveDict
-from typing import Optional, Union
+from typing import List, Optional, Union
+from urllib.parse import urlparse
 
 from stoobly_agent.lib.logger import bcolors, Logger
 from stoobly_agent.config.constants.custom_headers import MOCK_FIXTURE_PATH
 
-from .types import Fixtures
+from .types import Fixtures, MockOptions, PublicDirectoryPath, ResponseFixturesPath
 
 LOG_ID = 'Fixture'
 
-class Options():
-  public_directory_path: str
-  response_fixtures: Fixtures
-
-def eval_fixtures(request: MitmproxyRequest, **options: Options) -> Union[Response, None]:
+def eval_fixtures(request: MitmproxyRequest, **options: MockOptions) -> Union[Response, None]:
   fixture_path = request.headers.get(MOCK_FIXTURE_PATH)
   headers = CaseInsensitiveDict()
   status_code = 200
@@ -30,23 +28,49 @@ def eval_fixtures(request: MitmproxyRequest, **options: Options) -> Union[Respon
       return
   else:
     response_fixtures = options.get('response_fixtures')
-    fixture: dict = __eval_response_fixtures(request, response_fixtures)
+    
+    # Try response fixtures in order of preference
+    fixture = None
 
-    if not fixture:
-      public_directory_path = options.get('public_directory_path')
+    if response_fixtures:
+      fixture = __find_fixture_for_request(request, response_fixtures, request.method)
 
-      if not public_directory_path:
+    if options.get('response_fixtures_path'):
+      fixture = __eval_response_fixtures_from_paths(request, options['response_fixtures_path'])
+    
+    if not fixture:
+      raw_paths = options.get('public_directory_path')
+      public_directory_paths = __parse_public_directory_paths(raw_paths)
+      
+      if not public_directory_paths:
         return
 
       request_path = 'index' if request.path == '/' else request.path
-      _fixture_path = os.path.join(public_directory_path, request_path.lstrip('/'))
-      if request.headers.get('accept'):
-        fixture_path = __guess_file_path(_fixture_path, request.headers['accept'])
-
+      # Extract origin from request URL (e.g., https://example.com/path -> https://example.com)
+      request_origin = __request_origin(request)
+      
+      # Try to find a matching file in the public directory paths
+      fixture_path = None
+      for dir_path_config in public_directory_paths:
+        # Check if origin matches (if origin is specified)
+        if dir_path_config.get('origin') and request_origin:
+          if not __origin_matches(dir_path_config['origin'], request_origin):
+            continue
+        
+        # Try to find the file in this directory
+        _fixture_path = os.path.join(dir_path_config['path'], request_path.lstrip('/'))
+        if request.headers.get('accept'):
+          fixture_path = __guess_file_path(_fixture_path, request.headers['accept'])
+        
+        if not fixture_path:
+          fixture_path = _fixture_path
+        
+        if os.path.isfile(fixture_path):
+          break
+        else:
+          fixture_path = None
+      
       if not fixture_path:
-        fixture_path = _fixture_path
-
-      if not os.path.isfile(fixture_path):
         return
     else:
       fixture_path = fixture.get('path')
@@ -81,6 +105,53 @@ def eval_fixtures(request: MitmproxyRequest, **options: Options) -> Union[Respon
 
     return response
 
+def __eval_response_fixtures_from_paths(request: MitmproxyRequest, fixtures_paths: str):
+  """Iterate through response fixtures paths and return the first matching fixture."""
+  if not fixtures_paths:
+    return None
+
+  # Parse multiple response fixtures paths with optional origin specification
+  parsed_paths = __parse_response_fixtures_paths(fixtures_paths)
+  
+  # Extract origin from request URL
+  request_origin = __request_origin(request)
+  method = request.method
+
+  # Iterate through each fixtures path
+  for path_config in parsed_paths:
+    fixtures_path = path_config['path']
+    origin = path_config.get('origin')
+    
+    # If origin is specified, check if it matches the request origin
+    if origin and request_origin:
+      if not __origin_matches(origin, request_origin):
+        continue  # Skip this file if origin doesn't match
+    elif origin and not request_origin:
+      continue  # Skip origin-specific files if request has no origin
+    
+    # Load and parse this specific fixtures file
+    if not os.path.exists(fixtures_path):
+      Logger.instance(LOG_ID).error(f"Response fixtures {fixtures_path} does not exist")
+      continue
+
+    try:
+      with open(fixtures_path, 'r') as stream:
+        fixtures = yaml.safe_load(stream) or {}
+        
+        # Try to find a matching fixture in this file
+        fixture = __find_fixture_for_request(request, fixtures, method)
+        if fixture:
+          # Convert fixture path to absolute path if it's a relative path and if 'path' exists in the fixture
+          if fixture.get('path') and not os.path.isabs(fixture['path']):
+            fixture['path'] = os.path.join(os.path.dirname(fixtures_path), fixture['path'])
+          return fixture  # Return immediately on first match
+          
+    except yaml.YAMLError as exc:
+      Logger.instance(LOG_ID).error(f"Error parsing {fixtures_path}: {exc}")
+      continue
+  
+  return None  # No matching fixture found in any file
+
 def __guess_content_type(file_path):
   file_extension = os.path.splitext(file_path)[1]
   if not file_extension:
@@ -107,18 +178,22 @@ def __guess_file_path(file_path, content_type):
     if os.path.isfile(_file_path):
       return _file_path
 
-def __eval_response_fixtures(request: MitmproxyRequest, response_fixtures: Fixtures):
-  if not isinstance(response_fixtures, dict):
-    return
-
-  method = request.method
-  routes = response_fixtures.get(method)
-
+def __find_fixture_for_request(request: MitmproxyRequest, fixtures: dict, method: str):
+  """Find a fixture for the given request in the provided fixtures."""
+  if not fixtures:
+    return None
+  
+  return __find_fixture_in_routes(fixtures, method, request.path)
+
+def __find_fixture_in_routes(fixtures: dict, method: str, request_path: str):
+  """Find a fixture for the given method and path in the provided fixtures."""
+  routes = fixtures.get(method)
+  
   if not isinstance(routes, dict):
-    return
+    return None
 
   for path_pattern in routes:
-    if not re.match(path_pattern, request.path):
+    if not re.match(path_pattern, request_path):
       continue
       
     fixture = routes[path_pattern]
@@ -129,6 +204,8 @@ def __eval_response_fixtures(request: MitmproxyRequest, response_fixtures: Fixtu
 
     if path:
       return fixture
+  
+  return None
 
 def __choose_highest_priority_content_type(accept_header: str) -> Optional[str]:
     if not accept_header:
@@ -156,6 +233,9 @@ def __choose_highest_priority_content_type(accept_header: str) -> Optional[str]:
     types.sort(key=lambda x: -x[1])
     return types[0][0] if types else None
 
+def __origin_matches(pattern: str, request_origin: str) -> bool:
+    return bool(re.match(pattern, request_origin))
+
 def __parse_accept_header(accept_header):
     types = []
     for item in accept_header.split(","):
@@ -171,3 +251,73 @@ def __parse_accept_header(accept_header):
 
     # Sort by quality factor in descending order
     return [content_type for content_type, _ in sorted(types, key=lambda x: x[1], reverse=True)]
+
+def __parse_origin_path_item(path_item: str):
+  """Parse a single path:origin item and return (path, origin) tuple."""
+  path_item = path_item.strip()
+  
+  # Check if this looks like a path:origin format
+  # Format: <FILE-PATH>:<ORIGIN> where ORIGIN is scheme:hostname:port
+  # We need to find the colon that separates path from origin
+  if '://' in path_item:
+    # Find the colon that separates path from origin
+    # Origin format: scheme:hostname:port (e.g., https://api.example.com:8080)
+    origin_start = path_item.rfind('://')
+    if origin_start > 0:
+      # Look for the colon before the scheme
+      colon_before_scheme = path_item.rfind(':', 0, origin_start)
+      if colon_before_scheme != -1:
+        # Format: path:origin
+        path = path_item[:colon_before_scheme]
+        origin = path_item[colon_before_scheme + 1:]
+        return (path.strip(), origin.strip())
+    
+    # No colon before scheme found, treat entire string as path
+    return (path_item, None)
+  else:
+    # Check for path:hostname:port format (without scheme)
+    colons = [i for i, char in enumerate(path_item) if char == ':']
+    
+    if len(colons) >= 1:
+      # Find the first colon that separates path from origin
+      first_colon_idx = colons[0]
+      potential_origin = path_item[first_colon_idx + 1:]
+      
+      # Check if this looks like a hostname:port format
+      # A valid hostname:port should have another colon for the port
+      if ':' in potential_origin:
+        # Format: path:hostname:port
+        path = path_item[:first_colon_idx]
+        origin = potential_origin
+        return (path.strip(), origin.strip())
+    
+    # No valid path:origin format found, treat entire string as path
+    return (path_item, None)
+
+def __parse_public_directory_paths(raw_paths: str) -> List[PublicDirectoryPath]:
+  """Parse public directory paths from comma-separated string."""
+  if not raw_paths:
+    return []
+  
+  paths = []
+  for path_item in raw_paths.split(','):
+    path, origin = __parse_origin_path_item(path_item)
+    paths.append(PublicDirectoryPath(origin=origin, path=path))
+  
+  return paths
+
+def __parse_response_fixtures_paths(raw_paths: str) -> List[ResponseFixturesPath]:
+  """Parse response fixtures paths from comma-separated string."""
+  if not raw_paths:
+    return []
+  
+  paths = []
+  for path_item in raw_paths.split(','):
+    path, origin = __parse_origin_path_item(path_item)
+    paths.append(ResponseFixturesPath(origin=origin, path=path))
+  
+  return paths
+
+def __request_origin(request: MitmproxyRequest) -> str:
+  parsed_url = urlparse(request.url)
+  return f"{parsed_url.scheme}://{parsed_url.hostname}" + (f":{parsed_url.port}" if parsed_url.port else "")

stoobly_agent/app/proxy/mock/types/__init__.py

@@ -1,4 +1,4 @@
-from typing import TypedDict
+from typing import TypedDict, Optional
 
 class Route(TypedDict):
   path: str
@@ -8,3 +8,24 @@ class Fixtures(TypedDict):
   GET: Route
   POST: Route
   PUT: Route
+
+class PublicDirectoryPath(TypedDict):
+  """Represents a public directory path with optional origin specification.
+  
+  Format: <FOLDER-PATH>:<ORIGIN> where ORIGIN is scheme:hostname:port
+  """
+  origin: Optional[str]  # Optional origin (e.g., "https://api.example.com:8080")
+  path: str              # File system path to the public directory
+
+class ResponseFixturesPath(TypedDict):
+  """Represents a response fixtures file path with optional origin specification.
+  
+  Format: <FILE-PATH>:<ORIGIN> where ORIGIN is scheme:hostname:port
+  """
+  origin: Optional[str]  # Optional origin (e.g., "https://api.example.com:8080")
+  path: str              # File system path to the response fixtures file
+
+class MockOptions(TypedDict):
+  """Options for mock service."""
+  public_directory_path: str
+  response_fixtures_path: str                        # Comma-separated list of paths, optionally with origin prefix

stoobly_agent/app/proxy/replay/body_parser_service.py

@@ -69,13 +69,16 @@ def parse_json(content):
         return content
 
 def parse_multipart_form_data(content, content_type) -> Dict[bytes, bytes]:
-    headers = {'content-type': content_type}
-    decoded_multipart = multipart_decode(headers, content)
+    try:
+        headers = {'content-type': content_type}
+        decoded_multipart = multipart_decode(headers, content)
 
-    if not decoded_multipart:
+        if not decoded_multipart:
+            return content
+    
+        return MultiDict(decoded_multipart)
+    except:
         return content
- 
-    return MultiDict(decoded_multipart)
 
 def parse_www_form_urlencoded(content):
     try:

stoobly_agent/app/proxy/replay/multipart.py

@@ -3,7 +3,7 @@ import mimetypes
 import re
 import pdb
 
-from multipart import MultipartParser
+from multipart import MultipartParser, ParserError
 from urllib.parse import quote
 
 from mitmproxy.net.http import headers
@@ -51,29 +51,31 @@ def decode(hdrs, content):
         Takes a multipart boundary encoded string and returns list of (key, value) tuples.
     """
     if not isinstance(content, bytes) and not isinstance(content, str):
-        return
+        return content
 
     v = hdrs.get("content-type")
     if not v:
-        return
+        return content
 
     v = headers.parse_content_type(v)
     if not v:
-        return
+        return content
 
     try:
         boundary = v[2]["boundary"].encode("ascii")
     except (KeyError, UnicodeError):
-        return
+        return content
 
     r = []
-    parser = MultipartParser(io.BytesIO(content), boundary=boundary)
-    for part in parser.parts():
-        if part.content_type.lower() == 'application/octet-stream':
+
+    try:
+        parser = MultipartParser(io.BytesIO(content), boundary=boundary)
+
+        for part in parser.parts():
             r.append((part.name, part.raw))
-        else:
-            r.append((part.name, part.value))
 
-        # Free up resources after use
-        part.close()
-    return r
+            # Free up resources after use
+            part.close()
+        return r
+    except ParserError:
+        return content

stoobly_agent/app/proxy/replay/replay_request_service.py

@@ -27,11 +27,11 @@ class ReplayRequestOptions(TypedDict):
   after_replay: Union[Callable[[ReplayContext], Union[requests.Response, None]], None]
   project_key: Union[str, None]
   proxies: dict
-  public_directory_path: str
+  public_directory_path: str  # Comma-separated list of paths, optionally with origin prefix
   remote_project_key: str
   report_key: Union[str, None] 
   request_origin: Union[request_origin.CLI, None] 
-  response_fixtures_path: str
+  response_fixtures_path: str  # Comma-separated list of paths, optionally with origin prefix
   response_mode: Union[mode.RECORD, None]
   scenario_key: Union[str, None] 
   scheme: str

stoobly_agent/app/proxy/run.py

@@ -105,6 +105,9 @@ def __filter_options(options):
     if 'cert_passphrase' in options and not options['cert_passphrase']:
         del options['cert_passphrase']
 
+    if 'detached' in options:
+        del options['detached']
+
     if 'headless' in options:
         del options['headless']
 

stoobly_agent/app/proxy/test/context.py

@@ -166,10 +166,6 @@ class TestContext(TestContextABC):
   def response(self) -> TestContextResponse:
     return self.__response
 
-  @property
-  def response_fixtures(self):
-    return self.__intercept_settings.response_fixtures
-
   @property
   def response_fixtures_path(self):
     return self.__intercept_settings.response_fixtures_path

stoobly_agent/app/proxy/test/context_abc.py

@@ -149,11 +149,6 @@ class TestContextABC(abc.ABC):
   def response(self) -> TestContextResponse:
     pass
 
-  @property
-  @abc.abstractmethod
-  def response_fixtures(self):
-    pass
-
   @property
   @abc.abstractmethod
   def response_fixtures_path(self):

stoobly_agent/cli.py

@@ -9,8 +9,10 @@ from stoobly_agent import VERSION
 from stoobly_agent.app.cli.helpers.context import ReplayContext
 from stoobly_agent.app.cli.helpers.handle_mock_service import print_raw_response, RAW_FORMAT
 from stoobly_agent.app.cli.helpers.validations import validate_project_key, validate_scenario_key
+from stoobly_agent.app.cli.intercept_cli import mode_options
 from stoobly_agent.app.proxy.constants import custom_response_codes
 from stoobly_agent.app.proxy.replay.replay_request_service import replay as replay_request
+from stoobly_agent.app.settings.constants import intercept_mode
 from stoobly_agent.config.constants import env_vars, mode
 from stoobly_agent.config.data_dir import DataDir
 from stoobly_agent.lib.logger import Logger
@@ -80,7 +82,7 @@ def init(**kwargs):
     help="Run proxy and/or UI",
 )
 @ConditionalDecorator(lambda f: click.option('--api-url', help='API URL.')(f), is_remote)
-@click.option('--ca-certs-dir-path', default=DataDir.instance().ca_certs_dir_path, help='Path to ca certs directory used to sign SSL certs.')
+@click.option('--ca-certs-dir-path', default=os.path.join(os.path.expanduser('~'), '.mitmproxy'), help='Path to ca certs directory used to sign SSL certs.')
 @click.option('--certs', help='''
   SSL certificates of the form "[domain=]path". The domain may include a wildcard, and is equal to "*" if not specified. The file at path is a certificate in PEM format. If a private key is included in the
   PEM, it is used, else the default key in the conf dir is used. The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. May be passed multiple times.
@@ -90,6 +92,7 @@ def init(**kwargs):
   config.yaml to avoid this.
 ''')
 @click.option('--connection-strategy', help=', '.join(CONNECTION_STRATEGIES), type=click.Choice(CONNECTION_STRATEGIES))
+@click.option('--detached', type=click.Path(), help='Run in detached mode and redirect output to the specified file path.')
 @click.option('--flow-detail', default='1', type=click.Choice(['0', '1', '2', '3', '4']), help='''
   The display detail level for flows in mitmdump: 0 (quiet) to 4 (very verbose).
   0: no output
@@ -100,7 +103,7 @@ def init(**kwargs):
 ''')
 @click.option('--headless', is_flag=True, default=False, help='Disable starting UI.')
 @click.option('--intercept', is_flag=True, default=False, help='Enable intercept on run.')
-@click.option('--intercept-mode', help='Set intercept mode.')
+@click.option('--intercept-mode', type=click.Choice(mode_options), help='Set intercept mode.')
 @click.option('--log-level', default=logger.INFO, type=click.Choice([logger.DEBUG, logger.INFO, logger.WARNING, logger.ERROR]), help='''
     Log levels can be "debug", "info", "warning", or "error"
 ''')
@@ -118,8 +121,8 @@ def init(**kwargs):
   the form of "http[s]://host[:port]".
 ''')
 @click.option('--proxy-port', default=8080, type=click.IntRange(1, 65535), help='Proxy service port.')
-@click.option('--public-directory-path', help='Path to public files. Used for mocking requests.')
-@click.option('--response-fixtures-path', help='Path to response fixtures yaml. Used for mocking requests.')
+@click.option('--public-directory-path', multiple=True, help='Path to public files. Used for mocking requests. Can take the form <FOLDER-PATH>[:<ORIGIN>].')
+@click.option('--response-fixtures-path', multiple=True, help='Path to response fixtures yaml. Used for mocking requests. Can take the form <FILE-PATH>[:<ORIGIN>].')
 @click.option('--ssl-insecure', is_flag=True, default=False, help='Do not verify upstream server SSL/TLS certificates.')
 @click.option('--ui-host', default='0.0.0.0', help='Address to bind UI to.')
 @click.option('--ui-port', default=4200, type=click.IntRange(1, 65535), help='UI service port.')
@@ -130,6 +133,9 @@ def run(**kwargs):
     settings: Settings = Settings.instance()
     settings.watch()
 
+    if not os.path.exists(kwargs.get('ca_certs_dir_path')):
+      kwargs['ca_certs_dir_path'] = DataDir.instance().ca_certs_dir_path
+
     if kwargs.get('headless'):
       os.environ[env_vars.AGENT_HEADLESS] = '1'
 
@@ -144,10 +150,18 @@ def run(**kwargs):
       os.environ[env_vars.AGENT_LIFECYCLE_HOOKS_PATH] = kwargs['lifecycle_hooks_path']
 
     if kwargs.get('public_directory_path'):
-      os.environ[env_vars.AGENT_PUBLIC_DIRECTORY_PATH] = kwargs['public_directory_path']
+      # Join multiple paths with commas
+      public_dirs = kwargs['public_directory_path']
+      if isinstance(public_dirs, (list, tuple)):
+        os.environ[env_vars.AGENT_PUBLIC_DIRECTORY_PATH] = ','.join(public_dirs)
+      else:
+        os.environ[env_vars.AGENT_PUBLIC_DIRECTORY_PATH] = public_dirs
 
     if kwargs.get('response_fixtures_path'):
-      os.environ[env_vars.AGENT_RESPONSE_FIXTURES_PATH] = kwargs['response_fixtures_path']
+      response_fixtures_paths = kwargs.get('response_fixtures_path', ())
+      if response_fixtures_paths:
+        response_fixtures = ','.join(response_fixtures_paths)
+        os.environ[env_vars.AGENT_RESPONSE_FIXTURES_PATH] = response_fixtures
 
     if not os.getenv(env_vars.LOG_LEVEL):
       os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
@@ -166,15 +180,46 @@ def run(**kwargs):
       os.environ[env_vars.AGENT_PROXY_URL] = proxy_url
       settings.proxy.url = proxy_url
 
-    if not kwargs.get('headless'):
-      settings.commit()
-      run_api(**kwargs)
+    if kwargs.get('detached'):
+      # Run in detached mode with output redirection
+      import subprocess
+      import sys
+      
+      # Build the command to run in background
+      cmd = [sys.executable, '-m', 'stoobly_agent'] + sys.argv[1:]
+      # Remove the --detached flag and its value from the command
+      detached_index = None
+      for i, arg in enumerate(cmd):
+        if arg == '--detached':
+          detached_index = i
+          break
+      if detached_index is not None:
+        cmd.pop(detached_index)  # Remove --detached
+        if detached_index < len(cmd) and not cmd[detached_index].startswith('--'):
+          cmd.pop(detached_index)  # Remove the file path
+      
+      # Start the process in background with output redirection
+      with open(kwargs['detached'], 'w') as output_file:
+        process = subprocess.Popen(
+          cmd,
+          stdout=output_file,
+          stderr=subprocess.STDOUT,  # Redirect stderr to stdout
+          preexec_fn=os.setsid  # Create new process group
+        )
+      
+      print(process.pid)
+      return
+    else:
+      # Run in foreground mode
+      if not kwargs.get('headless'):
+        settings.commit()
+        run_api(**kwargs)
 
-    if not kwargs.get('proxyless'):
-      log_id = 'Proxy'
-      Logger.instance(log_id).info(f"starting with mode {kwargs['proxy_mode']} and listening at {kwargs['proxy_host']}:{kwargs['proxy_port']}")
-      Logger.instance(log_id).info(f"{'' if settings.proxy.intercept.active else 'not yet '}configured to {settings.proxy.intercept.mode}")
-      run_proxy(**kwargs)
+      if not kwargs.get('proxyless'):
+        log_id = 'Proxy'
+        Logger.instance(log_id).info(f"starting with mode {kwargs['proxy_mode']} and listening at {kwargs['proxy_host']}:{kwargs['proxy_port']}")
+        Logger.instance(log_id).info(f"{'' if settings.proxy.intercept.active else 'not yet '}configured to {settings.proxy.intercept.mode}")
+        run_proxy(**kwargs)
 
 @main.command(
   help="Mock request"
@@ -186,8 +231,8 @@ def run(**kwargs):
 @click.option('--lifecycle-hooks-path', help='Path to lifecycle hooks script.')
 @click.option('-o', '--output', help='Write to file instead of stdout')
 @ConditionalDecorator(lambda f: click.option('--project-key')(f), is_remote)
-@click.option('--public-directory-path', help='Path to public files. Used for mocking requests.')
-@click.option('--response-fixtures-path', help='Path to response fixtures yaml. Used for mocking requests.')
+@click.option('--public-directory-path', multiple=True, help='Path to public files. Used for mocking requests. Can take the form <FOLDER-PATH>[:<ORIGIN>].')
+@click.option('--response-fixtures-path', multiple=True, help='Path to response fixtures yaml. Used for mocking requests. Can take the form <FILE-PATH>[:<ORIGIN>].')
 @click.option('-X', '--request', default='GET', help='Specify request command to use')
 @click.option('--scenario-key')
 @click.argument('url')

stoobly_agent/config/data_dir.py

@@ -56,14 +56,6 @@ class DataDir:
 
     @property
     def path(self):
-        if not self.__path and os.environ.get(ENV) == 'test':
-            test_path = os.path.join(self.__data_dir_path, TMP_DIR_NAME, DATA_DIR_NAME)
-
-            if not os.path.exists(test_path):
-                os.makedirs(test_path, exist_ok=True)
-
-            return test_path
-
         return self.__data_dir_path
 
     @property