stoobly-agent 1.9.12__py3-none-any.whl → 1.10.0__py3-none-any.whl

stoobly_agent/app/proxy/run.py

@@ -16,8 +16,6 @@ tls.DEFAULT_OPTIONS |= 0x4
 from mitmproxy.options import Options
 from mitmproxy.tools.dump import DumpMaster
 
-from stoobly_agent.app.settings import Settings
-
 INTERCEPT_HANDLER_FILENAME = 'intercept_handler.py'
 
 def run(**kwargs):
@@ -67,7 +65,6 @@ def __with_static_options(config: MitmproxyConfig, cli_options):
     config.set(options)
 
 def __with_cli_options(config: MitmproxyConfig, cli_options: dict):
-    __commit_options(cli_options)
     __filter_options(cli_options)
 
     options = []
@@ -86,31 +83,6 @@ def __with_cli_options(config: MitmproxyConfig, cli_options: dict):
 
     config.set(tuple(options))
 
-def __commit_options(options: dict):
-    changed = False
-    service_name = os.environ.get(SERVICE_NAME_ENV)
-    settings = Settings.instance()
-
-    if not service_name or options.get('intercept'):
-        # In the case when service name is set,
-        # only update if intercept is explicitly enabled
-        intercept = not not options.get('intercept')
-        changed = settings.proxy.intercept.active != intercept
-        settings.proxy.intercept.active = intercept
-
-    if not service_name or service_name == CORE_MOCK_UI_SERVICE_NAME:
-        # Causes potentially unintended side effects when run as part of scaffold
-        # Defer to ui service for configuration in this case
-        if options.get('proxy_host') and options.get('proxy_port'):
-            settings.proxy.url = f"http://{options.get('proxy_host')}:{options.get('proxy_port')}"
-
-        settings.ui.active = not options.get('headless')
-
-        changed = True
-
-    if changed:
-        settings.commit()
-
 def __filter_options(options):
     ''' 
     Filter out non-mitmproxy options
@@ -139,6 +111,9 @@ def __filter_options(options):
     if 'intercept' in options:
         del options['intercept']
 
+    if 'intercept_mode' in options:
+        del options['intercept_mode']
+
     if 'lifecycle_hooks_path' in options:
         del options['lifecycle_hooks_path']
 

stoobly_agent/app/proxy/utils/allowed_request_service.py

@@ -5,13 +5,14 @@ from mitmproxy.http import Request as MitmproxyRequest
 from typing import List
 
 from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
+from stoobly_agent.app.settings.constants import intercept_mode
 from stoobly_agent.app.settings.firewall_rule import FirewallRule
-from stoobly_agent.config.constants import intercept_policy, request_origin
+from stoobly_agent.config.constants import mode, intercept_policy, request_origin
 from stoobly_agent.lib.logger import bcolors, Logger
 
 LOG_ID = 'Firewall'
 
-def get_active_mode_policy(request: MitmproxyRequest, intercept_settings: InterceptSettings):
+def get_active_mode_policy(request: MitmproxyRequest, intercept_settings: InterceptSettings) -> str:
     if intercept_settings.request_origin == request_origin.CLI:
         return intercept_settings.policy 
 

stoobly_agent/app/proxy/utils/minimize_headers.py

@@ -0,0 +1,47 @@
+import pdb
+
+from mitmproxy.http import Headers
+from mitmproxy.http import HTTPFlow as MitmproxyHTTPFlow
+
+from typing import Final
+
+
+REQUEST_HEADERS_ALLOWLIST: Final[dict[str]] = {
+  "Accept",
+  "Accept-Encoding",
+  "Accept-Language",
+  "Content-Length",
+  "Content-Type",
+  "Host",
+  "Origin",
+  "Referer",
+  "User-Agent",
+}
+
+RESPONSE_HEADERS_ALLOWLIST: Final[dict[str]] = {
+  "Content-Length",
+  "Content-Type",
+  "Date",
+  "Transfer-Encoding",
+  "Server",  # Sometimes required for HTTP/1.0, but not strictly mandatory
+}
+
+def minimize_headers(flow: MitmproxyHTTPFlow):
+  minimize_request_headers(flow)
+  minimize_response_headers(flow)
+
+def minimize_request_headers(flow: MitmproxyHTTPFlow) -> None:
+  remove_headers(flow.request.headers, REQUEST_HEADERS_ALLOWLIST)
+
+def minimize_response_headers(flow: MitmproxyHTTPFlow) -> None:
+  remove_headers(flow.response.headers, RESPONSE_HEADERS_ALLOWLIST)
+
+def remove_headers(headers: Headers, allowlist: dict[str]):
+  keys_to_remove = []
+
+  for key in headers:
+    if key.lower() not in {allowed_header.lower() for allowed_header in allowlist}:
+      keys_to_remove.append(key)
+
+  for key in keys_to_remove:
+    headers.pop(key)

stoobly_agent/app/proxy/utils/publish_change_service.py

@@ -1,8 +1,10 @@
+import os
 import pdb
 import threading
 
 from typing import TypedDict
 
+from stoobly_agent.app.settings import Settings
 from stoobly_agent.config.constants.statuses import REQUESTS_MODIFIED
 from stoobly_agent.lib.api.agent_api import AgentApi
 from stoobly_agent.lib.cache import Cache
@@ -12,15 +14,14 @@ class Options(TypedDict):
   sync: bool
 
 # Announce that a new request has been created
-def publish_change(status: str, value: any, **options: Options):
+def publish_change(status: str, value, **options: Options):
   if options.get('sync'):
     return __publish_change_sync(status, value)
 
-  from stoobly_agent.app.settings import Settings
-  settings = Settings.instance()
+  settings: Settings = Settings.instance()
 
   # If ui is not active, return
-  if not settings.ui.active:
+  if  not settings.ui.active:
     return False
 
   ui_url = settings.ui.url

stoobly_agent/app/proxy/utils/strategy.py

@@ -0,0 +1,16 @@
+import pdb
+
+from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
+from stoobly_agent.app.settings.constants import intercept_mode
+from stoobly_agent.config.constants import request_origin
+
+
+def get_active_mode_strategy(intercept_settings: InterceptSettings) -> str:
+    strategy = ""
+
+    if intercept_settings.mode == intercept_mode.RECORD:
+        strategy = intercept_settings.record_strategy
+    elif intercept_settings.mode == intercept_mode.TEST:
+        strategy = intercept_settings.test_strategy
+
+    return strategy

stoobly_agent/app/settings/__init__.py

@@ -3,7 +3,7 @@ import pdb
 import time
 import yaml
 
-from shutil import copyfile
+from filelock import FileLock
 from watchdog.observers import Observer
 from watchdog.events import PatternMatchingEventHandler
 from yamale import *
@@ -177,8 +177,14 @@ class Settings:
         self.__load_settings()
 
     def write(self, contents):
-        if contents:
-            with open(self.__settings_file_path, 'w') as fp:
+        if not contents:
+            return
+
+        path = self.__settings_file_path
+        lock = FileLock(path + ".lock")  # lock file alongside the target
+
+        with lock:
+            with open(path, 'w') as fp:
                 yaml.dump(contents, fp, allow_unicode=True)
 
     ### Helpers

stoobly_agent/app/settings/data_rules.py

@@ -1,4 +1,4 @@
-from stoobly_agent.config.constants import mock_policy, record_order, record_policy, replay_policy, test_strategy
+from stoobly_agent.config.constants import mock_policy, record_order, record_policy, record_strategy, replay_policy, test_strategy
 
 from .types.proxy_settings import DataRules as IDataRules
 
@@ -10,6 +10,7 @@ class DataRules:
     self.__mock_policy = self.__data_rules.get('mock_policy') or mock_policy.FOUND
     self.__record_order = self.__data_rules.get('record_order') or record_order.APPEND
     self.__record_policy = self.__data_rules.get('record_policy') or record_policy.ALL
+    self.__record_strategy = self.__data_rules.get('record_strategy') or record_strategy.FULL
     self.__replay_policy = self.__data_rules.get('replay_policy') or replay_policy.ALL
     self.__scenario_key = self.__data_rules.get('scenario_key')
     self.__test_policy = self.__data_rules.get('test_policy') or mock_policy.FOUND
@@ -46,6 +47,19 @@ class DataRules:
     self.__record_order = v
     self.__data_rules['record_order'] = v
 
+  @property
+  def record_strategy(self):
+    return self.__record_strategy
+
+  @record_strategy.setter
+  def record_strategy(self, v):
+    valid_strategies = [record_strategy.FULL, record_strategy.MINIMAL]
+    if v not in valid_strategies:
+      raise TypeError(f"record_strategy has to be one of {valid_strategies}, got {v}")
+
+    self.__record_strategy = v
+    self.__data_rules['record_strategy'] = v
+
   @property
   def replay_policy(self):
     return self.__replay_policy
@@ -79,11 +93,21 @@ class DataRules:
   def test_strategy(self):
     return self.__test_strategy 
 
+  @test_strategy.setter
+  def test_strategy(self, v):
+    valid_strategies = [test_strategy.CONTRACT, test_strategy.CUSTOM, test_strategy.DIFF, test_strategy.FUZZY]
+    if v not in valid_strategies:
+      raise TypeError(f"test_strategy has to be one of {valid_strategies}, got {v}")
+
+    self.__test_strategy = v
+    self.__data_rules['test_strategy'] = v
+
   def to_dict(self) -> IDataRules:
     return {
       'mock_policy': self.__mock_policy,
       'record_order': self.__record_order,
       'record_policy': self.__record_policy,
+      'record_strategy': self.__record_strategy,
       'replay_policy': self.__replay_policy,
       'scenario_key': self.__scenario_key,
       'test_policy': self.__test_policy,

stoobly_agent/app/settings/intercept_settings.py

@@ -19,6 +19,9 @@ class InterceptSettings:
 
   @property
   def active(self):
+    if os.environ.get(env_vars.AGENT_INTERCEPT_ACTIVE):
+      return os.environ[env_vars.AGENT_INTERCEPT_ACTIVE]
+
     return self.__active
 
   @active.setter
@@ -35,8 +38,8 @@ class InterceptSettings:
     if self.__mode != self.mode_before_change:
       return self.__mode
 
-    if os.environ.get(env_vars.AGENT_ACTIVE_MODE):
-        return os.environ[env_vars.AGENT_ACTIVE_MODE]
+    if os.environ.get(env_vars.AGENT_INTERCEPT_MODE):
+        return os.environ[env_vars.AGENT_INTERCEPT_MODE]
         
     return self.__mode
 

stoobly_agent/app/settings/types/__init__.py

@@ -84,7 +84,6 @@ class ISettings(TypedDict):
     api_url: str
     api_key: str
     mode: ISettingsMode
-    proxy_config_path: str
 
 Component = {
     'Header': 'Header',

stoobly_agent/app/settings/ui_settings.py

@@ -21,8 +21,8 @@ class UISettings:
     if self.__active != self.active_before_change:
       return self.__active
 
-    if os.environ.get(env_vars.AGENT_IS_HEADLESS):
-        return True
+    if os.environ.get(env_vars.AGENT_HEADLESS):
+        return False
 
     return self.__active or False
 
@@ -39,8 +39,8 @@ class UISettings:
     if self.__url != self.url_before_change:
         return self.__url
 
-    if os.environ.get(env_vars.AGENT_URL):
-        return os.environ[env_vars.AGENT_URL]
+    if os.environ.get(env_vars.AGENT_UI_URL):
+        return os.environ[env_vars.AGENT_UI_URL]
 
     return self.__url or ''
 
@@ -52,4 +52,4 @@ class UISettings:
     return {
       'active': self.__active,
       'url': self.__url,
-    }
+    }

stoobly_agent/cli.py

@@ -13,9 +13,10 @@ from stoobly_agent.app.proxy.constants import custom_response_codes
 from stoobly_agent.app.proxy.replay.replay_request_service import replay as replay_request
 from stoobly_agent.config.constants import env_vars, mode
 from stoobly_agent.config.data_dir import DataDir
+from stoobly_agent.lib.logger import Logger
 from stoobly_agent.lib.utils.conditional_decorator import ConditionalDecorator
 
-from .app.api import initialize as initialize_api, run as run_api
+from .app.api import run as run_api
 from .app.cli import ca_cert, config, endpoint, feature, intercept, MainGroup, request, scenario, scaffold, snapshot, trace
 from .app.cli.helpers.feature_flags import local, remote
 from .app.settings import Settings
@@ -99,6 +100,7 @@ def init(**kwargs):
 ''')
 @click.option('--headless', is_flag=True, default=False, help='Disable starting UI.')
 @click.option('--intercept', is_flag=True, default=False, help='Enable intercept on run.')
+@click.option('--intercept-mode', help='Set intercept mode.')
 @click.option('--log-level', default=logger.INFO, type=click.Choice([logger.DEBUG, logger.INFO, logger.WARNING, logger.ERROR]), help='''
     Log levels can be "debug", "info", "warning", or "error"
 ''')
@@ -124,7 +126,19 @@ def init(**kwargs):
 def run(**kwargs):
     from .app.proxy.run import run as run_proxy
 
-    os.environ[env_vars.AGENT_PROXY_URL] = f"http://{kwargs['proxy_host']}:{kwargs['proxy_port']}"
+    # Observe config for changes
+    settings: Settings = Settings.instance()
+    settings.watch()
+
+    if kwargs.get('headless'):
+      os.environ[env_vars.AGENT_HEADLESS] = '1'
+
+    if kwargs.get('intercept'):
+      os.environ[env_vars.AGENT_INTERCEPT_ACTIVE] = '1'
+
+    if kwargs.get('intercept_mode'):
+      os.environ[env_vars.AGENT_INTERCEPT_MODE] = kwargs['intercept_mode']
+      settings.proxy.intercept.mode = kwargs['intercept_mode']
 
     if kwargs.get('lifecycle_hooks_path'):
       os.environ[env_vars.AGENT_LIFECYCLE_HOOKS_PATH] = kwargs['lifecycle_hooks_path']
@@ -135,21 +149,32 @@ def run(**kwargs):
     if kwargs.get('response_fixtures_path'):
       os.environ[env_vars.AGENT_RESPONSE_FIXTURES_PATH] = kwargs['response_fixtures_path']
 
-    # Observe config for changes
-    Settings.instance().watch()
-
     if not os.getenv(env_vars.LOG_LEVEL):
-        os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
-
-    if 'api_url' in kwargs and kwargs['api_url']:
-        os.environ[env_vars.API_URL] = kwargs['api_url']
-
-    url = initialize_api(**kwargs)
-    if 'headless' in kwargs and not kwargs['headless']:
-        run_api(url)
-
-    if 'proxyless' in kwargs and not kwargs['proxyless']:
-        run_proxy(**kwargs)
+      os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
+
+    if kwargs.get('api_url'):
+      os.environ[env_vars.API_URL] = kwargs['api_url']
+
+    if not kwargs.get('headless'):
+      ui_url = f"http://{kwargs['ui_host']}:{kwargs['ui_port']}"
+      os.environ[env_vars.AGENT_UI_URL] = ui_url
+      settings.ui.active = True
+      settings.ui.url = ui_url
+
+    if not kwargs.get('proxyless'):
+      proxy_url = f"http://{kwargs['proxy_host']}:{kwargs['proxy_port']}"
+      os.environ[env_vars.AGENT_PROXY_URL] = proxy_url
+      settings.proxy.url = proxy_url
+
+    if not kwargs.get('headless'):
+      settings.commit()
+      run_api(**kwargs)
+
+    if not kwargs.get('proxyless'):
+      log_id = 'Proxy'
+      Logger.instance(log_id).info(f"starting with mode {kwargs['proxy_mode']} and listening at {kwargs['proxy_host']}:{kwargs['proxy_port']}")
+      Logger.instance(log_id).info(f"{'' if settings.proxy.intercept.active else 'not yet '}configured to {settings.proxy.intercept.mode}")
+      run_proxy(**kwargs)
 
 @main.command(
   help="Mock request"

stoobly_agent/config/constants/custom_headers.py

@@ -13,6 +13,7 @@ PROJECT_KEY = 'X-Stoobly-Project-Key'
 PROXY_MODE = 'X-Stoobly-Proxy-Mode'
 PUBLIC_DIRECTORY_PATH = 'X-Stoobly-Public-Directory-Path'
 RECORD_POLICY = 'X-Stoobly-Record-Policy'
+RECORD_STRATEGY = 'X-Stoobly-Record-Strategy'
 REPORT_KEY = 'X-Stoobly-Report-Key'
 REQUEST_ORIGIN = 'X-Stoobly-Request-Origin'
 RESPONSE_FIXTURES_PATH = 'X-Stoobly-Response-Fixtures-Path'

stoobly_agent/config/constants/env_vars.py

@@ -1,9 +1,10 @@
-AGENT_ACTIVE_MODE = 'STOOBLY_AGENT_ACTIVE_MODE'
 AGENT_CONFIG_PATH = 'STOOBLY_AGENT_CONFIG_PATH'
 AGENT_ENABLED = 'STOOBLY_AGENT_ENABLED'
 AGENT_SELF_INTERCEPT_ENABLED = 'STOOBLY_AGENT_SELF_INTERCEPT_ENABLED'
+AGENT_HEADLESS = 'STOOBLY_AGENT_HEADLESS'
 AGENT_INCLUDE_PATTERNS = 'STOOBLY_AGENT_INCLUDE_PATTERNS'
-AGENT_IS_HEADLESS = 'STOOBLY_AGENT_IS_HEADLESS'
+AGENT_INTERCEPT_ACTIVE = 'STOOBLY_AGENT_INTERCEPT_ACTIVE'
+AGENT_INTERCEPT_MODE = 'STOOBLY_AGENT_INTERCEPT_MODE'
 AGENT_EXCLUDE_PATTERNS = 'STOOBLY_AGENT_EXCLUDE_PATTERNS'
 AGENT_LIFECYCLE_HOOKS_PATH = 'STOOBLY_AGENT_LIFECYCLE_HOOKS_PATH'
 AGENT_POLICY = 'STOOBLY_AGENT_POLICY'
@@ -18,7 +19,7 @@ AGENT_RESPONSE_FIXTURES_PATH = 'STOOBLY_AGENT_RESPONSE_FIXTURES_PATH'
 AGENT_SERVICE_URL = 'STOOBLY_AGENT_SERVICE_URL'
 AGENT_SCENARIO_KEY = 'STOOBLY_AGENT_SCENARIO_KEY'
 AGENT_SIMULATE_LATENCY = 'STOOBLY_AGENT_SIMULATE_LATENCY'
-AGENT_URL = 'STOOBLY_AGENT_URL'
+AGENT_UI_URL = 'STOOBLY_AGENT_UI_URL'
 API_URL = 'STOOBLY_API_URL'
 API_KEY = 'STOOBLY_API_KEY'
 ENV = 'STOOBLY_AGENT_ENV'

stoobly_agent/config/constants/record_strategy.py

@@ -0,0 +1,6 @@
+from typing import Literal
+
+FULL = 'full'
+MINIMAL = 'minimal'
+
+RecordStrategy = Literal[FULL, MINIMAL]

stoobly_agent/config/settings.yml.sample

@@ -12,10 +12,9 @@ proxy:
         project_key: eyJpIjowLCJvIjowfQ==
     firewall: {}
     rewrite: {}
-proxy_config_path: config/settings.yml
 remote: 
     api_key: ''
-    api_url: 'http://localhost:3000'
+    api_url: ''
 ui:
     active: false
-    url: http://localhost:4200
+    url: ''

stoobly_agent/lib/logger.py

@@ -27,6 +27,7 @@ LogLevel = Literal[DEBUG, ERROR, INFO, WARNING]
 
 class Logger:
     _instance = None
+    _instances = {}
 
     def __init__(self):
         raise RuntimeError('Call instance() instead')
@@ -34,14 +35,23 @@ class Logger:
     @classmethod
     def instance(cls, name = None):
         if cls._instance is None:
+            logging.getLogger("mitmproxy").setLevel(logging.ERROR)
             cls._instance = cls.__new__(cls)
-
-        cls._instance.load()
+            cls._instance.load()
 
         if not name:
-            return logging
-        else:
-            return logging.getLogger(name)
+            name = 'root'
+
+        logger = logging.getLogger(name)
+
+        if name not in cls._instances:
+            handler = logging.StreamHandler()
+            handler.setFormatter(logging.Formatter(f"[%(levelname)s] {name} %(message)s"))
+            logger.addHandler(handler)
+            logger.propagate = False
+            cls._instances[name] = logger
+
+        return logger
 
     @classmethod
     def reload(cls):