stoobly-agent 1.9.12__py3-none-any.whl → 1.10.1__py3-none-any.whl

stoobly_agent/app/proxy/utils/minimize_headers.py

@@ -0,0 +1,47 @@
+import pdb
+
+from mitmproxy.http import Headers
+from mitmproxy.http import HTTPFlow as MitmproxyHTTPFlow
+
+from typing import Final
+
+
+REQUEST_HEADERS_ALLOWLIST: Final[dict[str]] = {
+  "Accept",
+  "Accept-Encoding",
+  "Accept-Language",
+  "Content-Length",
+  "Content-Type",
+  "Host",
+  "Origin",
+  "Referer",
+  "User-Agent",
+}
+
+RESPONSE_HEADERS_ALLOWLIST: Final[dict[str]] = {
+  "Content-Length",
+  "Content-Type",
+  "Date",
+  "Transfer-Encoding",
+  "Server",  # Sometimes required for HTTP/1.0, but not strictly mandatory
+}
+
+def minimize_headers(flow: MitmproxyHTTPFlow):
+  minimize_request_headers(flow)
+  minimize_response_headers(flow)
+
+def minimize_request_headers(flow: MitmproxyHTTPFlow) -> None:
+  remove_headers(flow.request.headers, REQUEST_HEADERS_ALLOWLIST)
+
+def minimize_response_headers(flow: MitmproxyHTTPFlow) -> None:
+  remove_headers(flow.response.headers, RESPONSE_HEADERS_ALLOWLIST)
+
+def remove_headers(headers: Headers, allowlist: dict[str]):
+  keys_to_remove = []
+
+  for key in headers:
+    if key.lower() not in {allowed_header.lower() for allowed_header in allowlist}:
+      keys_to_remove.append(key)
+
+  for key in keys_to_remove:
+    headers.pop(key)

stoobly_agent/app/proxy/utils/publish_change_service.py

@@ -1,40 +1,35 @@
 import pdb
+import requests
 import threading
 
-from typing import TypedDict
-
-from stoobly_agent.config.constants.statuses import REQUESTS_MODIFIED
+from stoobly_agent.app.settings import Settings
+from stoobly_agent.config.constants.statuses import REQUESTS_MODIFIED, SETTINGS_MODIFIED
 from stoobly_agent.lib.api.agent_api import AgentApi
 from stoobly_agent.lib.cache import Cache
 from stoobly_agent.lib.logger import Logger
 
-class Options(TypedDict):
-  sync: bool
-
-# Announce that a new request has been created
-def publish_change(status: str, value: any, **options: Options):
-  if options.get('sync'):
-    return __publish_change_sync(status, value)
+LOG_ID = 'PublishChange'
 
-  from stoobly_agent.app.settings import Settings
-  settings = Settings.instance()
+def publish_settings_modified(value):
+  return __publish_change_sync(SETTINGS_MODIFIED, value)
+ 
+def publish_requests_modified(value):
+  settings: Settings = Settings.instance()
 
-  # If ui is not active, return
-  if not settings.ui.active:
-    return False
+  # If not headless...
+  if settings.ui.active:
+    return __publish_change_sync(REQUESTS_MODIFIED, value)
 
   ui_url = settings.ui.url
-
   if not ui_url:
-    Logger.instance().warn('Settings.ui.url not configured')
     return False
-  else:
-    thread = threading.Thread(target=__put_status, args=(ui_url, status, value))
-    thread.start()
-    return True
 
-def publish_requests_modified(value, **options: Options):
-  return publish_change(REQUESTS_MODIFIED, value, **options)
+  return __publish_change_async(REQUESTS_MODIFIED, value, ui_url)
+
+def __publish_change_async(status, value, ui_url: str):
+  thread = threading.Thread(target=__put_status, args=(ui_url, status, value))
+  thread.start()
+  return True
 
 def __publish_change_sync(status: str, value):
   cache = Cache.instance()
@@ -43,4 +38,7 @@ def __publish_change_sync(status: str, value):
 
 def __put_status(ui_url, status, value):
     api: AgentApi = AgentApi(ui_url)
-    api.update_status(status, value)
+    try:
+      api.update_status(status, value)
+    except requests.exceptions.ConnectionError:
+      Logger.instance(LOG_ID).error(f"could not connect to {ui_url}")

stoobly_agent/app/proxy/utils/strategy.py

@@ -0,0 +1,16 @@
+import pdb
+
+from stoobly_agent.app.proxy.intercept_settings import InterceptSettings
+from stoobly_agent.app.settings.constants import intercept_mode
+from stoobly_agent.config.constants import request_origin
+
+
+def get_active_mode_strategy(intercept_settings: InterceptSettings) -> str:
+    strategy = ""
+
+    if intercept_settings.mode == intercept_mode.RECORD:
+        strategy = intercept_settings.record_strategy
+    elif intercept_settings.mode == intercept_mode.TEST:
+        strategy = intercept_settings.test_strategy
+
+    return strategy

stoobly_agent/app/settings/__init__.py

@@ -3,12 +3,12 @@ import pdb
 import time
 import yaml
 
-from shutil import copyfile
+from filelock import FileLock
 from watchdog.observers import Observer
 from watchdog.events import PatternMatchingEventHandler
 from yamale import *
 
-from stoobly_agent.config.constants import env_vars, statuses
+from stoobly_agent.config.constants import env_vars
 from stoobly_agent.config.data_dir import DataDir
 from stoobly_agent.config.source_dir import SourceDir
 from stoobly_agent.lib.logger import Logger
@@ -19,6 +19,7 @@ from .remote_settings import RemoteSettings
 from .ui_settings import UISettings
 
 LOG_ID = 'Settings'
+SETTINGS_YML = 'settings.yml'
 
 class Settings:
     _instances = None
@@ -102,7 +103,7 @@ class Settings:
         if self.__watching:
             return False
 
-        patterns = ['settings.yml']
+        patterns = [SETTINGS_YML]
         ignore_patterns = None
         ignore_directories = False
         case_sensitive = True
@@ -177,7 +178,14 @@ class Settings:
         self.__load_settings()
 
     def write(self, contents):
-        if contents:
+        if not contents:
+            return
+
+        lock_file = f".{SETTINGS_YML}.lock"
+        lock_file_path = os.path.join(os.path.dirname(self.__settings_file_path), lock_file)
+        lock = FileLock(lock_file_path)  # lock file alongside the target
+
+        with lock:
             with open(self.__settings_file_path, 'w') as fp:
                 yaml.dump(contents, fp, allow_unicode=True)
 
@@ -209,13 +217,14 @@ class Settings:
 
     def __reload_settings(self, event):
         if not self.__load_lock:
-            from stoobly_agent.app.proxy.utils.publish_change_service import publish_change
+            from stoobly_agent.app.proxy.utils.publish_change_service import publish_settings_modified
 
             self.__load_lock = True
 
             Logger.instance(LOG_ID).debug('Reloading settings')
             self.__load_settings()
 
-            publish_change(statuses.SETTINGS_MODIFIED, self.__settings, sync=True)
+            if self.__ui_settings.active:
+                publish_settings_modified(self.__settings)
 
         self.__load_lock = False

stoobly_agent/app/settings/data_rules.py

@@ -1,4 +1,4 @@
-from stoobly_agent.config.constants import mock_policy, record_order, record_policy, replay_policy, test_strategy
+from stoobly_agent.config.constants import mock_policy, record_order, record_policy, record_strategy, replay_policy, test_strategy
 
 from .types.proxy_settings import DataRules as IDataRules
 
@@ -10,6 +10,7 @@ class DataRules:
     self.__mock_policy = self.__data_rules.get('mock_policy') or mock_policy.FOUND
     self.__record_order = self.__data_rules.get('record_order') or record_order.APPEND
     self.__record_policy = self.__data_rules.get('record_policy') or record_policy.ALL
+    self.__record_strategy = self.__data_rules.get('record_strategy') or record_strategy.FULL
     self.__replay_policy = self.__data_rules.get('replay_policy') or replay_policy.ALL
     self.__scenario_key = self.__data_rules.get('scenario_key')
     self.__test_policy = self.__data_rules.get('test_policy') or mock_policy.FOUND
@@ -46,6 +47,19 @@ class DataRules:
     self.__record_order = v
     self.__data_rules['record_order'] = v
 
+  @property
+  def record_strategy(self):
+    return self.__record_strategy
+
+  @record_strategy.setter
+  def record_strategy(self, v):
+    valid_strategies = [record_strategy.FULL, record_strategy.MINIMAL]
+    if v not in valid_strategies:
+      raise TypeError(f"record_strategy has to be one of {valid_strategies}, got {v}")
+
+    self.__record_strategy = v
+    self.__data_rules['record_strategy'] = v
+
   @property
   def replay_policy(self):
     return self.__replay_policy
@@ -79,11 +93,21 @@ class DataRules:
   def test_strategy(self):
     return self.__test_strategy 
 
+  @test_strategy.setter
+  def test_strategy(self, v):
+    valid_strategies = [test_strategy.CONTRACT, test_strategy.CUSTOM, test_strategy.DIFF, test_strategy.FUZZY]
+    if v not in valid_strategies:
+      raise TypeError(f"test_strategy has to be one of {valid_strategies}, got {v}")
+
+    self.__test_strategy = v
+    self.__data_rules['test_strategy'] = v
+
   def to_dict(self) -> IDataRules:
     return {
       'mock_policy': self.__mock_policy,
       'record_order': self.__record_order,
       'record_policy': self.__record_policy,
+      'record_strategy': self.__record_strategy,
       'replay_policy': self.__replay_policy,
       'scenario_key': self.__scenario_key,
       'test_policy': self.__test_policy,

stoobly_agent/app/settings/intercept_settings.py

@@ -19,6 +19,9 @@ class InterceptSettings:
 
   @property
   def active(self):
+    if os.environ.get(env_vars.AGENT_INTERCEPT_ACTIVE):
+      return os.environ[env_vars.AGENT_INTERCEPT_ACTIVE]
+
     return self.__active
 
   @active.setter
@@ -35,8 +38,8 @@ class InterceptSettings:
     if self.__mode != self.mode_before_change:
       return self.__mode
 
-    if os.environ.get(env_vars.AGENT_ACTIVE_MODE):
-        return os.environ[env_vars.AGENT_ACTIVE_MODE]
+    if os.environ.get(env_vars.AGENT_INTERCEPT_MODE):
+        return os.environ[env_vars.AGENT_INTERCEPT_MODE]
         
     return self.__mode
 

stoobly_agent/app/settings/types/__init__.py

@@ -84,7 +84,6 @@ class ISettings(TypedDict):
     api_url: str
     api_key: str
     mode: ISettingsMode
-    proxy_config_path: str
 
 Component = {
     'Header': 'Header',

stoobly_agent/app/settings/ui_settings.py

@@ -21,8 +21,8 @@ class UISettings:
     if self.__active != self.active_before_change:
       return self.__active
 
-    if os.environ.get(env_vars.AGENT_IS_HEADLESS):
-        return True
+    if os.environ.get(env_vars.AGENT_HEADLESS):
+        return False
 
     return self.__active or False
 
@@ -39,8 +39,8 @@ class UISettings:
     if self.__url != self.url_before_change:
         return self.__url
 
-    if os.environ.get(env_vars.AGENT_URL):
-        return os.environ[env_vars.AGENT_URL]
+    if os.environ.get(env_vars.AGENT_UI_URL):
+        return os.environ[env_vars.AGENT_UI_URL]
 
     return self.__url or ''
 
@@ -52,4 +52,4 @@ class UISettings:
     return {
       'active': self.__active,
       'url': self.__url,
-    }
+    }

stoobly_agent/cli.py

@@ -13,9 +13,10 @@ from stoobly_agent.app.proxy.constants import custom_response_codes
 from stoobly_agent.app.proxy.replay.replay_request_service import replay as replay_request
 from stoobly_agent.config.constants import env_vars, mode
 from stoobly_agent.config.data_dir import DataDir
+from stoobly_agent.lib.logger import Logger
 from stoobly_agent.lib.utils.conditional_decorator import ConditionalDecorator
 
-from .app.api import initialize as initialize_api, run as run_api
+from .app.api import run as run_api
 from .app.cli import ca_cert, config, endpoint, feature, intercept, MainGroup, request, scenario, scaffold, snapshot, trace
 from .app.cli.helpers.feature_flags import local, remote
 from .app.settings import Settings
@@ -99,6 +100,7 @@ def init(**kwargs):
 ''')
 @click.option('--headless', is_flag=True, default=False, help='Disable starting UI.')
 @click.option('--intercept', is_flag=True, default=False, help='Enable intercept on run.')
+@click.option('--intercept-mode', help='Set intercept mode.')
 @click.option('--log-level', default=logger.INFO, type=click.Choice([logger.DEBUG, logger.INFO, logger.WARNING, logger.ERROR]), help='''
     Log levels can be "debug", "info", "warning", or "error"
 ''')
@@ -124,7 +126,19 @@ def init(**kwargs):
 def run(**kwargs):
     from .app.proxy.run import run as run_proxy
 
-    os.environ[env_vars.AGENT_PROXY_URL] = f"http://{kwargs['proxy_host']}:{kwargs['proxy_port']}"
+    # Observe config for changes
+    settings: Settings = Settings.instance()
+    settings.watch()
+
+    if kwargs.get('headless'):
+      os.environ[env_vars.AGENT_HEADLESS] = '1'
+
+    if kwargs.get('intercept'):
+      os.environ[env_vars.AGENT_INTERCEPT_ACTIVE] = '1'
+
+    if kwargs.get('intercept_mode'):
+      os.environ[env_vars.AGENT_INTERCEPT_MODE] = kwargs['intercept_mode']
+      settings.proxy.intercept.mode = kwargs['intercept_mode']
 
     if kwargs.get('lifecycle_hooks_path'):
       os.environ[env_vars.AGENT_LIFECYCLE_HOOKS_PATH] = kwargs['lifecycle_hooks_path']
@@ -135,21 +149,32 @@ def run(**kwargs):
     if kwargs.get('response_fixtures_path'):
       os.environ[env_vars.AGENT_RESPONSE_FIXTURES_PATH] = kwargs['response_fixtures_path']
 
-    # Observe config for changes
-    Settings.instance().watch()
-
     if not os.getenv(env_vars.LOG_LEVEL):
-        os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
-
-    if 'api_url' in kwargs and kwargs['api_url']:
-        os.environ[env_vars.API_URL] = kwargs['api_url']
-
-    url = initialize_api(**kwargs)
-    if 'headless' in kwargs and not kwargs['headless']:
-        run_api(url)
-
-    if 'proxyless' in kwargs and not kwargs['proxyless']:
-        run_proxy(**kwargs)
+      os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
+
+    if kwargs.get('api_url'):
+      os.environ[env_vars.API_URL] = kwargs['api_url']
+
+    if not kwargs.get('headless'):
+      ui_url = f"http://{kwargs['ui_host']}:{kwargs['ui_port']}"
+      os.environ[env_vars.AGENT_UI_URL] = ui_url
+      settings.ui.active = True
+      settings.ui.url = ui_url
+
+    if not kwargs.get('proxyless'):
+      proxy_url = f"http://{kwargs['proxy_host']}:{kwargs['proxy_port']}"
+      os.environ[env_vars.AGENT_PROXY_URL] = proxy_url
+      settings.proxy.url = proxy_url
+
+    if not kwargs.get('headless'):
+      settings.commit()
+      run_api(**kwargs)
+
+    if not kwargs.get('proxyless'):
+      log_id = 'Proxy'
+      Logger.instance(log_id).info(f"starting with mode {kwargs['proxy_mode']} and listening at {kwargs['proxy_host']}:{kwargs['proxy_port']}")
+      Logger.instance(log_id).info(f"{'' if settings.proxy.intercept.active else 'not yet '}configured to {settings.proxy.intercept.mode}")
+      run_proxy(**kwargs)
 
 @main.command(
   help="Mock request"

stoobly_agent/config/constants/custom_headers.py

@@ -13,6 +13,7 @@ PROJECT_KEY = 'X-Stoobly-Project-Key'
 PROXY_MODE = 'X-Stoobly-Proxy-Mode'
 PUBLIC_DIRECTORY_PATH = 'X-Stoobly-Public-Directory-Path'
 RECORD_POLICY = 'X-Stoobly-Record-Policy'
+RECORD_STRATEGY = 'X-Stoobly-Record-Strategy'
 REPORT_KEY = 'X-Stoobly-Report-Key'
 REQUEST_ORIGIN = 'X-Stoobly-Request-Origin'
 RESPONSE_FIXTURES_PATH = 'X-Stoobly-Response-Fixtures-Path'

stoobly_agent/config/constants/env_vars.py

@@ -1,9 +1,10 @@
-AGENT_ACTIVE_MODE = 'STOOBLY_AGENT_ACTIVE_MODE'
 AGENT_CONFIG_PATH = 'STOOBLY_AGENT_CONFIG_PATH'
 AGENT_ENABLED = 'STOOBLY_AGENT_ENABLED'
 AGENT_SELF_INTERCEPT_ENABLED = 'STOOBLY_AGENT_SELF_INTERCEPT_ENABLED'
+AGENT_HEADLESS = 'STOOBLY_AGENT_HEADLESS'
 AGENT_INCLUDE_PATTERNS = 'STOOBLY_AGENT_INCLUDE_PATTERNS'
-AGENT_IS_HEADLESS = 'STOOBLY_AGENT_IS_HEADLESS'
+AGENT_INTERCEPT_ACTIVE = 'STOOBLY_AGENT_INTERCEPT_ACTIVE'
+AGENT_INTERCEPT_MODE = 'STOOBLY_AGENT_INTERCEPT_MODE'
 AGENT_EXCLUDE_PATTERNS = 'STOOBLY_AGENT_EXCLUDE_PATTERNS'
 AGENT_LIFECYCLE_HOOKS_PATH = 'STOOBLY_AGENT_LIFECYCLE_HOOKS_PATH'
 AGENT_POLICY = 'STOOBLY_AGENT_POLICY'
@@ -18,7 +19,7 @@ AGENT_RESPONSE_FIXTURES_PATH = 'STOOBLY_AGENT_RESPONSE_FIXTURES_PATH'
 AGENT_SERVICE_URL = 'STOOBLY_AGENT_SERVICE_URL'
 AGENT_SCENARIO_KEY = 'STOOBLY_AGENT_SCENARIO_KEY'
 AGENT_SIMULATE_LATENCY = 'STOOBLY_AGENT_SIMULATE_LATENCY'
-AGENT_URL = 'STOOBLY_AGENT_URL'
+AGENT_UI_URL = 'STOOBLY_AGENT_UI_URL'
 API_URL = 'STOOBLY_API_URL'
 API_KEY = 'STOOBLY_API_KEY'
 ENV = 'STOOBLY_AGENT_ENV'

stoobly_agent/config/constants/record_strategy.py

@@ -0,0 +1,6 @@
+from typing import Literal
+
+FULL = 'full'
+MINIMAL = 'minimal'
+
+RecordStrategy = Literal[FULL, MINIMAL]

stoobly_agent/config/data_dir.py

@@ -202,6 +202,7 @@ class DataDir:
             with open(os.path.join(self.__data_dir_path, '.gitignore'), 'w') as fp:
                 fp.write(
                     "\n".join([
+                        '.settings.yml.lock',
                         'ca_certs',
                         'certs',
                         'db',

stoobly_agent/config/settings.yml.sample

@@ -12,10 +12,9 @@ proxy:
         project_key: eyJpIjowLCJvIjowfQ==
     firewall: {}
     rewrite: {}
-proxy_config_path: config/settings.yml
 remote: 
     api_key: ''
-    api_url: 'http://localhost:3000'
+    api_url: ''
 ui:
     active: false
-    url: http://localhost:4200
+    url: ''

stoobly_agent/lib/logger.py

@@ -27,6 +27,7 @@ LogLevel = Literal[DEBUG, ERROR, INFO, WARNING]
 
 class Logger:
     _instance = None
+    _instances = {}
 
     def __init__(self):
         raise RuntimeError('Call instance() instead')
@@ -34,14 +35,23 @@ class Logger:
     @classmethod
     def instance(cls, name = None):
         if cls._instance is None:
+            logging.getLogger("mitmproxy").setLevel(logging.ERROR)
             cls._instance = cls.__new__(cls)
-
-        cls._instance.load()
+            cls._instance.load()
 
         if not name:
-            return logging
-        else:
-            return logging.getLogger(name)
+            name = 'root'
+
+        logger = logging.getLogger(name)
+
+        if name not in cls._instances:
+            handler = logging.StreamHandler()
+            handler.setFormatter(logging.Formatter(f"[%(levelname)s] {name} %(message)s"))
+            logger.addHandler(handler)
+            logger.propagate = False
+            cls._instances[name] = logger
+
+        return logger
 
     @classmethod
     def reload(cls):

stoobly_agent/public/index.html

@@ -113,6 +113,6 @@
   </div>
 
   <root></root>
-<script src="runtime-es2015.f8c814b38b27708e91c1.js" type="module"></script><script src="runtime-es5.f8c814b38b27708e91c1.js" nomodule="" defer=""></script><script src="polyfills-es5.7530172ddcec11a10eb3.js" nomodule="" defer=""></script><script src="polyfills-es2015.8ce2adc69f283f6c4c5e.js" type="module"></script><script src="main-es2015.089b46f303768fbe864f.js" type="module"></script><script src="main-es5.089b46f303768fbe864f.js" nomodule="" defer=""></script></body>
+<script src="runtime-es2015.f8c814b38b27708e91c1.js" type="module"></script><script src="runtime-es5.f8c814b38b27708e91c1.js" nomodule="" defer=""></script><script src="polyfills-es5.7530172ddcec11a10eb3.js" nomodule="" defer=""></script><script src="polyfills-es2015.8ce2adc69f283f6c4c5e.js" type="module"></script><script src="main-es2015.5a9aa16433404c3f423a.js" type="module"></script><script src="main-es5.5a9aa16433404c3f423a.js" nomodule="" defer=""></script></body>
 
 </html>