stoobly-agent 1.2.3__py3-none-any.whl → 1.3.0__py3-none-any.whl

stoobly_agent/app/cli/scaffold/workflow_run_command.py

@@ -12,42 +12,46 @@ from stoobly_agent.lib.logger import Logger
 from .app import App
 from .constants import (
   APP_NETWORK_ENV, CA_CERTS_DIR_ENV, CERTS_DIR_ENV, CONTEXT_DIR_ENV, NAMESERVERS_FILE, 
-  SERVICE_DNS_ENV, SERVICE_NAME_ENV, USER_ID_ENV, WORKFLOW_NAME_ENV
+  SERVICE_DNS_ENV, SERVICE_NAME_ENV, USER_ID_ENV, WORKFLOW_NAME_ENV, WORKFLOW_NAMESPACE_ENV
 )
+from .docker.constants import DOCKERFILE_CONTEXT
 from .workflow_command import WorkflowCommand
 from .workflow_env import WorkflowEnv
 
 LOG_ID = 'WorkflowRunCommand'
 
-class UpOptions(TypedDict):
-  attached: bool
+class ComposeOptions(TypedDict):
   namespace: str
-  no_cache: bool
+  user_id: str
+
+class BuildOptions(ComposeOptions):
+  user_id: str
   verbose: bool
 
+class DownOptions(ComposeOptions):
+  rmi: bool
+
+class UpOptions(ComposeOptions):
+  attached: bool
+  pull: bool
+
 class WorkflowRunCommand(WorkflowCommand):
   def __init__(self, app: App, **kwargs):
     super().__init__(app, **kwargs)
 
     self.__current_working_dir = os.getcwd()
-    self.__ca_certs_dir_path = app.ca_certs_dir_path
-    self.__certs_dir_path = app.certs_dir_path
-    self.__context_dir_path = app.context_dir_path
+    self.__ca_certs_dir_path = kwargs.get('ca_certs_dir_path') or app.ca_certs_dir_path
+    self.__certs_dir_path = kwargs.get('certs_dir_path') or app.certs_dir_path
+    self.__context_dir_path = kwargs.get('context_dir_path') or app.context_dir_path
     self.__extra_compose_path = kwargs.get('extra_compose_path')
     self.__network = kwargs.get('network') or self.app_config.network
 
   @property
   def ca_certs_dir_path(self):
-    if not os.path.exists(self.__ca_certs_dir_path):
-      os.makedirs(self.__ca_certs_dir_path)
-
     return self.__ca_certs_dir_path
 
   @property
   def certs_dir_path(self):
-    if not os.path.exists(self.__certs_dir_path):
-      os.makedirs(self.__certs_dir_path)
-
     return self.__certs_dir_path
 
   @property
@@ -88,17 +92,43 @@ class WorkflowRunCommand(WorkflowCommand):
   def network(self):
     return self.__network
 
+  def create_image(self, **options: BuildOptions):
+    relative_namespace_path = os.path.relpath(self.scaffold_namespace_path, self.__current_working_dir)
+    dockerfile_path = os.path.join(relative_namespace_path, DOCKERFILE_CONTEXT)
+    user_id = options['user_id'] or os.getuid()
+    
+    command = ['docker', 'build']
+    command.append(f"-f {dockerfile_path}")
+    command.append(f"-t stoobly.{user_id}")
+    command.append(f"--build-arg USER_ID={user_id}")
+
+    if not os.environ.get('STOOBLY_IMAGE_USE_LOCAL'):
+      command.append('--pull')
+
+    if not options.get('verbose'):
+      command.append('--quiet')
+
+    # To avoid large context transfer times, should be a folder with relatively low number of files
+    command.append(relative_namespace_path) 
+
+    return ' '.join(command)
+
+  def remove_image(self, user_id: str = None):
+    user_id = user_id or os.getuid()
+    command = ['docker', 'rmi', f"stoobly.{user_id}", '&>', '/dev/null']
+    command.append('|| true')
+    return ' '.join(command)
+
   def create_network(self):
-    return f"docker network create {self.network} 2> /dev/null"
+    return f"docker network create {self.network} &> /dev/null"
 
   def remove_network(self):
-    return f"docker network rm {self.network} > /dev/null"
+    return f"docker network rm {self.network} &> /dev/null || true"
 
   def up(self, **options: UpOptions):
     if not os.path.exists(self.compose_path):
       return ''
 
-    build_command = ['docker', 'compose']
     command = ['COMPOSE_IGNORE_ORPHANS=true', 'docker', 'compose']
     command_options = []
 
@@ -128,22 +158,19 @@ class WorkflowRunCommand(WorkflowCommand):
 
     command_options.append(f"--profile {self.workflow_name}") 
 
-    if options.get('namespace'):
-      command_options.append(f"-p {options['namespace']}")
-
-    build_command += command_options
-    build_command.append('build')
-    build_command.append('--pull')
-
-    if not options.get('verbose'):
-      build_command.append('--quiet')
-
-    if options.get('no_cache'):
-      build_command.append('--no-cache')
+    if not options.get('namespace'):
+      options['namespace'] = self.workflow_name
+    command_options.append(f"-p {options['namespace']}")
 
     command += command_options
     command.append('up')
 
+    if options.get('build'):
+      command.append('--build')
+
+    if options.get('pull'):
+      command.append('--pull missing')
+
     if not options.get('attached'):
       command.append('-d')
     else:
@@ -159,11 +186,11 @@ class WorkflowRunCommand(WorkflowCommand):
       # Otherwise, even if a service exits with a non-zero exit code, exit code 0 is returned
       command.append(option)
 
-    self.write_env()
+    self.write_env(**options)
 
-    return ' && '.join([' '.join(build_command), ' '.join(command)]) 
+    return ' '.join(command)
 
-  def down(self, **options):
+  def down(self, **options: DownOptions):
     if not os.path.exists(self.compose_path):
       return ''
   
@@ -181,13 +208,16 @@ class WorkflowRunCommand(WorkflowCommand):
 
     command.append(f"--profile {self.workflow_name}") 
 
-    if options.get('namespace'):
-      command.append(f"-p {options['namespace']}")
+    if not options.get('namespace'):
+      options['namespace'] = self.workflow_name
+    command.append(f"-p {options['namespace']}")
 
     command.append('down')
 
     if options.get('rmi'):
-      command.append(f"--rmi local")
+      command.append('--rmi local')
+
+    self.write_env(**options)
 
     return ' '.join(command)
 
@@ -212,14 +242,20 @@ class WorkflowRunCommand(WorkflowCommand):
       if nameservers:
         fp.write("\n".join(nameservers))
 
-  def write_env(self):
+  def write_env(self, **options: ComposeOptions):
+    namespace = options.get('namespace')
+    user_id = options.get('user_id')
+
     _config = {}
     _config[CA_CERTS_DIR_ENV] = self.ca_certs_dir_path
     _config[CERTS_DIR_ENV] = self.certs_dir_path
     _config[CONTEXT_DIR_ENV] = self.context_dir_path
     _config[SERVICE_NAME_ENV] = self.service_name
-    _config[USER_ID_ENV] = os.getuid()
+    _config[USER_ID_ENV] = user_id or os.getuid()
     _config[WORKFLOW_NAME_ENV] = self.workflow_name
+    
+    if namespace:
+      _config[WORKFLOW_NAMESPACE_ENV] = namespace
 
     if self.network:
       _config[APP_NETWORK_ENV] = self.network

stoobly_agent/app/cli/scaffold_cli.py

@@ -62,14 +62,18 @@ def service(ctx):
 )
 @click.option('--app-dir-path', default=os.getcwd(), help='Path to create the app scaffold.')
 @click.option('--force', is_flag=True, help='Overwrite maintained scaffolded app files.')
+@click.option('--network', help='App default network name. Defaults to app name.')
 @click.argument('app_name')
 def create(**kwargs):
   __validate_app_dir(kwargs['app_dir_path'])
 
   app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE)
 
-  if kwargs['force'] or not os.path.exists(app.namespace_path):
-    __app_build(app, **kwargs)
+  if kwargs['force'] or not os.path.exists(app.scaffold_namespace_path):
+    if not kwargs['network']:
+      kwargs['network'] = kwargs['app_name']
+
+    AppCreateCommand(app, **kwargs).build()
   else:
     print(f"{kwargs['app_dir_path']} already exists, use option '--force' to continue ")
 
@@ -82,13 +86,7 @@ def create(**kwargs):
 @click.option('--context-dir-path', default=DataDir.instance().context_dir_path, help='Path to Stoobly data directory.')
 @click.option('--service', multiple=True, help='Select which services to run. Defaults to all.')
 def mkcert(**kwargs):
-  app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE, ca_certs_dir_path=kwargs['ca_certs_dir_path'])
-
-  if kwargs['certs_dir_path']:
-    app.certs_dir_path = kwargs['certs_dir_path']
-
-  if kwargs['context_dir_path']:
-    app.context_dir_path = kwargs['context_dir_path']
+  app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE, **kwargs)
 
   if not app.exists:
     print(f"Error: {app.dir_path} does not exist", file=sys.stderr)
@@ -249,20 +247,21 @@ def copy(**kwargs):
     Log levels can be "debug", "info", "warning", or "error"
 ''')
 @click.option('--namespace', help='Workflow namespace.')
-@click.option('--network', help='Workflow network namespace.')
+@click.option('--network', help='Workflow network name.')
 @click.option('--rmi', is_flag=True, help='Remove images used by containers.')
 @click.option('--service', multiple=True, help='Select which services to log. Defaults to all.')
+@click.option('--user-id', default=os.getuid(), help='OS user ID of the owner of context dir path.')
 @click.argument('workflow_name')
 def down(**kwargs):  
   cwd = os.getcwd()
 
-  if not os.getenv(env_vars.LOG_LEVEL):
-    os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
+  os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
 
-  app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE, skip_validate_path=True)
+  app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE, **kwargs)
 
-  if kwargs['context_dir_path']:
-    app.context_dir_path = kwargs['context_dir_path']
+  # If namespace is set, default network to namespace
+  if kwargs['namespace'] and not kwargs['network']:
+    kwargs['network'] = kwargs['namespace']
 
   if not app.exists:
     print(f"Error: {app.dir_path} does not exist", file=sys.stderr)
@@ -283,7 +282,7 @@ def down(**kwargs):
   for command in commands:
     __print_header(f"SERVICE {command.service_name}")
 
-    exec_command = command.down(namespace=kwargs['namespace'], rmi=kwargs['rmi'])
+    exec_command = command.down(namespace=kwargs['namespace'], rmi=kwargs['rmi'], user_id=kwargs['user_id'])
     if not exec_command:
       continue
 
@@ -295,11 +294,16 @@ def down(**kwargs):
   # After services are stopped, their network needs to be removed
   if len(commands) > 0:
     command: WorkflowRunCommand = commands[0]
-    remove_network_command = command.remove_network()
 
-    if not kwargs['dry_run']:
-      command.write_nameservers()
+    if kwargs['rmi']:
+      remove_image_command = command.remove_image(kwargs['user_id'])
+      if not kwargs['dry_run']:
+        exec_stream(remove_image_command)
+      else:
+        print(remove_image_command)
 
+    remove_network_command = command.remove_network()
+    if not kwargs['dry_run']:
       exec_stream(remove_network_command)
     else:
       print(remove_network_command)
@@ -311,10 +315,15 @@ def down(**kwargs):
 )
 @click.option('--dry-run', default=False, is_flag=True, help='If set, prints commands.')
 @click.option('--follow', is_flag=True, help='Follow last container log output.')
+@click.option('--log-level', default=INFO, type=click.Choice([DEBUG, INFO, WARNING, ERROR]), help='''
+    Log levels can be "debug", "info", "warning", or "error"
+''')
 @click.option('--namespace', help='Workflow namespace.')
 @click.option('--service', multiple=True, help='Select which services to log. Defaults to all.')
 @click.argument('workflow_name')
 def logs(**kwargs):
+  os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
+
   if len(kwargs['container']) == 0:
     kwargs['container'] = [WORKFLOW_CONTAINER_PROXY]
 
@@ -360,40 +369,34 @@ def logs(**kwargs):
  
 @workflow.command()
 @click.option('--app-dir-path', default=os.getcwd(), help='Path to application directory.')
+@click.option('--build', is_flag=True, help='Build images before starting containers.')
 @click.option('--ca-certs-dir-path', default=DataDir.instance().mitmproxy_conf_dir_path, help='Path to ca certs directory used to sign SSL certs. Defaults to ~/.mitmproxy')
 @click.option('--certs-dir-path', help='Path to certs directory. Defaults to the certs dir of the context.')
 @click.option('--context-dir-path', default=DataDir.instance().context_dir_path, help='Path to Stoobly data directory.')
 @click.option('--detached', is_flag=True, help='If set, will not run the highest priority service in the foreground.')
 @click.option('--dry-run', default=False, is_flag=True, help='If set, prints commands.')
 @click.option('--extra-compose-path', help='Path to extra compose configuration files.')
+@click.option('--from-make', is_flag=True, help='Set if run from scaffolded Makefile.')
 @click.option('--log-level', default=INFO, type=click.Choice([DEBUG, INFO, WARNING, ERROR]), help='''
     Log levels can be "debug", "info", "warning", or "error"
 ''')
 @click.option('--namespace', help='Workflow namespace.')
-@click.option('--network', help='Workflow network namespace.')
-@click.option('--no-cache', is_flag=True, help='Do not use cache when building images.')
+@click.option('--network', help='Workflow network name.')
+@click.option('--pull', is_flag=True, help='Pull image before running.')
 @click.option('--service', multiple=True, help='Select which services to run. Defaults to all.')
+@click.option('--user-id', default=os.getuid(), help='OS user ID of the owner of context dir path.')
 @click.option('--verbose', is_flag=True)
 @click.argument('workflow_name')
 def up(**kwargs):
   cwd = os.getcwd()
 
-  # Create the certs_dir_path if it doesn't exist
-  DataDir.instance().certs_dir_path
-
-  if not os.getenv(env_vars.LOG_LEVEL):
-    os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
-
-  app = App(
-    kwargs['app_dir_path'], DOCKER_NAMESPACE, 
-    ca_certs_dir_path=kwargs['ca_certs_dir_path'], skip_validate_path=kwargs['dry_run']
-  )
+  os.environ[env_vars.LOG_LEVEL] = kwargs['log_level']
 
-  if kwargs['certs_dir_path']:
-    app.certs_dir_path = kwargs['certs_dir_path']
+  app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE, **kwargs)
 
-  if kwargs['context_dir_path']:
-    app.context_dir_path = kwargs['context_dir_path']
+  # If namespace is set, default network to namespace
+  if kwargs['namespace'] and not kwargs['network']:
+    kwargs['network'] = kwargs['namespace']
 
   if not app.exists:
     print(f"Error: {app.dir_path} does not exist", file=sys.stderr)
@@ -413,17 +416,23 @@ def up(**kwargs):
     command.current_working_dir = cwd
     commands.append(command)
 
-  # Before services can be started, their network needs to be created
+  # Before services can be started, their image and network needs to be created
   if len(commands) > 0:
     command: WorkflowRunCommand = commands[0]
-    create_network_command = command.create_network()
 
-    if not kwargs['dry_run']:
-      command.write_nameservers()
+    init_commands = []
+    if not kwargs['from_make']:
+      create_image_command = command.create_image(user_id=kwargs['user_id'], verbose=kwargs['verbose'])
+      init_commands.append(create_image_command)
+
+    init_commands.append(command.create_network())
+    joined_command = ' && '.join(init_commands)
+    command.write_nameservers()
 
-      exec_stream(create_network_command)
+    if not kwargs['dry_run']:
+      exec_stream(joined_command)
     else:
-      print(create_network_command)
+      print(joined_command)
 
   commands = sorted(commands, key=lambda command: command.service_config.priority)
   for index, command in enumerate(commands):
@@ -433,7 +442,7 @@ def up(**kwargs):
     # However, this can change if the user has configured a service's priority to be higher
     attached = not kwargs['detached'] and index == len(commands) - 1
     exec_command = command.up(
-      attached=attached, namespace=kwargs['namespace'], no_cache=kwargs['no_cache'], verbose=kwargs['verbose']
+      attached=attached, build=kwargs['build'], namespace=kwargs['namespace'], pull=kwargs['pull'], user_id=kwargs['user_id']
     )
     if not exec_command:
       continue
@@ -498,9 +507,6 @@ def __get_services(services: List[str], **kwargs):
 def __print_header(text: str):
   Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}{text}{bcolors.ENDC}")
 
-def __app_build(app, **kwargs):
-  AppCreateCommand(app, **kwargs).build()
-
 def __scaffold_build(app, **kwargs):
   command = ServiceCreateCommand(app, **kwargs)
 

stoobly_agent/app/cli/snapshot_cli.py

@@ -36,11 +36,15 @@ def snapshot(ctx):
 @click.argument('uuid', required=False)
 def apply(**kwargs):
   apply = Apply(force=kwargs['force']).with_logger(print)
+  completed = True
 
   if kwargs.get('uuid'):
-    apply.single(kwargs['uuid'])
+    completed = apply.single(kwargs['uuid'])
   else:
-    apply.all()
+    completed = apply.all()
+
+  if not completed:
+    sys.exit(1)
 
 @snapshot.command(
   help="Copy snapshots to a different data directory."

stoobly_agent/app/models/adapters/joined_request_adapter.py

@@ -18,6 +18,12 @@ class JoinedRequestAdapter():
       payloads_delimitter = payloads_delimitter.encode()
       
     self.__split_joined_request_string = joined_request_string.split(payloads_delimitter)
+    if len(self.__split_joined_request_string) != 2:
+      self.__split_joined_request_string = joined_request_string.split(payloads_delimitter.replace(b"\n", b"\r\n"))
+
+    if len(self.__split_joined_request_string) != 2:
+      raise ValueError(f"Could not split by {payloads_delimitter}")
+
     self.__raw_request_string = None
     self.__raw_response_string = None
 

stoobly_agent/app/models/factories/resource/local_db/helpers/scenario_snapshot.py

@@ -34,7 +34,9 @@ class ScenarioSnapshot(Snapshot):
       try:
         return json.loads(fp.read())
       except Exception:
-        return {}
+        return {
+          'name': self.uuid
+        }
 
   @property
   def metadata_backup(self):

stoobly_agent/app/models/helpers/apply.py

@@ -61,15 +61,17 @@ class Apply():
       return
 
     last_processed_event = None
+    completed = True
 
     for event in unprocessed_events:
       if self.__logger:
-        self.__logger(f"Processing event {event.uuid}")
+        self.__logger(f"{bcolors.OKBLUE}Processing Event{bcolors.ENDC} {event.uuid}")
 
       results = event.apply(**self.__handlers())
       if results:
         status = results[1]
-        if status == 0 or status >= 500:
+        if status == 0 or status >= 400:
+          completed = False
           break
 
       last_processed_event = event
@@ -86,6 +88,8 @@ class Apply():
 
     log.lock()
 
+    return completed
+
   def request(self, uuid: str):
     result = self.__apply_put_request(uuid)
     if not result:
@@ -121,9 +125,13 @@ class Apply():
       return False
 
     if self.__logger:
-      self.__logger(f"Processing event {event.uuid}")
+      self.__logger(f"{bcolors.OKBLUE}Processing Event{bcolors.ENDC} {event.uuid}")
 
-    event.apply(**self.__handlers())
+    results = event.apply(**self.__handlers())
+    if results:
+      status = results[1]
+      if status == 0 or status >= 400:
+        return False
 
     return True
 
@@ -139,7 +147,7 @@ class Apply():
     res, status = self.request_model.destroy(uuid, force=self.__force)
 
     if status == 200:
-      self.__logger(f"{bcolors.WARNING}Deleted{bcolors.ENDC} request {uuid}") 
+      self.__logger(f"{bcolors.WARNING}Deleted Request{bcolors.ENDC} {uuid}") 
     else: 
       self.__logger(f"{bcolors.FAIL}{status}{bcolors.ENDC} {res}")
 
@@ -150,9 +158,9 @@ class Apply():
 
     raw_request = snapshot.request
     if not raw_request:
-      error = f"Snapshot for request {uuid} not found" 
-      self.__logger(f"{bcolors.FAIL}400{bcolors.ENDC} {error}")
-      return error, 400
+      error = f"snapshot for request {uuid} not found" 
+      self.__logger(f"{bcolors.WARNING}Skipping Request{bcolors.ENDC} {error}")
+      return error, 301
 
     return self.__put_request(uuid, raw_request)
 
@@ -160,7 +168,7 @@ class Apply():
     res, status = self.scenario_model.destroy(uuid, force=self.__force)
 
     if self.__logger and status == 200:
-      self.__logger(f"{bcolors.WARNING}Deleted{bcolors.ENDC} scenario {uuid}")
+      self.__logger(f"{bcolors.WARNING}Deleted Scenario{bcolors.ENDC} {uuid}")
     else: 
       self.__logger(f"{bcolors.FAIL}{status}{bcolors.ENDC} {res}")
 
@@ -171,9 +179,9 @@ class Apply():
     metadata = snapshot.metadata
 
     if not metadata:
-      error = f"Snapshot for scenario {uuid} not found"
-      self.__logger(f"{bcolors.FAIL}400{bcolors.ENDC} {error}")
-      return error, 400
+      error = f"snapshot for scenario {uuid} not found"
+      self.__logger(f"{bcolors.WARNING}Skipping Scenario{bcolors.ENDC} {error}")
+      return error, 301 
 
     res, status = self.scenario_model.show(uuid)
     if status == 404:
@@ -184,7 +192,7 @@ class Apply():
 
       if self.__logger:
         if status == 200:
-          self.__logger(f"{bcolors.OKGREEN}Created scenario{bcolors.ENDC} {res['name']}") 
+          self.__logger(f"{bcolors.OKGREEN}Created Scenario{bcolors.ENDC} {res['name']}") 
         else: 
           self.__logger(f"{bcolors.FAIL}{status}{bcolors.ENDC} {res}")
     else:
@@ -195,7 +203,7 @@ class Apply():
 
       if self.__logger:
         if status == 200:
-          self.__logger(f"{bcolors.OKBLUE}Updated{bcolors.ENDC} scenario {res['name']}") 
+          self.__logger(f"{bcolors.OKCYAN}Updated Scenario{bcolors.ENDC} {res['name']}") 
         else:
           self.__logger(f"{bcolors.FAIL}{status}{bcolors.ENDC} {res}")
 
@@ -221,6 +229,9 @@ class Apply():
       uuid = control.id
       res, status = self.__put_request(uuid, raw_request, scenario_id=scenario['id'])
 
+      if status != 200:
+        return res, status
+
       snapshot_requests[uuid] = res
 
     # Remove requests in scenario that don't exist in the snapshot
@@ -242,15 +253,21 @@ class Apply():
     res, status = self.request_model.show(uuid)
 
     if status == 404:
+      request_params = build_params(raw_request)
+
+      if not request_params:
+          self.__logger(f"{bcolors.FAIL}{status}{bcolors.ENDC} failed to join request {uuid}")
+          return res, status
+
       params = {
-        **build_params(raw_request),
+        **request_params,
         **base_params,
       }
 
       res, status = self.request_model.create(**params)
 
       if self.__logger and status == 200:
-        self.__logger(f"{bcolors.OKGREEN}Created{bcolors.ENDC} {res['list'][0]['url']}")
+        self.__logger(f"{bcolors.OKGREEN}Created Request{bcolors.ENDC} {res['list'][0]['url']}")
       else: 
         self.__logger(f"{bcolors.FAIL}{status}{bcolors.ENDC} {res}")
     elif status == 200:
@@ -263,7 +280,7 @@ class Apply():
 
       if self.__logger:
         if status == 200:
-          self.__logger(f"{bcolors.OKBLUE}Updated{bcolors.ENDC} {res['url']}")
+          self.__logger(f"{bcolors.OKCYAN}Updated Request{bcolors.ENDC} {res['url']}")
         else: 
           self.__logger(f"{bcolors.FAIL}{status}{bcolors.ENDC} {res}")
 

stoobly_agent/app/models/helpers/create_request_params_service.py

@@ -6,6 +6,9 @@ from stoobly_agent.app.models.adapters.python import PythonRequestAdapterFactory
 
 from stoobly_agent.app.proxy.record.join_request_service import InterceptSettings, join_request, MitmproxyRequestFacade, MitmproxyResponseFacade
 from stoobly_agent.app.settings import Settings
+from stoobly_agent.lib.logger import Logger
+
+LOG_ID = 'CreateRequestParamsService'
 
 class MitmproxyFlowMock():
     def __init__(self, request, response):
@@ -16,6 +19,7 @@ def build_params(raw_requests: str, payloads_delimitter = None):
   try:
     joined_request = JoinedRequestAdapter(raw_requests, payloads_delimitter).adapt()
   except Exception as e:
+    Logger.instance(LOG_ID).error(e)
     return
 
   request_adapter = RawHttpRequestAdapter(joined_request.request_string.get())

stoobly_agent/app/proxy/replay/body_parser_service.py

@@ -1,8 +1,8 @@
-import cgi
 import json
 import pdb
 import urllib.parse
 
+from email.message import Message
 from mitmproxy.coretypes.multidict import MultiDict
 from mitmproxy.net import encoding
 from typing import Dict, Union
@@ -108,11 +108,19 @@ def serialize_www_form_urlencoded(o):
 def normalize_header(header):
     if isinstance(header, bytes):
         header = header.decode('utf-8')
-    return cgi.parse_header(header)[0].lower()
+    return __parse_separated_header(header).lower()
 
 def is_traversable(content):
     return isinstance(content, list) or isinstance(content, dict) or isinstance(content, MultiDict)
 
 def is_json(content_type):
    _content_type = content_type.lower()
-   return _content_type == JSON or _content_type.startswith('application/x-amz-json') 
+   return _content_type == JSON or _content_type.startswith('application/x-amz-json') 
+
+
+def __parse_separated_header(header: str):
+    # Adapted from https://peps.python.org/pep-0594/#cgi
+    message = Message()
+    message['content-type'] = header 
+    return message.get_content_type()
+

stoobly_agent/config/data_dir.py

@@ -4,6 +4,7 @@ import shutil
 
 from stoobly_agent.config.constants.env_vars import ENV
 
+CERTS_DIR_NAME = 'certs'
 DATA_DIR_NAME = '.stoobly'
 DB_FILE_NAME = 'stoobly_agent.sqlite3'
 DB_VERSION_NAME = 'VERSION'
@@ -85,7 +86,7 @@ class DataDir:
 
     @property
     def certs_dir_path(self):
-        certs_dir_path = os.path.join(self.path, 'certs')
+        certs_dir_path = os.path.join(self.path, CERTS_DIR_NAME)
 
         if not os.path.exists(certs_dir_path):
             os.mkdir(certs_dir_path)