stoobly-agent 1.9.2__tar.gz → 1.9.4__tar.gz

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: stoobly-agent
-Version: 1.9.2
+Version: 1.9.4
 Summary: Record, mock, and test HTTP(s) requests. CLI agent for Stoobly
 License: Apache-2.0
 Author: Matt Le

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/pyproject.toml

@@ -10,7 +10,7 @@ description = "Record, mock, and test HTTP(s) requests. CLI agent for Stoobly"
 license = "Apache License 2.0"
 name = "stoobly-agent"
 readme = "README.md"
-version = "1.9.2"
+version = "1.9.4"
 
 [tool.poetry.dependencies]
 python = "^3.10"

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/__init__.py

@@ -1,2 +1,2 @@
 COMMAND = 'stoobly-agent'
-VERSION = '1.9.2'
+VERSION = '1.9.4'

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/app/cli/scaffold/app_config.py

@@ -1,5 +1,5 @@
 from .config import Config
-from .constants import APP_NAME_ENV, APP_NETWORK_ENV
+from .constants import APP_NAME_ENV, APP_NETWORK_ENV, APP_UI_PORT_ENV
 
 class AppConfig(Config):
 
@@ -27,11 +27,20 @@ class AppConfig(Config):
   def network(self, v):
     self.__network = v
 
+  @property
+  def ui_port(self):
+    return self.__ui_port
+
+  @ui_port.setter
+  def ui_port(self, v):
+    self.__ui_port = v
+
   def load(self, config = None):
     config = config or self.read()
 
     self.name = config.get(APP_NAME_ENV)
     self.network = config.get(APP_NETWORK_ENV)
+    self.ui_port = config.get(APP_UI_PORT_ENV)
     
   def write(self):
     config = {}
@@ -42,4 +51,7 @@ class AppConfig(Config):
     if self.network:
       config[APP_NETWORK_ENV] = self.network
 
+    if self.ui_port:
+      config[APP_UI_PORT_ENV] = self.ui_port
+
     super().write(config)

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/app/cli/scaffold/app_create_command.py

@@ -9,6 +9,7 @@ from .app_command import AppCommand
 class AppCreateOptions(TypedDict):
   name: str
   network: str
+  ui_port: int
 
 class AppCreateCommand(AppCommand):
 
@@ -21,6 +22,9 @@ class AppCreateCommand(AppCommand):
         if kwargs.get('network'):
             self.app_config.network = kwargs['network']
 
+        if kwargs.get('ui_port'):
+            self.app_config.ui_port = kwargs['ui_port']
+
     @property
     def app_name(self):
         return self.app_config.name
@@ -29,6 +33,10 @@ class AppCreateCommand(AppCommand):
     def app_network(self):
         return self.app_config.network
 
+    @property
+    def app_ui_port(self):
+        return self.app_config.ui_port
+
     def build(self):
         dest = self.scaffold_namespace_path
 

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/app/cli/scaffold/constants.py

@@ -8,6 +8,7 @@ APP_DIR = '${APP_DIR}'
 APP_DIR_ENV = 'APP_DIR'
 APP_NETWORK_ENV = 'APP_NETWORK'
 APP_NAME_ENV = 'APP_NAME'
+APP_UI_PORT_ENV = 'APP_UI_PORT'
 BIN_FOLDER_NAME = 'bin'
 CA_CERTS_DIR_ENV = 'CA_CERTS_DIR'
 CERTS_DIR_ENV = 'CERTS_DIR'

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/app/cli/scaffold/docker/service/configure_gateway.py

@@ -51,6 +51,8 @@ def __with_networks(config: dict, hostnames: List[str]):
   }
 
 def __with_traefik_config(service_paths: str, compose: dict, app_dir_path: str):
+  config_dest = '/etc/traefik/traefik.yml'
+
   if not compose['volumes']:
     compose['volumes'] = []
 
@@ -68,11 +70,15 @@ def __with_traefik_config(service_paths: str, compose: dict, app_dir_path: str):
     'providers': {
       'docker': {
         'exposedByDefault': False
+      },
+      'file': {
+        'fileName': config_dest,
+        'watch': False
       }
     },
     'tls': {
-      'certificates': certificates
-    }
+      'certificates': certificates,
+    },
   }
 
   for path in service_paths:
@@ -88,7 +94,7 @@ def __with_traefik_config(service_paths: str, compose: dict, app_dir_path: str):
     if config.scheme == 'https':
       certificates.append({
         'certFile': f"/certs/{config.hostname}.crt",
-        'keyFile': f"/certs/{config.hostname}.key"
+        'keyFile': f"/certs/{config.hostname}.key",
       })
 
   # Create traefik.yml in .stoobly/tmp
@@ -102,7 +108,7 @@ def __with_traefik_config(service_paths: str, compose: dict, app_dir_path: str):
     fp.write(yaml.dump(traefik_config))
 
   compose['volumes'].append(
-    f"{os.path.join(APP_DIR, traefik_template_relative_path)}:/etc/traefik/traefik.yml:ro"
+    f"{os.path.join(APP_DIR, traefik_template_relative_path)}:{config_dest}:ro"
   )
 
 def __find_hosts(service_paths):

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/app/cli/scaffold/templates/app/stoobly-ui/.docker-compose.base.yml

@@ -1,8 +1,5 @@
 services:
   stoobly_ui.base:
-    environment:
-      SERVICE_NAME: ${SERVICE_NAME}
-      WORKFLOW_NAME: ${WORKFLOW_NAME}
     extends:
       file: ../.docker-compose.base.yml
       service: context_base 

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/app/cli/scaffold/templates/app/stoobly-ui/mock/.docker-compose.mock.yml

@@ -6,6 +6,6 @@ services:
       service: stoobly_ui.base
     image: stoobly.${USER_ID}
     ports:
-      - '4200:4200'
+      - '${APP_UI_PORT}:4200'
     profiles:
       - mock

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/app/cli/scaffold/templates/app/stoobly-ui/record/.docker-compose.record.yml

@@ -6,7 +6,7 @@ services:
       service: stoobly_ui.base
     image: stoobly.${USER_ID}
     ports:
-      - '4200:4200'
+      - '${APP_UI_PORT}:4200'
     profiles:
       - record
 

stoobly_agent-1.9.4/stoobly_agent/app/cli/scaffold/templates/workflow/test/bin/init

@@ -0,0 +1,9 @@
+#! /bin/bash
+
+# Add custom initialization here
+
+app_path=$1 # Path to application source files
+
+# For example, the below commands copies files inside a dist folder into the public folder. 
+# The public folder is used when no recorded mocks are found.
+#cp -r $app_path/dist/. public

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/app/cli/scaffold_cli.py

@@ -6,6 +6,7 @@ import sys
 
 from io import TextIOWrapper
 from typing import List
+from urllib.parse import urlparse
 
 from stoobly_agent.app.cli.helpers.certificate_authority import CertificateAuthority
 from stoobly_agent.app.cli.helpers.shell import exec_stream
@@ -88,6 +89,7 @@ def hostname(ctx):
 @click.option('--app-dir-path', default=current_working_dir, help='Path to create the app scaffold.')
 @click.option('--force', is_flag=True, help='Overwrite maintained scaffolded app files.')
 @click.option('--network', help='App default network name. Defaults to app name.')
+@click.option('--ui-port', default=4200, type=click.IntRange(1, 65535), help='UI service port.')
 @click.argument('app_name')
 def create(**kwargs):
   __validate_app_dir(kwargs['app_dir_path'])
@@ -148,10 +150,10 @@ def create(**kwargs):
     sys.exit(1)
 
   if kwargs.get('hostname'):
-    hostname_regex = re.compile(r'^[a-zA-Z0-9.-]+$')
-    if not re.search(hostname_regex, kwargs['hostname']):
-      print(f"Error: {kwargs['hostname']} is invalid.", file=sys.stderr)
-      sys.exit(1)
+    __validate_hostname(kwargs.get('hostname'))
+
+  if kwargs.get("proxy_mode"):
+    __validate_proxy_mode(kwargs.get("proxy_mode"))
 
   app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE)
 
@@ -216,6 +218,13 @@ def delete(**kwargs):
 @click.option('--port', type=click.IntRange(1, 65535), help='Service port.')
 @click.option('--priority', default=5, type=click.FloatRange(1.0, 9.0), help='Determines the service run order. Lower values run first.')
 @click.option('--scheme', type=click.Choice(['http', 'https']), help='Defaults to https if hostname is set.')
+@click.option('--name', type=click.STRING, help='New name of the service to update to.')
+@click.option('--proxy-mode', help='''
+  Proxy mode can be "regular", "transparent", "socks5",
+  "reverse:SPEC", or "upstream:SPEC". For reverse and
+  upstream proxy modes, SPEC is host specification in
+  the form of "http[s]://host[:port]".
+''')
 @click.argument('service_name')
 def update(**kwargs):
   app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE)
@@ -228,7 +237,20 @@ def update(**kwargs):
   service_config = ServiceConfig(service.dir_path)
 
   if kwargs['hostname']:
-    service_config.hostname = kwargs['hostname']
+    __validate_hostname(kwargs['hostname'])
+
+    old_hostname = service_config.hostname
+
+    if old_hostname != kwargs['hostname']:
+      service_config.hostname = kwargs['hostname']
+
+      # If this is the default proxy_mode and the origin matches the original hostname, assume it is safe to update with the new hostname
+      if service_config.proxy_mode.startswith("reverse:"):
+        old_origin = service_config.proxy_mode.split("reverse:")[1]
+        parsed_origin_url = urlparse(old_origin)
+
+        if old_hostname == parsed_origin_url.hostname:
+          service_config.proxy_mode = service_config.proxy_mode.replace(old_hostname, service_config.hostname)
 
   if kwargs['priority']:
     service_config.priority = kwargs['priority']
@@ -239,6 +261,10 @@ def update(**kwargs):
   if kwargs['scheme']:
     service_config.scheme = kwargs['scheme']
 
+  if kwargs['proxy_mode']:
+    __validate_proxy_mode(kwargs['proxy_mode'])
+    service_config.proxy_mode = kwargs['proxy_mode']
+
   service_config.write()
 
 @workflow.command(
@@ -759,7 +785,42 @@ def __validate_app_dir(app_dir_path):
 
 def __validate_service_dir(service_dir_path):
   if not os.path.exists(service_dir_path):
-    print(f"Error: {service_dir_path} does not exist, please scaffold this service", file=sys.stderr)
+    print(f"Error: '{service_dir_path}' does not exist, please scaffold this service", file=sys.stderr)
+    sys.exit(1)
+
+def __validate_proxy_mode(proxy_mode: str) -> None:
+  valid_exact_matches = {
+    "regular": None,
+    "transparent": None,
+    "socks5": None,
+  }
+
+  valid_prefixes = {
+    "reverse": None,
+    "upstream": None
+  }
+
+  if proxy_mode in valid_exact_matches:
+    return
+
+  split_str = proxy_mode.split(":", 1)
+  if len(split_str) != 2:
+    print(f"Error: {proxy_mode} is invalid.", file=sys.stderr)
+    sys.exit(1)
+
+  prefix = split_str[0]
+  spec = split_str[1]
+
+  if prefix not in valid_prefixes:
+    print(f"Error: {proxy_mode} is invalid.", file=sys.stderr)
+    sys.exit(1)
+
+  # TODO: validate SPEC
+
+def __validate_hostname(hostname: str) -> None:
+  hostname_regex = re.compile(r'^[a-zA-Z0-9.-]+$')
+  if not re.search(hostname_regex, hostname):
+    print(f"Error: {hostname} is invalid.", file=sys.stderr)
     sys.exit(1)
 
 def __workflow_create(app, **kwargs):

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/app/models/adapters/joined_request_adapter.py

@@ -17,9 +17,7 @@ class JoinedRequestAdapter():
     if isinstance(payloads_delimitter, str):
       payloads_delimitter = payloads_delimitter.encode()
       
-    self.__split_joined_request_string = joined_request_string.split(payloads_delimitter)
-    if len(self.__split_joined_request_string) != 2:
-      self.__split_joined_request_string = joined_request_string.split(payloads_delimitter.replace(b"\n", b"\r\n"))
+    self.__split_joined_request_string = self.raw_request_split(joined_request_string, payloads_delimitter) 
 
     if len(self.__split_joined_request_string) != 2:
       raise ValueError(f"Could not split by {payloads_delimitter}")
@@ -47,7 +45,7 @@ class JoinedRequestAdapter():
     request_string = RequestString(None)
 
     delimitter = RequestStringCLRF
-    request_string_toks = self.__split_joined_request_string[0].split(delimitter)
+    request_string_toks = self.repaired_string_toks(self.__split_joined_request_string[0], delimitter)
     request_string.set(self.raw_request_string or delimitter.join(request_string_toks[1:]))
     request_string.control = request_string_toks[0]
 
@@ -57,7 +55,7 @@ class JoinedRequestAdapter():
     response_string = ResponseString(None, None)
 
     delimitter = ResponseStringCLRF
-    response_string_toks = self.__split_joined_request_string[1].split(delimitter)
+    response_string_toks = self.repaired_string_toks(self.__split_joined_request_string[1], delimitter)
     response_string.set(self.raw_response_string or delimitter.join(response_string_toks[1:]))
     response_string.control = response_string_toks[0]
 
@@ -68,4 +66,38 @@ class JoinedRequestAdapter():
 
     joined_request.request_string = self.build_request_string()
     joined_request.response_string = self.build_response_string()
-    return joined_request
+    return joined_request
+
+  # If all CRLF characters have been replaced with LF e.g. visual studio code
+  # Then try to repair the raw string, see https://github.com/Stoobly/stoobly-agent/issues/415
+  @staticmethod
+  def repaired_string_toks(raw_string: bytes, delimitter: bytes):
+    toks = raw_string.split(delimitter)
+
+    if len(toks) == 1:
+      lf = b"\n"
+      toks = raw_string.split(lf)
+
+      # See for request: https://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html
+      # See for response: https://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html 
+      i = 0
+      for line in toks:
+        i += 1
+
+        # On two lf characters, then the following lines are the body
+        if line == b'':
+          break
+
+      toks = toks[:i] + [lf.join(toks[i:])]
+
+    if len(toks) == 1:
+      raise ValueError(f"Could not split request by {delimitter}")
+
+    return toks
+
+  @staticmethod
+  def raw_request_split(raw_string: bytes, payloads_delimitter = REQUEST_DELIMITTER):
+    toks = raw_string.split(payloads_delimitter)
+    if len(toks) != 2:
+      toks = raw_string.split(payloads_delimitter.replace(b"\n", b"\r\n"))
+    return toks

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/app/models/helpers/apply.py

@@ -2,10 +2,10 @@ import pdb
 
 from typing import List
 
+from stoobly_agent.app.models.adapters.joined_request_adapter import JoinedRequestAdapter
 from stoobly_agent.app.models.factories.resource.local_db.helpers.log import Log
 from stoobly_agent.app.models.factories.resource.local_db.helpers.request_snapshot import RequestSnapshot
 from stoobly_agent.app.models.factories.resource.local_db.helpers.scenario_snapshot import ScenarioSnapshot
-from stoobly_agent.app.proxy.record import REQUEST_STRING_CLRF
 from stoobly_agent.app.settings import Settings
 from stoobly_agent.lib.logger import bcolors
 
@@ -165,7 +165,12 @@ class Apply():
       self.__logger(f"{bcolors.WARNING}Skipping Request{bcolors.ENDC} {error}")
       return error, 301
 
-    return self.__put_request(uuid, raw_request)
+    res, status = self.__put_request(uuid, raw_request)
+
+    if status != 200:
+      return f"{res} {snapshot.path}", status
+
+    return res, status
 
   def __apply_delete_scenario(self, uuid: str):
     res, status = self.scenario_model.destroy(uuid, force=self.__force)
@@ -227,8 +232,8 @@ class Apply():
 
       if not raw_request:
         return f"{request_snapshot.path} is missing", 400
-        
-      toks = raw_request.split(REQUEST_STRING_CLRF, 1)
+
+      toks = JoinedRequestAdapter.raw_request_split(raw_request)
       if len(toks) != 2:
         return f"{request_snapshot.path} contains an invalid request", 400
 
@@ -236,7 +241,7 @@ class Apply():
       res, status = self.__put_request(uuid, raw_request, scenario_id=scenario['id'])
 
       if status != 200:
-        return res, status
+        return f"{res} {request_snapshot.path}", status
 
       snapshot_requests[uuid] = res
 
@@ -274,8 +279,6 @@ class Apply():
 
       if self.__logger and status == 200:
         self.__logger(f"{bcolors.OKGREEN}Created Request{bcolors.ENDC} {res['list'][0]['url']}")
-      else: 
-        self.__logger(f"{bcolors.FAIL}{status}{bcolors.ENDC} {res}")
     elif status == 200:
       params = {
         'is_deleted': False,
@@ -287,7 +290,5 @@ class Apply():
       if self.__logger:
         if status == 200:
           self.__logger(f"{bcolors.OKCYAN}Updated Request{bcolors.ENDC} {res['url']}")
-        else: 
-          self.__logger(f"{bcolors.FAIL}{status}{bcolors.ENDC} {res}")
 
     return res, status

{stoobly_agent-1.9.2 → stoobly_agent-1.9.4}/stoobly_agent/cli.py

@@ -115,12 +115,12 @@ def init(**kwargs):
   upstream proxy modes, SPEC is host specification in
   the form of "http[s]://host[:port]".
 ''')
-@click.option('--proxy-port', default=8080, help='Proxy service port.')
+@click.option('--proxy-port', default=8080, type=click.IntRange(1, 65535), help='Proxy service port.')
 @click.option('--public-directory-path', help='Path to public files. Used for mocking requests.')
 @click.option('--response-fixtures-path', help='Path to response fixtures yaml. Used for mocking requests.')
 @click.option('--ssl-insecure', is_flag=True, default=False, help='Do not verify upstream server SSL/TLS certificates.')
 @click.option('--ui-host', default='0.0.0.0', help='Address to bind UI to.')
-@click.option('--ui-port', default=4200, help='UI service port.')
+@click.option('--ui-port', default=4200, type=click.IntRange(1, 65535), help='UI service port.')
 def run(**kwargs):
     from .app.proxy.run import run as run_proxy
 

stoobly_agent-1.9.4/stoobly_agent/test/app/models/adapters/joined_rquest_adapter_test.py

@@ -0,0 +1,38 @@
+import pytest
+
+from stoobly_agent.app.models.adapters.joined_request_adapter import RequestStringCLRF, JoinedRequestAdapter
+
+class TestJoindRequestStringAdapter():
+
+  @pytest.fixture(scope='class', autouse=True)
+  def corrupted_raw_string(self):
+    toks = [
+      b"control",
+      b"header1",
+      b"header2",
+      b'',
+      b'body1',
+      b'body2'
+    ]
+    return b"\n".join(toks)
+
+  @pytest.fixture(scope='class', autouse=True)
+  def repaired_toks(self, corrupted_raw_string: bytes):
+    # [b'control', b'header1', b'header2', b'', b'body1\nbody2']
+    return JoinedRequestAdapter.repaired_string_toks(corrupted_raw_string, RequestStringCLRF)
+
+  def test_control(self, repaired_toks: list):
+    assert repaired_toks[0] == b'control'
+
+  def test_header1(self, repaired_toks: list):
+    assert repaired_toks[1] == b'header1'
+
+  def test_header2(self, repaired_toks: list):
+    assert repaired_toks[2] == b'header2'
+
+  def test_clrf(self, repaired_toks: list):
+    assert repaired_toks[3] == b''
+
+  def test_body(self, repaired_toks: list):
+    assert repaired_toks[4] == b'body1\nbody2'
+    

stoobly_agent-1.9.4/stoobly_agent/test/app/models/schemas/.stoobly/db/VERSION

@@ -0,0 +1 @@
+1.9.3

stoobly_agent-1.9.2/stoobly_agent/app/cli/scaffold/templates/workflow/test/bin/init

@@ -1,5 +0,0 @@
-#! /bin/bash
-
-# Add custom initialization here
-
-app_path=$1 # Path to application source files

stoobly_agent-1.9.2/stoobly_agent/test/app/models/schemas/.stoobly/db/VERSION

@@ -1 +0,0 @@
-1.9.0