stoobly-agent 1.0.13__tar.gz → 1.1.0__tar.gz

{stoobly_agent-1.0.13 → stoobly_agent-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: stoobly-agent
-Version: 1.0.13
+Version: 1.1.0
 Summary: Record, mock, and test HTTP(s) requests. CLI agent for Stoobly
 License: Apache-2.0
 Author: Matt Le
@@ -13,24 +13,23 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: click (>=8.1.0,<9.0.0)
-Requires-Dist: diff-match-patch (>=20230430,<20230431)
-Requires-Dist: distro (>=1.6.0,<1.7.0)
-Requires-Dist: dnspython (>=2.6.1,<2.7.0)
+Requires-Dist: diff-match-patch (>=v20241021,<20241022)
+Requires-Dist: distro (>=1.9.0,<1.10)
+Requires-Dist: dnspython (>=2.7.0,<2.8)
 Requires-Dist: docker (>=7.1.0,<8.0)
 Requires-Dist: httptools (>=0.4.0)
 Requires-Dist: jmespath (>=1.0.0)
-Requires-Dist: mergedeep (>=1.3.0,<1.3.4)
-Requires-Dist: mitmproxy (>=10.0.0,<11.0.0)
-Requires-Dist: multipart (>=0.2.5,<0.3.0)
+Requires-Dist: mergedeep (>=1.3,<1.4)
+Requires-Dist: mitmproxy (>=11.0.2,<11.1.0)
+Requires-Dist: multipart (>=1.2.1,<1.3)
 Requires-Dist: openapi-core (>=0.19.0,<0.20.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: pyyaml (>=6.0.1)
 Requires-Dist: requests (>=2.31.0)
 Requires-Dist: stoobly_orator (>=0.9.12)
 Requires-Dist: tabulate (>=0.8.8)
-Requires-Dist: watchdog (>=2.1.0,<=2.1.3)
-Requires-Dist: werkzeug (<3.0.0)
-Requires-Dist: yamale (>=4.0.0,<=4.0.2)
+Requires-Dist: watchdog (>=6.0.0,<=6.1.0)
+Requires-Dist: yamale (>=6.0.0,<=7.0.0)
 Project-URL: Homepage, https://github.com/Stoobly/stoobly-agent
 Description-Content-Type: text/markdown
 

{stoobly_agent-1.0.13 → stoobly_agent-1.1.0}/pyproject.toml

@@ -10,30 +10,29 @@ description = "Record, mock, and test HTTP(s) requests. CLI agent for Stoobly"
 license = "Apache License 2.0"
 name = "stoobly-agent"
 readme = "README.md"
-version = "1.0.13"
+version = "1.1.0"
 
 [tool.poetry.dependencies]
 python = "^3.10"
 
 click = "^8.1.0"
 docker = ">=7.1.0,<8.0"
-diff-match-patch = "^20230430"
-distro = ">=1.6.0,<1.7.0"
-dnspython = ">=2.6.1,<2.7.0"
+diff-match-patch = "^v20241021"
+distro = ">=1.9.0,<1.10"
+dnspython = ">=2.7.0,<2.8"
 httptools = ">=0.4.0"
 jmespath = ">=1.0.0"
-mergedeep = ">=1.3.0,<1.3.4"
-mitmproxy = ">=10.0.0,<11.0.0"
+mergedeep = ">=1.3,<1.4"
+mitmproxy = ">=11.0.2,<11.1.0"
 openapi-core = ">=0.19.0,<0.20.0"
-multipart = "^0.2.5"
+multipart = ">=1.2.1,<1.3"
 python-dotenv = "^1.0.1"
 pyyaml = ">=6.0.1"
 requests = ">=2.31.0"
 stoobly_orator = ">=0.9.12"
 tabulate = ">=0.8.8"
-watchdog = ">=2.1.0,<=2.1.3"
-werkzeug = "<3.0.0" # Dependency resolution
-yamale = ">=4.0.0,<=4.0.2"
+watchdog = ">=6.0.0,<=6.1.0"
+yamale = ">=6.0.0,<=7.0.0"
 
 [tool.poetry.group.test.dependencies]
 mock = ">=5.0.0"

{stoobly_agent-1.0.13 → stoobly_agent-1.1.0}/stoobly_agent/__init__.py

@@ -1,2 +1,2 @@
 COMMAND = 'stoobly-agent'
-VERSION = '1.0.13'
+VERSION = '1.1.0'

{stoobly_agent-1.0.13 → stoobly_agent-1.1.0}/stoobly_agent/app/cli/scaffold/constants.py

@@ -36,7 +36,13 @@ USER_ID_ENV = 'USER_ID'
 VIRTUAL_HOST_ENV = 'VIRTUAL_HOST'
 VIRTUAL_PORT_ENV = 'VIRTUAL_PORT'
 VIRTUAL_PROTO_ENV = 'VIRTUAL_PROTO'
-WORKFLOW_CUSTOM_FILTER = 'custom'
+WORKFLOW_CONTAINER_CONFIGURE = 'configure'
+WORKFLOW_CONTAINER_INIT = 'init'
+WORKFLOW_CONTAINER_PROXY = 'proxy'
+WORKFLOW_CONTAINER_TEMPLATE = '{service_name}.{container}'
+WORKFLOW_CONTAINER_CONFIGURE_TEMPLATE = '{service_name}.' + WORKFLOW_CONTAINER_CONFIGURE
+WORKFLOW_CONTAINER_INIT_TEMPLATE = '{service_name}.' + WORKFLOW_CONTAINER_INIT
+WORKFLOW_CONTAINER_PROXY_TEMPLATE = '{service_name}.' + WORKFLOW_CONTAINER_PROXY
 WORKFLOW_MOCK_TYPE = 'mock'
 WORKFLOW_NAME_ENV = 'WORKFLOW_NAME'
 WORKFLOW_RECORD_TYPE = 'record'

{stoobly_agent-1.0.13 → stoobly_agent-1.1.0}/stoobly_agent/app/cli/scaffold/docker/workflow/builder.py

@@ -4,11 +4,11 @@ import pdb
 from typing import List
 
 from ...constants import (
-  COMPOSE_TEMPLATE, SERVICE_HOSTNAME, SERVICE_HOSTNAME_ENV, SERVICE_NAME_ENV, SERVICE_PORT, SERVICE_PORT_ENV, SERVICE_SCHEME, 
-  SERVICE_SCHEME_ENV, STOOBLY_HOME_DIR, WORKFLOW_NAME_ENV
+  COMPOSE_TEMPLATE, STOOBLY_HOME_DIR, SERVICE_HOSTNAME, SERVICE_HOSTNAME_ENV, SERVICE_NAME_ENV, 
+  SERVICE_PORT, SERVICE_PORT_ENV, SERVICE_SCHEME, SERVICE_SCHEME_ENV, 
+  WORKFLOW_CONTAINER_CONFIGURE_TEMPLATE, WORKFLOW_CONTAINER_INIT_TEMPLATE, WORKFLOW_CONTAINER_PROXY_TEMPLATE, WORKFLOW_NAME_ENV
 )
 from ...templates.constants import SERVICE_HOSTNAME_BUILD_ARG
-from ...workflow_env import WorkflowEnv
 from ..builder import Builder
 from ..service.builder import ServiceBuilder
 
@@ -44,7 +44,7 @@ class WorkflowBuilder(Builder):
 
   @property
   def init(self):
-    return f"{self.namespace}.init"
+    return WORKFLOW_CONTAINER_INIT_TEMPLATE.format(service_name=self.namespace)
 
   @property
   def config(self):
@@ -52,7 +52,7 @@ class WorkflowBuilder(Builder):
 
   @property
   def configure(self):
-    return f"{self.namespace}.configure"
+    return WORKFLOW_CONTAINER_CONFIGURE_TEMPLATE.format(service_name=self.namespace)
 
   @property
   def context(self):
@@ -79,7 +79,7 @@ class WorkflowBuilder(Builder):
 
   @property
   def proxy(self):
-    return f"{self.namespace}.proxy"
+    return WORKFLOW_CONTAINER_PROXY_TEMPLATE.format(service_name=self.namespace)
 
   @property
   def proxy_build(self):

stoobly_agent-1.1.0/stoobly_agent/app/cli/scaffold/hosts_file_reader.py

@@ -0,0 +1,65 @@
+import pdb
+import platform
+from dataclasses import dataclass
+from typing import Union
+
+
+class HostsFileReader():
+
+  @dataclass
+  class IpAddressToHostnames:
+    ip_address: str
+    hostnames: list[str]
+
+  def __get_hosts_file_path(self) -> str:
+    system = platform.system()
+    if system == 'Linux' or system == 'Darwin':
+      return '/etc/hosts'
+    else:
+      print(f"Unsupported system: {system}, for hosts file validation, skipping")
+
+    return ''
+
+  # Split IP address and hostnames. Don't include inline comments
+  def __split_hosts_line(self, line: str) -> list[str]:
+    ip_addr_hosts_split = line.split('#')[0].split()
+    return ip_addr_hosts_split
+
+
+  # Parses hosts file and returns a mapping of IP address to hostnames in a list.
+  def get_hosts(self) -> list[IpAddressToHostnames]:
+    hosts_file_path = self.__get_hosts_file_path()
+
+    if not hosts_file_path:
+      return []
+
+    with open(hosts_file_path, 'r') as f:
+      hostlines = f.readlines()
+
+    # Skip comments and empty lines
+    hostlines = [line.strip() for line in hostlines
+                 if not line.startswith('#') and line.strip() != '']
+
+    hosts = []
+    for line in hostlines:
+      ip_addr_hosts_split = self.__split_hosts_line(line)
+      ip_address = ip_addr_hosts_split[0]
+      hostnames = ip_addr_hosts_split[1:]
+      ipAddressToHostnames = self.IpAddressToHostnames(ip_address, hostnames)
+
+      hosts.append(ipAddressToHostnames)
+
+    return hosts
+
+  def find_host(self, hostname) -> Union[IpAddressToHostnames, None]:
+    hosts = self.get_hosts()
+
+    for mapping in hosts:
+      if ((mapping.ip_address == '0.0.0.0' or mapping.ip_address == '127.0.0.1') and
+        hostname in mapping.hostnames):
+
+        return mapping
+
+    return None
+
+

stoobly_agent-1.1.0/stoobly_agent/app/cli/scaffold/service_delete_command.py

@@ -0,0 +1,35 @@
+import glob
+import os
+import pdb
+import shutil
+
+from .app import App
+from .service_command import ServiceCommand
+from stoobly_agent.lib.logger import Logger
+
+
+LOG_ID = 'ServiceDeleteCommand'
+
+class ServiceDeleteCommand(ServiceCommand):
+  def __init__(self, app: App, **kwargs):
+    super().__init__(app, **kwargs)
+
+  def delete(self) -> None:
+    Logger.instance(LOG_ID).debug(f"Deleting service: {self.service_name}, path: {self.service_path}")
+
+    # Delete service directory
+    shutil.rmtree(self.service_path)
+
+    # Delete certs for that hostname if HTTPS
+    if self.service_config.scheme == 'https':
+      certs_dir_path_escaped = glob.escape(self.app.certs_dir_path)
+      hostname = self.service_config.hostname
+      pattern = os.path.join(certs_dir_path_escaped, f"{hostname}*")
+
+      cert_paths = glob.glob(pattern)
+      for cert_path in cert_paths:
+        Logger.instance(LOG_ID).debug(f"Deleting cert_path: {cert_path}")
+        os.remove(cert_path)
+
+    return None
+

{stoobly_agent-1.0.13 → stoobly_agent-1.1.0}/stoobly_agent/app/cli/scaffold/service_workflow_validate_command.py

@@ -1,6 +1,8 @@
 import os
 import pdb
+import socket
 import ssl
+import time
 from collections import Counter
 from pathlib import Path
 
@@ -20,6 +22,7 @@ from stoobly_agent.app.cli.scaffold.constants import (
   WORKFLOW_RECORD_TYPE,
   WORKFLOW_TEST_TYPE,
 )
+from stoobly_agent.app.cli.scaffold.hosts_file_reader import HostsFileReader
 from stoobly_agent.app.cli.scaffold.service_command import ServiceCommand
 from stoobly_agent.app.cli.scaffold.service_docker_compose import ServiceDockerCompose
 from stoobly_agent.app.cli.scaffold.validate_command import ValidateCommand
@@ -68,26 +71,40 @@ class ServiceWorkflowValidateCommand(ServiceCommand, ValidateCommand):
   def is_external(self):
     return not self.is_local()
 
-  def hostname_reachable(self, url: str) -> None:
-    # Retry HTTP request. Source: https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
-    session = requests.Session()
-    retries = Retry(total=5,
-                    backoff_factor=0.1,
-                    status_forcelist=[ 500, 502, 503, 504 ])
-    session.mount('http://', HTTPAdapter(max_retries=retries))
-    session.mount('https://', HTTPAdapter(max_retries=retries))
-
-    # Use default OpenSSL path to CA certs on the system
-    default_ssl_verify_paths = ssl.get_default_verify_paths()
-    default_capath = default_ssl_verify_paths.capath
-
-    response = session.get(url=url, verify=default_capath)
-    if not response.ok:
-      raise ScaffoldValidateException(f"Host is not reachable: {url}")
+  def hostname_reachable(self, hostname: str, port: int) -> bool:
+    print(f"Validating connection to hostname: {hostname}, port: {port}")
+    timeout = 1
+    retries = 15
+    delay = 0.100
+
+    for attempt in range(retries):
+      try:
+        with socket.create_connection((hostname, port), timeout=timeout):
+          return True
+      except (socket.timeout, socket.error):
+        if attempt < retries - 1:
+          time.sleep(delay)
+
+    raise ScaffoldValidateException(f"Connection failed to hostname: {hostname}, port: {port}")
+
+  # Check if hostname is defined in hosts file
+  def hostname_exists(self, hostname: str) -> bool:
+    print(f"Validating hostname exists in hosts file for hostname: {hostname}")
+
+    hosts_file_reader = HostsFileReader()
+    host_mapping = hosts_file_reader.find_host(hostname)
+    if host_mapping:
+      print(f"Correct hosts mapping found for {hostname}")
+      return True
+
+    raise ScaffoldValidateException(f"Missing hosts mapping for {hostname}")
  
-  def validate_hostname(self, url: str) -> None:
-    print(f"Validating hostname: {url}")
-    self.hostname_reachable(url)
+  def validate_hostname(self, hostname: str, port: int) -> None:
+    print(f"Validating hostname: {hostname}")
+
+    self.hostname_exists(hostname)
+
+    self.hostname_reachable(hostname, port)
 
     # TODO: check logs of proxy. lifecycle hook for custom logging? Does mitmproxy support json logging?
 
@@ -189,7 +206,7 @@ class ServiceWorkflowValidateCommand(ServiceCommand, ValidateCommand):
     url = f"{self.service_config.scheme}://{self.hostname}"
 
     if self.service_config.hostname and self.workflow_name not in [WORKFLOW_TEST_TYPE]:
-      self.validate_hostname(url)
+      self.validate_hostname(self.hostname, self.service_config.port)
     
     # Test workflow won't expose services that are detached and have a hostname to the host such as assets.
     # Need to test connection from inside the Docker network

{stoobly_agent-1.0.13 → stoobly_agent-1.1.0}/stoobly_agent/app/cli/scaffold/templates/app/.Makefile

@@ -28,7 +28,7 @@ docker_compose_command=docker compose
 source_env=set -a; [ -f .env ] && source .env; set +a
 
 docker_compose_file_path=$(app_data_dir)/docker/stoobly-ui/exec/.docker-compose.exec.yml
-stoobly_exec_args=--profile $(EXEC_WORKFLOW_NAME) -p $(EXEC_WORKFLOW_NAME) up --build --remove-orphans
+stoobly_exec_args=--profile $(EXEC_WORKFLOW_NAME) -p $(EXEC_WORKFLOW_NAME) up --build --remove-orphans --pull always 
 stoobly_exec_env=export CONTEXT_DIR=$(context_dir) && export USER_ID=$$UID && export CA_CERTS_DIR="$(ca_certs_dir)"
 stoobly_exec=$(stoobly_exec_env) && $(source_env) && $(docker_compose_command) -f "$(docker_compose_file_path)" $(stoobly_exec_args)
 
@@ -46,7 +46,7 @@ certs:
 	$(stoobly_exec)
 nameservers:
 	@if [ -f /etc/resolv.conf ]; then \
-		nameserver=$$(grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' /etc/resolv.conf | tr -d '\n') && \
+		nameserver=$$(grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' /etc/resolv.conf) && \
 		if [ "$$nameserver" = "127.0.0.53" ]; then \
 			echo "Nameserver is 127.0.0.53. Checking resolvectl status..."; \
 			nameserver=$$(resolvectl status | sed -n '/DNS Servers:/s/.*DNS Servers:\s*\([^ ]*\).*/\1/p' | head -n 1); \
@@ -68,6 +68,12 @@ mock: nameservers
 	export EXEC_ARGS="mock" && \
 	$(stoobly_exec_run) && \
 	$(workflow_run)
+mock/logs: 
+	@export EXEC_COMMAND=bin/.logs && \
+	export EXEC_OPTIONS="$(options)" && \
+	export EXEC_ARGS="mock" && \
+	$(stoobly_exec_run) && \
+	$(workflow_run)
 mock/stop:
 	@export EXEC_COMMAND=bin/.stop && \
 	export EXEC_OPTIONS="$(options)" && \
@@ -80,6 +86,12 @@ record: nameservers
 	export EXEC_ARGS="record" && \
 	$(stoobly_exec_run) && \
 	$(workflow_run)
+record/logs: 
+	@export EXEC_COMMAND=bin/.logs && \
+	export EXEC_OPTIONS="$(options)" && \
+	export EXEC_ARGS="record" && \
+	$(stoobly_exec_run) && \
+	$(workflow_run)
 record/stop:
 	@export EXEC_COMMAND=bin/.stop && \
 	export EXEC_OPTIONS="$(options)" && \
@@ -98,6 +110,11 @@ scenario/delete:
 	export EXEC_OPTIONS="$(options)" && \
 	export EXEC_ARGS="$(key)" && \
 	$(stoobly_exec)
+scenario/list:
+# List scenarios
+	@export EXEC_COMMAND=bin/.list && \
+	export EXEC_OPTIONS="$(options)" && \
+	$(stoobly_exec)
 scenario/reset:
 # Resets a scenario to its last snapshot
 	@export EXEC_COMMAND=bin/.reset && \:
@@ -116,6 +133,12 @@ test:
 	export EXEC_ARGS="test" && \
 	$(stoobly_exec_run) && \
 	$(workflow_run)
+test/logs: 
+	@export EXEC_COMMAND=bin/.logs && \
+	export EXEC_OPTIONS="$(options)" && \
+	export EXEC_ARGS="test" && \
+	$(stoobly_exec_run) && \
+	$(workflow_run)
 test/stop:
 	@export EXEC_COMMAND=bin/.stop && \
 	export EXEC_OPTIONS="$(options)" && \

{stoobly_agent-1.0.13 → stoobly_agent-1.1.0}/stoobly_agent/app/cli/scaffold/templates/app/gateway/.docker-compose.base.yml

@@ -1,9 +1,9 @@
 services:
   gateway_base:
+    environment:
+      TRUST_DOWNSTREAM_PROXY: true
     image: nginxproxy/nginx-proxy:1.5
-    ports:
-      - '80:80'
-      - '443:443' # Entry port
+    ports: {}
     profiles:
       - gateway_base
     volumes:

stoobly_agent-1.1.0/stoobly_agent/app/cli/scaffold/templates/app/stoobly-ui/exec/bin/.list

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+extra_options=$EXEC_OPTIONS
+
+stoobly-agent scenario list $extra_options

stoobly_agent-1.1.0/stoobly_agent/app/cli/scaffold/templates/app/stoobly-ui/exec/bin/.logs

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+extra_options=$EXEC_OPTIONS
+workflow=$1
+
+stoobly-agent scaffold workflow logs \
+  --app-dir-path "$(pwd)" \
+  --dry-run \
+  $extra_options \
+  $workflow > .stoobly/tmp/run.sh

{stoobly_agent-1.0.13 → stoobly_agent-1.1.0}/stoobly_agent/app/cli/scaffold/workflow_command.py

@@ -39,7 +39,6 @@ class WorkflowCommand(ServiceCommand):
 
   @property
   def containers(self):
-    _containers = []
     if not os.path.exists(self.compose_path):
       return []
 
@@ -49,11 +48,8 @@ class WorkflowCommand(ServiceCommand):
       if not isinstance(services, dict):
         return []
 
-      for service in services:
-        _containers.append(f"{self.workflow_name}-{service}-1")
+      return services
         
-    return _containers
-
   @property
   def custom_compose(self):
     if os.path.exists(self.custom_compose_path):

stoobly_agent-1.1.0/stoobly_agent/app/cli/scaffold/workflow_log_command.py

@@ -0,0 +1,46 @@
+import pdb
+
+from typing import TypedDict
+
+from stoobly_agent.lib.logger import DEBUG
+
+from .app import App
+from .constants import WORKFLOW_CONTAINER_TEMPLATE
+from .workflow_command import WorkflowCommand
+
+class WorkflowLogCommand(TypedDict):
+  container: str
+  follow: bool
+  namespace: str
+
+class WorkflowLogCommand(WorkflowCommand):
+  def __init__(self, app: App, **kwargs):
+    super().__init__(app, **kwargs)
+
+  def build(self, **options: WorkflowLogCommand):
+    commands = []
+    containers = self.containers
+    allowed_containers = list(
+      map(
+        lambda container: WORKFLOW_CONTAINER_TEMPLATE.format(
+          container=container, service_name=self.service_name
+        ), options.get('containers') or []
+      )
+    )
+
+    for index, container in enumerate(containers):
+      if container not in allowed_containers:
+        continue
+
+      container_name = self.container_name(container, options.get('namespace'))
+      commands.append(f"echo \"=== Logging {container_name}\"")
+      if options.get('follow') and index == len(containers) - 1:
+        command = ['docker', 'logs', '--follow', container_name]
+      else:
+        command = ['docker', 'logs', container_name]
+      commands.append(' '.join(command))
+
+    return commands
+
+  def container_name(self, container, namespace=None):
+    return f"{namespace or self.workflow_name}-{container}-1"

{stoobly_agent-1.0.13 → stoobly_agent-1.1.0}/stoobly_agent/app/cli/scaffold/workflow_run_command.py

@@ -129,13 +129,19 @@ class WorkflowRunCommand(WorkflowCommand):
     if not options.get('attached'):
       command.append('-d')
     else:
+      major_version = 2
+      minor_version = 27
+      patch_version = 0
+      min_version = major_version * 10000 + minor_version * 100 + patch_version
+      formula = "'{print $1*10000 + $2*100 + $3}'"
+      option = f"$(test $(echo $(docker compose version --short) | awk -F. {formula}) -ge {min_version} && echo '--abort-on-container-failure')"
+
       # This option enables docker compose to return exit code 1 
       # when one of the services exits with a non-zero exit code
       # Otherwise, even if a service exits with a non-zero exit code, exit code 0 is returned
-      command.append('--abort-on-container-failure')
+      command.append(option)
 
     command.append('--build')
-    command.append('--pull always')
 
     self.write_env()