stoobly-agent 1.0.10__tar.gz → 1.0.11__tar.gz

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: stoobly-agent
-Version: 1.0.10
+Version: 1.0.11
 Summary: Record, mock, and test HTTP(s) requests. CLI agent for Stoobly
 License: Apache-2.0
 Author: Matt Le

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/pyproject.toml

@@ -10,7 +10,7 @@ description = "Record, mock, and test HTTP(s) requests. CLI agent for Stoobly"
 license = "Apache License 2.0"
 name = "stoobly-agent"
 readme = "README.md"
-version = "1.0.10"
+version = "1.0.11"
 
 [tool.poetry.dependencies]
 python = "^3.10"

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/__init__.py

@@ -1,2 +1,2 @@
 COMMAND = 'stoobly-agent'
-VERSION = '1.0.10'
+VERSION = '1.0.11'

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/app.py

@@ -13,6 +13,7 @@ class App():
     self.__ca_certs_dir_path = kwargs.get('ca_certs_dir_path') or data_dir.mitmproxy_conf_dir_path
     self.__certs_dir_path = data_dir.certs_dir_path
     self.__context_dir_path = data_dir.context_dir_path
+    self.__data_dir = data_dir
     self.__dir_path = path
     self.__name = os.path.basename(self.__dir_path)
     self.__network = os.path.basename(self.__dir_path)
@@ -46,6 +47,10 @@ class App():
     self.__validate_path(v)
     self.__context_dir_path = v 
 
+  @property
+  def data_dir(self):
+    return self.__data_dir
+
   @property
   def data_dir_path(self):
     return os.path.join(self.context_dir_path, DATA_DIR_NAME)

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/app_create_command.py

@@ -19,6 +19,6 @@ class AppCreateCommand(AppCommand):
         self.app.copy_folders_and_hidden_files(self.app_templates_root_dir, dest)
 
         with open(os.path.join(dest, '.gitignore'), 'w') as fp:
-            fp.write("\n".join(['**/.env', '**/.config.yml']))
+            fp.write("\n".join(['**/.env']))
 
         self.app_config.write()

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/constants.py

@@ -4,6 +4,7 @@ from stoobly_agent.config.data_dir import DATA_DIR_NAME
 
 
 APP_NETWORK_ENV = 'APP_NETWORK'
+BIN_FOLDER_NAME = 'bin'
 CA_CERTS_DIR_ENV = 'CA_CERTS_DIR'
 CERTS_DIR_ENV = 'CERTS_DIR'
 COMPOSE_TEMPLATE = '.docker-compose.{workflow}.yml'
@@ -12,6 +13,7 @@ CONTEXT_DIR_ENV = 'CONTEXT_DIR'
 DOCKER_NAMESPACE = 'docker'
 ENV_FILE = '.env'
 FIXTURES_FOLDER_NAME = 'fixtures'
+NAMESERVERS_FILE = '.nameservers'
 SERVICE_DETACHED = '${SERVICE_DETACHED}'
 SERVICE_DETACHED_ENV = 'SERVICE_DETACHED'
 SERVICE_DOCKER_COMPOSE_PATH = '${SERVICE_DOCKER_COMPOSE_PATH}'

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/docker/workflow/builder.py

@@ -7,9 +7,10 @@ from ...constants import (
   COMPOSE_TEMPLATE, SERVICE_HOSTNAME, SERVICE_HOSTNAME_ENV, SERVICE_NAME_ENV, SERVICE_PORT, SERVICE_PORT_ENV, SERVICE_SCHEME, 
   SERVICE_SCHEME_ENV, STOOBLY_HOME_DIR, WORKFLOW_NAME_ENV
 )
+from ...templates.constants import SERVICE_HOSTNAME_BUILD_ARG
+from ...workflow_env import WorkflowEnv
 from ..builder import Builder
 from ..service.builder import ServiceBuilder
-from ...templates.constants import SERVICE_HOSTNAME_BUILD_ARG
 
 class WorkflowBuilder(Builder):
 

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/docker/workflow/reverse_proxy_decorator.py

@@ -2,7 +2,9 @@ import pdb
 
 from urllib.parse import urlparse
 
-from ...constants import SERVICE_DNS, SERVICE_HOSTNAME, SERVICE_PORT
+from stoobly_agent.config.data_dir import DataDir
+
+from ...constants import SERVICE_DNS, SERVICE_DNS_ENV, SERVICE_HOSTNAME, SERVICE_PORT
 from .builder import WorkflowBuilder
 
 class ReverseProxyDecorator():
@@ -57,7 +59,7 @@ class ReverseProxyDecorator():
     # If we are reverse proxying to potentially an external host,
     # Docker's embedded DNS will use the host's /etc/host file as part of the resolution process
     # This can lead the hostname to resolve to localhost instead of the service's actual IP address
-    if config.dns and not config.detached:
+    if not config.detached:
       service['dns'] = SERVICE_DNS
 
     services[proxy_name] = service 

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/service_config.py

@@ -1,16 +1,11 @@
 # Wraps the .config.yml file in the service folder
-
-import dns.resolver
 import pdb
-import subprocess
-import re
 
 from .config import Config
 from .constants import (
   SERVICE_DETACHED_ENV,
   SERVICE_DOCKER_COMPOSE_PATH_ENV,
   SERVICE_HOSTNAME_ENV,
-  SERVICE_DNS_ENV,
   SERVICE_PRIORITY_ENV,
   SERVICE_PORT_ENV,
   SERVICE_PROXY_MODE_ENV,
@@ -25,7 +20,6 @@ class ServiceConfig(Config):
     self.__detached = None
     self.__docker_compose_path = None
     self.__hostname = None
-    self.__dns = None
     self.__port = None
     self.__priority = None
     self.__proxy_mode = None
@@ -75,28 +69,6 @@ class ServiceConfig(Config):
   def hostname(self, v):
     self.__hostname = v
 
-  @property
-  def dns(self):
-    # If hostname is set then the service is external and we will need to configure the container's DNS.
-    # If we don't configure the container's DNS, then Docker's embedded DNS will potentially
-    # use configuration from the host's /etc/hosts file. The user may have configured their
-    # /etc/hosts file to resolve requests to localhost
-    #
-    # See: 
-    # https://forums.docker.com/t/docker-127-0-0-11-resolver-should-use-host-etc-hosts-file/55157
-    # https://docs.docker.com/network/#dns-services
-    # 
-    # TODO: ideally we want to know if the service is built locally, if so, then no need to set DNS
-    # since Docker's embedded DNS will resolve to it
-    if self.hostname and not self.__dns:
-      nameservers = self.__find_dns()
-      self.__dns = nameservers[0] if nameservers else None
-    return self.__dns
-
-  @dns.setter
-  def dns(self, v):
-    self.__dns = v
-
   @property
   def port(self):
     if not self.__port:
@@ -147,9 +119,6 @@ class ServiceConfig(Config):
   def load(self, config = None):
     config = config or self.read()
 
-    # Do not load dns from config, have it dynamically determined
-    #self.dns = config.get(SERVICE_DNS_ENV)
-
     self.detached = config.get(SERVICE_DETACHED_ENV)
     self.docker_compose_path = config.get(SERVICE_DOCKER_COMPOSE_PATH_ENV)
     self.hostname = config.get(SERVICE_HOSTNAME_ENV)
@@ -167,9 +136,6 @@ class ServiceConfig(Config):
     if self.hostname:
       config[SERVICE_HOSTNAME_ENV] = self.hostname
 
-    if self.dns:
-      config[SERVICE_DNS_ENV] = self.dns
-    
     if self.port:
       config[SERVICE_PORT_ENV] = self.port
 
@@ -183,32 +149,4 @@ class ServiceConfig(Config):
 
     config[SERVICE_PROXY_MODE_ENV] = self.proxy_mode
 
-    super().write(config)
-
-  def __find_dns(self):
-    dns_resolver = dns.resolver.Resolver()
-    nameservers = dns_resolver.nameservers
-
-    # If systemd-resolved is not used
-    if nameservers != ['127.0.0.53']:
-      return nameservers
-
-    # Run the `resolvectl status` command and capture its output
-    result = subprocess.run(['resolvectl', 'status'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-
-    # Check if the command ran successfully
-    if result.returncode != 0:
-      return []
-
-    # Extract the DNS servers using a regular expression
-    #dns_servers = re.findall(r'DNS Servers: ([\d.]+(?:, [\d.]+)*)', result.stdout)
-    pattern = re.compile('DNS Servers:(.*?)DNS Domain', re.DOTALL)
-    match = re.findall(pattern, result.stdout)
-
-    if not match:
-      return []
-      
-    # Split the DNS servers string into a list
-    dns_servers = match[0].strip().split("\n")
-    return list(map(lambda dns_server: dns_server.strip(), dns_servers))
-
+    super().write(config)

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/templates/app/.Makefile

@@ -21,6 +21,7 @@ workflow_run_options=$${STOOBLY_WORKFLOW_RUN_OPTIONS:+$$STOOBLY_WORKFLOW_RUN_OPT
 
 app_data_dir=$(app_dir)/.stoobly
 data_dir=$(context_dir)/.stoobly
+app_tmp_dir=$(app_data_dir)/tmp
 
 # Commands
 docker_compose_command=docker compose
@@ -41,71 +42,82 @@ workflow_run_env=export APP_DIR="$(app_dir)" && export CERTS_DIR="$(certs_dir)"
 workflow_run=$(workflow_run_env) && $(source_env) && bash "$(workflow_run_script)"
 
 certs:
-	export EXEC_COMMAND=bin/.mkcert && \
+	@export EXEC_COMMAND=bin/.mkcert && \
 	$(stoobly_exec)
+nameservers:
+	@if [ -f /etc/resolv.conf ]; then \
+		nameserver=$$(grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' /etc/resolv.conf | tr -d '\n') && \
+		if [ "$$nameserver" = "127.0.0.53" ]; then \
+			echo "Nameserver is 127.0.0.53. Checking resolvectl status..."; \
+			nameserver=$$(resolvectl status | sed -n '/DNS Servers:/s/.*DNS Servers:\s*\([^ ]*\).*/\1/p' | head -n 1); \
+		fi; \
+		echo "$$nameserver" > $(app_tmp_dir)/.nameservers; \
+	else \
+		echo "/etc/resolv.conf not found."; \
+	fi
 intercept/disable:
-	export EXEC_COMMAND=bin/.disable && \
+	@export EXEC_COMMAND=bin/.disable && \
 	$(stoobly_exec)
 intercept/enable:
-	export EXEC_COMMAND=bin/.enable && \
+	@export EXEC_COMMAND=bin/.enable && \
 	export EXEC_ARGS=$(scenario_key) && \
 	$(stoobly_exec)
-mock: certs
-	export EXEC_COMMAND=bin/.run && \
+mock: nameservers
+	@export EXEC_COMMAND=bin/.run && \
 	export EXEC_OPTIONS="$(workflow_run_options)$(options)" && \
 	export EXEC_ARGS="mock" && \
 	$(stoobly_exec_run) && \
 	$(workflow_run)
 mock/stop:
-	export EXEC_COMMAND=bin/.stop && \
+	@export EXEC_COMMAND=bin/.stop && \
 	export EXEC_OPTIONS="$(options)" && \
 	export EXEC_ARGS="mock" && \
 	$(stoobly_exec_run) && \
 	$(workflow_run)
-record: certs
-	export EXEC_COMMAND=bin/.run && \
+record: nameservers
+	@export EXEC_COMMAND=bin/.run && \
 	export EXEC_OPTIONS="$(workflow_run_options)$(options)" && \
 	export EXEC_ARGS="record" && \
 	$(stoobly_exec_run) && \
 	$(workflow_run)
 record/stop:
-	export EXEC_COMMAND=bin/.stop && \
+	@export EXEC_COMMAND=bin/.stop && \
 	export EXEC_OPTIONS="$(options)" && \
 	export EXEC_ARGS="record" && \
 	$(stoobly_exec_run) && \
 	$(workflow_run)
 scenario/create:
 # Create a scenario
-	export EXEC_COMMAND=bin/.create && \
+	@export EXEC_COMMAND=bin/.create && \
 	export EXEC_OPTIONS="$(options)" && \
 	export EXEC_ARGS="$(name)" && \
 	$(stoobly_exec)
 scenario/delete:
 # Delete a scenario
-	export EXEC_COMMAND=bin/.delete && \
+	@export EXEC_COMMAND=bin/.delete && \
 	export EXEC_OPTIONS="$(options)" && \
 	export EXEC_ARGS="$(key)" && \
 	$(stoobly_exec)
 scenario/reset:
 # Resets a scenario to its last snapshot
-	export EXEC_COMMAND=bin/.reset && \:
+	@export EXEC_COMMAND=bin/.reset && \:
 	export EXEC_OPTIONS="$(options)" && \
 	export EXEC_ARGS="$(key)" && \
 	$(stoobly_exec)
 scenario/snapshot:
 # Create committable files for a scenario
-	export EXEC_COMMAND=bin/.snapshot && \
+	@export EXEC_COMMAND=bin/.snapshot && \
 	export EXEC_OPTIONS="$(options)" && \
 	export EXEC_ARGS="$(key)" && \
 	$(stoobly_exec)
 test:
-	export EXEC_COMMAND=bin/.run && \
+	@export EXEC_COMMAND=bin/.run && \
 	export EXEC_OPTIONS="$(workflow_run_options)$(options)" && \
 	export EXEC_ARGS="test" && \
 	$(stoobly_exec_run) && \
 	$(workflow_run)
 test/stop:
-	export EXEC_COMMAND=bin/.stop && \
+	@export EXEC_COMMAND=bin/.stop && \
 	export EXEC_OPTIONS="$(options)" && \
 	export EXEC_ARGS="test" && \
 	$(stoobly_exec_run) && \

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/templates/app/stoobly-ui/exec/bin/.run

@@ -5,6 +5,8 @@ workflow=$1
 
 mkdir -p .stoobly/tmp
 
+stoobly-agent scaffold app mkcert
+
 stoobly-agent scaffold workflow run \
   --app-dir-path "$(pwd)" \
   --dry-run \

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/templates/app/stoobly-ui/exec/bin/.stop

@@ -3,6 +3,8 @@
 extra_options=$EXEC_OPTIONS
 workflow=$1
 
+mkdir -p .stoobly/tmp
+
 stoobly-agent scaffold workflow stop \
   --app-dir-path "$(pwd)" \
   --dry-run \

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/validate_command.py

@@ -1,4 +1,5 @@
 import pdb
+import re
 from time import sleep
 
 import docker
@@ -31,7 +32,7 @@ class ValidateCommand():
 
     init_container = self.__get_container(init_container_name)
     logs = init_container.logs()
-    if logs:
+    if logs and re.search('error', str(logs), re.IGNORECASE):
       raise ScaffoldValidateException(f"Error logs potentially detected in: {init_container_name}")
     if init_container.status != 'exited' or init_container.attrs['State']['ExitCode'] != 0:
       raise ScaffoldValidateException(f"init container has not exited like expected: {init_container_name}")

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/workflow_command.py

@@ -6,7 +6,7 @@ from stoobly_agent.lib.logger import Logger
 
 from .app import App
 from .config import Config
-from .constants import COMPOSE_TEMPLATE, CONFIG_FILE, ENV_FILE, FIXTURES_FOLDER_NAME
+from .constants import BIN_FOLDER_NAME, COMPOSE_TEMPLATE, CONFIG_FILE, ENV_FILE, FIXTURES_FOLDER_NAME
 from .docker.constants import DOCKER_COMPOSE_CUSTOM
 from .service_command import ServiceCommand
 
@@ -19,6 +19,10 @@ class WorkflowCommand(ServiceCommand):
 
     self.__workflow_name = kwargs['workflow_name']
 
+  @property
+  def bin_dir_path(self):
+    return os.path.join(self.workflow_path, BIN_FOLDER_NAME)
+
   @property
   def compose_path(self):
     return os.path.join(

stoobly_agent-1.0.11/stoobly_agent/app/cli/scaffold/workflow_env.py

@@ -0,0 +1,20 @@
+from .config import Config
+from .constants import ENV_FILE
+from .env import Env
+
+class WorkflowEnv(Config):
+
+  def __init__(self, dir: str):
+    super().__init__(dir, ENV_FILE)
+
+    self.__env_vars = {}
+
+  def get(self, key):
+    return self.__env_vars.get(key)
+
+  def load(self):
+    self.__env_vars = self.read()
+
+  def write(self, env_vars: dict):
+    Env(self.path).write(env_vars)
+    self.__env_vars = env_vars

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold/workflow_run_command.py

@@ -1,5 +1,8 @@
+import dns.resolver
 import os
 import pdb
+import subprocess
+import re
 
 from typing import TypedDict
 
@@ -7,9 +10,12 @@ from stoobly_agent.config.data_dir import DataDir
 from stoobly_agent.lib.logger import Logger
 
 from .app import App
-from .constants import APP_NETWORK_ENV, CA_CERTS_DIR_ENV, CERTS_DIR_ENV, CONTEXT_DIR_ENV, SERVICE_NAME_ENV, USER_ID_ENV, WORKFLOW_NAME_ENV
-from .env import Env
+from .constants import (
+  APP_NETWORK_ENV, CA_CERTS_DIR_ENV, CERTS_DIR_ENV, CONTEXT_DIR_ENV, NAMESERVERS_FILE, 
+  SERVICE_DNS_ENV, SERVICE_NAME_ENV, USER_ID_ENV, WORKFLOW_NAME_ENV
+)
 from .workflow_command import WorkflowCommand
+from .workflow_env import WorkflowEnv
 
 LOG_ID = 'WorkflowRunCommand'
 
@@ -62,6 +68,20 @@ class WorkflowRunCommand(WorkflowCommand):
   def extra_compose_path(self):
     return self.__extra_compose_path
 
+  @property
+  def nameservers(self):
+    path = self.nameservers_path
+    if not os.path.exists(path):
+      return []
+
+    with open(path, 'r') as fp:
+      nameservers = fp.read()
+      return nameservers.split("\n")
+
+  @property
+  def nameservers_path(self):
+    return os.path.join(self.app.data_dir.tmp_dir_path, NAMESERVERS_FILE)
+
   @property
   def network(self):
     return self.__network
@@ -137,6 +157,27 @@ class WorkflowRunCommand(WorkflowCommand):
 
     return ' '.join(command)
 
+  def write_nameservers(self):
+    # If hostname is set then the service is external and we will need to configure the container's DNS.
+    # If we don't configure the container's DNS, then Docker's embedded DNS will potentially
+    # use configuration from the host's /etc/hosts file. The user may have configured their
+    # /etc/hosts file to resolve requests to localhost
+    #
+    # See: 
+    # https://forums.docker.com/t/docker-127-0-0-11-resolver-should-use-host-etc-hosts-file/55157
+    # https://docs.docker.com/network/#dns-services
+    # 
+    # TODO: ideally we want to know if the service is built locally, if so, then no need to set DNS
+    # since Docker's embedded DNS will resolve to it
+    dns_resolver = dns.resolver.Resolver()
+
+    with open(self.nameservers_path, 'w') as fp:
+      nameservers = self.__find_nameservers(dns_resolver)
+      ipv4_pattern = re.compile(r'^((25[0-5]|2[0-4][0-9]|1?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|1?[0-9][0-9]?)$')
+      nameservers = [ip for ip in nameservers if ipv4_pattern.match(ip)]
+      if nameservers:
+        fp.write("\n".join(nameservers))
+
   def write_env(self):
     _config = {}
     _config[CA_CERTS_DIR_ENV] = self.ca_certs_dir_path
@@ -149,7 +190,36 @@ class WorkflowRunCommand(WorkflowCommand):
     if self.network:
       _config[APP_NETWORK_ENV] = self.network
 
-    env_vars = self.config(_config)
-    env_path = self.workflow_env_path
-    Env(env_path).write(env_vars)
+    nameservers = self.nameservers
+    if nameservers:
+      _config[SERVICE_DNS_ENV] = nameservers[0]
 
+    env_vars = self.config(_config)
+    WorkflowEnv(self.workflow_path).write(env_vars)
+    return env_vars
+
+  def __find_nameservers(self, dns_resolver: dns.resolver.Resolver):
+    nameservers = dns_resolver.nameservers
+
+    # If systemd-resolved is not used
+    if nameservers != ['127.0.0.53']:
+      return nameservers
+
+    # Run the `resolvectl status` command and capture its output
+    result = subprocess.run(['resolvectl', 'status'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+    # Check if the command ran successfully
+    if result.returncode != 0:
+      return []
+
+    # Extract the DNS servers using a regular expression
+    #dns_servers = re.findall(r'DNS Servers: ([\d.]+(?:, [\d.]+)*)', result.stdout)
+    pattern = re.compile('DNS Servers:(.*?)DNS Domain', re.DOTALL)
+    match = re.findall(pattern, result.stdout)
+
+    if not match:
+      return []
+      
+    # Split the DNS servers string into a list
+    dns_servers = match[0].strip().split("\n")
+    return list(map(lambda dns_server: dns_server.strip(), dns_servers))

{stoobly_agent-1.0.10 → stoobly_agent-1.0.11}/stoobly_agent/app/cli/scaffold_cli.py

@@ -367,8 +367,12 @@ def run(**kwargs):
 
   # Before services can be started, their network needs to be created
   if len(commands) > 0:
-    create_network_command = commands[0].create_network()
+    command = commands[0]
+    create_network_command = command.create_network()
+
     if not kwargs['dry_run']:
+      command.write_nameservers()
+
       exec_stream(create_network_command)
     else:
       print(create_network_command)
@@ -418,7 +422,6 @@ def validate(**kwargs):
     print(f"\nFatal Scaffold Validation Exception: {sve}", file=sys.stderr)
     sys.exit(1)
 
-
 scaffold.add_command(app)
 scaffold.add_command(service)
 scaffold.add_command(workflow)
@@ -473,5 +476,4 @@ def __workflow_build(app, **kwargs):
     headless=kwargs['headless'],
     template=kwargs['template'],
     workflow_decorators=workflow_decorators
-  )
-
+  )