stoobly-agent 1.4.0__tar.gz → 1.4.2__tar.gz

{stoobly_agent-1.4.0 → stoobly_agent-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: stoobly-agent
-Version: 1.4.0
+Version: 1.4.2
 Summary: Record, mock, and test HTTP(s) requests. CLI agent for Stoobly
 License: Apache-2.0
 Author: Matt Le

{stoobly_agent-1.4.0 → stoobly_agent-1.4.2}/pyproject.toml

@@ -10,7 +10,7 @@ description = "Record, mock, and test HTTP(s) requests. CLI agent for Stoobly"
 license = "Apache License 2.0"
 name = "stoobly-agent"
 readme = "README.md"
-version = "1.4.0"
+version = "1.4.2"
 
 [tool.poetry.dependencies]
 python = "^3.10"

{stoobly_agent-1.4.0 → stoobly_agent-1.4.2}/stoobly_agent/__init__.py

@@ -1,2 +1,2 @@
 COMMAND = 'stoobly-agent'
-VERSION = '1.4.0'
+VERSION = '1.4.2'

{stoobly_agent-1.4.0 → stoobly_agent-1.4.2}/stoobly_agent/app/cli/scaffold/docker/workflow/mock_decorator.py

@@ -1,7 +1,7 @@
 import os
 import pdb
 
-from ...constants import SERVICE_HOSTNAME, SERVICE_PORT, STOOBLY_CERTS_DIR
+from ...constants import SERVICE_HOSTNAME, SERVICE_PORT, SERVICE_PROXY_MODE, STOOBLY_CERTS_DIR
 from .builder import WorkflowBuilder
 
 class MockDecorator():
@@ -24,7 +24,7 @@ class MockDecorator():
       '--headless',
       '--intercept',
       '--lifecycle-hooks-path', 'lifecycle_hooks.py',
-      '--proxy-mode', config.proxy_mode,
+      '--proxy-mode', SERVICE_PROXY_MODE,
       '--proxy-port', f"{SERVICE_PORT}",
       '--response-fixtures-path', 'fixtures.yml',
       '--ssl-insecure'

{stoobly_agent-1.4.0 → stoobly_agent-1.4.2}/stoobly_agent/app/cli/scaffold/docker/workflow/reverse_proxy_decorator.py

@@ -3,7 +3,7 @@ import pdb
 
 from urllib.parse import urlparse
 
-from ...constants import SERVICE_HOSTNAME, SERVICE_PORT, STOOBLY_CERTS_DIR
+from ...constants import SERVICE_HOSTNAME, SERVICE_PORT, SERVICE_PROXY_MODE, STOOBLY_CERTS_DIR
 from .builder import WorkflowBuilder
 
 class ReverseProxyDecorator():
@@ -25,7 +25,7 @@ class ReverseProxyDecorator():
     command = [
       '--headless',
       '--lifecycle-hooks-path', 'lifecycle_hooks.py',
-      '--proxy-mode', config.proxy_mode,
+      '--proxy-mode', SERVICE_PROXY_MODE,
       '--proxy-port', f"{SERVICE_PORT}",
       '--ssl-insecure'
     ]

stoobly_agent-1.4.2/stoobly_agent/app/cli/scaffold/hosts_file_manager.py

@@ -0,0 +1,185 @@
+import os
+import pdb
+import sys
+
+from dataclasses import dataclass
+from typing import Union
+
+SCAFFOLD_HOSTS_DELIMITTER_BEGIN = "##### STOOBLY SCAFFOLD HOSTS BEGIN #####\n"
+SCAFFOLD_HOSTS_DELIMITTER_END = "##### STOOBLY SCAFFOLD HOSTS END #####\n"
+
+class HostsFileManager():
+
+  @dataclass
+  class IpAddressToHostnames:
+    ip_address: str
+    hostnames: list[str]
+
+  def __get_hosts_file_path(self) -> str:
+    file_path = '/etc/hosts'
+    if not os.path.exists(file_path):
+      print(f"Error: File {file_path} not found.", file=sys.stderr)
+      sys.exit(1)
+    return file_path
+
+  # Split IP address and hostnames. Don't include inline comments
+  def __split_hosts_line(self, line: str) -> list[str]:
+    ip_addr_hosts_split = line.split('#')[0].split()
+    return ip_addr_hosts_split
+
+  # Parses hosts file and returns a mapping of IP address to hostnames in a list.
+  def get_hosts(self) -> list[IpAddressToHostnames]:
+    hosts_file_path = self.__get_hosts_file_path()
+
+    if not hosts_file_path:
+      return []
+
+    with open(hosts_file_path, 'r') as f:
+      hostlines = f.readlines()
+
+    # Skip comments and empty lines
+    hostlines = [line.strip() for line in hostlines
+                 if not line.startswith('#') and line.strip() != '']
+
+    hosts = []
+    for line in hostlines:
+      ip_addr_hosts_split = self.__split_hosts_line(line)
+      ip_address = ip_addr_hosts_split[0]
+      hostnames = ip_addr_hosts_split[1:]
+      ipAddressToHostnames = self.IpAddressToHostnames(ip_address, hostnames)
+
+      hosts.append(ipAddressToHostnames)
+
+    return hosts
+
+  def find_host(self, hostname) -> Union[IpAddressToHostnames, None]:
+    hosts = self.get_hosts()
+
+    for mapping in hosts:
+      if ((mapping.ip_address == '0.0.0.0' or mapping.ip_address == '127.0.0.1') and
+        hostname in mapping.hostnames):
+
+        return mapping
+
+    return None
+
+  def install_hostnames(self, hostnames: list[str]) -> None:
+    hosts_file_path = self.__get_hosts_file_path()
+
+    self.__add_lines_between_markers(
+      hosts_file_path, SCAFFOLD_HOSTS_DELIMITTER_BEGIN, SCAFFOLD_HOSTS_DELIMITTER_END, hostnames
+    )
+
+  def uninstall_hostnames(self, hostnames: list[str] = []) -> None:
+    hosts_file_path = self.__get_hosts_file_path()
+
+    self.__remove_lines_between_markers(
+      hosts_file_path, SCAFFOLD_HOSTS_DELIMITTER_BEGIN, SCAFFOLD_HOSTS_DELIMITTER_END, hostnames
+    ) 
+
+  def __remove_lines_between_markers(self, file_path, start_marker, end_marker, hostnames = []):
+    with open(file_path, "r") as file:
+      lines = file.readlines()
+
+    filtered_lines = []
+    index = 0
+
+    # Continue until we reach start_marker
+    for line in lines:
+      index += 1
+
+      if start_marker in line:
+        break
+
+      filtered_lines.append(line)
+
+    # Continue until we reach end_marker
+    found_hostnames = {}
+    section = []
+    for line in lines[index:]:
+      index += 1
+
+      if end_marker in line:
+        break
+        
+      found = False
+      for hostname in hostnames:
+        if hostname in line:
+          if hostname not in found_hostnames:
+            print(f"Removing hostname {hostname}")
+
+          found_hostnames[hostname] = True
+          found = True
+
+      if not found: 
+        section.append(line)
+
+    # If there are still lines in the section
+    if len(section):
+      filtered_lines.append(start_marker)
+      section.append(end_marker)    
+      filtered_lines += section
+
+    for line in lines[index:]:
+      filtered_lines.append(line)
+
+    with open(file_path, "w") as file:
+      file.writelines(filtered_lines)
+
+  def __add_lines_between_markers(self, file_path, start_marker, end_marker, hostnames = []):
+    with open(file_path, "r") as file:
+      lines = file.readlines()
+
+    filtered_lines = []
+    index = 0
+
+    # Continue until we reach start_marker
+    for line in lines:
+      index += 1
+      if start_marker in line:
+        break
+
+      filtered_lines.append(line)
+
+    # If no empty line before start_marker, add one
+    if len(filtered_lines):
+      last_line = filtered_lines[-1]
+
+      if last_line != "\n":
+        filtered_lines.append("\n")
+
+    filtered_lines.append(start_marker)
+
+    # Continue until we reach end_marker
+    found_hostnames = {}
+    for line in lines[index:]:
+      index += 1
+
+      if end_marker in line:
+        break
+        
+      found = False
+      for hostname in hostnames:
+        if hostname in line:
+          filtered_lines.append(line)
+          found_hostnames[hostname] = True
+          found = True
+
+      if not found:
+        filtered_lines.append(line)
+      
+    for hostname in hostnames:
+      if hostname in found_hostnames:
+        continue
+
+      print(f"Installing hostname {hostname}")
+      filtered_lines.append(f"127.0.0.1 {hostname}\n")
+      filtered_lines.append(f"::1       {hostname}\n")
+
+    filtered_lines.append(end_marker)    
+
+    for line in lines[index:]:
+      filtered_lines.append(line)
+
+    with open(file_path, "w") as file:
+      file.writelines(filtered_lines)

{stoobly_agent-1.4.0 → stoobly_agent-1.4.2}/stoobly_agent/app/cli/scaffold/service_config.py

@@ -92,7 +92,21 @@ class ServiceConfig(Config):
     if self.__proxy_mode:
       return (self.__proxy_mode or '').strip()
 
-    return f"reverse:{self.scheme}://{self.hostname}"
+    if not self.hostname:
+      return ''
+
+    _proxy_mode = f"reverse:{self.scheme}://{self.hostname}"
+
+    if not self.port:
+      return _proxy_mode
+
+    if self.scheme == 'http' and self.port == '80': 
+      return _proxy_mode
+
+    if self.scheme == 'https' and self.port == '443':
+      return _proxy_mode
+
+    return f"{_proxy_mode}:{self.port}"
 
   @proxy_mode.setter
   def proxy_mode(self, v):
@@ -123,7 +137,7 @@ class ServiceConfig(Config):
       'port': self.port,
       'priority': self.priority,
       'proxy_mode': self.proxy_mode,
-      'scheme': self.scheme,
+      'scheme': self.scheme if self.hostname else '',
     }
 
   def write(self):

{stoobly_agent-1.4.0 → stoobly_agent-1.4.2}/stoobly_agent/app/cli/scaffold/templates/app/.Makefile

@@ -61,8 +61,15 @@ stoobly_exec_run_env=$(source_env) && $(exec_env) && export CONTEXT_DIR="$(app_d
 workflow_run=$(source_env) && bash "$(workflow_run_script)"
 
 ca-cert/install: stoobly/install
-	@echo "Running stoobly-agent ca-cert install..."; \
-	stoobly-agent ca-cert install
+	@if [ ! -d "$$HOME/.mitmproxy" ]; then \
+		read -p "Installing CA certificate is required for $(WORKFLOW)ing requests, continue? (y/N) " confirm && \
+		if [ "$$confirm" = "y" ] || [ "$$confirm" = "Y" ]; then \
+			echo "Running stoobly-agent ca-cert install..."; \
+			stoobly-agent ca-cert install; \
+		else \
+			echo "You can install the CA certificate later by running: stoobly-agent ca-cert install"; \
+		fi \
+	fi
 certs:
 	@export EXEC_COMMAND=bin/.mkcert && \
 	$(stoobly_exec)
@@ -89,7 +96,7 @@ intercept/enable:
 	@export EXEC_COMMAND=bin/.enable && \
 	export EXEC_ARGS=$(scenario_key) && \
 	$(stoobly_exec)
-mock: workflow/mock workflow/hostname/install nameservers workflow/up
+mock: workflow/mock ca-cert/install workflow/hostname/install nameservers workflow/up
 mock/services: workflow/mock workflow/services
 mock/logs: workflow/mock workflow/logs
 mock/down: workflow/mock workflow/down workflow/hostname/uninstall
@@ -103,7 +110,7 @@ python/validate:
 		echo "Error: Python 3.10, 3.11, or 3.12 is required."; \
 		exit 1; \
 	fi
-record: workflow/record workflow/hostname/install nameservers workflow/up
+record: workflow/record ca-cert/install workflow/hostname/install nameservers workflow/up
 record/down: workflow/record workflow/down workflow/hostname/uninstall
 record/services: workflow/record workflow/services
 record/logs: workflow/record workflow/logs
@@ -139,7 +146,7 @@ scenario/snapshot:
 stoobly/install: python/validate pipx/install 
 	@if ! pipx list 2> /dev/null | grep -q 'stoobly-agent'; then \
 		echo "stoobly-agent not found. Installing..."; \
-		pipx install stoobly-agent; \
+		pipx install stoobly-agent || { echo "Failed to install stoobly-agent"; exit 1; }; \
 	fi
 test: workflow/test workflow/up
 test/services: workflow/test workflow/services
@@ -158,12 +165,12 @@ workflow/hostname: stoobly/install
 	if [ "$$confirm" = "y" ] || [ "$$confirm" = "Y" ]; then \
 		CURRENT_VERSION=$$(stoobly-agent --version); \
 		REQUIRED_VERSION="1.4.0"; \
-		if [ "$$(printf '%s\n' "$$REQUIRED_VERSION" "$$CURRENT_VERSION" | sort -V | tail -n 1)" = "$$CURRENT_VERSION" ]; then \
+		if [ "$$(printf '%s\n' "$$REQUIRED_VERSION" "$$CURRENT_VERSION" | sort -V | head -n 1)" != "$$REQUIRED_VERSION" ]; then \
 			echo "stoobly-agent version $$REQUIRED_VERSION required. Please run: pipx upgrade stoobly-agent"; \
 			exit 1; \
 		fi; \
-		echo "Running stoobly-agent scaffold hostname $(COMMAND)..."; \
-		stoobly-agent scaffold hostname $(COMMAND) --app-dir-path $(app_dir) --workflow $(WORKFLOW); \
+		echo "Running stoobly-agent scaffold hostname $(COMMAND) $(workflow_service_options)"; \
+		stoobly-agent scaffold hostname $(COMMAND) --app-dir-path $(app_dir) --workflow $(WORKFLOW) $(workflow_service_options); \
 	fi
 workflow/hostname/install: command/install workflow/hostname
 workflow/hostname/uninstall: command/uninstall workflow/hostname  

{stoobly_agent-1.4.0 → stoobly_agent-1.4.2}/stoobly_agent/app/cli/scaffold_cli.py

@@ -108,14 +108,19 @@ def create(**kwargs):
 @click.option('--certs-dir-path', help='Path to certs directory. Defaults to the certs dir of the context.')
 @click.option('--context-dir-path', default=data_dir.context_dir_path, help='Path to Stoobly data directory.')
 @click.option('--service', multiple=True, help='Select which services to run. Defaults to all.')
+@click.option('--workflow', multiple=True, help='Specify services by workflow(s). Defaults to all.')
 def mkcert(**kwargs):
   app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE, **kwargs)
   __validate_app(app)
 
-  services = __get_services(app.services, service=kwargs['service'])
+  services = __get_services(
+    app, service=kwargs['service'], without_core=True, workflow=kwargs['workflow']
+  )
 
   for service_name in services:
     service = Service(service_name, app)
+    __validate_service_dir(service.dir_path)
+
     service_config = ServiceConfig(service.dir_path)
 
     if service_config.scheme != 'https':
@@ -170,19 +175,23 @@ def create(**kwargs):
 @click.option('--select', multiple=True, help='Select column(s) to display.')
 @click.option('--service', multiple=True, help='Select specific services.')
 @click.option('--without-headers', is_flag=True, default=False, help='Disable printing column headers.')
-@click.option('--workflow', multiple=True, help='Specify workflow(s) to filter the services by.')
+@click.option('--workflow', multiple=True, help='Specify workflow(s) to filter the services by. Defaults to all.')
 def _list(**kwargs):
-  __validate_app_dir(kwargs['app_dir_path'])
-
   app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE)
   __validate_app(app)
 
+  services = __get_services(app, service=kwargs['service'], workflow=kwargs['workflow'])
+
   rows = []
-  for service_name in __get_workflow_services(app, **kwargs): 
+  for service_name in services: 
     service = Service(service_name, app)
     __validate_service_dir(service.dir_path)
+
     service_config = ServiceConfig(service.dir_path)
-    rows.append(service_config.to_dict())
+    rows.append({
+      'name': service_name,
+      **service_config.to_dict()
+    })
 
   print_services(rows, **select_print_options(kwargs))
 
@@ -192,9 +201,9 @@ def _list(**kwargs):
 @click.option('--app-dir-path', default=current_working_dir, help='Path to application directory.')
 @click.argument('service_name')
 def delete(**kwargs):
-  __validate_app_dir(kwargs['app_dir_path'])
-
   app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE)
+  __validate_app(app)
+
   service = Service(kwargs['service_name'], app)
 
   if not os.path.exists(service.dir_path):
@@ -211,9 +220,9 @@ def delete(**kwargs):
 @click.option('--priority', help='Determines the service run order.')
 @click.argument('service_name')
 def update(**kwargs):
-  __validate_app_dir(kwargs['app_dir_path'])
-
   app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE)
+  __validate_app(app)
+
   service = Service(kwargs['service_name'], app)
 
   __validate_service_dir(service.dir_path)
@@ -304,8 +313,9 @@ def down(**kwargs):
   if kwargs['namespace'] and not kwargs['network']:
     kwargs['network'] = kwargs['namespace']
 
-  workflow = Workflow(kwargs['workflow_name'], app)
-  services = __get_services(workflow.services, service=kwargs['service'])
+  services = __get_services(
+    app, service=kwargs['service'], workflow=[kwargs['workflow_name']]
+  )
 
   commands: List[WorkflowRunCommand] = []
   for service in services:
@@ -379,8 +389,9 @@ def logs(**kwargs):
   if len(kwargs['container']) == 0:
     kwargs['container'] = [WORKFLOW_CONTAINER_PROXY]
 
-  workflow = Workflow(kwargs['workflow_name'], app)
-  services = __get_services(workflow.services, service=kwargs['service'], without_core=True)
+  services = __get_services(
+    app, service=kwargs['service'], without_core=True, workflow=[kwargs['workflow_name']]
+  )
 
   commands: List[WorkflowLogCommand] = []
   for service in services:
@@ -445,10 +456,12 @@ def up(**kwargs):
   if kwargs['namespace'] and not kwargs['network']:
     kwargs['network'] = kwargs['namespace']
 
-  workflow = Workflow(kwargs['workflow_name'], app)
-  services = __get_services(workflow.services, service=kwargs['service'])
+  services = __get_services(
+    app, service=kwargs['service'], workflow=[kwargs['workflow_name']]
+  )
 
   # Gateway ports are dynamically set depending on the workflow run
+  workflow = Workflow(kwargs['workflow_name'], app)
   set_gateway_ports(workflow.service_paths_from_services(services))
 
   commands: List[WorkflowRunCommand] = []
@@ -541,13 +554,15 @@ def validate(**kwargs):
   help="Update the system hosts file for all scaffold service hostnames"
 )
 @click.option('--app-dir-path', default=current_working_dir, help='Path to application directory.')
-@click.option('--service', multiple=True, help='Select specific services.')
-@click.option('--workflow', multiple=True, help='Specify services by workflow(s).')
+@click.option('--service', multiple=True, help='Select specific services. Defaults to all.')
+@click.option('--workflow', multiple=True, help='Specify services by workflow(s). Defaults to all.')
 def install(**kwargs):
   app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE)
   __validate_app(app)
 
-  services = __get_workflow_services(app, **kwargs)
+  services = __get_services(
+    app, service=kwargs['service'], without_core=True, workflow=kwargs['workflow']
+  )
 
   hostnames = []
   for service_name in services: 
@@ -565,25 +580,39 @@ def install(**kwargs):
     hosts_file_manager.install_hostnames(hostnames)
   except PermissionError:
     print("Permission denied. Please run this command with sudo.", file=sys.stderr)
+    sys.exit(1)
 
 @hostname.command(
   help="Delete from the system hosts file all scaffold service hostnames"
 )
 @click.option('--app-dir-path', default=current_working_dir, help='Path to application directory.')
+@click.option('--service', multiple=True, help='Select specific services. Defaults to all.')
+@click.option('--workflow', multiple=True, help='Specify services by workflow(s). Defaults to all.')
 def uninstall(**kwargs):
   app = App(kwargs['app_dir_path'], DOCKER_NAMESPACE)
   __validate_app(app)
 
+  services = __get_services(
+    app, service=kwargs['service'], without_core=True, workflow=kwargs['workflow']
+  )
+
+  hostnames = []
+  for service_name in services: 
+    service = Service(service_name, app)
+    __validate_service_dir(service.dir_path)
+
+    service_config = ServiceConfig(service.dir_path)
+    if service_config.hostname:
+      hostnames.append(service_config.hostname)
+
   __elevate_sudo()
 
   try:
     hosts_file_manager = HostsFileManager()
-    hosts_file_manager.uninstall_hostnames()
-  except OSError as e:
-    if e.errno == errno.EACCES or e.errno == errno.EPERM:
-      print("Permission denied. Please run this command with sudo.", file=sys.stderr)
-    else:
-      print(f"An unexpected error occurred: {e}", file=sys.stderr)
+    hosts_file_manager.uninstall_hostnames(hostnames)
+  except PermissionError:
+    print("Permission denied. Please run this command with sudo.", file=sys.stderr)
+    sys.exit(1)
 
 scaffold.add_command(app)
 scaffold.add_command(service)
@@ -597,39 +626,37 @@ def __elevate_sudo():
     subprocess.run(["sudo", sys.executable] + sys.argv)
     sys.exit(0)
 
-def __get_services(services: List[str], **kwargs) -> List[str]:
+def __get_services(app: App, **kwargs):
   selected_services = list(kwargs['service'])
 
-  # If service is specified, run only those services
-  if selected_services:
-    missing_services = [service for service in selected_services if service not in services]
+  if not selected_services:
+    selected_services = app.services
+  else:
+    selected_services += CORE_SERVICES
+    missing_services = [service for service in selected_services if service not in app.services]
 
-    # Remove services that don't exist
     if missing_services:
+      # Warn if an invalid service is provided
       Logger.instance(LOG_ID).warn(f"Service(s) {','.join(missing_services)} are not found")
-      selected_services = list(set(selected_services) - set(missing_services))
 
-    services = selected_services
+    # Remove services that don't exist
+      selected_services = list(set(selected_services) - set(missing_services))
 
-  services += CORE_SERVICES
+  # If without_score is set, filter out CORE_SERVICES
+  if kwargs.get('without_core'):
+    selected_services = list(set(selected_services) - set(CORE_SERVICES))
 
-  services_index = {}
-  for service in services:
-    if kwargs.get('without_core') and service in CORE_SERVICES:
-        continue
-    services_index[service] = True
- 
-  return services_index.keys()
-
-def __get_workflow_services(app: App, **kwargs):
-  selected_services = []
-  if not kwargs['workflow']:
-    selected_services += __get_services(app.services, service=kwargs['service'], without_core=True)
-  else:
+  # If workflow is set, keep only services in the workflow
+  if kwargs.get('workflow'):
+    workflow_services = []
     for workflow_name in kwargs['workflow']:
       workflow = Workflow(workflow_name, app)
-      selected_services += __get_services(workflow.services, service=kwargs['service'], without_core=True)
-  return set(selected_services)
+      workflow_services += workflow.services
+
+    # Intersection
+    selected_services = list(filter(lambda x: x in workflow_services, selected_services))
+  
+  return list(set(selected_services))
 
 def __print_header(text: str):
   Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}{text}{bcolors.ENDC}")

stoobly_agent-1.4.2/stoobly_agent/test/app/models/schemas/.stoobly/db/VERSION

@@ -0,0 +1 @@
+1.4.1