stoobly-agent 1.10.6__tar.gz → 1.11.0__tar.gz

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stoobly-agent
-Version: 1.10.6
+Version: 1.11.0
 Summary: Record, mock, and test HTTP(s) requests. CLI agent for Stoobly
 License: Apache-2.0
 License-File: LICENSE

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/pyproject.toml

@@ -10,7 +10,7 @@ description = "Record, mock, and test HTTP(s) requests. CLI agent for Stoobly"
 license = "Apache License 2.0"
 name = "stoobly-agent"
 readme = "README.md"
-version = "1.10.6"
+version = "1.11.0"
 
 [tool.poetry.dependencies]
 python = "^3.10"

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/__init__.py

@@ -1,2 +1,2 @@
 COMMAND = 'stoobly-agent'
-VERSION = '1.10.6'
+VERSION = '1.11.0'

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/intercept_cli.py

@@ -128,7 +128,7 @@ def configure(**kwargs):
         active_mode = settings.proxy.intercept.mode
         valid_policies = __get_policy_options(active_mode)
 
-        if not kwargs['policy'] in valid_policies:
+        if kwargs['policy'] not in valid_policies:
             print("Error: Valid policies for {active_mode} are {valid_policies}", file=sys.stderr)
             sys.exit(1)
 

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/request_cli.py

@@ -1,6 +1,7 @@
 import click
 
 from stoobly_agent.app.cli.helpers.handle_replay_service import BODY_FORMAT, JSON_FORMAT
+from stoobly_agent.lib.intercepted_requests_logger import InterceptedRequestsLogger
 from stoobly_agent.app.models.factories.resource.local_db.helpers.log_event import DELETE_ACTION, PUT_ACTION
 from stoobly_agent.app.settings import Settings
 from stoobly_agent.config.constants import alias_resolve_strategy, test_filter, test_output_level, test_strategy
@@ -170,4 +171,21 @@ def get(**kwargs) -> None:
 def query(**kwargs):
   query_handler(kwargs)
 
-request.add_command(response)
+@click.group(
+  epilog="Run 'stoobly-agent request log COMMAND --help' for more information on a command.",
+  help="Manage intercepted requests logs"
+)
+@click.pass_context
+def log(ctx):
+    pass
+
+@log.command(name="list", help="List intercepted requests log entries")
+def log_list(**kwargs):
+  InterceptedRequestsLogger.dump_logs()
+
+@log.command(name="delete", help="Delete intercepted requests log entries")
+def log_delete(**kwargs):
+  InterceptedRequestsLogger.truncate()
+
+request.add_command(response)
+request.add_command(log)

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/docker/service/builder.py

@@ -1,21 +1,18 @@
 import os
 import pdb
 
-from typing import List
-
 from stoobly_agent.config.data_dir import DATA_DIR_NAME
 
 from ...app_config import AppConfig
 from ...constants import (
-  APP_DIR, SERVICES_NAMESPACE, 
+  APP_DIR, SERVICES_NAMESPACE,
   SERVICE_HOSTNAME, SERVICE_HOSTNAME_ENV,
-  SERVICE_NAME, SERVICE_NAME_ENV, 
   SERVICE_ID,
-  SERVICE_PORT, SERVICE_PORT_ENV, 
+  SERVICE_PORT, SERVICE_PORT_ENV,
   SERVICE_SCHEME, SERVICE_SCHEME_ENV,
   SERVICE_UPSTREAM_HOSTNAME, SERVICE_UPSTREAM_HOSTNAME_ENV, SERVICE_UPSTREAM_PORT, SERVICE_UPSTREAM_PORT_ENV, SERVICE_UPSTREAM_SCHEME, SERVICE_UPSTREAM_SCHEME_ENV,
   STOOBLY_HOME_DIR, 
-  WORKFLOW_NAME, WORKFLOW_NAME_ENV, WORKFLOW_SCRIPTS, WORKFLOW_TEMPLATE
+  WORKFLOW_SCRIPTS, WORKFLOW_TEMPLATE
 )
 from ...service_config import ServiceConfig
 from ...local.service.builder import ServiceBuilder

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/docker/workflow/builder.py

@@ -4,7 +4,7 @@ import pdb
 from typing import List, Union
 
 from ...constants import (
-  WORKFLOW_CONTAINER_CONFIGURE_TEMPLATE, WORKFLOW_CONTAINER_INIT_TEMPLATE, WORKFLOW_CONTAINER_PROXY_TEMPLATE, WORKFLOW_NAME, WORKFLOW_NAME_ENV
+  WORKFLOW_CONTAINER_CONFIGURE_TEMPLATE, WORKFLOW_CONTAINER_INIT_TEMPLATE, WORKFLOW_CONTAINER_PROXY_TEMPLATE, WORKFLOW_NAME
 )
 from ...local.workflow.builder import WorkflowBuilder
 from ..builder import Builder
@@ -13,6 +13,7 @@ from ..service.builder import DockerServiceBuilder
 from .mock_decorator import MockDecorator
 from .reverse_proxy_decorator import ReverseProxyDecorator
 
+
 class DockerWorkflowBuilder(Builder, WorkflowBuilder):
 
   def __init__(self, workflow_path: str, service_builder: DockerServiceBuilder):

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/docker/workflow/mock_decorator.py

@@ -34,6 +34,8 @@ class MockDecorator(CommandDecorator):
       command.append('--certs')
       command.append(os.path.join(STOOBLY_CERTS_DIR, f"{SERVICE_HOSTNAME}-joined.pem"))
 
+    command.append('--request-log-enable')
+
     services = self.workflow_builder.services
     proxy_name = self.workflow_builder.proxy
     proxy_service = services.get(proxy_name) or {}

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/local/service/builder.py

@@ -3,17 +3,10 @@ from typing import List
 
 from stoobly_agent.config.data_dir import DATA_DIR_NAME
 
-from ...app_config import AppConfig
 from ...constants import (
-  APP_DIR, SERVICES_NAMESPACE, 
-  SERVICE_HOSTNAME, SERVICE_HOSTNAME_ENV,
-  SERVICE_NAME, SERVICE_NAME_ENV, 
-  SERVICE_ID,
-  SERVICE_PORT, SERVICE_PORT_ENV, 
-  SERVICE_SCHEME, SERVICE_SCHEME_ENV,
-  SERVICE_UPSTREAM_HOSTNAME, SERVICE_UPSTREAM_HOSTNAME_ENV, SERVICE_UPSTREAM_PORT, SERVICE_UPSTREAM_PORT_ENV, SERVICE_UPSTREAM_SCHEME, SERVICE_UPSTREAM_SCHEME_ENV,
+  SERVICES_NAMESPACE, SERVICE_NAME, SERVICE_NAME_ENV, 
   STOOBLY_HOME_DIR, 
-  WORKFLOW_NAME, WORKFLOW_NAME_ENV, WORKFLOW_SCRIPTS, WORKFLOW_TEMPLATE
+  WORKFLOW_NAME, WORKFLOW_NAME_ENV, WORKFLOW_NAMESPACE_ENV
 )
 from ...service_config import ServiceConfig
 
@@ -23,7 +16,7 @@ class ServiceBuilder():
     self.__config = config
     self.__dir_path = config.dir
     self.__upstream_port = None
-    self.__env = [SERVICE_NAME_ENV, WORKFLOW_NAME_ENV]
+    self.__env = [SERVICE_NAME_ENV, WORKFLOW_NAME_ENV, WORKFLOW_NAMESPACE_ENV]
     self.__service_name = os.path.basename(config.dir)
     self.__working_dir = os.path.join(
       STOOBLY_HOME_DIR, DATA_DIR_NAME, SERVICES_NAMESPACE, SERVICE_NAME, WORKFLOW_NAME

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/local/workflow/builder.py

@@ -1,14 +1,12 @@
 import os
 
-from typing import Union
-
-from ...constants import SERVICE_NAME_ENV, WORKFLOW_NAME_ENV
+from ...constants import SERVICE_NAME_ENV, WORKFLOW_NAME_ENV, WORKFLOW_NAMESPACE_ENV
 from ..service.builder import ServiceBuilder
 
 class WorkflowBuilder():
 
   def __init__(self, workflow_path: str, service_builder: ServiceBuilder):
-    self._env = [SERVICE_NAME_ENV, WORKFLOW_NAME_ENV]
+    self._env = [SERVICE_NAME_ENV, WORKFLOW_NAME_ENV, WORKFLOW_NAMESPACE_ENV]
     self._service_builder = service_builder
     self._workflow_name = os.path.basename(workflow_path)
 

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/service_workflow_validate_command.py

@@ -135,7 +135,7 @@ class ServiceWorkflowValidateCommand(ServiceCommand, ValidateCommand):
     # connection is working, but we haven't recorded anything yet
     logs = output.decode('ascii')
     if logs != '200' and  logs != '499':
-      raise ScaffoldValidateException(f"Error reaching {url} from inside Docker network")
+      raise ScaffoldValidateException(f"Error reaching {url} from inside Docker network with logs: \n{logs}")
 
   # Check public folder exists in container
   def validate_public_folder(self, container: Container):

stoobly_agent-1.11.0/stoobly_agent/app/cli/scaffold/templates/app/.Dockerfile.context

@@ -0,0 +1,6 @@
+FROM stoobly/agent:1.11
+
+ARG USER_ID
+
+# Change user id of stoobly user to that of host's user id
+RUN if [ -n "$USER_ID" ]; then usermod -u $USER_ID stoobly; fi

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/templates/app/.Makefile

@@ -111,9 +111,10 @@ intercept/enable:
 	@export EXEC_COMMAND=intercept/.enable EXEC_OPTIONS="" EXEC_ARGS=$(scenario_key) && \
 	$(stoobly_exec)
 mock: workflow/mock ca-cert/install workflow/up nameservers workflow/hostname/install workflow/up/run
-mock/services: workflow/mock workflow/services
-mock/logs: workflow/mock workflow/logs workflow/logs/run
 mock/down: workflow/mock workflow/down workflow/down/run workflow/hostname/uninstall
+mock/logs: workflow/mock workflow/logs workflow/logs/run
+mock/report: workflow/mock workflow/report
+mock/services: workflow/mock workflow/services
 pipx/install:
 	@if ! command -v pipx >/dev/null 2>&1; then \
 		echo "pipx is not installed. Installing pipx..."; \
@@ -126,8 +127,9 @@ python/validate:
 	fi
 record: workflow/record ca-cert/install workflow/up nameservers workflow/hostname/install workflow/up/run
 record/down: workflow/record workflow/down workflow/down/run workflow/hostname/uninstall
-record/services: workflow/record workflow/services
 record/logs: workflow/record workflow/logs workflow/logs/run
+record/report: workflow/record workflow/report
+record/services: workflow/record workflow/services
 scenario/create:
 # Create a scenario
 	@export EXEC_COMMAND=scenario/.create EXEC_OPTIONS="$(options)" EXEC_ARGS="$(name)" && \
@@ -158,9 +160,10 @@ stoobly/install: python/validate pipx/install
 		pipx install stoobly-agent || { echo "Failed to install stoobly-agent"; exit 1; }; \
 	fi
 test: workflow/test workflow/up workflow/up/run
-test/services: workflow/test workflow/services
-test/logs: workflow/test workflow/logs workflow/logs/run
 test/down: workflow/test workflow/down workflow/down/run
+test/logs: workflow/test workflow/logs workflow/logs/run
+test/report: workflow/test workflow/report
+test/services: workflow/test workflow/services
 tmpdir:
 	@mkdir -p $(app_tmp_dir)
 workflow/down: dotenv
@@ -197,6 +200,9 @@ workflow/namespace: tmpdir
 	@mkdir -p $(workflow_namespace_dir)
 workflow/record:
 	$(eval workflow=record)
+workflow/report:
+	@export EXEC_COMMAND=request/log/.list EXEC_OPTIONS="$(options)" EXEC_ARGS="" && \
+	$(stoobly_exec)
 workflow/services:
 	@export EXEC_COMMAND=scaffold/.services EXEC_OPTIONS="$(workflow_service_options) $(options)" EXEC_ARGS="$(workflow)" && \
 	$(stoobly_exec_run)

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/templates/app/.docker-compose.base.yml

@@ -4,6 +4,8 @@ services:
       service: stoobly_base
     volumes:
       - ${CONTEXT_DIR}/.stoobly:/home/stoobly/.stoobly
+      - ${APP_DIR}/.stoobly/ca_certs:/home/stoobly/.stoobly/ca_certs
+      - ${APP_DIR}/.stoobly/certs:/home/stoobly/.stoobly/certs
       - ${APP_DIR}/.stoobly/services:/home/stoobly/.stoobly/services
   stoobly_base:
     image: stoobly.${USER_ID}

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/templates/app/stoobly-ui/mock/.docker-compose.yml

@@ -5,8 +5,8 @@ services:
       file: ../.docker-compose.base.yml
       service: stoobly_ui.base
     networks:
-      app.egress: 
-        aliases: 
+      app.egress:
+        aliases:
           - local.stoobly.com
     ports:
       - '${APP_UI_PORT}:4200'

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/templates/app/stoobly-ui/record/.docker-compose.yml

@@ -5,8 +5,8 @@ services:
       file: ../.docker-compose.base.yml
       service: stoobly_ui.base
     networks:
-      app.egress: 
-        aliases: 
+      app.egress:
+        aliases:
           - local.stoobly.com
     ports:
       - '${APP_UI_PORT}:4200'

stoobly_agent-1.11.0/stoobly_agent/app/cli/scaffold/templates/build/workflows/exec/request/log/.list

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+extra_options=$EXEC_OPTIONS
+
+stoobly-agent request log list $extra_options

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/cli/scaffold/workflow_run_command.py

@@ -8,8 +8,8 @@ from stoobly_agent.config.data_dir import DataDir
 
 from .app import App
 from .constants import (
-  APP_DIR_ENV, APP_NETWORK_ENV, APP_PLUGINS_ENV, CA_CERTS_DIR_ENV, CERTS_DIR_ENV, CONTEXT_DIR_ENV, 
-  SERVICE_DNS_ENV, SERVICE_NAME_ENV, SERVICE_SCRIPTS_DIR,  SERVICE_SCRIPTS_ENV, USER_ID_ENV, 
+  APP_DIR_ENV, APP_NETWORK_ENV, CA_CERTS_DIR_ENV, CERTS_DIR_ENV, CONTEXT_DIR_ENV,
+  SERVICE_DNS_ENV, SERVICE_NAME_ENV, SERVICE_SCRIPTS_DIR,  SERVICE_SCRIPTS_ENV, USER_ID_ENV,
   WORKFLOW_NAME_ENV, WORKFLOW_NAMESPACE_ENV, WORKFLOW_SCRIPTS_DIR, WORKFLOW_SCRIPTS_ENV, WORKFLOW_TEMPLATE_ENV
 )
 
@@ -139,9 +139,8 @@ class WorkflowRunCommand(WorkflowCommand):
     _config[WORKFLOW_NAME_ENV] = self.workflow_name
     _config[WORKFLOW_SCRIPTS_ENV] = WORKFLOW_SCRIPTS_DIR
     _config[WORKFLOW_TEMPLATE_ENV] = self.workflow_name
-    
-    if namespace:
-      _config[WORKFLOW_NAMESPACE_ENV] = namespace
+    # Default to the workflow name if a namespace isn't given
+    _config[WORKFLOW_NAMESPACE_ENV] = namespace if namespace else _config[WORKFLOW_NAME_ENV]
 
     if self.network:
       _config[APP_NETWORK_ENV] = self.network

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/proxy/handle_mock_service.py

@@ -10,8 +10,8 @@ from stoobly_agent.app.models.request_model import RequestModel
 from stoobly_agent.app.proxy.mitmproxy.request_facade import MitmproxyRequestFacade
 from stoobly_agent.app.proxy.utils.rewrite_rules_to_ignored_components_service import rewrite_rules_to_ignored_components
 from stoobly_agent.config.constants import custom_headers, env_vars, lifecycle_hooks, mock_policy, request_origin
-from stoobly_agent.lib.logger import bcolors, Logger
-
+from stoobly_agent.lib.logger import bcolors, Logger, DEBUG, INFO, WARNING, ERROR
+from stoobly_agent.lib.intercepted_requests_logger import InterceptedRequestsLogger
 from .constants import custom_response_codes
 from .mock.context import MockContext
 from .mock.eval_fixtures_service import eval_fixtures
@@ -153,6 +153,9 @@ def handle_response_mock(context: MockContext):
 def __handle_mock_failure(context: MockContext) -> None:
     flow = context.flow
     request = flow.request
+    response = context.response
+
+    InterceptedRequestsLogger.error("Mock failure", request=request, response=response)
 
     if request.method.upper() != 'OPTIONS':
         return False
@@ -187,10 +190,12 @@ def __handle_mock_success(context: MockContext) -> None:
         request_key = response.headers.get(custom_headers.MOCK_REQUEST_KEY)
         if request_key:
             Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Mocked{bcolors.ENDC} {request.url} -> {request_key}")
+            InterceptedRequestsLogger.info("Mock success", request=request, response=response, request_key=request_key)
 
         fixture_path = response.headers.get(custom_headers.MOCK_FIXTURE_PATH)
         if fixture_path:
             Logger.instance(LOG_ID).info(f"{bcolors.OKBLUE}Mocked{bcolors.ENDC} {request.url} -> {fixture_path}")
+            InterceptedRequestsLogger.info("Mock success", request=request, response=response, fixture_path=fixture_path)
 
     if os.environ.get(env_vars.AGENT_SIMULATE_LATENCY):
         response = context.response

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/app/proxy/run.py

@@ -138,4 +138,7 @@ def __filter_options(options):
     del options['log_level']
     del options['proxy_host']
     del options['proxy_mode']
-    del options['proxy_port']
+    del options['proxy_port']
+    del options['request_log_enable']
+    del options['request_log_level']
+    del options['request_log_truncate']

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/cli.py

@@ -12,9 +12,9 @@ from stoobly_agent.app.cli.helpers.validations import validate_project_key, vali
 from stoobly_agent.app.cli.intercept_cli import mode_options
 from stoobly_agent.app.proxy.constants import custom_response_codes
 from stoobly_agent.app.proxy.replay.replay_request_service import replay as replay_request
-from stoobly_agent.app.settings.constants import intercept_mode
 from stoobly_agent.config.constants import env_vars, mode
 from stoobly_agent.config.data_dir import DataDir
+from stoobly_agent.lib.intercepted_requests_logger import InterceptedRequestsLogger
 from stoobly_agent.lib.logger import Logger
 from stoobly_agent.lib.utils.conditional_decorator import ConditionalDecorator
 
@@ -123,6 +123,9 @@ def init(**kwargs):
 @click.option('--proxy-port', default=8080, type=click.IntRange(1, 65535), help='Proxy service port.')
 @click.option('--public-directory-path', multiple=True, help='Path to public files. Used for mocking requests. Can take the form <FOLDER-PATH>[:<ORIGIN>].')
 @click.option('--response-fixtures-path', multiple=True, help='Path to response fixtures yaml. Used for mocking requests. Can take the form <FILE-PATH>[:<ORIGIN>].')
+@click.option('--request-log-enable', is_flag=True, default=False, required=False, help='Enable intercepted requests logging')
+@click.option('--request-log-level', default=logger.INFO, type=click.Choice([logger.DEBUG, logger.INFO, logger.WARNING, logger.ERROR]), help='Log level for intercepted requests.')
+@click.option('--request-log-truncate', is_flag=True, default=True, required=False, help='Truncate the intercepted requests log')
 @click.option('--ssl-insecure', is_flag=True, default=False, help='Do not verify upstream server SSL/TLS certificates.')
 @click.option('--ui-host', default='0.0.0.0', help='Address to bind UI to.')
 @click.option('--ui-port', default=4200, type=click.IntRange(1, 65535), help='UI service port.')
@@ -180,6 +183,18 @@ def run(**kwargs):
       os.environ[env_vars.AGENT_PROXY_URL] = proxy_url
       settings.proxy.url = proxy_url
 
+    if kwargs.get('request_log_enable'):
+      # If truncating, do that first (it handles enable internally)
+      if kwargs.get('request_log_truncate'):
+        InterceptedRequestsLogger.truncate()
+      else:
+        InterceptedRequestsLogger.enable_logger_file()
+
+      # Set log level after logger is enabled
+      request_log_level = kwargs.get('request_log_level')
+      if request_log_level:
+        InterceptedRequestsLogger.set_log_level(request_log_level)
+
     if kwargs.get('detached'):
       # Run in detached mode with output redirection
       import subprocess

{stoobly_agent-1.10.6 → stoobly_agent-1.11.0}/stoobly_agent/config/constants/custom_headers.py

@@ -25,6 +25,7 @@ SESSION_ID = 'X-Stoobly-Session-Id'
 SERVICE_URL = 'X-Stoobly-Service-Url'
 TEST_FILTER = 'X-Stoobly-Test-Filter'
 TEST_ID = 'X-Stoobly-Test-Id'
+TEST_NAME = 'X-Stoobly-Test-Name'
 TEST_POLICY = 'X-Stoobly-Test-Policy'
 TEST_SAVE_RESULTS = 'X-Stoobly-Test-Save-Results'
 TEST_SKIP = 'X-Stoobly-Test-Skip'