stoobly-agent 0.34.13__py3-none-any.whl → 1.0.0__py3-none-any.whl

stoobly_agent/app/cli/scaffold/service_config.py

@@ -0,0 +1,207 @@
+# Wraps the .config.yml file in the service folder
+
+import dns.resolver
+import pdb
+import subprocess
+import re
+
+from .config import Config
+from .constants import (
+  SERVICE_DETACHED_ENV,
+  SERVICE_DOCKER_COMPOSE_PATH_ENV,
+  SERVICE_HOSTNAME_ENV,
+  SERVICE_DNS_ENV,
+  SERVICE_PRIORITY_ENV,
+  SERVICE_PORT_ENV,
+  SERVICE_PROXY_MODE_ENV,
+  SERVICE_SCHEME_ENV
+)
+
+class ServiceConfig(Config):
+
+  def __init__(self, dir: str, **kwargs):
+    super().__init__(dir)
+
+    self.__detached = None
+    self.__docker_compose_path = None
+    self.__hostname = None
+    self.__dns = None
+    self.__port = None
+    self.__priority = None
+    self.__proxy_mode = None
+    self.__scheme = None
+
+    self.load()
+
+    if 'detached' in kwargs:
+      self.__detached = bool(kwargs.get('detached'))
+    
+    if 'hostname' in kwargs:
+      self.__hostname = kwargs.get('hostname')
+
+    if 'port' in kwargs:
+      self.__port = kwargs.get('port')
+
+    if 'priority' in kwargs:
+      self.__priority = kwargs.get('priority')
+
+    if 'proxy_mode' in kwargs:
+      self.__proxy_mode = kwargs.get('proxy_mode')
+
+    if 'scheme' in kwargs:
+      self.__scheme = kwargs.get('scheme')
+
+  @property
+  def detached(self):
+    return self.__detached
+
+  @detached.setter
+  def detached(self, v):
+    self.__detached = v
+
+  @property
+  def docker_compose_path(self):
+    return self.__docker_compose_path
+
+  @docker_compose_path.setter
+  def docker_compose_path(self, v):
+    self.__docker_compose_path = v
+
+  @property
+  def hostname(self):
+    return self.__hostname
+
+  @hostname.setter
+  def hostname(self, v):
+    self.__hostname = v
+
+  @property
+  def dns(self):
+    # If hostname is set then the service is external and we will need to configure the container's DNS.
+    # If we don't configure the container's DNS, then Docker's embedded DNS will potentially
+    # use configuration from the host's /etc/hosts file. The user may have configured their
+    # /etc/hosts file to resolve requests to localhost
+    #
+    # See: 
+    # https://forums.docker.com/t/docker-127-0-0-11-resolver-should-use-host-etc-hosts-file/55157
+    # https://docs.docker.com/network/#dns-services
+    # 
+    # TODO: ideally we want to know if the service is built locally, if so, then no need to set DNS
+    # since Docker's embedded DNS will resolve to it
+    if self.hostname and not self.__dns:
+      nameservers = self.__find_dns()
+      self.__dns = nameservers[0] if nameservers else None
+    return self.__dns
+
+  @dns.setter
+  def dns(self, v):
+    self.__dns = v
+
+  @property
+  def port(self):
+    return self.__port
+
+  @port.setter
+  def port(self, v):
+    self.__port = v
+
+  @property
+  def priority(self):
+    _priority = self.__priority
+    _priority = float('inf') if _priority == None else float(_priority)
+
+    if _priority == float('inf'):
+      return _priority
+
+    return float(_priority)
+
+  @priority.setter
+  def priority(self, v):
+    self.__priority = v
+
+  @property
+  def proxy_mode(self):
+    if self.__proxy_mode:
+      return self.__proxy_mode
+
+    return f"reverse:{self.scheme}://{self.hostname}"
+
+  @proxy_mode.setter
+  def proxy_mode(self, v):
+    self.__proxy_mode = v
+
+  @property
+  def scheme(self):
+    return self.__scheme or 'https'
+
+  @scheme.setter
+  def scheme(self, v):
+    self.__scheme = v
+
+  def load(self, config = None):
+    config = config or self.read()
+
+    # Do not load dns from config, have it dynamically determined
+    #self.dns = config.get(SERVICE_DNS_ENV)
+
+    self.detached = config.get(SERVICE_DETACHED_ENV)
+    self.docker_compose_path = config.get(SERVICE_DOCKER_COMPOSE_PATH_ENV)
+    self.hostname = config.get(SERVICE_HOSTNAME_ENV)
+    self.port = config.get(SERVICE_PORT_ENV)
+    self.priority = config.get(SERVICE_PRIORITY_ENV)
+    self.proxy_mode = config.get(SERVICE_PROXY_MODE_ENV)
+    self.scheme = config.get(SERVICE_SCHEME_ENV)
+
+  def write(self):
+    config = {}
+
+    if self.docker_compose_path:
+      config[SERVICE_DOCKER_COMPOSE_PATH_ENV] = self.docker_compose_path
+
+    if self.hostname:
+      config[SERVICE_HOSTNAME_ENV] = self.hostname
+
+    if self.dns:
+      config[SERVICE_DNS_ENV] = self.dns
+    
+    if self.port:
+      config[SERVICE_PORT_ENV] = self.port
+
+    if self.priority:
+      config[SERVICE_PRIORITY_ENV] = self.priority
+
+    if self.scheme:
+      config[SERVICE_SCHEME_ENV] = self.scheme
+
+    config[SERVICE_DETACHED_ENV] = bool(self.detached)
+
+    config[SERVICE_PROXY_MODE_ENV] = self.proxy_mode
+
+    super().write(config)
+
+  def __find_dns(self):
+    dns_resolver = dns.resolver.Resolver()
+    nameservers = dns_resolver.nameservers
+
+    # If systemd-resolved is not used
+    if nameservers != ['127.0.0.53']:
+      return nameservers
+
+    # Run the `resolvectl status` command and capture its output
+    result = subprocess.run(['resolvectl', 'status'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+
+    # Check if the command ran successfully
+    if result.returncode != 0:
+      return []
+
+    # Extract the DNS servers using a regular expression
+    #dns_servers = re.findall(r'DNS Servers: ([\d.]+(?:, [\d.]+)*)', result.stdout)
+    pattern = re.compile('DNS Servers:(.*?)DNS Domain', re.DOTALL)
+    match = re.findall(pattern, result.stdout)
+
+    if not match:
+      return []
+      
+    # Split the DNS servers string into a list
+    dns_servers = match[0].strip().split("\n")
+    return list(map(lambda dns_server: dns_server.strip(), dns_servers))

stoobly_agent/app/cli/scaffold/service_create_command.py

@@ -0,0 +1,77 @@
+import os
+import pdb
+import shutil
+
+from .app import App
+from .constants import WORKFLOW_MOCK_TYPE, WORKFLOW_RECORD_TYPE, WORKFLOW_TEST_TYPE
+from .docker.service.builder import ServiceBuilder
+from .docker.workflow.decorators_factory import get_workflow_decorators
+from .service_command import ServiceCommand
+from .workflow_create_command import WorkflowCreateCommand
+
+class ServiceCreateCommand(ServiceCommand):
+
+  def __init__(self, app: App, **kwargs):
+    super().__init__(app, **kwargs)
+
+    self.__env_vars = kwargs.get('env') or []
+    self.__workflows = kwargs.get('workflow') or []
+
+  @property
+  def env_vars(self):
+    return self.__env_vars
+
+  @property
+  def workflows(self):
+    return self.__workflows
+
+  def build(self):
+    service_builder = ServiceBuilder(self.service_config)
+    service_decorators = []
+
+    for service_decorator in service_decorators:
+      service_decorator(service_builder).decorate()
+
+    service_builder.write()
+
+    workflow_kwargs = {
+      'app_dir_path': self.app_dir_path,
+      'env': self.env_vars,
+      'namespace': self.namespace, 
+      'service_name': self.service_name,
+    }
+
+    if WORKFLOW_MOCK_TYPE in self.workflows:
+      self.__build_with_mock_workflow(service_builder, **workflow_kwargs)
+
+    if WORKFLOW_RECORD_TYPE in self.workflows:
+      self.__build_with_record_workflow(service_builder, **workflow_kwargs)
+
+    if WORKFLOW_TEST_TYPE in self.workflows:
+      self.__build_with_test_workflow(service_builder, **workflow_kwargs)
+
+    self.service_config.write()
+
+  def reset(self):
+    dest = self.service_path
+
+    if os.path.exists(dest):
+      shutil.rmtree(dest)
+
+  def __build_with_mock_workflow(self, service_builder: ServiceBuilder, **kwargs):
+    mock_workflow = WorkflowCreateCommand(self.app, **{ **kwargs, **{ 'workflow_name': WORKFLOW_MOCK_TYPE }})
+
+    workflow_decorators = get_workflow_decorators(WORKFLOW_MOCK_TYPE, self.service_config)
+    mock_workflow.build(service_builder=service_builder, workflow_decorators=workflow_decorators)
+
+  def __build_with_record_workflow(self, service_builder: ServiceBuilder, **kwargs):
+    record_workflow = WorkflowCreateCommand(self.app, **{ **kwargs, **{ 'workflow_name': WORKFLOW_RECORD_TYPE }})
+
+    workflow_decorators = get_workflow_decorators(WORKFLOW_RECORD_TYPE, self.service_config)
+    record_workflow.build(service_builder=service_builder, workflow_decorators=workflow_decorators)
+
+  def __build_with_test_workflow(self, service_builder: ServiceBuilder, **kwargs):
+    mock_workflow = WorkflowCreateCommand(self.app, **{ **kwargs, **{ 'workflow_name': WORKFLOW_TEST_TYPE }})
+
+    workflow_decorators = get_workflow_decorators(WORKFLOW_TEST_TYPE, self.service_config)
+    mock_workflow.build(service_builder=service_builder, workflow_decorators=workflow_decorators)

stoobly_agent/app/cli/scaffold/service_workflow.py

@@ -0,0 +1,18 @@
+import os
+
+from .app import App
+from .workflow import Workflow
+
+class ServiceWorkflow(Workflow):
+
+  def __init__(self, service_name: str, workflow_name: str, app: App):
+    super().__init__(workflow_name, app)
+    self._service_name = service_name
+
+  @property
+  def path(self):
+    return os.path.join(self.app.scaffold_namespace_path, self.service_name, self.workflow_name)
+
+  @property
+  def service_name(self):
+    return self._service_name

stoobly_agent/app/cli/scaffold/templates/app/.Dockerfile.context

@@ -0,0 +1,8 @@
+FROM stoobly/agent:0.34
+
+ARG USER_ID
+
+# Change user id of stoobly user to that of host's user id
+RUN if [ -n "$USER_ID" ]; then usermod -u $USER_ID stoobly; fi
+
+COPY --chown=stoobly:stoobly . .

stoobly_agent/app/cli/scaffold/templates/app/.Dockerfile.proxy

@@ -0,0 +1,35 @@
+FROM ubuntu:23.10 AS build
+
+WORKDIR /tmp
+
+RUN apt-get update && apt-get install -y curl
+RUN curl -JLO "https://dl.filippo.io/mkcert/v1.4.4?for=linux/amd64" && \
+  chmod +x mkcert-v1.4.4-linux-amd64
+
+FROM ubuntu:23.10 AS cert
+
+ARG SERVICE_HOSTNAME
+
+COPY --from=build /tmp/mkcert-v1.4.4-linux-amd64 /bin/mkcert
+
+RUN useradd -m mkcert
+
+USER mkcert
+WORKDIR /home/mkcert
+
+RUN mkcert $SERVICE_HOSTNAME && mkcert -pkcs12 $SERVICE_HOSTNAME && cat $SERVICE_HOSTNAME.pem >> $SERVICE_HOSTNAME-joined.pem && cat $SERVICE_HOSTNAME-key.pem >> $SERVICE_HOSTNAME-joined.pem
+
+FROM stoobly/agent:0.34
+
+ARG USER_ID
+ARG SERVICE_HOSTNAME
+
+# Allow running stoobly on port 443
+RUN apt-get update && apt-get install -y curl libcap2-bin
+RUN setcap 'cap_net_bind_service=+ep' /usr/local/bin/python3.8
+
+# Change user id of stoobly user to that of host's user id
+RUN if [ -n "$USER_ID" ]; then groupmod -g $USER_ID stoobly; usermod -u $USER_ID stoobly; fi
+
+COPY --chown=stoobly:stoobly . .
+COPY --chown=stoobly:stoobly --from=cert /home/mkcert/$SERVICE_HOSTNAME-joined.pem .

stoobly_agent/app/cli/scaffold/templates/app/.Makefile

@@ -0,0 +1,118 @@
+# Overridable Environment Variables
+#
+# STOOBLY_APP_DIR: path to the application source code directory
+# STOOBLY_CONTEXT_DIR: path to the folder containing the .stoobly folder
+# STOOBLY_CERTS_DIR: path to a folder to store certs
+# STOOBLY_WORKFLOW_RUN_OPTIONS: extra options to pass to 'stoobly-agent scaffold workflow run' command
+
+# Constants
+DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
+WORKFLOW_NAME := exec
+
+CONTEXT_DIR_DEFAULT := $(realpath $(DIR)/../..)
+
+# Configuration
+app_dir=$$(realpath "$${STOOBLY_APP_DIR:-$(CONTEXT_DIR_DEFAULT)}")
+certs_dir=$$(realpath "$${STOOBLY_CERTS_DIR:-$(app_data_dir)/certs}")
+context_dir=$$(realpath "$${STOOBLY_CONTEXT_DIR:-$(CONTEXT_DIR_DEFAULT)}")
+workflow_run_options=$${STOOBLY_WORKFLOW_RUN_OPTIONS:+$$STOOBLY_WORKFLOW_RUN_OPTIONS }
+
+app_data_dir=$(app_dir)/.stoobly
+data_dir=$(context_dir)/.stoobly
+
+# Commands
+docker_compose_command=docker compose
+source_env=set -a; [ -f .env ] && source .env; set +a
+
+docker_compose_file_path=$(app_data_dir)/docker/stoobly-ui/exec/.docker-compose.exec.yml
+stoobly_exec_args=--profile $(WORKFLOW_NAME) -p $(WORKFLOW_NAME) up --build --remove-orphans
+stoobly_exec_env=export CONTEXT_DIR=$(context_dir) && export USER_ID=$$UID
+stoobly_exec=$(stoobly_exec_env) && $(source_env) && $(docker_compose_command) -f "$(docker_compose_file_path)" $(stoobly_exec_args)
+
+# Because scaffold is stored in the APP_DIR, when running a scaffold command from within a container,
+# it needs access to APP_DIR rather than CONTEXT_DIR
+stoobly_exec_run_env=export USER_ID=$$UID
+stoobly_exec_run=$(stoobly_exec_run_env) && $(source_env) && CONTEXT_DIR=$(app_dir) $(docker_compose_command) -f "$(docker_compose_file_path)" $(stoobly_exec_args)
+
+run_script=$(app_data_dir)/tmp/run.sh
+run_env=export APP_DIR="$(app_dir)" && export CERTS_DIR="$(certs_dir)" && export CONTEXT_DIR="$(context_dir)"
+workflow_run=$(run_env) && $(source_env) && bash "$(run_script)"
+
+cert:
+	if [ -z "$$(which mkcert)" ]; then \
+		echo "Error: missing mkcert command"; \
+	else \
+		mkdir -p "$(certs_dir)" && \
+		cd "$(certs_dir)" && \
+		mkcert --install && \
+		mkcert $(hostname) && \
+		cp $(hostname).pem $(hostname).crt && \
+		cp $(hostname)-key.pem $(hostname).key; \
+	fi
+intercept/disable:
+	export EXEC_COMMAND=bin/.disable && \
+	$(stoobly_exec)
+intercept/enable:
+	export EXEC_COMMAND=bin/.enable && \
+	export EXEC_ARGS=$(scenario_key) && \
+	$(stoobly_exec)
+mock:
+	export EXEC_COMMAND=bin/.run && \
+	export EXEC_OPTIONS="$(workflow_run_options)$(options)" && \
+	export EXEC_ARGS="mock" && \
+	$(stoobly_exec_run) && \
+	$(workflow_run)
+mock/stop:
+	export EXEC_COMMAND=bin/.stop && \
+	export EXEC_OPTIONS="$(options)" && \
+	export EXEC_ARGS="mock" && \
+	$(stoobly_exec_run) && \
+	$(workflow_run)
+record:
+	export EXEC_COMMAND=bin/.run && \
+	export EXEC_OPTIONS="$(workflow_run_options)$(options)" && \
+	export EXEC_ARGS="record" && \
+	$(stoobly_exec_run) && \
+	$(workflow_run)
+record/stop:
+	export EXEC_COMMAND=bin/.stop && \
+	export EXEC_OPTIONS="$(options)" && \
+	export EXEC_ARGS="record" && \
+	$(stoobly_exec_run) && \
+	$(workflow_run)
+scenario/create:
+# Create a scenario
+	export EXEC_COMMAND=bin/.create && \
+	export EXEC_OPTIONS="$(options)" && \
+	export EXEC_ARGS="$(name)" && \
+	$(stoobly_exec)
+scenario/delete:
+# Delete a scenario
+	export EXEC_COMMAND=bin/.delete && \
+	export EXEC_OPTIONS="$(options)" && \
+	export EXEC_ARGS="$(key)" && \
+	$(stoobly_exec)
+scenario/reset:
+# Resets a scenario to its last snapshot
+	export EXEC_COMMAND=bin/.reset && \:
+	export EXEC_OPTIONS="$(options)" && \
+	export EXEC_ARGS="$(key)" && \
+	$(stoobly_exec)
+scenario/snapshot:
+# Create committable files for a scenario
+	export EXEC_COMMAND=bin/.snapshot && \
+	export EXEC_OPTIONS="$(options)" && \
+	export EXEC_ARGS="$(key)" && \
+	$(stoobly_exec)
+test:
+	export EXEC_COMMAND=bin/.run && \
+	export EXEC_OPTIONS="$(workflow_run_options)$(options)" && \
+	export EXEC_ARGS="test" && \
+	$(stoobly_exec_run) && \
+	$(workflow_run)
+test/stop:
+	export EXEC_COMMAND=bin/.stop && \
+	export EXEC_OPTIONS="$(options)" && \
+	export EXEC_ARGS="test" && \
+	$(stoobly_exec_run) && \
+	$(workflow_run)

stoobly_agent/app/cli/scaffold/templates/app/.docker-compose.base.yml

@@ -0,0 +1,15 @@
+services:
+  context_base:
+    build:
+      args:
+        USER_ID: ${USER_ID}
+      dockerfile: ./.Dockerfile.context
+    volumes:
+      - ${CONTEXT_DIR}/.stoobly:/home/stoobly/.stoobly   
+  proxy_base:
+    build:
+      args:
+        USER_ID: ${USER_ID}
+      dockerfile: ./.Dockerfile.proxy
+    volumes:
+      - ${CONTEXT_DIR}/.stoobly:/home/stoobly/.stoobly

stoobly_agent/app/cli/scaffold/templates/app/Makefile

@@ -0,0 +1,3 @@
+DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
+
+include $(DIR)/.Makefile

stoobly_agent/app/cli/scaffold/templates/app/build/.config.yml

@@ -0,0 +1 @@
+SERVICE_PRIORITY: 0

stoobly_agent/app/cli/scaffold/templates/app/build/.docker-compose.base.yml

@@ -0,0 +1,11 @@
+services:
+  build.init_base:
+    command: ['bin/.init']
+    extends:
+      file: ../.docker-compose.base.yml
+      service: proxy_base
+  build.configure_base:
+    command: ['bin/.configure']
+    extends:
+      file: ../.docker-compose.base.yml
+      service: proxy_base

stoobly_agent/app/cli/scaffold/templates/app/build/mock/.docker-compose.mock.yml

@@ -0,0 +1,22 @@
+services:
+  build.init:
+    build:
+      context: ./
+      dockerfile: ../../.Dockerfile.context
+    extends:
+      file: ../.docker-compose.base.yml
+      service: build.init_base
+    profiles:
+      - mock
+  build.configure:
+    build:
+      context: ./
+      dockerfile: ../../.Dockerfile.context
+    depends_on:
+      build.init:
+        condition: service_completed_successfully
+    extends:
+      file: ../.docker-compose.base.yml
+      service: build.configure_base
+    profiles:
+      - mock

stoobly_agent/app/cli/scaffold/templates/app/build/mock/bin/.configure

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+stoobly-agent config reset
+
+cd $(dirname -- "$0") && ./configure

stoobly_agent/app/cli/scaffold/templates/app/build/mock/bin/.init

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+stoobly-agent snapshot apply
+
+cd $(dirname -- "$0") && ./init

stoobly_agent/app/cli/scaffold/templates/app/build/mock/bin/configure

@@ -0,0 +1 @@
+#!/bin/bash

stoobly_agent/app/cli/scaffold/templates/app/build/mock/bin/init

@@ -0,0 +1 @@
+#! /bin/bash

stoobly_agent/app/cli/scaffold/templates/app/build/record/.docker-compose.record.yml

@@ -0,0 +1,22 @@
+services:
+  build.init:
+    build:
+      context: ./
+      dockerfile: ../../.Dockerfile.context
+    extends:
+      file: ../.docker-compose.base.yml
+      service: build.init_base
+    profiles:
+      - record
+  build.configure:
+    build:
+      context: ./
+      dockerfile: ../../.Dockerfile.context
+    depends_on:
+      build.init:
+        condition: service_completed_successfully
+    extends:
+      file: ../.docker-compose.base.yml
+      service: build.configure_base
+    profiles:
+      - record

stoobly_agent/app/cli/scaffold/templates/app/build/record/bin/.configure

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+stoobly-agent config reset
+
+cd $(dirname -- "$0") && ./configure

stoobly_agent/app/cli/scaffold/templates/app/build/record/bin/.init

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+stoobly-agent snapshot apply
+
+cd $(dirname -- "$0") && ./init

stoobly_agent/app/cli/scaffold/templates/app/build/record/bin/configure

@@ -0,0 +1 @@
+#!/bin/bash

stoobly_agent/app/cli/scaffold/templates/app/build/record/bin/init

@@ -0,0 +1 @@
+#! /bin/bash

stoobly_agent/app/cli/scaffold/templates/app/build/test/.docker-compose.test.yml

@@ -0,0 +1,22 @@
+services:
+  build.init:
+    build:
+      context: ./
+      dockerfile: ../../.Dockerfile.context
+    extends:
+      file: ../.docker-compose.base.yml
+      service: build.init_base
+    profiles:
+      - test
+  build.configure:
+    build:
+      context: ./
+      dockerfile: ../../.Dockerfile.context
+    depends_on:
+      build.init:
+        condition: service_completed_successfully
+    extends:
+      file: ../.docker-compose.base.yml
+      service: build.configure_base
+    profiles:
+      - test

stoobly_agent/app/cli/scaffold/templates/app/build/test/bin/.configure

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+stoobly-agent config reset
+
+cd $(dirname -- "$0") && ./configure

stoobly_agent/app/cli/scaffold/templates/app/build/test/bin/.init

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+stoobly-agent snapshot apply
+
+cd $(dirname -- "$0") && ./init

stoobly_agent/app/cli/scaffold/templates/app/build/test/bin/configure

@@ -0,0 +1 @@
+#!/bin/bash

stoobly_agent/app/cli/scaffold/templates/app/build/test/bin/init

@@ -0,0 +1 @@
+#! /bin/bash

stoobly_agent/app/cli/scaffold/templates/app/entrypoint/.config.yml

@@ -0,0 +1 @@
+SERVICE_PRIORITY: 10.0

stoobly_agent/app/cli/scaffold/templates/app/entrypoint/.docker-compose.base.yml

@@ -0,0 +1,17 @@
+networks: {}
+services:
+  entrypoint.configure_base:
+    command:
+    - bin/.configure
+    environment: {}
+    extends:
+      file: ../.docker-compose.base.yml
+      service: context_base
+  entrypoint.init_base:
+    command:
+    - bin/.init
+    - dist
+    environment: {}
+    extends:
+      file: ../.docker-compose.base.yml
+      service: context_base