stoobly-agent 0.34.13__py3-none-any.whl → 1.0.0__py3-none-any.whl

stoobly_agent/app/cli/scaffold/docker/service/builder.py

@@ -0,0 +1,117 @@
+import os
+import pdb
+
+from ...app_config import AppConfig
+from ...constants import SERVICE_HOSTNAME, SERVICE_HOSTNAME_ENV, SERVICE_PORT, SERVICE_PORT_ENV, SERVICE_SCHEME, SERVICE_SCHEME_ENV
+from ...service_config import ServiceConfig
+from ..app_builder import AppBuilder
+from ..builder import Builder
+from ..constants import DOCKER_COMPOSE_BASE
+
+class ServiceBuilder(Builder):
+
+  def __init__(self, config: ServiceConfig, app_builder: AppBuilder = None):
+    service_path = config.dir
+    super().__init__(service_path, DOCKER_COMPOSE_BASE)
+
+    if not app_builder:
+      app_dir = os.path.dirname(service_path)
+      app_builder = AppBuilder(AppConfig(app_dir))
+    self.app_builder = app_builder
+
+    self.__config = config
+    self.__service_name = os.path.basename(service_path)
+
+    self.load()
+
+  @property
+  def app_base(self):
+    return f"{self.service_name}.app_base"
+
+  @property
+  def init_base(self):
+    return f"{self.service_name}.init_base"
+
+  @property
+  def config(self):
+    return self.__config
+
+  @property
+  def configure_base(self):
+    return f"{self.service_name}.configure_base"
+
+  @property
+  def configure_base_service(self):
+    return self.services.get(self.configure_base)
+
+  @property
+  def proxy_base(self):
+    return f"{self.service_name}.proxy_base"
+
+  @property
+  def proxy_base_service(self):
+    return self.services.get(self.proxy_base)
+
+  @property
+  def service_name(self):
+    return self.__service_name
+
+  def build_extends_init_base(self, source_dir: str):
+    return self.build_extends(self.init_base, source_dir)
+
+  def build_extends_configure_base(self, source_dir: str):
+    return self.build_extends(self.configure_base, source_dir)
+
+  def build_extends_proxy_base(self, source_dir: str):
+    return self.build_extends(self.proxy_base, source_dir)
+
+  def build_proxy_base(self):
+    if not self.config.hostname:
+      return
+
+    args = {}
+    self.with_service(self.proxy_base, {
+      'build': {
+        'args': args,
+      },
+      'extends': {
+        'file': os.path.relpath(self.app_builder.compose_file_path, self.dir_path),
+        'service': self.app_builder.proxy_base
+      }
+    })
+
+    args[SERVICE_HOSTNAME_ENV] = f"{SERVICE_HOSTNAME}"
+
+  def build_init_base(self):
+    environment = {}
+    self.with_service(self.init_base, {
+      'command': ['bin/.init', 'dist'],
+      'environment': environment,
+      'extends': {
+        'file': os.path.relpath(self.app_builder.compose_file_path, self.dir_path),
+        'service': self.app_builder.context_base
+      }
+    })
+
+  def build_configure_base(self):
+    environment = {}
+    self.with_service(self.configure_base, {
+      'command': ['bin/.configure'],
+      'environment': environment,
+      'extends': {
+        'file': os.path.relpath(self.app_builder.compose_file_path, self.dir_path),
+        'service': self.app_builder.context_base
+      }
+    })
+
+  def write(self):
+    self.build_init_base()
+    self.build_configure_base()
+
+    if self.config.hostname:
+      self.build_proxy_base()
+
+    super().write({
+      'networks': self.networks,
+      'services': self.services,
+    })

stoobly_agent/app/cli/scaffold/docker/service/set_gateway_ports.py

@@ -0,0 +1,47 @@
+import os
+import pdb
+import yaml
+
+from typing import List
+
+from stoobly_agent.app.cli.scaffold.service_config import ServiceConfig
+from stoobly_agent.app.cli.scaffold.docker.constants import DOCKER_COMPOSE_BASE
+from stoobly_agent.app.cli.scaffold.templates.constants import CORE_GATEWAY_SERVICE_NAME
+
+def set_gateway_ports(service_paths: List[str]):
+  if len(service_paths) == 0:
+    return
+
+  ports = []
+  for path in service_paths:
+    config = ServiceConfig(path)
+
+    try:
+      port = int(config.port)
+    except Exception:
+      continue
+
+    if port > 0 and port <= 65535 and port not in ports:
+      ports.append(port)
+
+  app_dir_path = os.path.dirname(service_paths[0])
+  gateway_service_path = os.path.join(app_dir_path, CORE_GATEWAY_SERVICE_NAME)
+  docker_compose_path = os.path.join(gateway_service_path, DOCKER_COMPOSE_BASE)
+
+  with open(docker_compose_path, 'r+') as fp:
+    compose = yaml.safe_load(fp)
+    services = compose.get('services')
+
+    if not services:
+      return
+    
+    gateway_base = services.get('gateway_base')
+
+    if not gateway_base:
+      return
+
+    gateway_base['ports'] = list(map(lambda port: f"{port}:{port}", ports))
+
+    fp.seek(0)
+    yaml.dump(compose, fp)
+    fp.truncate()

stoobly_agent/app/cli/scaffold/docker/service/types.py

@@ -0,0 +1,4 @@
+from typing import TypedDict
+
+class BuildDecoratorOptions(TypedDict):
+  build_args: list

stoobly_agent/app/cli/scaffold/docker/workflow/build_decorator.py

@@ -0,0 +1,28 @@
+import pdb
+
+from .builder import WorkflowBuilder
+
+class BuildDecorator():
+
+  def __init__(self, workflow_builder: WorkflowBuilder):
+    self.__workflow_builder = workflow_builder
+
+  @property
+  def workflow_builder(self):
+    return self.__workflow_builder
+
+  def decorate(self):
+    service_builder = self.__workflow_builder.service_builder
+
+    services = self.workflow_builder.services
+    app_name = self.workflow_builder.app
+    app_service = services.get(app_name) or {}
+
+    services[app_name] = { 
+      **app_service,
+      **{ 
+          'extends': service_builder.build_extends_proxy_base(self.workflow_builder.dir_path),
+          'hostname': service_builder.config.hostname,
+          'profiles': self.workflow_builder.profiles,
+        } 
+    }

stoobly_agent/app/cli/scaffold/docker/workflow/builder.py

@@ -0,0 +1,259 @@
+import os
+import pdb
+
+from typing import List
+
+from ...constants import (
+  COMPOSE_TEMPLATE, SERVICE_HOSTNAME, SERVICE_HOSTNAME_ENV, SERVICE_NAME_ENV, SERVICE_PORT, SERVICE_PORT_ENV, SERVICE_SCHEME, 
+  SERVICE_SCHEME_ENV, STOOBLY_HOME_DIR, WORKFLOW_NAME_ENV
+)
+from ..builder import Builder
+from ..service.builder import ServiceBuilder
+from ...templates.constants import SERVICE_HOSTNAME_BUILD_ARG
+
+class WorkflowBuilder(Builder):
+
+  def __init__(self, workflow_path: str, service_builder: ServiceBuilder):
+    self._env = [SERVICE_NAME_ENV, WORKFLOW_NAME_ENV]
+    self.__workflow_name = os.path.basename(workflow_path)
+    super().__init__(workflow_path, COMPOSE_TEMPLATE.format(workflow=self.__workflow_name))
+
+    self.__context = '../'
+    self.__profiles = [self.__workflow_name]
+    self.__workdir = os.path.join(STOOBLY_HOME_DIR, self.workflow_name)
+
+    if not service_builder:
+      service_path = os.path.dirname(workflow_path)
+      service_builder = ServiceBuilder(service_path)
+
+    self.__service_builder = service_builder
+
+    if self.config.hostname:
+      self.with_public_network()
+
+    self.load()
+  
+  @property
+  def app(self):
+    return f"{self.namespace}.app"
+
+  @property
+  def base_compose_file_path(self):
+    return os.path.relpath(self.service_builder.compose_file_path, self.dir_path)
+
+  @property
+  def init(self):
+    return f"{self.namespace}.init"
+
+  @property
+  def config(self):
+    return self.service_builder.config
+
+  @property
+  def configure(self):
+    return f"{self.namespace}.configure"
+
+  @property
+  def context(self):
+    return self.__context
+
+  @property
+  def context_build(self):
+    return {
+      'context': self.context,
+      'dockerfile': self.context_docker_file_path,
+    }
+
+  @property
+  def context_docker_file_path(self):
+    return os.path.relpath(self.service_builder.app_builder.context_docker_file_path, self.service_path)
+
+  @property
+  def namespace(self):
+    return f"{self.service_builder.service_name}"
+
+  @property
+  def profiles(self):
+    return self.__profiles
+
+  @property
+  def proxy(self):
+    return f"{self.namespace}.proxy"
+
+  @property
+  def proxy_build(self):
+    args = {}
+    args[SERVICE_HOSTNAME_BUILD_ARG] = SERVICE_HOSTNAME
+
+    return {
+      'args': args,
+      'context': self.context,
+      'dockerfile': self.proxy_docker_file_path,
+    }
+
+  @property
+  def proxy_docker_file_path(self):
+    return os.path.relpath(self.service_builder.app_builder.proxy_docker_file_path, self.service_path)
+
+  @property
+  def service_builder(self):
+    return self.__service_builder
+
+  @property
+  def service_path(self):
+    return self.__service_builder.dir_path
+
+  @property
+  def workflow_name(self):
+    return self.__workflow_name
+
+  def build_all(self):
+    # Resources
+    self.with_network(self.service_builder.service_name)
+    
+    if self.config.detached:
+      self.with_volume(self.service_builder.service_name)
+
+    # Services
+    self.build_init()
+    self.build_configure()
+
+    if self.config.hostname:
+      self.with_public_network()
+      self.build_proxy() # Depends on configure, must call build_configure first
+
+  def build_init(self):
+    environment = { **self.env_dict() }
+    volumes = []
+
+    service = {
+      'build': self.context_build,
+      'environment': environment,
+      'extends': self.service_builder.build_extends_init_base(self.dir_path),
+      'profiles': self.profiles,
+      'volumes': volumes,
+      'working_dir': self.__workdir,
+    }
+
+    if self.config.hostname:
+      self.__with_url_environment(environment)
+
+    if self.config.detached:
+      volumes.append(f"{self.service_builder.service_name}:{STOOBLY_HOME_DIR}/.stoobly")
+
+    self.with_service(self.init, service)
+
+  def build_configure(self):
+    # If the configure_base service does not exist, we can't extend from it, return
+    if not self.service_builder.configure_base_service:
+      return
+
+    depends_on = {}
+    environment = { **self.env_dict() }
+    volumes = []
+
+    service = {
+      'build': self.context_build,
+      'depends_on': depends_on,
+      'environment': environment,
+      'extends': self.service_builder.build_extends_configure_base(self.dir_path),
+      'profiles': self.profiles,
+      'volumes': volumes,
+      'working_dir': self.__workdir,
+    }
+
+    if self.config.hostname:
+      self.__with_url_environment(environment)
+
+    if self.init in self.services:
+      depends_on[self.init] = {
+        'condition': 'service_completed_successfully',
+      }
+
+    if self.config.detached:
+      volumes.append(f"{self.service_builder.service_name}:{STOOBLY_HOME_DIR}/.stoobly")
+
+    self.with_service(self.configure, service)
+
+  def build_proxy(self):
+    # If the proxy_base service does not exist, we can't extend from it, return
+    if not self.service_builder.proxy_base_service:
+      return
+
+    depends_on = {}
+    environment = { **self.env_dict() }
+    extra_hosts = []
+    networks = [self.service_builder.service_name]
+    volumes = []
+
+    service = {
+      'build': self.proxy_build, 
+      'depends_on': depends_on,
+      'environment': environment,
+      'extra_hosts': extra_hosts,
+      'extends': self.service_builder.build_extends_proxy_base(self.dir_path),
+      'networks': networks,
+      'profiles': self.profiles,
+      'volumes': volumes,
+      'working_dir': self.__workdir,
+    }
+
+    if self.configure in self.services:
+      depends_on[self.init] = {
+        'condition': 'service_completed_successfully',
+      }
+
+      depends_on[self.configure] = {
+        'condition': 'service_completed_successfully',
+      }
+
+    if self.config.hostname:
+      environment['VIRTUAL_HOST'] = SERVICE_HOSTNAME
+      environment['VIRTUAL_PORT'] = SERVICE_PORT
+      environment['VIRTUAL_PROTO'] = SERVICE_SCHEME
+
+      # Expose this container service to the public network 
+      # so that it is accessible to other Stoobly services
+      networks.append(self.public_network_name)
+
+    if self.config.detached:
+      volumes.append(f"{self.service_builder.service_name}:{STOOBLY_HOME_DIR}/.stoobly")
+
+    self.with_service(self.proxy, service)
+
+  def env_dict(self):
+    env = {}
+    for e in self._env:
+      env[e] = '${' + e + '}'
+    return env
+
+  def initialize_custom_file(self):
+    dest = self.custom_compose_file_path
+
+    if not os.path.exists(dest):
+      super().write({
+        'networks': self.networks,
+        'services': {}
+      }, dest)
+
+  def with_env(self, v: List[str]): 
+    if not isinstance(v, list):
+      return self
+    self._env += v
+    return self
+
+  def write(self):
+    super().write({
+      'networks': self.networks,
+      'services': self.services,
+      'volumes': self.volumes,
+    })
+
+  def __with_url_environment(self, environment):
+    environment[SERVICE_HOSTNAME_ENV] = SERVICE_HOSTNAME
+
+    if self.config.scheme:
+      environment[SERVICE_SCHEME_ENV] = SERVICE_SCHEME
+
+    if self.config.port:
+      environment[SERVICE_PORT_ENV] = SERVICE_PORT

stoobly_agent/app/cli/scaffold/docker/workflow/decorators_factory.py

@@ -0,0 +1,17 @@
+from stoobly_agent.app.cli.scaffold.service_config import ServiceConfig
+
+from ...constants import WORKFLOW_MOCK_TYPE, WORKFLOW_RECORD_TYPE
+from .mock_decorator import MockDecorator
+from .reverse_proxy_decorator import ReverseProxyDecorator
+
+def get_workflow_decorators(workflow: str, service_config: ServiceConfig):
+  workflow_decorators = []
+
+  if workflow == WORKFLOW_RECORD_TYPE:
+    if service_config.hostname:
+      workflow_decorators.append(ReverseProxyDecorator)
+  else:
+    if service_config.hostname:
+      workflow_decorators.append(ReverseProxyDecorator if service_config.detached else MockDecorator) 
+
+  return workflow_decorators

stoobly_agent/app/cli/scaffold/docker/workflow/mock_decorator.py

@@ -0,0 +1,40 @@
+import pdb
+
+from ...constants import SERVICE_HOSTNAME, SERVICE_PORT, SERVICE_SCHEME
+from .builder import WorkflowBuilder
+
+class MockDecorator():
+
+  def __init__(self, workflow_builder: WorkflowBuilder):
+    self.__workflow_builder = workflow_builder
+
+  @property
+  def workflow_builder(self):
+    return self.__workflow_builder
+
+  @property
+  def service_builder(self):
+    return self.workflow_builder.service_builder
+
+  def decorate(self):
+    config = self.service_builder.config
+
+    command = [
+      '--certs', f"../{SERVICE_HOSTNAME}-joined.pem",
+      '--headless',
+      '--intercept',
+      '--lifecycle-hooks-path', 'lifecycle_hooks.py',
+      '--proxy-mode', config.proxy_mode,
+      '--proxy-port', f"{SERVICE_PORT}",
+      '--response-fixtures-path', 'fixtures.yml',
+      '--ssl-insecure'
+    ]
+
+    services = self.__workflow_builder.services
+    proxy_name = self.__workflow_builder.proxy
+    proxy_service = services.get(proxy_name) or {}
+
+    services[proxy_name] = { 
+      **proxy_service,
+      **{ 'command': command, 'hostname': f"{SERVICE_HOSTNAME}" },
+    }

stoobly_agent/app/cli/scaffold/docker/workflow/reverse_proxy_decorator.py

@@ -0,0 +1,51 @@
+import pdb
+
+from ...constants import SERVICE_DNS, SERVICE_HOSTNAME, SERVICE_PORT
+from .builder import WorkflowBuilder
+
+class ReverseProxyDecorator():
+
+  def __init__(self, workflow_builder: WorkflowBuilder):
+    self.__workflow_builder = workflow_builder
+
+  @property
+  def workflow_builder(self):
+    return self.__workflow_builder
+
+  @property
+  def service_builder(self):
+    return self.workflow_builder.service_builder
+
+  def decorate(self):
+    config = self.service_builder.config
+
+    command = [
+      '--certs', f"../{SERVICE_HOSTNAME}-joined.pem",
+      '--headless',
+      '--lifecycle-hooks-path', 'lifecycle_hooks.py',
+      '--proxy-mode', config.proxy_mode,
+      '--proxy-port', f"{SERVICE_PORT}",
+      '--ssl-insecure'
+    ]
+
+    services = self.workflow_builder.services
+    proxy_name = self.workflow_builder.proxy
+    proxy_service = services.get(proxy_name) or {}
+
+    # proxying forwards requests to the actual service
+    # If we set the 'hostname' property, this will cause an "infinite loop"
+
+    service = { 
+      **proxy_service,
+      **{ 'command': command },
+    }
+
+    # If we are reverse proxying to potentially an external host,
+    # Docker's embedded DNS will use the host's /etc/host file as part of the resolution process
+    # This can lead the hostname to resolve to localhost instead of the service's actual IP address
+    if config.dns and not config.detached:
+      service['dns'] = SERVICE_DNS
+
+    services[proxy_name] = service 
+
+

stoobly_agent/app/cli/scaffold/env.py

@@ -0,0 +1,49 @@
+import os
+
+from .constants import ENV_FILE
+
+DELIMITTER = "\n"
+
+class Env():
+
+  def __init__(self, path: str):
+    self.__path = path
+
+  @property
+  def path(self):
+    return self.__path
+
+  def read(self, source: str = None):
+    if not source:
+      source = self.path
+
+    env_vars = {}
+
+    if os.path.exists(source):
+      with open(source, 'r') as fp:
+        contents = fp.read()
+
+        rows = contents.split(DELIMITTER)
+        for row in rows:
+          el = row.split("=")
+          if len(el) != 2:
+            continue
+          env_vars[el[0]] = el[1]
+
+    return env_vars
+
+  # Priority: env_vars, source_env_vars, dest_env_vars
+  def merge(self, defaults_path: str, env_vars: dict = {}):
+    source_env_vars = self.read()
+    dest_env_vars = self.read(defaults_path)
+    dest_env_vars.update(source_env_vars)
+    dest_env_vars.update(env_vars)
+    return env_vars
+
+  def write(self, env_vars: dict):
+    env_file_path = os.path.join(self.path)
+    with open(env_file_path, 'w') as fp:
+      contents = []
+      for var in env_vars:
+          contents.append(f"{var}={env_vars[var]}")
+      fp.write(DELIMITTER.join(contents))

stoobly_agent/app/cli/scaffold/service.py

@@ -0,0 +1,25 @@
+import os
+
+from .app import App
+
+class Service():
+
+  def __init__(self, service_name: str, app: App):
+    self.__app = app
+    self.__service_name = service_name
+
+  @property
+  def app(self):
+    return self.__app
+
+  @property
+  def dir_path(self):
+    return os.path.join(self.app.namespace_path, self.service_name)
+
+  @property
+  def service_name(self):
+    return self.__service_name
+
+  @property
+  def workflow_dir_path(self, workflow_name: str):
+    return os.path.join(self.dir_path, workflow_name)

stoobly_agent/app/cli/scaffold/service_command.py

@@ -0,0 +1,50 @@
+import os
+import pdb
+
+from .app import App
+from .app_command import AppCommand
+from .service_config import ServiceConfig
+
+class ServiceCommand(AppCommand):
+
+  def __init__(self, app: App, **kwargs):
+    super().__init__(app)
+    self.__service_name = kwargs.get('service_name')
+
+    self.__config = ServiceConfig(self.service_path, **kwargs)
+
+  @property
+  def service_config(self):
+    return self.__config
+
+  @property
+  def service_config_path(self):
+    return self.__config.path
+
+  @property
+  def service_name(self):
+    return self.__service_name
+
+  @property
+  def service_exists(self):
+    return os.path.exists(self.service_path)
+
+  @property
+  def service_path(self):
+    return os.path.join(
+      self.scaffold_dir_path,
+      self.service_relative_path
+    )
+
+  @property
+  def service_relative_path(self):
+    return os.path.join(
+      self.namespace,
+      self.service_name,
+    )
+
+  def config(self, _c: dict):
+    _config = self.app_config.read()
+    _config.update(self.service_config.read())
+    _config.update(_c)
+    return _config