seto 2.5.0__py3-none-any.whl

seto/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2024 Sébastien Demanou. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================

seto/__main__.py

@@ -0,0 +1,224 @@
+# Copyright 2024 Sébastien Demanou. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+import argparse
+import os
+import sys
+
+from paramiko.ssh_exception import AuthenticationException
+
+from .commands.config import execute_config_command
+from .commands.deploy import execute_deploy_command
+from .commands.down import execute_down_command
+from .commands.mount import execute_mount_volumes_command
+from .commands.setup import execute_setup_command
+from .commands.setup import print_ssh_copy_id_commands
+from .commands.umount import execute_umount_volumes_command
+from .commands.volumes import execute_create_volumes_command
+from .core.shell import Setting
+from .drivers.gluster import GlusterDriver
+from .drivers.nfs import NFSDriver
+from .shells.local import LocalShell
+from .shells.remote import RemoteShell
+
+DriverType = type[str]
+ShellConnectionString = type[str]
+
+
+driver_shemes = ['nfs://', 'glusterfs://']
+driver_examples = [f'{sheme}username:password@hostname' for sheme in driver_shemes]
+DRIVER_EXAMPLES_STR = ' or '.join(driver_examples)
+
+
+def manager_type(value: str) -> tuple[DriverType, ShellConnectionString]:
+  if not any(value.startswith(scheme) for scheme in driver_shemes):
+    raise argparse.ArgumentError('manager', f'{value} is not a valid manager ({DRIVER_EXAMPLES_STR})')
+
+  return value.split('://')
+
+
+def replica_type(value: str) -> list[Setting]:
+  connection_strings = value.split(' ')
+
+  try:
+    return [
+      Setting.from_connection_string(item.strip()) for item in connection_strings
+    ]
+  except Exception as exception:
+    raise argparse.ArgumentError('replica', f'{value} is not a replica value (username:password@hostname)') from exception
+
+
+def nodes_type(value: str) -> list[Setting]:
+  connection_strings = value.split(' ')
+
+  try:
+    return [
+      Setting.from_connection_string(item.strip()) for item in connection_strings
+    ]
+  except Exception as exception:
+    raise argparse.ArgumentError('nodes', f'{value} is not a nodes value (username:password@hostname)') from exception
+
+
+def main() -> None:
+  parser = argparse.ArgumentParser('seto', description='')
+  subparsers = parser.add_subparsers()
+
+  parser.add_argument('--project', required=True, help='Project name')
+  parser.add_argument('--stack', help='Stack name')
+
+  parser.add_argument(
+    '--manager',
+    type=manager_type,
+    required=True,
+    help=f'Storage manager URI to use. Can be {DRIVER_EXAMPLES_STR}',
+  )
+
+  parser.add_argument(
+    '--key',
+    default=os.path.expanduser('~/.ssh/id_rsa'),
+    help='Path to the SSH private key file',
+  )
+
+  parser.add_argument(
+    '--debug',
+    action='store_true',
+    help='Enable debug mode',
+  )
+
+  #
+  # Setup command
+  setup_parser = subparsers.add_parser('setup', description='Setup manager and replica nodes')
+  setup_parser.set_defaults(func=execute_setup_command)
+
+  setup_parser.add_argument(
+    '--replica',
+    type=replica_type,
+    nargs='+',
+    required=True,
+    help='Replica nodes to setup',
+  )
+
+  setup_parser.add_argument(
+    '--clients',
+    type=nodes_type,
+    nargs='+',
+    required=True,
+    help='Client nodes to setup',
+  )
+
+  setup_parser.add_argument(
+    '--force',
+    action='store_true',
+    help='Force setup tasks',
+  )
+
+  #
+  # Create Volumes command
+  create_volumes_parser = subparsers.add_parser('create-volumes', description='Create and sync shared volumes')
+  create_volumes_parser.set_defaults(func=execute_create_volumes_command)
+  create_volumes_parser.add_argument(
+    '--replica',
+    type=replica_type,
+    nargs='+',
+    required=True,
+    help='Nodes where volumes will be created',
+  )
+
+  create_volumes_parser.add_argument(
+    '--force',
+    action='store_true',
+    help='Force volumes data sync',
+  )
+
+  #
+  # Mount Volumes command
+  mount_volumes_parser = subparsers.add_parser('mount-volumes', description='Mount shared volumes')
+  mount_volumes_parser.set_defaults(func=execute_mount_volumes_command)
+  mount_volumes_parser.add_argument(
+    '--clients',
+    type=nodes_type,
+    nargs='+',
+    required=True,
+    help='Client nodes on which volumes will be mounted',
+  )
+
+  #
+  # Unmount Volumes command
+  umount_volumes_parser = subparsers.add_parser('unmount-volumes', description='Unmount shared volumes')
+  umount_volumes_parser.set_defaults(func=execute_umount_volumes_command)
+  umount_volumes_parser.add_argument(
+    '--clients',
+    type=nodes_type,
+    nargs='+',
+    required=True,
+    help='Client nodes on which volumes will be unmounted',
+  )
+
+  #
+  # Config command
+  config_parser = subparsers.add_parser('config', description='Parse, resolve and render compose file in canonical format')
+  config_parser.set_defaults(func=execute_config_command)
+
+  config_parser.add_argument(
+    '--compose',
+    action='store_true',
+    help='Resolve for Docker Compose',
+  )
+
+  #
+  # Deploy command
+  deploy_parser = subparsers.add_parser('deploy', description='Deploy a new stack or update an existing stack')
+  deploy_parser.set_defaults(func=execute_deploy_command)
+
+  #
+  # Down command
+  down_parser = subparsers.add_parser('down', description='Stop and remove containers, networks')
+  down_parser.set_defaults(func=execute_down_command)
+
+  #
+  # Parsing
+  try:
+    args = parser.parse_args()
+  except (Exception, argparse.ArgumentError) as exception:
+    print(exception)
+    sys.exit(40)
+
+  driver_name, connection_string = args.manager
+  setting = Setting.from_connection_string(connection_string)
+  create_shell = LocalShell if setting.local else RemoteShell
+  create_driver = GlusterDriver if driver_name == 'gluster' else NFSDriver
+  shell = create_shell(setting, args.key)
+  driver = create_driver(args.stack, project=args.project, shell=shell)
+
+  # Call the function associated with the selected subcommand
+  if hasattr(args, 'func'):
+    if args.debug:
+      args.func(args, driver)
+    else:
+      try:
+        args.func(args, driver)
+      except AuthenticationException as exception:
+        print(f'\nError: {exception}')
+        print_ssh_copy_id_commands(args)
+        sys.exit(41)
+      except Exception as exception:
+        print(exception)
+        sys.exit(50)
+  else:
+    parser.print_help()
+
+  sys.exit(0)
+
+
+main()

seto/commands/config.py

@@ -0,0 +1,97 @@
+# Copyright 2024-2025 Sébastien Demanou. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from collections.abc import Callable
+
+import yaml
+
+from ..core.driver import Driver
+from ..core.network import get_global_external_networks
+from ..core.network import resolve_networks
+from ..core.parser import parse_services
+from ..core.shell import Shell
+from ..core.volume import Volume
+
+
+def resolve(
+  args,
+  driver: Driver,
+  *,
+  inject: bool = False,
+  execute: Callable[[dict, str], None] | None = None,
+) -> dict:
+  config_networks = get_global_external_networks()
+  global_networks = resolve_networks(args.project)
+
+  for network_name, network_definition in global_networks.items():
+    config_networks[network_name] = {
+      'name': network_definition['name'],
+      'external': True,
+    }
+
+  compose = {
+    'x-placement': None,
+    'x-placement-hostname': None,
+    'configs': {},
+    'networks': config_networks,
+    'volumes': {},
+    'secrets': {},
+    'services': {},
+  }
+
+  def parse(resolved_compose_data: dict, volumes: list[Volume]):
+    placement_hostname = resolved_compose_data.get('x-placement-hostname', None)
+    placement = resolved_compose_data.get('x-placement', None)
+    networks_ = resolved_compose_data.get('networks', {})
+    services = resolved_compose_data.get('services', {})
+    volumes = resolved_compose_data.get('volumes', {})
+    configs = resolved_compose_data.get('configs', {})
+    secrets = resolved_compose_data.get('secrets', {})
+
+    resolved_compose_data['networks'] = {**config_networks, **networks_}
+    compose['x-placement-hostname'] = placement_hostname
+    compose['x-placement'] = placement
+    compose['networks'].update(networks_)
+    compose['services'].update(services)
+    compose['volumes'].update(volumes)
+    compose['configs'].update(configs)
+    compose['secrets'].update(secrets)
+
+    if args.compose and not placement and not placement_hostname:
+      raise ValueError('Missing required x-placement or x-placement-hostname field')
+
+    if execute:
+      execute(resolved_compose_data, placement_hostname or placement)
+
+  parse_services(
+    driver=driver,
+    stack=args.stack or args.project,
+    execute=parse,
+    inject=inject,
+    mode=['compose'] if args.compose else ['swarm'],
+  )
+
+  return compose
+
+
+def execute_config_command(args, driver: Driver) -> None:
+  compose = resolve(args, driver)
+  compose_output = yaml.dump(compose)
+
+  if args.compose:
+    command = f'docker compose -p {driver.stack_id} -f - config'
+  else:
+    command = 'docker stack config -c -'
+
+  Shell.pipe_exec(command, pipe_input=compose_output)

seto/commands/deploy.py

@@ -0,0 +1,332 @@
+# Copyright 2024-2025 Sébastien Demanou. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+import json
+import os
+import random
+import re
+from typing import Any
+
+from docker import DockerClient
+
+from ..core.dns import resolve_hostname
+from ..core.docker import DockerCompose
+from ..core.docker import DockerSwarm
+from ..core.driver import Driver
+from ..core.network import GLOBAL_NETWORKS
+from ..core.network import resolve_networks
+from ..core.parser import resolve_compose_file
+from ..core.shell import Setting
+from ..core.traefik import convert_middlewares_to_dict
+from .config import resolve
+
+
+# Define the regular expression pattern to match {{ .Node.Hostname }} with optional spaces
+NODE_HOSTNAME_RE = r'\{\{\s*\.Node\.Hostname\s*\}\}'
+HTTP_PROVIDER_SERVICENAME = 'seto-http-provider'
+
+
+def parse_service_vars(entries: dict[str, Any], hostname: str) -> None:
+  for key, value in entries.items():
+    if isinstance(value, str):
+      entries[key] = re.sub(NODE_HOSTNAME_RE, hostname, value)
+
+
+def pick_label_value(labels: dict[str, Any], name: str) -> Any | str:
+  for label, value in labels.items():
+    if label.endswith(name):
+      del labels[label]
+      return value
+  return ''
+
+
+def parse_compose_config(
+  args,
+  driver: Driver,
+  client: DockerClient,
+  networks_list: list[str],
+  swarm_config: dict,
+  *,
+  compose_config: dict,
+  placement: str,
+  composes: list[DockerCompose],
+  traefik_http_provider_routers: dict,
+  traefik_http_provider_services: dict,
+  traefik_http_provider_middlewares: dict,
+) -> None:
+  if compose_config['services']:
+    compose = DockerCompose(
+      client=client,
+      driver=driver,
+      config=compose_config,
+    )
+
+    composes.append(compose)
+
+    for service_name, service in compose_config['services'].items():
+      service_bridge_name = f'{service_name}_bridge'
+      service_environment = service.get('environment', {})
+      service_labels = service.get('labels', {})
+      service_networks = service.get('networks', [])
+      service_ports = service.get('ports', [])
+      service_deploy = {
+        'placement': {
+          'constraints': [
+            f'node.labels.{placement}' if '=' in placement else f'node.hostname == {placement}',
+          ],
+        },
+      }
+
+      # Use a shadow service to make sure all networks are created on all node
+      swarm_config['services'][service_bridge_name] = {
+        'image': 'traefik/whoami',
+        'networks': service_networks,
+        'deploy': service_deploy,
+      }
+
+      if compose.node_hostname:
+        parse_service_vars(service_labels, compose.node_hostname)
+        parse_service_vars(service_environment, compose.node_hostname)
+
+      service['ports'] = service_ports
+      service_traefik_rule = pick_label_value(service_labels, '.rule')
+      service_traefik_middlewares = pick_label_value(service_labels, '.middlewares')
+      service_traefik_port = pick_label_value(service_labels, '.loadbalancer.server.port')
+      service_traefik_entryPoints = pick_label_value(service_labels, '.entryPoints')
+      service_traefik_tls_certresolver = pick_label_value(service_labels, '.tls.certresolver')
+      service_traefik_service = pick_label_value(service_labels, '.service') or service_name
+      published_port = random.randint(53100, 64200)
+
+      if not service_traefik_port:
+        print(f'WARN: Service "{service_name}" has no defined port. Skipped from Traefik HTTP Provider')
+        continue
+
+      service_ports.append(f'{published_port}:{service_traefik_port}')
+
+      traefik_http_provider_routers[service_traefik_service] = {
+        'entryPoints': [service_traefik_entryPoints],
+        'service': service_traefik_service,
+        'rule': service_traefik_rule,
+        'middlewares': [item for item in service_traefik_middlewares.split(',') if item],
+        'tls': {
+          'certresolver': service_traefik_tls_certresolver,
+        },
+      }
+
+      node_ip = resolve_hostname(compose.node_hostname)
+      traefik_http_provider_services[service_traefik_service] = {
+        'loadBalancer': {
+          'servers': [
+            {
+              'url': f'http://{node_ip}:{published_port}',
+            },
+          ],
+        },
+      }
+
+      traefik_http_provider_middlewares.update(
+        convert_middlewares_to_dict(service_labels),
+      )
+
+
+def deploy_seto_stack(args, driver: Driver, replica: list[Setting]) -> None:
+  # Temporary rewrite driver config
+  driver.project = 'seto'
+  driver.stack = None
+
+  # Building seto config
+  client = DockerClient.from_env()
+  config_networks = resolve_networks(args.project)
+
+  config_networks.update(GLOBAL_NETWORKS)
+
+  print('Configuring seto-http-provider...')
+  traefik_http_provider_data_path = '/data'
+  internal_stack = {
+    'networks': config_networks,
+    'services': {
+      HTTP_PROVIDER_SERVICENAME: {
+        'image': 'demsking/traefik-http-provider',
+        'networks': list(config_networks.keys()),
+        'environment': [
+          'WORKER=1',
+          'EXPIRATION_MINUTES=10',
+          f'DATA_PATH={traefik_http_provider_data_path}',
+        ],
+        'volumes-nfs': {
+          f'data:{traefik_http_provider_data_path}',
+        },
+        'deploy': {
+          'mode': 'global',
+          'labels': {
+            'traefik.discovery.enable': True,
+          },
+        },
+      },
+    },
+  }
+
+  # Resolving compose local volumes
+  resolved_compose_data, volumes = resolve_compose_file(
+    driver=driver,
+    compose_data=internal_stack,
+    inject=True,
+  )
+
+  print('Creating seto volumes...')
+  driver.create_volumes(replica=replica, volumes=volumes, force=args.force)
+
+  print('Deploying seto services...')
+  swarm = DockerSwarm(
+    client=client,
+    driver=driver,
+    config=resolved_compose_data,
+  )
+
+  swarm.info()
+  swarm.deploy()
+
+  # Restore initial driver config
+  driver.project = args.project
+  driver.stack = args.stack
+
+
+def execute_deploy_command(args, driver: Driver) -> None:
+  client = DockerClient.from_env()
+
+  # Docker Swarm
+  print(f'Resolving {driver.stack_id} services...')
+  setattr(args, 'compose', False)
+  swarm_config = resolve(args, driver)
+
+  swarm = DockerSwarm(
+    client=client,
+    driver=driver,
+    config=swarm_config,
+  )
+
+  # Docker Compose
+  setattr(args, 'compose', True)
+  networks_list = list(swarm_config['networks'].keys())
+  composes_items: list[DockerCompose] = []
+
+  bridges_path = 'bridges'
+  traefik_http_provider_filename = os.path.join(bridges_path, 'traefik-http-provider.json')
+  traefik_http_provider_target = '/traefik/config.json'
+  traefik_http_provider_routers = {}
+  traefik_http_provider_services = {}
+  traefik_http_provider_middlewares = {}
+  traefik_http_provider = {
+    'http': {
+      'routers': traefik_http_provider_routers,
+      'services': traefik_http_provider_services,
+      'middlewares': traefik_http_provider_middlewares,
+    },
+  }
+
+  register_command = ' && '.join([
+    f'echo "Registering service {driver.stack_id}..."',
+    f'curl -s -X POST http://{HTTP_PROVIDER_SERVICENAME}:6116/api/config/{driver.stack_id} -H "Content-Type: application/json" -d @{traefik_http_provider_target} > /dev/nul',
+  ])
+
+  entrypoint = ' && '.join([
+    # Log the start of the initial endpoint call
+    'echo "Starting initial call to the provider endpoint..."',
+
+    # Call the POST endpoint at startup
+    register_command,
+
+    # Log the end of the initial endpoint call
+    'echo "Initial call to the provider endpoint completed. Running cron job..."',
+
+    # Set up the cron job
+    f'echo "*/1 * * * * {register_command}" | crontab -',
+
+    # Start the cron service
+    'crond -f',
+  ])
+
+  seto_agent_compose_data, _ = resolve_compose_file(
+    driver=driver,
+    compose_data={
+      'services': {
+        'seto_agent': {
+          'image': 'curlimages/curl:latest',
+          'user': 'root',  # Ensure it runs as root to avoid issue `crontab: must be suid to work properly`
+          'networks': networks_list,
+          'entrypoint': f"sh -c '{entrypoint}'",
+          'volumes-image': [
+            f'{traefik_http_provider_filename}:{traefik_http_provider_target}',
+          ],
+        },
+      },
+    },
+  )
+
+  swarm_config['services'].update(seto_agent_compose_data['services'])
+
+  resolve(
+    args,
+    driver,
+    inject=True,
+    execute=lambda config, placement: parse_compose_config(
+      args,
+      driver,
+      client,
+      networks_list,
+      swarm_config,
+      compose_config=config,
+      placement=placement,
+      composes=composes_items,
+      traefik_http_provider_routers=traefik_http_provider_routers,
+      traefik_http_provider_services=traefik_http_provider_services,
+      traefik_http_provider_middlewares=traefik_http_provider_middlewares,
+    ),
+  )
+
+  if not os.path.exists(bridges_path):
+    os.mkdir(bridges_path)
+
+  with open(traefik_http_provider_filename, 'w', encoding='utf-8') as file:
+    file.write(json.dumps(traefik_http_provider, indent='  '))
+
+  print(f'Building {driver.stack_id} swarm images...')
+  # swarm.info()
+  swarm.build()
+
+  print(f'Pushing {driver.stack_id} images...')
+  swarm.push()
+
+  # print(f'Creating {driver.stack_id} volumes...')
+  # driver.create_volumes(replica=replica, volumes=volumes, force=args.force)
+
+  print(f'Deploying {driver.stack_id} swarm environment...')
+  swarm.deploy()
+  swarm.ps()
+
+  if composes_items:
+    print(f'Building {driver.stack_id} compose images...')
+    for compose in composes_items:
+      # compose.info()
+      compose.build()
+
+    print(f'Pulling {driver.stack_id} compose images...')
+    for compose in composes_items:
+      compose.pull()
+
+    print(f'Deploying {driver.stack_id} compose environment...')
+    for compose in composes_items:
+      compose.deploy()
+      compose.ps()
+      compose.logs()