seto 1.0.0__py3-none-any.whl

seto/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2024 Sébastien Demanou. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================

seto/__main__.py

@@ -0,0 +1,216 @@
+# Copyright 2024 Sébastien Demanou. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+import argparse
+import os
+import sys
+
+from paramiko.ssh_exception import AuthenticationException
+
+from .commands.config import execute_config_command
+from .commands.deploy import execute_deploy_command
+from .commands.down import execute_down_command
+from .commands.mount import execute_mount_volumes_command
+from .commands.setup import execute_setup_command
+from .commands.setup import print_ssh_copy_id_commands
+from .commands.umount import execute_umount_volumes_command
+from .commands.volumes import execute_create_volumes_command
+from .core.shell import Setting
+from .drivers.gluster import GlusterDriver
+from .drivers.nfs import NFSDriver
+from .shells.local import LocalShell
+from .shells.remote import RemoteShell
+
+DriverType = type[str]
+ShellConnectionString = type[str]
+
+
+driver_shemes = ['nfs://', 'glusterfs://']
+driver_examples = [f'{sheme}username:password@hostname' for sheme in driver_shemes]
+DRIVER_EXAMPLES_STR = ' or '.join(driver_examples)
+
+
+def manager_type(value: str) -> tuple[DriverType, ShellConnectionString]:
+  if not any(value.startswith(scheme) for scheme in driver_shemes):
+    raise argparse.ArgumentError('manager', f'{value} is not a valid manager ({DRIVER_EXAMPLES_STR})')
+
+  return value.split('://')
+
+
+def replica_type(value: str) -> list[Setting]:
+  connection_strings = value.split(' ')
+
+  try:
+    return [
+      Setting.from_connection_string(item.strip()) for item in connection_strings
+    ]
+  except Exception as exception:
+    raise argparse.ArgumentError('replica', f'{value} is not a replica value (username:password@hostname)') from exception
+
+
+def nodes_type(value: str) -> list[Setting]:
+  connection_strings = value.split(' ')
+
+  try:
+    return [
+      Setting.from_connection_string(item.strip()) for item in connection_strings
+    ]
+  except Exception as exception:
+    raise argparse.ArgumentError('nodes', f'{value} is not a nodes value (username:password@hostname)') from exception
+
+
+def main() -> None:
+  parser = argparse.ArgumentParser('seto', description='')
+  subparsers = parser.add_subparsers()
+
+  parser.add_argument('--stack', required=True, help='Stack name')
+  parser.add_argument(
+    '--manager',
+    type=manager_type,
+    required=True,
+    help=f'Storage manager URI to use. Can be {DRIVER_EXAMPLES_STR}',
+  )
+
+  parser.add_argument('--key', default=os.path.expanduser('~/.ssh/id_rsa'), help='Path to the SSH private key file')
+
+  parser.add_argument(
+    '--debug',
+    action='store_true',
+    help='Enable debug mode',
+  )
+
+  #
+  # Setup command
+  setup_parser = subparsers.add_parser('setup', description='Setup manager and replica nodes')
+  setup_parser.set_defaults(func=execute_setup_command)
+
+  setup_parser.add_argument(
+    '--replica',
+    type=replica_type,
+    nargs='+',
+    required=True,
+    help='Replica nodes to setup',
+  )
+
+  setup_parser.add_argument(
+    '--clients',
+    type=nodes_type,
+    nargs='+',
+    required=True,
+    help='Client nodes to setup',
+  )
+
+  setup_parser.add_argument(
+    '--force',
+    action='store_true',
+    help='Force setup tasks',
+  )
+
+  #
+  # Create Volumes command
+  create_volumes_parser = subparsers.add_parser('create-volumes', description='Create and sync shared volumes')
+  create_volumes_parser.set_defaults(func=execute_create_volumes_command)
+  create_volumes_parser.add_argument(
+    '--replica',
+    type=replica_type,
+    nargs='+',
+    required=True,
+    help='Nodes where volumes will be created',
+  )
+
+  create_volumes_parser.add_argument(
+    '--force',
+    action='store_true',
+    help='Force volumes data sync',
+  )
+
+  #
+  # Mount Volumes command
+  mount_volumes_parser = subparsers.add_parser('mount-volumes', description='Mount shared volumes')
+  mount_volumes_parser.set_defaults(func=execute_mount_volumes_command)
+  mount_volumes_parser.add_argument(
+    '--clients',
+    type=nodes_type,
+    nargs='+',
+    required=True,
+    help='Client nodes on which volumes will be mounted',
+  )
+
+  #
+  # Unmount Volumes command
+  umount_volumes_parser = subparsers.add_parser('unmount-volumes', description='Unmount shared volumes')
+  umount_volumes_parser.set_defaults(func=execute_umount_volumes_command)
+  umount_volumes_parser.add_argument(
+    '--clients',
+    type=nodes_type,
+    nargs='+',
+    required=True,
+    help='Client nodes on which volumes will be unmounted',
+  )
+
+  #
+  # Config command
+  config_parser = subparsers.add_parser('config', description='Parse, resolve and render compose file in canonical format')
+  config_parser.set_defaults(func=execute_config_command)
+
+  config_parser.add_argument(
+    '--compose',
+    action='store_true',
+    help='Resolve for Docker Compose',
+  )
+
+  #
+  # Deploy command
+  deploy_parser = subparsers.add_parser('deploy', description='Deploy a new stack or update an existing stack')
+  deploy_parser.set_defaults(func=execute_deploy_command)
+
+  #
+  # Down command
+  down_parser = subparsers.add_parser('down', description='Stop and remove containers, networks')
+  down_parser.set_defaults(func=execute_down_command)
+
+  #
+  # Parsing
+  try:
+    args = parser.parse_args()
+  except (Exception, argparse.ArgumentError) as exception:
+    print(exception)
+    sys.exit(40)
+
+  driver_name, connection_string = args.manager
+  setting = Setting.from_connection_string(connection_string)
+  create_shell = LocalShell if setting.local else RemoteShell
+  create_driver = GlusterDriver if driver_name == 'gluster' else NFSDriver
+  shell = create_shell(setting, args.key)
+  driver = create_driver(args.stack, shell)
+
+  # Call the function associated with the selected subcommand
+  if hasattr(args, 'func'):
+    if args.debug:
+      args.func(args, driver)
+    else:
+      try:
+        args.func(args, driver)
+      except AuthenticationException as exception:
+        print(f'\nError: {exception}')
+        print_ssh_copy_id_commands(args)
+        sys.exit(41)
+      except Exception as exception:
+        print(exception)
+        sys.exit(50)
+  else:
+    parser.print_help()
+
+
+main()

seto/commands/config.py

@@ -0,0 +1,102 @@
+# Copyright 2024 Sébastien Demanou. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from collections.abc import Callable
+
+import yaml
+
+from ..core.driver import Driver
+from ..core.parser import parse_services
+from ..core.parser import resolve_networks
+from ..core.shell import Shell
+from ..core.volume import Volume
+
+
+def resolve(
+  args,
+  driver: Driver,
+  *,
+  inject: bool = False,
+  execute: Callable[[dict, str], None] | None = None,
+) -> dict:
+  config_networks = resolve_networks(args.stack)
+
+  config_networks['cloud-public'] = {
+    'name': 'seto-cloud-public',
+    'driver': 'overlay',
+    'attachable': True,
+  }
+
+  config_networks['cloud-edge'] = {
+    'name': 'seto-cloud-edge',
+    'driver': 'overlay',
+    'attachable': True,
+  }
+
+  if args.compose:
+    external_networks_keys = config_networks.keys()
+    config_networks = resolve_networks(args.stack, external_networks_keys)
+
+  compose = {
+    'x-placement': None,
+    'configs': {},
+    'networks': config_networks,
+    'volumes': {},
+    'secrets': {},
+    'services': {},
+  }
+
+  def parse(resolved_compose_data: dict, volumes: list[Volume]):
+    placement = resolved_compose_data.get('x-placement', None)
+    networks_ = resolved_compose_data.get('networks', {})
+    services = resolved_compose_data.get('services', {})
+    volumes = resolved_compose_data.get('volumes', {})
+    configs = resolved_compose_data.get('configs', {})
+    secrets = resolved_compose_data.get('secrets', {})
+
+    resolved_compose_data['networks'] = {*config_networks, *networks_}
+    compose['x-placement'] = placement
+    compose['networks'].update(networks_)
+    compose['services'].update(services)
+    compose['volumes'].update(volumes)
+    compose['configs'].update(configs)
+    compose['secrets'].update(secrets)
+
+    if args.compose and not placement:
+      raise ValueError('Missing required x-placement field')
+
+    if execute:
+      execute(resolved_compose_data, placement)
+
+  parse_services(
+    driver=driver,
+    stack=args.stack,
+    execute=parse,
+    inject=inject,
+    mode=['compose'] if args.compose else ['swarm'],
+  )
+
+  return compose
+
+
+def execute_config_command(args, driver: Driver) -> None:
+  compose = resolve(args, driver)
+  compose_output = yaml.dump(compose)
+
+  if args.compose:
+    command = f'docker compose -p {args.stack} -f - config'
+  else:
+    command = 'docker stack config -c -'
+
+  Shell.pipe_exec(command, pipe_input=compose_output)

seto/commands/deploy.py

@@ -0,0 +1,238 @@
+# Copyright 2024 Sébastien Demanou. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+import json
+import os
+import re
+from typing import Any
+
+from docker import DockerClient
+
+from ..core.dns import resolve_hostname
+from ..core.docker import DockerCompose
+from ..core.docker import DockerSwarm
+from ..core.driver import Driver
+from .config import resolve
+
+# Define the regular expression pattern to match {{ .Node.Hostname }} with optional spaces
+NODE_HOSTNAME_RE = r'\{\{\s*\.Node\.Hostname\s*\}\}'
+
+
+def parse_service_vars(entries: dict[str, Any], hostname: str) -> None:
+  for key, value in entries.items():
+    if isinstance(value, str):
+      entries[key] = re.sub(NODE_HOSTNAME_RE, hostname, value)
+
+
+def get_label_value(labels: dict[str, Any], name: str) -> Any | None:
+  for label, value in labels.items():
+    if label.endswith(name):
+      return value
+  return None
+
+
+def parse_compose_config(
+  args,
+  driver: Driver,
+  client: DockerClient,
+  networks_list: list[str],
+  swarm_config: dict,
+  *,
+  compose_config: dict,
+  placement: str,
+  composes: list[DockerCompose],
+  traefik_http_provider_routers: dict,
+  traefik_http_provider_services: dict,
+) -> None:
+  if compose_config['services']:
+    compose = DockerCompose(
+      stack=args.stack,
+      client=client,
+      driver=driver,
+      config=compose_config,
+    )
+
+    composes.append(compose)
+
+    for service_name, service in compose_config['services'].items():
+      service_bridge_name = f'{service_name}_bridge'
+      service_environment = service.get('environment', {})
+      service_labels = service.get('labels', {})
+      service_networks = service.get('networks', [])
+      service_ports = service.get('ports', [])
+      service_deploy = {
+        'labels': [
+          'traefik.discovery.enable=false',
+        ],
+        'placement': {
+          'constraints': [
+            f'node.labels.{placement}',
+          ],
+        },
+      }
+
+      # Use a shadow service to make sure all networks are created on all node
+      swarm_config['services'][service_bridge_name] = {
+        'image': 'traefik/whoami',
+        'networks': service_networks,
+        'deploy': service_deploy,
+      }
+
+      parse_service_vars(service_labels, compose.node_hostname)
+      parse_service_vars(service_environment, compose.node_hostname)
+
+      if service_ports:
+        service_traefik_rule = get_label_value(service_labels, '.rule')
+        service_traefik_middlewares = get_label_value(service_labels, '.middlewares')
+        service_traefik_port = get_label_value(service_labels, '.loadbalancer.server.port')
+        service_traefik_entryPoints = get_label_value(service_labels, '.entryPoints')
+        service_traefik_tls_certresolver = get_label_value(service_labels, '.tls.certresolver')
+        service_traefik_service = get_label_value(service_labels, '.service')
+
+        if not service_traefik_port:
+          raise ValueError(f'Service "{service_name}" error: port is missing')
+
+        published_port = -1
+
+        for entry in service_ports:
+          source_port, target_port = entry.split(':')
+
+          if int(target_port) == service_traefik_port:
+            published_port = source_port
+
+        if published_port == -1:
+          raise ValueError(f'No exposed port for {service_name}:{service_traefik_port}')
+
+        traefik_http_provider_routers[service_name] = {
+          'entryPoints': [service_traefik_entryPoints],
+          'service': service_traefik_service or service_name,
+          'rule': service_traefik_rule,
+          'middlewares': service_traefik_middlewares,
+          'tls': {
+            'certresolver': service_traefik_tls_certresolver,
+          },
+        }
+
+        node_ip = resolve_hostname(compose.node_hostname)
+        traefik_http_provider_services[service_name] = {
+          'loadBalancer': {
+            'servers': [
+              {
+                'url': f'http://{node_ip}:{published_port}',
+              },
+            ],
+          },
+        }
+
+
+def execute_deploy_command(args, driver: Driver) -> None:
+  client = DockerClient.from_env()
+
+  # Docker Swarm
+  print('Resolving services...')
+  setattr(args, 'compose', False)
+  swarm_config = resolve(args, driver)
+  swarm = DockerSwarm(
+    stack=args.stack,
+    client=client,
+    driver=driver,
+    config=swarm_config,
+  )
+
+  # Docker Compose
+  setattr(args, 'compose', True)
+  networks_list = list(swarm_config['networks'].keys())
+  composes_items: list[DockerCompose] = []
+
+  bridges_path = 'bridges'
+  traefik_http_provider_name = 'traefik-http-provider'
+  traefik_http_provider_filename = os.path.join(bridges_path, 'traefik-http-provider.json')
+  traefik_http_provider_routers = {}
+  traefik_http_provider_services = {}
+  traefik_http_provider = {
+    'http': {
+      'routers': traefik_http_provider_routers,
+      'services': traefik_http_provider_services,
+    },
+  }
+
+  swarm_config['configs'][traefik_http_provider_name] = {
+    'file': traefik_http_provider_filename,
+  }
+
+  swarm_config['services']['compose-provider'] = {
+    'image': 'httpd:alpine',
+    'networks': networks_list,
+    'configs': [
+      {
+        'source': traefik_http_provider_name,
+        'target': '/usr/local/apache2/htdocs/bridge.json',
+      },
+    ],
+    'deploy': {
+      'labels': [
+        'traefik.discovery.enable=false',
+      ],
+    },
+  }
+
+  resolve(
+    args,
+    driver,
+    inject=True,
+    execute=lambda config, placement: parse_compose_config(
+      args,
+      driver,
+      client,
+      networks_list,
+      swarm_config,
+      compose_config=config,
+      placement=placement,
+      composes=composes_items,
+      traefik_http_provider_routers=traefik_http_provider_routers,
+      traefik_http_provider_services=traefik_http_provider_services,
+    ),
+  )
+
+  if not os.path.exists(bridges_path):
+    os.mkdir(bridges_path)
+
+  with open(traefik_http_provider_filename, 'w', encoding='utf-8') as file:
+    file.write(json.dumps(traefik_http_provider, indent='  '))
+
+  print('Building swarm images...')
+  # swarm.info()
+  swarm.build()
+  swarm.push()
+
+  print('Deploying swarm environment...')
+  swarm.deploy()
+  swarm.ps()
+
+  if composes_items:
+    print('Building compose images...')
+    for compose in composes_items:
+      # compose.info()
+      compose.build()
+      compose.push()
+
+    print('Pulling compose images...')
+    for compose in composes_items:
+      compose.pull()
+
+    print('Deploying compose environment...')
+    for compose in composes_items:
+      compose.deploy()
+      compose.ps()
+      compose.logs()

seto/commands/down.py

@@ -0,0 +1,51 @@
+# Copyright 2024 Sébastien Demanou. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from docker import DockerClient
+
+from ..core.docker import DockerCompose
+from ..core.docker import DockerSwarm
+from ..core.driver import Driver
+from .config import resolve
+
+
+def execute_down_command(args, driver: Driver) -> None:
+  client = DockerClient.from_env()
+
+  # Docker Swarm
+  print('Stoping environment...')
+  setattr(args, 'compose', False)
+
+  swarm = DockerSwarm(
+    stack=args.stack,
+    client=client,
+    driver=driver,
+    config=resolve(args, driver),
+  )
+
+  swarm.down()
+
+  # Docker Compose
+  setattr(args, 'compose', True)
+  config = resolve(args, driver, inject=True)
+
+  if config['services']:
+    compose = DockerCompose(
+      stack=args.stack,
+      client=client,
+      driver=driver,
+      config=config,
+    )
+
+    compose.down()

seto/commands/mount.py

@@ -0,0 +1,24 @@
+# Copyright 2024 Sébastien Demanou. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from ..core.command import parse_volumes_args
+from ..core.driver import Driver
+from ..core.shell import Setting
+
+
+def execute_mount_volumes_command(args, driver: Driver) -> None:
+  clients: list[Setting] = [items[0] for items in args.clients]
+  all_volumes = parse_volumes_args(args, driver)
+
+  driver.mount_volumes(clients=clients, volumes=all_volumes)