ominfra 0.0.0.dev7__py3-none-any.whl

ominfra/deploy/configs.py

@@ -0,0 +1,19 @@
+import dataclasses as dc
+
+
+@dc.dataclass(frozen=True)
+class DeployConfig:
+    python_bin: str
+    app_name: str
+    repo_url: str
+    revision: str
+    requirements_txt: str
+    entrypoint: str
+
+
+@dc.dataclass(frozen=True)
+class HostConfig:
+    username: str = 'deploy'
+
+    global_supervisor_conf_file_path: str = '/etc/supervisor/conf.d/supervisord.conf'
+    global_nginx_conf_file_path: str = '/etc/nginx/sites-enabled/deploy.conf'

ominfra/deploy/executor/__init__.py

@@ -0,0 +1 @@
+# @omlish-lite

ominfra/deploy/executor/base.py

@@ -0,0 +1,115 @@
+# ruff: noqa: UP006
+import abc
+import dataclasses as dc
+import enum
+import os.path
+import shlex
+import typing as ta
+
+from omlish.lite.cached import cached_nullary
+from omlish.lite.logs import log
+from omlish.lite.subprocesses import subprocess_check_call
+
+from ..configs import DeployConfig
+from ..configs import HostConfig
+
+
+##
+
+
+class Phase(enum.Enum):
+    HOST = enum.auto()
+    ENV = enum.auto()
+    BACKEND = enum.auto()
+    FRONTEND = enum.auto()
+    START_BACKEND = enum.auto()
+    START_FRONTEND = enum.auto()
+
+
+def run_in_phase(*ps: Phase):
+    def inner(fn):
+        fn.__deployment_phases__ = ps
+        return fn
+    return inner
+
+
+class Concern(abc.ABC):
+    def __init__(self, d: 'Deployment') -> None:
+        super().__init__()
+        self._d = d
+
+    _phase_fns: ta.ClassVar[ta.Mapping[Phase, ta.Sequence[ta.Callable]]]
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        dct: ta.Dict[Phase, ta.List[ta.Callable]] = {}
+        for fn, ps in [
+            (v, ps)
+            for a in dir(cls)
+            if not (a.startswith('__') and a.endswith('__'))
+            for v in [getattr(cls, a, None)]
+            for ps in [getattr(v, '__deployment_phases__', None)]
+            if ps
+        ]:
+            dct.update({p: [*dct.get(p, []), fn] for p in ps})
+        cls._phase_fns = dct
+
+    @dc.dataclass(frozen=True)
+    class Output(abc.ABC):
+        path: str
+        is_file: bool
+
+    def outputs(self) -> ta.Sequence[Output]:
+        return ()
+
+    def run_phase(self, p: Phase) -> None:
+        for fn in self._phase_fns.get(p, ()):
+            fn.__get__(self, type(self))()
+
+
+##
+
+
+class Deployment:
+
+    def __init__(
+            self,
+            cfg: DeployConfig,
+            concern_cls_list: ta.List[ta.Type[Concern]],
+            host_cfg: HostConfig = HostConfig(),
+    ) -> None:
+        super().__init__()
+        self._cfg = cfg
+        self._host_cfg = host_cfg
+
+        self._concerns: ta.List[Concern] = [cls(self) for cls in concern_cls_list]
+
+    @property
+    def cfg(self) -> DeployConfig:
+        return self._cfg
+
+    @property
+    def host_cfg(self) -> HostConfig:
+        return self._host_cfg
+
+    def sh(self, *ss: str) -> None:
+        s = ' && '.join(ss)
+        log.info('Executing: %s', s)
+        subprocess_check_call(s, shell=True)
+
+    def ush(self, *ss: str) -> None:
+        s = ' && '.join(ss)
+        self.sh(f'su - {self._host_cfg.username} -c {shlex.quote(s)}')
+
+    @cached_nullary
+    def home_dir(self) -> str:
+        return os.path.expanduser(f'~{self._host_cfg.username}')
+
+    @cached_nullary
+    def deploy(self) -> None:
+        for p in Phase:
+            log.info('Phase %s', p.name)
+            for c in self._concerns:
+                c.run_phase(p)
+
+        log.info('Shitty deploy complete!')

ominfra/deploy/executor/concerns/dirs.py

@@ -0,0 +1,28 @@
+import os.path
+import pwd
+
+from omlish.lite.logs import log
+
+from ..base import Concern
+from ..base import Phase
+from ..base import run_in_phase
+
+
+class DirsConcern(Concern):
+    @run_in_phase(Phase.HOST)
+    def create_dirs(self) -> None:
+        pwn = pwd.getpwnam(self._d.host_cfg.username)
+
+        for dn in [
+            'app',
+            'conf',
+            'conf/env',
+            'conf/nginx',
+            'conf/supervisor',
+            'venv',
+        ]:
+            fp = os.path.join(self._d.home_dir(), dn)
+            if not os.path.exists(fp):
+                log.info('Creating directory: %s', fp)
+                os.mkdir(fp)
+                os.chown(fp, pwn.pw_uid, pwn.pw_gid)

ominfra/deploy/executor/concerns/nginx.py

@@ -0,0 +1,47 @@
+"""
+TODO:
+ - https://stackoverflow.com/questions/3011067/restart-nginx-without-sudo
+"""
+import os.path
+import textwrap
+
+from omlish.lite.logs import log
+
+from ..base import Concern
+from ..base import Phase
+from ..base import run_in_phase
+
+
+class GlobalNginxConcern(Concern):
+    @run_in_phase(Phase.HOST)
+    def create_global_nginx_conf(self) -> None:
+        nginx_conf_dir = os.path.join(self._d.home_dir(), 'conf/nginx')
+        if not os.path.isfile(self._d.host_cfg.global_nginx_conf_file_path):
+            log.info('Writing global nginx conf at %s', self._d.host_cfg.global_nginx_conf_file_path)
+            with open(self._d.host_cfg.global_nginx_conf_file_path, 'w') as f:
+                f.write(f'include {nginx_conf_dir}/*.conf;\n')
+
+
+class NginxConcern(Concern):
+    @run_in_phase(Phase.FRONTEND)
+    def create_nginx_conf(self) -> None:
+        nginx_conf = textwrap.dedent(f"""
+            server {{
+                listen 80;
+                location / {{
+                    proxy_pass http://127.0.0.1:8000/;
+                }}
+            }}
+        """)
+        nginx_conf_file = os.path.join(self._d.home_dir(), f'conf/nginx/{self._d.cfg.app_name}.conf')
+        log.info('Writing nginx conf to %s', nginx_conf_file)
+        with open(nginx_conf_file, 'w') as f:
+            f.write(nginx_conf)
+
+    @run_in_phase(Phase.START_FRONTEND)
+    def poke_nginx(self) -> None:
+        log.info('Starting nginx')
+        self._d.sh('service nginx start')
+
+        log.info('Poking nginx')
+        self._d.sh('nginx -s reload')

ominfra/deploy/executor/concerns/repo.py

@@ -0,0 +1,17 @@
+from ..base import Concern
+from ..base import Phase
+from ..base import run_in_phase
+
+
+class RepoConcern(Concern):
+    @run_in_phase(Phase.ENV)
+    def clone_repo(self) -> None:
+        clone_submodules = False
+        self._d.ush(
+            'cd ~/app',
+            f'git clone --depth 1 {self._d.cfg.repo_url} {self._d.cfg.app_name}',
+            *([
+                f'cd {self._d.cfg.app_name}',
+                'git submodule update --init',
+            ] if clone_submodules else []),
+        )

ominfra/deploy/executor/concerns/supervisor.py

@@ -0,0 +1,46 @@
+import os.path
+import textwrap
+
+from omlish.lite.logs import log
+
+from ..base import Concern
+from ..base import Phase
+from ..base import run_in_phase
+
+
+class GlobalSupervisorConcern(Concern):
+    @run_in_phase(Phase.HOST)
+    def create_global_supervisor_conf(self) -> None:
+        sup_conf_dir = os.path.join(self._d.home_dir(), 'conf/supervisor')
+        with open(self._d.host_cfg.global_supervisor_conf_file_path) as f:
+            glo_sup_conf = f.read()
+        if sup_conf_dir not in glo_sup_conf:
+            log.info('Updating global supervisor conf at %s', self._d.host_cfg.global_supervisor_conf_file_path)  # noqa
+            glo_sup_conf += textwrap.dedent(f"""
+                [include]
+                files = {self._d.home_dir()}/conf/supervisor/*.conf
+            """)
+            with open(self._d.host_cfg.global_supervisor_conf_file_path, 'w') as f:
+                f.write(glo_sup_conf)
+
+
+class SupervisorConcern(Concern):
+    @run_in_phase(Phase.BACKEND)
+    def create_supervisor_conf(self) -> None:
+        sup_conf = textwrap.dedent(f"""
+            [program:{self._d.cfg.app_name}]
+            command={self._d.home_dir()}/venv/{self._d.cfg.app_name}/bin/python -m {self._d.cfg.entrypoint}
+            directory={self._d.home_dir()}/app/{self._d.cfg.app_name}
+            user={self._d.host_cfg.username}
+            autostart=true
+            autorestart=true
+        """)
+        sup_conf_file = os.path.join(self._d.home_dir(), f'conf/supervisor/{self._d.cfg.app_name}.conf')
+        log.info('Writing supervisor conf to %s', sup_conf_file)
+        with open(sup_conf_file, 'w') as f:
+            f.write(sup_conf)
+
+    @run_in_phase(Phase.START_BACKEND)
+    def poke_supervisor(self) -> None:
+        log.info('Poking supervisor')
+        self._d.sh('kill -HUP 1')

ominfra/deploy/executor/concerns/systemd.py

@@ -0,0 +1,88 @@
+"""
+# https://serverfault.com/questions/617823/how-to-set-systemd-service-dependencies
+PIDFile=/run/nginx.pid
+ExecStartPre=/usr/sbin/nginx -t
+ExecStart=/usr/sbin/nginx
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s QUIT $MAINPID
+PrivateTmp=true
+
+# https://gist.github.com/clemensg/7dd024169efe8ce6e7fa4a0b3caa3780
+Type=forking
+PIDFile=/var/run/nginx.pid
+ExecStartPre=/usr/sbin/nginx -t
+ExecStart=/usr/sbin/nginx
+ExecReload=/usr/bin/kill -s HUP $MAINPID
+ExecStop=/usr/bin/kill -s QUIT $MAINPID
+# Hardening
+InaccessiblePaths=/etc/gnupg /etc/shadow /etc/ssh
+ProtectSystem=full
+ProtectKernelTunables=yes
+ProtectControlGroups=yes
+SystemCallFilter=~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
+RestrictRealtime=yes
+"""
+import os.path
+import textwrap
+
+from omlish.lite.logs import log
+
+from ..base import Concern
+from ..base import Phase
+from ..base import run_in_phase
+
+
+class GlobalSystemdConcern(Concern):
+    @run_in_phase(Phase.HOST)
+    def enable_user_linger(self) -> None:
+        log.info('Enabling user linger')
+        self._d.sh(f'loginctl enable-linger {self._d.host_cfg.username}')
+
+
+class SystemdConcern(Concern):
+    service_name: str
+
+    @run_in_phase(Phase.HOST)
+    def create_systemd_path(self) -> None:
+        sd_svc_dir = os.path.join(self._d.home_dir(), '.config/systemd/user')
+        if not os.path.exists(sd_svc_dir):
+            log.info('Creating directory: %s', sd_svc_dir)
+            os.makedirs(sd_svc_dir)
+
+    @run_in_phase(Phase.BACKEND)
+    def create_systemd_service(self) -> None:
+        sd_svc = textwrap.dedent(f"""
+            [Unit]
+            Description={self.service_name}
+            After= \
+                syslog.target \
+                network.target \
+                remote-fs.target \
+                nss-lookup.target \
+                network-online.target
+
+            [Service]
+            Type=simple
+            StandardOutput=journal
+            ExecStart={self._d.home_dir()}/venv/{self._d.cfg.app_name}/bin/python -m {self._d.cfg.entrypoint}
+            WorkingDirectory={self._d.home_dir()}/app/{self._d.cfg.app_name}
+
+            Restart=always
+            RestartSec=3
+
+            [Install]
+            WantedBy=multi-user.target
+        """)
+        sd_svc_file = os.path.join(self._d.home_dir(), f'.config/systemd/user/{self.service_name}.service')
+        log.info('Writing systemd service to %s', sd_svc_file)
+        with open(sd_svc_file, 'w') as f:
+            f.write(sd_svc)
+
+    @run_in_phase(Phase.START_BACKEND)
+    def poke_systemd(self) -> None:
+        log.info('Poking systemd')
+        self._d.sh('systemctl --user daemon-reload')
+        self._d.sh(f'systemctl --user enable {self.service_name}')
+        self._d.sh(f'systemctl --user restart {self.service_name}')

ominfra/deploy/executor/concerns/user.py

@@ -0,0 +1,25 @@
+import pwd
+
+from omlish.lite.logs import log
+
+from ..base import Concern
+from ..base import Phase
+from ..base import run_in_phase
+
+
+class UserConcern(Concern):
+    @run_in_phase(Phase.HOST)
+    def create_user(self) -> None:
+        try:
+            pwd.getpwnam(self._d.host_cfg.username)
+        except KeyError:
+            log.info('Creating user %s', self._d.host_cfg.username)
+            self._d.sh(' '.join([
+                'adduser',
+                '--system',
+                '--disabled-password',
+                '--group',
+                '--shell /bin/bash',
+                self._d.host_cfg.username,
+            ]))
+            pwd.getpwnam(self._d.host_cfg.username)

ominfra/deploy/executor/concerns/venv.py

@@ -0,0 +1,22 @@
+"""
+TODO:
+ - use LinuxInterpResolver lol
+"""
+from ..base import Concern
+from ..base import Phase
+from ..base import run_in_phase
+
+
+class VenvConcern(Concern):
+    @run_in_phase(Phase.ENV)
+    def setup_venv(self) -> None:
+        self._d.ush(
+            'cd ~/venv',
+            f'{self._d.cfg.python_bin} -mvenv {self._d.cfg.app_name}',
+
+            # https://stackoverflow.com/questions/77364550/attributeerror-module-pkgutil-has-no-attribute-impimporter-did-you-mean
+            f'{self._d.cfg.app_name}/bin/python -m ensurepip',
+            f'{self._d.cfg.app_name}/bin/python -mpip install --upgrade setuptools pip',
+
+            f'{self._d.cfg.app_name}/bin/python -mpip install -r ~deploy/app/{self._d.cfg.app_name}/{self._d.cfg.requirements_txt}',  # noqa
+        )

ominfra/deploy/executor/main.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# @omdev-amalg ../_executor.py
+r"""
+TODO:
+ - flock
+ - interp.py
+ - systemd
+
+deployment matrix
+ - os: ubuntu / amzn / generic
+ - arch: amd64 / arm64
+ - host: bare / docker
+ - init: supervisor-provided / supervisor-must-configure / systemd (/ self?)
+ - interp: system / pyenv / interp.py
+ - venv: none / yes
+ - nginx: no / provided / must-configure
+
+==
+
+~deploy
+  deploy.pid (flock)
+  /app
+    /<appspec> - shallow clone
+  /conf
+    /env
+      <appspec>.env
+    /nginx
+      <appspec>.conf
+    /supervisor
+      <appspec>.conf
+  /venv
+    /<appspec>
+
+?
+  /logs
+    /wrmsr--omlish--<spec>
+
+spec = <name>--<rev>--<when>
+
+https://docs.docker.com/config/containers/multi-service_container/#use-a-process-manager
+https://serverfault.com/questions/211525/supervisor-not-loading-new-configuration-files
+"""  # noqa
+# ruff: noqa: UP007
+import argparse
+import json
+import sys
+import typing as ta
+
+from omlish.lite.logs import configure_standard_logging
+from omlish.lite.marshal import unmarshal_obj
+from omlish.lite.runtime import check_runtime_version
+
+from ..configs import DeployConfig
+from .base import Deployment
+from .concerns.dirs import DirsConcern
+from .concerns.nginx import GlobalNginxConcern
+from .concerns.nginx import NginxConcern
+from .concerns.repo import RepoConcern
+from .concerns.supervisor import GlobalSupervisorConcern
+from .concerns.supervisor import SupervisorConcern
+from .concerns.user import UserConcern
+from .concerns.venv import VenvConcern
+
+
+##
+
+
+def _deploy_cmd(args) -> None:
+    dct = json.loads(args.cfg)
+    cfg: DeployConfig = unmarshal_obj(dct, DeployConfig)
+    dp = Deployment(
+        cfg,
+        [
+            UserConcern,
+            DirsConcern,
+            GlobalNginxConcern,
+            GlobalSupervisorConcern,
+            RepoConcern,
+            VenvConcern,
+            SupervisorConcern,
+            NginxConcern,
+        ],
+    )
+    dp.deploy()
+
+
+##
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers()
+
+    parser_resolve = subparsers.add_parser('deploy')
+    parser_resolve.add_argument('cfg')
+    parser_resolve.set_defaults(func=_deploy_cmd)
+
+    return parser
+
+
+def _main(argv: ta.Optional[ta.Sequence[str]] = None) -> None:
+    check_runtime_version()
+
+    if getattr(sys, 'platform') != 'linux':  # noqa
+        raise OSError('must run on linux')
+
+    configure_standard_logging()
+
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    if not getattr(args, 'func', None):
+        parser.print_help()
+    else:
+        args.func(args)
+
+
+if __name__ == '__main__':
+    _main()

ominfra/deploy/poly/__init__.py

@@ -0,0 +1 @@
+# @omlish-lite