ominfra 0.0.0.dev138__py3-none-any.whl → 0.0.0.dev139__py3-none-any.whl

ominfra/manage/deploy/configs.py

@@ -1,19 +0,0 @@
-import dataclasses as dc
-
-
-@dc.dataclass(frozen=True)
-class DeployConfig:
-    python_bin: str
-    app_name: str
-    repo_url: str
-    revision: str
-    requirements_txt: str
-    entrypoint: str
-
-
-@dc.dataclass(frozen=True)
-class HostConfig:
-    username: str = 'deploy'
-
-    global_supervisor_conf_file_path: str = '/etc/supervisor/conf.d/supervisord.conf'
-    global_nginx_conf_file_path: str = '/etc/nginx/sites-enabled/deploy.conf'

ominfra/manage/deploy/executor/__init__.py

@@ -1 +0,0 @@
-# @omlish-lite

ominfra/manage/deploy/executor/base.py

@@ -1,115 +0,0 @@
-# ruff: noqa: UP006
-import abc
-import dataclasses as dc
-import enum
-import os.path
-import shlex
-import typing as ta
-
-from omlish.lite.cached import cached_nullary
-from omlish.lite.logs import log
-from omlish.lite.subprocesses import subprocess_check_call
-
-from ..configs import DeployConfig
-from ..configs import HostConfig
-
-
-##
-
-
-class Phase(enum.Enum):
-    HOST = enum.auto()
-    ENV = enum.auto()
-    BACKEND = enum.auto()
-    FRONTEND = enum.auto()
-    START_BACKEND = enum.auto()
-    START_FRONTEND = enum.auto()
-
-
-def run_in_phase(*ps: Phase):
-    def inner(fn):
-        fn.__deployment_phases__ = ps
-        return fn
-    return inner
-
-
-class Concern(abc.ABC):
-    def __init__(self, d: 'Deployment') -> None:
-        super().__init__()
-        self._d = d
-
-    _phase_fns: ta.ClassVar[ta.Mapping[Phase, ta.Sequence[ta.Callable]]]
-
-    def __init_subclass__(cls, **kwargs):
-        super().__init_subclass__(**kwargs)
-        dct: ta.Dict[Phase, ta.List[ta.Callable]] = {}
-        for fn, ps in [
-            (v, ps)
-            for a in dir(cls)
-            if not (a.startswith('__') and a.endswith('__'))
-            for v in [getattr(cls, a, None)]
-            for ps in [getattr(v, '__deployment_phases__', None)]
-            if ps
-        ]:
-            dct.update({p: [*dct.get(p, []), fn] for p in ps})
-        cls._phase_fns = dct
-
-    @dc.dataclass(frozen=True)
-    class Output(abc.ABC):
-        path: str
-        is_file: bool
-
-    def outputs(self) -> ta.Sequence[Output]:
-        return ()
-
-    def run_phase(self, p: Phase) -> None:
-        for fn in self._phase_fns.get(p, ()):
-            fn.__get__(self, type(self))()
-
-
-##
-
-
-class Deployment:
-
-    def __init__(
-            self,
-            cfg: DeployConfig,
-            concern_cls_list: ta.List[ta.Type[Concern]],
-            host_cfg: HostConfig = HostConfig(),
-    ) -> None:
-        super().__init__()
-        self._cfg = cfg
-        self._host_cfg = host_cfg
-
-        self._concerns: ta.List[Concern] = [cls(self) for cls in concern_cls_list]
-
-    @property
-    def cfg(self) -> DeployConfig:
-        return self._cfg
-
-    @property
-    def host_cfg(self) -> HostConfig:
-        return self._host_cfg
-
-    def sh(self, *ss: str) -> None:
-        s = ' && '.join(ss)
-        log.info('Executing: %s', s)
-        subprocess_check_call(s, shell=True)
-
-    def ush(self, *ss: str) -> None:
-        s = ' && '.join(ss)
-        self.sh(f'su - {self._host_cfg.username} -c {shlex.quote(s)}')
-
-    @cached_nullary
-    def home_dir(self) -> str:
-        return os.path.expanduser(f'~{self._host_cfg.username}')
-
-    @cached_nullary
-    def deploy(self) -> None:
-        for p in Phase:
-            log.info('Phase %s', p.name)
-            for c in self._concerns:
-                c.run_phase(p)
-
-        log.info('Shitty deploy complete!')

ominfra/manage/deploy/executor/concerns/dirs.py

@@ -1,28 +0,0 @@
-import os.path
-import pwd
-
-from omlish.lite.logs import log
-
-from ..base import Concern
-from ..base import Phase
-from ..base import run_in_phase
-
-
-class DirsConcern(Concern):
-    @run_in_phase(Phase.HOST)
-    def create_dirs(self) -> None:
-        pwn = pwd.getpwnam(self._d.host_cfg.username)
-
-        for dn in [
-            'app',
-            'conf',
-            'conf/env',
-            'conf/nginx',
-            'conf/supervisor',
-            'venv',
-        ]:
-            fp = os.path.join(self._d.home_dir(), dn)
-            if not os.path.exists(fp):
-                log.info('Creating directory: %s', fp)
-                os.mkdir(fp)
-                os.chown(fp, pwn.pw_uid, pwn.pw_gid)

ominfra/manage/deploy/executor/concerns/nginx.py

@@ -1,47 +0,0 @@
-"""
-TODO:
- - https://stackoverflow.com/questions/3011067/restart-nginx-without-sudo
-"""
-import os.path
-import textwrap
-
-from omlish.lite.logs import log
-
-from ..base import Concern
-from ..base import Phase
-from ..base import run_in_phase
-
-
-class GlobalNginxConcern(Concern):
-    @run_in_phase(Phase.HOST)
-    def create_global_nginx_conf(self) -> None:
-        nginx_conf_dir = os.path.join(self._d.home_dir(), 'conf/nginx')
-        if not os.path.isfile(self._d.host_cfg.global_nginx_conf_file_path):
-            log.info('Writing global nginx conf at %s', self._d.host_cfg.global_nginx_conf_file_path)
-            with open(self._d.host_cfg.global_nginx_conf_file_path, 'w') as f:
-                f.write(f'include {nginx_conf_dir}/*.conf;\n')
-
-
-class NginxConcern(Concern):
-    @run_in_phase(Phase.FRONTEND)
-    def create_nginx_conf(self) -> None:
-        nginx_conf = textwrap.dedent(f"""
-            server {{
-                listen 80;
-                location / {{
-                    proxy_pass http://127.0.0.1:8000/;
-                }}
-            }}
-        """)
-        nginx_conf_file = os.path.join(self._d.home_dir(), f'conf/nginx/{self._d.cfg.app_name}.conf')
-        log.info('Writing nginx conf to %s', nginx_conf_file)
-        with open(nginx_conf_file, 'w') as f:
-            f.write(nginx_conf)
-
-    @run_in_phase(Phase.START_FRONTEND)
-    def poke_nginx(self) -> None:
-        log.info('Starting nginx')
-        self._d.sh('service nginx start')
-
-        log.info('Poking nginx')
-        self._d.sh('nginx -s reload')

ominfra/manage/deploy/executor/concerns/repo.py

@@ -1,17 +0,0 @@
-from ..base import Concern
-from ..base import Phase
-from ..base import run_in_phase
-
-
-class RepoConcern(Concern):
-    @run_in_phase(Phase.ENV)
-    def clone_repo(self) -> None:
-        clone_submodules = False
-        self._d.ush(
-            'cd ~/app',
-            f'git clone --depth 1 {self._d.cfg.repo_url} {self._d.cfg.app_name}',
-            *([
-                f'cd {self._d.cfg.app_name}',
-                'git submodule update --init',
-            ] if clone_submodules else []),
-        )

ominfra/manage/deploy/executor/concerns/supervisor.py

@@ -1,46 +0,0 @@
-import os.path
-import textwrap
-
-from omlish.lite.logs import log
-
-from ..base import Concern
-from ..base import Phase
-from ..base import run_in_phase
-
-
-class GlobalSupervisorConcern(Concern):
-    @run_in_phase(Phase.HOST)
-    def create_global_supervisor_conf(self) -> None:
-        sup_conf_dir = os.path.join(self._d.home_dir(), 'conf/supervisor')
-        with open(self._d.host_cfg.global_supervisor_conf_file_path) as f:
-            glo_sup_conf = f.read()
-        if sup_conf_dir not in glo_sup_conf:
-            log.info('Updating global supervisor conf at %s', self._d.host_cfg.global_supervisor_conf_file_path)  # noqa
-            glo_sup_conf += textwrap.dedent(f"""
-                [include]
-                files = {self._d.home_dir()}/conf/supervisor/*.conf
-            """)
-            with open(self._d.host_cfg.global_supervisor_conf_file_path, 'w') as f:
-                f.write(glo_sup_conf)
-
-
-class SupervisorConcern(Concern):
-    @run_in_phase(Phase.BACKEND)
-    def create_supervisor_conf(self) -> None:
-        sup_conf = textwrap.dedent(f"""
-            [program:{self._d.cfg.app_name}]
-            command={self._d.home_dir()}/venv/{self._d.cfg.app_name}/bin/python -m {self._d.cfg.entrypoint}
-            directory={self._d.home_dir()}/app/{self._d.cfg.app_name}
-            user={self._d.host_cfg.username}
-            autostart=true
-            autorestart=true
-        """)
-        sup_conf_file = os.path.join(self._d.home_dir(), f'conf/supervisor/{self._d.cfg.app_name}.conf')
-        log.info('Writing supervisor conf to %s', sup_conf_file)
-        with open(sup_conf_file, 'w') as f:
-            f.write(sup_conf)
-
-    @run_in_phase(Phase.START_BACKEND)
-    def poke_supervisor(self) -> None:
-        log.info('Poking supervisor')
-        self._d.sh('kill -HUP 1')

ominfra/manage/deploy/executor/concerns/systemd.py

@@ -1,88 +0,0 @@
-"""
-# https://serverfault.com/questions/617823/how-to-set-systemd-service-dependencies
-PIDFile=/run/nginx.pid
-ExecStartPre=/usr/sbin/nginx -t
-ExecStart=/usr/sbin/nginx
-ExecReload=/bin/kill -s HUP $MAINPID
-ExecStop=/bin/kill -s QUIT $MAINPID
-PrivateTmp=true
-
-# https://gist.github.com/clemensg/7dd024169efe8ce6e7fa4a0b3caa3780
-Type=forking
-PIDFile=/var/run/nginx.pid
-ExecStartPre=/usr/sbin/nginx -t
-ExecStart=/usr/sbin/nginx
-ExecReload=/usr/bin/kill -s HUP $MAINPID
-ExecStop=/usr/bin/kill -s QUIT $MAINPID
-# Hardening
-InaccessiblePaths=/etc/gnupg /etc/shadow /etc/ssh
-ProtectSystem=full
-ProtectKernelTunables=yes
-ProtectControlGroups=yes
-SystemCallFilter=~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io
-MemoryDenyWriteExecute=yes
-RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
-RestrictRealtime=yes
-"""
-import os.path
-import textwrap
-
-from omlish.lite.logs import log
-
-from ..base import Concern
-from ..base import Phase
-from ..base import run_in_phase
-
-
-class GlobalSystemdConcern(Concern):
-    @run_in_phase(Phase.HOST)
-    def enable_user_linger(self) -> None:
-        log.info('Enabling user linger')
-        self._d.sh(f'loginctl enable-linger {self._d.host_cfg.username}')
-
-
-class SystemdConcern(Concern):
-    service_name: str
-
-    @run_in_phase(Phase.HOST)
-    def create_systemd_path(self) -> None:
-        sd_svc_dir = os.path.join(self._d.home_dir(), '.config/systemd/user')
-        if not os.path.exists(sd_svc_dir):
-            log.info('Creating directory: %s', sd_svc_dir)
-            os.makedirs(sd_svc_dir)
-
-    @run_in_phase(Phase.BACKEND)
-    def create_systemd_service(self) -> None:
-        sd_svc = textwrap.dedent(f"""
-            [Unit]
-            Description={self.service_name}
-            After= \
-                syslog.target \
-                network.target \
-                remote-fs.target \
-                nss-lookup.target \
-                network-online.target
-
-            [Service]
-            Type=simple
-            StandardOutput=journal
-            ExecStart={self._d.home_dir()}/venv/{self._d.cfg.app_name}/bin/python -m {self._d.cfg.entrypoint}
-            WorkingDirectory={self._d.home_dir()}/app/{self._d.cfg.app_name}
-
-            Restart=always
-            RestartSec=3
-
-            [Install]
-            WantedBy=multi-user.target
-        """)
-        sd_svc_file = os.path.join(self._d.home_dir(), f'.config/systemd/user/{self.service_name}.service')
-        log.info('Writing systemd service to %s', sd_svc_file)
-        with open(sd_svc_file, 'w') as f:
-            f.write(sd_svc)
-
-    @run_in_phase(Phase.START_BACKEND)
-    def poke_systemd(self) -> None:
-        log.info('Poking systemd')
-        self._d.sh('systemctl --user daemon-reload')
-        self._d.sh(f'systemctl --user enable {self.service_name}')
-        self._d.sh(f'systemctl --user restart {self.service_name}')

ominfra/manage/deploy/executor/concerns/user.py

@@ -1,25 +0,0 @@
-import pwd
-
-from omlish.lite.logs import log
-
-from ..base import Concern
-from ..base import Phase
-from ..base import run_in_phase
-
-
-class UserConcern(Concern):
-    @run_in_phase(Phase.HOST)
-    def create_user(self) -> None:
-        try:
-            pwd.getpwnam(self._d.host_cfg.username)
-        except KeyError:
-            log.info('Creating user %s', self._d.host_cfg.username)
-            self._d.sh(' '.join([
-                'adduser',
-                '--system',
-                '--disabled-password',
-                '--group',
-                '--shell /bin/bash',
-                self._d.host_cfg.username,
-            ]))
-            pwd.getpwnam(self._d.host_cfg.username)

ominfra/manage/deploy/executor/concerns/venv.py

@@ -1,22 +0,0 @@
-"""
-TODO:
- - use LinuxInterpResolver lol
-"""
-from ..base import Concern
-from ..base import Phase
-from ..base import run_in_phase
-
-
-class VenvConcern(Concern):
-    @run_in_phase(Phase.ENV)
-    def setup_venv(self) -> None:
-        self._d.ush(
-            'cd ~/venv',
-            f'{self._d.cfg.python_bin} -mvenv {self._d.cfg.app_name}',
-
-            # https://stackoverflow.com/questions/77364550/attributeerror-module-pkgutil-has-no-attribute-impimporter-did-you-mean
-            f'{self._d.cfg.app_name}/bin/python -m ensurepip',
-            f'{self._d.cfg.app_name}/bin/python -mpip install --upgrade setuptools pip',
-
-            f'{self._d.cfg.app_name}/bin/python -mpip install -r ~deploy/app/{self._d.cfg.app_name}/{self._d.cfg.requirements_txt}',  # noqa
-        )

ominfra/manage/deploy/executor/main.py

@@ -1,119 +0,0 @@
-#!/usr/bin/env python3
-# @omlish-amalg ../_executor.py
-r"""
-TODO:
- - flock
- - interp.py
- - systemd
-
-deployment matrix
- - os: ubuntu / amzn / generic
- - arch: amd64 / arm64
- - host: bare / docker
- - init: supervisor-provided / supervisor-must-configure / systemd (/ self?)
- - interp: system / pyenv / interp.py
- - venv: none / yes
- - nginx: no / provided / must-configure
-
-==
-
-~deploy
-  deploy.pid (flock)
-  /app
-    /<appspec> - shallow clone
-  /conf
-    /env
-      <appspec>.env
-    /nginx
-      <appspec>.conf
-    /supervisor
-      <appspec>.conf
-  /venv
-    /<appspec>
-
-?
-  /logs
-    /wrmsr--omlish--<spec>
-
-spec = <name>--<rev>--<when>
-
-https://docs.docker.com/config/containers/multi-service_container/#use-a-process-manager
-https://serverfault.com/questions/211525/supervisor-not-loading-new-configuration-files
-"""  # noqa
-# ruff: noqa: UP007
-import argparse
-import json
-import sys
-import typing as ta
-
-from omlish.lite.logs import configure_standard_logging
-from omlish.lite.marshal import unmarshal_obj
-from omlish.lite.runtime import check_runtime_version
-
-from ..configs import DeployConfig
-from .base import Deployment
-from .concerns.dirs import DirsConcern
-from .concerns.nginx import GlobalNginxConcern
-from .concerns.nginx import NginxConcern
-from .concerns.repo import RepoConcern
-from .concerns.supervisor import GlobalSupervisorConcern
-from .concerns.supervisor import SupervisorConcern
-from .concerns.user import UserConcern
-from .concerns.venv import VenvConcern
-
-
-##
-
-
-def _deploy_cmd(args) -> None:
-    dct = json.loads(args.cfg)
-    cfg: DeployConfig = unmarshal_obj(dct, DeployConfig)
-    dp = Deployment(
-        cfg,
-        [
-            UserConcern,
-            DirsConcern,
-            GlobalNginxConcern,
-            GlobalSupervisorConcern,
-            RepoConcern,
-            VenvConcern,
-            SupervisorConcern,
-            NginxConcern,
-        ],
-    )
-    dp.deploy()
-
-
-##
-
-
-def _build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser()
-
-    subparsers = parser.add_subparsers()
-
-    parser_resolve = subparsers.add_parser('deploy')
-    parser_resolve.add_argument('cfg')
-    parser_resolve.set_defaults(func=_deploy_cmd)
-
-    return parser
-
-
-def _main(argv: ta.Optional[ta.Sequence[str]] = None) -> None:
-    check_runtime_version()
-
-    if getattr(sys, 'platform') != 'linux':  # noqa
-        raise OSError('must run on linux')
-
-    configure_standard_logging()
-
-    parser = _build_parser()
-    args = parser.parse_args(argv)
-    if not getattr(args, 'func', None):
-        parser.print_help()
-    else:
-        args.func(args)
-
-
-if __name__ == '__main__':
-    _main()

ominfra/manage/deploy/poly/__init__.py

@@ -1 +0,0 @@
-# @omlish-lite