ominfra 0.0.0.dev166__py3-none-any.whl → 0.0.0.dev168__py3-none-any.whl

ominfra/manage/deploy/paths/paths.py

@@ -0,0 +1,216 @@
+# ruff: noqa: UP006 UP007
+"""
+TODO:
+ - run/{.pid,.sock}
+ - logs/...
+ - current symlink
+ - conf/{nginx,supervisor}
+ - env/?
+ - apps/<app>/shared
+"""
+import abc
+import dataclasses as dc
+import itertools
+import typing as ta
+
+from omlish.lite.check import check
+from omlish.lite.strings import split_keep_delimiter
+
+from ..types import DeployPathKind
+from ..types import DeployPathPlaceholder
+
+
+##
+
+
+DEPLOY_PATH_PLACEHOLDER_SIGIL = '@'
+DEPLOY_PATH_PLACEHOLDER_SEPARATOR = '--'
+
+DEPLOY_PATH_PLACEHOLDER_DELIMITERS: ta.AbstractSet[str] = frozenset([
+    DEPLOY_PATH_PLACEHOLDER_SEPARATOR,
+    '.',
+])
+
+DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
+    'app',
+    'tag',
+    'conf',
+])
+
+
+class DeployPathError(Exception):
+    pass
+
+
+class DeployPathRenderable(abc.ABC):
+    @abc.abstractmethod
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+        raise NotImplementedError
+
+
+##
+
+
+class DeployPathNamePart(DeployPathRenderable, abc.ABC):
+    @classmethod
+    def parse(cls, s: str) -> 'DeployPathNamePart':
+        check.non_empty_str(s)
+        if s.startswith(DEPLOY_PATH_PLACEHOLDER_SIGIL):
+            return PlaceholderDeployPathNamePart(s[1:])
+        elif s in DEPLOY_PATH_PLACEHOLDER_DELIMITERS:
+            return DelimiterDeployPathNamePart(s)
+        else:
+            return ConstDeployPathNamePart(s)
+
+
+@dc.dataclass(frozen=True)
+class PlaceholderDeployPathNamePart(DeployPathNamePart):
+    placeholder: str  # DeployPathPlaceholder
+
+    def __post_init__(self) -> None:
+        check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
+
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+        if placeholders is not None:
+            return placeholders[self.placeholder]  # type: ignore
+        else:
+            return DEPLOY_PATH_PLACEHOLDER_SIGIL + self.placeholder
+
+
+@dc.dataclass(frozen=True)
+class DelimiterDeployPathNamePart(DeployPathNamePart):
+    delimiter: str
+
+    def __post_init__(self) -> None:
+        check.in_(self.delimiter, DEPLOY_PATH_PLACEHOLDER_DELIMITERS)
+
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+        return self.delimiter
+
+
+@dc.dataclass(frozen=True)
+class ConstDeployPathNamePart(DeployPathNamePart):
+    const: str
+
+    def __post_init__(self) -> None:
+        check.non_empty_str(self.const)
+        for c in [*DEPLOY_PATH_PLACEHOLDER_DELIMITERS, DEPLOY_PATH_PLACEHOLDER_SIGIL, '/']:
+            check.not_in(c, self.const)
+
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+        return self.const
+
+
+@dc.dataclass(frozen=True)
+class DeployPathName(DeployPathRenderable):
+    parts: ta.Sequence[DeployPathNamePart]
+
+    def __post_init__(self) -> None:
+        hash(self)
+        check.not_empty(self.parts)
+        for k, g in itertools.groupby(self.parts, type):
+            if len(gl := list(g)) > 1:
+                raise DeployPathError(f'May not have consecutive path name part types: {k} {gl}')
+
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+        return ''.join(p.render(placeholders) for p in self.parts)
+
+    @classmethod
+    def parse(cls, s: str) -> 'DeployPathName':
+        check.non_empty_str(s)
+        check.not_in('/', s)
+
+        i = 0
+        ps = []
+        while i < len(s):
+            ns = [(n, d) for d in DEPLOY_PATH_PLACEHOLDER_DELIMITERS if (n := s.find(d, i)) >= 0]
+            if not ns:
+                ps.append(s[i:])
+                break
+            n, d = min(ns)
+            ps.append(check.non_empty_str(s[i:n]))
+            ps.append(s[n:n + len(d)])
+            i = n + len(d)
+
+        return cls(tuple(DeployPathNamePart.parse(p) for p in ps))
+
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployPathPart(DeployPathRenderable, abc.ABC):  # noqa
+    name: DeployPathName
+
+    @property
+    @abc.abstractmethod
+    def kind(self) -> DeployPathKind:
+        raise NotImplementedError
+
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+        return self.name.render(placeholders) + ('/' if self.kind == 'dir' else '')
+
+    @classmethod
+    def parse(cls, s: str) -> 'DeployPathPart':
+        if (is_dir := s.endswith('/')):
+            s = s[:-1]
+        check.non_empty_str(s)
+        check.not_in('/', s)
+
+        n = DeployPathName.parse(s)
+        if is_dir:
+            return DirDeployPathPart(n)
+        else:
+            return FileDeployPathPart(n)
+
+
+class DirDeployPathPart(DeployPathPart):
+    @property
+    def kind(self) -> DeployPathKind:
+        return 'dir'
+
+
+class FileDeployPathPart(DeployPathPart):
+    @property
+    def kind(self) -> DeployPathKind:
+        return 'file'
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class DeployPath:
+    parts: ta.Sequence[DeployPathPart]
+
+    @property
+    def name_parts(self) -> ta.Iterator[DeployPathNamePart]:
+        for p in self.parts:
+            yield from p.name.parts
+
+    def __post_init__(self) -> None:
+        hash(self)
+        check.not_empty(self.parts)
+        for p in self.parts[:-1]:
+            check.equal(p.kind, 'dir')
+
+        pd: ta.Dict[DeployPathPlaceholder, ta.List[int]] = {}
+        for i, np in enumerate(self.name_parts):
+            if isinstance(np, PlaceholderDeployPathNamePart):
+                pd.setdefault(ta.cast(DeployPathPlaceholder, np.placeholder), []).append(i)
+
+        # if 'tag' in pd and 'app' not in pd:
+        #     raise DeployPathError('Tag placeholder in path without app', self)
+
+    @property
+    def kind(self) -> ta.Literal['file', 'dir']:
+        return self.parts[-1].kind
+
+    def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
+        return ''.join([p.render(placeholders) for p in self.parts])
+
+    @classmethod
+    def parse(cls, s: str) -> 'DeployPath':
+        check.non_empty_str(s)
+        ps = split_keep_delimiter(s, '/')
+        return cls(tuple(DeployPathPart.parse(p) for p in ps))

ominfra/manage/deploy/specs.py

@@ -1,4 +1,5 @@
 # ruff: noqa: UP006 UP007
+import abc
 import dataclasses as dc
 import hashlib
 import typing as ta
@@ -14,6 +15,17 @@ from .types import DeployRev
 ##
 
 
+def check_valid_deploy_spec_path(s: str) -> str:
+    check.non_empty_str(s)
+    for c in ['..', '//']:
+        check.not_in(c, s)
+    check.arg(not s.startswith('/'))
+    return s
+
+
+##
+
+
 @dc.dataclass(frozen=True)
 class DeployGitRepo:
     host: ta.Optional[str] = None
@@ -26,14 +38,13 @@ class DeployGitRepo:
 
 
 @dc.dataclass(frozen=True)
-class DeployGitCheckout:
+class DeployGitSpec:
     repo: DeployGitRepo
     rev: DeployRev
 
     subtrees: ta.Optional[ta.Sequence[str]] = None
 
     def __post_init__(self) -> None:
-        hash(self)
         check.non_empty_str(self.rev)
         if self.subtrees is not None:
             for st in self.subtrees:
@@ -52,8 +63,62 @@ class DeployVenvSpec:
 
     use_uv: bool = False
 
+
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployConfFile:
+    path: str
+    body: str
+
     def __post_init__(self) -> None:
-        hash(self)
+        check_valid_deploy_spec_path(self.path)
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class DeployConfLink(abc.ABC):  # noqa
+    """
+    May be either:
+     - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
+     - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
+     - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
+    """
+
+    src: str
+
+    def __post_init__(self) -> None:
+        check_valid_deploy_spec_path(self.src)
+        if '/' in self.src:
+            check.equal(self.src.count('/'), 1)
+
+
+class AppDeployConfLink(DeployConfLink):
+    pass
+
+
+class TagDeployConfLink(DeployConfLink):
+    pass
+
+
+#
+
+
+@dc.dataclass(frozen=True)
+class DeployConfSpec:
+    files: ta.Optional[ta.Sequence[DeployConfFile]] = None
+
+    links: ta.Optional[ta.Sequence[DeployConfLink]] = None
+
+    def __post_init__(self) -> None:
+        if self.files:
+            seen: ta.Set[str] = set()
+            for f in self.files:
+                check.not_in(f.path, seen)
+                seen.add(f.path)
 
 
 ##
@@ -62,12 +127,12 @@ class DeployVenvSpec:
 @dc.dataclass(frozen=True)
 class DeploySpec:
     app: DeployApp
-    checkout: DeployGitCheckout
+
+    git: DeployGitSpec
 
     venv: ta.Optional[DeployVenvSpec] = None
 
-    def __post_init__(self) -> None:
-        hash(self)
+    conf: ta.Optional[DeployConfSpec] = None
 
     @cached_nullary
     def key(self) -> DeployKey:

ominfra/manage/deploy/tmp.py

@@ -8,7 +8,7 @@ from omlish.os.atomics import AtomicPathSwapKind
 from omlish.os.atomics import AtomicPathSwapping
 from omlish.os.atomics import TempDirAtomicPathSwapping
 
-from .paths import SingleDirDeployPathOwner
+from .paths.owners import SingleDirDeployPathOwner
 from .types import DeployHome
 
 

ominfra/manage/deploy/types.py

@@ -1,5 +1,15 @@
+import dataclasses as dc
 import typing as ta
 
+from omlish.lite.check import check
+
+
+DeployPathKind = ta.Literal['dir', 'file']  # ta.TypeAlias
+DeployPathPlaceholder = ta.Literal['app', 'tag', 'conf']  # ta.TypeAlias
+
+
+##
+
 
 DeployHome = ta.NewType('DeployHome', str)
 
@@ -9,6 +19,21 @@ DeployRev = ta.NewType('DeployRev', str)
 DeployKey = ta.NewType('DeployKey', str)
 
 
-class DeployAppTag(ta.NamedTuple):
+##
+
+
+@dc.dataclass(frozen=True)
+class DeployAppTag:
     app: DeployApp
     tag: DeployTag
+
+    def __post_init__(self) -> None:
+        for s in [self.app, self.tag]:
+            check.non_empty_str(s)
+            check.equal(s, s.strip())
+
+    def placeholders(self) -> ta.Mapping[DeployPathPlaceholder, str]:
+        return {
+            'app': self.app,
+            'tag': self.tag,
+        }

ominfra/manage/deploy/venvs.py

@@ -5,46 +5,28 @@ TODO:
  - share more code with pyproject?
 """
 import os.path
-import typing as ta
 
 from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
-from omlish.lite.cached import cached_nullary
-from omlish.lite.check import check
 from omlish.os.atomics import AtomicPathSwapping
 
-from .paths import DeployPath
-from .paths import DeployPathOwner
 from .specs import DeployVenvSpec
-from .types import DeployAppTag
-from .types import DeployHome
 
 
-class DeployVenvManager(DeployPathOwner):
+class DeployVenvManager:
     def __init__(
             self,
             *,
-            deploy_home: ta.Optional[DeployHome] = None,
             atomics: AtomicPathSwapping,
     ) -> None:
         super().__init__()
 
-        self._deploy_home = deploy_home
         self._atomics = atomics
 
-    @cached_nullary
-    def _dir(self) -> str:
-        return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
-
-    def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
-        return {
-            DeployPath.parse('venvs/@app/@tag/'),
-        }
-
     async def setup_venv(
             self,
-            app_dir: str,
-            venv_dir: str,
             spec: DeployVenvSpec,
+            git_dir: str,
+            venv_dir: str,
     ) -> None:
         sys_exe = 'python3'
 
@@ -58,7 +40,7 @@ class DeployVenvManager(DeployPathOwner):
 
         #
 
-        reqs_txt = os.path.join(app_dir, 'requirements.txt')
+        reqs_txt = os.path.join(git_dir, 'requirements.txt')
 
         if os.path.isfile(reqs_txt):
             if spec.use_uv:
@@ -68,14 +50,3 @@ class DeployVenvManager(DeployPathOwner):
                 pip_cmd = ['-m', 'pip']
 
             await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
-
-    async def setup_app_venv(
-            self,
-            app_tag: DeployAppTag,
-            spec: DeployVenvSpec,
-    ) -> None:
-        await self.setup_venv(
-            os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
-            os.path.join(self._dir(), app_tag.app, app_tag.tag),
-            spec,
-        )

ominfra/scripts/journald2aws.py

@@ -961,6 +961,8 @@ def async_cached_nullary(fn):  # ta.Callable[..., T]) -> ta.Callable[..., T]:
 """
 TODO:
  - def maybe(v: lang.Maybe[T])
+ - def not_ ?
+ - ** class @dataclass Raise - user message should be able to be an exception type or instance or factory
 """
 
 
@@ -1527,6 +1529,37 @@ def snake_case(name: str) -> str:
 ##
 
 
+def strip_with_newline(s: str) -> str:
+    if not s:
+        return ''
+    return s.strip() + '\n'
+
+
+@ta.overload
+def split_keep_delimiter(s: str, d: str) -> str:
+    ...
+
+
+@ta.overload
+def split_keep_delimiter(s: bytes, d: bytes) -> bytes:
+    ...
+
+
+def split_keep_delimiter(s, d):
+    ps = []
+    i = 0
+    while i < len(s):
+        if (n := s.find(d, i)) < i:
+            ps.append(s[i:])
+            break
+        ps.append(s[i:n + 1])
+        i = n + 1
+    return ps
+
+
+##
+
+
 def is_dunder(name: str) -> bool:
     return (
         name[:2] == name[-2:] == '__' and