ominfra 0.0.0.dev166__py3-none-any.whl → 0.0.0.dev167__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/manage/deploy/apps.py +57 -6
- ominfra/manage/deploy/conf.py +143 -0
- ominfra/manage/deploy/git.py +11 -7
- ominfra/manage/deploy/inject.py +32 -7
- ominfra/manage/deploy/paths.py +5 -1
- ominfra/manage/deploy/specs.py +71 -5
- ominfra/manage/deploy/venvs.py +4 -33
- ominfra/scripts/journald2aws.py +9 -0
- ominfra/scripts/manage.py +348 -54
- ominfra/scripts/supervisor.py +10 -0
- ominfra/supervisor/configs.py +1 -0
- {ominfra-0.0.0.dev166.dist-info → ominfra-0.0.0.dev167.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev166.dist-info → ominfra-0.0.0.dev167.dist-info}/RECORD +17 -16
- {ominfra-0.0.0.dev166.dist-info → ominfra-0.0.0.dev167.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev166.dist-info → ominfra-0.0.0.dev167.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev166.dist-info → ominfra-0.0.0.dev167.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev166.dist-info → ominfra-0.0.0.dev167.dist-info}/top_level.txt +0 -0
ominfra/manage/deploy/apps.py
CHANGED
@@ -5,7 +5,9 @@ import typing as ta
|
|
5
5
|
|
6
6
|
from omlish.lite.cached import cached_nullary
|
7
7
|
from omlish.lite.check import check
|
8
|
+
from omlish.os.paths import relative_symlink
|
8
9
|
|
10
|
+
from .conf import DeployConfManager
|
9
11
|
from .git import DeployGitManager
|
10
12
|
from .paths import DeployPath
|
11
13
|
from .paths import DeployPathOwner
|
@@ -36,12 +38,16 @@ class DeployAppManager(DeployPathOwner):
|
|
36
38
|
self,
|
37
39
|
*,
|
38
40
|
deploy_home: ta.Optional[DeployHome] = None,
|
41
|
+
|
42
|
+
conf: DeployConfManager,
|
39
43
|
git: DeployGitManager,
|
40
44
|
venvs: DeployVenvManager,
|
41
45
|
) -> None:
|
42
46
|
super().__init__()
|
43
47
|
|
44
48
|
self._deploy_home = deploy_home
|
49
|
+
|
50
|
+
self._conf = conf
|
45
51
|
self._git = git
|
46
52
|
self._venvs = venvs
|
47
53
|
|
@@ -51,24 +57,69 @@ class DeployAppManager(DeployPathOwner):
|
|
51
57
|
|
52
58
|
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
53
59
|
return {
|
54
|
-
DeployPath.parse('apps/@app
|
60
|
+
DeployPath.parse('apps/@app/current'),
|
61
|
+
DeployPath.parse('apps/@app/deploying'),
|
62
|
+
|
63
|
+
DeployPath.parse('apps/@app/tags/@tag/conf/'),
|
64
|
+
DeployPath.parse('apps/@app/tags/@tag/git/'),
|
65
|
+
DeployPath.parse('apps/@app/tags/@tag/venv/'),
|
55
66
|
}
|
56
67
|
|
57
68
|
async def prepare_app(
|
58
69
|
self,
|
59
70
|
spec: DeploySpec,
|
60
71
|
) -> None:
|
61
|
-
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.
|
62
|
-
|
72
|
+
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.git.rev, spec.key()))
|
73
|
+
|
74
|
+
#
|
75
|
+
|
76
|
+
app_dir = os.path.join(self._dir(), spec.app)
|
77
|
+
os.makedirs(app_dir, exist_ok=True)
|
63
78
|
|
64
79
|
#
|
65
80
|
|
81
|
+
tag_dir = os.path.join(app_dir, 'tags', app_tag.tag)
|
82
|
+
os.makedirs(tag_dir)
|
83
|
+
|
84
|
+
#
|
85
|
+
|
86
|
+
deploying_file = os.path.join(app_dir, 'deploying')
|
87
|
+
current_file = os.path.join(app_dir, 'current')
|
88
|
+
|
89
|
+
if os.path.exists(deploying_file):
|
90
|
+
os.unlink(deploying_file)
|
91
|
+
relative_symlink(tag_dir, deploying_file, target_is_directory=True)
|
92
|
+
|
93
|
+
#
|
94
|
+
|
95
|
+
git_dir = os.path.join(tag_dir, 'git')
|
66
96
|
await self._git.checkout(
|
67
|
-
spec.
|
68
|
-
|
97
|
+
spec.git,
|
98
|
+
git_dir,
|
69
99
|
)
|
70
100
|
|
71
101
|
#
|
72
102
|
|
73
103
|
if spec.venv is not None:
|
74
|
-
|
104
|
+
venv_dir = os.path.join(tag_dir, 'venv')
|
105
|
+
await self._venvs.setup_venv(
|
106
|
+
spec.venv,
|
107
|
+
git_dir,
|
108
|
+
venv_dir,
|
109
|
+
)
|
110
|
+
|
111
|
+
#
|
112
|
+
|
113
|
+
if spec.conf is not None:
|
114
|
+
conf_dir = os.path.join(tag_dir, 'conf')
|
115
|
+
conf_link_dir = os.path.join(current_file, 'conf')
|
116
|
+
await self._conf.write_conf(
|
117
|
+
spec.conf,
|
118
|
+
conf_dir,
|
119
|
+
app_tag,
|
120
|
+
conf_link_dir,
|
121
|
+
)
|
122
|
+
|
123
|
+
#
|
124
|
+
|
125
|
+
os.replace(deploying_file, current_file)
|
@@ -0,0 +1,143 @@
|
|
1
|
+
# ruff: noqa: UP006 UP007
|
2
|
+
"""
|
3
|
+
TODO:
|
4
|
+
- @conf DeployPathPlaceholder? :|
|
5
|
+
- post-deploy: remove any dir_links not present in new spec
|
6
|
+
- * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
|
7
|
+
- no such thing as 'previously present'.. build a 'deploy state' and pass it back?
|
8
|
+
- ** whole thing can be atomic **
|
9
|
+
- 1) new atomic temp dir
|
10
|
+
- 2) for each subdir not needing modification, hardlink into temp dir
|
11
|
+
- 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
|
12
|
+
- 4) write (or if deleting, omit) new files
|
13
|
+
- 5) swap top level
|
14
|
+
- ** whole deploy can be atomic(-ish) - do this for everything **
|
15
|
+
- just a '/deploy/current' dir
|
16
|
+
- some things (venvs) cannot be moved, thus the /deploy/venvs dir
|
17
|
+
- ** ensure (enforce) equivalent relpath nesting
|
18
|
+
"""
|
19
|
+
import os.path
|
20
|
+
import typing as ta
|
21
|
+
|
22
|
+
from omlish.lite.check import check
|
23
|
+
from omlish.os.paths import is_path_in_dir
|
24
|
+
from omlish.os.paths import relative_symlink
|
25
|
+
|
26
|
+
from .paths import SingleDirDeployPathOwner
|
27
|
+
from .specs import AppDeployConfLink
|
28
|
+
from .specs import DeployConfFile
|
29
|
+
from .specs import DeployConfLink
|
30
|
+
from .specs import DeployConfSpec
|
31
|
+
from .specs import TagDeployConfLink
|
32
|
+
from .types import DeployAppTag
|
33
|
+
from .types import DeployHome
|
34
|
+
|
35
|
+
|
36
|
+
class DeployConfManager(SingleDirDeployPathOwner):
|
37
|
+
def __init__(
|
38
|
+
self,
|
39
|
+
*,
|
40
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
41
|
+
) -> None:
|
42
|
+
super().__init__(
|
43
|
+
owned_dir='conf',
|
44
|
+
deploy_home=deploy_home,
|
45
|
+
)
|
46
|
+
|
47
|
+
async def _write_conf_file(
|
48
|
+
self,
|
49
|
+
cf: DeployConfFile,
|
50
|
+
conf_dir: str,
|
51
|
+
) -> None:
|
52
|
+
conf_file = os.path.join(conf_dir, cf.path)
|
53
|
+
check.arg(is_path_in_dir(conf_dir, conf_file))
|
54
|
+
|
55
|
+
os.makedirs(os.path.dirname(conf_file), exist_ok=True)
|
56
|
+
|
57
|
+
with open(conf_file, 'w') as f: # noqa
|
58
|
+
f.write(cf.body)
|
59
|
+
|
60
|
+
async def _make_conf_link(
|
61
|
+
self,
|
62
|
+
link: DeployConfLink,
|
63
|
+
conf_dir: str,
|
64
|
+
app_tag: DeployAppTag,
|
65
|
+
link_dir: str,
|
66
|
+
) -> None:
|
67
|
+
link_src = os.path.join(conf_dir, link.src)
|
68
|
+
check.arg(is_path_in_dir(conf_dir, link_src))
|
69
|
+
|
70
|
+
is_link_dir = link.src.endswith('/')
|
71
|
+
if is_link_dir:
|
72
|
+
check.arg(link.src.count('/') == 1)
|
73
|
+
check.arg(os.path.isdir(link_src))
|
74
|
+
link_dst_pfx = link.src
|
75
|
+
link_dst_sfx = ''
|
76
|
+
|
77
|
+
elif '/' in link.src:
|
78
|
+
check.arg(os.path.isfile(link_src))
|
79
|
+
d, f = os.path.split(link.src)
|
80
|
+
# TODO: check filename :|
|
81
|
+
link_dst_pfx = d + '/'
|
82
|
+
link_dst_sfx = '-' + f
|
83
|
+
|
84
|
+
else:
|
85
|
+
check.arg(os.path.isfile(link_src))
|
86
|
+
if '.' in link.src:
|
87
|
+
l, _, r = link.src.partition('.')
|
88
|
+
link_dst_pfx = l + '/'
|
89
|
+
link_dst_sfx = '.' + r
|
90
|
+
else:
|
91
|
+
link_dst_pfx = link.src + '/'
|
92
|
+
link_dst_sfx = ''
|
93
|
+
|
94
|
+
if isinstance(link, AppDeployConfLink):
|
95
|
+
link_dst_mid = str(app_tag.app)
|
96
|
+
sym_root = link_dir
|
97
|
+
elif isinstance(link, TagDeployConfLink):
|
98
|
+
link_dst_mid = '-'.join([app_tag.app, app_tag.tag])
|
99
|
+
sym_root = conf_dir
|
100
|
+
else:
|
101
|
+
raise TypeError(link)
|
102
|
+
|
103
|
+
link_dst = ''.join([
|
104
|
+
link_dst_pfx,
|
105
|
+
link_dst_mid,
|
106
|
+
link_dst_sfx,
|
107
|
+
])
|
108
|
+
|
109
|
+
root_conf_dir = self._make_dir()
|
110
|
+
sym_src = os.path.join(sym_root, link.src)
|
111
|
+
sym_dst = os.path.join(root_conf_dir, link_dst)
|
112
|
+
check.arg(is_path_in_dir(root_conf_dir, sym_dst))
|
113
|
+
|
114
|
+
os.makedirs(os.path.dirname(sym_dst), exist_ok=True)
|
115
|
+
relative_symlink(sym_src, sym_dst, target_is_directory=is_link_dir)
|
116
|
+
|
117
|
+
async def write_conf(
|
118
|
+
self,
|
119
|
+
spec: DeployConfSpec,
|
120
|
+
conf_dir: str,
|
121
|
+
app_tag: DeployAppTag,
|
122
|
+
link_dir: str,
|
123
|
+
) -> None:
|
124
|
+
conf_dir = os.path.abspath(conf_dir)
|
125
|
+
os.makedirs(conf_dir)
|
126
|
+
|
127
|
+
#
|
128
|
+
|
129
|
+
for cf in spec.files or []:
|
130
|
+
await self._write_conf_file(
|
131
|
+
cf,
|
132
|
+
conf_dir,
|
133
|
+
)
|
134
|
+
|
135
|
+
#
|
136
|
+
|
137
|
+
for link in spec.links or []:
|
138
|
+
await self._make_conf_link(
|
139
|
+
link,
|
140
|
+
conf_dir,
|
141
|
+
app_tag,
|
142
|
+
link_dir,
|
143
|
+
)
|
ominfra/manage/deploy/git.py
CHANGED
@@ -18,8 +18,8 @@ from omlish.lite.check import check
|
|
18
18
|
from omlish.os.atomics import AtomicPathSwapping
|
19
19
|
|
20
20
|
from .paths import SingleDirDeployPathOwner
|
21
|
-
from .specs import DeployGitCheckout
|
22
21
|
from .specs import DeployGitRepo
|
22
|
+
from .specs import DeployGitSpec
|
23
23
|
from .types import DeployHome
|
24
24
|
from .types import DeployRev
|
25
25
|
|
@@ -95,7 +95,7 @@ class DeployGitManager(SingleDirDeployPathOwner):
|
|
95
95
|
|
96
96
|
#
|
97
97
|
|
98
|
-
async def checkout(self,
|
98
|
+
async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
|
99
99
|
check.state(not os.path.exists(dst_dir))
|
100
100
|
with self._git._atomics.begin_atomic_path_swap( # noqa
|
101
101
|
'dir',
|
@@ -103,14 +103,14 @@ class DeployGitManager(SingleDirDeployPathOwner):
|
|
103
103
|
auto_commit=True,
|
104
104
|
make_dirs=True,
|
105
105
|
) as dst_swap:
|
106
|
-
await self.fetch(
|
106
|
+
await self.fetch(spec.rev)
|
107
107
|
|
108
108
|
dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
|
109
109
|
await dst_call('git', 'init')
|
110
110
|
|
111
111
|
await dst_call('git', 'remote', 'add', 'local', self._dir)
|
112
|
-
await dst_call('git', 'fetch', '--depth=1', 'local',
|
113
|
-
await dst_call('git', 'checkout',
|
112
|
+
await dst_call('git', 'fetch', '--depth=1', 'local', spec.rev)
|
113
|
+
await dst_call('git', 'checkout', spec.rev, *(spec.subtrees or []))
|
114
114
|
|
115
115
|
def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
|
116
116
|
try:
|
@@ -119,5 +119,9 @@ class DeployGitManager(SingleDirDeployPathOwner):
|
|
119
119
|
repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
|
120
120
|
return repo_dir
|
121
121
|
|
122
|
-
async def checkout(
|
123
|
-
|
122
|
+
async def checkout(
|
123
|
+
self,
|
124
|
+
spec: DeployGitSpec,
|
125
|
+
dst_dir: str,
|
126
|
+
) -> None:
|
127
|
+
await self.get_repo_dir(spec.repo).checkout(spec, dst_dir)
|
ominfra/manage/deploy/inject.py
CHANGED
@@ -11,10 +11,13 @@ from ..commands.inject import bind_command
|
|
11
11
|
from .apps import DeployAppManager
|
12
12
|
from .commands import DeployCommand
|
13
13
|
from .commands import DeployCommandExecutor
|
14
|
+
from .conf import DeployConfManager
|
14
15
|
from .config import DeployConfig
|
15
16
|
from .git import DeployGitManager
|
16
17
|
from .interp import InterpCommand
|
17
18
|
from .interp import InterpCommandExecutor
|
19
|
+
from .paths import DeployPathOwner
|
20
|
+
from .paths import DeployPathOwners
|
18
21
|
from .tmp import DeployTmpManager
|
19
22
|
from .types import DeployHome
|
20
23
|
from .venvs import DeployVenvManager
|
@@ -26,23 +29,45 @@ def bind_deploy(
|
|
26
29
|
) -> InjectorBindings:
|
27
30
|
lst: ta.List[InjectorBindingOrBindings] = [
|
28
31
|
inj.bind(deploy_config),
|
32
|
+
]
|
33
|
+
|
34
|
+
#
|
35
|
+
|
36
|
+
def bind_manager(cls: type) -> InjectorBindings:
|
37
|
+
return inj.as_bindings(
|
38
|
+
inj.bind(cls, singleton=True),
|
39
|
+
|
40
|
+
*([inj.bind(DeployPathOwner, to_key=cls, array=True)] if issubclass(cls, DeployPathOwner) else []),
|
41
|
+
)
|
29
42
|
|
30
|
-
|
43
|
+
lst.extend([
|
44
|
+
inj.bind_array(DeployPathOwner),
|
45
|
+
inj.bind_array_type(DeployPathOwner, DeployPathOwners),
|
46
|
+
])
|
31
47
|
|
32
|
-
|
48
|
+
#
|
33
49
|
|
34
|
-
|
50
|
+
lst.extend([
|
51
|
+
bind_manager(DeployAppManager),
|
35
52
|
|
36
|
-
|
53
|
+
bind_manager(DeployConfManager),
|
54
|
+
|
55
|
+
bind_manager(DeployGitManager),
|
56
|
+
|
57
|
+
bind_manager(DeployTmpManager),
|
37
58
|
inj.bind(AtomicPathSwapping, to_key=DeployTmpManager),
|
38
59
|
|
39
|
-
|
60
|
+
bind_manager(DeployVenvManager),
|
61
|
+
])
|
40
62
|
|
41
|
-
|
63
|
+
#
|
42
64
|
|
65
|
+
lst.extend([
|
43
66
|
bind_command(DeployCommand, DeployCommandExecutor),
|
44
67
|
bind_command(InterpCommand, InterpCommandExecutor),
|
45
|
-
]
|
68
|
+
])
|
69
|
+
|
70
|
+
#
|
46
71
|
|
47
72
|
if (dh := deploy_config.deploy_home) is not None:
|
48
73
|
dh = os.path.abspath(os.path.expanduser(dh))
|
ominfra/manage/deploy/paths.py
CHANGED
@@ -1,11 +1,12 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
2
|
"""
|
3
3
|
TODO:
|
4
|
-
- run/
|
4
|
+
- run/{.pid,.sock}
|
5
5
|
- logs/...
|
6
6
|
- current symlink
|
7
7
|
- conf/{nginx,supervisor}
|
8
8
|
- env/?
|
9
|
+
- apps/<app>/shared
|
9
10
|
"""
|
10
11
|
import abc
|
11
12
|
import dataclasses as dc
|
@@ -208,6 +209,9 @@ class DeployPathOwner(abc.ABC):
|
|
208
209
|
raise NotImplementedError
|
209
210
|
|
210
211
|
|
212
|
+
DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
|
213
|
+
|
214
|
+
|
211
215
|
class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
|
212
216
|
def __init__(
|
213
217
|
self,
|
ominfra/manage/deploy/specs.py
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
+
import abc
|
2
3
|
import dataclasses as dc
|
3
4
|
import hashlib
|
4
5
|
import typing as ta
|
@@ -14,6 +15,17 @@ from .types import DeployRev
|
|
14
15
|
##
|
15
16
|
|
16
17
|
|
18
|
+
def check_valid_deploy_spec_path(s: str) -> str:
|
19
|
+
check.non_empty_str(s)
|
20
|
+
for c in ['..', '//']:
|
21
|
+
check.not_in(c, s)
|
22
|
+
check.arg(not s.startswith('/'))
|
23
|
+
return s
|
24
|
+
|
25
|
+
|
26
|
+
##
|
27
|
+
|
28
|
+
|
17
29
|
@dc.dataclass(frozen=True)
|
18
30
|
class DeployGitRepo:
|
19
31
|
host: ta.Optional[str] = None
|
@@ -26,7 +38,7 @@ class DeployGitRepo:
|
|
26
38
|
|
27
39
|
|
28
40
|
@dc.dataclass(frozen=True)
|
29
|
-
class
|
41
|
+
class DeployGitSpec:
|
30
42
|
repo: DeployGitRepo
|
31
43
|
rev: DeployRev
|
32
44
|
|
@@ -52,8 +64,62 @@ class DeployVenvSpec:
|
|
52
64
|
|
53
65
|
use_uv: bool = False
|
54
66
|
|
67
|
+
|
68
|
+
##
|
69
|
+
|
70
|
+
|
71
|
+
@dc.dataclass(frozen=True)
|
72
|
+
class DeployConfFile:
|
73
|
+
path: str
|
74
|
+
body: str
|
75
|
+
|
55
76
|
def __post_init__(self) -> None:
|
56
|
-
|
77
|
+
check_valid_deploy_spec_path(self.path)
|
78
|
+
|
79
|
+
|
80
|
+
#
|
81
|
+
|
82
|
+
|
83
|
+
@dc.dataclass(frozen=True)
|
84
|
+
class DeployConfLink(abc.ABC): # noqa
|
85
|
+
"""
|
86
|
+
May be either:
|
87
|
+
- @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
|
88
|
+
- @conf/file - links a single file in a single subdir to conf/@conf/@dst-file
|
89
|
+
- @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
|
90
|
+
"""
|
91
|
+
|
92
|
+
src: str
|
93
|
+
|
94
|
+
def __post_init__(self) -> None:
|
95
|
+
check_valid_deploy_spec_path(self.src)
|
96
|
+
if '/' in self.src:
|
97
|
+
check.equal(self.src.count('/'), 1)
|
98
|
+
|
99
|
+
|
100
|
+
class AppDeployConfLink(DeployConfLink):
|
101
|
+
pass
|
102
|
+
|
103
|
+
|
104
|
+
class TagDeployConfLink(DeployConfLink):
|
105
|
+
pass
|
106
|
+
|
107
|
+
|
108
|
+
#
|
109
|
+
|
110
|
+
|
111
|
+
@dc.dataclass(frozen=True)
|
112
|
+
class DeployConfSpec:
|
113
|
+
files: ta.Optional[ta.Sequence[DeployConfFile]] = None
|
114
|
+
|
115
|
+
links: ta.Optional[ta.Sequence[DeployConfLink]] = None
|
116
|
+
|
117
|
+
def __post_init__(self) -> None:
|
118
|
+
if self.files:
|
119
|
+
seen: ta.Set[str] = set()
|
120
|
+
for f in self.files:
|
121
|
+
check.not_in(f.path, seen)
|
122
|
+
seen.add(f.path)
|
57
123
|
|
58
124
|
|
59
125
|
##
|
@@ -62,12 +128,12 @@ class DeployVenvSpec:
|
|
62
128
|
@dc.dataclass(frozen=True)
|
63
129
|
class DeploySpec:
|
64
130
|
app: DeployApp
|
65
|
-
|
131
|
+
|
132
|
+
git: DeployGitSpec
|
66
133
|
|
67
134
|
venv: ta.Optional[DeployVenvSpec] = None
|
68
135
|
|
69
|
-
|
70
|
-
hash(self)
|
136
|
+
conf: ta.Optional[DeployConfSpec] = None
|
71
137
|
|
72
138
|
@cached_nullary
|
73
139
|
def key(self) -> DeployKey:
|
ominfra/manage/deploy/venvs.py
CHANGED
@@ -5,46 +5,28 @@ TODO:
|
|
5
5
|
- share more code with pyproject?
|
6
6
|
"""
|
7
7
|
import os.path
|
8
|
-
import typing as ta
|
9
8
|
|
10
9
|
from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
|
11
|
-
from omlish.lite.cached import cached_nullary
|
12
|
-
from omlish.lite.check import check
|
13
10
|
from omlish.os.atomics import AtomicPathSwapping
|
14
11
|
|
15
|
-
from .paths import DeployPath
|
16
|
-
from .paths import DeployPathOwner
|
17
12
|
from .specs import DeployVenvSpec
|
18
|
-
from .types import DeployAppTag
|
19
|
-
from .types import DeployHome
|
20
13
|
|
21
14
|
|
22
|
-
class DeployVenvManager
|
15
|
+
class DeployVenvManager:
|
23
16
|
def __init__(
|
24
17
|
self,
|
25
18
|
*,
|
26
|
-
deploy_home: ta.Optional[DeployHome] = None,
|
27
19
|
atomics: AtomicPathSwapping,
|
28
20
|
) -> None:
|
29
21
|
super().__init__()
|
30
22
|
|
31
|
-
self._deploy_home = deploy_home
|
32
23
|
self._atomics = atomics
|
33
24
|
|
34
|
-
@cached_nullary
|
35
|
-
def _dir(self) -> str:
|
36
|
-
return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
|
37
|
-
|
38
|
-
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
39
|
-
return {
|
40
|
-
DeployPath.parse('venvs/@app/@tag/'),
|
41
|
-
}
|
42
|
-
|
43
25
|
async def setup_venv(
|
44
26
|
self,
|
45
|
-
app_dir: str,
|
46
|
-
venv_dir: str,
|
47
27
|
spec: DeployVenvSpec,
|
28
|
+
git_dir: str,
|
29
|
+
venv_dir: str,
|
48
30
|
) -> None:
|
49
31
|
sys_exe = 'python3'
|
50
32
|
|
@@ -58,7 +40,7 @@ class DeployVenvManager(DeployPathOwner):
|
|
58
40
|
|
59
41
|
#
|
60
42
|
|
61
|
-
reqs_txt = os.path.join(
|
43
|
+
reqs_txt = os.path.join(git_dir, 'requirements.txt')
|
62
44
|
|
63
45
|
if os.path.isfile(reqs_txt):
|
64
46
|
if spec.use_uv:
|
@@ -68,14 +50,3 @@ class DeployVenvManager(DeployPathOwner):
|
|
68
50
|
pip_cmd = ['-m', 'pip']
|
69
51
|
|
70
52
|
await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
|
71
|
-
|
72
|
-
async def setup_app_venv(
|
73
|
-
self,
|
74
|
-
app_tag: DeployAppTag,
|
75
|
-
spec: DeployVenvSpec,
|
76
|
-
) -> None:
|
77
|
-
await self.setup_venv(
|
78
|
-
os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
|
79
|
-
os.path.join(self._dir(), app_tag.app, app_tag.tag),
|
80
|
-
spec,
|
81
|
-
)
|
ominfra/scripts/journald2aws.py
CHANGED
@@ -1527,6 +1527,15 @@ def snake_case(name: str) -> str:
|
|
1527
1527
|
##
|
1528
1528
|
|
1529
1529
|
|
1530
|
+
def strip_with_newline(s: str) -> str:
|
1531
|
+
if not s:
|
1532
|
+
return ''
|
1533
|
+
return s.strip() + '\n'
|
1534
|
+
|
1535
|
+
|
1536
|
+
##
|
1537
|
+
|
1538
|
+
|
1530
1539
|
def is_dunder(name: str) -> bool:
|
1531
1540
|
return (
|
1532
1541
|
name[:2] == name[-2:] == '__' and
|
ominfra/scripts/manage.py
CHANGED
@@ -2732,6 +2732,15 @@ def snake_case(name: str) -> str:
|
|
2732
2732
|
##
|
2733
2733
|
|
2734
2734
|
|
2735
|
+
def strip_with_newline(s: str) -> str:
|
2736
|
+
if not s:
|
2737
|
+
return ''
|
2738
|
+
return s.strip() + '\n'
|
2739
|
+
|
2740
|
+
|
2741
|
+
##
|
2742
|
+
|
2743
|
+
|
2735
2744
|
def is_dunder(name: str) -> bool:
|
2736
2745
|
return (
|
2737
2746
|
name[:2] == name[-2:] == '__' and
|
@@ -3392,6 +3401,38 @@ class LinuxOsRelease:
|
|
3392
3401
|
return dct
|
3393
3402
|
|
3394
3403
|
|
3404
|
+
########################################
|
3405
|
+
# ../../../omlish/os/paths.py
|
3406
|
+
|
3407
|
+
|
3408
|
+
def abs_real_path(p: str) -> str:
|
3409
|
+
return os.path.abspath(os.path.realpath(p))
|
3410
|
+
|
3411
|
+
|
3412
|
+
def is_path_in_dir(base_dir: str, target_path: str) -> bool:
|
3413
|
+
base_dir = abs_real_path(base_dir)
|
3414
|
+
target_path = abs_real_path(target_path)
|
3415
|
+
|
3416
|
+
return target_path.startswith(base_dir + os.path.sep)
|
3417
|
+
|
3418
|
+
|
3419
|
+
def relative_symlink(
|
3420
|
+
src: str,
|
3421
|
+
dst: str,
|
3422
|
+
*,
|
3423
|
+
target_is_directory: bool = False,
|
3424
|
+
dir_fd: ta.Optional[int] = None,
|
3425
|
+
**kwargs: ta.Any,
|
3426
|
+
) -> None:
|
3427
|
+
os.symlink(
|
3428
|
+
os.path.relpath(src, os.path.dirname(dst)),
|
3429
|
+
dst,
|
3430
|
+
target_is_directory=target_is_directory,
|
3431
|
+
dir_fd=dir_fd,
|
3432
|
+
**kwargs,
|
3433
|
+
)
|
3434
|
+
|
3435
|
+
|
3395
3436
|
########################################
|
3396
3437
|
# ../../../omdev/packaging/specifiers.py
|
3397
3438
|
# Copyright (c) Donald Stufft and individual contributors.
|
@@ -4068,11 +4109,12 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
|
|
4068
4109
|
# ../deploy/paths.py
|
4069
4110
|
"""
|
4070
4111
|
TODO:
|
4071
|
-
- run/
|
4112
|
+
- run/{.pid,.sock}
|
4072
4113
|
- logs/...
|
4073
4114
|
- current symlink
|
4074
4115
|
- conf/{nginx,supervisor}
|
4075
4116
|
- env/?
|
4117
|
+
- apps/<app>/shared
|
4076
4118
|
"""
|
4077
4119
|
|
4078
4120
|
|
@@ -4262,6 +4304,9 @@ class DeployPathOwner(abc.ABC):
|
|
4262
4304
|
raise NotImplementedError
|
4263
4305
|
|
4264
4306
|
|
4307
|
+
DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
|
4308
|
+
|
4309
|
+
|
4265
4310
|
class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
|
4266
4311
|
def __init__(
|
4267
4312
|
self,
|
@@ -4300,6 +4345,17 @@ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
|
|
4300
4345
|
##
|
4301
4346
|
|
4302
4347
|
|
4348
|
+
def check_valid_deploy_spec_path(s: str) -> str:
|
4349
|
+
check.non_empty_str(s)
|
4350
|
+
for c in ['..', '//']:
|
4351
|
+
check.not_in(c, s)
|
4352
|
+
check.arg(not s.startswith('/'))
|
4353
|
+
return s
|
4354
|
+
|
4355
|
+
|
4356
|
+
##
|
4357
|
+
|
4358
|
+
|
4303
4359
|
@dc.dataclass(frozen=True)
|
4304
4360
|
class DeployGitRepo:
|
4305
4361
|
host: ta.Optional[str] = None
|
@@ -4312,7 +4368,7 @@ class DeployGitRepo:
|
|
4312
4368
|
|
4313
4369
|
|
4314
4370
|
@dc.dataclass(frozen=True)
|
4315
|
-
class
|
4371
|
+
class DeployGitSpec:
|
4316
4372
|
repo: DeployGitRepo
|
4317
4373
|
rev: DeployRev
|
4318
4374
|
|
@@ -4338,8 +4394,62 @@ class DeployVenvSpec:
|
|
4338
4394
|
|
4339
4395
|
use_uv: bool = False
|
4340
4396
|
|
4397
|
+
|
4398
|
+
##
|
4399
|
+
|
4400
|
+
|
4401
|
+
@dc.dataclass(frozen=True)
|
4402
|
+
class DeployConfFile:
|
4403
|
+
path: str
|
4404
|
+
body: str
|
4405
|
+
|
4341
4406
|
def __post_init__(self) -> None:
|
4342
|
-
|
4407
|
+
check_valid_deploy_spec_path(self.path)
|
4408
|
+
|
4409
|
+
|
4410
|
+
#
|
4411
|
+
|
4412
|
+
|
4413
|
+
@dc.dataclass(frozen=True)
|
4414
|
+
class DeployConfLink(abc.ABC): # noqa
|
4415
|
+
"""
|
4416
|
+
May be either:
|
4417
|
+
- @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
|
4418
|
+
- @conf/file - links a single file in a single subdir to conf/@conf/@dst-file
|
4419
|
+
- @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
|
4420
|
+
"""
|
4421
|
+
|
4422
|
+
src: str
|
4423
|
+
|
4424
|
+
def __post_init__(self) -> None:
|
4425
|
+
check_valid_deploy_spec_path(self.src)
|
4426
|
+
if '/' in self.src:
|
4427
|
+
check.equal(self.src.count('/'), 1)
|
4428
|
+
|
4429
|
+
|
4430
|
+
class AppDeployConfLink(DeployConfLink):
|
4431
|
+
pass
|
4432
|
+
|
4433
|
+
|
4434
|
+
class TagDeployConfLink(DeployConfLink):
|
4435
|
+
pass
|
4436
|
+
|
4437
|
+
|
4438
|
+
#
|
4439
|
+
|
4440
|
+
|
4441
|
+
@dc.dataclass(frozen=True)
|
4442
|
+
class DeployConfSpec:
|
4443
|
+
files: ta.Optional[ta.Sequence[DeployConfFile]] = None
|
4444
|
+
|
4445
|
+
links: ta.Optional[ta.Sequence[DeployConfLink]] = None
|
4446
|
+
|
4447
|
+
def __post_init__(self) -> None:
|
4448
|
+
if self.files:
|
4449
|
+
seen: ta.Set[str] = set()
|
4450
|
+
for f in self.files:
|
4451
|
+
check.not_in(f.path, seen)
|
4452
|
+
seen.add(f.path)
|
4343
4453
|
|
4344
4454
|
|
4345
4455
|
##
|
@@ -4348,12 +4458,12 @@ class DeployVenvSpec:
|
|
4348
4458
|
@dc.dataclass(frozen=True)
|
4349
4459
|
class DeploySpec:
|
4350
4460
|
app: DeployApp
|
4351
|
-
|
4461
|
+
|
4462
|
+
git: DeployGitSpec
|
4352
4463
|
|
4353
4464
|
venv: ta.Optional[DeployVenvSpec] = None
|
4354
4465
|
|
4355
|
-
|
4356
|
-
hash(self)
|
4466
|
+
conf: ta.Optional[DeployConfSpec] = None
|
4357
4467
|
|
4358
4468
|
@cached_nullary
|
4359
4469
|
def key(self) -> DeployKey:
|
@@ -6435,7 +6545,7 @@ class AtomicPathSwapping(abc.ABC):
|
|
6435
6545
|
##
|
6436
6546
|
|
6437
6547
|
|
6438
|
-
class
|
6548
|
+
class OsReplaceAtomicPathSwap(AtomicPathSwap):
|
6439
6549
|
def __init__(
|
6440
6550
|
self,
|
6441
6551
|
kind: AtomicPathSwapKind,
|
@@ -6463,7 +6573,7 @@ class OsRenameAtomicPathSwap(AtomicPathSwap):
|
|
6463
6573
|
return self._tmp_path
|
6464
6574
|
|
6465
6575
|
def _commit(self) -> None:
|
6466
|
-
os.
|
6576
|
+
os.replace(self._tmp_path, self._dst_path)
|
6467
6577
|
|
6468
6578
|
def _abort(self) -> None:
|
6469
6579
|
shutil.rmtree(self._tmp_path, ignore_errors=True)
|
@@ -6510,7 +6620,7 @@ class TempDirAtomicPathSwapping(AtomicPathSwapping):
|
|
6510
6620
|
else:
|
6511
6621
|
raise TypeError(kind)
|
6512
6622
|
|
6513
|
-
return
|
6623
|
+
return OsReplaceAtomicPathSwap(
|
6514
6624
|
kind,
|
6515
6625
|
dst_path,
|
6516
6626
|
tmp_path,
|
@@ -6748,6 +6858,137 @@ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
|
|
6748
6858
|
CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
|
6749
6859
|
|
6750
6860
|
|
6861
|
+
########################################
|
6862
|
+
# ../deploy/conf.py
|
6863
|
+
"""
|
6864
|
+
TODO:
|
6865
|
+
- @conf DeployPathPlaceholder? :|
|
6866
|
+
- post-deploy: remove any dir_links not present in new spec
|
6867
|
+
- * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
|
6868
|
+
- no such thing as 'previously present'.. build a 'deploy state' and pass it back?
|
6869
|
+
- ** whole thing can be atomic **
|
6870
|
+
- 1) new atomic temp dir
|
6871
|
+
- 2) for each subdir not needing modification, hardlink into temp dir
|
6872
|
+
- 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
|
6873
|
+
- 4) write (or if deleting, omit) new files
|
6874
|
+
- 5) swap top level
|
6875
|
+
- ** whole deploy can be atomic(-ish) - do this for everything **
|
6876
|
+
- just a '/deploy/current' dir
|
6877
|
+
- some things (venvs) cannot be moved, thus the /deploy/venvs dir
|
6878
|
+
- ** ensure (enforce) equivalent relpath nesting
|
6879
|
+
"""
|
6880
|
+
|
6881
|
+
|
6882
|
+
class DeployConfManager(SingleDirDeployPathOwner):
|
6883
|
+
def __init__(
|
6884
|
+
self,
|
6885
|
+
*,
|
6886
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
6887
|
+
) -> None:
|
6888
|
+
super().__init__(
|
6889
|
+
owned_dir='conf',
|
6890
|
+
deploy_home=deploy_home,
|
6891
|
+
)
|
6892
|
+
|
6893
|
+
async def _write_conf_file(
|
6894
|
+
self,
|
6895
|
+
cf: DeployConfFile,
|
6896
|
+
conf_dir: str,
|
6897
|
+
) -> None:
|
6898
|
+
conf_file = os.path.join(conf_dir, cf.path)
|
6899
|
+
check.arg(is_path_in_dir(conf_dir, conf_file))
|
6900
|
+
|
6901
|
+
os.makedirs(os.path.dirname(conf_file), exist_ok=True)
|
6902
|
+
|
6903
|
+
with open(conf_file, 'w') as f: # noqa
|
6904
|
+
f.write(cf.body)
|
6905
|
+
|
6906
|
+
async def _make_conf_link(
|
6907
|
+
self,
|
6908
|
+
link: DeployConfLink,
|
6909
|
+
conf_dir: str,
|
6910
|
+
app_tag: DeployAppTag,
|
6911
|
+
link_dir: str,
|
6912
|
+
) -> None:
|
6913
|
+
link_src = os.path.join(conf_dir, link.src)
|
6914
|
+
check.arg(is_path_in_dir(conf_dir, link_src))
|
6915
|
+
|
6916
|
+
is_link_dir = link.src.endswith('/')
|
6917
|
+
if is_link_dir:
|
6918
|
+
check.arg(link.src.count('/') == 1)
|
6919
|
+
check.arg(os.path.isdir(link_src))
|
6920
|
+
link_dst_pfx = link.src
|
6921
|
+
link_dst_sfx = ''
|
6922
|
+
|
6923
|
+
elif '/' in link.src:
|
6924
|
+
check.arg(os.path.isfile(link_src))
|
6925
|
+
d, f = os.path.split(link.src)
|
6926
|
+
# TODO: check filename :|
|
6927
|
+
link_dst_pfx = d + '/'
|
6928
|
+
link_dst_sfx = '-' + f
|
6929
|
+
|
6930
|
+
else:
|
6931
|
+
check.arg(os.path.isfile(link_src))
|
6932
|
+
if '.' in link.src:
|
6933
|
+
l, _, r = link.src.partition('.')
|
6934
|
+
link_dst_pfx = l + '/'
|
6935
|
+
link_dst_sfx = '.' + r
|
6936
|
+
else:
|
6937
|
+
link_dst_pfx = link.src + '/'
|
6938
|
+
link_dst_sfx = ''
|
6939
|
+
|
6940
|
+
if isinstance(link, AppDeployConfLink):
|
6941
|
+
link_dst_mid = str(app_tag.app)
|
6942
|
+
sym_root = link_dir
|
6943
|
+
elif isinstance(link, TagDeployConfLink):
|
6944
|
+
link_dst_mid = '-'.join([app_tag.app, app_tag.tag])
|
6945
|
+
sym_root = conf_dir
|
6946
|
+
else:
|
6947
|
+
raise TypeError(link)
|
6948
|
+
|
6949
|
+
link_dst = ''.join([
|
6950
|
+
link_dst_pfx,
|
6951
|
+
link_dst_mid,
|
6952
|
+
link_dst_sfx,
|
6953
|
+
])
|
6954
|
+
|
6955
|
+
root_conf_dir = self._make_dir()
|
6956
|
+
sym_src = os.path.join(sym_root, link.src)
|
6957
|
+
sym_dst = os.path.join(root_conf_dir, link_dst)
|
6958
|
+
check.arg(is_path_in_dir(root_conf_dir, sym_dst))
|
6959
|
+
|
6960
|
+
os.makedirs(os.path.dirname(sym_dst), exist_ok=True)
|
6961
|
+
relative_symlink(sym_src, sym_dst, target_is_directory=is_link_dir)
|
6962
|
+
|
6963
|
+
async def write_conf(
|
6964
|
+
self,
|
6965
|
+
spec: DeployConfSpec,
|
6966
|
+
conf_dir: str,
|
6967
|
+
app_tag: DeployAppTag,
|
6968
|
+
link_dir: str,
|
6969
|
+
) -> None:
|
6970
|
+
conf_dir = os.path.abspath(conf_dir)
|
6971
|
+
os.makedirs(conf_dir)
|
6972
|
+
|
6973
|
+
#
|
6974
|
+
|
6975
|
+
for cf in spec.files or []:
|
6976
|
+
await self._write_conf_file(
|
6977
|
+
cf,
|
6978
|
+
conf_dir,
|
6979
|
+
)
|
6980
|
+
|
6981
|
+
#
|
6982
|
+
|
6983
|
+
for link in spec.links or []:
|
6984
|
+
await self._make_conf_link(
|
6985
|
+
link,
|
6986
|
+
conf_dir,
|
6987
|
+
app_tag,
|
6988
|
+
link_dir,
|
6989
|
+
)
|
6990
|
+
|
6991
|
+
|
6751
6992
|
########################################
|
6752
6993
|
# ../deploy/tmp.py
|
6753
6994
|
|
@@ -8240,7 +8481,7 @@ class DeployGitManager(SingleDirDeployPathOwner):
|
|
8240
8481
|
|
8241
8482
|
#
|
8242
8483
|
|
8243
|
-
async def checkout(self,
|
8484
|
+
async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
|
8244
8485
|
check.state(not os.path.exists(dst_dir))
|
8245
8486
|
with self._git._atomics.begin_atomic_path_swap( # noqa
|
8246
8487
|
'dir',
|
@@ -8248,14 +8489,14 @@ class DeployGitManager(SingleDirDeployPathOwner):
|
|
8248
8489
|
auto_commit=True,
|
8249
8490
|
make_dirs=True,
|
8250
8491
|
) as dst_swap:
|
8251
|
-
await self.fetch(
|
8492
|
+
await self.fetch(spec.rev)
|
8252
8493
|
|
8253
8494
|
dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
|
8254
8495
|
await dst_call('git', 'init')
|
8255
8496
|
|
8256
8497
|
await dst_call('git', 'remote', 'add', 'local', self._dir)
|
8257
|
-
await dst_call('git', 'fetch', '--depth=1', 'local',
|
8258
|
-
await dst_call('git', 'checkout',
|
8498
|
+
await dst_call('git', 'fetch', '--depth=1', 'local', spec.rev)
|
8499
|
+
await dst_call('git', 'checkout', spec.rev, *(spec.subtrees or []))
|
8259
8500
|
|
8260
8501
|
def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
|
8261
8502
|
try:
|
@@ -8264,8 +8505,12 @@ class DeployGitManager(SingleDirDeployPathOwner):
|
|
8264
8505
|
repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
|
8265
8506
|
return repo_dir
|
8266
8507
|
|
8267
|
-
async def checkout(
|
8268
|
-
|
8508
|
+
async def checkout(
|
8509
|
+
self,
|
8510
|
+
spec: DeployGitSpec,
|
8511
|
+
dst_dir: str,
|
8512
|
+
) -> None:
|
8513
|
+
await self.get_repo_dir(spec.repo).checkout(spec, dst_dir)
|
8269
8514
|
|
8270
8515
|
|
8271
8516
|
########################################
|
@@ -8277,32 +8522,21 @@ TODO:
|
|
8277
8522
|
"""
|
8278
8523
|
|
8279
8524
|
|
8280
|
-
class DeployVenvManager
|
8525
|
+
class DeployVenvManager:
|
8281
8526
|
def __init__(
|
8282
8527
|
self,
|
8283
8528
|
*,
|
8284
|
-
deploy_home: ta.Optional[DeployHome] = None,
|
8285
8529
|
atomics: AtomicPathSwapping,
|
8286
8530
|
) -> None:
|
8287
8531
|
super().__init__()
|
8288
8532
|
|
8289
|
-
self._deploy_home = deploy_home
|
8290
8533
|
self._atomics = atomics
|
8291
8534
|
|
8292
|
-
@cached_nullary
|
8293
|
-
def _dir(self) -> str:
|
8294
|
-
return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
|
8295
|
-
|
8296
|
-
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
8297
|
-
return {
|
8298
|
-
DeployPath.parse('venvs/@app/@tag/'),
|
8299
|
-
}
|
8300
|
-
|
8301
8535
|
async def setup_venv(
|
8302
8536
|
self,
|
8303
|
-
app_dir: str,
|
8304
|
-
venv_dir: str,
|
8305
8537
|
spec: DeployVenvSpec,
|
8538
|
+
git_dir: str,
|
8539
|
+
venv_dir: str,
|
8306
8540
|
) -> None:
|
8307
8541
|
sys_exe = 'python3'
|
8308
8542
|
|
@@ -8316,7 +8550,7 @@ class DeployVenvManager(DeployPathOwner):
|
|
8316
8550
|
|
8317
8551
|
#
|
8318
8552
|
|
8319
|
-
reqs_txt = os.path.join(
|
8553
|
+
reqs_txt = os.path.join(git_dir, 'requirements.txt')
|
8320
8554
|
|
8321
8555
|
if os.path.isfile(reqs_txt):
|
8322
8556
|
if spec.use_uv:
|
@@ -8327,17 +8561,6 @@ class DeployVenvManager(DeployPathOwner):
|
|
8327
8561
|
|
8328
8562
|
await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
|
8329
8563
|
|
8330
|
-
async def setup_app_venv(
|
8331
|
-
self,
|
8332
|
-
app_tag: DeployAppTag,
|
8333
|
-
spec: DeployVenvSpec,
|
8334
|
-
) -> None:
|
8335
|
-
await self.setup_venv(
|
8336
|
-
os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
|
8337
|
-
os.path.join(self._dir(), app_tag.app, app_tag.tag),
|
8338
|
-
spec,
|
8339
|
-
)
|
8340
|
-
|
8341
8564
|
|
8342
8565
|
########################################
|
8343
8566
|
# ../remote/_main.py
|
@@ -8886,12 +9109,16 @@ class DeployAppManager(DeployPathOwner):
|
|
8886
9109
|
self,
|
8887
9110
|
*,
|
8888
9111
|
deploy_home: ta.Optional[DeployHome] = None,
|
9112
|
+
|
9113
|
+
conf: DeployConfManager,
|
8889
9114
|
git: DeployGitManager,
|
8890
9115
|
venvs: DeployVenvManager,
|
8891
9116
|
) -> None:
|
8892
9117
|
super().__init__()
|
8893
9118
|
|
8894
9119
|
self._deploy_home = deploy_home
|
9120
|
+
|
9121
|
+
self._conf = conf
|
8895
9122
|
self._git = git
|
8896
9123
|
self._venvs = venvs
|
8897
9124
|
|
@@ -8901,27 +9128,72 @@ class DeployAppManager(DeployPathOwner):
|
|
8901
9128
|
|
8902
9129
|
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
8903
9130
|
return {
|
8904
|
-
DeployPath.parse('apps/@app
|
9131
|
+
DeployPath.parse('apps/@app/current'),
|
9132
|
+
DeployPath.parse('apps/@app/deploying'),
|
9133
|
+
|
9134
|
+
DeployPath.parse('apps/@app/tags/@tag/conf/'),
|
9135
|
+
DeployPath.parse('apps/@app/tags/@tag/git/'),
|
9136
|
+
DeployPath.parse('apps/@app/tags/@tag/venv/'),
|
8905
9137
|
}
|
8906
9138
|
|
8907
9139
|
async def prepare_app(
|
8908
9140
|
self,
|
8909
9141
|
spec: DeploySpec,
|
8910
9142
|
) -> None:
|
8911
|
-
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.
|
8912
|
-
|
9143
|
+
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.git.rev, spec.key()))
|
9144
|
+
|
9145
|
+
#
|
9146
|
+
|
9147
|
+
app_dir = os.path.join(self._dir(), spec.app)
|
9148
|
+
os.makedirs(app_dir, exist_ok=True)
|
8913
9149
|
|
8914
9150
|
#
|
8915
9151
|
|
9152
|
+
tag_dir = os.path.join(app_dir, 'tags', app_tag.tag)
|
9153
|
+
os.makedirs(tag_dir)
|
9154
|
+
|
9155
|
+
#
|
9156
|
+
|
9157
|
+
deploying_file = os.path.join(app_dir, 'deploying')
|
9158
|
+
current_file = os.path.join(app_dir, 'current')
|
9159
|
+
|
9160
|
+
if os.path.exists(deploying_file):
|
9161
|
+
os.unlink(deploying_file)
|
9162
|
+
relative_symlink(tag_dir, deploying_file, target_is_directory=True)
|
9163
|
+
|
9164
|
+
#
|
9165
|
+
|
9166
|
+
git_dir = os.path.join(tag_dir, 'git')
|
8916
9167
|
await self._git.checkout(
|
8917
|
-
spec.
|
8918
|
-
|
9168
|
+
spec.git,
|
9169
|
+
git_dir,
|
8919
9170
|
)
|
8920
9171
|
|
8921
9172
|
#
|
8922
9173
|
|
8923
9174
|
if spec.venv is not None:
|
8924
|
-
|
9175
|
+
venv_dir = os.path.join(tag_dir, 'venv')
|
9176
|
+
await self._venvs.setup_venv(
|
9177
|
+
spec.venv,
|
9178
|
+
git_dir,
|
9179
|
+
venv_dir,
|
9180
|
+
)
|
9181
|
+
|
9182
|
+
#
|
9183
|
+
|
9184
|
+
if spec.conf is not None:
|
9185
|
+
conf_dir = os.path.join(tag_dir, 'conf')
|
9186
|
+
conf_link_dir = os.path.join(current_file, 'conf')
|
9187
|
+
await self._conf.write_conf(
|
9188
|
+
spec.conf,
|
9189
|
+
conf_dir,
|
9190
|
+
app_tag,
|
9191
|
+
conf_link_dir,
|
9192
|
+
)
|
9193
|
+
|
9194
|
+
#
|
9195
|
+
|
9196
|
+
os.replace(deploying_file, current_file)
|
8925
9197
|
|
8926
9198
|
|
8927
9199
|
########################################
|
@@ -10043,23 +10315,45 @@ def bind_deploy(
|
|
10043
10315
|
) -> InjectorBindings:
|
10044
10316
|
lst: ta.List[InjectorBindingOrBindings] = [
|
10045
10317
|
inj.bind(deploy_config),
|
10318
|
+
]
|
10046
10319
|
|
10047
|
-
|
10320
|
+
#
|
10048
10321
|
|
10049
|
-
|
10322
|
+
def bind_manager(cls: type) -> InjectorBindings:
|
10323
|
+
return inj.as_bindings(
|
10324
|
+
inj.bind(cls, singleton=True),
|
10050
10325
|
|
10051
|
-
|
10326
|
+
*([inj.bind(DeployPathOwner, to_key=cls, array=True)] if issubclass(cls, DeployPathOwner) else []),
|
10327
|
+
)
|
10328
|
+
|
10329
|
+
lst.extend([
|
10330
|
+
inj.bind_array(DeployPathOwner),
|
10331
|
+
inj.bind_array_type(DeployPathOwner, DeployPathOwners),
|
10332
|
+
])
|
10333
|
+
|
10334
|
+
#
|
10335
|
+
|
10336
|
+
lst.extend([
|
10337
|
+
bind_manager(DeployAppManager),
|
10052
10338
|
|
10053
|
-
|
10339
|
+
bind_manager(DeployConfManager),
|
10340
|
+
|
10341
|
+
bind_manager(DeployGitManager),
|
10342
|
+
|
10343
|
+
bind_manager(DeployTmpManager),
|
10054
10344
|
inj.bind(AtomicPathSwapping, to_key=DeployTmpManager),
|
10055
10345
|
|
10056
|
-
|
10346
|
+
bind_manager(DeployVenvManager),
|
10347
|
+
])
|
10057
10348
|
|
10058
|
-
|
10349
|
+
#
|
10059
10350
|
|
10351
|
+
lst.extend([
|
10060
10352
|
bind_command(DeployCommand, DeployCommandExecutor),
|
10061
10353
|
bind_command(InterpCommand, InterpCommandExecutor),
|
10062
|
-
]
|
10354
|
+
])
|
10355
|
+
|
10356
|
+
#
|
10063
10357
|
|
10064
10358
|
if (dh := deploy_config.deploy_home) is not None:
|
10065
10359
|
dh = os.path.abspath(os.path.expanduser(dh))
|
ominfra/scripts/supervisor.py
CHANGED
@@ -2406,6 +2406,15 @@ def snake_case(name: str) -> str:
|
|
2406
2406
|
##
|
2407
2407
|
|
2408
2408
|
|
2409
|
+
def strip_with_newline(s: str) -> str:
|
2410
|
+
if not s:
|
2411
|
+
return ''
|
2412
|
+
return s.strip() + '\n'
|
2413
|
+
|
2414
|
+
|
2415
|
+
##
|
2416
|
+
|
2417
|
+
|
2409
2418
|
def is_dunder(name: str) -> bool:
|
2410
2419
|
return (
|
2411
2420
|
name[:2] == name[-2:] == '__' and
|
@@ -6110,6 +6119,7 @@ class ServerConfig:
|
|
6110
6119
|
|
6111
6120
|
groups: ta.Optional[ta.Sequence[ProcessGroupConfig]] = None
|
6112
6121
|
|
6122
|
+
# TODO: implement - make sure to accept broken symlinks
|
6113
6123
|
group_config_dirs: ta.Optional[ta.Sequence[str]] = None
|
6114
6124
|
|
6115
6125
|
@classmethod
|
ominfra/supervisor/configs.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: ominfra
|
3
|
-
Version: 0.0.0.
|
3
|
+
Version: 0.0.0.dev167
|
4
4
|
Summary: ominfra
|
5
5
|
Author: wrmsr
|
6
6
|
License: BSD-3-Clause
|
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
|
|
12
12
|
Classifier: Operating System :: POSIX
|
13
13
|
Requires-Python: >=3.12
|
14
14
|
License-File: LICENSE
|
15
|
-
Requires-Dist: omdev==0.0.0.
|
16
|
-
Requires-Dist: omlish==0.0.0.
|
15
|
+
Requires-Dist: omdev==0.0.0.dev167
|
16
|
+
Requires-Dist: omlish==0.0.0.dev167
|
17
17
|
Provides-Extra: all
|
18
18
|
Requires-Dist: paramiko~=3.5; extra == "all"
|
19
19
|
Requires-Dist: asyncssh~=2.18; extra == "all"
|
@@ -44,17 +44,18 @@ ominfra/manage/commands/ping.py,sha256=DVZFzL1Z_f-Bq53vxMrL3xOi0iK_nMonJE4KvQf9w
|
|
44
44
|
ominfra/manage/commands/subprocess.py,sha256=yHGMbAI-xKe_9BUs5IZ3Yav8qRE-I9aGnBtTwW15Pnw,2440
|
45
45
|
ominfra/manage/commands/types.py,sha256=XFZPeqeIBAaIIQF3pdPbGxLlb-LCrz6WtlDWO2q_vz0,210
|
46
46
|
ominfra/manage/deploy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
47
|
-
ominfra/manage/deploy/apps.py,sha256=
|
47
|
+
ominfra/manage/deploy/apps.py,sha256=jLvam6sVcWfRmyyLeJyTxz0IErWTn98rf1-8lIEEYlM,3292
|
48
48
|
ominfra/manage/deploy/commands.py,sha256=N9qVntnRgJ_IneI7rEQB2Za0oU7gouPfm-sl2MCwW1E,764
|
49
|
+
ominfra/manage/deploy/conf.py,sha256=HxKLWIT95StavJ0uHhMhuvWio1gTYBlDJv6N1_XdxEo,4417
|
49
50
|
ominfra/manage/deploy/config.py,sha256=aR6ubMEWqkTI55XtcG1Cczn6YhCVN6eSL8DT5EHQJN0,166
|
50
|
-
ominfra/manage/deploy/git.py,sha256=
|
51
|
-
ominfra/manage/deploy/inject.py,sha256=
|
51
|
+
ominfra/manage/deploy/git.py,sha256=BGht0UmY1d76kSGHESWuG8qxNKLuOGrwey5V9M4pfCY,3746
|
52
|
+
ominfra/manage/deploy/inject.py,sha256=o-bgWvziUuE5XzsE9_rcPSdWqPO9AAf2SbXamEpir2k,1978
|
52
53
|
ominfra/manage/deploy/interp.py,sha256=OKkenH8YKEW_mEDR6X7_ZLxK9a1Ox6KHSwFPTHT6OzA,1029
|
53
|
-
ominfra/manage/deploy/paths.py,sha256=
|
54
|
-
ominfra/manage/deploy/specs.py,sha256=
|
54
|
+
ominfra/manage/deploy/paths.py,sha256=Rq9OsK3MpporH6tfuumzLO51e4Yz_wi6RiXEg82iyLY,6947
|
55
|
+
ominfra/manage/deploy/specs.py,sha256=Xv1OQ3yQxOTR39Gj5ACgAn7vpgL-zVuGk1CHk_xmt5o,2883
|
55
56
|
ominfra/manage/deploy/tmp.py,sha256=L0pIfQuxQ7_6gC_AAv7eubI37_IPzCVR29hkn1MHL2Q,1230
|
56
57
|
ominfra/manage/deploy/types.py,sha256=o95wqvTGNRq8Cxx7VpqeX-9x1tI8k8BpqPFvJZkJYBA,305
|
57
|
-
ominfra/manage/deploy/venvs.py,sha256=
|
58
|
+
ominfra/manage/deploy/venvs.py,sha256=1co8l3eSxl2WccrF-XOTqeltoMccRH63o69NblV3yok,1366
|
58
59
|
ominfra/manage/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
59
60
|
ominfra/manage/remote/_main.py,sha256=p5KoiS2WMw6QAqlDl_Zun-JybmCsy8awIfpBMLBjGMY,4356
|
60
61
|
ominfra/manage/remote/channel.py,sha256=36xR9Ti9ZA8TUBtxmY0u7_3Lv7E6wzQTxlZl7gLR5GE,2224
|
@@ -75,13 +76,13 @@ ominfra/manage/targets/connection.py,sha256=5e8h9Miej2DKJxZfLyxpGe8y-Y0V_b_AuUW1
|
|
75
76
|
ominfra/manage/targets/inject.py,sha256=P4597xWM-V3I_gCt2O71OLhYQkkXtuJvkYRsIbhhMcE,1561
|
76
77
|
ominfra/manage/targets/targets.py,sha256=CFl8Uirgn3gfowO1Fn-LBK-6qYqEMFJ9snPUl0gCRuM,1753
|
77
78
|
ominfra/scripts/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
78
|
-
ominfra/scripts/journald2aws.py,sha256=
|
79
|
-
ominfra/scripts/manage.py,sha256=
|
80
|
-
ominfra/scripts/supervisor.py,sha256=
|
79
|
+
ominfra/scripts/journald2aws.py,sha256=Cfxnq9N2PaRLBrGSPq4qi39SOGldKZXT_Ve2_uPWcpY,155191
|
80
|
+
ominfra/scripts/manage.py,sha256=Yb_Ok9rB8zMycxED0fjzHQVNmVQ2qc14pASuzpves68,301283
|
81
|
+
ominfra/scripts/supervisor.py,sha256=y9o5NyKKb7ugwk3ZEywzlqVFd7QDhsSO2_C8oSqO_28,273957
|
81
82
|
ominfra/supervisor/LICENSE.txt,sha256=yvqaMNsDhWxziHa9ien6qCW1SkZv-DQlAg96XjfSee8,1746
|
82
83
|
ominfra/supervisor/__init__.py,sha256=Y3l4WY4JRi2uLG6kgbGp93fuGfkxkKwZDvhsa0Rwgtk,15
|
83
84
|
ominfra/supervisor/__main__.py,sha256=I0yFw-C08OOiZ3BF6lF1Oiv789EQXu-_j6whDhQUTEA,66
|
84
|
-
ominfra/supervisor/configs.py,sha256=
|
85
|
+
ominfra/supervisor/configs.py,sha256=UxcZ1yaw7CiwiSb7A1AegwcWKFXqtswaKaHlIN8Bt_Q,13771
|
85
86
|
ominfra/supervisor/dispatchers.py,sha256=zXLwQS4Vc6dWw5o9QOL04UMDt7w6CKu9wf19CjUiS2Q,1005
|
86
87
|
ominfra/supervisor/dispatchersimpl.py,sha256=q3dEyOHWTPKm28nmAGisjgIW1BX6O3-SzbYa7nWuTEs,11349
|
87
88
|
ominfra/supervisor/events.py,sha256=XGrtzHr1xm0dwjz329fn9eR0_Ap-LQL6Sk8LJ8eVDEo,6692
|
@@ -119,9 +120,9 @@ ominfra/tailscale/api.py,sha256=C5-t_b6jZXUWcy5k8bXm7CFnk73pSdrlMOgGDeGVrpw,1370
|
|
119
120
|
ominfra/tailscale/cli.py,sha256=h6akQJMl0KuWLHS7Ur6WcBZ2JwF0DJQhsPTnFBdGyNk,3571
|
120
121
|
ominfra/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
121
122
|
ominfra/tools/listresources.py,sha256=4qVg5txsb10EHhvqXXeM6gJ2jx9LbroEnPydDv1uXs0,6176
|
122
|
-
ominfra-0.0.0.
|
123
|
-
ominfra-0.0.0.
|
124
|
-
ominfra-0.0.0.
|
125
|
-
ominfra-0.0.0.
|
126
|
-
ominfra-0.0.0.
|
127
|
-
ominfra-0.0.0.
|
123
|
+
ominfra-0.0.0.dev167.dist-info/LICENSE,sha256=B_hVtavaA8zCYDW99DYdcpDLKz1n3BBRjZrcbv8uG8c,1451
|
124
|
+
ominfra-0.0.0.dev167.dist-info/METADATA,sha256=JFB_UaK98rzNZ6vQWBEVI9f7zjJ4DeSf00RpqMQrczs,731
|
125
|
+
ominfra-0.0.0.dev167.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
126
|
+
ominfra-0.0.0.dev167.dist-info/entry_points.txt,sha256=kgecQ2MgGrM9qK744BoKS3tMesaC3yjLnl9pa5CRczg,37
|
127
|
+
ominfra-0.0.0.dev167.dist-info/top_level.txt,sha256=E-b2OHkk_AOBLXHYZQ2EOFKl-_6uOGd8EjeG-Zy6h_w,8
|
128
|
+
ominfra-0.0.0.dev167.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|