ominfra 0.0.0.dev157__py3-none-any.whl → 0.0.0.dev158__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/main.py +1 -1
- ominfra/journald/tailer.py +2 -2
- ominfra/manage/bootstrap_.py +1 -1
- ominfra/manage/commands/subprocess.py +4 -4
- ominfra/manage/deploy/apps.py +14 -15
- ominfra/manage/deploy/config.py +3 -0
- ominfra/manage/deploy/git.py +11 -27
- ominfra/manage/deploy/paths.py +48 -48
- ominfra/manage/deploy/specs.py +32 -0
- ominfra/manage/deploy/venvs.py +10 -5
- ominfra/manage/remote/spawning.py +3 -3
- ominfra/manage/system/packages.py +1 -1
- ominfra/pyremote.py +26 -26
- ominfra/scripts/journald2aws.py +461 -350
- ominfra/scripts/manage.py +1783 -1693
- ominfra/scripts/supervisor.py +353 -332
- ominfra/supervisor/http.py +1 -1
- ominfra/supervisor/main.py +2 -2
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/RECORD +24 -23
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev158.dist-info}/top_level.txt +0 -0
@@ -5,7 +5,7 @@ import dataclasses as dc
|
|
5
5
|
import os.path
|
6
6
|
import sys
|
7
7
|
|
8
|
-
from omlish.
|
8
|
+
from omlish.logs.standard import configure_standard_logging
|
9
9
|
|
10
10
|
from ....configs import read_config_file
|
11
11
|
from .driver import JournalctlToAwsDriver
|
ominfra/journald/tailer.py
CHANGED
@@ -410,8 +410,8 @@ import typing as ta
|
|
410
410
|
from omlish.lite.cached import cached_nullary
|
411
411
|
from omlish.lite.check import check
|
412
412
|
from omlish.lite.logs import log
|
413
|
-
from omlish.
|
414
|
-
from omlish.
|
413
|
+
from omlish.subprocesses import subprocess_close
|
414
|
+
from omlish.subprocesses import subprocess_shell_wrap_exec
|
415
415
|
|
416
416
|
from ..threadworkers import ThreadWorker
|
417
417
|
from .messages import JournalctlMessage # noqa
|
ominfra/manage/bootstrap_.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
2
|
from omlish.lite.inject import Injector
|
3
3
|
from omlish.lite.inject import inj
|
4
|
-
from omlish.
|
4
|
+
from omlish.logs.standard import configure_standard_logging
|
5
5
|
|
6
6
|
from .bootstrap import MainBootstrap
|
7
7
|
from .inject import bind_main
|
@@ -6,11 +6,11 @@ import subprocess
|
|
6
6
|
import time
|
7
7
|
import typing as ta
|
8
8
|
|
9
|
-
from omlish.
|
9
|
+
from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
|
10
10
|
from omlish.lite.check import check
|
11
|
-
from omlish.
|
12
|
-
from omlish.
|
13
|
-
from omlish.
|
11
|
+
from omlish.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
|
12
|
+
from omlish.subprocesses import SubprocessChannelOption
|
13
|
+
from omlish.subprocesses import subprocess_maybe_shell_wrap_exec
|
14
14
|
|
15
15
|
from .base import Command
|
16
16
|
from .base import CommandExecutor
|
ominfra/manage/deploy/apps.py
CHANGED
@@ -3,12 +3,13 @@ import datetime
|
|
3
3
|
import os.path
|
4
4
|
import typing as ta
|
5
5
|
|
6
|
+
from omlish.lite.cached import cached_nullary
|
7
|
+
from omlish.lite.check import check
|
8
|
+
|
6
9
|
from .git import DeployGitManager
|
7
|
-
from .git import DeployGitRepo
|
8
|
-
from .git import DeployGitSpec
|
9
10
|
from .paths import DeployPath
|
10
11
|
from .paths import DeployPathOwner
|
11
|
-
from .
|
12
|
+
from .specs import DeploySpec
|
12
13
|
from .types import DeployAppTag
|
13
14
|
from .types import DeployHome
|
14
15
|
from .types import DeployRev
|
@@ -24,14 +25,14 @@ def make_deploy_tag(
|
|
24
25
|
now = datetime.datetime.utcnow() # noqa
|
25
26
|
now_fmt = '%Y%m%dT%H%M%S'
|
26
27
|
now_str = now.strftime(now_fmt)
|
27
|
-
return DeployTag('-'.join([
|
28
|
+
return DeployTag('-'.join([now_str, rev]))
|
28
29
|
|
29
30
|
|
30
31
|
class DeployAppManager(DeployPathOwner):
|
31
32
|
def __init__(
|
32
33
|
self,
|
33
34
|
*,
|
34
|
-
deploy_home: DeployHome,
|
35
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
35
36
|
git: DeployGitManager,
|
36
37
|
venvs: DeployVenvManager,
|
37
38
|
) -> None:
|
@@ -41,7 +42,9 @@ class DeployAppManager(DeployPathOwner):
|
|
41
42
|
self._git = git
|
42
43
|
self._venvs = venvs
|
43
44
|
|
44
|
-
|
45
|
+
@cached_nullary
|
46
|
+
def _dir(self) -> str:
|
47
|
+
return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
|
45
48
|
|
46
49
|
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
47
50
|
return {
|
@@ -50,20 +53,16 @@ class DeployAppManager(DeployPathOwner):
|
|
50
53
|
|
51
54
|
async def prepare_app(
|
52
55
|
self,
|
53
|
-
|
54
|
-
rev: DeployRev,
|
55
|
-
repo: DeployGitRepo,
|
56
|
+
spec: DeploySpec,
|
56
57
|
):
|
57
|
-
app_tag = DeployAppTag(app, make_deploy_tag(rev))
|
58
|
-
app_dir = os.path.join(self._dir, app, app_tag.tag)
|
58
|
+
app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev))
|
59
|
+
app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
|
59
60
|
|
60
61
|
#
|
61
62
|
|
62
63
|
await self._git.checkout(
|
63
|
-
|
64
|
-
|
65
|
-
rev=rev,
|
66
|
-
),
|
64
|
+
spec.repo,
|
65
|
+
spec.rev,
|
67
66
|
app_dir,
|
68
67
|
)
|
69
68
|
|
ominfra/manage/deploy/config.py
CHANGED
ominfra/manage/deploy/git.py
CHANGED
@@ -8,17 +8,18 @@ git/github.com/wrmsr/omlish <- bootstrap repo
|
|
8
8
|
|
9
9
|
github.com/wrmsr/omlish@rev
|
10
10
|
"""
|
11
|
-
import dataclasses as dc
|
12
11
|
import functools
|
13
12
|
import os.path
|
14
13
|
import typing as ta
|
15
14
|
|
16
|
-
from omlish.
|
15
|
+
from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
|
17
16
|
from omlish.lite.cached import async_cached_nullary
|
17
|
+
from omlish.lite.cached import cached_nullary
|
18
18
|
from omlish.lite.check import check
|
19
19
|
|
20
20
|
from .paths import DeployPath
|
21
21
|
from .paths import DeployPathOwner
|
22
|
+
from .specs import DeployGitRepo
|
22
23
|
from .types import DeployHome
|
23
24
|
from .types import DeployRev
|
24
25
|
|
@@ -26,39 +27,22 @@ from .types import DeployRev
|
|
26
27
|
##
|
27
28
|
|
28
29
|
|
29
|
-
@dc.dataclass(frozen=True)
|
30
|
-
class DeployGitRepo:
|
31
|
-
host: ta.Optional[str] = None
|
32
|
-
username: ta.Optional[str] = None
|
33
|
-
path: ta.Optional[str] = None
|
34
|
-
|
35
|
-
def __post_init__(self) -> None:
|
36
|
-
check.not_in('..', check.non_empty_str(self.host))
|
37
|
-
check.not_in('.', check.non_empty_str(self.path))
|
38
|
-
|
39
|
-
|
40
|
-
@dc.dataclass(frozen=True)
|
41
|
-
class DeployGitSpec:
|
42
|
-
repo: DeployGitRepo
|
43
|
-
rev: DeployRev
|
44
|
-
|
45
|
-
|
46
|
-
##
|
47
|
-
|
48
|
-
|
49
30
|
class DeployGitManager(DeployPathOwner):
|
50
31
|
def __init__(
|
51
32
|
self,
|
52
33
|
*,
|
53
|
-
deploy_home: DeployHome,
|
34
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
54
35
|
) -> None:
|
55
36
|
super().__init__()
|
56
37
|
|
57
38
|
self._deploy_home = deploy_home
|
58
|
-
self._dir = os.path.join(deploy_home, 'git')
|
59
39
|
|
60
40
|
self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
|
61
41
|
|
42
|
+
@cached_nullary
|
43
|
+
def _dir(self) -> str:
|
44
|
+
return os.path.join(check.non_empty_str(self._deploy_home), 'git')
|
45
|
+
|
62
46
|
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
63
47
|
return {
|
64
48
|
DeployPath.parse('git'),
|
@@ -75,7 +59,7 @@ class DeployGitManager(DeployPathOwner):
|
|
75
59
|
self._git = git
|
76
60
|
self._repo = repo
|
77
61
|
self._dir = os.path.join(
|
78
|
-
self._git._dir, # noqa
|
62
|
+
self._git._dir(), # noqa
|
79
63
|
check.non_empty_str(repo.host),
|
80
64
|
check.non_empty_str(repo.path),
|
81
65
|
)
|
@@ -132,5 +116,5 @@ class DeployGitManager(DeployPathOwner):
|
|
132
116
|
repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
|
133
117
|
return repo_dir
|
134
118
|
|
135
|
-
async def checkout(self,
|
136
|
-
await self.get_repo_dir(
|
119
|
+
async def checkout(self, repo: DeployGitRepo, rev: DeployRev, dst_dir: str) -> None:
|
120
|
+
await self.get_repo_dir(repo).checkout(rev, dst_dir)
|
ominfra/manage/deploy/paths.py
CHANGED
@@ -3,22 +3,22 @@
|
|
3
3
|
~deploy
|
4
4
|
deploy.pid (flock)
|
5
5
|
/app
|
6
|
-
/<
|
6
|
+
/<appplaceholder> - shallow clone
|
7
7
|
/conf
|
8
8
|
/env
|
9
|
-
<
|
9
|
+
<appplaceholder>.env
|
10
10
|
/nginx
|
11
|
-
<
|
11
|
+
<appplaceholder>.conf
|
12
12
|
/supervisor
|
13
|
-
<
|
13
|
+
<appplaceholder>.conf
|
14
14
|
/venv
|
15
|
-
/<
|
15
|
+
/<appplaceholder>
|
16
16
|
|
17
17
|
?
|
18
18
|
/logs
|
19
|
-
/wrmsr--omlish--<
|
19
|
+
/wrmsr--omlish--<placeholder>
|
20
20
|
|
21
|
-
|
21
|
+
placeholder = <name>--<rev>--<when>
|
22
22
|
|
23
23
|
==
|
24
24
|
|
@@ -43,16 +43,16 @@ from omlish.lite.check import check
|
|
43
43
|
|
44
44
|
|
45
45
|
DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
|
46
|
-
|
46
|
+
DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
|
47
47
|
|
48
48
|
|
49
49
|
##
|
50
50
|
|
51
51
|
|
52
|
-
|
53
|
-
|
52
|
+
DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER = '@'
|
53
|
+
DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
|
54
54
|
|
55
|
-
|
55
|
+
DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
|
56
56
|
'app',
|
57
57
|
'tag', # <rev>-<dt>
|
58
58
|
])
|
@@ -70,7 +70,7 @@ class DeployPathPart(abc.ABC): # noqa
|
|
70
70
|
raise NotImplementedError
|
71
71
|
|
72
72
|
@abc.abstractmethod
|
73
|
-
def render(self,
|
73
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
74
74
|
raise NotImplementedError
|
75
75
|
|
76
76
|
|
@@ -84,9 +84,9 @@ class DirDeployPathPart(DeployPathPart, abc.ABC):
|
|
84
84
|
|
85
85
|
@classmethod
|
86
86
|
def parse(cls, s: str) -> 'DirDeployPathPart':
|
87
|
-
if
|
88
|
-
check.equal(s[0],
|
89
|
-
return
|
87
|
+
if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
|
88
|
+
check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
|
89
|
+
return PlaceholderDirDeployPathPart(s[1:])
|
90
90
|
else:
|
91
91
|
return ConstDirDeployPathPart(s)
|
92
92
|
|
@@ -98,13 +98,13 @@ class FileDeployPathPart(DeployPathPart, abc.ABC):
|
|
98
98
|
|
99
99
|
@classmethod
|
100
100
|
def parse(cls, s: str) -> 'FileDeployPathPart':
|
101
|
-
if
|
102
|
-
check.equal(s[0],
|
103
|
-
if not any(c in s for c in
|
104
|
-
return
|
101
|
+
if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
|
102
|
+
check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
|
103
|
+
if not any(c in s for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS):
|
104
|
+
return PlaceholderFileDeployPathPart(s[1:], '')
|
105
105
|
else:
|
106
|
-
p = min(f for c in
|
107
|
-
return
|
106
|
+
p = min(f for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS if (f := s.find(c)) > 0)
|
107
|
+
return PlaceholderFileDeployPathPart(s[1:p], s[p:])
|
108
108
|
else:
|
109
109
|
return ConstFileDeployPathPart(s)
|
110
110
|
|
@@ -119,9 +119,9 @@ class ConstDeployPathPart(DeployPathPart, abc.ABC):
|
|
119
119
|
def __post_init__(self) -> None:
|
120
120
|
check.non_empty_str(self.name)
|
121
121
|
check.not_in('/', self.name)
|
122
|
-
check.not_in(
|
122
|
+
check.not_in(DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, self.name)
|
123
123
|
|
124
|
-
def render(self,
|
124
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
125
125
|
return self.name
|
126
126
|
|
127
127
|
|
@@ -137,40 +137,40 @@ class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
|
|
137
137
|
|
138
138
|
|
139
139
|
@dc.dataclass(frozen=True)
|
140
|
-
class
|
141
|
-
|
140
|
+
class PlaceholderDeployPathPart(DeployPathPart, abc.ABC):
|
141
|
+
placeholder: str # DeployPathPlaceholder
|
142
142
|
|
143
143
|
def __post_init__(self) -> None:
|
144
|
-
check.non_empty_str(self.
|
145
|
-
for c in [*
|
146
|
-
check.not_in(c, self.
|
147
|
-
check.in_(self.
|
148
|
-
|
149
|
-
def
|
150
|
-
if
|
151
|
-
return
|
144
|
+
check.non_empty_str(self.placeholder)
|
145
|
+
for c in [*DEPLOY_PATH_PLACEHOLDER_SEPARATORS, DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
|
146
|
+
check.not_in(c, self.placeholder)
|
147
|
+
check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
|
148
|
+
|
149
|
+
def _render_placeholder(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
150
|
+
if placeholders is not None:
|
151
|
+
return placeholders[self.placeholder] # type: ignore
|
152
152
|
else:
|
153
|
-
return
|
153
|
+
return DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER + self.placeholder
|
154
154
|
|
155
155
|
|
156
156
|
@dc.dataclass(frozen=True)
|
157
|
-
class
|
158
|
-
def render(self,
|
159
|
-
return self.
|
157
|
+
class PlaceholderDirDeployPathPart(PlaceholderDeployPathPart, DirDeployPathPart):
|
158
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
159
|
+
return self._render_placeholder(placeholders)
|
160
160
|
|
161
161
|
|
162
162
|
@dc.dataclass(frozen=True)
|
163
|
-
class
|
163
|
+
class PlaceholderFileDeployPathPart(PlaceholderDeployPathPart, FileDeployPathPart):
|
164
164
|
suffix: str
|
165
165
|
|
166
166
|
def __post_init__(self) -> None:
|
167
167
|
super().__post_init__()
|
168
168
|
if self.suffix:
|
169
|
-
for c in [
|
169
|
+
for c in [DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
|
170
170
|
check.not_in(c, self.suffix)
|
171
171
|
|
172
|
-
def render(self,
|
173
|
-
return self.
|
172
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
173
|
+
return self._render_placeholder(placeholders) + self.suffix
|
174
174
|
|
175
175
|
|
176
176
|
##
|
@@ -187,22 +187,22 @@ class DeployPath:
|
|
187
187
|
|
188
188
|
pd = {}
|
189
189
|
for i, p in enumerate(self.parts):
|
190
|
-
if isinstance(p,
|
191
|
-
if p.
|
192
|
-
raise DeployPathError('Duplicate
|
193
|
-
pd[p.
|
190
|
+
if isinstance(p, PlaceholderDeployPathPart):
|
191
|
+
if p.placeholder in pd:
|
192
|
+
raise DeployPathError('Duplicate placeholders in path', self)
|
193
|
+
pd[p.placeholder] = i
|
194
194
|
|
195
195
|
if 'tag' in pd:
|
196
196
|
if 'app' not in pd or pd['app'] >= pd['tag']:
|
197
|
-
raise DeployPathError('Tag
|
197
|
+
raise DeployPathError('Tag placeholder in path without preceding app', self)
|
198
198
|
|
199
199
|
@property
|
200
200
|
def kind(self) -> ta.Literal['file', 'dir']:
|
201
201
|
return self.parts[-1].kind
|
202
202
|
|
203
|
-
def render(self,
|
203
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
204
204
|
return os.path.join( # noqa
|
205
|
-
*[p.render(
|
205
|
+
*[p.render(placeholders) for p in self.parts],
|
206
206
|
*([''] if self.kind == 'dir' else []),
|
207
207
|
)
|
208
208
|
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# ruff: noqa: UP006 UP007
|
2
|
+
import dataclasses as dc
|
3
|
+
import typing as ta
|
4
|
+
|
5
|
+
from omlish.lite.check import check
|
6
|
+
|
7
|
+
from .types import DeployApp
|
8
|
+
from .types import DeployRev
|
9
|
+
|
10
|
+
|
11
|
+
##
|
12
|
+
|
13
|
+
|
14
|
+
@dc.dataclass(frozen=True)
|
15
|
+
class DeployGitRepo:
|
16
|
+
host: ta.Optional[str] = None
|
17
|
+
username: ta.Optional[str] = None
|
18
|
+
path: ta.Optional[str] = None
|
19
|
+
|
20
|
+
def __post_init__(self) -> None:
|
21
|
+
check.not_in('..', check.non_empty_str(self.host))
|
22
|
+
check.not_in('.', check.non_empty_str(self.path))
|
23
|
+
|
24
|
+
|
25
|
+
##
|
26
|
+
|
27
|
+
|
28
|
+
@dc.dataclass(frozen=True)
|
29
|
+
class DeploySpec:
|
30
|
+
app: DeployApp
|
31
|
+
repo: DeployGitRepo
|
32
|
+
rev: DeployRev
|
ominfra/manage/deploy/venvs.py
CHANGED
@@ -7,7 +7,9 @@ TODO:
|
|
7
7
|
import os.path
|
8
8
|
import typing as ta
|
9
9
|
|
10
|
-
from omlish.
|
10
|
+
from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
|
11
|
+
from omlish.lite.cached import cached_nullary
|
12
|
+
from omlish.lite.check import check
|
11
13
|
|
12
14
|
from .paths import DeployPath
|
13
15
|
from .paths import DeployPathOwner
|
@@ -19,12 +21,15 @@ class DeployVenvManager(DeployPathOwner):
|
|
19
21
|
def __init__(
|
20
22
|
self,
|
21
23
|
*,
|
22
|
-
deploy_home: DeployHome,
|
24
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
23
25
|
) -> None:
|
24
26
|
super().__init__()
|
25
27
|
|
26
28
|
self._deploy_home = deploy_home
|
27
|
-
|
29
|
+
|
30
|
+
@cached_nullary
|
31
|
+
def _dir(self) -> str:
|
32
|
+
return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
|
28
33
|
|
29
34
|
def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
30
35
|
return {
|
@@ -61,6 +66,6 @@ class DeployVenvManager(DeployPathOwner):
|
|
61
66
|
|
62
67
|
async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
|
63
68
|
await self.setup_venv(
|
64
|
-
os.path.join(self._deploy_home, 'apps', app_tag.app, app_tag.tag),
|
65
|
-
os.path.join(self.
|
69
|
+
os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
|
70
|
+
os.path.join(self._dir(), app_tag.app, app_tag.tag),
|
66
71
|
)
|
@@ -7,10 +7,10 @@ import shlex
|
|
7
7
|
import subprocess
|
8
8
|
import typing as ta
|
9
9
|
|
10
|
-
from omlish.
|
10
|
+
from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
|
11
11
|
from omlish.lite.check import check
|
12
|
-
from omlish.
|
13
|
-
from omlish.
|
12
|
+
from omlish.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
|
13
|
+
from omlish.subprocesses import SubprocessChannelOption
|
14
14
|
|
15
15
|
|
16
16
|
##
|
ominfra/pyremote.py
CHANGED
@@ -162,7 +162,7 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
|
|
162
162
|
# Get pid
|
163
163
|
pid = os.getpid()
|
164
164
|
|
165
|
-
# Two copies of
|
165
|
+
# Two copies of payload src to be sent to parent
|
166
166
|
r0, w0 = os.pipe()
|
167
167
|
r1, w1 = os.pipe()
|
168
168
|
|
@@ -201,17 +201,17 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
|
|
201
201
|
# Write pid
|
202
202
|
os.write(1, struct.pack('<Q', pid))
|
203
203
|
|
204
|
-
# Read
|
205
|
-
|
206
|
-
if len(
|
204
|
+
# Read payload src from stdin
|
205
|
+
payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
|
206
|
+
if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
|
207
207
|
raise EOFError
|
208
|
-
|
208
|
+
payload_src = zlib.decompress(payload_z)
|
209
209
|
|
210
|
-
# Write both copies of
|
211
|
-
# and block and need to be drained by pyremote_bootstrap_finalize running in parent.
|
210
|
+
# Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
|
211
|
+
# fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
|
212
212
|
for w in [w0, w1]:
|
213
213
|
fp = os.fdopen(w, 'wb', 0)
|
214
|
-
fp.write(
|
214
|
+
fp.write(payload_src)
|
215
215
|
fp.close()
|
216
216
|
|
217
217
|
# Write second ack
|
@@ -275,7 +275,7 @@ class PyremotePayloadRuntime:
|
|
275
275
|
input: ta.BinaryIO
|
276
276
|
output: ta.BinaryIO
|
277
277
|
context_name: str
|
278
|
-
|
278
|
+
payload_src: str
|
279
279
|
options: PyremoteBootstrapOptions
|
280
280
|
env_info: PyremoteEnvInfo
|
281
281
|
|
@@ -283,9 +283,9 @@ class PyremotePayloadRuntime:
|
|
283
283
|
def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
284
284
|
# If src file var is not present we need to do initial finalization
|
285
285
|
if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
|
286
|
-
# Read second copy of
|
286
|
+
# Read second copy of payload src
|
287
287
|
r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
|
288
|
-
|
288
|
+
payload_src = r1.read().decode('utf-8')
|
289
289
|
r1.close()
|
290
290
|
|
291
291
|
# Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
|
@@ -303,7 +303,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
303
303
|
# Write temp source file
|
304
304
|
import tempfile
|
305
305
|
tfd, tfn = tempfile.mkstemp('-pyremote.py')
|
306
|
-
os.write(tfd,
|
306
|
+
os.write(tfd, payload_src.encode('utf-8'))
|
307
307
|
os.close(tfd)
|
308
308
|
|
309
309
|
# Set vars
|
@@ -322,7 +322,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
322
322
|
|
323
323
|
# Read temp source file
|
324
324
|
with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
|
325
|
-
|
325
|
+
payload_src = sf.read()
|
326
326
|
|
327
327
|
# Restore vars
|
328
328
|
sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
|
@@ -355,7 +355,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
355
355
|
input=input,
|
356
356
|
output=output,
|
357
357
|
context_name=context_name,
|
358
|
-
|
358
|
+
payload_src=payload_src,
|
359
359
|
options=options,
|
360
360
|
env_info=env_info,
|
361
361
|
)
|
@@ -367,31 +367,31 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
367
367
|
class PyremoteBootstrapDriver:
|
368
368
|
def __init__(
|
369
369
|
self,
|
370
|
-
|
370
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
371
371
|
options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
|
372
372
|
) -> None:
|
373
373
|
super().__init__()
|
374
374
|
|
375
|
-
self.
|
375
|
+
self._payload_src = payload_src
|
376
376
|
self._options = options
|
377
377
|
|
378
|
-
self.
|
379
|
-
self.
|
378
|
+
self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
|
379
|
+
self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
|
380
380
|
|
381
381
|
self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
|
382
382
|
#
|
383
383
|
|
384
384
|
@classmethod
|
385
|
-
def
|
385
|
+
def _prepare_payload_src(
|
386
386
|
cls,
|
387
|
-
|
387
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
388
388
|
options: PyremoteBootstrapOptions,
|
389
389
|
) -> str:
|
390
390
|
parts: ta.List[str]
|
391
|
-
if isinstance(
|
392
|
-
parts = [
|
391
|
+
if isinstance(payload_src, str):
|
392
|
+
parts = [payload_src]
|
393
393
|
else:
|
394
|
-
parts = list(
|
394
|
+
parts = list(payload_src)
|
395
395
|
|
396
396
|
if (mn := options.main_name_override) is not None:
|
397
397
|
parts.insert(0, f'__name__ = {mn!r}')
|
@@ -427,9 +427,9 @@ class PyremoteBootstrapDriver:
|
|
427
427
|
d = yield from self._read(8)
|
428
428
|
pid = struct.unpack('<Q', d)[0]
|
429
429
|
|
430
|
-
# Write
|
431
|
-
yield from self._write(struct.pack('<I', len(self.
|
432
|
-
yield from self._write(self.
|
430
|
+
# Write payload src
|
431
|
+
yield from self._write(struct.pack('<I', len(self._payload_z)))
|
432
|
+
yield from self._write(self._payload_z)
|
433
433
|
|
434
434
|
# Read second ack (after writing src copies)
|
435
435
|
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
|