ominfra 0.0.0.dev157__py3-none-any.whl → 0.0.0.dev159__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/main.py +1 -1
- ominfra/journald/tailer.py +2 -2
- ominfra/manage/bootstrap_.py +1 -1
- ominfra/manage/commands/subprocess.py +4 -4
- ominfra/manage/deploy/apps.py +23 -21
- ominfra/manage/deploy/atomics.py +207 -0
- ominfra/manage/deploy/config.py +3 -0
- ominfra/manage/deploy/git.py +27 -47
- ominfra/manage/deploy/inject.py +11 -0
- ominfra/manage/deploy/paths.py +89 -51
- ominfra/manage/deploy/specs.py +42 -0
- ominfra/manage/deploy/tmp.py +46 -0
- ominfra/manage/deploy/types.py +1 -0
- ominfra/manage/deploy/venvs.py +16 -6
- ominfra/manage/remote/spawning.py +3 -3
- ominfra/manage/system/packages.py +1 -1
- ominfra/pyremote.py +26 -26
- ominfra/scripts/journald2aws.py +467 -354
- ominfra/scripts/manage.py +1426 -1037
- ominfra/scripts/supervisor.py +359 -336
- ominfra/supervisor/http.py +1 -1
- ominfra/supervisor/main.py +2 -2
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/RECORD +28 -25
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev157.dist-info → ominfra-0.0.0.dev159.dist-info}/top_level.txt +0 -0
ominfra/manage/deploy/paths.py
CHANGED
@@ -3,22 +3,24 @@
|
|
3
3
|
~deploy
|
4
4
|
deploy.pid (flock)
|
5
5
|
/app
|
6
|
-
/<
|
6
|
+
/<appplaceholder> - shallow clone
|
7
7
|
/conf
|
8
8
|
/env
|
9
|
-
<
|
9
|
+
<appplaceholder>.env
|
10
10
|
/nginx
|
11
|
-
<
|
11
|
+
<appplaceholder>.conf
|
12
12
|
/supervisor
|
13
|
-
<
|
13
|
+
<appplaceholder>.conf
|
14
14
|
/venv
|
15
|
-
/<
|
15
|
+
/<appplaceholder>
|
16
|
+
|
17
|
+
/tmp
|
16
18
|
|
17
19
|
?
|
18
20
|
/logs
|
19
|
-
/wrmsr--omlish--<
|
21
|
+
/wrmsr--omlish--<placeholder>
|
20
22
|
|
21
|
-
|
23
|
+
placeholder = <name>--<rev>--<when>
|
22
24
|
|
23
25
|
==
|
24
26
|
|
@@ -39,20 +41,23 @@ import dataclasses as dc
|
|
39
41
|
import os.path
|
40
42
|
import typing as ta
|
41
43
|
|
44
|
+
from omlish.lite.cached import cached_nullary
|
42
45
|
from omlish.lite.check import check
|
43
46
|
|
47
|
+
from .types import DeployHome
|
48
|
+
|
44
49
|
|
45
50
|
DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
|
46
|
-
|
51
|
+
DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
|
47
52
|
|
48
53
|
|
49
54
|
##
|
50
55
|
|
51
56
|
|
52
|
-
|
53
|
-
|
57
|
+
DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER = '@'
|
58
|
+
DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
|
54
59
|
|
55
|
-
|
60
|
+
DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
|
56
61
|
'app',
|
57
62
|
'tag', # <rev>-<dt>
|
58
63
|
])
|
@@ -70,7 +75,7 @@ class DeployPathPart(abc.ABC): # noqa
|
|
70
75
|
raise NotImplementedError
|
71
76
|
|
72
77
|
@abc.abstractmethod
|
73
|
-
def render(self,
|
78
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
74
79
|
raise NotImplementedError
|
75
80
|
|
76
81
|
|
@@ -84,9 +89,9 @@ class DirDeployPathPart(DeployPathPart, abc.ABC):
|
|
84
89
|
|
85
90
|
@classmethod
|
86
91
|
def parse(cls, s: str) -> 'DirDeployPathPart':
|
87
|
-
if
|
88
|
-
check.equal(s[0],
|
89
|
-
return
|
92
|
+
if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
|
93
|
+
check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
|
94
|
+
return PlaceholderDirDeployPathPart(s[1:])
|
90
95
|
else:
|
91
96
|
return ConstDirDeployPathPart(s)
|
92
97
|
|
@@ -98,13 +103,13 @@ class FileDeployPathPart(DeployPathPart, abc.ABC):
|
|
98
103
|
|
99
104
|
@classmethod
|
100
105
|
def parse(cls, s: str) -> 'FileDeployPathPart':
|
101
|
-
if
|
102
|
-
check.equal(s[0],
|
103
|
-
if not any(c in s for c in
|
104
|
-
return
|
106
|
+
if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
|
107
|
+
check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
|
108
|
+
if not any(c in s for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS):
|
109
|
+
return PlaceholderFileDeployPathPart(s[1:], '')
|
105
110
|
else:
|
106
|
-
p = min(f for c in
|
107
|
-
return
|
111
|
+
p = min(f for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS if (f := s.find(c)) > 0)
|
112
|
+
return PlaceholderFileDeployPathPart(s[1:p], s[p:])
|
108
113
|
else:
|
109
114
|
return ConstFileDeployPathPart(s)
|
110
115
|
|
@@ -119,9 +124,9 @@ class ConstDeployPathPart(DeployPathPart, abc.ABC):
|
|
119
124
|
def __post_init__(self) -> None:
|
120
125
|
check.non_empty_str(self.name)
|
121
126
|
check.not_in('/', self.name)
|
122
|
-
check.not_in(
|
127
|
+
check.not_in(DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, self.name)
|
123
128
|
|
124
|
-
def render(self,
|
129
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
125
130
|
return self.name
|
126
131
|
|
127
132
|
|
@@ -137,40 +142,40 @@ class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
|
|
137
142
|
|
138
143
|
|
139
144
|
@dc.dataclass(frozen=True)
|
140
|
-
class
|
141
|
-
|
145
|
+
class PlaceholderDeployPathPart(DeployPathPart, abc.ABC):
|
146
|
+
placeholder: str # DeployPathPlaceholder
|
142
147
|
|
143
148
|
def __post_init__(self) -> None:
|
144
|
-
check.non_empty_str(self.
|
145
|
-
for c in [*
|
146
|
-
check.not_in(c, self.
|
147
|
-
check.in_(self.
|
148
|
-
|
149
|
-
def
|
150
|
-
if
|
151
|
-
return
|
149
|
+
check.non_empty_str(self.placeholder)
|
150
|
+
for c in [*DEPLOY_PATH_PLACEHOLDER_SEPARATORS, DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
|
151
|
+
check.not_in(c, self.placeholder)
|
152
|
+
check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
|
153
|
+
|
154
|
+
def _render_placeholder(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
155
|
+
if placeholders is not None:
|
156
|
+
return placeholders[self.placeholder] # type: ignore
|
152
157
|
else:
|
153
|
-
return
|
158
|
+
return DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER + self.placeholder
|
154
159
|
|
155
160
|
|
156
161
|
@dc.dataclass(frozen=True)
|
157
|
-
class
|
158
|
-
def render(self,
|
159
|
-
return self.
|
162
|
+
class PlaceholderDirDeployPathPart(PlaceholderDeployPathPart, DirDeployPathPart):
|
163
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
164
|
+
return self._render_placeholder(placeholders)
|
160
165
|
|
161
166
|
|
162
167
|
@dc.dataclass(frozen=True)
|
163
|
-
class
|
168
|
+
class PlaceholderFileDeployPathPart(PlaceholderDeployPathPart, FileDeployPathPart):
|
164
169
|
suffix: str
|
165
170
|
|
166
171
|
def __post_init__(self) -> None:
|
167
172
|
super().__post_init__()
|
168
173
|
if self.suffix:
|
169
|
-
for c in [
|
174
|
+
for c in [DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
|
170
175
|
check.not_in(c, self.suffix)
|
171
176
|
|
172
|
-
def render(self,
|
173
|
-
return self.
|
177
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
178
|
+
return self._render_placeholder(placeholders) + self.suffix
|
174
179
|
|
175
180
|
|
176
181
|
##
|
@@ -181,28 +186,30 @@ class DeployPath:
|
|
181
186
|
parts: ta.Sequence[DeployPathPart]
|
182
187
|
|
183
188
|
def __post_init__(self) -> None:
|
189
|
+
hash(self)
|
190
|
+
|
184
191
|
check.not_empty(self.parts)
|
185
192
|
for p in self.parts[:-1]:
|
186
193
|
check.equal(p.kind, 'dir')
|
187
194
|
|
188
195
|
pd = {}
|
189
196
|
for i, p in enumerate(self.parts):
|
190
|
-
if isinstance(p,
|
191
|
-
if p.
|
192
|
-
raise DeployPathError('Duplicate
|
193
|
-
pd[p.
|
197
|
+
if isinstance(p, PlaceholderDeployPathPart):
|
198
|
+
if p.placeholder in pd:
|
199
|
+
raise DeployPathError('Duplicate placeholders in path', self)
|
200
|
+
pd[p.placeholder] = i
|
194
201
|
|
195
202
|
if 'tag' in pd:
|
196
203
|
if 'app' not in pd or pd['app'] >= pd['tag']:
|
197
|
-
raise DeployPathError('Tag
|
204
|
+
raise DeployPathError('Tag placeholder in path without preceding app', self)
|
198
205
|
|
199
206
|
@property
|
200
207
|
def kind(self) -> ta.Literal['file', 'dir']:
|
201
208
|
return self.parts[-1].kind
|
202
209
|
|
203
|
-
def render(self,
|
210
|
+
def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
|
204
211
|
return os.path.join( # noqa
|
205
|
-
*[p.render(
|
212
|
+
*[p.render(placeholders) for p in self.parts],
|
206
213
|
*([''] if self.kind == 'dir' else []),
|
207
214
|
)
|
208
215
|
|
@@ -215,10 +222,10 @@ class DeployPath:
|
|
215
222
|
else:
|
216
223
|
tail_parse = FileDeployPathPart.parse
|
217
224
|
ps = check.non_empty_str(s).split('/')
|
218
|
-
return cls(
|
225
|
+
return cls((
|
219
226
|
*([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
|
220
227
|
tail_parse(ps[-1]),
|
221
|
-
|
228
|
+
))
|
222
229
|
|
223
230
|
|
224
231
|
##
|
@@ -226,5 +233,36 @@ class DeployPath:
|
|
226
233
|
|
227
234
|
class DeployPathOwner(abc.ABC):
|
228
235
|
@abc.abstractmethod
|
229
|
-
def
|
236
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
230
237
|
raise NotImplementedError
|
238
|
+
|
239
|
+
|
240
|
+
class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
|
241
|
+
def __init__(
|
242
|
+
self,
|
243
|
+
*args: ta.Any,
|
244
|
+
owned_dir: str,
|
245
|
+
deploy_home: ta.Optional[DeployHome],
|
246
|
+
**kwargs: ta.Any,
|
247
|
+
) -> None:
|
248
|
+
super().__init__(*args, **kwargs)
|
249
|
+
|
250
|
+
check.not_in('/', owned_dir)
|
251
|
+
self._owned_dir: str = check.non_empty_str(owned_dir)
|
252
|
+
|
253
|
+
self._deploy_home = deploy_home
|
254
|
+
|
255
|
+
self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
|
256
|
+
|
257
|
+
@cached_nullary
|
258
|
+
def _dir(self) -> str:
|
259
|
+
return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
|
260
|
+
|
261
|
+
@cached_nullary
|
262
|
+
def _make_dir(self) -> str:
|
263
|
+
if not os.path.isdir(d := self._dir()):
|
264
|
+
os.makedirs(d, exist_ok=True)
|
265
|
+
return d
|
266
|
+
|
267
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
268
|
+
return self._owned_deploy_paths
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# ruff: noqa: UP006 UP007
|
2
|
+
import dataclasses as dc
|
3
|
+
import hashlib
|
4
|
+
import typing as ta
|
5
|
+
|
6
|
+
from omlish.lite.cached import cached_nullary
|
7
|
+
from omlish.lite.check import check
|
8
|
+
|
9
|
+
from .types import DeployApp
|
10
|
+
from .types import DeployKey
|
11
|
+
from .types import DeployRev
|
12
|
+
|
13
|
+
|
14
|
+
##
|
15
|
+
|
16
|
+
|
17
|
+
@dc.dataclass(frozen=True)
|
18
|
+
class DeployGitRepo:
|
19
|
+
host: ta.Optional[str] = None
|
20
|
+
username: ta.Optional[str] = None
|
21
|
+
path: ta.Optional[str] = None
|
22
|
+
|
23
|
+
def __post_init__(self) -> None:
|
24
|
+
check.not_in('..', check.non_empty_str(self.host))
|
25
|
+
check.not_in('.', check.non_empty_str(self.path))
|
26
|
+
|
27
|
+
|
28
|
+
##
|
29
|
+
|
30
|
+
|
31
|
+
@dc.dataclass(frozen=True)
|
32
|
+
class DeploySpec:
|
33
|
+
app: DeployApp
|
34
|
+
repo: DeployGitRepo
|
35
|
+
rev: DeployRev
|
36
|
+
|
37
|
+
def __post_init__(self) -> None:
|
38
|
+
hash(self)
|
39
|
+
|
40
|
+
@cached_nullary
|
41
|
+
def key(self) -> DeployKey:
|
42
|
+
return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# ruff: noqa: UP006 UP007
|
2
|
+
import typing as ta
|
3
|
+
|
4
|
+
from omlish.lite.cached import cached_nullary
|
5
|
+
from omlish.lite.check import check
|
6
|
+
|
7
|
+
from .atomics import DeployAtomicPathSwap
|
8
|
+
from .atomics import DeployAtomicPathSwapKind
|
9
|
+
from .atomics import DeployAtomicPathSwapping
|
10
|
+
from .atomics import TempDirDeployAtomicPathSwapping
|
11
|
+
from .paths import SingleDirDeployPathOwner
|
12
|
+
from .types import DeployHome
|
13
|
+
|
14
|
+
|
15
|
+
class DeployTmpManager(
|
16
|
+
SingleDirDeployPathOwner,
|
17
|
+
DeployAtomicPathSwapping,
|
18
|
+
):
|
19
|
+
def __init__(
|
20
|
+
self,
|
21
|
+
*,
|
22
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
23
|
+
) -> None:
|
24
|
+
super().__init__(
|
25
|
+
owned_dir='tmp',
|
26
|
+
deploy_home=deploy_home,
|
27
|
+
)
|
28
|
+
|
29
|
+
@cached_nullary
|
30
|
+
def _swapping(self) -> DeployAtomicPathSwapping:
|
31
|
+
return TempDirDeployAtomicPathSwapping(
|
32
|
+
temp_dir=self._make_dir(),
|
33
|
+
root_dir=check.non_empty_str(self._deploy_home),
|
34
|
+
)
|
35
|
+
|
36
|
+
def begin_atomic_path_swap(
|
37
|
+
self,
|
38
|
+
kind: DeployAtomicPathSwapKind,
|
39
|
+
dst_path: str,
|
40
|
+
**kwargs: ta.Any,
|
41
|
+
) -> DeployAtomicPathSwap:
|
42
|
+
return self._swapping().begin_atomic_path_swap(
|
43
|
+
kind,
|
44
|
+
dst_path,
|
45
|
+
**kwargs,
|
46
|
+
)
|
ominfra/manage/deploy/types.py
CHANGED
ominfra/manage/deploy/venvs.py
CHANGED
@@ -7,8 +7,11 @@ TODO:
|
|
7
7
|
import os.path
|
8
8
|
import typing as ta
|
9
9
|
|
10
|
-
from omlish.
|
10
|
+
from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
|
11
|
+
from omlish.lite.cached import cached_nullary
|
12
|
+
from omlish.lite.check import check
|
11
13
|
|
14
|
+
from .atomics import DeployAtomicPathSwapping
|
12
15
|
from .paths import DeployPath
|
13
16
|
from .paths import DeployPathOwner
|
14
17
|
from .types import DeployAppTag
|
@@ -19,14 +22,19 @@ class DeployVenvManager(DeployPathOwner):
|
|
19
22
|
def __init__(
|
20
23
|
self,
|
21
24
|
*,
|
22
|
-
deploy_home: DeployHome,
|
25
|
+
deploy_home: ta.Optional[DeployHome] = None,
|
26
|
+
atomics: DeployAtomicPathSwapping,
|
23
27
|
) -> None:
|
24
28
|
super().__init__()
|
25
29
|
|
26
30
|
self._deploy_home = deploy_home
|
27
|
-
self.
|
31
|
+
self._atomics = atomics
|
28
32
|
|
29
|
-
|
33
|
+
@cached_nullary
|
34
|
+
def _dir(self) -> str:
|
35
|
+
return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
|
36
|
+
|
37
|
+
def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
|
30
38
|
return {
|
31
39
|
DeployPath.parse('venvs/@app/@tag/'),
|
32
40
|
}
|
@@ -40,6 +48,8 @@ class DeployVenvManager(DeployPathOwner):
|
|
40
48
|
) -> None:
|
41
49
|
sys_exe = 'python3'
|
42
50
|
|
51
|
+
# !! NOTE: (most) venvs cannot be relocated, so an atomic swap can't be used. it's up to the path manager to
|
52
|
+
# garbage collect orphaned dirs.
|
43
53
|
await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
|
44
54
|
|
45
55
|
#
|
@@ -61,6 +71,6 @@ class DeployVenvManager(DeployPathOwner):
|
|
61
71
|
|
62
72
|
async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
|
63
73
|
await self.setup_venv(
|
64
|
-
os.path.join(self._deploy_home, 'apps', app_tag.app, app_tag.tag),
|
65
|
-
os.path.join(self.
|
74
|
+
os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
|
75
|
+
os.path.join(self._dir(), app_tag.app, app_tag.tag),
|
66
76
|
)
|
@@ -7,10 +7,10 @@ import shlex
|
|
7
7
|
import subprocess
|
8
8
|
import typing as ta
|
9
9
|
|
10
|
-
from omlish.
|
10
|
+
from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
|
11
11
|
from omlish.lite.check import check
|
12
|
-
from omlish.
|
13
|
-
from omlish.
|
12
|
+
from omlish.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
|
13
|
+
from omlish.subprocesses import SubprocessChannelOption
|
14
14
|
|
15
15
|
|
16
16
|
##
|
ominfra/pyremote.py
CHANGED
@@ -162,7 +162,7 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
|
|
162
162
|
# Get pid
|
163
163
|
pid = os.getpid()
|
164
164
|
|
165
|
-
# Two copies of
|
165
|
+
# Two copies of payload src to be sent to parent
|
166
166
|
r0, w0 = os.pipe()
|
167
167
|
r1, w1 = os.pipe()
|
168
168
|
|
@@ -201,17 +201,17 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
|
|
201
201
|
# Write pid
|
202
202
|
os.write(1, struct.pack('<Q', pid))
|
203
203
|
|
204
|
-
# Read
|
205
|
-
|
206
|
-
if len(
|
204
|
+
# Read payload src from stdin
|
205
|
+
payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
|
206
|
+
if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
|
207
207
|
raise EOFError
|
208
|
-
|
208
|
+
payload_src = zlib.decompress(payload_z)
|
209
209
|
|
210
|
-
# Write both copies of
|
211
|
-
# and block and need to be drained by pyremote_bootstrap_finalize running in parent.
|
210
|
+
# Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
|
211
|
+
# fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
|
212
212
|
for w in [w0, w1]:
|
213
213
|
fp = os.fdopen(w, 'wb', 0)
|
214
|
-
fp.write(
|
214
|
+
fp.write(payload_src)
|
215
215
|
fp.close()
|
216
216
|
|
217
217
|
# Write second ack
|
@@ -275,7 +275,7 @@ class PyremotePayloadRuntime:
|
|
275
275
|
input: ta.BinaryIO
|
276
276
|
output: ta.BinaryIO
|
277
277
|
context_name: str
|
278
|
-
|
278
|
+
payload_src: str
|
279
279
|
options: PyremoteBootstrapOptions
|
280
280
|
env_info: PyremoteEnvInfo
|
281
281
|
|
@@ -283,9 +283,9 @@ class PyremotePayloadRuntime:
|
|
283
283
|
def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
284
284
|
# If src file var is not present we need to do initial finalization
|
285
285
|
if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
|
286
|
-
# Read second copy of
|
286
|
+
# Read second copy of payload src
|
287
287
|
r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
|
288
|
-
|
288
|
+
payload_src = r1.read().decode('utf-8')
|
289
289
|
r1.close()
|
290
290
|
|
291
291
|
# Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
|
@@ -303,7 +303,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
303
303
|
# Write temp source file
|
304
304
|
import tempfile
|
305
305
|
tfd, tfn = tempfile.mkstemp('-pyremote.py')
|
306
|
-
os.write(tfd,
|
306
|
+
os.write(tfd, payload_src.encode('utf-8'))
|
307
307
|
os.close(tfd)
|
308
308
|
|
309
309
|
# Set vars
|
@@ -322,7 +322,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
322
322
|
|
323
323
|
# Read temp source file
|
324
324
|
with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
|
325
|
-
|
325
|
+
payload_src = sf.read()
|
326
326
|
|
327
327
|
# Restore vars
|
328
328
|
sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
|
@@ -355,7 +355,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
355
355
|
input=input,
|
356
356
|
output=output,
|
357
357
|
context_name=context_name,
|
358
|
-
|
358
|
+
payload_src=payload_src,
|
359
359
|
options=options,
|
360
360
|
env_info=env_info,
|
361
361
|
)
|
@@ -367,31 +367,31 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
|
|
367
367
|
class PyremoteBootstrapDriver:
|
368
368
|
def __init__(
|
369
369
|
self,
|
370
|
-
|
370
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
371
371
|
options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
|
372
372
|
) -> None:
|
373
373
|
super().__init__()
|
374
374
|
|
375
|
-
self.
|
375
|
+
self._payload_src = payload_src
|
376
376
|
self._options = options
|
377
377
|
|
378
|
-
self.
|
379
|
-
self.
|
378
|
+
self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
|
379
|
+
self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
|
380
380
|
|
381
381
|
self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
|
382
382
|
#
|
383
383
|
|
384
384
|
@classmethod
|
385
|
-
def
|
385
|
+
def _prepare_payload_src(
|
386
386
|
cls,
|
387
|
-
|
387
|
+
payload_src: ta.Union[str, ta.Sequence[str]],
|
388
388
|
options: PyremoteBootstrapOptions,
|
389
389
|
) -> str:
|
390
390
|
parts: ta.List[str]
|
391
|
-
if isinstance(
|
392
|
-
parts = [
|
391
|
+
if isinstance(payload_src, str):
|
392
|
+
parts = [payload_src]
|
393
393
|
else:
|
394
|
-
parts = list(
|
394
|
+
parts = list(payload_src)
|
395
395
|
|
396
396
|
if (mn := options.main_name_override) is not None:
|
397
397
|
parts.insert(0, f'__name__ = {mn!r}')
|
@@ -427,9 +427,9 @@ class PyremoteBootstrapDriver:
|
|
427
427
|
d = yield from self._read(8)
|
428
428
|
pid = struct.unpack('<Q', d)[0]
|
429
429
|
|
430
|
-
# Write
|
431
|
-
yield from self._write(struct.pack('<I', len(self.
|
432
|
-
yield from self._write(self.
|
430
|
+
# Write payload src
|
431
|
+
yield from self._write(struct.pack('<I', len(self._payload_z)))
|
432
|
+
yield from self._write(self._payload_z)
|
433
433
|
|
434
434
|
# Read second ack (after writing src copies)
|
435
435
|
yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
|