ominfra 0.0.0.dev140__tar.gz → 0.0.0.dev142__tar.gz

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. {ominfra-0.0.0.dev140/ominfra.egg-info → ominfra-0.0.0.dev142}/PKG-INFO +3 -3
  2. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/commands/subprocess.py +18 -17
  3. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/main.py +25 -63
  4. ominfra-0.0.0.dev142/ominfra/manage/protocol.py +45 -0
  5. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/spawning.py +4 -8
  6. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/pyremote.py +13 -11
  7. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/scripts/journald2aws.py +123 -86
  8. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/scripts/manage.py +219 -176
  9. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/scripts/supervisor.py +110 -86
  10. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142/ominfra.egg-info}/PKG-INFO +3 -3
  11. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra.egg-info/SOURCES.txt +1 -0
  12. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra.egg-info/requires.txt +2 -2
  13. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/pyproject.toml +3 -3
  14. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/LICENSE +0 -0
  15. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/MANIFEST.in +0 -0
  16. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/README.rst +0 -0
  17. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/.manifests.json +0 -0
  18. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/__about__.py +0 -0
  19. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/__init__.py +0 -0
  20. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/__init__.py +0 -0
  21. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/__init__.py +0 -0
  22. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/__main__.py +0 -0
  23. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/auth.py +0 -0
  24. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/cli.py +0 -0
  25. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/dataclasses.py +0 -0
  26. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/__init__.py +0 -0
  27. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/__main__.py +0 -0
  28. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/cursor.py +0 -0
  29. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/driver.py +0 -0
  30. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/main.py +0 -0
  31. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/journald2aws/poster.py +0 -0
  32. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/logs.py +0 -0
  33. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/aws/metadata.py +0 -0
  34. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/gcp/__init__.py +0 -0
  35. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/clouds/gcp/auth.py +0 -0
  36. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/cmds.py +0 -0
  37. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/configs.py +0 -0
  38. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/journald/__init__.py +0 -0
  39. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/journald/fields.py +0 -0
  40. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/journald/genmessages.py +0 -0
  41. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/journald/messages.py +0 -0
  42. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/journald/tailer.py +0 -0
  43. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/__init__.py +0 -0
  44. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/commands/__init__.py +0 -0
  45. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/commands/base.py +0 -0
  46. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/manage/payload.py +0 -0
  47. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/scripts/__init__.py +0 -0
  48. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/ssh.py +0 -0
  49. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/LICENSE.txt +0 -0
  50. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/__init__.py +0 -0
  51. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/__main__.py +0 -0
  52. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/configs.py +0 -0
  53. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/dispatchers.py +0 -0
  54. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/dispatchersimpl.py +0 -0
  55. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/events.py +0 -0
  56. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/exceptions.py +0 -0
  57. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/groups.py +0 -0
  58. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/groupsimpl.py +0 -0
  59. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/http.py +0 -0
  60. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/inject.py +0 -0
  61. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/io.py +0 -0
  62. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/main.py +0 -0
  63. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/pipes.py +0 -0
  64. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/privileges.py +0 -0
  65. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/process.py +0 -0
  66. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/processimpl.py +0 -0
  67. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/setup.py +0 -0
  68. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/setupimpl.py +0 -0
  69. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/signals.py +0 -0
  70. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/spawning.py +0 -0
  71. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/spawningimpl.py +0 -0
  72. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/states.py +0 -0
  73. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/supervisor.py +0 -0
  74. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/types.py +0 -0
  75. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/__init__.py +0 -0
  76. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/collections.py +0 -0
  77. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/diag.py +0 -0
  78. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/fds.py +0 -0
  79. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/fs.py +0 -0
  80. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/os.py +0 -0
  81. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/ostypes.py +0 -0
  82. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/signals.py +0 -0
  83. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/strings.py +0 -0
  84. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/supervisor/utils/users.py +0 -0
  85. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/tailscale/__init__.py +0 -0
  86. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/tailscale/api.py +0 -0
  87. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/tailscale/cli.py +0 -0
  88. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/threadworkers.py +0 -0
  89. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/tools/__init__.py +0 -0
  90. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra/tools/listresources.py +0 -0
  91. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra.egg-info/dependency_links.txt +0 -0
  92. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra.egg-info/entry_points.txt +0 -0
  93. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/ominfra.egg-info/top_level.txt +0 -0
  94. {ominfra-0.0.0.dev140 → ominfra-0.0.0.dev142}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ominfra
3
- Version: 0.0.0.dev140
3
+ Version: 0.0.0.dev142
4
4
  Summary: ominfra
5
5
  Author: wrmsr
6
6
  License: BSD-3-Clause
@@ -12,8 +12,8 @@ Classifier: Operating System :: OS Independent
12
12
  Classifier: Operating System :: POSIX
13
13
  Requires-Python: >=3.12
14
14
  License-File: LICENSE
15
- Requires-Dist: omdev==0.0.0.dev140
16
- Requires-Dist: omlish==0.0.0.dev140
15
+ Requires-Dist: omdev==0.0.0.dev142
16
+ Requires-Dist: omlish==0.0.0.dev142
17
17
  Provides-Extra: all
18
18
  Requires-Dist: paramiko~=3.5; extra == "all"
19
19
  Requires-Dist: asyncssh~=2.18; extra == "all"
@@ -5,6 +5,8 @@ import subprocess
5
5
  import time
6
6
  import typing as ta
7
7
 
8
+ from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
9
+ from omlish.lite.subprocesses import SubprocessChannelOption
8
10
  from omlish.lite.subprocesses import subprocess_maybe_shell_wrap_exec
9
11
 
10
12
  from .base import Command
@@ -16,21 +18,21 @@ from .base import CommandExecutor
16
18
 
17
19
  @dc.dataclass(frozen=True)
18
20
  class SubprocessCommand(Command['SubprocessCommand.Output']):
19
- args: ta.Sequence[str]
21
+ cmd: ta.Sequence[str]
20
22
 
21
23
  shell: bool = False
22
24
  cwd: ta.Optional[str] = None
23
25
  env: ta.Optional[ta.Mapping[str, str]] = None
24
26
 
25
- capture_stdout: bool = False
26
- capture_stderr: bool = False
27
+ stdout: str = 'pipe' # SubprocessChannelOption
28
+ stderr: str = 'pipe' # SubprocessChannelOption
27
29
 
28
30
  input: ta.Optional[bytes] = None
29
31
  timeout: ta.Optional[float] = None
30
32
 
31
33
  def __post_init__(self) -> None:
32
- if isinstance(self.args, str):
33
- raise TypeError(self.args)
34
+ if isinstance(self.cmd, str):
35
+ raise TypeError(self.cmd)
34
36
 
35
37
  @dc.dataclass(frozen=True)
36
38
  class Output(Command.Output):
@@ -48,24 +50,23 @@ class SubprocessCommand(Command['SubprocessCommand.Output']):
48
50
 
49
51
  class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
50
52
  def execute(self, inp: SubprocessCommand) -> SubprocessCommand.Output:
51
- proc = subprocess.Popen(
52
- subprocess_maybe_shell_wrap_exec(*inp.args),
53
+ with subprocess.Popen(
54
+ subprocess_maybe_shell_wrap_exec(*inp.cmd),
53
55
 
54
56
  shell=inp.shell,
55
57
  cwd=inp.cwd,
56
58
  env={**os.environ, **(inp.env or {})},
57
59
 
58
60
  stdin=subprocess.PIPE if inp.input is not None else None,
59
- stdout=subprocess.PIPE if inp.capture_stdout else None,
60
- stderr=subprocess.PIPE if inp.capture_stderr else None,
61
- )
62
-
63
- start_time = time.time()
64
- stdout, stderr = proc.communicate(
65
- input=inp.input,
66
- timeout=inp.timeout,
67
- )
68
- end_time = time.time()
61
+ stdout=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stdout)],
62
+ stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[ta.cast(SubprocessChannelOption, inp.stderr)],
63
+ ) as proc:
64
+ start_time = time.time()
65
+ stdout, stderr = proc.communicate(
66
+ input=inp.input,
67
+ timeout=inp.timeout,
68
+ )
69
+ end_time = time.time()
69
70
 
70
71
  return SubprocessCommand.Output(
71
72
  rc=proc.returncode,
@@ -5,17 +5,9 @@
5
5
  manage.py -s 'docker run -i python:3.12'
6
6
  manage.py -s 'ssh -i /foo/bar.pem foo@bar.baz' -q --python=python3.8
7
7
  """
8
- import json
9
- import struct
10
- import typing as ta
11
-
12
- from omlish.lite.cached import static_init
13
- from omlish.lite.json import json_dumps_compact
8
+ from omlish.lite.marshal import OBJ_MARSHALER_MANAGER
9
+ from omlish.lite.marshal import ObjMarshalerManager
14
10
  from omlish.lite.marshal import PolymorphicObjMarshaler
15
- from omlish.lite.marshal import get_obj_marshaler
16
- from omlish.lite.marshal import marshal_obj
17
- from omlish.lite.marshal import register_opj_marshaler
18
- from omlish.lite.marshal import unmarshal_obj
19
11
 
20
12
  from ..pyremote import PyremoteBootstrapDriver
21
13
  from ..pyremote import PyremoteBootstrapOptions
@@ -25,6 +17,7 @@ from .commands.base import Command
25
17
  from .commands.subprocess import SubprocessCommand
26
18
  from .commands.subprocess import SubprocessCommandExecutor
27
19
  from .payload import get_payload_src
20
+ from .protocol import Channel
28
21
  from .spawning import PySpawner
29
22
 
30
23
 
@@ -36,51 +29,28 @@ _COMMAND_TYPES = {
36
29
  }
37
30
 
38
31
 
39
- @static_init
40
- def _register_command_marshaling() -> None:
32
+ ##
33
+
34
+
35
+ def register_command_marshaling(msh: ObjMarshalerManager) -> None:
41
36
  for fn in [
42
37
  lambda c: c,
43
38
  lambda c: c.Output,
44
39
  ]:
45
- register_opj_marshaler(
40
+ msh.register_opj_marshaler(
46
41
  fn(Command),
47
42
  PolymorphicObjMarshaler.of([
48
43
  PolymorphicObjMarshaler.Impl(
49
44
  fn(cty),
50
45
  k,
51
- get_obj_marshaler(fn(cty)),
46
+ msh.get_obj_marshaler(fn(cty)),
52
47
  )
53
48
  for k, cty in _COMMAND_TYPES.items()
54
49
  ]),
55
50
  )
56
51
 
57
52
 
58
- ##
59
-
60
-
61
- def _send_obj(f: ta.IO, o: ta.Any, ty: ta.Any = None) -> None:
62
- j = json_dumps_compact(marshal_obj(o, ty))
63
- d = j.encode('utf-8')
64
-
65
- f.write(struct.pack('<I', len(d)))
66
- f.write(d)
67
- f.flush()
68
-
69
-
70
- def _recv_obj(f: ta.IO, ty: ta.Any) -> ta.Any:
71
- d = f.read(4)
72
- if not d:
73
- return None
74
- if len(d) != 4:
75
- raise EOFError
76
-
77
- sz = struct.unpack('<I', d)[0]
78
- d = f.read(sz)
79
- if len(d) != sz:
80
- raise EOFError
81
-
82
- j = json.loads(d.decode('utf-8'))
83
- return unmarshal_obj(j, ty)
53
+ register_command_marshaling(OBJ_MARSHALER_MANAGER)
84
54
 
85
55
 
86
56
  ##
@@ -88,9 +58,10 @@ def _recv_obj(f: ta.IO, ty: ta.Any) -> ta.Any:
88
58
 
89
59
  def _remote_main() -> None:
90
60
  rt = pyremote_bootstrap_finalize() # noqa
61
+ chan = Channel(rt.input, rt.output)
91
62
 
92
63
  while True:
93
- i = _recv_obj(rt.input, Command)
64
+ i = chan.recv_obj(Command)
94
65
  if i is None:
95
66
  break
96
67
 
@@ -99,7 +70,7 @@ def _remote_main() -> None:
99
70
  else:
100
71
  raise TypeError(i)
101
72
 
102
- _send_obj(rt.output, o, Command.Output)
73
+ chan.send_obj(o, Command.Output)
103
74
 
104
75
 
105
76
  ##
@@ -110,11 +81,13 @@ def _main() -> None:
110
81
 
111
82
  parser = argparse.ArgumentParser()
112
83
 
84
+ parser.add_argument('--_payload-file')
85
+
113
86
  parser.add_argument('-s', '--shell')
114
87
  parser.add_argument('-q', '--shell-quote', action='store_true')
115
88
  parser.add_argument('--python', default='python3')
89
+
116
90
  parser.add_argument('--debug', action='store_true')
117
- parser.add_argument('--_payload-file')
118
91
 
119
92
  args = parser.parse_args()
120
93
 
@@ -122,8 +95,6 @@ def _main() -> None:
122
95
 
123
96
  payload_src = get_payload_src(file=args._payload_file) # noqa
124
97
 
125
- #
126
-
127
98
  remote_src = '\n\n'.join([
128
99
  '__name__ = "__remote__"',
129
100
  payload_src,
@@ -132,41 +103,32 @@ def _main() -> None:
132
103
 
133
104
  #
134
105
 
135
- bs_src = pyremote_build_bootstrap_cmd(__package__ or 'manage')
136
-
137
- #
138
-
139
106
  spawner = PySpawner(
140
- bs_src,
107
+ pyremote_build_bootstrap_cmd(__package__ or 'manage'),
141
108
  shell=args.shell,
142
109
  shell_quote=args.shell_quote,
143
110
  python=args.python,
144
111
  )
112
+
145
113
  with spawner.spawn() as proc:
146
114
  res = PyremoteBootstrapDriver( # noqa
147
115
  remote_src,
148
116
  PyremoteBootstrapOptions(
149
117
  debug=args.debug,
150
118
  ),
151
- ).run(proc.stdin, proc.stdout)
152
- # print(res)
119
+ ).run(proc.stdout, proc.stdin)
120
+
121
+ chan = Channel(proc.stdout, proc.stdin)
153
122
 
154
123
  #
155
124
 
156
125
  for ci in [
157
- SubprocessCommand(
158
- args=['python3', '-'],
159
- input=b'print(1)\n',
160
- capture_stdout=True,
161
- ),
162
- SubprocessCommand(
163
- args=['uname'],
164
- capture_stdout=True,
165
- ),
126
+ SubprocessCommand(['python3', '-'], input=b'print(1)\n'),
127
+ SubprocessCommand(['uname']),
166
128
  ]:
167
- _send_obj(proc.stdin, ci, Command)
129
+ chan.send_obj(ci, Command)
168
130
 
169
- o = _recv_obj(proc.stdout, Command.Output)
131
+ o = chan.recv_obj(Command.Output)
170
132
 
171
133
  print(o)
172
134
 
@@ -0,0 +1,45 @@
1
+ import json
2
+ import struct
3
+ import typing as ta
4
+
5
+ from omlish.lite.json import json_dumps_compact
6
+ from omlish.lite.marshal import OBJ_MARSHALER_MANAGER
7
+ from omlish.lite.marshal import ObjMarshalerManager
8
+
9
+
10
+ class Channel:
11
+ def __init__(
12
+ self,
13
+ input: ta.IO, # noqa
14
+ output: ta.IO,
15
+ *,
16
+ msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
17
+ ) -> None:
18
+ super().__init__()
19
+
20
+ self._input = input
21
+ self._output = output
22
+ self._msh = msh
23
+
24
+ def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
25
+ j = json_dumps_compact(self._msh.marshal_obj(o, ty))
26
+ d = j.encode('utf-8')
27
+
28
+ self._output.write(struct.pack('<I', len(d)))
29
+ self._output.write(d)
30
+ self._output.flush()
31
+
32
+ def recv_obj(self, ty: ta.Any) -> ta.Any:
33
+ d = self._input.read(4)
34
+ if not d:
35
+ return None
36
+ if len(d) != 4:
37
+ raise EOFError
38
+
39
+ sz = struct.unpack('<I', d)[0]
40
+ d = self._input.read(sz)
41
+ if len(d) != sz:
42
+ raise EOFError
43
+
44
+ j = json.loads(d.decode('utf-8'))
45
+ return self._msh.unmarshal_obj(j, ty)
@@ -6,6 +6,8 @@ import subprocess
6
6
  import typing as ta
7
7
 
8
8
  from omlish.lite.check import check_not_none
9
+ from omlish.lite.subprocesses import SUBPROCESS_CHANNEL_OPTION_VALUES
10
+ from omlish.lite.subprocesses import SubprocessChannelOption
9
11
  from omlish.lite.subprocesses import subprocess_maybe_shell_wrap_exec
10
12
 
11
13
 
@@ -19,7 +21,7 @@ class PySpawner:
19
21
  shell: ta.Optional[str] = None,
20
22
  shell_quote: bool = False,
21
23
  python: str = DEFAULT_PYTHON,
22
- stderr: ta.Optional[ta.Literal['pipe', 'stdout', 'devnull']] = None,
24
+ stderr: ta.Optional[SubprocessChannelOption] = None,
23
25
  ) -> None:
24
26
  super().__init__()
25
27
 
@@ -54,12 +56,6 @@ class PySpawner:
54
56
 
55
57
  #
56
58
 
57
- _STDERR_KWARG_MAP: ta.Mapping[str, int] = {
58
- 'pipe': subprocess.PIPE,
59
- 'stdout': subprocess.STDOUT,
60
- 'devnull': subprocess.DEVNULL,
61
- }
62
-
63
59
  @dc.dataclass(frozen=True)
64
60
  class Spawned:
65
61
  stdin: ta.IO
@@ -79,7 +75,7 @@ class PySpawner:
79
75
  shell=pc.shell,
80
76
  stdin=subprocess.PIPE,
81
77
  stdout=subprocess.PIPE,
82
- stderr=self._STDERR_KWARG_MAP[self._stderr] if self._stderr is not None else None,
78
+ stderr=SUBPROCESS_CHANNEL_OPTION_VALUES[self._stderr] if self._stderr is not None else None,
83
79
  ) as proc:
84
80
  stdin = check_not_none(proc.stdin)
85
81
  stdout = check_not_none(proc.stdout)
@@ -177,12 +177,14 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
177
177
  os.close(f)
178
178
 
179
179
  # Save vars
180
- os.environ[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
181
- os.environ[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = sys.executable
182
- os.environ[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
180
+ env = os.environ
181
+ exe = sys.executable
182
+ env[_PYREMOTE_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
183
+ env[_PYREMOTE_BOOTSTRAP_ARGV0_VAR] = exe
184
+ env[_PYREMOTE_BOOTSTRAP_CONTEXT_NAME_VAR] = context_name
183
185
 
184
186
  # Start repl reading stdin from r0
185
- os.execl(sys.executable, sys.executable + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
187
+ os.execl(exe, exe + (_PYREMOTE_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
186
188
 
187
189
  else:
188
190
  # Child process
@@ -246,12 +248,12 @@ def pyremote_build_bootstrap_cmd(context_name: str) -> str:
246
248
  if cl.strip()
247
249
  )
248
250
 
249
- bs_z = zlib.compress(bs_src.encode('utf-8'))
250
- bs_z64 = base64.encodebytes(bs_z).replace(b'\n', b'')
251
+ bs_z = zlib.compress(bs_src.encode('utf-8'), 9)
252
+ bs_z85 = base64.b85encode(bs_z).replace(b'\n', b'')
251
253
 
252
254
  stmts = [
253
255
  f'import {", ".join(_PYREMOTE_BOOTSTRAP_IMPORTS)}',
254
- f'exec(zlib.decompress(base64.decodebytes({bs_z64!r})))',
256
+ f'exec(zlib.decompress(base64.b85decode({bs_z85!r})))',
255
257
  f'_pyremote_bootstrap_main({context_name!r})',
256
258
  ]
257
259
 
@@ -440,7 +442,7 @@ class PyremoteBootstrapDriver:
440
442
 
441
443
  #
442
444
 
443
- def run(self, stdin: ta.IO, stdout: ta.IO) -> Result:
445
+ def run(self, input: ta.IO, output: ta.IO) -> Result: # noqa
444
446
  gen = self.gen()
445
447
 
446
448
  gi: ta.Optional[bytes] = None
@@ -454,11 +456,11 @@ class PyremoteBootstrapDriver:
454
456
  return e.value
455
457
 
456
458
  if isinstance(go, self.Read):
457
- if len(gi := stdout.read(go.sz)) != go.sz:
459
+ if len(gi := input.read(go.sz)) != go.sz:
458
460
  raise EOFError
459
461
  elif isinstance(go, self.Write):
460
462
  gi = None
461
- stdin.write(go.d)
462
- stdin.flush()
463
+ output.write(go.d)
464
+ output.flush()
463
465
  else:
464
466
  raise TypeError(go)
@@ -72,6 +72,9 @@ ConfigMapping = ta.Mapping[str, ta.Any]
72
72
  # ../../../threadworkers.py
73
73
  ThreadWorkerT = ta.TypeVar('ThreadWorkerT', bound='ThreadWorker')
74
74
 
75
+ # ../../../../omlish/lite/subprocesses.py
76
+ SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull']
77
+
75
78
 
76
79
  ########################################
77
80
  # ../../../../../omdev/toml/parser.py
@@ -2382,120 +2385,144 @@ _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
2382
2385
  }
2383
2386
 
2384
2387
 
2385
- def _make_obj_marshaler(
2386
- ty: ta.Any,
2387
- rec: ta.Callable[[ta.Any], ObjMarshaler],
2388
- *,
2389
- nonstrict_dataclasses: bool = False,
2390
- ) -> ObjMarshaler:
2391
- if isinstance(ty, type):
2392
- if abc.ABC in ty.__bases__:
2393
- return PolymorphicObjMarshaler.of([ # type: ignore
2394
- PolymorphicObjMarshaler.Impl(
2395
- ity,
2396
- ity.__qualname__,
2397
- rec(ity),
2398
- )
2399
- for ity in deep_subclasses(ty)
2400
- if abc.ABC not in ity.__bases__
2401
- ])
2402
-
2403
- if issubclass(ty, enum.Enum):
2404
- return EnumObjMarshaler(ty)
2405
-
2406
- if dc.is_dataclass(ty):
2407
- return DataclassObjMarshaler(
2408
- ty,
2409
- {f.name: rec(f.type) for f in dc.fields(ty)},
2410
- nonstrict=nonstrict_dataclasses,
2411
- )
2388
+ ##
2412
2389
 
2413
- if is_generic_alias(ty):
2414
- try:
2415
- mt = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES[ta.get_origin(ty)]
2416
- except KeyError:
2417
- pass
2418
- else:
2419
- k, v = ta.get_args(ty)
2420
- return MappingObjMarshaler(mt, rec(k), rec(v))
2421
2390
 
2422
- try:
2423
- st = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES[ta.get_origin(ty)]
2424
- except KeyError:
2425
- pass
2426
- else:
2427
- [e] = ta.get_args(ty)
2428
- return IterableObjMarshaler(st, rec(e))
2391
+ class ObjMarshalerManager:
2392
+ def __init__(
2393
+ self,
2394
+ *,
2395
+ default_obj_marshalers: ta.Dict[ta.Any, ObjMarshaler] = _DEFAULT_OBJ_MARSHALERS, # noqa
2396
+ generic_mapping_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES, # noqa
2397
+ generic_iterable_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES, # noqa
2398
+ ) -> None:
2399
+ super().__init__()
2429
2400
 
2430
- if is_union_alias(ty):
2431
- return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
2401
+ self._obj_marshalers = dict(default_obj_marshalers)
2402
+ self._generic_mapping_types = generic_mapping_types
2403
+ self._generic_iterable_types = generic_iterable_types
2432
2404
 
2433
- raise TypeError(ty)
2405
+ self._lock = threading.RLock()
2406
+ self._marshalers: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
2407
+ self._proxies: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
2434
2408
 
2409
+ #
2435
2410
 
2436
- ##
2411
+ def make_obj_marshaler(
2412
+ self,
2413
+ ty: ta.Any,
2414
+ rec: ta.Callable[[ta.Any], ObjMarshaler],
2415
+ *,
2416
+ nonstrict_dataclasses: bool = False,
2417
+ ) -> ObjMarshaler:
2418
+ if isinstance(ty, type):
2419
+ if abc.ABC in ty.__bases__:
2420
+ return PolymorphicObjMarshaler.of([ # type: ignore
2421
+ PolymorphicObjMarshaler.Impl(
2422
+ ity,
2423
+ ity.__qualname__,
2424
+ rec(ity),
2425
+ )
2426
+ for ity in deep_subclasses(ty)
2427
+ if abc.ABC not in ity.__bases__
2428
+ ])
2429
+
2430
+ if issubclass(ty, enum.Enum):
2431
+ return EnumObjMarshaler(ty)
2432
+
2433
+ if dc.is_dataclass(ty):
2434
+ return DataclassObjMarshaler(
2435
+ ty,
2436
+ {f.name: rec(f.type) for f in dc.fields(ty)},
2437
+ nonstrict=nonstrict_dataclasses,
2438
+ )
2437
2439
 
2440
+ if is_generic_alias(ty):
2441
+ try:
2442
+ mt = self._generic_mapping_types[ta.get_origin(ty)]
2443
+ except KeyError:
2444
+ pass
2445
+ else:
2446
+ k, v = ta.get_args(ty)
2447
+ return MappingObjMarshaler(mt, rec(k), rec(v))
2438
2448
 
2439
- _OBJ_MARSHALERS_LOCK = threading.RLock()
2449
+ try:
2450
+ st = self._generic_iterable_types[ta.get_origin(ty)]
2451
+ except KeyError:
2452
+ pass
2453
+ else:
2454
+ [e] = ta.get_args(ty)
2455
+ return IterableObjMarshaler(st, rec(e))
2440
2456
 
2441
- _OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
2457
+ if is_union_alias(ty):
2458
+ return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
2442
2459
 
2443
- _OBJ_MARSHALER_PROXIES: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
2460
+ raise TypeError(ty)
2444
2461
 
2462
+ #
2445
2463
 
2446
- def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
2447
- with _OBJ_MARSHALERS_LOCK:
2448
- if ty in _OBJ_MARSHALERS:
2449
- raise KeyError(ty)
2450
- _OBJ_MARSHALERS[ty] = m
2464
+ def register_opj_marshaler(self, ty: ta.Any, m: ObjMarshaler) -> None:
2465
+ with self._lock:
2466
+ if ty in self._obj_marshalers:
2467
+ raise KeyError(ty)
2468
+ self._obj_marshalers[ty] = m
2451
2469
 
2470
+ def get_obj_marshaler(
2471
+ self,
2472
+ ty: ta.Any,
2473
+ *,
2474
+ no_cache: bool = False,
2475
+ **kwargs: ta.Any,
2476
+ ) -> ObjMarshaler:
2477
+ with self._lock:
2478
+ if not no_cache:
2479
+ try:
2480
+ return self._obj_marshalers[ty]
2481
+ except KeyError:
2482
+ pass
2452
2483
 
2453
- def get_obj_marshaler(
2454
- ty: ta.Any,
2455
- *,
2456
- no_cache: bool = False,
2457
- **kwargs: ta.Any,
2458
- ) -> ObjMarshaler:
2459
- with _OBJ_MARSHALERS_LOCK:
2460
- if not no_cache:
2461
2484
  try:
2462
- return _OBJ_MARSHALERS[ty]
2485
+ return self._proxies[ty]
2463
2486
  except KeyError:
2464
2487
  pass
2465
2488
 
2466
- try:
2467
- return _OBJ_MARSHALER_PROXIES[ty]
2468
- except KeyError:
2469
- pass
2489
+ rec = functools.partial(
2490
+ self.get_obj_marshaler,
2491
+ no_cache=no_cache,
2492
+ **kwargs,
2493
+ )
2470
2494
 
2471
- rec = functools.partial(
2472
- get_obj_marshaler,
2473
- no_cache=no_cache,
2474
- **kwargs,
2475
- )
2495
+ p = ProxyObjMarshaler()
2496
+ self._proxies[ty] = p
2497
+ try:
2498
+ m = self.make_obj_marshaler(ty, rec, **kwargs)
2499
+ finally:
2500
+ del self._proxies[ty]
2501
+ p.m = m
2476
2502
 
2477
- p = ProxyObjMarshaler()
2478
- _OBJ_MARSHALER_PROXIES[ty] = p
2479
- try:
2480
- m = _make_obj_marshaler(ty, rec, **kwargs)
2481
- finally:
2482
- del _OBJ_MARSHALER_PROXIES[ty]
2483
- p.m = m
2503
+ if not no_cache:
2504
+ self._obj_marshalers[ty] = m
2505
+ return m
2506
+
2507
+ #
2484
2508
 
2485
- if not no_cache:
2486
- _OBJ_MARSHALERS[ty] = m
2487
- return m
2509
+ def marshal_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Any:
2510
+ return self.get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
2511
+
2512
+ def unmarshal_obj(self, o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
2513
+ return self.get_obj_marshaler(ty).unmarshal(o)
2488
2514
 
2489
2515
 
2490
2516
  ##
2491
2517
 
2492
2518
 
2493
- def marshal_obj(o: ta.Any, ty: ta.Any = None) -> ta.Any:
2494
- return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
2519
+ OBJ_MARSHALER_MANAGER = ObjMarshalerManager()
2495
2520
 
2521
+ register_opj_marshaler = OBJ_MARSHALER_MANAGER.register_opj_marshaler
2522
+ get_obj_marshaler = OBJ_MARSHALER_MANAGER.get_obj_marshaler
2496
2523
 
2497
- def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
2498
- return get_obj_marshaler(ty).unmarshal(o)
2524
+ marshal_obj = OBJ_MARSHALER_MANAGER.marshal_obj
2525
+ unmarshal_obj = OBJ_MARSHALER_MANAGER.unmarshal_obj
2499
2526
 
2500
2527
 
2501
2528
  ########################################
@@ -3058,6 +3085,16 @@ class ThreadWorkerGroup:
3058
3085
  ##
3059
3086
 
3060
3087
 
3088
+ SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
3089
+ 'pipe': subprocess.PIPE,
3090
+ 'stdout': subprocess.STDOUT,
3091
+ 'devnull': subprocess.DEVNULL,
3092
+ }
3093
+
3094
+
3095
+ ##
3096
+
3097
+
3061
3098
  _SUBPROCESS_SHELL_WRAP_EXECS = False
3062
3099
 
3063
3100