ominfra 0.0.0.dev102__py3-none-any.whl → 0.0.0.dev103__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ominfra/clouds/aws/journald2aws/cursor.py +47 -0
- ominfra/clouds/aws/journald2aws/driver.py +210 -0
- ominfra/clouds/aws/journald2aws/main.py +9 -262
- ominfra/clouds/aws/journald2aws/poster.py +89 -0
- ominfra/clouds/aws/logs.py +19 -13
- ominfra/deploy/_executor.py +25 -3
- ominfra/deploy/poly/_main.py +26 -2
- ominfra/journald/genmessages.py +14 -2
- ominfra/journald/tailer.py +39 -33
- ominfra/pyremote/_runcommands.py +25 -3
- ominfra/scripts/journald2aws.py +428 -195
- ominfra/scripts/supervisor.py +1 -1
- ominfra/threadworkers.py +139 -0
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/METADATA +3 -3
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/RECORD +19 -16
- ominfra/threadworker.py +0 -67
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/LICENSE +0 -0
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/WHEEL +0 -0
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/entry_points.txt +0 -0
- {ominfra-0.0.0.dev102.dist-info → ominfra-0.0.0.dev103.dist-info}/top_level.txt +0 -0
ominfra/clouds/aws/logs.py
CHANGED
@@ -62,7 +62,7 @@ class AwsPutLogEventsResponse(AwsDataclass):
|
|
62
62
|
##
|
63
63
|
|
64
64
|
|
65
|
-
class
|
65
|
+
class AwsLogMessageBuilder:
|
66
66
|
"""
|
67
67
|
TODO:
|
68
68
|
- max_items
|
@@ -88,7 +88,7 @@ class AwsLogMessagePoster:
|
|
88
88
|
log_group_name: str,
|
89
89
|
log_stream_name: str,
|
90
90
|
region_name: str,
|
91
|
-
credentials: AwsSigner.Credentials,
|
91
|
+
credentials: ta.Optional[AwsSigner.Credentials],
|
92
92
|
|
93
93
|
url: ta.Optional[str] = None,
|
94
94
|
service_name: str = DEFAULT_SERVICE_NAME,
|
@@ -110,11 +110,16 @@ class AwsLogMessagePoster:
|
|
110
110
|
headers = {**headers, **extra_headers}
|
111
111
|
self._headers = {k: [v] for k, v in headers.items()}
|
112
112
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
113
|
+
signer: ta.Optional[V4AwsSigner]
|
114
|
+
if credentials is not None:
|
115
|
+
signer = V4AwsSigner(
|
116
|
+
credentials,
|
117
|
+
region_name,
|
118
|
+
service_name,
|
119
|
+
)
|
120
|
+
else:
|
121
|
+
signer = None
|
122
|
+
self._signer = signer
|
118
123
|
|
119
124
|
#
|
120
125
|
|
@@ -158,13 +163,14 @@ class AwsLogMessagePoster:
|
|
158
163
|
payload=body,
|
159
164
|
)
|
160
165
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
+
if (signer := self._signer) is not None:
|
167
|
+
sig_headers = signer.sign(
|
168
|
+
sig_req,
|
169
|
+
sign_payload=False,
|
170
|
+
)
|
171
|
+
sig_req = dc.replace(sig_req, headers={**sig_req.headers, **sig_headers})
|
166
172
|
|
167
|
-
post =
|
173
|
+
post = AwsLogMessageBuilder.Post(
|
168
174
|
url=self._url,
|
169
175
|
headers={k: check_single(v) for k, v in sig_req.headers.items()},
|
170
176
|
data=sig_req.payload,
|
ominfra/deploy/_executor.py
CHANGED
@@ -82,7 +82,7 @@ if sys.version_info < (3, 8):
|
|
82
82
|
########################################
|
83
83
|
|
84
84
|
|
85
|
-
# ../../../../omlish/lite/
|
85
|
+
# ../../../../omlish/lite/cached.py
|
86
86
|
T = ta.TypeVar('T')
|
87
87
|
|
88
88
|
|
@@ -112,7 +112,7 @@ class HostConfig:
|
|
112
112
|
# ../../../../omlish/lite/cached.py
|
113
113
|
|
114
114
|
|
115
|
-
class
|
115
|
+
class _cached_nullary: # noqa
|
116
116
|
def __init__(self, fn):
|
117
117
|
super().__init__()
|
118
118
|
self._fn = fn
|
@@ -129,6 +129,10 @@ class cached_nullary: # noqa
|
|
129
129
|
return bound
|
130
130
|
|
131
131
|
|
132
|
+
def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
|
133
|
+
return _cached_nullary(fn)
|
134
|
+
|
135
|
+
|
132
136
|
########################################
|
133
137
|
# ../../../../omlish/lite/check.py
|
134
138
|
|
@@ -661,7 +665,7 @@ class DataclassObjMarshaler(ObjMarshaler):
|
|
661
665
|
return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
|
662
666
|
|
663
667
|
def unmarshal(self, o: ta.Any) -> ta.Any:
|
664
|
-
return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if self.nonstrict or k in self.fs})
|
668
|
+
return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if not self.nonstrict or k in self.fs})
|
665
669
|
|
666
670
|
|
667
671
|
@dc.dataclass(frozen=True)
|
@@ -956,6 +960,24 @@ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
|
956
960
|
return out.decode().strip() if out is not None else None
|
957
961
|
|
958
962
|
|
963
|
+
##
|
964
|
+
|
965
|
+
|
966
|
+
def subprocess_close(
|
967
|
+
proc: subprocess.Popen,
|
968
|
+
timeout: ta.Optional[float] = None,
|
969
|
+
) -> None:
|
970
|
+
# TODO: terminate, sleep, kill
|
971
|
+
if proc.stdout:
|
972
|
+
proc.stdout.close()
|
973
|
+
if proc.stderr:
|
974
|
+
proc.stderr.close()
|
975
|
+
if proc.stdin:
|
976
|
+
proc.stdin.close()
|
977
|
+
|
978
|
+
proc.wait(timeout)
|
979
|
+
|
980
|
+
|
959
981
|
########################################
|
960
982
|
# ../base.py
|
961
983
|
|
ominfra/deploy/poly/_main.py
CHANGED
@@ -34,8 +34,10 @@ if sys.version_info < (3, 8):
|
|
34
34
|
########################################
|
35
35
|
|
36
36
|
|
37
|
-
#
|
37
|
+
# ../../../../omlish/lite/cached.py
|
38
38
|
T = ta.TypeVar('T')
|
39
|
+
|
40
|
+
# ../base.py
|
39
41
|
ConcernT = ta.TypeVar('ConcernT')
|
40
42
|
ConfigT = ta.TypeVar('ConfigT')
|
41
43
|
SiteConcernConfigT = ta.TypeVar('SiteConcernConfigT', bound='SiteConcernConfig')
|
@@ -84,7 +86,7 @@ class DeployConfig:
|
|
84
86
|
# ../../../../omlish/lite/cached.py
|
85
87
|
|
86
88
|
|
87
|
-
class
|
89
|
+
class _cached_nullary: # noqa
|
88
90
|
def __init__(self, fn):
|
89
91
|
super().__init__()
|
90
92
|
self._fn = fn
|
@@ -101,6 +103,10 @@ class cached_nullary: # noqa
|
|
101
103
|
return bound
|
102
104
|
|
103
105
|
|
106
|
+
def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
|
107
|
+
return _cached_nullary(fn)
|
108
|
+
|
109
|
+
|
104
110
|
########################################
|
105
111
|
# ../../../../omlish/lite/json.py
|
106
112
|
|
@@ -809,6 +815,24 @@ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
|
809
815
|
return out.decode().strip() if out is not None else None
|
810
816
|
|
811
817
|
|
818
|
+
##
|
819
|
+
|
820
|
+
|
821
|
+
def subprocess_close(
|
822
|
+
proc: subprocess.Popen,
|
823
|
+
timeout: ta.Optional[float] = None,
|
824
|
+
) -> None:
|
825
|
+
# TODO: terminate, sleep, kill
|
826
|
+
if proc.stdout:
|
827
|
+
proc.stdout.close()
|
828
|
+
if proc.stderr:
|
829
|
+
proc.stderr.close()
|
830
|
+
if proc.stdin:
|
831
|
+
proc.stdin.close()
|
832
|
+
|
833
|
+
proc.wait(timeout)
|
834
|
+
|
835
|
+
|
812
836
|
########################################
|
813
837
|
# ../runtime.py
|
814
838
|
|
ominfra/journald/genmessages.py
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
import argparse
|
4
4
|
import datetime
|
5
5
|
import json
|
6
|
+
import os
|
6
7
|
import re
|
7
8
|
import sys
|
8
9
|
import time
|
@@ -34,6 +35,11 @@ def _main() -> None:
|
|
34
35
|
else:
|
35
36
|
start = 0
|
36
37
|
|
38
|
+
stdout_fd = sys.stdout.fileno()
|
39
|
+
out_fd = os.dup(stdout_fd)
|
40
|
+
null_fd = os.open('/dev/null', os.O_WRONLY)
|
41
|
+
os.dup2(null_fd, stdout_fd)
|
42
|
+
|
37
43
|
for i in range(start, args.n):
|
38
44
|
if args.sleep_s:
|
39
45
|
if not args.sleep_n or (i and i % args.sleep_n == 0):
|
@@ -46,8 +52,14 @@ def _main() -> None:
|
|
46
52
|
'__CURSOR': f'cursor:{i}',
|
47
53
|
'_SOURCE_REALTIME_TIMESTAMP': str(int(ts_us)),
|
48
54
|
}
|
49
|
-
|
50
|
-
|
55
|
+
|
56
|
+
buf = json.dumps(dct, indent=None, separators=(',', ':')).encode()
|
57
|
+
|
58
|
+
try:
|
59
|
+
os.write(out_fd, buf)
|
60
|
+
os.write(out_fd, b'\n')
|
61
|
+
except BrokenPipeError:
|
62
|
+
break
|
51
63
|
|
52
64
|
|
53
65
|
if __name__ == '__main__':
|
ominfra/journald/tailer.py
CHANGED
@@ -347,9 +347,10 @@ import typing as ta
|
|
347
347
|
from omlish.lite.cached import cached_nullary
|
348
348
|
from omlish.lite.check import check_not_none
|
349
349
|
from omlish.lite.logs import log
|
350
|
+
from omlish.lite.subprocesses import subprocess_close
|
350
351
|
from omlish.lite.subprocesses import subprocess_shell_wrap_exec
|
351
352
|
|
352
|
-
from ..
|
353
|
+
from ..threadworkers import ThreadWorker
|
353
354
|
from .messages import JournalctlMessage # noqa
|
354
355
|
from .messages import JournalctlMessageBuilder
|
355
356
|
|
@@ -385,7 +386,7 @@ class JournalctlTailerWorker(ThreadWorker):
|
|
385
386
|
self._read_size = read_size
|
386
387
|
self._sleep_s = sleep_s
|
387
388
|
|
388
|
-
self.
|
389
|
+
self._builder = JournalctlMessageBuilder()
|
389
390
|
|
390
391
|
self._proc: ta.Optional[subprocess.Popen] = None
|
391
392
|
|
@@ -409,45 +410,50 @@ class JournalctlTailerWorker(ThreadWorker):
|
|
409
410
|
|
410
411
|
return cmd
|
411
412
|
|
413
|
+
def _read_loop(self, stdout: ta.IO) -> None:
|
414
|
+
while stdout.readable():
|
415
|
+
self._heartbeat()
|
416
|
+
|
417
|
+
buf = stdout.read(self._read_size)
|
418
|
+
if not buf:
|
419
|
+
log.debug('Journalctl empty read')
|
420
|
+
break
|
421
|
+
|
422
|
+
log.debug('Journalctl read buffer: %r', buf)
|
423
|
+
msgs = self._builder.feed(buf)
|
424
|
+
if msgs:
|
425
|
+
while True:
|
426
|
+
try:
|
427
|
+
self._output.put(msgs, timeout=1.)
|
428
|
+
except queue.Full:
|
429
|
+
self._heartbeat()
|
430
|
+
else:
|
431
|
+
break
|
432
|
+
|
412
433
|
def _run(self) -> None:
|
413
434
|
with subprocess.Popen(
|
414
435
|
self._full_cmd(),
|
415
436
|
stdout=subprocess.PIPE,
|
416
437
|
) as self._proc:
|
417
|
-
|
438
|
+
try:
|
439
|
+
stdout = check_not_none(self._proc.stdout)
|
440
|
+
|
441
|
+
fd = stdout.fileno()
|
442
|
+
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
|
443
|
+
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
|
418
444
|
|
419
|
-
|
420
|
-
|
421
|
-
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
|
445
|
+
while True:
|
446
|
+
self._heartbeat()
|
422
447
|
|
423
|
-
|
424
|
-
if not self._heartbeat():
|
425
|
-
return
|
448
|
+
self._read_loop(stdout)
|
426
449
|
|
427
|
-
|
428
|
-
|
450
|
+
log.debug('Journalctl not readable')
|
451
|
+
|
452
|
+
if self._proc.poll() is not None:
|
453
|
+
log.critical('Journalctl process terminated')
|
429
454
|
return
|
430
455
|
|
431
|
-
|
432
|
-
if not buf:
|
433
|
-
log.debug('Journalctl empty read')
|
434
|
-
break
|
456
|
+
time.sleep(self._sleep_s)
|
435
457
|
|
436
|
-
|
437
|
-
|
438
|
-
if msgs:
|
439
|
-
while True:
|
440
|
-
try:
|
441
|
-
self._output.put(msgs, timeout=1.)
|
442
|
-
except queue.Full:
|
443
|
-
if not self._heartbeat():
|
444
|
-
return
|
445
|
-
else:
|
446
|
-
break
|
447
|
-
|
448
|
-
if self._proc.poll() is not None:
|
449
|
-
log.critical('Journalctl process terminated')
|
450
|
-
return
|
451
|
-
|
452
|
-
log.debug('Journalctl readable')
|
453
|
-
time.sleep(self._sleep_s)
|
458
|
+
finally:
|
459
|
+
subprocess_close(self._proc)
|
ominfra/pyremote/_runcommands.py
CHANGED
@@ -41,7 +41,7 @@ if sys.version_info < (3, 8):
|
|
41
41
|
########################################
|
42
42
|
|
43
43
|
|
44
|
-
# ../../../omlish/lite/
|
44
|
+
# ../../../omlish/lite/cached.py
|
45
45
|
T = ta.TypeVar('T')
|
46
46
|
|
47
47
|
|
@@ -195,7 +195,7 @@ def post_boostrap() -> PostBoostrap:
|
|
195
195
|
# ../../../omlish/lite/cached.py
|
196
196
|
|
197
197
|
|
198
|
-
class
|
198
|
+
class _cached_nullary: # noqa
|
199
199
|
def __init__(self, fn):
|
200
200
|
super().__init__()
|
201
201
|
self._fn = fn
|
@@ -212,6 +212,10 @@ class cached_nullary: # noqa
|
|
212
212
|
return bound
|
213
213
|
|
214
214
|
|
215
|
+
def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
|
216
|
+
return _cached_nullary(fn)
|
217
|
+
|
218
|
+
|
215
219
|
########################################
|
216
220
|
# ../../../omlish/lite/check.py
|
217
221
|
|
@@ -744,7 +748,7 @@ class DataclassObjMarshaler(ObjMarshaler):
|
|
744
748
|
return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
|
745
749
|
|
746
750
|
def unmarshal(self, o: ta.Any) -> ta.Any:
|
747
|
-
return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if self.nonstrict or k in self.fs})
|
751
|
+
return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if not self.nonstrict or k in self.fs})
|
748
752
|
|
749
753
|
|
750
754
|
@dc.dataclass(frozen=True)
|
@@ -1039,6 +1043,24 @@ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
|
|
1039
1043
|
return out.decode().strip() if out is not None else None
|
1040
1044
|
|
1041
1045
|
|
1046
|
+
##
|
1047
|
+
|
1048
|
+
|
1049
|
+
def subprocess_close(
|
1050
|
+
proc: subprocess.Popen,
|
1051
|
+
timeout: ta.Optional[float] = None,
|
1052
|
+
) -> None:
|
1053
|
+
# TODO: terminate, sleep, kill
|
1054
|
+
if proc.stdout:
|
1055
|
+
proc.stdout.close()
|
1056
|
+
if proc.stderr:
|
1057
|
+
proc.stderr.close()
|
1058
|
+
if proc.stdin:
|
1059
|
+
proc.stdin.close()
|
1060
|
+
|
1061
|
+
proc.wait(timeout)
|
1062
|
+
|
1063
|
+
|
1042
1064
|
########################################
|
1043
1065
|
# runcommands.py
|
1044
1066
|
|