ominfra 0.0.0.dev102__py3-none-any.whl → 0.0.0.dev103__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -62,7 +62,7 @@ class AwsPutLogEventsResponse(AwsDataclass):
62
62
  ##
63
63
 
64
64
 
65
- class AwsLogMessagePoster:
65
+ class AwsLogMessageBuilder:
66
66
  """
67
67
  TODO:
68
68
  - max_items
@@ -88,7 +88,7 @@ class AwsLogMessagePoster:
88
88
  log_group_name: str,
89
89
  log_stream_name: str,
90
90
  region_name: str,
91
- credentials: AwsSigner.Credentials,
91
+ credentials: ta.Optional[AwsSigner.Credentials],
92
92
 
93
93
  url: ta.Optional[str] = None,
94
94
  service_name: str = DEFAULT_SERVICE_NAME,
@@ -110,11 +110,16 @@ class AwsLogMessagePoster:
110
110
  headers = {**headers, **extra_headers}
111
111
  self._headers = {k: [v] for k, v in headers.items()}
112
112
 
113
- self._signer = V4AwsSigner(
114
- credentials,
115
- region_name,
116
- service_name,
117
- )
113
+ signer: ta.Optional[V4AwsSigner]
114
+ if credentials is not None:
115
+ signer = V4AwsSigner(
116
+ credentials,
117
+ region_name,
118
+ service_name,
119
+ )
120
+ else:
121
+ signer = None
122
+ self._signer = signer
118
123
 
119
124
  #
120
125
 
@@ -158,13 +163,14 @@ class AwsLogMessagePoster:
158
163
  payload=body,
159
164
  )
160
165
 
161
- sig_headers = self._signer.sign(
162
- sig_req,
163
- sign_payload=False,
164
- )
165
- sig_req = dc.replace(sig_req, headers={**sig_req.headers, **sig_headers})
166
+ if (signer := self._signer) is not None:
167
+ sig_headers = signer.sign(
168
+ sig_req,
169
+ sign_payload=False,
170
+ )
171
+ sig_req = dc.replace(sig_req, headers={**sig_req.headers, **sig_headers})
166
172
 
167
- post = AwsLogMessagePoster.Post(
173
+ post = AwsLogMessageBuilder.Post(
168
174
  url=self._url,
169
175
  headers={k: check_single(v) for k, v in sig_req.headers.items()},
170
176
  data=sig_req.payload,
@@ -82,7 +82,7 @@ if sys.version_info < (3, 8):
82
82
  ########################################
83
83
 
84
84
 
85
- # ../../../../omlish/lite/check.py
85
+ # ../../../../omlish/lite/cached.py
86
86
  T = ta.TypeVar('T')
87
87
 
88
88
 
@@ -112,7 +112,7 @@ class HostConfig:
112
112
  # ../../../../omlish/lite/cached.py
113
113
 
114
114
 
115
- class cached_nullary: # noqa
115
+ class _cached_nullary: # noqa
116
116
  def __init__(self, fn):
117
117
  super().__init__()
118
118
  self._fn = fn
@@ -129,6 +129,10 @@ class cached_nullary: # noqa
129
129
  return bound
130
130
 
131
131
 
132
+ def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
133
+ return _cached_nullary(fn)
134
+
135
+
132
136
  ########################################
133
137
  # ../../../../omlish/lite/check.py
134
138
 
@@ -661,7 +665,7 @@ class DataclassObjMarshaler(ObjMarshaler):
661
665
  return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
662
666
 
663
667
  def unmarshal(self, o: ta.Any) -> ta.Any:
664
- return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if self.nonstrict or k in self.fs})
668
+ return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if not self.nonstrict or k in self.fs})
665
669
 
666
670
 
667
671
  @dc.dataclass(frozen=True)
@@ -956,6 +960,24 @@ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
956
960
  return out.decode().strip() if out is not None else None
957
961
 
958
962
 
963
+ ##
964
+
965
+
966
+ def subprocess_close(
967
+ proc: subprocess.Popen,
968
+ timeout: ta.Optional[float] = None,
969
+ ) -> None:
970
+ # TODO: terminate, sleep, kill
971
+ if proc.stdout:
972
+ proc.stdout.close()
973
+ if proc.stderr:
974
+ proc.stderr.close()
975
+ if proc.stdin:
976
+ proc.stdin.close()
977
+
978
+ proc.wait(timeout)
979
+
980
+
959
981
  ########################################
960
982
  # ../base.py
961
983
 
@@ -34,8 +34,10 @@ if sys.version_info < (3, 8):
34
34
  ########################################
35
35
 
36
36
 
37
- # ../base.py
37
+ # ../../../../omlish/lite/cached.py
38
38
  T = ta.TypeVar('T')
39
+
40
+ # ../base.py
39
41
  ConcernT = ta.TypeVar('ConcernT')
40
42
  ConfigT = ta.TypeVar('ConfigT')
41
43
  SiteConcernConfigT = ta.TypeVar('SiteConcernConfigT', bound='SiteConcernConfig')
@@ -84,7 +86,7 @@ class DeployConfig:
84
86
  # ../../../../omlish/lite/cached.py
85
87
 
86
88
 
87
- class cached_nullary: # noqa
89
+ class _cached_nullary: # noqa
88
90
  def __init__(self, fn):
89
91
  super().__init__()
90
92
  self._fn = fn
@@ -101,6 +103,10 @@ class cached_nullary: # noqa
101
103
  return bound
102
104
 
103
105
 
106
+ def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
107
+ return _cached_nullary(fn)
108
+
109
+
104
110
  ########################################
105
111
  # ../../../../omlish/lite/json.py
106
112
 
@@ -809,6 +815,24 @@ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
809
815
  return out.decode().strip() if out is not None else None
810
816
 
811
817
 
818
+ ##
819
+
820
+
821
+ def subprocess_close(
822
+ proc: subprocess.Popen,
823
+ timeout: ta.Optional[float] = None,
824
+ ) -> None:
825
+ # TODO: terminate, sleep, kill
826
+ if proc.stdout:
827
+ proc.stdout.close()
828
+ if proc.stderr:
829
+ proc.stderr.close()
830
+ if proc.stdin:
831
+ proc.stdin.close()
832
+
833
+ proc.wait(timeout)
834
+
835
+
812
836
  ########################################
813
837
  # ../runtime.py
814
838
 
@@ -3,6 +3,7 @@
3
3
  import argparse
4
4
  import datetime
5
5
  import json
6
+ import os
6
7
  import re
7
8
  import sys
8
9
  import time
@@ -34,6 +35,11 @@ def _main() -> None:
34
35
  else:
35
36
  start = 0
36
37
 
38
+ stdout_fd = sys.stdout.fileno()
39
+ out_fd = os.dup(stdout_fd)
40
+ null_fd = os.open('/dev/null', os.O_WRONLY)
41
+ os.dup2(null_fd, stdout_fd)
42
+
37
43
  for i in range(start, args.n):
38
44
  if args.sleep_s:
39
45
  if not args.sleep_n or (i and i % args.sleep_n == 0):
@@ -46,8 +52,14 @@ def _main() -> None:
46
52
  '__CURSOR': f'cursor:{i}',
47
53
  '_SOURCE_REALTIME_TIMESTAMP': str(int(ts_us)),
48
54
  }
49
- print(json.dumps(dct, indent=None, separators=(',', ':')))
50
- sys.stdout.flush()
55
+
56
+ buf = json.dumps(dct, indent=None, separators=(',', ':')).encode()
57
+
58
+ try:
59
+ os.write(out_fd, buf)
60
+ os.write(out_fd, b'\n')
61
+ except BrokenPipeError:
62
+ break
51
63
 
52
64
 
53
65
  if __name__ == '__main__':
@@ -347,9 +347,10 @@ import typing as ta
347
347
  from omlish.lite.cached import cached_nullary
348
348
  from omlish.lite.check import check_not_none
349
349
  from omlish.lite.logs import log
350
+ from omlish.lite.subprocesses import subprocess_close
350
351
  from omlish.lite.subprocesses import subprocess_shell_wrap_exec
351
352
 
352
- from ..threadworker import ThreadWorker
353
+ from ..threadworkers import ThreadWorker
353
354
  from .messages import JournalctlMessage # noqa
354
355
  from .messages import JournalctlMessageBuilder
355
356
 
@@ -385,7 +386,7 @@ class JournalctlTailerWorker(ThreadWorker):
385
386
  self._read_size = read_size
386
387
  self._sleep_s = sleep_s
387
388
 
388
- self._mb = JournalctlMessageBuilder()
389
+ self._builder = JournalctlMessageBuilder()
389
390
 
390
391
  self._proc: ta.Optional[subprocess.Popen] = None
391
392
 
@@ -409,45 +410,50 @@ class JournalctlTailerWorker(ThreadWorker):
409
410
 
410
411
  return cmd
411
412
 
413
+ def _read_loop(self, stdout: ta.IO) -> None:
414
+ while stdout.readable():
415
+ self._heartbeat()
416
+
417
+ buf = stdout.read(self._read_size)
418
+ if not buf:
419
+ log.debug('Journalctl empty read')
420
+ break
421
+
422
+ log.debug('Journalctl read buffer: %r', buf)
423
+ msgs = self._builder.feed(buf)
424
+ if msgs:
425
+ while True:
426
+ try:
427
+ self._output.put(msgs, timeout=1.)
428
+ except queue.Full:
429
+ self._heartbeat()
430
+ else:
431
+ break
432
+
412
433
  def _run(self) -> None:
413
434
  with subprocess.Popen(
414
435
  self._full_cmd(),
415
436
  stdout=subprocess.PIPE,
416
437
  ) as self._proc:
417
- stdout = check_not_none(self._proc.stdout)
438
+ try:
439
+ stdout = check_not_none(self._proc.stdout)
440
+
441
+ fd = stdout.fileno()
442
+ fl = fcntl.fcntl(fd, fcntl.F_GETFL)
443
+ fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
418
444
 
419
- fd = stdout.fileno()
420
- fl = fcntl.fcntl(fd, fcntl.F_GETFL)
421
- fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
445
+ while True:
446
+ self._heartbeat()
422
447
 
423
- while True:
424
- if not self._heartbeat():
425
- return
448
+ self._read_loop(stdout)
426
449
 
427
- while stdout.readable():
428
- if not self._heartbeat():
450
+ log.debug('Journalctl not readable')
451
+
452
+ if self._proc.poll() is not None:
453
+ log.critical('Journalctl process terminated')
429
454
  return
430
455
 
431
- buf = stdout.read(self._read_size)
432
- if not buf:
433
- log.debug('Journalctl empty read')
434
- break
456
+ time.sleep(self._sleep_s)
435
457
 
436
- log.debug('Journalctl read buffer: %r', buf)
437
- msgs = self._mb.feed(buf)
438
- if msgs:
439
- while True:
440
- try:
441
- self._output.put(msgs, timeout=1.)
442
- except queue.Full:
443
- if not self._heartbeat():
444
- return
445
- else:
446
- break
447
-
448
- if self._proc.poll() is not None:
449
- log.critical('Journalctl process terminated')
450
- return
451
-
452
- log.debug('Journalctl readable')
453
- time.sleep(self._sleep_s)
458
+ finally:
459
+ subprocess_close(self._proc)
@@ -41,7 +41,7 @@ if sys.version_info < (3, 8):
41
41
  ########################################
42
42
 
43
43
 
44
- # ../../../omlish/lite/check.py
44
+ # ../../../omlish/lite/cached.py
45
45
  T = ta.TypeVar('T')
46
46
 
47
47
 
@@ -195,7 +195,7 @@ def post_boostrap() -> PostBoostrap:
195
195
  # ../../../omlish/lite/cached.py
196
196
 
197
197
 
198
- class cached_nullary: # noqa
198
+ class _cached_nullary: # noqa
199
199
  def __init__(self, fn):
200
200
  super().__init__()
201
201
  self._fn = fn
@@ -212,6 +212,10 @@ class cached_nullary: # noqa
212
212
  return bound
213
213
 
214
214
 
215
+ def cached_nullary(fn: ta.Callable[..., T]) -> ta.Callable[..., T]:
216
+ return _cached_nullary(fn)
217
+
218
+
215
219
  ########################################
216
220
  # ../../../omlish/lite/check.py
217
221
 
@@ -744,7 +748,7 @@ class DataclassObjMarshaler(ObjMarshaler):
744
748
  return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
745
749
 
746
750
  def unmarshal(self, o: ta.Any) -> ta.Any:
747
- return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if self.nonstrict or k in self.fs})
751
+ return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if not self.nonstrict or k in self.fs})
748
752
 
749
753
 
750
754
  @dc.dataclass(frozen=True)
@@ -1039,6 +1043,24 @@ def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
1039
1043
  return out.decode().strip() if out is not None else None
1040
1044
 
1041
1045
 
1046
+ ##
1047
+
1048
+
1049
+ def subprocess_close(
1050
+ proc: subprocess.Popen,
1051
+ timeout: ta.Optional[float] = None,
1052
+ ) -> None:
1053
+ # TODO: terminate, sleep, kill
1054
+ if proc.stdout:
1055
+ proc.stdout.close()
1056
+ if proc.stderr:
1057
+ proc.stderr.close()
1058
+ if proc.stdin:
1059
+ proc.stdin.close()
1060
+
1061
+ proc.wait(timeout)
1062
+
1063
+
1042
1064
  ########################################
1043
1065
  # runcommands.py
1044
1066