ominfra 0.0.0.dev132__py3-none-any.whl → 0.0.0.dev134__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. ominfra/{deploy → manage/deploy}/_executor.py +10 -10
  2. ominfra/{deploy → manage/deploy}/poly/_main.py +6 -6
  3. ominfra/{deploy → manage/deploy}/remote.py +1 -1
  4. ominfra/pyremote.py +389 -0
  5. ominfra/scripts/supervisor.py +1 -1
  6. ominfra/supervisor/http.py +1 -1
  7. {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/METADATA +3 -3
  8. {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/RECORD +36 -39
  9. ominfra/pyremote/__init__.py +0 -0
  10. ominfra/pyremote/_runcommands.py +0 -1201
  11. ominfra/pyremote/bootstrap.py +0 -149
  12. ominfra/pyremote/runcommands.py +0 -56
  13. /ominfra/{deploy → manage/deploy}/__init__.py +0 -0
  14. /ominfra/{deploy → manage/deploy}/configs.py +0 -0
  15. /ominfra/{deploy → manage/deploy}/executor/__init__.py +0 -0
  16. /ominfra/{deploy → manage/deploy}/executor/base.py +0 -0
  17. /ominfra/{deploy → manage/deploy}/executor/concerns/__init__.py +0 -0
  18. /ominfra/{deploy → manage/deploy}/executor/concerns/dirs.py +0 -0
  19. /ominfra/{deploy → manage/deploy}/executor/concerns/nginx.py +0 -0
  20. /ominfra/{deploy → manage/deploy}/executor/concerns/repo.py +0 -0
  21. /ominfra/{deploy → manage/deploy}/executor/concerns/supervisor.py +0 -0
  22. /ominfra/{deploy → manage/deploy}/executor/concerns/systemd.py +0 -0
  23. /ominfra/{deploy → manage/deploy}/executor/concerns/user.py +0 -0
  24. /ominfra/{deploy → manage/deploy}/executor/concerns/venv.py +0 -0
  25. /ominfra/{deploy → manage/deploy}/executor/main.py +0 -0
  26. /ominfra/{deploy → manage/deploy}/poly/__init__.py +0 -0
  27. /ominfra/{deploy → manage/deploy}/poly/base.py +0 -0
  28. /ominfra/{deploy → manage/deploy}/poly/configs.py +0 -0
  29. /ominfra/{deploy → manage/deploy}/poly/deploy.py +0 -0
  30. /ominfra/{deploy → manage/deploy}/poly/main.py +0 -0
  31. /ominfra/{deploy → manage/deploy}/poly/nginx.py +0 -0
  32. /ominfra/{deploy → manage/deploy}/poly/repo.py +0 -0
  33. /ominfra/{deploy → manage/deploy}/poly/runtime.py +0 -0
  34. /ominfra/{deploy → manage/deploy}/poly/site.py +0 -0
  35. /ominfra/{deploy → manage/deploy}/poly/supervisor.py +0 -0
  36. /ominfra/{deploy → manage/deploy}/poly/venv.py +0 -0
  37. {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/LICENSE +0 -0
  38. {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/WHEEL +0 -0
  39. {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/entry_points.txt +0 -0
  40. {ominfra-0.0.0.dev132.dist-info → ominfra-0.0.0.dev134.dist-info}/top_level.txt +0 -0
@@ -1,1201 +0,0 @@
1
- #!/usr/bin/env python3
2
- # noinspection DuplicatedCode
3
- # @omlish-lite
4
- # @omlish-script
5
- # @omlish-amalg-output runcommands.py
6
- # ruff: noqa: N802 UP006 UP007 UP036
7
- import abc
8
- import base64
9
- import collections.abc
10
- import contextlib
11
- import dataclasses as dc
12
- import datetime
13
- import decimal
14
- import enum
15
- import fractions
16
- import functools
17
- import inspect
18
- import io
19
- import json
20
- import logging
21
- import os
22
- import shlex
23
- import subprocess
24
- import sys
25
- import textwrap
26
- import threading
27
- import types
28
- import typing as ta
29
- import uuid
30
- import weakref # noqa
31
- import zlib
32
-
33
-
34
- ########################################
35
-
36
-
37
- if sys.version_info < (3, 8):
38
- raise OSError(f'Requires python (3, 8), got {sys.version_info} from {sys.executable}') # noqa
39
-
40
-
41
- ########################################
42
-
43
-
44
- # ../../omlish/lite/cached.py
45
- T = ta.TypeVar('T')
46
-
47
- # ../../omlish/lite/check.py
48
- SizedT = ta.TypeVar('SizedT', bound=ta.Sized)
49
-
50
-
51
- ########################################
52
- # ../bootstrap.py
53
- """
54
- Basically this: https://mitogen.networkgenomics.com/howitworks.html
55
- """
56
-
57
-
58
- ##
59
-
60
-
61
- _BOOTSTRAP_COMM_FD = 100
62
- _BOOTSTRAP_SRC_FD = 101
63
-
64
- _BOOTSTRAP_CHILD_PID_VAR = '_OPYR_CPID'
65
- _BOOTSTRAP_ARGV0_VAR = '_OPYR_ARGV0'
66
-
67
- BOOTSTRAP_ACK0 = b'OPYR000\n'
68
- BOOTSTRAP_ACK1 = b'OPYR001\n'
69
-
70
- _BOOTSTRAP_PROC_TITLE_FMT = '(pyremote:%s)'
71
-
72
- _BOOTSTRAP_IMPORTS = [
73
- 'base64',
74
- 'os',
75
- 'sys',
76
- 'zlib',
77
- ]
78
-
79
-
80
- def _bootstrap_main(context_name: str, main_z_len: int) -> None:
81
- # Two copies of main src to be sent to parent
82
- r0, w0 = os.pipe()
83
- r1, w1 = os.pipe()
84
-
85
- if (cp := os.fork()):
86
- # Parent process
87
-
88
- # Dup original stdin to comm_fd for use as comm channel
89
- os.dup2(0, _BOOTSTRAP_COMM_FD)
90
-
91
- # Overwrite stdin (fed to python repl) with first copy of src
92
- os.dup2(r0, 0)
93
-
94
- # Dup second copy of src to src_fd to recover after launch
95
- os.dup2(r1, _BOOTSTRAP_SRC_FD)
96
-
97
- # Close remaining fd's
98
- for f in [r0, w0, r1, w1]:
99
- os.close(f)
100
-
101
- # Save child pid to close after relaunch
102
- os.environ[_BOOTSTRAP_CHILD_PID_VAR] = str(cp)
103
-
104
- # Save original argv0
105
- os.environ[_BOOTSTRAP_ARGV0_VAR] = sys.executable
106
-
107
- # Start repl reading stdin from r0
108
- os.execl(sys.executable, sys.executable + (_BOOTSTRAP_PROC_TITLE_FMT % (context_name,)))
109
-
110
- else:
111
- # Child process
112
-
113
- # Write first ack
114
- os.write(1, BOOTSTRAP_ACK0)
115
-
116
- # Read main src from stdin
117
- main_src = zlib.decompress(os.fdopen(0, 'rb').read(main_z_len))
118
-
119
- # Write both copies of main src
120
- for w in [w0, w1]:
121
- fp = os.fdopen(w, 'wb', 0)
122
- fp.write(main_src)
123
- fp.close()
124
-
125
- # Write second ack
126
- os.write(1, BOOTSTRAP_ACK1)
127
-
128
- sys.exit(0)
129
-
130
-
131
- #
132
-
133
-
134
- def bootstrap_payload(context_name: str, main_z_len: int) -> str:
135
- bs_src = textwrap.dedent(inspect.getsource(_bootstrap_main))
136
-
137
- for gl in [
138
- '_BOOTSTRAP_COMM_FD',
139
- '_BOOTSTRAP_SRC_FD',
140
-
141
- '_BOOTSTRAP_CHILD_PID_VAR',
142
- '_BOOTSTRAP_ARGV0_VAR',
143
-
144
- 'BOOTSTRAP_ACK0',
145
- 'BOOTSTRAP_ACK1',
146
-
147
- '_BOOTSTRAP_PROC_TITLE_FMT',
148
- ]:
149
- bs_src = bs_src.replace(gl, repr(globals()[gl]))
150
-
151
- bs_src = '\n'.join(
152
- cl
153
- for l in bs_src.splitlines()
154
- if (cl := (l.split('#')[0]).rstrip())
155
- if cl.strip()
156
- )
157
-
158
- bs_z = zlib.compress(bs_src.encode('utf-8'))
159
- bs_z64 = base64.encodebytes(bs_z).replace(b'\n', b'')
160
-
161
- stmts = [
162
- f'import {", ".join(_BOOTSTRAP_IMPORTS)}',
163
- f'exec(zlib.decompress(base64.decodebytes({bs_z64!r})))',
164
- f'_bootstrap_main({context_name!r}, {main_z_len})',
165
- ]
166
-
167
- cmd = '; '.join(stmts)
168
- return cmd
169
-
170
-
171
- #
172
-
173
-
174
- class PostBoostrap(ta.NamedTuple):
175
- input: ta.BinaryIO
176
- main_src: str
177
-
178
-
179
- def post_boostrap() -> PostBoostrap:
180
- # Restore original argv0
181
- sys.executable = os.environ.pop(_BOOTSTRAP_ARGV0_VAR)
182
-
183
- # Reap boostrap child
184
- os.waitpid(int(os.environ.pop(_BOOTSTRAP_CHILD_PID_VAR)), 0)
185
-
186
- # Read second copy of main src
187
- r1 = os.fdopen(_BOOTSTRAP_SRC_FD, 'rb', 0)
188
- main_src = r1.read().decode('utf-8')
189
- r1.close()
190
-
191
- return PostBoostrap(
192
- input=os.fdopen(_BOOTSTRAP_COMM_FD, 'rb', 0),
193
- main_src=main_src,
194
- )
195
-
196
-
197
- ########################################
198
- # ../../../omlish/lite/cached.py
199
-
200
-
201
- class _cached_nullary: # noqa
202
- def __init__(self, fn):
203
- super().__init__()
204
- self._fn = fn
205
- self._value = self._missing = object()
206
- functools.update_wrapper(self, fn)
207
-
208
- def __call__(self, *args, **kwargs): # noqa
209
- if self._value is self._missing:
210
- self._value = self._fn()
211
- return self._value
212
-
213
- def __get__(self, instance, owner): # noqa
214
- bound = instance.__dict__[self._fn.__name__] = self.__class__(self._fn.__get__(instance, owner))
215
- return bound
216
-
217
-
218
- def cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
219
- return _cached_nullary(fn)
220
-
221
-
222
- ########################################
223
- # ../../../omlish/lite/check.py
224
-
225
-
226
- def check_isinstance(v: ta.Any, spec: ta.Union[ta.Type[T], tuple]) -> T:
227
- if not isinstance(v, spec):
228
- raise TypeError(v)
229
- return v
230
-
231
-
232
- def check_not_isinstance(v: T, spec: ta.Union[type, tuple]) -> T:
233
- if isinstance(v, spec):
234
- raise TypeError(v)
235
- return v
236
-
237
-
238
- def check_none(v: T) -> None:
239
- if v is not None:
240
- raise ValueError(v)
241
-
242
-
243
- def check_not_none(v: ta.Optional[T]) -> T:
244
- if v is None:
245
- raise ValueError
246
- return v
247
-
248
-
249
- def check_not(v: ta.Any) -> None:
250
- if v:
251
- raise ValueError(v)
252
- return v
253
-
254
-
255
- def check_non_empty_str(v: ta.Optional[str]) -> str:
256
- if not v:
257
- raise ValueError
258
- return v
259
-
260
-
261
- def check_state(v: bool, msg: str = 'Illegal state') -> None:
262
- if not v:
263
- raise ValueError(msg)
264
-
265
-
266
- def check_equal(l: T, r: T) -> T:
267
- if l != r:
268
- raise ValueError(l, r)
269
- return l
270
-
271
-
272
- def check_not_equal(l: T, r: T) -> T:
273
- if l == r:
274
- raise ValueError(l, r)
275
- return l
276
-
277
-
278
- def check_is(l: T, r: T) -> T:
279
- if l is not r:
280
- raise ValueError(l, r)
281
- return l
282
-
283
-
284
- def check_is_not(l: T, r: ta.Any) -> T:
285
- if l is r:
286
- raise ValueError(l, r)
287
- return l
288
-
289
-
290
- def check_in(v: T, c: ta.Container[T]) -> T:
291
- if v not in c:
292
- raise ValueError(v, c)
293
- return v
294
-
295
-
296
- def check_not_in(v: T, c: ta.Container[T]) -> T:
297
- if v in c:
298
- raise ValueError(v, c)
299
- return v
300
-
301
-
302
- def check_single(vs: ta.Iterable[T]) -> T:
303
- [v] = vs
304
- return v
305
-
306
-
307
- def check_empty(v: SizedT) -> SizedT:
308
- if len(v):
309
- raise ValueError(v)
310
- return v
311
-
312
-
313
- def check_non_empty(v: SizedT) -> SizedT:
314
- if not len(v):
315
- raise ValueError(v)
316
- return v
317
-
318
-
319
- ########################################
320
- # ../../../omlish/lite/json.py
321
-
322
-
323
- ##
324
-
325
-
326
- JSON_PRETTY_INDENT = 2
327
-
328
- JSON_PRETTY_KWARGS: ta.Mapping[str, ta.Any] = dict(
329
- indent=JSON_PRETTY_INDENT,
330
- )
331
-
332
- json_dump_pretty: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_PRETTY_KWARGS) # type: ignore
333
- json_dumps_pretty: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_PRETTY_KWARGS)
334
-
335
-
336
- ##
337
-
338
-
339
- JSON_COMPACT_SEPARATORS = (',', ':')
340
-
341
- JSON_COMPACT_KWARGS: ta.Mapping[str, ta.Any] = dict(
342
- indent=None,
343
- separators=JSON_COMPACT_SEPARATORS,
344
- )
345
-
346
- json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON_COMPACT_KWARGS) # type: ignore
347
- json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
348
-
349
-
350
- ########################################
351
- # ../../../omlish/lite/reflect.py
352
-
353
-
354
- _GENERIC_ALIAS_TYPES = (
355
- ta._GenericAlias, # type: ignore # noqa
356
- *([ta._SpecialGenericAlias] if hasattr(ta, '_SpecialGenericAlias') else []), # noqa
357
- )
358
-
359
-
360
- def is_generic_alias(obj, *, origin: ta.Any = None) -> bool:
361
- return (
362
- isinstance(obj, _GENERIC_ALIAS_TYPES) and
363
- (origin is None or ta.get_origin(obj) is origin)
364
- )
365
-
366
-
367
- is_union_alias = functools.partial(is_generic_alias, origin=ta.Union)
368
- is_callable_alias = functools.partial(is_generic_alias, origin=ta.Callable)
369
-
370
-
371
- def is_optional_alias(spec: ta.Any) -> bool:
372
- return (
373
- isinstance(spec, _GENERIC_ALIAS_TYPES) and # noqa
374
- ta.get_origin(spec) is ta.Union and
375
- len(ta.get_args(spec)) == 2 and
376
- any(a in (None, type(None)) for a in ta.get_args(spec))
377
- )
378
-
379
-
380
- def get_optional_alias_arg(spec: ta.Any) -> ta.Any:
381
- [it] = [it for it in ta.get_args(spec) if it not in (None, type(None))]
382
- return it
383
-
384
-
385
- def is_new_type(spec: ta.Any) -> bool:
386
- if isinstance(ta.NewType, type):
387
- return isinstance(spec, ta.NewType)
388
- else:
389
- # Before https://github.com/python/cpython/commit/c2f33dfc83ab270412bf243fb21f724037effa1a
390
- return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1] # type: ignore # noqa
391
-
392
-
393
- def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
394
- seen = set()
395
- todo = list(reversed(cls.__subclasses__()))
396
- while todo:
397
- cur = todo.pop()
398
- if cur in seen:
399
- continue
400
- seen.add(cur)
401
- yield cur
402
- todo.extend(reversed(cur.__subclasses__()))
403
-
404
-
405
- ########################################
406
- # ../../../omlish/lite/logs.py
407
- """
408
- TODO:
409
- - translate json keys
410
- - debug
411
- """
412
-
413
-
414
- log = logging.getLogger(__name__)
415
-
416
-
417
- ##
418
-
419
-
420
- class TidLogFilter(logging.Filter):
421
-
422
- def filter(self, record):
423
- record.tid = threading.get_native_id()
424
- return True
425
-
426
-
427
- ##
428
-
429
-
430
- class JsonLogFormatter(logging.Formatter):
431
-
432
- KEYS: ta.Mapping[str, bool] = {
433
- 'name': False,
434
- 'msg': False,
435
- 'args': False,
436
- 'levelname': False,
437
- 'levelno': False,
438
- 'pathname': False,
439
- 'filename': False,
440
- 'module': False,
441
- 'exc_info': True,
442
- 'exc_text': True,
443
- 'stack_info': True,
444
- 'lineno': False,
445
- 'funcName': False,
446
- 'created': False,
447
- 'msecs': False,
448
- 'relativeCreated': False,
449
- 'thread': False,
450
- 'threadName': False,
451
- 'processName': False,
452
- 'process': False,
453
- }
454
-
455
- def format(self, record: logging.LogRecord) -> str:
456
- dct = {
457
- k: v
458
- for k, o in self.KEYS.items()
459
- for v in [getattr(record, k)]
460
- if not (o and v is None)
461
- }
462
- return json_dumps_compact(dct)
463
-
464
-
465
- ##
466
-
467
-
468
- STANDARD_LOG_FORMAT_PARTS = [
469
- ('asctime', '%(asctime)-15s'),
470
- ('process', 'pid=%(process)-6s'),
471
- ('thread', 'tid=%(thread)x'),
472
- ('levelname', '%(levelname)s'),
473
- ('name', '%(name)s'),
474
- ('separator', '::'),
475
- ('message', '%(message)s'),
476
- ]
477
-
478
-
479
- class StandardLogFormatter(logging.Formatter):
480
-
481
- @staticmethod
482
- def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
483
- return ' '.join(v for k, v in parts)
484
-
485
- converter = datetime.datetime.fromtimestamp # type: ignore
486
-
487
- def formatTime(self, record, datefmt=None):
488
- ct = self.converter(record.created) # type: ignore
489
- if datefmt:
490
- return ct.strftime(datefmt) # noqa
491
- else:
492
- t = ct.strftime('%Y-%m-%d %H:%M:%S')
493
- return '%s.%03d' % (t, record.msecs) # noqa
494
-
495
-
496
- ##
497
-
498
-
499
- class ProxyLogFilterer(logging.Filterer):
500
- def __init__(self, underlying: logging.Filterer) -> None: # noqa
501
- self._underlying = underlying
502
-
503
- @property
504
- def underlying(self) -> logging.Filterer:
505
- return self._underlying
506
-
507
- @property
508
- def filters(self):
509
- return self._underlying.filters
510
-
511
- @filters.setter
512
- def filters(self, filters):
513
- self._underlying.filters = filters
514
-
515
- def addFilter(self, filter): # noqa
516
- self._underlying.addFilter(filter)
517
-
518
- def removeFilter(self, filter): # noqa
519
- self._underlying.removeFilter(filter)
520
-
521
- def filter(self, record):
522
- return self._underlying.filter(record)
523
-
524
-
525
- class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
526
- def __init__(self, underlying: logging.Handler) -> None: # noqa
527
- ProxyLogFilterer.__init__(self, underlying)
528
-
529
- _underlying: logging.Handler
530
-
531
- @property
532
- def underlying(self) -> logging.Handler:
533
- return self._underlying
534
-
535
- def get_name(self):
536
- return self._underlying.get_name()
537
-
538
- def set_name(self, name):
539
- self._underlying.set_name(name)
540
-
541
- @property
542
- def name(self):
543
- return self._underlying.name
544
-
545
- @property
546
- def level(self):
547
- return self._underlying.level
548
-
549
- @level.setter
550
- def level(self, level):
551
- self._underlying.level = level
552
-
553
- @property
554
- def formatter(self):
555
- return self._underlying.formatter
556
-
557
- @formatter.setter
558
- def formatter(self, formatter):
559
- self._underlying.formatter = formatter
560
-
561
- def createLock(self):
562
- self._underlying.createLock()
563
-
564
- def acquire(self):
565
- self._underlying.acquire()
566
-
567
- def release(self):
568
- self._underlying.release()
569
-
570
- def setLevel(self, level):
571
- self._underlying.setLevel(level)
572
-
573
- def format(self, record):
574
- return self._underlying.format(record)
575
-
576
- def emit(self, record):
577
- self._underlying.emit(record)
578
-
579
- def handle(self, record):
580
- return self._underlying.handle(record)
581
-
582
- def setFormatter(self, fmt):
583
- self._underlying.setFormatter(fmt)
584
-
585
- def flush(self):
586
- self._underlying.flush()
587
-
588
- def close(self):
589
- self._underlying.close()
590
-
591
- def handleError(self, record):
592
- self._underlying.handleError(record)
593
-
594
-
595
- ##
596
-
597
-
598
- class StandardLogHandler(ProxyLogHandler):
599
- pass
600
-
601
-
602
- ##
603
-
604
-
605
- @contextlib.contextmanager
606
- def _locking_logging_module_lock() -> ta.Iterator[None]:
607
- if hasattr(logging, '_acquireLock'):
608
- logging._acquireLock() # noqa
609
- try:
610
- yield
611
- finally:
612
- logging._releaseLock() # type: ignore # noqa
613
-
614
- elif hasattr(logging, '_lock'):
615
- # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
616
- with logging._lock: # noqa
617
- yield
618
-
619
- else:
620
- raise Exception("Can't find lock in logging module")
621
-
622
-
623
- def configure_standard_logging(
624
- level: ta.Union[int, str] = logging.INFO,
625
- *,
626
- json: bool = False,
627
- target: ta.Optional[logging.Logger] = None,
628
- force: bool = False,
629
- handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
630
- ) -> ta.Optional[StandardLogHandler]:
631
- with _locking_logging_module_lock():
632
- if target is None:
633
- target = logging.root
634
-
635
- #
636
-
637
- if not force:
638
- if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
639
- return None
640
-
641
- #
642
-
643
- if handler_factory is not None:
644
- handler = handler_factory()
645
- else:
646
- handler = logging.StreamHandler()
647
-
648
- #
649
-
650
- formatter: logging.Formatter
651
- if json:
652
- formatter = JsonLogFormatter()
653
- else:
654
- formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
655
- handler.setFormatter(formatter)
656
-
657
- #
658
-
659
- handler.addFilter(TidLogFilter())
660
-
661
- #
662
-
663
- target.addHandler(handler)
664
-
665
- #
666
-
667
- if level is not None:
668
- target.setLevel(level)
669
-
670
- #
671
-
672
- return StandardLogHandler(handler)
673
-
674
-
675
- ########################################
676
- # ../../../omlish/lite/marshal.py
677
- """
678
- TODO:
679
- - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
680
- - nonstrict toggle
681
- """
682
-
683
-
684
- ##
685
-
686
-
687
- class ObjMarshaler(abc.ABC):
688
- @abc.abstractmethod
689
- def marshal(self, o: ta.Any) -> ta.Any:
690
- raise NotImplementedError
691
-
692
- @abc.abstractmethod
693
- def unmarshal(self, o: ta.Any) -> ta.Any:
694
- raise NotImplementedError
695
-
696
-
697
- class NopObjMarshaler(ObjMarshaler):
698
- def marshal(self, o: ta.Any) -> ta.Any:
699
- return o
700
-
701
- def unmarshal(self, o: ta.Any) -> ta.Any:
702
- return o
703
-
704
-
705
- @dc.dataclass()
706
- class ProxyObjMarshaler(ObjMarshaler):
707
- m: ta.Optional[ObjMarshaler] = None
708
-
709
- def marshal(self, o: ta.Any) -> ta.Any:
710
- return check_not_none(self.m).marshal(o)
711
-
712
- def unmarshal(self, o: ta.Any) -> ta.Any:
713
- return check_not_none(self.m).unmarshal(o)
714
-
715
-
716
- @dc.dataclass(frozen=True)
717
- class CastObjMarshaler(ObjMarshaler):
718
- ty: type
719
-
720
- def marshal(self, o: ta.Any) -> ta.Any:
721
- return o
722
-
723
- def unmarshal(self, o: ta.Any) -> ta.Any:
724
- return self.ty(o)
725
-
726
-
727
- class DynamicObjMarshaler(ObjMarshaler):
728
- def marshal(self, o: ta.Any) -> ta.Any:
729
- return marshal_obj(o)
730
-
731
- def unmarshal(self, o: ta.Any) -> ta.Any:
732
- return o
733
-
734
-
735
- @dc.dataclass(frozen=True)
736
- class Base64ObjMarshaler(ObjMarshaler):
737
- ty: type
738
-
739
- def marshal(self, o: ta.Any) -> ta.Any:
740
- return base64.b64encode(o).decode('ascii')
741
-
742
- def unmarshal(self, o: ta.Any) -> ta.Any:
743
- return self.ty(base64.b64decode(o))
744
-
745
-
746
- @dc.dataclass(frozen=True)
747
- class EnumObjMarshaler(ObjMarshaler):
748
- ty: type
749
-
750
- def marshal(self, o: ta.Any) -> ta.Any:
751
- return o.name
752
-
753
- def unmarshal(self, o: ta.Any) -> ta.Any:
754
- return self.ty.__members__[o] # type: ignore
755
-
756
-
757
- @dc.dataclass(frozen=True)
758
- class OptionalObjMarshaler(ObjMarshaler):
759
- item: ObjMarshaler
760
-
761
- def marshal(self, o: ta.Any) -> ta.Any:
762
- if o is None:
763
- return None
764
- return self.item.marshal(o)
765
-
766
- def unmarshal(self, o: ta.Any) -> ta.Any:
767
- if o is None:
768
- return None
769
- return self.item.unmarshal(o)
770
-
771
-
772
- @dc.dataclass(frozen=True)
773
- class MappingObjMarshaler(ObjMarshaler):
774
- ty: type
775
- km: ObjMarshaler
776
- vm: ObjMarshaler
777
-
778
- def marshal(self, o: ta.Any) -> ta.Any:
779
- return {self.km.marshal(k): self.vm.marshal(v) for k, v in o.items()}
780
-
781
- def unmarshal(self, o: ta.Any) -> ta.Any:
782
- return self.ty((self.km.unmarshal(k), self.vm.unmarshal(v)) for k, v in o.items())
783
-
784
-
785
- @dc.dataclass(frozen=True)
786
- class IterableObjMarshaler(ObjMarshaler):
787
- ty: type
788
- item: ObjMarshaler
789
-
790
- def marshal(self, o: ta.Any) -> ta.Any:
791
- return [self.item.marshal(e) for e in o]
792
-
793
- def unmarshal(self, o: ta.Any) -> ta.Any:
794
- return self.ty(self.item.unmarshal(e) for e in o)
795
-
796
-
797
- @dc.dataclass(frozen=True)
798
- class DataclassObjMarshaler(ObjMarshaler):
799
- ty: type
800
- fs: ta.Mapping[str, ObjMarshaler]
801
- nonstrict: bool = False
802
-
803
- def marshal(self, o: ta.Any) -> ta.Any:
804
- return {k: m.marshal(getattr(o, k)) for k, m in self.fs.items()}
805
-
806
- def unmarshal(self, o: ta.Any) -> ta.Any:
807
- return self.ty(**{k: self.fs[k].unmarshal(v) for k, v in o.items() if not self.nonstrict or k in self.fs})
808
-
809
-
810
- @dc.dataclass(frozen=True)
811
- class PolymorphicObjMarshaler(ObjMarshaler):
812
- class Impl(ta.NamedTuple):
813
- ty: type
814
- tag: str
815
- m: ObjMarshaler
816
-
817
- impls_by_ty: ta.Mapping[type, Impl]
818
- impls_by_tag: ta.Mapping[str, Impl]
819
-
820
- def marshal(self, o: ta.Any) -> ta.Any:
821
- impl = self.impls_by_ty[type(o)]
822
- return {impl.tag: impl.m.marshal(o)}
823
-
824
- def unmarshal(self, o: ta.Any) -> ta.Any:
825
- [(t, v)] = o.items()
826
- impl = self.impls_by_tag[t]
827
- return impl.m.unmarshal(v)
828
-
829
-
830
- @dc.dataclass(frozen=True)
831
- class DatetimeObjMarshaler(ObjMarshaler):
832
- ty: type
833
-
834
- def marshal(self, o: ta.Any) -> ta.Any:
835
- return o.isoformat()
836
-
837
- def unmarshal(self, o: ta.Any) -> ta.Any:
838
- return self.ty.fromisoformat(o) # type: ignore
839
-
840
-
841
- class DecimalObjMarshaler(ObjMarshaler):
842
- def marshal(self, o: ta.Any) -> ta.Any:
843
- return str(check_isinstance(o, decimal.Decimal))
844
-
845
- def unmarshal(self, v: ta.Any) -> ta.Any:
846
- return decimal.Decimal(check_isinstance(v, str))
847
-
848
-
849
- class FractionObjMarshaler(ObjMarshaler):
850
- def marshal(self, o: ta.Any) -> ta.Any:
851
- fr = check_isinstance(o, fractions.Fraction)
852
- return [fr.numerator, fr.denominator]
853
-
854
- def unmarshal(self, v: ta.Any) -> ta.Any:
855
- num, denom = check_isinstance(v, list)
856
- return fractions.Fraction(num, denom)
857
-
858
-
859
- class UuidObjMarshaler(ObjMarshaler):
860
- def marshal(self, o: ta.Any) -> ta.Any:
861
- return str(o)
862
-
863
- def unmarshal(self, o: ta.Any) -> ta.Any:
864
- return uuid.UUID(o)
865
-
866
-
867
- ##
868
-
869
-
870
- _DEFAULT_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
871
- **{t: NopObjMarshaler() for t in (type(None),)},
872
- **{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
873
- **{t: Base64ObjMarshaler(t) for t in (bytes, bytearray)},
874
- **{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
875
- **{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
876
-
877
- ta.Any: DynamicObjMarshaler(),
878
-
879
- **{t: DatetimeObjMarshaler(t) for t in (datetime.date, datetime.time, datetime.datetime)},
880
- decimal.Decimal: DecimalObjMarshaler(),
881
- fractions.Fraction: FractionObjMarshaler(),
882
- uuid.UUID: UuidObjMarshaler(),
883
- }
884
-
885
- _OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
886
- **{t: t for t in (dict,)},
887
- **{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
888
- }
889
-
890
- _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
891
- **{t: t for t in (list, tuple, set, frozenset)},
892
- collections.abc.Set: frozenset,
893
- collections.abc.MutableSet: set,
894
- collections.abc.Sequence: tuple,
895
- collections.abc.MutableSequence: list,
896
- }
897
-
898
-
899
- def _make_obj_marshaler(
900
- ty: ta.Any,
901
- rec: ta.Callable[[ta.Any], ObjMarshaler],
902
- *,
903
- nonstrict_dataclasses: bool = False,
904
- ) -> ObjMarshaler:
905
- if isinstance(ty, type):
906
- if abc.ABC in ty.__bases__:
907
- impls = [ # type: ignore
908
- PolymorphicObjMarshaler.Impl(
909
- ity,
910
- ity.__qualname__,
911
- rec(ity),
912
- )
913
- for ity in deep_subclasses(ty)
914
- if abc.ABC not in ity.__bases__
915
- ]
916
- return PolymorphicObjMarshaler(
917
- {i.ty: i for i in impls},
918
- {i.tag: i for i in impls},
919
- )
920
-
921
- if issubclass(ty, enum.Enum):
922
- return EnumObjMarshaler(ty)
923
-
924
- if dc.is_dataclass(ty):
925
- return DataclassObjMarshaler(
926
- ty,
927
- {f.name: rec(f.type) for f in dc.fields(ty)},
928
- nonstrict=nonstrict_dataclasses,
929
- )
930
-
931
- if is_generic_alias(ty):
932
- try:
933
- mt = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES[ta.get_origin(ty)]
934
- except KeyError:
935
- pass
936
- else:
937
- k, v = ta.get_args(ty)
938
- return MappingObjMarshaler(mt, rec(k), rec(v))
939
-
940
- try:
941
- st = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES[ta.get_origin(ty)]
942
- except KeyError:
943
- pass
944
- else:
945
- [e] = ta.get_args(ty)
946
- return IterableObjMarshaler(st, rec(e))
947
-
948
- if is_union_alias(ty):
949
- return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
950
-
951
- raise TypeError(ty)
952
-
953
-
954
- ##
955
-
956
-
957
- _OBJ_MARSHALERS_LOCK = threading.RLock()
958
-
959
- _OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
960
-
961
- _OBJ_MARSHALER_PROXIES: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
962
-
963
-
964
- def register_opj_marshaler(ty: ta.Any, m: ObjMarshaler) -> None:
965
- with _OBJ_MARSHALERS_LOCK:
966
- if ty in _OBJ_MARSHALERS:
967
- raise KeyError(ty)
968
- _OBJ_MARSHALERS[ty] = m
969
-
970
-
971
- def get_obj_marshaler(
972
- ty: ta.Any,
973
- *,
974
- no_cache: bool = False,
975
- **kwargs: ta.Any,
976
- ) -> ObjMarshaler:
977
- with _OBJ_MARSHALERS_LOCK:
978
- if not no_cache:
979
- try:
980
- return _OBJ_MARSHALERS[ty]
981
- except KeyError:
982
- pass
983
-
984
- try:
985
- return _OBJ_MARSHALER_PROXIES[ty]
986
- except KeyError:
987
- pass
988
-
989
- rec = functools.partial(
990
- get_obj_marshaler,
991
- no_cache=no_cache,
992
- **kwargs,
993
- )
994
-
995
- p = ProxyObjMarshaler()
996
- _OBJ_MARSHALER_PROXIES[ty] = p
997
- try:
998
- m = _make_obj_marshaler(ty, rec, **kwargs)
999
- finally:
1000
- del _OBJ_MARSHALER_PROXIES[ty]
1001
- p.m = m
1002
-
1003
- if not no_cache:
1004
- _OBJ_MARSHALERS[ty] = m
1005
- return m
1006
-
1007
-
1008
- ##
1009
-
1010
-
1011
- def marshal_obj(o: ta.Any, ty: ta.Any = None) -> ta.Any:
1012
- return get_obj_marshaler(ty if ty is not None else type(o)).marshal(o)
1013
-
1014
-
1015
- def unmarshal_obj(o: ta.Any, ty: ta.Union[ta.Type[T], ta.Any]) -> T:
1016
- return get_obj_marshaler(ty).unmarshal(o)
1017
-
1018
-
1019
- ########################################
1020
- # ../../../omlish/lite/runtime.py
1021
-
1022
-
1023
- @cached_nullary
1024
- def is_debugger_attached() -> bool:
1025
- return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
1026
-
1027
-
1028
- REQUIRED_PYTHON_VERSION = (3, 8)
1029
-
1030
-
1031
- def check_runtime_version() -> None:
1032
- if sys.version_info < REQUIRED_PYTHON_VERSION:
1033
- raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
1034
-
1035
-
1036
- ########################################
1037
- # ../../../omlish/lite/subprocesses.py
1038
-
1039
-
1040
- ##
1041
-
1042
-
1043
- _SUBPROCESS_SHELL_WRAP_EXECS = False
1044
-
1045
-
1046
- def subprocess_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
1047
- return ('sh', '-c', ' '.join(map(shlex.quote, args)))
1048
-
1049
-
1050
- def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
1051
- if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
1052
- return subprocess_shell_wrap_exec(*args)
1053
- else:
1054
- return args
1055
-
1056
-
1057
- def _prepare_subprocess_invocation(
1058
- *args: str,
1059
- env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
1060
- extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
1061
- quiet: bool = False,
1062
- shell: bool = False,
1063
- **kwargs: ta.Any,
1064
- ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
1065
- log.debug(args)
1066
- if extra_env:
1067
- log.debug(extra_env)
1068
-
1069
- if extra_env:
1070
- env = {**(env if env is not None else os.environ), **extra_env}
1071
-
1072
- if quiet and 'stderr' not in kwargs:
1073
- if not log.isEnabledFor(logging.DEBUG):
1074
- kwargs['stderr'] = subprocess.DEVNULL
1075
-
1076
- if not shell:
1077
- args = subprocess_maybe_shell_wrap_exec(*args)
1078
-
1079
- return args, dict(
1080
- env=env,
1081
- shell=shell,
1082
- **kwargs,
1083
- )
1084
-
1085
-
1086
- def subprocess_check_call(*args: str, stdout=sys.stderr, **kwargs: ta.Any) -> None:
1087
- args, kwargs = _prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
1088
- return subprocess.check_call(args, **kwargs) # type: ignore
1089
-
1090
-
1091
- def subprocess_check_output(*args: str, **kwargs: ta.Any) -> bytes:
1092
- args, kwargs = _prepare_subprocess_invocation(*args, **kwargs)
1093
- return subprocess.check_output(args, **kwargs)
1094
-
1095
-
1096
- def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
1097
- return subprocess_check_output(*args, **kwargs).decode().strip()
1098
-
1099
-
1100
- ##
1101
-
1102
-
1103
- DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
1104
- FileNotFoundError,
1105
- subprocess.CalledProcessError,
1106
- )
1107
-
1108
-
1109
- def subprocess_try_call(
1110
- *args: str,
1111
- try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
1112
- **kwargs: ta.Any,
1113
- ) -> bool:
1114
- try:
1115
- subprocess_check_call(*args, **kwargs)
1116
- except try_exceptions as e: # noqa
1117
- if log.isEnabledFor(logging.DEBUG):
1118
- log.exception('command failed')
1119
- return False
1120
- else:
1121
- return True
1122
-
1123
-
1124
- def subprocess_try_output(
1125
- *args: str,
1126
- try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
1127
- **kwargs: ta.Any,
1128
- ) -> ta.Optional[bytes]:
1129
- try:
1130
- return subprocess_check_output(*args, **kwargs)
1131
- except try_exceptions as e: # noqa
1132
- if log.isEnabledFor(logging.DEBUG):
1133
- log.exception('command failed')
1134
- return None
1135
-
1136
-
1137
- def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
1138
- out = subprocess_try_output(*args, **kwargs)
1139
- return out.decode().strip() if out is not None else None
1140
-
1141
-
1142
- ##
1143
-
1144
-
1145
- def subprocess_close(
1146
- proc: subprocess.Popen,
1147
- timeout: ta.Optional[float] = None,
1148
- ) -> None:
1149
- # TODO: terminate, sleep, kill
1150
- if proc.stdout:
1151
- proc.stdout.close()
1152
- if proc.stderr:
1153
- proc.stderr.close()
1154
- if proc.stdin:
1155
- proc.stdin.close()
1156
-
1157
- proc.wait(timeout)
1158
-
1159
-
1160
- ########################################
1161
- # runcommands.py
1162
-
1163
-
1164
- @dc.dataclass(frozen=True)
1165
- class CommandRequest:
1166
- cmd: ta.Sequence[str]
1167
- in_: ta.Optional[bytes] = None
1168
-
1169
-
1170
- @dc.dataclass(frozen=True)
1171
- class CommandResponse:
1172
- req: CommandRequest
1173
- rc: int
1174
- out: bytes
1175
- err: bytes
1176
-
1177
-
1178
- def _run_commands_loop(input: ta.BinaryIO, output: ta.BinaryIO = sys.stdout.buffer) -> None: # noqa
1179
- while (l := input.readline().decode('utf-8').strip()):
1180
- req: CommandRequest = unmarshal_obj(json.loads(l), CommandRequest)
1181
- proc = subprocess.Popen( # type: ignore
1182
- subprocess_maybe_shell_wrap_exec(*req.cmd),
1183
- **(dict(stdin=io.BytesIO(req.in_)) if req.in_ is not None else {}),
1184
- stdout=subprocess.PIPE,
1185
- stderr=subprocess.PIPE,
1186
- )
1187
- out, err = proc.communicate()
1188
- resp = CommandResponse(
1189
- req=req,
1190
- rc=proc.returncode,
1191
- out=out, # noqa
1192
- err=err, # noqa
1193
- )
1194
- output.write(json_dumps_compact(marshal_obj(resp)).encode('utf-8'))
1195
- output.write(b'\n')
1196
- output.flush()
1197
-
1198
-
1199
- def run_commands_main() -> None:
1200
- bs = post_boostrap()
1201
- _run_commands_loop(bs.input)