ominfra 0.0.0.dev157__py3-none-any.whl → 0.0.0.dev158__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
ominfra/scripts/manage.py CHANGED
@@ -100,7 +100,7 @@ CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
100
100
 
101
101
  # deploy/paths.py
102
102
  DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
103
- DeployPathSpec = ta.Literal['app', 'tag'] # ta.TypeAlias
103
+ DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
104
104
 
105
105
  # ../../omlish/argparse/cli.py
106
106
  ArgparseCommandFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
@@ -118,7 +118,7 @@ InjectorBindingOrBindings = ta.Union['InjectorBinding', 'InjectorBindings']
118
118
  # ../configs.py
119
119
  ConfigMapping = ta.Mapping[str, ta.Any]
120
120
 
121
- # ../../omlish/lite/subprocesses.py
121
+ # ../../omlish/subprocesses.py
122
122
  SubprocessChannelOption = ta.Literal['pipe', 'stdout', 'devnull'] # ta.TypeAlias
123
123
 
124
124
  # system/packages.py
@@ -1365,6 +1365,9 @@ class MainConfig:
1365
1365
  # ../deploy/config.py
1366
1366
 
1367
1367
 
1368
+ ##
1369
+
1370
+
1368
1371
  @dc.dataclass(frozen=True)
1369
1372
  class DeployConfig:
1370
1373
  deploy_home: ta.Optional[str] = None
@@ -1539,7 +1542,7 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
1539
1542
  # Get pid
1540
1543
  pid = os.getpid()
1541
1544
 
1542
- # Two copies of main src to be sent to parent
1545
+ # Two copies of payload src to be sent to parent
1543
1546
  r0, w0 = os.pipe()
1544
1547
  r1, w1 = os.pipe()
1545
1548
 
@@ -1578,17 +1581,17 @@ def _pyremote_bootstrap_main(context_name: str) -> None:
1578
1581
  # Write pid
1579
1582
  os.write(1, struct.pack('<Q', pid))
1580
1583
 
1581
- # Read main src from stdin
1582
- main_z_len = struct.unpack('<I', os.read(0, 4))[0]
1583
- if len(main_z := os.fdopen(0, 'rb').read(main_z_len)) != main_z_len:
1584
+ # Read payload src from stdin
1585
+ payload_z_len = struct.unpack('<I', os.read(0, 4))[0]
1586
+ if len(payload_z := os.fdopen(0, 'rb').read(payload_z_len)) != payload_z_len:
1584
1587
  raise EOFError
1585
- main_src = zlib.decompress(main_z)
1588
+ payload_src = zlib.decompress(payload_z)
1586
1589
 
1587
- # Write both copies of main src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely fill
1588
- # and block and need to be drained by pyremote_bootstrap_finalize running in parent.
1590
+ # Write both copies of payload src. Must write to w0 (parent stdin) before w1 (copy pipe) as pipe will likely
1591
+ # fill and block and need to be drained by pyremote_bootstrap_finalize running in parent.
1589
1592
  for w in [w0, w1]:
1590
1593
  fp = os.fdopen(w, 'wb', 0)
1591
- fp.write(main_src)
1594
+ fp.write(payload_src)
1592
1595
  fp.close()
1593
1596
 
1594
1597
  # Write second ack
@@ -1652,7 +1655,7 @@ class PyremotePayloadRuntime:
1652
1655
  input: ta.BinaryIO
1653
1656
  output: ta.BinaryIO
1654
1657
  context_name: str
1655
- main_src: str
1658
+ payload_src: str
1656
1659
  options: PyremoteBootstrapOptions
1657
1660
  env_info: PyremoteEnvInfo
1658
1661
 
@@ -1660,9 +1663,9 @@ class PyremotePayloadRuntime:
1660
1663
  def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
1661
1664
  # If src file var is not present we need to do initial finalization
1662
1665
  if _PYREMOTE_BOOTSTRAP_SRC_FILE_VAR not in os.environ:
1663
- # Read second copy of main src
1666
+ # Read second copy of payload src
1664
1667
  r1 = os.fdopen(_PYREMOTE_BOOTSTRAP_SRC_FD, 'rb', 0)
1665
- main_src = r1.read().decode('utf-8')
1668
+ payload_src = r1.read().decode('utf-8')
1666
1669
  r1.close()
1667
1670
 
1668
1671
  # Reap boostrap child. Must be done after reading second copy of source because source may be too big to fit in
@@ -1680,7 +1683,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
1680
1683
  # Write temp source file
1681
1684
  import tempfile
1682
1685
  tfd, tfn = tempfile.mkstemp('-pyremote.py')
1683
- os.write(tfd, main_src.encode('utf-8'))
1686
+ os.write(tfd, payload_src.encode('utf-8'))
1684
1687
  os.close(tfd)
1685
1688
 
1686
1689
  # Set vars
@@ -1699,7 +1702,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
1699
1702
 
1700
1703
  # Read temp source file
1701
1704
  with open(os.environ.pop(_PYREMOTE_BOOTSTRAP_SRC_FILE_VAR)) as sf:
1702
- main_src = sf.read()
1705
+ payload_src = sf.read()
1703
1706
 
1704
1707
  # Restore vars
1705
1708
  sys.executable = os.environ.pop(_PYREMOTE_BOOTSTRAP_ARGV0_VAR)
@@ -1732,7 +1735,7 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
1732
1735
  input=input,
1733
1736
  output=output,
1734
1737
  context_name=context_name,
1735
- main_src=main_src,
1738
+ payload_src=payload_src,
1736
1739
  options=options,
1737
1740
  env_info=env_info,
1738
1741
  )
@@ -1744,31 +1747,31 @@ def pyremote_bootstrap_finalize() -> PyremotePayloadRuntime:
1744
1747
  class PyremoteBootstrapDriver:
1745
1748
  def __init__(
1746
1749
  self,
1747
- main_src: ta.Union[str, ta.Sequence[str]],
1750
+ payload_src: ta.Union[str, ta.Sequence[str]],
1748
1751
  options: PyremoteBootstrapOptions = PyremoteBootstrapOptions(),
1749
1752
  ) -> None:
1750
1753
  super().__init__()
1751
1754
 
1752
- self._main_src = main_src
1755
+ self._payload_src = payload_src
1753
1756
  self._options = options
1754
1757
 
1755
- self._prepared_main_src = self._prepare_main_src(main_src, options)
1756
- self._main_z = zlib.compress(self._prepared_main_src.encode('utf-8'))
1758
+ self._prepared_payload_src = self._prepare_payload_src(payload_src, options)
1759
+ self._payload_z = zlib.compress(self._prepared_payload_src.encode('utf-8'))
1757
1760
 
1758
1761
  self._options_json = json.dumps(dc.asdict(options), indent=None, separators=(',', ':')).encode('utf-8') # noqa
1759
1762
  #
1760
1763
 
1761
1764
  @classmethod
1762
- def _prepare_main_src(
1765
+ def _prepare_payload_src(
1763
1766
  cls,
1764
- main_src: ta.Union[str, ta.Sequence[str]],
1767
+ payload_src: ta.Union[str, ta.Sequence[str]],
1765
1768
  options: PyremoteBootstrapOptions,
1766
1769
  ) -> str:
1767
1770
  parts: ta.List[str]
1768
- if isinstance(main_src, str):
1769
- parts = [main_src]
1771
+ if isinstance(payload_src, str):
1772
+ parts = [payload_src]
1770
1773
  else:
1771
- parts = list(main_src)
1774
+ parts = list(payload_src)
1772
1775
 
1773
1776
  if (mn := options.main_name_override) is not None:
1774
1777
  parts.insert(0, f'__name__ = {mn!r}')
@@ -1804,9 +1807,9 @@ class PyremoteBootstrapDriver:
1804
1807
  d = yield from self._read(8)
1805
1808
  pid = struct.unpack('<Q', d)[0]
1806
1809
 
1807
- # Write main src
1808
- yield from self._write(struct.pack('<I', len(self._main_z)))
1809
- yield from self._write(self._main_z)
1810
+ # Write payload src
1811
+ yield from self._write(struct.pack('<I', len(self._payload_z)))
1812
+ yield from self._write(self._payload_z)
1810
1813
 
1811
1814
  # Read second ack (after writing src copies)
1812
1815
  yield from self._expect(_PYREMOTE_BOOTSTRAP_ACK1)
@@ -2540,6 +2543,13 @@ json_dump_compact: ta.Callable[..., bytes] = functools.partial(json.dump, **JSON
2540
2543
  json_dumps_compact: ta.Callable[..., str] = functools.partial(json.dumps, **JSON_COMPACT_KWARGS)
2541
2544
 
2542
2545
 
2546
+ ########################################
2547
+ # ../../../omlish/lite/logs.py
2548
+
2549
+
2550
+ log = logging.getLogger(__name__)
2551
+
2552
+
2543
2553
  ########################################
2544
2554
  # ../../../omlish/lite/maybes.py
2545
2555
 
@@ -2754,6 +2764,116 @@ def format_num_bytes(num_bytes: int) -> str:
2754
2764
  return f'{num_bytes / 1024 ** (len(FORMAT_NUM_BYTES_SUFFIXES) - 1):.2f}{FORMAT_NUM_BYTES_SUFFIXES[-1]}'
2755
2765
 
2756
2766
 
2767
+ ########################################
2768
+ # ../../../omlish/logs/filters.py
2769
+
2770
+
2771
+ class TidLogFilter(logging.Filter):
2772
+ def filter(self, record):
2773
+ record.tid = threading.get_native_id()
2774
+ return True
2775
+
2776
+
2777
+ ########################################
2778
+ # ../../../omlish/logs/proxy.py
2779
+
2780
+
2781
+ class ProxyLogFilterer(logging.Filterer):
2782
+ def __init__(self, underlying: logging.Filterer) -> None: # noqa
2783
+ self._underlying = underlying
2784
+
2785
+ @property
2786
+ def underlying(self) -> logging.Filterer:
2787
+ return self._underlying
2788
+
2789
+ @property
2790
+ def filters(self):
2791
+ return self._underlying.filters
2792
+
2793
+ @filters.setter
2794
+ def filters(self, filters):
2795
+ self._underlying.filters = filters
2796
+
2797
+ def addFilter(self, filter): # noqa
2798
+ self._underlying.addFilter(filter)
2799
+
2800
+ def removeFilter(self, filter): # noqa
2801
+ self._underlying.removeFilter(filter)
2802
+
2803
+ def filter(self, record):
2804
+ return self._underlying.filter(record)
2805
+
2806
+
2807
+ class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
2808
+ def __init__(self, underlying: logging.Handler) -> None: # noqa
2809
+ ProxyLogFilterer.__init__(self, underlying)
2810
+
2811
+ _underlying: logging.Handler
2812
+
2813
+ @property
2814
+ def underlying(self) -> logging.Handler:
2815
+ return self._underlying
2816
+
2817
+ def get_name(self):
2818
+ return self._underlying.get_name()
2819
+
2820
+ def set_name(self, name):
2821
+ self._underlying.set_name(name)
2822
+
2823
+ @property
2824
+ def name(self):
2825
+ return self._underlying.name
2826
+
2827
+ @property
2828
+ def level(self):
2829
+ return self._underlying.level
2830
+
2831
+ @level.setter
2832
+ def level(self, level):
2833
+ self._underlying.level = level
2834
+
2835
+ @property
2836
+ def formatter(self):
2837
+ return self._underlying.formatter
2838
+
2839
+ @formatter.setter
2840
+ def formatter(self, formatter):
2841
+ self._underlying.formatter = formatter
2842
+
2843
+ def createLock(self):
2844
+ self._underlying.createLock()
2845
+
2846
+ def acquire(self):
2847
+ self._underlying.acquire()
2848
+
2849
+ def release(self):
2850
+ self._underlying.release()
2851
+
2852
+ def setLevel(self, level):
2853
+ self._underlying.setLevel(level)
2854
+
2855
+ def format(self, record):
2856
+ return self._underlying.format(record)
2857
+
2858
+ def emit(self, record):
2859
+ self._underlying.emit(record)
2860
+
2861
+ def handle(self, record):
2862
+ return self._underlying.handle(record)
2863
+
2864
+ def setFormatter(self, fmt):
2865
+ self._underlying.setFormatter(fmt)
2866
+
2867
+ def flush(self):
2868
+ self._underlying.flush()
2869
+
2870
+ def close(self):
2871
+ self._underlying.close()
2872
+
2873
+ def handleError(self, record):
2874
+ self._underlying.handleError(record)
2875
+
2876
+
2757
2877
  ########################################
2758
2878
  # ../../../omlish/os/deathsig.py
2759
2879
 
@@ -3939,22 +4059,22 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
3939
4059
  ~deploy
3940
4060
  deploy.pid (flock)
3941
4061
  /app
3942
- /<appspec> - shallow clone
4062
+ /<appplaceholder> - shallow clone
3943
4063
  /conf
3944
4064
  /env
3945
- <appspec>.env
4065
+ <appplaceholder>.env
3946
4066
  /nginx
3947
- <appspec>.conf
4067
+ <appplaceholder>.conf
3948
4068
  /supervisor
3949
- <appspec>.conf
4069
+ <appplaceholder>.conf
3950
4070
  /venv
3951
- /<appspec>
4071
+ /<appplaceholder>
3952
4072
 
3953
4073
  ?
3954
4074
  /logs
3955
- /wrmsr--omlish--<spec>
4075
+ /wrmsr--omlish--<placeholder>
3956
4076
 
3957
- spec = <name>--<rev>--<when>
4077
+ placeholder = <name>--<rev>--<when>
3958
4078
 
3959
4079
  ==
3960
4080
 
@@ -3975,10 +4095,10 @@ for dn in [
3975
4095
  ##
3976
4096
 
3977
4097
 
3978
- DEPLOY_PATH_SPEC_PLACEHOLDER = '@'
3979
- DEPLOY_PATH_SPEC_SEPARATORS = '-.'
4098
+ DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER = '@'
4099
+ DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
3980
4100
 
3981
- DEPLOY_PATH_SPECS: ta.FrozenSet[str] = frozenset([
4101
+ DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
3982
4102
  'app',
3983
4103
  'tag', # <rev>-<dt>
3984
4104
  ])
@@ -3996,7 +4116,7 @@ class DeployPathPart(abc.ABC): # noqa
3996
4116
  raise NotImplementedError
3997
4117
 
3998
4118
  @abc.abstractmethod
3999
- def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
4119
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4000
4120
  raise NotImplementedError
4001
4121
 
4002
4122
 
@@ -4010,9 +4130,9 @@ class DirDeployPathPart(DeployPathPart, abc.ABC):
4010
4130
 
4011
4131
  @classmethod
4012
4132
  def parse(cls, s: str) -> 'DirDeployPathPart':
4013
- if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
4014
- check.equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
4015
- return SpecDirDeployPathPart(s[1:])
4133
+ if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
4134
+ check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
4135
+ return PlaceholderDirDeployPathPart(s[1:])
4016
4136
  else:
4017
4137
  return ConstDirDeployPathPart(s)
4018
4138
 
@@ -4024,13 +4144,13 @@ class FileDeployPathPart(DeployPathPart, abc.ABC):
4024
4144
 
4025
4145
  @classmethod
4026
4146
  def parse(cls, s: str) -> 'FileDeployPathPart':
4027
- if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
4028
- check.equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
4029
- if not any(c in s for c in DEPLOY_PATH_SPEC_SEPARATORS):
4030
- return SpecFileDeployPathPart(s[1:], '')
4147
+ if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
4148
+ check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
4149
+ if not any(c in s for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS):
4150
+ return PlaceholderFileDeployPathPart(s[1:], '')
4031
4151
  else:
4032
- p = min(f for c in DEPLOY_PATH_SPEC_SEPARATORS if (f := s.find(c)) > 0)
4033
- return SpecFileDeployPathPart(s[1:p], s[p:])
4152
+ p = min(f for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS if (f := s.find(c)) > 0)
4153
+ return PlaceholderFileDeployPathPart(s[1:p], s[p:])
4034
4154
  else:
4035
4155
  return ConstFileDeployPathPart(s)
4036
4156
 
@@ -4045,9 +4165,9 @@ class ConstDeployPathPart(DeployPathPart, abc.ABC):
4045
4165
  def __post_init__(self) -> None:
4046
4166
  check.non_empty_str(self.name)
4047
4167
  check.not_in('/', self.name)
4048
- check.not_in(DEPLOY_PATH_SPEC_PLACEHOLDER, self.name)
4168
+ check.not_in(DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, self.name)
4049
4169
 
4050
- def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
4170
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4051
4171
  return self.name
4052
4172
 
4053
4173
 
@@ -4063,40 +4183,40 @@ class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
4063
4183
 
4064
4184
 
4065
4185
  @dc.dataclass(frozen=True)
4066
- class SpecDeployPathPart(DeployPathPart, abc.ABC):
4067
- spec: str # DeployPathSpec
4186
+ class PlaceholderDeployPathPart(DeployPathPart, abc.ABC):
4187
+ placeholder: str # DeployPathPlaceholder
4068
4188
 
4069
4189
  def __post_init__(self) -> None:
4070
- check.non_empty_str(self.spec)
4071
- for c in [*DEPLOY_PATH_SPEC_SEPARATORS, DEPLOY_PATH_SPEC_PLACEHOLDER, '/']:
4072
- check.not_in(c, self.spec)
4073
- check.in_(self.spec, DEPLOY_PATH_SPECS)
4074
-
4075
- def _render_spec(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
4076
- if specs is not None:
4077
- return specs[self.spec] # type: ignore
4190
+ check.non_empty_str(self.placeholder)
4191
+ for c in [*DEPLOY_PATH_PLACEHOLDER_SEPARATORS, DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
4192
+ check.not_in(c, self.placeholder)
4193
+ check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
4194
+
4195
+ def _render_placeholder(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4196
+ if placeholders is not None:
4197
+ return placeholders[self.placeholder] # type: ignore
4078
4198
  else:
4079
- return DEPLOY_PATH_SPEC_PLACEHOLDER + self.spec
4199
+ return DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER + self.placeholder
4080
4200
 
4081
4201
 
4082
4202
  @dc.dataclass(frozen=True)
4083
- class SpecDirDeployPathPart(SpecDeployPathPart, DirDeployPathPart):
4084
- def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
4085
- return self._render_spec(specs)
4203
+ class PlaceholderDirDeployPathPart(PlaceholderDeployPathPart, DirDeployPathPart):
4204
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4205
+ return self._render_placeholder(placeholders)
4086
4206
 
4087
4207
 
4088
4208
  @dc.dataclass(frozen=True)
4089
- class SpecFileDeployPathPart(SpecDeployPathPart, FileDeployPathPart):
4209
+ class PlaceholderFileDeployPathPart(PlaceholderDeployPathPart, FileDeployPathPart):
4090
4210
  suffix: str
4091
4211
 
4092
4212
  def __post_init__(self) -> None:
4093
4213
  super().__post_init__()
4094
4214
  if self.suffix:
4095
- for c in [DEPLOY_PATH_SPEC_PLACEHOLDER, '/']:
4215
+ for c in [DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
4096
4216
  check.not_in(c, self.suffix)
4097
4217
 
4098
- def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
4099
- return self._render_spec(specs) + self.suffix
4218
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4219
+ return self._render_placeholder(placeholders) + self.suffix
4100
4220
 
4101
4221
 
4102
4222
  ##
@@ -4113,22 +4233,22 @@ class DeployPath:
4113
4233
 
4114
4234
  pd = {}
4115
4235
  for i, p in enumerate(self.parts):
4116
- if isinstance(p, SpecDeployPathPart):
4117
- if p.spec in pd:
4118
- raise DeployPathError('Duplicate specs in path', self)
4119
- pd[p.spec] = i
4236
+ if isinstance(p, PlaceholderDeployPathPart):
4237
+ if p.placeholder in pd:
4238
+ raise DeployPathError('Duplicate placeholders in path', self)
4239
+ pd[p.placeholder] = i
4120
4240
 
4121
4241
  if 'tag' in pd:
4122
4242
  if 'app' not in pd or pd['app'] >= pd['tag']:
4123
- raise DeployPathError('Tag spec in path without preceding app', self)
4243
+ raise DeployPathError('Tag placeholder in path without preceding app', self)
4124
4244
 
4125
4245
  @property
4126
4246
  def kind(self) -> ta.Literal['file', 'dir']:
4127
4247
  return self.parts[-1].kind
4128
4248
 
4129
- def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
4249
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4130
4250
  return os.path.join( # noqa
4131
- *[p.render(specs) for p in self.parts],
4251
+ *[p.render(placeholders) for p in self.parts],
4132
4252
  *([''] if self.kind == 'dir' else []),
4133
4253
  )
4134
4254
 
@@ -4156,6 +4276,34 @@ class DeployPathOwner(abc.ABC):
4156
4276
  raise NotImplementedError
4157
4277
 
4158
4278
 
4279
+ ########################################
4280
+ # ../deploy/specs.py
4281
+
4282
+
4283
+ ##
4284
+
4285
+
4286
+ @dc.dataclass(frozen=True)
4287
+ class DeployGitRepo:
4288
+ host: ta.Optional[str] = None
4289
+ username: ta.Optional[str] = None
4290
+ path: ta.Optional[str] = None
4291
+
4292
+ def __post_init__(self) -> None:
4293
+ check.not_in('..', check.non_empty_str(self.host))
4294
+ check.not_in('.', check.non_empty_str(self.path))
4295
+
4296
+
4297
+ ##
4298
+
4299
+
4300
+ @dc.dataclass(frozen=True)
4301
+ class DeploySpec:
4302
+ app: DeployApp
4303
+ repo: DeployGitRepo
4304
+ rev: DeployRev
4305
+
4306
+
4159
4307
  ########################################
4160
4308
  # ../remote/config.py
4161
4309
 
@@ -4216,6 +4364,75 @@ def get_remote_payload_src(
4216
4364
  return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
4217
4365
 
4218
4366
 
4367
+ ########################################
4368
+ # ../system/platforms.py
4369
+
4370
+
4371
+ ##
4372
+
4373
+
4374
+ @dc.dataclass(frozen=True)
4375
+ class Platform(abc.ABC): # noqa
4376
+ pass
4377
+
4378
+
4379
+ class LinuxPlatform(Platform, abc.ABC):
4380
+ pass
4381
+
4382
+
4383
+ class UbuntuPlatform(LinuxPlatform):
4384
+ pass
4385
+
4386
+
4387
+ class AmazonLinuxPlatform(LinuxPlatform):
4388
+ pass
4389
+
4390
+
4391
+ class GenericLinuxPlatform(LinuxPlatform):
4392
+ pass
4393
+
4394
+
4395
+ class DarwinPlatform(Platform):
4396
+ pass
4397
+
4398
+
4399
+ class UnknownPlatform(Platform):
4400
+ pass
4401
+
4402
+
4403
+ ##
4404
+
4405
+
4406
+ def _detect_system_platform() -> Platform:
4407
+ plat = sys.platform
4408
+
4409
+ if plat == 'linux':
4410
+ if (osr := LinuxOsRelease.read()) is None:
4411
+ return GenericLinuxPlatform()
4412
+
4413
+ if osr.id == 'amzn':
4414
+ return AmazonLinuxPlatform()
4415
+
4416
+ elif osr.id == 'ubuntu':
4417
+ return UbuntuPlatform()
4418
+
4419
+ else:
4420
+ return GenericLinuxPlatform()
4421
+
4422
+ elif plat == 'darwin':
4423
+ return DarwinPlatform()
4424
+
4425
+ else:
4426
+ return UnknownPlatform()
4427
+
4428
+
4429
+ @cached_nullary
4430
+ def detect_system_platform() -> Platform:
4431
+ platform = _detect_system_platform()
4432
+ log.info('Detected platform: %r', platform)
4433
+ return platform
4434
+
4435
+
4219
4436
  ########################################
4220
4437
  # ../targets/targets.py
4221
4438
  """
@@ -5540,1150 +5757,994 @@ inj = Injection
5540
5757
 
5541
5758
 
5542
5759
  ########################################
5543
- # ../../../omlish/lite/logs.py
5760
+ # ../../../omlish/lite/marshal.py
5544
5761
  """
5545
5762
  TODO:
5546
- - translate json keys
5547
- - debug
5763
+ - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
5764
+ - namedtuple
5765
+ - literals
5766
+ - newtypes?
5548
5767
  """
5549
5768
 
5550
5769
 
5551
- log = logging.getLogger(__name__)
5770
+ ##
5552
5771
 
5553
5772
 
5554
- ##
5773
+ @dc.dataclass(frozen=True)
5774
+ class ObjMarshalOptions:
5775
+ raw_bytes: bool = False
5776
+ nonstrict_dataclasses: bool = False
5555
5777
 
5556
5778
 
5557
- class TidLogFilter(logging.Filter):
5779
+ class ObjMarshaler(abc.ABC):
5780
+ @abc.abstractmethod
5781
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5782
+ raise NotImplementedError
5558
5783
 
5559
- def filter(self, record):
5560
- record.tid = threading.get_native_id()
5561
- return True
5784
+ @abc.abstractmethod
5785
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5786
+ raise NotImplementedError
5562
5787
 
5563
5788
 
5564
- ##
5789
+ class NopObjMarshaler(ObjMarshaler):
5790
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5791
+ return o
5565
5792
 
5793
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5794
+ return o
5566
5795
 
5567
- class JsonLogFormatter(logging.Formatter):
5568
5796
 
5569
- KEYS: ta.Mapping[str, bool] = {
5570
- 'name': False,
5571
- 'msg': False,
5572
- 'args': False,
5573
- 'levelname': False,
5574
- 'levelno': False,
5575
- 'pathname': False,
5576
- 'filename': False,
5577
- 'module': False,
5578
- 'exc_info': True,
5579
- 'exc_text': True,
5580
- 'stack_info': True,
5581
- 'lineno': False,
5582
- 'funcName': False,
5583
- 'created': False,
5584
- 'msecs': False,
5585
- 'relativeCreated': False,
5586
- 'thread': False,
5587
- 'threadName': False,
5588
- 'processName': False,
5589
- 'process': False,
5590
- }
5797
+ @dc.dataclass()
5798
+ class ProxyObjMarshaler(ObjMarshaler):
5799
+ m: ta.Optional[ObjMarshaler] = None
5591
5800
 
5592
- def format(self, record: logging.LogRecord) -> str:
5593
- dct = {
5594
- k: v
5595
- for k, o in self.KEYS.items()
5596
- for v in [getattr(record, k)]
5597
- if not (o and v is None)
5598
- }
5599
- return json_dumps_compact(dct)
5801
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5802
+ return check.not_none(self.m).marshal(o, ctx)
5600
5803
 
5804
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5805
+ return check.not_none(self.m).unmarshal(o, ctx)
5601
5806
 
5602
- ##
5603
5807
 
5808
+ @dc.dataclass(frozen=True)
5809
+ class CastObjMarshaler(ObjMarshaler):
5810
+ ty: type
5604
5811
 
5605
- STANDARD_LOG_FORMAT_PARTS = [
5606
- ('asctime', '%(asctime)-15s'),
5607
- ('process', 'pid=%(process)-6s'),
5608
- ('thread', 'tid=%(thread)x'),
5609
- ('levelname', '%(levelname)s'),
5610
- ('name', '%(name)s'),
5611
- ('separator', '::'),
5612
- ('message', '%(message)s'),
5613
- ]
5812
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5813
+ return o
5614
5814
 
5815
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5816
+ return self.ty(o)
5615
5817
 
5616
- class StandardLogFormatter(logging.Formatter):
5617
5818
 
5618
- @staticmethod
5619
- def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
5620
- return ' '.join(v for k, v in parts)
5819
+ class DynamicObjMarshaler(ObjMarshaler):
5820
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5821
+ return ctx.manager.marshal_obj(o, opts=ctx.options)
5621
5822
 
5622
- converter = datetime.datetime.fromtimestamp # type: ignore
5823
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5824
+ return o
5623
5825
 
5624
- def formatTime(self, record, datefmt=None):
5625
- ct = self.converter(record.created) # type: ignore
5626
- if datefmt:
5627
- return ct.strftime(datefmt) # noqa
5628
- else:
5629
- t = ct.strftime('%Y-%m-%d %H:%M:%S')
5630
- return '%s.%03d' % (t, record.msecs) # noqa
5631
5826
 
5827
+ @dc.dataclass(frozen=True)
5828
+ class Base64ObjMarshaler(ObjMarshaler):
5829
+ ty: type
5632
5830
 
5633
- ##
5831
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5832
+ return base64.b64encode(o).decode('ascii')
5634
5833
 
5834
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5835
+ return self.ty(base64.b64decode(o))
5635
5836
 
5636
- class ProxyLogFilterer(logging.Filterer):
5637
- def __init__(self, underlying: logging.Filterer) -> None: # noqa
5638
- self._underlying = underlying
5639
5837
 
5640
- @property
5641
- def underlying(self) -> logging.Filterer:
5642
- return self._underlying
5838
+ @dc.dataclass(frozen=True)
5839
+ class BytesSwitchedObjMarshaler(ObjMarshaler):
5840
+ m: ObjMarshaler
5643
5841
 
5644
- @property
5645
- def filters(self):
5646
- return self._underlying.filters
5842
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5843
+ if ctx.options.raw_bytes:
5844
+ return o
5845
+ return self.m.marshal(o, ctx)
5647
5846
 
5648
- @filters.setter
5649
- def filters(self, filters):
5650
- self._underlying.filters = filters
5847
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5848
+ if ctx.options.raw_bytes:
5849
+ return o
5850
+ return self.m.unmarshal(o, ctx)
5651
5851
 
5652
- def addFilter(self, filter): # noqa
5653
- self._underlying.addFilter(filter)
5654
5852
 
5655
- def removeFilter(self, filter): # noqa
5656
- self._underlying.removeFilter(filter)
5853
+ @dc.dataclass(frozen=True)
5854
+ class EnumObjMarshaler(ObjMarshaler):
5855
+ ty: type
5657
5856
 
5658
- def filter(self, record):
5659
- return self._underlying.filter(record)
5857
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5858
+ return o.name
5660
5859
 
5860
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5861
+ return self.ty.__members__[o] # type: ignore
5661
5862
 
5662
- class ProxyLogHandler(ProxyLogFilterer, logging.Handler):
5663
- def __init__(self, underlying: logging.Handler) -> None: # noqa
5664
- ProxyLogFilterer.__init__(self, underlying)
5665
5863
 
5666
- _underlying: logging.Handler
5864
+ @dc.dataclass(frozen=True)
5865
+ class OptionalObjMarshaler(ObjMarshaler):
5866
+ item: ObjMarshaler
5667
5867
 
5668
- @property
5669
- def underlying(self) -> logging.Handler:
5670
- return self._underlying
5868
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5869
+ if o is None:
5870
+ return None
5871
+ return self.item.marshal(o, ctx)
5671
5872
 
5672
- def get_name(self):
5673
- return self._underlying.get_name()
5873
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5874
+ if o is None:
5875
+ return None
5876
+ return self.item.unmarshal(o, ctx)
5674
5877
 
5675
- def set_name(self, name):
5676
- self._underlying.set_name(name)
5677
5878
 
5678
- @property
5679
- def name(self):
5680
- return self._underlying.name
5879
+ @dc.dataclass(frozen=True)
5880
+ class MappingObjMarshaler(ObjMarshaler):
5881
+ ty: type
5882
+ km: ObjMarshaler
5883
+ vm: ObjMarshaler
5681
5884
 
5682
- @property
5683
- def level(self):
5684
- return self._underlying.level
5885
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5886
+ return {self.km.marshal(k, ctx): self.vm.marshal(v, ctx) for k, v in o.items()}
5685
5887
 
5686
- @level.setter
5687
- def level(self, level):
5688
- self._underlying.level = level
5888
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5889
+ return self.ty((self.km.unmarshal(k, ctx), self.vm.unmarshal(v, ctx)) for k, v in o.items())
5689
5890
 
5690
- @property
5691
- def formatter(self):
5692
- return self._underlying.formatter
5693
5891
 
5694
- @formatter.setter
5695
- def formatter(self, formatter):
5696
- self._underlying.formatter = formatter
5892
+ @dc.dataclass(frozen=True)
5893
+ class IterableObjMarshaler(ObjMarshaler):
5894
+ ty: type
5895
+ item: ObjMarshaler
5697
5896
 
5698
- def createLock(self):
5699
- self._underlying.createLock()
5897
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5898
+ return [self.item.marshal(e, ctx) for e in o]
5700
5899
 
5701
- def acquire(self):
5702
- self._underlying.acquire()
5900
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5901
+ return self.ty(self.item.unmarshal(e, ctx) for e in o)
5703
5902
 
5704
- def release(self):
5705
- self._underlying.release()
5706
5903
 
5707
- def setLevel(self, level):
5708
- self._underlying.setLevel(level)
5904
+ @dc.dataclass(frozen=True)
5905
+ class DataclassObjMarshaler(ObjMarshaler):
5906
+ ty: type
5907
+ fs: ta.Mapping[str, ObjMarshaler]
5908
+ nonstrict: bool = False
5709
5909
 
5710
- def format(self, record):
5711
- return self._underlying.format(record)
5910
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5911
+ return {
5912
+ k: m.marshal(getattr(o, k), ctx)
5913
+ for k, m in self.fs.items()
5914
+ }
5712
5915
 
5713
- def emit(self, record):
5714
- self._underlying.emit(record)
5916
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5917
+ return self.ty(**{
5918
+ k: self.fs[k].unmarshal(v, ctx)
5919
+ for k, v in o.items()
5920
+ if not (self.nonstrict or ctx.options.nonstrict_dataclasses) or k in self.fs
5921
+ })
5715
5922
 
5716
- def handle(self, record):
5717
- return self._underlying.handle(record)
5718
5923
 
5719
- def setFormatter(self, fmt):
5720
- self._underlying.setFormatter(fmt)
5924
+ @dc.dataclass(frozen=True)
5925
+ class PolymorphicObjMarshaler(ObjMarshaler):
5926
+ class Impl(ta.NamedTuple):
5927
+ ty: type
5928
+ tag: str
5929
+ m: ObjMarshaler
5721
5930
 
5722
- def flush(self):
5723
- self._underlying.flush()
5931
+ impls_by_ty: ta.Mapping[type, Impl]
5932
+ impls_by_tag: ta.Mapping[str, Impl]
5724
5933
 
5725
- def close(self):
5726
- self._underlying.close()
5934
+ @classmethod
5935
+ def of(cls, impls: ta.Iterable[Impl]) -> 'PolymorphicObjMarshaler':
5936
+ return cls(
5937
+ {i.ty: i for i in impls},
5938
+ {i.tag: i for i in impls},
5939
+ )
5727
5940
 
5728
- def handleError(self, record):
5729
- self._underlying.handleError(record)
5941
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5942
+ impl = self.impls_by_ty[type(o)]
5943
+ return {impl.tag: impl.m.marshal(o, ctx)}
5730
5944
 
5945
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5946
+ [(t, v)] = o.items()
5947
+ impl = self.impls_by_tag[t]
5948
+ return impl.m.unmarshal(v, ctx)
5731
5949
 
5732
- ##
5733
5950
 
5951
+ @dc.dataclass(frozen=True)
5952
+ class DatetimeObjMarshaler(ObjMarshaler):
5953
+ ty: type
5734
5954
 
5735
- class StandardLogHandler(ProxyLogHandler):
5736
- pass
5955
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5956
+ return o.isoformat()
5737
5957
 
5958
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5959
+ return self.ty.fromisoformat(o) # type: ignore
5738
5960
 
5739
- ##
5740
5961
 
5962
+ class DecimalObjMarshaler(ObjMarshaler):
5963
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5964
+ return str(check.isinstance(o, decimal.Decimal))
5741
5965
 
5742
- @contextlib.contextmanager
5743
- def _locking_logging_module_lock() -> ta.Iterator[None]:
5744
- if hasattr(logging, '_acquireLock'):
5745
- logging._acquireLock() # noqa
5746
- try:
5747
- yield
5748
- finally:
5749
- logging._releaseLock() # type: ignore # noqa
5966
+ def unmarshal(self, v: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5967
+ return decimal.Decimal(check.isinstance(v, str))
5750
5968
 
5751
- elif hasattr(logging, '_lock'):
5752
- # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
5753
- with logging._lock: # noqa
5754
- yield
5755
5969
 
5756
- else:
5757
- raise Exception("Can't find lock in logging module")
5970
+ class FractionObjMarshaler(ObjMarshaler):
5971
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5972
+ fr = check.isinstance(o, fractions.Fraction)
5973
+ return [fr.numerator, fr.denominator]
5758
5974
 
5975
+ def unmarshal(self, v: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5976
+ num, denom = check.isinstance(v, list)
5977
+ return fractions.Fraction(num, denom)
5759
5978
 
5760
- def configure_standard_logging(
5761
- level: ta.Union[int, str] = logging.INFO,
5762
- *,
5763
- json: bool = False,
5764
- target: ta.Optional[logging.Logger] = None,
5765
- force: bool = False,
5766
- handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
5767
- ) -> ta.Optional[StandardLogHandler]:
5768
- with _locking_logging_module_lock():
5769
- if target is None:
5770
- target = logging.root
5771
5979
 
5772
- #
5980
+ class UuidObjMarshaler(ObjMarshaler):
5981
+ def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5982
+ return str(o)
5773
5983
 
5774
- if not force:
5775
- if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
5776
- return None
5984
+ def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5985
+ return uuid.UUID(o)
5777
5986
 
5778
- #
5779
5987
 
5780
- if handler_factory is not None:
5781
- handler = handler_factory()
5782
- else:
5783
- handler = logging.StreamHandler()
5988
+ ##
5784
5989
 
5785
- #
5786
5990
 
5787
- formatter: logging.Formatter
5788
- if json:
5789
- formatter = JsonLogFormatter()
5790
- else:
5791
- formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
5792
- handler.setFormatter(formatter)
5991
+ _DEFAULT_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
5992
+ **{t: NopObjMarshaler() for t in (type(None),)},
5993
+ **{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
5994
+ **{t: BytesSwitchedObjMarshaler(Base64ObjMarshaler(t)) for t in (bytes, bytearray)},
5995
+ **{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
5996
+ **{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
5793
5997
 
5794
- #
5998
+ ta.Any: DynamicObjMarshaler(),
5795
5999
 
5796
- handler.addFilter(TidLogFilter())
6000
+ **{t: DatetimeObjMarshaler(t) for t in (datetime.date, datetime.time, datetime.datetime)},
6001
+ decimal.Decimal: DecimalObjMarshaler(),
6002
+ fractions.Fraction: FractionObjMarshaler(),
6003
+ uuid.UUID: UuidObjMarshaler(),
6004
+ }
5797
6005
 
5798
- #
6006
+ _OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
6007
+ **{t: t for t in (dict,)},
6008
+ **{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
6009
+ }
5799
6010
 
5800
- target.addHandler(handler)
6011
+ _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
6012
+ **{t: t for t in (list, tuple, set, frozenset)},
6013
+ collections.abc.Set: frozenset,
6014
+ collections.abc.MutableSet: set,
6015
+ collections.abc.Sequence: tuple,
6016
+ collections.abc.MutableSequence: list,
6017
+ }
5801
6018
 
5802
- #
5803
6019
 
5804
- if level is not None:
5805
- target.setLevel(level)
6020
+ ##
5806
6021
 
5807
- #
5808
6022
 
5809
- return StandardLogHandler(handler)
6023
+ class ObjMarshalerManager:
6024
+ def __init__(
6025
+ self,
6026
+ *,
6027
+ default_options: ObjMarshalOptions = ObjMarshalOptions(),
5810
6028
 
6029
+ default_obj_marshalers: ta.Dict[ta.Any, ObjMarshaler] = _DEFAULT_OBJ_MARSHALERS, # noqa
6030
+ generic_mapping_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES, # noqa
6031
+ generic_iterable_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES, # noqa
6032
+ ) -> None:
6033
+ super().__init__()
5811
6034
 
5812
- ########################################
5813
- # ../../../omlish/lite/marshal.py
5814
- """
5815
- TODO:
5816
- - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
5817
- - namedtuple
5818
- - literals
5819
- - newtypes?
5820
- """
6035
+ self._default_options = default_options
5821
6036
 
6037
+ self._obj_marshalers = dict(default_obj_marshalers)
6038
+ self._generic_mapping_types = generic_mapping_types
6039
+ self._generic_iterable_types = generic_iterable_types
5822
6040
 
5823
- ##
6041
+ self._lock = threading.RLock()
6042
+ self._marshalers: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
6043
+ self._proxies: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
5824
6044
 
6045
+ #
5825
6046
 
5826
- @dc.dataclass(frozen=True)
5827
- class ObjMarshalOptions:
5828
- raw_bytes: bool = False
5829
- nonstrict_dataclasses: bool = False
6047
+ def make_obj_marshaler(
6048
+ self,
6049
+ ty: ta.Any,
6050
+ rec: ta.Callable[[ta.Any], ObjMarshaler],
6051
+ *,
6052
+ nonstrict_dataclasses: bool = False,
6053
+ ) -> ObjMarshaler:
6054
+ if isinstance(ty, type):
6055
+ if abc.ABC in ty.__bases__:
6056
+ impls = [ity for ity in deep_subclasses(ty) if abc.ABC not in ity.__bases__] # type: ignore
6057
+ if all(ity.__qualname__.endswith(ty.__name__) for ity in impls):
6058
+ ins = {ity: snake_case(ity.__qualname__[:-len(ty.__name__)]) for ity in impls}
6059
+ else:
6060
+ ins = {ity: ity.__qualname__ for ity in impls}
6061
+ return PolymorphicObjMarshaler.of([
6062
+ PolymorphicObjMarshaler.Impl(
6063
+ ity,
6064
+ itn,
6065
+ rec(ity),
6066
+ )
6067
+ for ity, itn in ins.items()
6068
+ ])
5830
6069
 
6070
+ if issubclass(ty, enum.Enum):
6071
+ return EnumObjMarshaler(ty)
5831
6072
 
5832
- class ObjMarshaler(abc.ABC):
5833
- @abc.abstractmethod
5834
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5835
- raise NotImplementedError
6073
+ if dc.is_dataclass(ty):
6074
+ return DataclassObjMarshaler(
6075
+ ty,
6076
+ {f.name: rec(f.type) for f in dc.fields(ty)},
6077
+ nonstrict=nonstrict_dataclasses,
6078
+ )
5836
6079
 
5837
- @abc.abstractmethod
5838
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5839
- raise NotImplementedError
6080
+ if is_generic_alias(ty):
6081
+ try:
6082
+ mt = self._generic_mapping_types[ta.get_origin(ty)]
6083
+ except KeyError:
6084
+ pass
6085
+ else:
6086
+ k, v = ta.get_args(ty)
6087
+ return MappingObjMarshaler(mt, rec(k), rec(v))
5840
6088
 
6089
+ try:
6090
+ st = self._generic_iterable_types[ta.get_origin(ty)]
6091
+ except KeyError:
6092
+ pass
6093
+ else:
6094
+ [e] = ta.get_args(ty)
6095
+ return IterableObjMarshaler(st, rec(e))
5841
6096
 
5842
- class NopObjMarshaler(ObjMarshaler):
5843
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5844
- return o
6097
+ if is_union_alias(ty):
6098
+ return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
5845
6099
 
5846
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5847
- return o
6100
+ raise TypeError(ty)
5848
6101
 
6102
+ #
5849
6103
 
5850
- @dc.dataclass()
5851
- class ProxyObjMarshaler(ObjMarshaler):
5852
- m: ta.Optional[ObjMarshaler] = None
6104
+ def register_opj_marshaler(self, ty: ta.Any, m: ObjMarshaler) -> None:
6105
+ with self._lock:
6106
+ if ty in self._obj_marshalers:
6107
+ raise KeyError(ty)
6108
+ self._obj_marshalers[ty] = m
5853
6109
 
5854
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5855
- return check.not_none(self.m).marshal(o, ctx)
6110
+ def get_obj_marshaler(
6111
+ self,
6112
+ ty: ta.Any,
6113
+ *,
6114
+ no_cache: bool = False,
6115
+ **kwargs: ta.Any,
6116
+ ) -> ObjMarshaler:
6117
+ with self._lock:
6118
+ if not no_cache:
6119
+ try:
6120
+ return self._obj_marshalers[ty]
6121
+ except KeyError:
6122
+ pass
5856
6123
 
5857
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5858
- return check.not_none(self.m).unmarshal(o, ctx)
6124
+ try:
6125
+ return self._proxies[ty]
6126
+ except KeyError:
6127
+ pass
5859
6128
 
6129
+ rec = functools.partial(
6130
+ self.get_obj_marshaler,
6131
+ no_cache=no_cache,
6132
+ **kwargs,
6133
+ )
5860
6134
 
5861
- @dc.dataclass(frozen=True)
5862
- class CastObjMarshaler(ObjMarshaler):
5863
- ty: type
6135
+ p = ProxyObjMarshaler()
6136
+ self._proxies[ty] = p
6137
+ try:
6138
+ m = self.make_obj_marshaler(ty, rec, **kwargs)
6139
+ finally:
6140
+ del self._proxies[ty]
6141
+ p.m = m
5864
6142
 
5865
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5866
- return o
6143
+ if not no_cache:
6144
+ self._obj_marshalers[ty] = m
6145
+ return m
5867
6146
 
5868
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5869
- return self.ty(o)
6147
+ #
5870
6148
 
6149
+ def _make_context(self, opts: ta.Optional[ObjMarshalOptions]) -> 'ObjMarshalContext':
6150
+ return ObjMarshalContext(
6151
+ options=opts or self._default_options,
6152
+ manager=self,
6153
+ )
5871
6154
 
5872
- class DynamicObjMarshaler(ObjMarshaler):
5873
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5874
- return ctx.manager.marshal_obj(o, opts=ctx.options)
6155
+ def marshal_obj(
6156
+ self,
6157
+ o: ta.Any,
6158
+ ty: ta.Any = None,
6159
+ opts: ta.Optional[ObjMarshalOptions] = None,
6160
+ ) -> ta.Any:
6161
+ m = self.get_obj_marshaler(ty if ty is not None else type(o))
6162
+ return m.marshal(o, self._make_context(opts))
5875
6163
 
5876
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5877
- return o
6164
+ def unmarshal_obj(
6165
+ self,
6166
+ o: ta.Any,
6167
+ ty: ta.Union[ta.Type[T], ta.Any],
6168
+ opts: ta.Optional[ObjMarshalOptions] = None,
6169
+ ) -> T:
6170
+ m = self.get_obj_marshaler(ty)
6171
+ return m.unmarshal(o, self._make_context(opts))
6172
+
6173
+ def roundtrip_obj(
6174
+ self,
6175
+ o: ta.Any,
6176
+ ty: ta.Any = None,
6177
+ opts: ta.Optional[ObjMarshalOptions] = None,
6178
+ ) -> ta.Any:
6179
+ if ty is None:
6180
+ ty = type(o)
6181
+ m: ta.Any = self.marshal_obj(o, ty, opts)
6182
+ u: ta.Any = self.unmarshal_obj(m, ty, opts)
6183
+ return u
5878
6184
 
5879
6185
 
5880
6186
  @dc.dataclass(frozen=True)
5881
- class Base64ObjMarshaler(ObjMarshaler):
5882
- ty: type
6187
+ class ObjMarshalContext:
6188
+ options: ObjMarshalOptions
6189
+ manager: ObjMarshalerManager
5883
6190
 
5884
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5885
- return base64.b64encode(o).decode('ascii')
5886
6191
 
5887
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5888
- return self.ty(base64.b64decode(o))
6192
+ ##
5889
6193
 
5890
6194
 
5891
- @dc.dataclass(frozen=True)
5892
- class BytesSwitchedObjMarshaler(ObjMarshaler):
5893
- m: ObjMarshaler
6195
+ OBJ_MARSHALER_MANAGER = ObjMarshalerManager()
5894
6196
 
5895
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5896
- if ctx.options.raw_bytes:
5897
- return o
5898
- return self.m.marshal(o, ctx)
6197
+ register_opj_marshaler = OBJ_MARSHALER_MANAGER.register_opj_marshaler
6198
+ get_obj_marshaler = OBJ_MARSHALER_MANAGER.get_obj_marshaler
5899
6199
 
5900
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5901
- if ctx.options.raw_bytes:
5902
- return o
5903
- return self.m.unmarshal(o, ctx)
6200
+ marshal_obj = OBJ_MARSHALER_MANAGER.marshal_obj
6201
+ unmarshal_obj = OBJ_MARSHALER_MANAGER.unmarshal_obj
5904
6202
 
5905
6203
 
5906
- @dc.dataclass(frozen=True)
5907
- class EnumObjMarshaler(ObjMarshaler):
5908
- ty: type
6204
+ ########################################
6205
+ # ../../../omlish/lite/runtime.py
5909
6206
 
5910
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5911
- return o.name
5912
6207
 
5913
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5914
- return self.ty.__members__[o] # type: ignore
6208
+ @cached_nullary
6209
+ def is_debugger_attached() -> bool:
6210
+ return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
5915
6211
 
5916
6212
 
5917
- @dc.dataclass(frozen=True)
5918
- class OptionalObjMarshaler(ObjMarshaler):
5919
- item: ObjMarshaler
6213
+ REQUIRED_PYTHON_VERSION = (3, 8)
5920
6214
 
5921
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5922
- if o is None:
5923
- return None
5924
- return self.item.marshal(o, ctx)
5925
6215
 
5926
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5927
- if o is None:
5928
- return None
5929
- return self.item.unmarshal(o, ctx)
6216
+ def check_runtime_version() -> None:
6217
+ if sys.version_info < REQUIRED_PYTHON_VERSION:
6218
+ raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
5930
6219
 
5931
6220
 
5932
- @dc.dataclass(frozen=True)
5933
- class MappingObjMarshaler(ObjMarshaler):
5934
- ty: type
5935
- km: ObjMarshaler
5936
- vm: ObjMarshaler
6221
+ ########################################
6222
+ # ../../../omlish/logs/json.py
6223
+ """
6224
+ TODO:
6225
+ - translate json keys
6226
+ """
5937
6227
 
5938
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5939
- return {self.km.marshal(k, ctx): self.vm.marshal(v, ctx) for k, v in o.items()}
5940
6228
 
5941
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5942
- return self.ty((self.km.unmarshal(k, ctx), self.vm.unmarshal(v, ctx)) for k, v in o.items())
6229
+ class JsonLogFormatter(logging.Formatter):
6230
+ KEYS: ta.Mapping[str, bool] = {
6231
+ 'name': False,
6232
+ 'msg': False,
6233
+ 'args': False,
6234
+ 'levelname': False,
6235
+ 'levelno': False,
6236
+ 'pathname': False,
6237
+ 'filename': False,
6238
+ 'module': False,
6239
+ 'exc_info': True,
6240
+ 'exc_text': True,
6241
+ 'stack_info': True,
6242
+ 'lineno': False,
6243
+ 'funcName': False,
6244
+ 'created': False,
6245
+ 'msecs': False,
6246
+ 'relativeCreated': False,
6247
+ 'thread': False,
6248
+ 'threadName': False,
6249
+ 'processName': False,
6250
+ 'process': False,
6251
+ }
5943
6252
 
6253
+ def __init__(
6254
+ self,
6255
+ *args: ta.Any,
6256
+ json_dumps: ta.Optional[ta.Callable[[ta.Any], str]] = None,
6257
+ **kwargs: ta.Any,
6258
+ ) -> None:
6259
+ super().__init__(*args, **kwargs)
5944
6260
 
5945
- @dc.dataclass(frozen=True)
5946
- class IterableObjMarshaler(ObjMarshaler):
5947
- ty: type
5948
- item: ObjMarshaler
6261
+ if json_dumps is None:
6262
+ json_dumps = json_dumps_compact
6263
+ self._json_dumps = json_dumps
5949
6264
 
5950
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5951
- return [self.item.marshal(e, ctx) for e in o]
6265
+ def format(self, record: logging.LogRecord) -> str:
6266
+ dct = {
6267
+ k: v
6268
+ for k, o in self.KEYS.items()
6269
+ for v in [getattr(record, k)]
6270
+ if not (o and v is None)
6271
+ }
6272
+ return self._json_dumps(dct)
5952
6273
 
5953
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5954
- return self.ty(self.item.unmarshal(e, ctx) for e in o)
6274
+
6275
+ ########################################
6276
+ # ../../../omdev/interp/types.py
6277
+
6278
+
6279
+ # See https://peps.python.org/pep-3149/
6280
+ INTERP_OPT_GLYPHS_BY_ATTR: ta.Mapping[str, str] = collections.OrderedDict([
6281
+ ('debug', 'd'),
6282
+ ('threaded', 't'),
6283
+ ])
6284
+
6285
+ INTERP_OPT_ATTRS_BY_GLYPH: ta.Mapping[str, str] = collections.OrderedDict(
6286
+ (g, a) for a, g in INTERP_OPT_GLYPHS_BY_ATTR.items()
6287
+ )
5955
6288
 
5956
6289
 
5957
6290
  @dc.dataclass(frozen=True)
5958
- class DataclassObjMarshaler(ObjMarshaler):
5959
- ty: type
5960
- fs: ta.Mapping[str, ObjMarshaler]
5961
- nonstrict: bool = False
6291
+ class InterpOpts:
6292
+ threaded: bool = False
6293
+ debug: bool = False
5962
6294
 
5963
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5964
- return {
5965
- k: m.marshal(getattr(o, k), ctx)
5966
- for k, m in self.fs.items()
5967
- }
6295
+ def __str__(self) -> str:
6296
+ return ''.join(g for a, g in INTERP_OPT_GLYPHS_BY_ATTR.items() if getattr(self, a))
5968
6297
 
5969
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5970
- return self.ty(**{
5971
- k: self.fs[k].unmarshal(v, ctx)
5972
- for k, v in o.items()
5973
- if not (self.nonstrict or ctx.options.nonstrict_dataclasses) or k in self.fs
5974
- })
6298
+ @classmethod
6299
+ def parse(cls, s: str) -> 'InterpOpts':
6300
+ return cls(**{INTERP_OPT_ATTRS_BY_GLYPH[g]: True for g in s})
6301
+
6302
+ @classmethod
6303
+ def parse_suffix(cls, s: str) -> ta.Tuple[str, 'InterpOpts']:
6304
+ kw = {}
6305
+ while s and (a := INTERP_OPT_ATTRS_BY_GLYPH.get(s[-1])):
6306
+ s, kw[a] = s[:-1], True
6307
+ return s, cls(**kw)
5975
6308
 
5976
6309
 
5977
6310
  @dc.dataclass(frozen=True)
5978
- class PolymorphicObjMarshaler(ObjMarshaler):
5979
- class Impl(ta.NamedTuple):
5980
- ty: type
5981
- tag: str
5982
- m: ObjMarshaler
6311
+ class InterpVersion:
6312
+ version: Version
6313
+ opts: InterpOpts
5983
6314
 
5984
- impls_by_ty: ta.Mapping[type, Impl]
5985
- impls_by_tag: ta.Mapping[str, Impl]
6315
+ def __str__(self) -> str:
6316
+ return str(self.version) + str(self.opts)
5986
6317
 
5987
6318
  @classmethod
5988
- def of(cls, impls: ta.Iterable[Impl]) -> 'PolymorphicObjMarshaler':
6319
+ def parse(cls, s: str) -> 'InterpVersion':
6320
+ s, o = InterpOpts.parse_suffix(s)
6321
+ v = Version(s)
5989
6322
  return cls(
5990
- {i.ty: i for i in impls},
5991
- {i.tag: i for i in impls},
6323
+ version=v,
6324
+ opts=o,
5992
6325
  )
5993
6326
 
5994
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5995
- impl = self.impls_by_ty[type(o)]
5996
- return {impl.tag: impl.m.marshal(o, ctx)}
5997
-
5998
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5999
- [(t, v)] = o.items()
6000
- impl = self.impls_by_tag[t]
6001
- return impl.m.unmarshal(v, ctx)
6327
+ @classmethod
6328
+ def try_parse(cls, s: str) -> ta.Optional['InterpVersion']:
6329
+ try:
6330
+ return cls.parse(s)
6331
+ except (KeyError, InvalidVersion):
6332
+ return None
6002
6333
 
6003
6334
 
6004
6335
  @dc.dataclass(frozen=True)
6005
- class DatetimeObjMarshaler(ObjMarshaler):
6006
- ty: type
6336
+ class InterpSpecifier:
6337
+ specifier: Specifier
6338
+ opts: InterpOpts
6007
6339
 
6008
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
6009
- return o.isoformat()
6340
+ def __str__(self) -> str:
6341
+ return str(self.specifier) + str(self.opts)
6010
6342
 
6011
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
6012
- return self.ty.fromisoformat(o) # type: ignore
6343
+ @classmethod
6344
+ def parse(cls, s: str) -> 'InterpSpecifier':
6345
+ s, o = InterpOpts.parse_suffix(s)
6346
+ if not any(s.startswith(o) for o in Specifier.OPERATORS):
6347
+ s = '~=' + s
6348
+ if s.count('.') < 2:
6349
+ s += '.0'
6350
+ return cls(
6351
+ specifier=Specifier(s),
6352
+ opts=o,
6353
+ )
6013
6354
 
6355
+ def contains(self, iv: InterpVersion) -> bool:
6356
+ return self.specifier.contains(iv.version) and self.opts == iv.opts
6014
6357
 
6015
- class DecimalObjMarshaler(ObjMarshaler):
6016
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
6017
- return str(check.isinstance(o, decimal.Decimal))
6358
+ def __contains__(self, iv: InterpVersion) -> bool:
6359
+ return self.contains(iv)
6018
6360
 
6019
- def unmarshal(self, v: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
6020
- return decimal.Decimal(check.isinstance(v, str))
6021
6361
 
6362
+ @dc.dataclass(frozen=True)
6363
+ class Interp:
6364
+ exe: str
6365
+ version: InterpVersion
6022
6366
 
6023
- class FractionObjMarshaler(ObjMarshaler):
6024
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
6025
- fr = check.isinstance(o, fractions.Fraction)
6026
- return [fr.numerator, fr.denominator]
6027
6367
 
6028
- def unmarshal(self, v: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
6029
- num, denom = check.isinstance(v, list)
6030
- return fractions.Fraction(num, denom)
6368
+ ########################################
6369
+ # ../../configs.py
6031
6370
 
6032
6371
 
6033
- class UuidObjMarshaler(ObjMarshaler):
6034
- def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
6035
- return str(o)
6372
+ def parse_config_file(
6373
+ name: str,
6374
+ f: ta.TextIO,
6375
+ ) -> ConfigMapping:
6376
+ if name.endswith('.toml'):
6377
+ return toml_loads(f.read())
6036
6378
 
6037
- def unmarshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
6038
- return uuid.UUID(o)
6379
+ elif any(name.endswith(e) for e in ('.yml', '.yaml')):
6380
+ yaml = __import__('yaml')
6381
+ return yaml.safe_load(f)
6039
6382
 
6383
+ elif name.endswith('.ini'):
6384
+ import configparser
6385
+ cp = configparser.ConfigParser()
6386
+ cp.read_file(f)
6387
+ config_dct: ta.Dict[str, ta.Any] = {}
6388
+ for sec in cp.sections():
6389
+ cd = config_dct
6390
+ for k in sec.split('.'):
6391
+ cd = cd.setdefault(k, {})
6392
+ cd.update(cp.items(sec))
6393
+ return config_dct
6040
6394
 
6041
- ##
6395
+ else:
6396
+ return json.loads(f.read())
6042
6397
 
6043
6398
 
6044
- _DEFAULT_OBJ_MARSHALERS: ta.Dict[ta.Any, ObjMarshaler] = {
6045
- **{t: NopObjMarshaler() for t in (type(None),)},
6046
- **{t: CastObjMarshaler(t) for t in (int, float, str, bool)},
6047
- **{t: BytesSwitchedObjMarshaler(Base64ObjMarshaler(t)) for t in (bytes, bytearray)},
6048
- **{t: IterableObjMarshaler(t, DynamicObjMarshaler()) for t in (list, tuple, set, frozenset)},
6049
- **{t: MappingObjMarshaler(t, DynamicObjMarshaler(), DynamicObjMarshaler()) for t in (dict,)},
6050
-
6051
- ta.Any: DynamicObjMarshaler(),
6399
+ def read_config_file(
6400
+ path: str,
6401
+ cls: ta.Type[T],
6402
+ *,
6403
+ prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
6404
+ ) -> T:
6405
+ with open(path) as cf:
6406
+ config_dct = parse_config_file(os.path.basename(path), cf)
6052
6407
 
6053
- **{t: DatetimeObjMarshaler(t) for t in (datetime.date, datetime.time, datetime.datetime)},
6054
- decimal.Decimal: DecimalObjMarshaler(),
6055
- fractions.Fraction: FractionObjMarshaler(),
6056
- uuid.UUID: UuidObjMarshaler(),
6057
- }
6408
+ if prepare is not None:
6409
+ config_dct = prepare(config_dct)
6058
6410
 
6059
- _OBJ_MARSHALER_GENERIC_MAPPING_TYPES: ta.Dict[ta.Any, type] = {
6060
- **{t: t for t in (dict,)},
6061
- **{t: dict for t in (collections.abc.Mapping, collections.abc.MutableMapping)},
6062
- }
6411
+ return unmarshal_obj(config_dct, cls)
6063
6412
 
6064
- _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES: ta.Dict[ta.Any, type] = {
6065
- **{t: t for t in (list, tuple, set, frozenset)},
6066
- collections.abc.Set: frozenset,
6067
- collections.abc.MutableSet: set,
6068
- collections.abc.Sequence: tuple,
6069
- collections.abc.MutableSequence: list,
6070
- }
6071
6413
 
6414
+ def build_config_named_children(
6415
+ o: ta.Union[
6416
+ ta.Sequence[ConfigMapping],
6417
+ ta.Mapping[str, ConfigMapping],
6418
+ None,
6419
+ ],
6420
+ *,
6421
+ name_key: str = 'name',
6422
+ ) -> ta.Optional[ta.Sequence[ConfigMapping]]:
6423
+ if o is None:
6424
+ return None
6072
6425
 
6073
- ##
6426
+ lst: ta.List[ConfigMapping] = []
6427
+ if isinstance(o, ta.Mapping):
6428
+ for k, v in o.items():
6429
+ check.isinstance(v, ta.Mapping)
6430
+ if name_key in v:
6431
+ n = v[name_key]
6432
+ if k != n:
6433
+ raise KeyError(f'Given names do not match: {n} != {k}')
6434
+ lst.append(v)
6435
+ else:
6436
+ lst.append({name_key: k, **v})
6074
6437
 
6438
+ else:
6439
+ check.not_isinstance(o, str)
6440
+ lst.extend(o)
6075
6441
 
6076
- class ObjMarshalerManager:
6077
- def __init__(
6078
- self,
6079
- *,
6080
- default_options: ObjMarshalOptions = ObjMarshalOptions(),
6442
+ seen = set()
6443
+ for d in lst:
6444
+ n = d['name']
6445
+ if n in d:
6446
+ raise KeyError(f'Duplicate name: {n}')
6447
+ seen.add(n)
6081
6448
 
6082
- default_obj_marshalers: ta.Dict[ta.Any, ObjMarshaler] = _DEFAULT_OBJ_MARSHALERS, # noqa
6083
- generic_mapping_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_MAPPING_TYPES, # noqa
6084
- generic_iterable_types: ta.Dict[ta.Any, type] = _OBJ_MARSHALER_GENERIC_ITERABLE_TYPES, # noqa
6085
- ) -> None:
6086
- super().__init__()
6449
+ return lst
6087
6450
 
6088
- self._default_options = default_options
6089
6451
 
6090
- self._obj_marshalers = dict(default_obj_marshalers)
6091
- self._generic_mapping_types = generic_mapping_types
6092
- self._generic_iterable_types = generic_iterable_types
6452
+ ########################################
6453
+ # ../commands/marshal.py
6093
6454
 
6094
- self._lock = threading.RLock()
6095
- self._marshalers: ta.Dict[ta.Any, ObjMarshaler] = dict(_DEFAULT_OBJ_MARSHALERS)
6096
- self._proxies: ta.Dict[ta.Any, ProxyObjMarshaler] = {}
6097
6455
 
6098
- #
6456
+ def install_command_marshaling(
6457
+ cmds: CommandNameMap,
6458
+ msh: ObjMarshalerManager,
6459
+ ) -> None:
6460
+ for fn in [
6461
+ lambda c: c,
6462
+ lambda c: c.Output,
6463
+ ]:
6464
+ msh.register_opj_marshaler(
6465
+ fn(Command),
6466
+ PolymorphicObjMarshaler.of([
6467
+ PolymorphicObjMarshaler.Impl(
6468
+ fn(cmd),
6469
+ name,
6470
+ msh.get_obj_marshaler(fn(cmd)),
6471
+ )
6472
+ for name, cmd in cmds.items()
6473
+ ]),
6474
+ )
6099
6475
 
6100
- def make_obj_marshaler(
6101
- self,
6102
- ty: ta.Any,
6103
- rec: ta.Callable[[ta.Any], ObjMarshaler],
6104
- *,
6105
- nonstrict_dataclasses: bool = False,
6106
- ) -> ObjMarshaler:
6107
- if isinstance(ty, type):
6108
- if abc.ABC in ty.__bases__:
6109
- impls = [ity for ity in deep_subclasses(ty) if abc.ABC not in ity.__bases__] # type: ignore
6110
- if all(ity.__qualname__.endswith(ty.__name__) for ity in impls):
6111
- ins = {ity: snake_case(ity.__qualname__[:-len(ty.__name__)]) for ity in impls}
6112
- else:
6113
- ins = {ity: ity.__qualname__ for ity in impls}
6114
- return PolymorphicObjMarshaler.of([
6115
- PolymorphicObjMarshaler.Impl(
6116
- ity,
6117
- itn,
6118
- rec(ity),
6119
- )
6120
- for ity, itn in ins.items()
6121
- ])
6122
6476
 
6123
- if issubclass(ty, enum.Enum):
6124
- return EnumObjMarshaler(ty)
6477
+ ########################################
6478
+ # ../commands/ping.py
6125
6479
 
6126
- if dc.is_dataclass(ty):
6127
- return DataclassObjMarshaler(
6128
- ty,
6129
- {f.name: rec(f.type) for f in dc.fields(ty)},
6130
- nonstrict=nonstrict_dataclasses,
6131
- )
6132
6480
 
6133
- if is_generic_alias(ty):
6134
- try:
6135
- mt = self._generic_mapping_types[ta.get_origin(ty)]
6136
- except KeyError:
6137
- pass
6138
- else:
6139
- k, v = ta.get_args(ty)
6140
- return MappingObjMarshaler(mt, rec(k), rec(v))
6481
+ ##
6141
6482
 
6142
- try:
6143
- st = self._generic_iterable_types[ta.get_origin(ty)]
6144
- except KeyError:
6145
- pass
6146
- else:
6147
- [e] = ta.get_args(ty)
6148
- return IterableObjMarshaler(st, rec(e))
6149
6483
 
6150
- if is_union_alias(ty):
6151
- return OptionalObjMarshaler(rec(get_optional_alias_arg(ty)))
6484
+ @dc.dataclass(frozen=True)
6485
+ class PingCommand(Command['PingCommand.Output']):
6486
+ time: float = dc.field(default_factory=time.time)
6152
6487
 
6153
- raise TypeError(ty)
6488
+ @dc.dataclass(frozen=True)
6489
+ class Output(Command.Output):
6490
+ time: float
6154
6491
 
6155
- #
6156
6492
 
6157
- def register_opj_marshaler(self, ty: ta.Any, m: ObjMarshaler) -> None:
6158
- with self._lock:
6159
- if ty in self._obj_marshalers:
6160
- raise KeyError(ty)
6161
- self._obj_marshalers[ty] = m
6493
+ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
6494
+ async def execute(self, cmd: PingCommand) -> PingCommand.Output:
6495
+ return PingCommand.Output(cmd.time)
6162
6496
 
6163
- def get_obj_marshaler(
6164
- self,
6165
- ty: ta.Any,
6166
- *,
6167
- no_cache: bool = False,
6168
- **kwargs: ta.Any,
6169
- ) -> ObjMarshaler:
6170
- with self._lock:
6171
- if not no_cache:
6172
- try:
6173
- return self._obj_marshalers[ty]
6174
- except KeyError:
6175
- pass
6176
6497
 
6177
- try:
6178
- return self._proxies[ty]
6179
- except KeyError:
6180
- pass
6498
+ ########################################
6499
+ # ../commands/types.py
6181
6500
 
6182
- rec = functools.partial(
6183
- self.get_obj_marshaler,
6184
- no_cache=no_cache,
6185
- **kwargs,
6186
- )
6187
6501
 
6188
- p = ProxyObjMarshaler()
6189
- self._proxies[ty] = p
6190
- try:
6191
- m = self.make_obj_marshaler(ty, rec, **kwargs)
6192
- finally:
6193
- del self._proxies[ty]
6194
- p.m = m
6502
+ CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
6195
6503
 
6196
- if not no_cache:
6197
- self._obj_marshalers[ty] = m
6198
- return m
6199
6504
 
6200
- #
6505
+ ########################################
6506
+ # ../deploy/commands.py
6201
6507
 
6202
- def _make_context(self, opts: ta.Optional[ObjMarshalOptions]) -> 'ObjMarshalContext':
6203
- return ObjMarshalContext(
6204
- options=opts or self._default_options,
6205
- manager=self,
6206
- )
6207
6508
 
6208
- def marshal_obj(
6209
- self,
6210
- o: ta.Any,
6211
- ty: ta.Any = None,
6212
- opts: ta.Optional[ObjMarshalOptions] = None,
6213
- ) -> ta.Any:
6214
- m = self.get_obj_marshaler(ty if ty is not None else type(o))
6215
- return m.marshal(o, self._make_context(opts))
6509
+ ##
6216
6510
 
6217
- def unmarshal_obj(
6218
- self,
6219
- o: ta.Any,
6220
- ty: ta.Union[ta.Type[T], ta.Any],
6221
- opts: ta.Optional[ObjMarshalOptions] = None,
6222
- ) -> T:
6223
- m = self.get_obj_marshaler(ty)
6224
- return m.unmarshal(o, self._make_context(opts))
6225
6511
 
6226
- def roundtrip_obj(
6227
- self,
6228
- o: ta.Any,
6229
- ty: ta.Any = None,
6230
- opts: ta.Optional[ObjMarshalOptions] = None,
6231
- ) -> ta.Any:
6232
- if ty is None:
6233
- ty = type(o)
6234
- m: ta.Any = self.marshal_obj(o, ty, opts)
6235
- u: ta.Any = self.unmarshal_obj(m, ty, opts)
6236
- return u
6512
+ @dc.dataclass(frozen=True)
6513
+ class DeployCommand(Command['DeployCommand.Output']):
6514
+ @dc.dataclass(frozen=True)
6515
+ class Output(Command.Output):
6516
+ pass
6237
6517
 
6238
6518
 
6239
- @dc.dataclass(frozen=True)
6240
- class ObjMarshalContext:
6241
- options: ObjMarshalOptions
6242
- manager: ObjMarshalerManager
6519
+ class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
6520
+ async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
6521
+ log.info('Deploying!')
6243
6522
 
6523
+ return DeployCommand.Output()
6244
6524
 
6245
- ##
6246
6525
 
6526
+ ########################################
6527
+ # ../marshal.py
6247
6528
 
6248
- OBJ_MARSHALER_MANAGER = ObjMarshalerManager()
6249
6529
 
6250
- register_opj_marshaler = OBJ_MARSHALER_MANAGER.register_opj_marshaler
6251
- get_obj_marshaler = OBJ_MARSHALER_MANAGER.get_obj_marshaler
6252
-
6253
- marshal_obj = OBJ_MARSHALER_MANAGER.marshal_obj
6254
- unmarshal_obj = OBJ_MARSHALER_MANAGER.unmarshal_obj
6530
+ @dc.dataclass(frozen=True)
6531
+ class ObjMarshalerInstaller:
6532
+ fn: ta.Callable[[ObjMarshalerManager], None]
6255
6533
 
6256
6534
 
6257
- ########################################
6258
- # ../../../omlish/lite/runtime.py
6535
+ ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMarshalerInstaller])
6259
6536
 
6260
6537
 
6261
- @cached_nullary
6262
- def is_debugger_attached() -> bool:
6263
- return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
6538
+ ########################################
6539
+ # ../remote/channel.py
6264
6540
 
6265
6541
 
6266
- REQUIRED_PYTHON_VERSION = (3, 8)
6542
+ ##
6267
6543
 
6268
6544
 
6269
- def check_runtime_version() -> None:
6270
- if sys.version_info < REQUIRED_PYTHON_VERSION:
6271
- raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
6545
+ class RemoteChannel(abc.ABC):
6546
+ @abc.abstractmethod
6547
+ def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
6548
+ raise NotImplementedError
6272
6549
 
6550
+ @abc.abstractmethod
6551
+ def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
6552
+ raise NotImplementedError
6273
6553
 
6274
- ########################################
6275
- # ../../../omdev/interp/types.py
6554
+ def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
6555
+ pass
6276
6556
 
6277
6557
 
6278
- # See https://peps.python.org/pep-3149/
6279
- INTERP_OPT_GLYPHS_BY_ATTR: ta.Mapping[str, str] = collections.OrderedDict([
6280
- ('debug', 'd'),
6281
- ('threaded', 't'),
6282
- ])
6558
+ ##
6283
6559
 
6284
- INTERP_OPT_ATTRS_BY_GLYPH: ta.Mapping[str, str] = collections.OrderedDict(
6285
- (g, a) for a, g in INTERP_OPT_GLYPHS_BY_ATTR.items()
6286
- )
6287
6560
 
6561
+ class RemoteChannelImpl(RemoteChannel):
6562
+ def __init__(
6563
+ self,
6564
+ input: asyncio.StreamReader, # noqa
6565
+ output: asyncio.StreamWriter,
6566
+ *,
6567
+ msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
6568
+ ) -> None:
6569
+ super().__init__()
6288
6570
 
6289
- @dc.dataclass(frozen=True)
6290
- class InterpOpts:
6291
- threaded: bool = False
6292
- debug: bool = False
6571
+ self._input = input
6572
+ self._output = output
6573
+ self._msh = msh
6293
6574
 
6294
- def __str__(self) -> str:
6295
- return ''.join(g for a, g in INTERP_OPT_GLYPHS_BY_ATTR.items() if getattr(self, a))
6575
+ self._input_lock = asyncio.Lock()
6576
+ self._output_lock = asyncio.Lock()
6296
6577
 
6297
- @classmethod
6298
- def parse(cls, s: str) -> 'InterpOpts':
6299
- return cls(**{INTERP_OPT_ATTRS_BY_GLYPH[g]: True for g in s})
6578
+ def set_marshaler(self, msh: ObjMarshalerManager) -> None:
6579
+ self._msh = msh
6300
6580
 
6301
- @classmethod
6302
- def parse_suffix(cls, s: str) -> ta.Tuple[str, 'InterpOpts']:
6303
- kw = {}
6304
- while s and (a := INTERP_OPT_ATTRS_BY_GLYPH.get(s[-1])):
6305
- s, kw[a] = s[:-1], True
6306
- return s, cls(**kw)
6581
+ #
6307
6582
 
6583
+ async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
6584
+ j = json_dumps_compact(self._msh.marshal_obj(o, ty))
6585
+ d = j.encode('utf-8')
6308
6586
 
6309
- @dc.dataclass(frozen=True)
6310
- class InterpVersion:
6311
- version: Version
6312
- opts: InterpOpts
6587
+ self._output.write(struct.pack('<I', len(d)))
6588
+ self._output.write(d)
6589
+ await self._output.drain()
6313
6590
 
6314
- def __str__(self) -> str:
6315
- return str(self.version) + str(self.opts)
6591
+ async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
6592
+ async with self._output_lock:
6593
+ return await self._send_obj(o, ty)
6316
6594
 
6317
- @classmethod
6318
- def parse(cls, s: str) -> 'InterpVersion':
6319
- s, o = InterpOpts.parse_suffix(s)
6320
- v = Version(s)
6321
- return cls(
6322
- version=v,
6323
- opts=o,
6324
- )
6595
+ #
6325
6596
 
6326
- @classmethod
6327
- def try_parse(cls, s: str) -> ta.Optional['InterpVersion']:
6328
- try:
6329
- return cls.parse(s)
6330
- except (KeyError, InvalidVersion):
6597
+ async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
6598
+ d = await self._input.read(4)
6599
+ if not d:
6331
6600
  return None
6601
+ if len(d) != 4:
6602
+ raise EOFError
6332
6603
 
6604
+ sz = struct.unpack('<I', d)[0]
6605
+ d = await self._input.read(sz)
6606
+ if len(d) != sz:
6607
+ raise EOFError
6333
6608
 
6334
- @dc.dataclass(frozen=True)
6335
- class InterpSpecifier:
6336
- specifier: Specifier
6337
- opts: InterpOpts
6338
-
6339
- def __str__(self) -> str:
6340
- return str(self.specifier) + str(self.opts)
6609
+ j = json.loads(d.decode('utf-8'))
6610
+ return self._msh.unmarshal_obj(j, ty)
6341
6611
 
6342
- @classmethod
6343
- def parse(cls, s: str) -> 'InterpSpecifier':
6344
- s, o = InterpOpts.parse_suffix(s)
6345
- if not any(s.startswith(o) for o in Specifier.OPERATORS):
6346
- s = '~=' + s
6347
- if s.count('.') < 2:
6348
- s += '.0'
6349
- return cls(
6350
- specifier=Specifier(s),
6351
- opts=o,
6352
- )
6612
+ async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
6613
+ async with self._input_lock:
6614
+ return await self._recv_obj(ty)
6353
6615
 
6354
- def contains(self, iv: InterpVersion) -> bool:
6355
- return self.specifier.contains(iv.version) and self.opts == iv.opts
6356
6616
 
6357
- def __contains__(self, iv: InterpVersion) -> bool:
6358
- return self.contains(iv)
6617
+ ########################################
6618
+ # ../system/config.py
6359
6619
 
6360
6620
 
6361
6621
  @dc.dataclass(frozen=True)
6362
- class Interp:
6363
- exe: str
6364
- version: InterpVersion
6622
+ class SystemConfig:
6623
+ platform: ta.Optional[Platform] = None
6365
6624
 
6366
6625
 
6367
6626
  ########################################
6368
- # ../../configs.py
6627
+ # ../../../omlish/logs/standard.py
6628
+ """
6629
+ TODO:
6630
+ - structured
6631
+ - prefixed
6632
+ - debug
6633
+ """
6369
6634
 
6370
6635
 
6371
- def parse_config_file(
6372
- name: str,
6373
- f: ta.TextIO,
6374
- ) -> ConfigMapping:
6375
- if name.endswith('.toml'):
6376
- return toml_loads(f.read())
6636
+ ##
6377
6637
 
6378
- elif any(name.endswith(e) for e in ('.yml', '.yaml')):
6379
- yaml = __import__('yaml')
6380
- return yaml.safe_load(f)
6381
6638
 
6382
- elif name.endswith('.ini'):
6383
- import configparser
6384
- cp = configparser.ConfigParser()
6385
- cp.read_file(f)
6386
- config_dct: ta.Dict[str, ta.Any] = {}
6387
- for sec in cp.sections():
6388
- cd = config_dct
6389
- for k in sec.split('.'):
6390
- cd = cd.setdefault(k, {})
6391
- cd.update(cp.items(sec))
6392
- return config_dct
6639
+ STANDARD_LOG_FORMAT_PARTS = [
6640
+ ('asctime', '%(asctime)-15s'),
6641
+ ('process', 'pid=%(process)-6s'),
6642
+ ('thread', 'tid=%(thread)x'),
6643
+ ('levelname', '%(levelname)s'),
6644
+ ('name', '%(name)s'),
6645
+ ('separator', '::'),
6646
+ ('message', '%(message)s'),
6647
+ ]
6393
6648
 
6394
- else:
6395
- return json.loads(f.read())
6396
6649
 
6650
+ class StandardLogFormatter(logging.Formatter):
6651
+ @staticmethod
6652
+ def build_log_format(parts: ta.Iterable[ta.Tuple[str, str]]) -> str:
6653
+ return ' '.join(v for k, v in parts)
6397
6654
 
6398
- def read_config_file(
6399
- path: str,
6400
- cls: ta.Type[T],
6401
- *,
6402
- prepare: ta.Optional[ta.Callable[[ConfigMapping], ConfigMapping]] = None,
6403
- ) -> T:
6404
- with open(path) as cf:
6405
- config_dct = parse_config_file(os.path.basename(path), cf)
6655
+ converter = datetime.datetime.fromtimestamp # type: ignore
6406
6656
 
6407
- if prepare is not None:
6408
- config_dct = prepare(config_dct)
6657
+ def formatTime(self, record, datefmt=None):
6658
+ ct = self.converter(record.created) # type: ignore
6659
+ if datefmt:
6660
+ return ct.strftime(datefmt) # noqa
6661
+ else:
6662
+ t = ct.strftime('%Y-%m-%d %H:%M:%S')
6663
+ return '%s.%03d' % (t, record.msecs) # noqa
6409
6664
 
6410
- return unmarshal_obj(config_dct, cls)
6411
6665
 
6666
+ ##
6412
6667
 
6413
- def build_config_named_children(
6414
- o: ta.Union[
6415
- ta.Sequence[ConfigMapping],
6416
- ta.Mapping[str, ConfigMapping],
6417
- None,
6418
- ],
6419
- *,
6420
- name_key: str = 'name',
6421
- ) -> ta.Optional[ta.Sequence[ConfigMapping]]:
6422
- if o is None:
6423
- return None
6424
6668
 
6425
- lst: ta.List[ConfigMapping] = []
6426
- if isinstance(o, ta.Mapping):
6427
- for k, v in o.items():
6428
- check.isinstance(v, ta.Mapping)
6429
- if name_key in v:
6430
- n = v[name_key]
6431
- if k != n:
6432
- raise KeyError(f'Given names do not match: {n} != {k}')
6433
- lst.append(v)
6434
- else:
6435
- lst.append({name_key: k, **v})
6669
+ class StandardLogHandler(ProxyLogHandler):
6670
+ pass
6436
6671
 
6437
- else:
6438
- check.not_isinstance(o, str)
6439
- lst.extend(o)
6440
6672
 
6441
- seen = set()
6442
- for d in lst:
6443
- n = d['name']
6444
- if n in d:
6445
- raise KeyError(f'Duplicate name: {n}')
6446
- seen.add(n)
6673
+ ##
6447
6674
 
6448
- return lst
6449
6675
 
6676
+ @contextlib.contextmanager
6677
+ def _locking_logging_module_lock() -> ta.Iterator[None]:
6678
+ if hasattr(logging, '_acquireLock'):
6679
+ logging._acquireLock() # noqa
6680
+ try:
6681
+ yield
6682
+ finally:
6683
+ logging._releaseLock() # type: ignore # noqa
6450
6684
 
6451
- ########################################
6452
- # ../commands/marshal.py
6685
+ elif hasattr(logging, '_lock'):
6686
+ # https://github.com/python/cpython/commit/74723e11109a320e628898817ab449b3dad9ee96
6687
+ with logging._lock: # noqa
6688
+ yield
6453
6689
 
6454
-
6455
- def install_command_marshaling(
6456
- cmds: CommandNameMap,
6457
- msh: ObjMarshalerManager,
6458
- ) -> None:
6459
- for fn in [
6460
- lambda c: c,
6461
- lambda c: c.Output,
6462
- ]:
6463
- msh.register_opj_marshaler(
6464
- fn(Command),
6465
- PolymorphicObjMarshaler.of([
6466
- PolymorphicObjMarshaler.Impl(
6467
- fn(cmd),
6468
- name,
6469
- msh.get_obj_marshaler(fn(cmd)),
6470
- )
6471
- for name, cmd in cmds.items()
6472
- ]),
6473
- )
6474
-
6475
-
6476
- ########################################
6477
- # ../commands/ping.py
6478
-
6479
-
6480
- ##
6481
-
6482
-
6483
- @dc.dataclass(frozen=True)
6484
- class PingCommand(Command['PingCommand.Output']):
6485
- time: float = dc.field(default_factory=time.time)
6486
-
6487
- @dc.dataclass(frozen=True)
6488
- class Output(Command.Output):
6489
- time: float
6490
-
6491
-
6492
- class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
6493
- async def execute(self, cmd: PingCommand) -> PingCommand.Output:
6494
- return PingCommand.Output(cmd.time)
6495
-
6496
-
6497
- ########################################
6498
- # ../commands/types.py
6499
-
6500
-
6501
- CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
6502
-
6503
-
6504
- ########################################
6505
- # ../deploy/commands.py
6506
-
6507
-
6508
- ##
6509
-
6510
-
6511
- @dc.dataclass(frozen=True)
6512
- class DeployCommand(Command['DeployCommand.Output']):
6513
- @dc.dataclass(frozen=True)
6514
- class Output(Command.Output):
6515
- pass
6516
-
6517
-
6518
- class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
6519
- async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
6520
- log.info('Deploying!')
6521
-
6522
- return DeployCommand.Output()
6523
-
6524
-
6525
- ########################################
6526
- # ../marshal.py
6527
-
6528
-
6529
- @dc.dataclass(frozen=True)
6530
- class ObjMarshalerInstaller:
6531
- fn: ta.Callable[[ObjMarshalerManager], None]
6532
-
6533
-
6534
- ObjMarshalerInstallers = ta.NewType('ObjMarshalerInstallers', ta.Sequence[ObjMarshalerInstaller])
6535
-
6536
-
6537
- ########################################
6538
- # ../remote/channel.py
6539
-
6540
-
6541
- ##
6542
-
6543
-
6544
- class RemoteChannel(abc.ABC):
6545
- @abc.abstractmethod
6546
- def send_obj(self, o: ta.Any, ty: ta.Any = None) -> ta.Awaitable[None]:
6547
- raise NotImplementedError
6548
-
6549
- @abc.abstractmethod
6550
- def recv_obj(self, ty: ta.Type[T]) -> ta.Awaitable[ta.Optional[T]]:
6551
- raise NotImplementedError
6552
-
6553
- def set_marshaler(self, msh: ObjMarshalerManager) -> None: # noqa
6554
- pass
6555
-
6556
-
6557
- ##
6558
-
6559
-
6560
- class RemoteChannelImpl(RemoteChannel):
6561
- def __init__(
6562
- self,
6563
- input: asyncio.StreamReader, # noqa
6564
- output: asyncio.StreamWriter,
6565
- *,
6566
- msh: ObjMarshalerManager = OBJ_MARSHALER_MANAGER,
6567
- ) -> None:
6568
- super().__init__()
6569
-
6570
- self._input = input
6571
- self._output = output
6572
- self._msh = msh
6573
-
6574
- self._input_lock = asyncio.Lock()
6575
- self._output_lock = asyncio.Lock()
6576
-
6577
- def set_marshaler(self, msh: ObjMarshalerManager) -> None:
6578
- self._msh = msh
6579
-
6580
- #
6581
-
6582
- async def _send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
6583
- j = json_dumps_compact(self._msh.marshal_obj(o, ty))
6584
- d = j.encode('utf-8')
6585
-
6586
- self._output.write(struct.pack('<I', len(d)))
6587
- self._output.write(d)
6588
- await self._output.drain()
6589
-
6590
- async def send_obj(self, o: ta.Any, ty: ta.Any = None) -> None:
6591
- async with self._output_lock:
6592
- return await self._send_obj(o, ty)
6593
-
6594
- #
6595
-
6596
- async def _recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
6597
- d = await self._input.read(4)
6598
- if not d:
6599
- return None
6600
- if len(d) != 4:
6601
- raise EOFError
6602
-
6603
- sz = struct.unpack('<I', d)[0]
6604
- d = await self._input.read(sz)
6605
- if len(d) != sz:
6606
- raise EOFError
6607
-
6608
- j = json.loads(d.decode('utf-8'))
6609
- return self._msh.unmarshal_obj(j, ty)
6610
-
6611
- async def recv_obj(self, ty: ta.Type[T]) -> ta.Optional[T]:
6612
- async with self._input_lock:
6613
- return await self._recv_obj(ty)
6690
+ else:
6691
+ raise Exception("Can't find lock in logging module")
6614
6692
 
6615
6693
 
6616
- ########################################
6617
- # ../system/platforms.py
6618
-
6619
-
6620
- ##
6621
-
6622
-
6623
- @dc.dataclass(frozen=True)
6624
- class Platform(abc.ABC): # noqa
6625
- pass
6626
-
6627
-
6628
- class LinuxPlatform(Platform, abc.ABC):
6629
- pass
6630
-
6631
-
6632
- class UbuntuPlatform(LinuxPlatform):
6633
- pass
6634
-
6635
-
6636
- class AmazonLinuxPlatform(LinuxPlatform):
6637
- pass
6638
-
6639
-
6640
- class GenericLinuxPlatform(LinuxPlatform):
6641
- pass
6642
-
6643
-
6644
- class DarwinPlatform(Platform):
6645
- pass
6694
+ def configure_standard_logging(
6695
+ level: ta.Union[int, str] = logging.INFO,
6696
+ *,
6697
+ json: bool = False,
6698
+ target: ta.Optional[logging.Logger] = None,
6699
+ force: bool = False,
6700
+ handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
6701
+ ) -> ta.Optional[StandardLogHandler]:
6702
+ with _locking_logging_module_lock():
6703
+ if target is None:
6704
+ target = logging.root
6646
6705
 
6706
+ #
6647
6707
 
6648
- class UnknownPlatform(Platform):
6649
- pass
6708
+ if not force:
6709
+ if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
6710
+ return None
6650
6711
 
6712
+ #
6651
6713
 
6652
- ##
6714
+ if handler_factory is not None:
6715
+ handler = handler_factory()
6716
+ else:
6717
+ handler = logging.StreamHandler()
6653
6718
 
6719
+ #
6654
6720
 
6655
- def _detect_system_platform() -> Platform:
6656
- plat = sys.platform
6721
+ formatter: logging.Formatter
6722
+ if json:
6723
+ formatter = JsonLogFormatter()
6724
+ else:
6725
+ formatter = StandardLogFormatter(StandardLogFormatter.build_log_format(STANDARD_LOG_FORMAT_PARTS))
6726
+ handler.setFormatter(formatter)
6657
6727
 
6658
- if plat == 'linux':
6659
- if (osr := LinuxOsRelease.read()) is None:
6660
- return GenericLinuxPlatform()
6728
+ #
6661
6729
 
6662
- if osr.id == 'amzn':
6663
- return AmazonLinuxPlatform()
6730
+ handler.addFilter(TidLogFilter())
6664
6731
 
6665
- elif osr.id == 'ubuntu':
6666
- return UbuntuPlatform()
6732
+ #
6667
6733
 
6668
- else:
6669
- return GenericLinuxPlatform()
6734
+ target.addHandler(handler)
6670
6735
 
6671
- elif plat == 'darwin':
6672
- return DarwinPlatform()
6736
+ #
6673
6737
 
6674
- else:
6675
- return UnknownPlatform()
6738
+ if level is not None:
6739
+ target.setLevel(level)
6676
6740
 
6741
+ #
6677
6742
 
6678
- @cached_nullary
6679
- def detect_system_platform() -> Platform:
6680
- platform = _detect_system_platform()
6681
- log.info('Detected platform: %r', platform)
6682
- return platform
6743
+ return StandardLogHandler(handler)
6683
6744
 
6684
6745
 
6685
6746
  ########################################
6686
- # ../../../omlish/lite/subprocesses.py
6747
+ # ../../../omlish/subprocesses.py
6687
6748
 
6688
6749
 
6689
6750
  ##
@@ -6734,8 +6795,8 @@ def subprocess_close(
6734
6795
  ##
6735
6796
 
6736
6797
 
6737
- class AbstractSubprocesses(abc.ABC): # noqa
6738
- DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = log
6798
+ class BaseSubprocesses(abc.ABC): # noqa
6799
+ DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = None
6739
6800
 
6740
6801
  def __init__(
6741
6802
  self,
@@ -6748,6 +6809,9 @@ class AbstractSubprocesses(abc.ABC): # noqa
6748
6809
  self._log = log if log is not None else self.DEFAULT_LOGGER
6749
6810
  self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
6750
6811
 
6812
+ def set_logger(self, log: ta.Optional[logging.Logger]) -> None:
6813
+ self._log = log
6814
+
6751
6815
  #
6752
6816
 
6753
6817
  def prepare_args(
@@ -6859,23 +6923,25 @@ class AbstractSubprocesses(abc.ABC): # noqa
6859
6923
  ##
6860
6924
 
6861
6925
 
6862
- class Subprocesses(AbstractSubprocesses):
6926
+ class AbstractSubprocesses(BaseSubprocesses, abc.ABC):
6927
+ @abc.abstractmethod
6863
6928
  def check_call(
6864
6929
  self,
6865
6930
  *cmd: str,
6866
6931
  stdout: ta.Any = sys.stderr,
6867
6932
  **kwargs: ta.Any,
6868
6933
  ) -> None:
6869
- with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs): # noqa
6870
- subprocess.check_call(cmd, **kwargs)
6934
+ raise NotImplementedError
6871
6935
 
6936
+ @abc.abstractmethod
6872
6937
  def check_output(
6873
6938
  self,
6874
6939
  *cmd: str,
6875
6940
  **kwargs: ta.Any,
6876
6941
  ) -> bytes:
6877
- with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs): # noqa
6878
- return subprocess.check_output(cmd, **kwargs)
6942
+ raise NotImplementedError
6943
+
6944
+ #
6879
6945
 
6880
6946
  def check_output_str(
6881
6947
  self,
@@ -6917,9 +6983,109 @@ class Subprocesses(AbstractSubprocesses):
6917
6983
  return ret.decode().strip()
6918
6984
 
6919
6985
 
6986
+ ##
6987
+
6988
+
6989
+ class Subprocesses(AbstractSubprocesses):
6990
+ def check_call(
6991
+ self,
6992
+ *cmd: str,
6993
+ stdout: ta.Any = sys.stderr,
6994
+ **kwargs: ta.Any,
6995
+ ) -> None:
6996
+ with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs): # noqa
6997
+ subprocess.check_call(cmd, **kwargs)
6998
+
6999
+ def check_output(
7000
+ self,
7001
+ *cmd: str,
7002
+ **kwargs: ta.Any,
7003
+ ) -> bytes:
7004
+ with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs): # noqa
7005
+ return subprocess.check_output(cmd, **kwargs)
7006
+
7007
+
6920
7008
  subprocesses = Subprocesses()
6921
7009
 
6922
7010
 
7011
+ ##
7012
+
7013
+
7014
+ class AbstractAsyncSubprocesses(BaseSubprocesses):
7015
+ @abc.abstractmethod
7016
+ async def check_call(
7017
+ self,
7018
+ *cmd: str,
7019
+ stdout: ta.Any = sys.stderr,
7020
+ **kwargs: ta.Any,
7021
+ ) -> None:
7022
+ raise NotImplementedError
7023
+
7024
+ @abc.abstractmethod
7025
+ async def check_output(
7026
+ self,
7027
+ *cmd: str,
7028
+ **kwargs: ta.Any,
7029
+ ) -> bytes:
7030
+ raise NotImplementedError
7031
+
7032
+ #
7033
+
7034
+ async def check_output_str(
7035
+ self,
7036
+ *cmd: str,
7037
+ **kwargs: ta.Any,
7038
+ ) -> str:
7039
+ return (await self.check_output(*cmd, **kwargs)).decode().strip()
7040
+
7041
+ #
7042
+
7043
+ async def try_call(
7044
+ self,
7045
+ *cmd: str,
7046
+ **kwargs: ta.Any,
7047
+ ) -> bool:
7048
+ if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
7049
+ return False
7050
+ else:
7051
+ return True
7052
+
7053
+ async def try_output(
7054
+ self,
7055
+ *cmd: str,
7056
+ **kwargs: ta.Any,
7057
+ ) -> ta.Optional[bytes]:
7058
+ if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
7059
+ return None
7060
+ else:
7061
+ return ret
7062
+
7063
+ async def try_output_str(
7064
+ self,
7065
+ *cmd: str,
7066
+ **kwargs: ta.Any,
7067
+ ) -> ta.Optional[str]:
7068
+ if (ret := await self.try_output(*cmd, **kwargs)) is None:
7069
+ return None
7070
+ else:
7071
+ return ret.decode().strip()
7072
+
7073
+
7074
+ ########################################
7075
+ # ../bootstrap.py
7076
+
7077
+
7078
+ @dc.dataclass(frozen=True)
7079
+ class MainBootstrap:
7080
+ main_config: MainConfig = MainConfig()
7081
+
7082
+ deploy_config: DeployConfig = DeployConfig()
7083
+
7084
+ remote_config: RemoteConfig = RemoteConfig()
7085
+
7086
+ system_config: SystemConfig = SystemConfig()
7087
+
7088
+
6923
7089
  ########################################
6924
7090
  # ../commands/local.py
6925
7091
 
@@ -7337,16 +7503,7 @@ class RemoteCommandExecutor(CommandExecutor):
7337
7503
 
7338
7504
 
7339
7505
  ########################################
7340
- # ../system/config.py
7341
-
7342
-
7343
- @dc.dataclass(frozen=True)
7344
- class SystemConfig:
7345
- platform: ta.Optional[Platform] = None
7346
-
7347
-
7348
- ########################################
7349
- # ../../../omlish/lite/asyncio/subprocesses.py
7506
+ # ../../../omlish/asyncs/asyncio/subprocesses.py
7350
7507
 
7351
7508
 
7352
7509
  ##
@@ -7357,6 +7514,8 @@ class AsyncioProcessCommunicator:
7357
7514
  self,
7358
7515
  proc: asyncio.subprocess.Process,
7359
7516
  loop: ta.Optional[ta.Any] = None,
7517
+ *,
7518
+ log: ta.Optional[logging.Logger] = None,
7360
7519
  ) -> None:
7361
7520
  super().__init__()
7362
7521
 
@@ -7365,6 +7524,7 @@ class AsyncioProcessCommunicator:
7365
7524
 
7366
7525
  self._proc = proc
7367
7526
  self._loop = loop
7527
+ self._log = log
7368
7528
 
7369
7529
  self._transport: asyncio.base_subprocess.BaseSubprocessTransport = check.isinstance(
7370
7530
  proc._transport, # type: ignore # noqa
@@ -7380,19 +7540,19 @@ class AsyncioProcessCommunicator:
7380
7540
  try:
7381
7541
  if input is not None:
7382
7542
  stdin.write(input)
7383
- if self._debug:
7384
- log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
7543
+ if self._debug and self._log is not None:
7544
+ self._log.debug('%r communicate: feed stdin (%s bytes)', self, len(input))
7385
7545
 
7386
7546
  await stdin.drain()
7387
7547
 
7388
7548
  except (BrokenPipeError, ConnectionResetError) as exc:
7389
7549
  # communicate() ignores BrokenPipeError and ConnectionResetError. write() and drain() can raise these
7390
7550
  # exceptions.
7391
- if self._debug:
7392
- log.debug('%r communicate: stdin got %r', self, exc)
7551
+ if self._debug and self._log is not None:
7552
+ self._log.debug('%r communicate: stdin got %r', self, exc)
7393
7553
 
7394
- if self._debug:
7395
- log.debug('%r communicate: close stdin', self)
7554
+ if self._debug and self._log is not None:
7555
+ self._log.debug('%r communicate: close stdin', self)
7396
7556
 
7397
7557
  stdin.close()
7398
7558
 
@@ -7408,15 +7568,15 @@ class AsyncioProcessCommunicator:
7408
7568
  check.equal(fd, 1)
7409
7569
  stream = check.not_none(self._proc.stdout)
7410
7570
 
7411
- if self._debug:
7571
+ if self._debug and self._log is not None:
7412
7572
  name = 'stdout' if fd == 1 else 'stderr'
7413
- log.debug('%r communicate: read %s', self, name)
7573
+ self._log.debug('%r communicate: read %s', self, name)
7414
7574
 
7415
7575
  output = await stream.read()
7416
7576
 
7417
- if self._debug:
7577
+ if self._debug and self._log is not None:
7418
7578
  name = 'stdout' if fd == 1 else 'stderr'
7419
- log.debug('%r communicate: close %s', self, name)
7579
+ self._log.debug('%r communicate: close %s', self, name)
7420
7580
 
7421
7581
  transport.close()
7422
7582
 
@@ -7465,7 +7625,7 @@ class AsyncioProcessCommunicator:
7465
7625
  ##
7466
7626
 
7467
7627
 
7468
- class AsyncioSubprocesses(AbstractSubprocesses):
7628
+ class AsyncioSubprocesses(AbstractAsyncSubprocesses):
7469
7629
  async def communicate(
7470
7630
  self,
7471
7631
  proc: asyncio.subprocess.Process,
@@ -7562,45 +7722,6 @@ class AsyncioSubprocesses(AbstractSubprocesses):
7562
7722
  with self.prepare_and_wrap(*cmd, stdout=subprocess.PIPE, check=True, **kwargs) as (cmd, kwargs): # noqa
7563
7723
  return check.not_none((await self.run(*cmd, **kwargs)).stdout)
7564
7724
 
7565
- async def check_output_str(
7566
- self,
7567
- *cmd: str,
7568
- **kwargs: ta.Any,
7569
- ) -> str:
7570
- return (await self.check_output(*cmd, **kwargs)).decode().strip()
7571
-
7572
- #
7573
-
7574
- async def try_call(
7575
- self,
7576
- *cmd: str,
7577
- **kwargs: ta.Any,
7578
- ) -> bool:
7579
- if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
7580
- return False
7581
- else:
7582
- return True
7583
-
7584
- async def try_output(
7585
- self,
7586
- *cmd: str,
7587
- **kwargs: ta.Any,
7588
- ) -> ta.Optional[bytes]:
7589
- if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
7590
- return None
7591
- else:
7592
- return ret
7593
-
7594
- async def try_output_str(
7595
- self,
7596
- *cmd: str,
7597
- **kwargs: ta.Any,
7598
- ) -> ta.Optional[str]:
7599
- if (ret := await self.try_output(*cmd, **kwargs)) is None:
7600
- return None
7601
- else:
7602
- return ret.decode().strip()
7603
-
7604
7725
 
7605
7726
  asyncio_subprocesses = AsyncioSubprocesses()
7606
7727
 
@@ -7700,21 +7821,6 @@ class InterpInspector:
7700
7821
  INTERP_INSPECTOR = InterpInspector()
7701
7822
 
7702
7823
 
7703
- ########################################
7704
- # ../bootstrap.py
7705
-
7706
-
7707
- @dc.dataclass(frozen=True)
7708
- class MainBootstrap:
7709
- main_config: MainConfig = MainConfig()
7710
-
7711
- deploy_config: DeployConfig = DeployConfig()
7712
-
7713
- remote_config: RemoteConfig = RemoteConfig()
7714
-
7715
- system_config: SystemConfig = SystemConfig()
7716
-
7717
-
7718
7824
  ########################################
7719
7825
  # ../commands/subprocess.py
7720
7826
 
@@ -7801,39 +7907,22 @@ github.com/wrmsr/omlish@rev
7801
7907
  ##
7802
7908
 
7803
7909
 
7804
- @dc.dataclass(frozen=True)
7805
- class DeployGitRepo:
7806
- host: ta.Optional[str] = None
7807
- username: ta.Optional[str] = None
7808
- path: ta.Optional[str] = None
7809
-
7810
- def __post_init__(self) -> None:
7811
- check.not_in('..', check.non_empty_str(self.host))
7812
- check.not_in('.', check.non_empty_str(self.path))
7813
-
7814
-
7815
- @dc.dataclass(frozen=True)
7816
- class DeployGitSpec:
7817
- repo: DeployGitRepo
7818
- rev: DeployRev
7819
-
7820
-
7821
- ##
7822
-
7823
-
7824
7910
  class DeployGitManager(DeployPathOwner):
7825
7911
  def __init__(
7826
7912
  self,
7827
7913
  *,
7828
- deploy_home: DeployHome,
7914
+ deploy_home: ta.Optional[DeployHome] = None,
7829
7915
  ) -> None:
7830
7916
  super().__init__()
7831
7917
 
7832
7918
  self._deploy_home = deploy_home
7833
- self._dir = os.path.join(deploy_home, 'git')
7834
7919
 
7835
7920
  self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
7836
7921
 
7922
+ @cached_nullary
7923
+ def _dir(self) -> str:
7924
+ return os.path.join(check.non_empty_str(self._deploy_home), 'git')
7925
+
7837
7926
  def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
7838
7927
  return {
7839
7928
  DeployPath.parse('git'),
@@ -7850,7 +7939,7 @@ class DeployGitManager(DeployPathOwner):
7850
7939
  self._git = git
7851
7940
  self._repo = repo
7852
7941
  self._dir = os.path.join(
7853
- self._git._dir, # noqa
7942
+ self._git._dir(), # noqa
7854
7943
  check.non_empty_str(repo.host),
7855
7944
  check.non_empty_str(repo.path),
7856
7945
  )
@@ -7907,8 +7996,8 @@ class DeployGitManager(DeployPathOwner):
7907
7996
  repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
7908
7997
  return repo_dir
7909
7998
 
7910
- async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
7911
- await self.get_repo_dir(spec.repo).checkout(spec.rev, dst_dir)
7999
+ async def checkout(self, repo: DeployGitRepo, rev: DeployRev, dst_dir: str) -> None:
8000
+ await self.get_repo_dir(repo).checkout(rev, dst_dir)
7912
8001
 
7913
8002
 
7914
8003
  ########################################
@@ -7924,12 +8013,15 @@ class DeployVenvManager(DeployPathOwner):
7924
8013
  def __init__(
7925
8014
  self,
7926
8015
  *,
7927
- deploy_home: DeployHome,
8016
+ deploy_home: ta.Optional[DeployHome] = None,
7928
8017
  ) -> None:
7929
8018
  super().__init__()
7930
8019
 
7931
8020
  self._deploy_home = deploy_home
7932
- self._dir = os.path.join(deploy_home, 'venvs')
8021
+
8022
+ @cached_nullary
8023
+ def _dir(self) -> str:
8024
+ return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
7933
8025
 
7934
8026
  def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
7935
8027
  return {
@@ -7966,66 +8058,208 @@ class DeployVenvManager(DeployPathOwner):
7966
8058
 
7967
8059
  async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
7968
8060
  await self.setup_venv(
7969
- os.path.join(self._deploy_home, 'apps', app_tag.app, app_tag.tag),
7970
- os.path.join(self._deploy_home, 'venvs', app_tag.app, app_tag.tag),
8061
+ os.path.join(check.non_empty_str(self._deploy_home), 'apps', app_tag.app, app_tag.tag),
8062
+ os.path.join(self._dir(), app_tag.app, app_tag.tag),
7971
8063
  )
7972
8064
 
7973
8065
 
7974
8066
  ########################################
7975
- # ../remote/spawning.py
8067
+ # ../remote/_main.py
7976
8068
 
7977
8069
 
7978
8070
  ##
7979
8071
 
7980
8072
 
7981
- class RemoteSpawning(abc.ABC):
7982
- @dc.dataclass(frozen=True)
7983
- class Target:
7984
- shell: ta.Optional[str] = None
7985
- shell_quote: bool = False
8073
+ class _RemoteExecutionLogHandler(logging.Handler):
8074
+ def __init__(self, fn: ta.Callable[[str], None]) -> None:
8075
+ super().__init__()
8076
+ self._fn = fn
7986
8077
 
7987
- DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
7988
- python: str = DEFAULT_PYTHON
8078
+ def emit(self, record):
8079
+ msg = self.format(record)
8080
+ self._fn(msg)
7989
8081
 
7990
- stderr: ta.Optional[str] = None # SubprocessChannelOption
7991
8082
 
7992
- @dc.dataclass(frozen=True)
7993
- class Spawned:
7994
- stdin: asyncio.StreamWriter
7995
- stdout: asyncio.StreamReader
7996
- stderr: ta.Optional[asyncio.StreamReader]
8083
+ ##
7997
8084
 
7998
- @abc.abstractmethod
7999
- def spawn(
8085
+
8086
+ class _RemoteExecutionMain:
8087
+ def __init__(
8000
8088
  self,
8001
- tgt: Target,
8002
- src: str,
8003
- *,
8004
- timeout: ta.Optional[float] = None,
8005
- debug: bool = False,
8006
- ) -> ta.AsyncContextManager[Spawned]:
8007
- raise NotImplementedError
8089
+ chan: RemoteChannel,
8090
+ ) -> None:
8091
+ super().__init__()
8008
8092
 
8093
+ self._chan = chan
8009
8094
 
8010
- ##
8095
+ self.__bootstrap: ta.Optional[MainBootstrap] = None
8096
+ self.__injector: ta.Optional[Injector] = None
8011
8097
 
8098
+ @property
8099
+ def _bootstrap(self) -> MainBootstrap:
8100
+ return check.not_none(self.__bootstrap)
8012
8101
 
8013
- class SubprocessRemoteSpawning(RemoteSpawning):
8014
- class _PreparedCmd(ta.NamedTuple): # noqa
8015
- cmd: ta.Sequence[str]
8016
- shell: bool
8102
+ @property
8103
+ def _injector(self) -> Injector:
8104
+ return check.not_none(self.__injector)
8017
8105
 
8018
- def _prepare_cmd(
8019
- self,
8020
- tgt: RemoteSpawning.Target,
8021
- src: str,
8022
- ) -> _PreparedCmd:
8023
- if tgt.shell is not None:
8024
- sh_src = f'{tgt.python} -c {shlex.quote(src)}'
8025
- if tgt.shell_quote:
8026
- sh_src = shlex.quote(sh_src)
8027
- sh_cmd = f'{tgt.shell} {sh_src}'
8028
- return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
8106
+ #
8107
+
8108
+ def _timebomb_main(
8109
+ self,
8110
+ delay_s: float,
8111
+ *,
8112
+ sig: int = signal.SIGINT,
8113
+ code: int = 1,
8114
+ ) -> None:
8115
+ time.sleep(delay_s)
8116
+
8117
+ if (pgid := os.getpgid(0)) == os.getpid():
8118
+ os.killpg(pgid, sig)
8119
+
8120
+ os._exit(code) # noqa
8121
+
8122
+ @cached_nullary
8123
+ def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
8124
+ if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
8125
+ return None
8126
+
8127
+ thr = threading.Thread(
8128
+ target=functools.partial(self._timebomb_main, tbd),
8129
+ name=f'{self.__class__.__name__}.timebomb',
8130
+ daemon=True,
8131
+ )
8132
+
8133
+ thr.start()
8134
+
8135
+ log.debug('Started timebomb thread: %r', thr)
8136
+
8137
+ return thr
8138
+
8139
+ #
8140
+
8141
+ @cached_nullary
8142
+ def _log_handler(self) -> _RemoteLogHandler:
8143
+ return _RemoteLogHandler(self._chan)
8144
+
8145
+ #
8146
+
8147
+ async def _setup(self) -> None:
8148
+ check.none(self.__bootstrap)
8149
+ check.none(self.__injector)
8150
+
8151
+ # Bootstrap
8152
+
8153
+ self.__bootstrap = check.not_none(await self._chan.recv_obj(MainBootstrap))
8154
+
8155
+ if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
8156
+ pycharm_debug_connect(prd)
8157
+
8158
+ self.__injector = main_bootstrap(self._bootstrap)
8159
+
8160
+ self._chan.set_marshaler(self._injector[ObjMarshalerManager])
8161
+
8162
+ # Post-bootstrap
8163
+
8164
+ if self._bootstrap.remote_config.set_pgid:
8165
+ if os.getpgid(0) != os.getpid():
8166
+ log.debug('Setting pgid')
8167
+ os.setpgid(0, 0)
8168
+
8169
+ if (ds := self._bootstrap.remote_config.deathsig) is not None:
8170
+ log.debug('Setting deathsig: %s', ds)
8171
+ set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
8172
+
8173
+ self._timebomb_thread()
8174
+
8175
+ if self._bootstrap.remote_config.forward_logging:
8176
+ log.debug('Installing log forwarder')
8177
+ logging.root.addHandler(self._log_handler())
8178
+
8179
+ #
8180
+
8181
+ async def run(self) -> None:
8182
+ await self._setup()
8183
+
8184
+ executor = self._injector[LocalCommandExecutor]
8185
+
8186
+ handler = _RemoteCommandHandler(self._chan, executor)
8187
+
8188
+ await handler.run()
8189
+
8190
+
8191
+ def _remote_execution_main() -> None:
8192
+ rt = pyremote_bootstrap_finalize() # noqa
8193
+
8194
+ async def inner() -> None:
8195
+ input = await asyncio_open_stream_reader(rt.input) # noqa
8196
+ output = await asyncio_open_stream_writer(rt.output)
8197
+
8198
+ chan = RemoteChannelImpl(
8199
+ input,
8200
+ output,
8201
+ )
8202
+
8203
+ await _RemoteExecutionMain(chan).run()
8204
+
8205
+ asyncio.run(inner())
8206
+
8207
+
8208
+ ########################################
8209
+ # ../remote/spawning.py
8210
+
8211
+
8212
+ ##
8213
+
8214
+
8215
+ class RemoteSpawning(abc.ABC):
8216
+ @dc.dataclass(frozen=True)
8217
+ class Target:
8218
+ shell: ta.Optional[str] = None
8219
+ shell_quote: bool = False
8220
+
8221
+ DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
8222
+ python: str = DEFAULT_PYTHON
8223
+
8224
+ stderr: ta.Optional[str] = None # SubprocessChannelOption
8225
+
8226
+ @dc.dataclass(frozen=True)
8227
+ class Spawned:
8228
+ stdin: asyncio.StreamWriter
8229
+ stdout: asyncio.StreamReader
8230
+ stderr: ta.Optional[asyncio.StreamReader]
8231
+
8232
+ @abc.abstractmethod
8233
+ def spawn(
8234
+ self,
8235
+ tgt: Target,
8236
+ src: str,
8237
+ *,
8238
+ timeout: ta.Optional[float] = None,
8239
+ debug: bool = False,
8240
+ ) -> ta.AsyncContextManager[Spawned]:
8241
+ raise NotImplementedError
8242
+
8243
+
8244
+ ##
8245
+
8246
+
8247
+ class SubprocessRemoteSpawning(RemoteSpawning):
8248
+ class _PreparedCmd(ta.NamedTuple): # noqa
8249
+ cmd: ta.Sequence[str]
8250
+ shell: bool
8251
+
8252
+ def _prepare_cmd(
8253
+ self,
8254
+ tgt: RemoteSpawning.Target,
8255
+ src: str,
8256
+ ) -> _PreparedCmd:
8257
+ if tgt.shell is not None:
8258
+ sh_src = f'{tgt.python} -c {shlex.quote(src)}'
8259
+ if tgt.shell_quote:
8260
+ sh_src = shlex.quote(sh_src)
8261
+ sh_cmd = f'{tgt.shell} {sh_src}'
8262
+ return SubprocessRemoteSpawning._PreparedCmd([sh_cmd], shell=True)
8029
8263
 
8030
8264
  else:
8031
8265
  return SubprocessRemoteSpawning._PreparedCmd([tgt.python, '-c', src], shell=False)
@@ -8366,14 +8600,14 @@ def make_deploy_tag(
8366
8600
  now = datetime.datetime.utcnow() # noqa
8367
8601
  now_fmt = '%Y%m%dT%H%M%S'
8368
8602
  now_str = now.strftime(now_fmt)
8369
- return DeployTag('-'.join([rev, now_str]))
8603
+ return DeployTag('-'.join([now_str, rev]))
8370
8604
 
8371
8605
 
8372
8606
  class DeployAppManager(DeployPathOwner):
8373
8607
  def __init__(
8374
8608
  self,
8375
8609
  *,
8376
- deploy_home: DeployHome,
8610
+ deploy_home: ta.Optional[DeployHome] = None,
8377
8611
  git: DeployGitManager,
8378
8612
  venvs: DeployVenvManager,
8379
8613
  ) -> None:
@@ -8383,7 +8617,9 @@ class DeployAppManager(DeployPathOwner):
8383
8617
  self._git = git
8384
8618
  self._venvs = venvs
8385
8619
 
8386
- self._dir = os.path.join(deploy_home, 'apps')
8620
+ @cached_nullary
8621
+ def _dir(self) -> str:
8622
+ return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
8387
8623
 
8388
8624
  def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
8389
8625
  return {
@@ -8392,20 +8628,16 @@ class DeployAppManager(DeployPathOwner):
8392
8628
 
8393
8629
  async def prepare_app(
8394
8630
  self,
8395
- app: DeployApp,
8396
- rev: DeployRev,
8397
- repo: DeployGitRepo,
8631
+ spec: DeploySpec,
8398
8632
  ):
8399
- app_tag = DeployAppTag(app, make_deploy_tag(rev))
8400
- app_dir = os.path.join(self._dir, app, app_tag.tag)
8633
+ app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev))
8634
+ app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
8401
8635
 
8402
8636
  #
8403
8637
 
8404
8638
  await self._git.checkout(
8405
- DeployGitSpec(
8406
- repo=repo,
8407
- rev=rev,
8408
- ),
8639
+ spec.repo,
8640
+ spec.rev,
8409
8641
  app_dir,
8410
8642
  )
8411
8643
 
@@ -8415,145 +8647,122 @@ class DeployAppManager(DeployPathOwner):
8415
8647
 
8416
8648
 
8417
8649
  ########################################
8418
- # ../remote/_main.py
8419
-
8420
-
8421
- ##
8422
-
8423
-
8424
- class _RemoteExecutionLogHandler(logging.Handler):
8425
- def __init__(self, fn: ta.Callable[[str], None]) -> None:
8426
- super().__init__()
8427
- self._fn = fn
8428
-
8429
- def emit(self, record):
8430
- msg = self.format(record)
8431
- self._fn(msg)
8650
+ # ../remote/connection.py
8432
8651
 
8433
8652
 
8434
8653
  ##
8435
8654
 
8436
8655
 
8437
- class _RemoteExecutionMain:
8656
+ class PyremoteRemoteExecutionConnector:
8438
8657
  def __init__(
8439
8658
  self,
8440
- chan: RemoteChannel,
8659
+ *,
8660
+ spawning: RemoteSpawning,
8661
+ msh: ObjMarshalerManager,
8662
+ payload_file: ta.Optional[RemoteExecutionPayloadFile] = None,
8441
8663
  ) -> None:
8442
8664
  super().__init__()
8443
8665
 
8444
- self._chan = chan
8666
+ self._spawning = spawning
8667
+ self._msh = msh
8668
+ self._payload_file = payload_file
8445
8669
 
8446
- self.__bootstrap: ta.Optional[MainBootstrap] = None
8447
- self.__injector: ta.Optional[Injector] = None
8670
+ #
8448
8671
 
8449
- @property
8450
- def _bootstrap(self) -> MainBootstrap:
8451
- return check.not_none(self.__bootstrap)
8672
+ @cached_nullary
8673
+ def _payload_src(self) -> str:
8674
+ return get_remote_payload_src(file=self._payload_file)
8452
8675
 
8453
- @property
8454
- def _injector(self) -> Injector:
8455
- return check.not_none(self.__injector)
8676
+ @cached_nullary
8677
+ def _remote_src(self) -> ta.Sequence[str]:
8678
+ return [
8679
+ self._payload_src(),
8680
+ '_remote_execution_main()',
8681
+ ]
8682
+
8683
+ @cached_nullary
8684
+ def _spawn_src(self) -> str:
8685
+ return pyremote_build_bootstrap_cmd(__package__ or 'manage')
8456
8686
 
8457
8687
  #
8458
8688
 
8459
- def _timebomb_main(
8689
+ @contextlib.asynccontextmanager
8690
+ async def connect(
8460
8691
  self,
8461
- delay_s: float,
8462
- *,
8463
- sig: int = signal.SIGINT,
8464
- code: int = 1,
8465
- ) -> None:
8466
- time.sleep(delay_s)
8692
+ tgt: RemoteSpawning.Target,
8693
+ bs: MainBootstrap,
8694
+ ) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
8695
+ spawn_src = self._spawn_src()
8696
+ remote_src = self._remote_src()
8467
8697
 
8468
- if (pgid := os.getpgid(0)) == os.getpid():
8469
- os.killpg(pgid, sig)
8698
+ async with self._spawning.spawn(
8699
+ tgt,
8700
+ spawn_src,
8701
+ debug=bs.main_config.debug,
8702
+ ) as proc:
8703
+ res = await PyremoteBootstrapDriver( # noqa
8704
+ remote_src,
8705
+ PyremoteBootstrapOptions(
8706
+ debug=bs.main_config.debug,
8707
+ ),
8708
+ ).async_run(
8709
+ proc.stdout,
8710
+ proc.stdin,
8711
+ )
8470
8712
 
8471
- os._exit(code) # noqa
8713
+ chan = RemoteChannelImpl(
8714
+ proc.stdout,
8715
+ proc.stdin,
8716
+ msh=self._msh,
8717
+ )
8472
8718
 
8473
- @cached_nullary
8474
- def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
8475
- if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
8476
- return None
8719
+ await chan.send_obj(bs)
8477
8720
 
8478
- thr = threading.Thread(
8479
- target=functools.partial(self._timebomb_main, tbd),
8480
- name=f'{self.__class__.__name__}.timebomb',
8481
- daemon=True,
8482
- )
8721
+ rce: RemoteCommandExecutor
8722
+ async with aclosing(RemoteCommandExecutor(chan)) as rce:
8723
+ await rce.start()
8483
8724
 
8484
- thr.start()
8485
-
8486
- log.debug('Started timebomb thread: %r', thr)
8487
-
8488
- return thr
8489
-
8490
- #
8491
-
8492
- @cached_nullary
8493
- def _log_handler(self) -> _RemoteLogHandler:
8494
- return _RemoteLogHandler(self._chan)
8495
-
8496
- #
8497
-
8498
- async def _setup(self) -> None:
8499
- check.none(self.__bootstrap)
8500
- check.none(self.__injector)
8501
-
8502
- # Bootstrap
8503
-
8504
- self.__bootstrap = check.not_none(await self._chan.recv_obj(MainBootstrap))
8505
-
8506
- if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
8507
- pycharm_debug_connect(prd)
8508
-
8509
- self.__injector = main_bootstrap(self._bootstrap)
8510
-
8511
- self._chan.set_marshaler(self._injector[ObjMarshalerManager])
8512
-
8513
- # Post-bootstrap
8514
-
8515
- if self._bootstrap.remote_config.set_pgid:
8516
- if os.getpgid(0) != os.getpid():
8517
- log.debug('Setting pgid')
8518
- os.setpgid(0, 0)
8519
-
8520
- if (ds := self._bootstrap.remote_config.deathsig) is not None:
8521
- log.debug('Setting deathsig: %s', ds)
8522
- set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
8523
-
8524
- self._timebomb_thread()
8525
-
8526
- if self._bootstrap.remote_config.forward_logging:
8527
- log.debug('Installing log forwarder')
8528
- logging.root.addHandler(self._log_handler())
8725
+ yield rce
8529
8726
 
8530
- #
8531
8727
 
8532
- async def run(self) -> None:
8533
- await self._setup()
8534
-
8535
- executor = self._injector[LocalCommandExecutor]
8728
+ ##
8536
8729
 
8537
- handler = _RemoteCommandHandler(self._chan, executor)
8538
8730
 
8539
- await handler.run()
8731
+ class InProcessRemoteExecutionConnector:
8732
+ def __init__(
8733
+ self,
8734
+ *,
8735
+ msh: ObjMarshalerManager,
8736
+ local_executor: LocalCommandExecutor,
8737
+ ) -> None:
8738
+ super().__init__()
8540
8739
 
8740
+ self._msh = msh
8741
+ self._local_executor = local_executor
8541
8742
 
8542
- def _remote_execution_main() -> None:
8543
- rt = pyremote_bootstrap_finalize() # noqa
8743
+ @contextlib.asynccontextmanager
8744
+ async def connect(self) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
8745
+ r0, w0 = asyncio_create_bytes_channel()
8746
+ r1, w1 = asyncio_create_bytes_channel()
8544
8747
 
8545
- async def inner() -> None:
8546
- input = await asyncio_open_stream_reader(rt.input) # noqa
8547
- output = await asyncio_open_stream_writer(rt.output)
8748
+ remote_chan = RemoteChannelImpl(r0, w1, msh=self._msh)
8749
+ local_chan = RemoteChannelImpl(r1, w0, msh=self._msh)
8548
8750
 
8549
- chan = RemoteChannelImpl(
8550
- input,
8551
- output,
8751
+ rch = _RemoteCommandHandler(
8752
+ remote_chan,
8753
+ self._local_executor,
8552
8754
  )
8755
+ rch_task = asyncio.create_task(rch.run()) # noqa
8756
+ try:
8757
+ rce: RemoteCommandExecutor
8758
+ async with aclosing(RemoteCommandExecutor(local_chan)) as rce:
8759
+ await rce.start()
8553
8760
 
8554
- await _RemoteExecutionMain(chan).run()
8761
+ yield rce
8555
8762
 
8556
- asyncio.run(inner())
8763
+ finally:
8764
+ rch.stop()
8765
+ await rch_task
8557
8766
 
8558
8767
 
8559
8768
  ########################################
@@ -8899,521 +9108,256 @@ class PyenvVersionInstaller:
8899
9108
  self._pyenv.exe(),
8900
9109
  'install',
8901
9110
  *conf_args,
8902
- ]
8903
-
8904
- await asyncio_subprocesses.check_call(
8905
- *full_args,
8906
- env=env,
8907
- )
8908
-
8909
- exe = os.path.join(await self.install_dir(), 'bin', 'python')
8910
- if not os.path.isfile(exe):
8911
- raise RuntimeError(f'Interpreter not found: {exe}')
8912
- return exe
8913
-
8914
-
8915
- ##
8916
-
8917
-
8918
- class PyenvInterpProvider(InterpProvider):
8919
- def __init__(
8920
- self,
8921
- pyenv: Pyenv = Pyenv(),
8922
-
8923
- inspect: bool = False,
8924
- inspector: InterpInspector = INTERP_INSPECTOR,
8925
-
8926
- *,
8927
-
8928
- try_update: bool = False,
8929
- ) -> None:
8930
- super().__init__()
8931
-
8932
- self._pyenv = pyenv
8933
-
8934
- self._inspect = inspect
8935
- self._inspector = inspector
8936
-
8937
- self._try_update = try_update
8938
-
8939
- #
8940
-
8941
- @staticmethod
8942
- def guess_version(s: str) -> ta.Optional[InterpVersion]:
8943
- def strip_sfx(s: str, sfx: str) -> ta.Tuple[str, bool]:
8944
- if s.endswith(sfx):
8945
- return s[:-len(sfx)], True
8946
- return s, False
8947
- ok = {}
8948
- s, ok['debug'] = strip_sfx(s, '-debug')
8949
- s, ok['threaded'] = strip_sfx(s, 't')
8950
- try:
8951
- v = Version(s)
8952
- except InvalidVersion:
8953
- return None
8954
- return InterpVersion(v, InterpOpts(**ok))
8955
-
8956
- class Installed(ta.NamedTuple):
8957
- name: str
8958
- exe: str
8959
- version: InterpVersion
8960
-
8961
- async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
8962
- iv: ta.Optional[InterpVersion]
8963
- if self._inspect:
8964
- try:
8965
- iv = check.not_none(await self._inspector.inspect(ep)).iv
8966
- except Exception as e: # noqa
8967
- return None
8968
- else:
8969
- iv = self.guess_version(vn)
8970
- if iv is None:
8971
- return None
8972
- return PyenvInterpProvider.Installed(
8973
- name=vn,
8974
- exe=ep,
8975
- version=iv,
8976
- )
8977
-
8978
- async def installed(self) -> ta.Sequence[Installed]:
8979
- ret: ta.List[PyenvInterpProvider.Installed] = []
8980
- for vn, ep in await self._pyenv.version_exes():
8981
- if (i := await self._make_installed(vn, ep)) is None:
8982
- log.debug('Invalid pyenv version: %s', vn)
8983
- continue
8984
- ret.append(i)
8985
- return ret
8986
-
8987
- #
8988
-
8989
- async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
8990
- return [i.version for i in await self.installed()]
8991
-
8992
- async def get_installed_version(self, version: InterpVersion) -> Interp:
8993
- for i in await self.installed():
8994
- if i.version == version:
8995
- return Interp(
8996
- exe=i.exe,
8997
- version=i.version,
8998
- )
8999
- raise KeyError(version)
9000
-
9001
- #
9002
-
9003
- async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
9004
- lst = []
9005
-
9006
- for vs in await self._pyenv.installable_versions():
9007
- if (iv := self.guess_version(vs)) is None:
9008
- continue
9009
- if iv.opts.debug:
9010
- raise Exception('Pyenv installable versions not expected to have debug suffix')
9011
- for d in [False, True]:
9012
- lst.append(dc.replace(iv, opts=dc.replace(iv.opts, debug=d)))
9013
-
9014
- return lst
9015
-
9016
- async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
9017
- lst = await self._get_installable_versions(spec)
9018
-
9019
- if self._try_update and not any(v in spec for v in lst):
9020
- if self._pyenv.update():
9021
- lst = await self._get_installable_versions(spec)
9022
-
9023
- return lst
9024
-
9025
- async def install_version(self, version: InterpVersion) -> Interp:
9026
- inst_version = str(version.version)
9027
- inst_opts = version.opts
9028
- if inst_opts.threaded:
9029
- inst_version += 't'
9030
- inst_opts = dc.replace(inst_opts, threaded=False)
9031
-
9032
- installer = PyenvVersionInstaller(
9033
- inst_version,
9034
- interp_opts=inst_opts,
9035
- )
9036
-
9037
- exe = await installer.install()
9038
- return Interp(exe, version)
9039
-
9040
-
9041
- ########################################
9042
- # ../../../omdev/interp/system.py
9043
- """
9044
- TODO:
9045
- - python, python3, python3.12, ...
9046
- - check if path py's are venvs: sys.prefix != sys.base_prefix
9047
- """
9048
-
9049
-
9050
- ##
9051
-
9052
-
9053
- @dc.dataclass(frozen=True)
9054
- class SystemInterpProvider(InterpProvider):
9055
- cmd: str = 'python3'
9056
- path: ta.Optional[str] = None
9057
-
9058
- inspect: bool = False
9059
- inspector: InterpInspector = INTERP_INSPECTOR
9060
-
9061
- #
9062
-
9063
- @staticmethod
9064
- def _re_which(
9065
- pat: re.Pattern,
9066
- *,
9067
- mode: int = os.F_OK | os.X_OK,
9068
- path: ta.Optional[str] = None,
9069
- ) -> ta.List[str]:
9070
- if path is None:
9071
- path = os.environ.get('PATH', None)
9072
- if path is None:
9073
- try:
9074
- path = os.confstr('CS_PATH')
9075
- except (AttributeError, ValueError):
9076
- path = os.defpath
9077
-
9078
- if not path:
9079
- return []
9080
-
9081
- path = os.fsdecode(path)
9082
- pathlst = path.split(os.pathsep)
9083
-
9084
- def _access_check(fn: str, mode: int) -> bool:
9085
- return os.path.exists(fn) and os.access(fn, mode)
9086
-
9087
- out = []
9088
- seen = set()
9089
- for d in pathlst:
9090
- normdir = os.path.normcase(d)
9091
- if normdir not in seen:
9092
- seen.add(normdir)
9093
- if not _access_check(normdir, mode):
9094
- continue
9095
- for thefile in os.listdir(d):
9096
- name = os.path.join(d, thefile)
9097
- if not (
9098
- os.path.isfile(name) and
9099
- pat.fullmatch(thefile) and
9100
- _access_check(name, mode)
9101
- ):
9102
- continue
9103
- out.append(name)
9104
-
9105
- return out
9106
-
9107
- @cached_nullary
9108
- def exes(self) -> ta.List[str]:
9109
- return self._re_which(
9110
- re.compile(r'python3(\.\d+)?'),
9111
- path=self.path,
9112
- )
9113
-
9114
- #
9115
-
9116
- async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
9117
- if not self.inspect:
9118
- s = os.path.basename(exe)
9119
- if s.startswith('python'):
9120
- s = s[len('python'):]
9121
- if '.' in s:
9122
- try:
9123
- return InterpVersion.parse(s)
9124
- except InvalidVersion:
9125
- pass
9126
- ii = await self.inspector.inspect(exe)
9127
- return ii.iv if ii is not None else None
9128
-
9129
- async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
9130
- lst = []
9131
- for e in self.exes():
9132
- if (ev := await self.get_exe_version(e)) is None:
9133
- log.debug('Invalid system version: %s', e)
9134
- continue
9135
- lst.append((e, ev))
9136
- return lst
9137
-
9138
- #
9139
-
9140
- async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
9141
- return [ev for e, ev in await self.exe_versions()]
9142
-
9143
- async def get_installed_version(self, version: InterpVersion) -> Interp:
9144
- for e, ev in await self.exe_versions():
9145
- if ev != version:
9146
- continue
9147
- return Interp(
9148
- exe=e,
9149
- version=ev,
9150
- )
9151
- raise KeyError(version)
9152
-
9153
-
9154
- ########################################
9155
- # ../remote/connection.py
9156
-
9157
-
9158
- ##
9159
-
9160
-
9161
- class PyremoteRemoteExecutionConnector:
9162
- def __init__(
9163
- self,
9164
- *,
9165
- spawning: RemoteSpawning,
9166
- msh: ObjMarshalerManager,
9167
- payload_file: ta.Optional[RemoteExecutionPayloadFile] = None,
9168
- ) -> None:
9169
- super().__init__()
9170
-
9171
- self._spawning = spawning
9172
- self._msh = msh
9173
- self._payload_file = payload_file
9174
-
9175
- #
9176
-
9177
- @cached_nullary
9178
- def _payload_src(self) -> str:
9179
- return get_remote_payload_src(file=self._payload_file)
9180
-
9181
- @cached_nullary
9182
- def _remote_src(self) -> ta.Sequence[str]:
9183
- return [
9184
- self._payload_src(),
9185
- '_remote_execution_main()',
9186
- ]
9187
-
9188
- @cached_nullary
9189
- def _spawn_src(self) -> str:
9190
- return pyremote_build_bootstrap_cmd(__package__ or 'manage')
9191
-
9192
- #
9193
-
9194
- @contextlib.asynccontextmanager
9195
- async def connect(
9196
- self,
9197
- tgt: RemoteSpawning.Target,
9198
- bs: MainBootstrap,
9199
- ) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
9200
- spawn_src = self._spawn_src()
9201
- remote_src = self._remote_src()
9202
-
9203
- async with self._spawning.spawn(
9204
- tgt,
9205
- spawn_src,
9206
- debug=bs.main_config.debug,
9207
- ) as proc:
9208
- res = await PyremoteBootstrapDriver( # noqa
9209
- remote_src,
9210
- PyremoteBootstrapOptions(
9211
- debug=bs.main_config.debug,
9212
- ),
9213
- ).async_run(
9214
- proc.stdout,
9215
- proc.stdin,
9216
- )
9217
-
9218
- chan = RemoteChannelImpl(
9219
- proc.stdout,
9220
- proc.stdin,
9221
- msh=self._msh,
9222
- )
9223
-
9224
- await chan.send_obj(bs)
9225
-
9226
- rce: RemoteCommandExecutor
9227
- async with aclosing(RemoteCommandExecutor(chan)) as rce:
9228
- await rce.start()
9111
+ ]
9229
9112
 
9230
- yield rce
9113
+ await asyncio_subprocesses.check_call(
9114
+ *full_args,
9115
+ env=env,
9116
+ )
9117
+
9118
+ exe = os.path.join(await self.install_dir(), 'bin', 'python')
9119
+ if not os.path.isfile(exe):
9120
+ raise RuntimeError(f'Interpreter not found: {exe}')
9121
+ return exe
9231
9122
 
9232
9123
 
9233
9124
  ##
9234
9125
 
9235
9126
 
9236
- class InProcessRemoteExecutionConnector:
9127
+ class PyenvInterpProvider(InterpProvider):
9237
9128
  def __init__(
9238
9129
  self,
9130
+ pyenv: Pyenv = Pyenv(),
9131
+
9132
+ inspect: bool = False,
9133
+ inspector: InterpInspector = INTERP_INSPECTOR,
9134
+
9239
9135
  *,
9240
- msh: ObjMarshalerManager,
9241
- local_executor: LocalCommandExecutor,
9136
+
9137
+ try_update: bool = False,
9242
9138
  ) -> None:
9243
9139
  super().__init__()
9244
9140
 
9245
- self._msh = msh
9246
- self._local_executor = local_executor
9141
+ self._pyenv = pyenv
9247
9142
 
9248
- @contextlib.asynccontextmanager
9249
- async def connect(self) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
9250
- r0, w0 = asyncio_create_bytes_channel()
9251
- r1, w1 = asyncio_create_bytes_channel()
9143
+ self._inspect = inspect
9144
+ self._inspector = inspector
9252
9145
 
9253
- remote_chan = RemoteChannelImpl(r0, w1, msh=self._msh)
9254
- local_chan = RemoteChannelImpl(r1, w0, msh=self._msh)
9146
+ self._try_update = try_update
9255
9147
 
9256
- rch = _RemoteCommandHandler(
9257
- remote_chan,
9258
- self._local_executor,
9259
- )
9260
- rch_task = asyncio.create_task(rch.run()) # noqa
9148
+ #
9149
+
9150
+ @staticmethod
9151
+ def guess_version(s: str) -> ta.Optional[InterpVersion]:
9152
+ def strip_sfx(s: str, sfx: str) -> ta.Tuple[str, bool]:
9153
+ if s.endswith(sfx):
9154
+ return s[:-len(sfx)], True
9155
+ return s, False
9156
+ ok = {}
9157
+ s, ok['debug'] = strip_sfx(s, '-debug')
9158
+ s, ok['threaded'] = strip_sfx(s, 't')
9261
9159
  try:
9262
- rce: RemoteCommandExecutor
9263
- async with aclosing(RemoteCommandExecutor(local_chan)) as rce:
9264
- await rce.start()
9160
+ v = Version(s)
9161
+ except InvalidVersion:
9162
+ return None
9163
+ return InterpVersion(v, InterpOpts(**ok))
9265
9164
 
9266
- yield rce
9165
+ class Installed(ta.NamedTuple):
9166
+ name: str
9167
+ exe: str
9168
+ version: InterpVersion
9267
9169
 
9268
- finally:
9269
- rch.stop()
9270
- await rch_task
9170
+ async def _make_installed(self, vn: str, ep: str) -> ta.Optional[Installed]:
9171
+ iv: ta.Optional[InterpVersion]
9172
+ if self._inspect:
9173
+ try:
9174
+ iv = check.not_none(await self._inspector.inspect(ep)).iv
9175
+ except Exception as e: # noqa
9176
+ return None
9177
+ else:
9178
+ iv = self.guess_version(vn)
9179
+ if iv is None:
9180
+ return None
9181
+ return PyenvInterpProvider.Installed(
9182
+ name=vn,
9183
+ exe=ep,
9184
+ version=iv,
9185
+ )
9271
9186
 
9187
+ async def installed(self) -> ta.Sequence[Installed]:
9188
+ ret: ta.List[PyenvInterpProvider.Installed] = []
9189
+ for vn, ep in await self._pyenv.version_exes():
9190
+ if (i := await self._make_installed(vn, ep)) is None:
9191
+ log.debug('Invalid pyenv version: %s', vn)
9192
+ continue
9193
+ ret.append(i)
9194
+ return ret
9272
9195
 
9273
- ########################################
9274
- # ../system/inject.py
9196
+ #
9275
9197
 
9198
+ async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
9199
+ return [i.version for i in await self.installed()]
9276
9200
 
9277
- def bind_system(
9278
- *,
9279
- system_config: SystemConfig,
9280
- ) -> InjectorBindings:
9281
- lst: ta.List[InjectorBindingOrBindings] = [
9282
- inj.bind(system_config),
9283
- ]
9201
+ async def get_installed_version(self, version: InterpVersion) -> Interp:
9202
+ for i in await self.installed():
9203
+ if i.version == version:
9204
+ return Interp(
9205
+ exe=i.exe,
9206
+ version=i.version,
9207
+ )
9208
+ raise KeyError(version)
9284
9209
 
9285
9210
  #
9286
9211
 
9287
- platform = system_config.platform or detect_system_platform()
9288
- lst.append(inj.bind(platform, key=Platform))
9212
+ async def _get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
9213
+ lst = []
9289
9214
 
9290
- #
9215
+ for vs in await self._pyenv.installable_versions():
9216
+ if (iv := self.guess_version(vs)) is None:
9217
+ continue
9218
+ if iv.opts.debug:
9219
+ raise Exception('Pyenv installable versions not expected to have debug suffix')
9220
+ for d in [False, True]:
9221
+ lst.append(dc.replace(iv, opts=dc.replace(iv.opts, debug=d)))
9291
9222
 
9292
- if isinstance(platform, AmazonLinuxPlatform):
9293
- lst.extend([
9294
- inj.bind(YumSystemPackageManager, singleton=True),
9295
- inj.bind(SystemPackageManager, to_key=YumSystemPackageManager),
9296
- ])
9223
+ return lst
9297
9224
 
9298
- elif isinstance(platform, LinuxPlatform):
9299
- lst.extend([
9300
- inj.bind(AptSystemPackageManager, singleton=True),
9301
- inj.bind(SystemPackageManager, to_key=AptSystemPackageManager),
9302
- ])
9225
+ async def get_installable_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
9226
+ lst = await self._get_installable_versions(spec)
9303
9227
 
9304
- elif isinstance(platform, DarwinPlatform):
9305
- lst.extend([
9306
- inj.bind(BrewSystemPackageManager, singleton=True),
9307
- inj.bind(SystemPackageManager, to_key=BrewSystemPackageManager),
9308
- ])
9228
+ if self._try_update and not any(v in spec for v in lst):
9229
+ if self._pyenv.update():
9230
+ lst = await self._get_installable_versions(spec)
9309
9231
 
9310
- #
9232
+ return lst
9311
9233
 
9312
- lst.extend([
9313
- bind_command(CheckSystemPackageCommand, CheckSystemPackageCommandExecutor),
9314
- ])
9234
+ async def install_version(self, version: InterpVersion) -> Interp:
9235
+ inst_version = str(version.version)
9236
+ inst_opts = version.opts
9237
+ if inst_opts.threaded:
9238
+ inst_version += 't'
9239
+ inst_opts = dc.replace(inst_opts, threaded=False)
9315
9240
 
9316
- #
9241
+ installer = PyenvVersionInstaller(
9242
+ inst_version,
9243
+ interp_opts=inst_opts,
9244
+ )
9317
9245
 
9318
- return inj.as_bindings(*lst)
9246
+ exe = await installer.install()
9247
+ return Interp(exe, version)
9319
9248
 
9320
9249
 
9321
9250
  ########################################
9322
- # ../../../omdev/interp/resolvers.py
9323
-
9324
-
9325
- INTERP_PROVIDER_TYPES_BY_NAME: ta.Mapping[str, ta.Type[InterpProvider]] = {
9326
- cls.name: cls for cls in deep_subclasses(InterpProvider) if abc.ABC not in cls.__bases__ # type: ignore
9327
- }
9251
+ # ../../../omdev/interp/system.py
9252
+ """
9253
+ TODO:
9254
+ - python, python3, python3.12, ...
9255
+ - check if path py's are venvs: sys.prefix != sys.base_prefix
9256
+ """
9328
9257
 
9329
9258
 
9330
- class InterpResolver:
9331
- def __init__(
9332
- self,
9333
- providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
9334
- ) -> None:
9335
- super().__init__()
9259
+ ##
9336
9260
 
9337
- self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
9338
9261
 
9339
- async def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
9340
- lst = [
9341
- (i, si)
9342
- for i, p in enumerate(self._providers.values())
9343
- for si in await p.get_installed_versions(spec)
9344
- if spec.contains(si)
9345
- ]
9262
+ @dc.dataclass(frozen=True)
9263
+ class SystemInterpProvider(InterpProvider):
9264
+ cmd: str = 'python3'
9265
+ path: ta.Optional[str] = None
9346
9266
 
9347
- slst = sorted(lst, key=lambda t: (-t[0], t[1].version))
9348
- if not slst:
9349
- return None
9267
+ inspect: bool = False
9268
+ inspector: InterpInspector = INTERP_INSPECTOR
9350
9269
 
9351
- bi, bv = slst[-1]
9352
- bp = list(self._providers.values())[bi]
9353
- return (bp, bv)
9270
+ #
9354
9271
 
9355
- async def resolve(
9356
- self,
9357
- spec: InterpSpecifier,
9272
+ @staticmethod
9273
+ def _re_which(
9274
+ pat: re.Pattern,
9358
9275
  *,
9359
- install: bool = False,
9360
- ) -> ta.Optional[Interp]:
9361
- tup = await self._resolve_installed(spec)
9362
- if tup is not None:
9363
- bp, bv = tup
9364
- return await bp.get_installed_version(bv)
9276
+ mode: int = os.F_OK | os.X_OK,
9277
+ path: ta.Optional[str] = None,
9278
+ ) -> ta.List[str]:
9279
+ if path is None:
9280
+ path = os.environ.get('PATH', None)
9281
+ if path is None:
9282
+ try:
9283
+ path = os.confstr('CS_PATH')
9284
+ except (AttributeError, ValueError):
9285
+ path = os.defpath
9365
9286
 
9366
- if not install:
9367
- return None
9287
+ if not path:
9288
+ return []
9368
9289
 
9369
- tp = list(self._providers.values())[0] # noqa
9290
+ path = os.fsdecode(path)
9291
+ pathlst = path.split(os.pathsep)
9370
9292
 
9371
- sv = sorted(
9372
- [s for s in await tp.get_installable_versions(spec) if s in spec],
9373
- key=lambda s: s.version,
9374
- )
9375
- if not sv:
9376
- return None
9293
+ def _access_check(fn: str, mode: int) -> bool:
9294
+ return os.path.exists(fn) and os.access(fn, mode)
9295
+
9296
+ out = []
9297
+ seen = set()
9298
+ for d in pathlst:
9299
+ normdir = os.path.normcase(d)
9300
+ if normdir not in seen:
9301
+ seen.add(normdir)
9302
+ if not _access_check(normdir, mode):
9303
+ continue
9304
+ for thefile in os.listdir(d):
9305
+ name = os.path.join(d, thefile)
9306
+ if not (
9307
+ os.path.isfile(name) and
9308
+ pat.fullmatch(thefile) and
9309
+ _access_check(name, mode)
9310
+ ):
9311
+ continue
9312
+ out.append(name)
9377
9313
 
9378
- bv = sv[-1]
9379
- return await tp.install_version(bv)
9314
+ return out
9380
9315
 
9381
- async def list(self, spec: InterpSpecifier) -> None:
9382
- print('installed:')
9383
- for n, p in self._providers.items():
9384
- lst = [
9385
- si
9386
- for si in await p.get_installed_versions(spec)
9387
- if spec.contains(si)
9388
- ]
9389
- if lst:
9390
- print(f' {n}')
9391
- for si in lst:
9392
- print(f' {si}')
9316
+ @cached_nullary
9317
+ def exes(self) -> ta.List[str]:
9318
+ return self._re_which(
9319
+ re.compile(r'python3(\.\d+)?'),
9320
+ path=self.path,
9321
+ )
9393
9322
 
9394
- print()
9323
+ #
9395
9324
 
9396
- print('installable:')
9397
- for n, p in self._providers.items():
9398
- lst = [
9399
- si
9400
- for si in await p.get_installable_versions(spec)
9401
- if spec.contains(si)
9402
- ]
9403
- if lst:
9404
- print(f' {n}')
9405
- for si in lst:
9406
- print(f' {si}')
9325
+ async def get_exe_version(self, exe: str) -> ta.Optional[InterpVersion]:
9326
+ if not self.inspect:
9327
+ s = os.path.basename(exe)
9328
+ if s.startswith('python'):
9329
+ s = s[len('python'):]
9330
+ if '.' in s:
9331
+ try:
9332
+ return InterpVersion.parse(s)
9333
+ except InvalidVersion:
9334
+ pass
9335
+ ii = await self.inspector.inspect(exe)
9336
+ return ii.iv if ii is not None else None
9407
9337
 
9338
+ async def exe_versions(self) -> ta.Sequence[ta.Tuple[str, InterpVersion]]:
9339
+ lst = []
9340
+ for e in self.exes():
9341
+ if (ev := await self.get_exe_version(e)) is None:
9342
+ log.debug('Invalid system version: %s', e)
9343
+ continue
9344
+ lst.append((e, ev))
9345
+ return lst
9408
9346
 
9409
- DEFAULT_INTERP_RESOLVER = InterpResolver([(p.name, p) for p in [
9410
- # pyenv is preferred to system interpreters as it tends to have more support for things like tkinter
9411
- PyenvInterpProvider(try_update=True),
9347
+ #
9412
9348
 
9413
- RunningInterpProvider(),
9349
+ async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
9350
+ return [ev for e, ev in await self.exe_versions()]
9414
9351
 
9415
- SystemInterpProvider(),
9416
- ]])
9352
+ async def get_installed_version(self, version: InterpVersion) -> Interp:
9353
+ for e, ev in await self.exe_versions():
9354
+ if ev != version:
9355
+ continue
9356
+ return Interp(
9357
+ exe=e,
9358
+ version=ev,
9359
+ )
9360
+ raise KeyError(version)
9417
9361
 
9418
9362
 
9419
9363
  ########################################
@@ -9444,6 +9388,54 @@ def bind_remote(
9444
9388
  return inj.as_bindings(*lst)
9445
9389
 
9446
9390
 
9391
+ ########################################
9392
+ # ../system/inject.py
9393
+
9394
+
9395
+ def bind_system(
9396
+ *,
9397
+ system_config: SystemConfig,
9398
+ ) -> InjectorBindings:
9399
+ lst: ta.List[InjectorBindingOrBindings] = [
9400
+ inj.bind(system_config),
9401
+ ]
9402
+
9403
+ #
9404
+
9405
+ platform = system_config.platform or detect_system_platform()
9406
+ lst.append(inj.bind(platform, key=Platform))
9407
+
9408
+ #
9409
+
9410
+ if isinstance(platform, AmazonLinuxPlatform):
9411
+ lst.extend([
9412
+ inj.bind(YumSystemPackageManager, singleton=True),
9413
+ inj.bind(SystemPackageManager, to_key=YumSystemPackageManager),
9414
+ ])
9415
+
9416
+ elif isinstance(platform, LinuxPlatform):
9417
+ lst.extend([
9418
+ inj.bind(AptSystemPackageManager, singleton=True),
9419
+ inj.bind(SystemPackageManager, to_key=AptSystemPackageManager),
9420
+ ])
9421
+
9422
+ elif isinstance(platform, DarwinPlatform):
9423
+ lst.extend([
9424
+ inj.bind(BrewSystemPackageManager, singleton=True),
9425
+ inj.bind(SystemPackageManager, to_key=BrewSystemPackageManager),
9426
+ ])
9427
+
9428
+ #
9429
+
9430
+ lst.extend([
9431
+ bind_command(CheckSystemPackageCommand, CheckSystemPackageCommandExecutor),
9432
+ ])
9433
+
9434
+ #
9435
+
9436
+ return inj.as_bindings(*lst)
9437
+
9438
+
9447
9439
  ########################################
9448
9440
  # ../targets/connection.py
9449
9441
 
@@ -9579,33 +9571,101 @@ class SshManageTargetConnector(ManageTargetConnector):
9579
9571
 
9580
9572
 
9581
9573
  ########################################
9582
- # ../deploy/interp.py
9574
+ # ../../../omdev/interp/resolvers.py
9583
9575
 
9584
9576
 
9585
- ##
9577
+ INTERP_PROVIDER_TYPES_BY_NAME: ta.Mapping[str, ta.Type[InterpProvider]] = {
9578
+ cls.name: cls for cls in deep_subclasses(InterpProvider) if abc.ABC not in cls.__bases__ # type: ignore
9579
+ }
9586
9580
 
9587
9581
 
9588
- @dc.dataclass(frozen=True)
9589
- class InterpCommand(Command['InterpCommand.Output']):
9590
- spec: str
9591
- install: bool = False
9582
+ class InterpResolver:
9583
+ def __init__(
9584
+ self,
9585
+ providers: ta.Sequence[ta.Tuple[str, InterpProvider]],
9586
+ ) -> None:
9587
+ super().__init__()
9592
9588
 
9593
- @dc.dataclass(frozen=True)
9594
- class Output(Command.Output):
9595
- exe: str
9596
- version: str
9597
- opts: InterpOpts
9589
+ self._providers: ta.Mapping[str, InterpProvider] = collections.OrderedDict(providers)
9598
9590
 
9591
+ async def _resolve_installed(self, spec: InterpSpecifier) -> ta.Optional[ta.Tuple[InterpProvider, InterpVersion]]:
9592
+ lst = [
9593
+ (i, si)
9594
+ for i, p in enumerate(self._providers.values())
9595
+ for si in await p.get_installed_versions(spec)
9596
+ if spec.contains(si)
9597
+ ]
9599
9598
 
9600
- class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
9601
- async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
9602
- i = InterpSpecifier.parse(check.not_none(cmd.spec))
9603
- o = check.not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
9604
- return InterpCommand.Output(
9605
- exe=o.exe,
9606
- version=str(o.version.version),
9607
- opts=o.version.opts,
9599
+ slst = sorted(lst, key=lambda t: (-t[0], t[1].version))
9600
+ if not slst:
9601
+ return None
9602
+
9603
+ bi, bv = slst[-1]
9604
+ bp = list(self._providers.values())[bi]
9605
+ return (bp, bv)
9606
+
9607
+ async def resolve(
9608
+ self,
9609
+ spec: InterpSpecifier,
9610
+ *,
9611
+ install: bool = False,
9612
+ ) -> ta.Optional[Interp]:
9613
+ tup = await self._resolve_installed(spec)
9614
+ if tup is not None:
9615
+ bp, bv = tup
9616
+ return await bp.get_installed_version(bv)
9617
+
9618
+ if not install:
9619
+ return None
9620
+
9621
+ tp = list(self._providers.values())[0] # noqa
9622
+
9623
+ sv = sorted(
9624
+ [s for s in await tp.get_installable_versions(spec) if s in spec],
9625
+ key=lambda s: s.version,
9608
9626
  )
9627
+ if not sv:
9628
+ return None
9629
+
9630
+ bv = sv[-1]
9631
+ return await tp.install_version(bv)
9632
+
9633
+ async def list(self, spec: InterpSpecifier) -> None:
9634
+ print('installed:')
9635
+ for n, p in self._providers.items():
9636
+ lst = [
9637
+ si
9638
+ for si in await p.get_installed_versions(spec)
9639
+ if spec.contains(si)
9640
+ ]
9641
+ if lst:
9642
+ print(f' {n}')
9643
+ for si in lst:
9644
+ print(f' {si}')
9645
+
9646
+ print()
9647
+
9648
+ print('installable:')
9649
+ for n, p in self._providers.items():
9650
+ lst = [
9651
+ si
9652
+ for si in await p.get_installable_versions(spec)
9653
+ if spec.contains(si)
9654
+ ]
9655
+ if lst:
9656
+ print(f' {n}')
9657
+ for si in lst:
9658
+ print(f' {si}')
9659
+
9660
+
9661
+ DEFAULT_INTERP_RESOLVER = InterpResolver([(p.name, p) for p in [
9662
+ # pyenv is preferred to system interpreters as it tends to have more support for things like tkinter
9663
+ PyenvInterpProvider(try_update=True),
9664
+
9665
+ RunningInterpProvider(),
9666
+
9667
+ SystemInterpProvider(),
9668
+ ]])
9609
9669
 
9610
9670
 
9611
9671
  ########################################
@@ -9637,6 +9697,36 @@ def bind_targets() -> InjectorBindings:
9637
9697
  return inj.as_bindings(*lst)
9638
9698
 
9639
9699
 
9700
+ ########################################
9701
+ # ../deploy/interp.py
9702
+
9703
+
9704
+ ##
9705
+
9706
+
9707
+ @dc.dataclass(frozen=True)
9708
+ class InterpCommand(Command['InterpCommand.Output']):
9709
+ spec: str
9710
+ install: bool = False
9711
+
9712
+ @dc.dataclass(frozen=True)
9713
+ class Output(Command.Output):
9714
+ exe: str
9715
+ version: str
9716
+ opts: InterpOpts
9717
+
9718
+
9719
+ class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
9720
+ async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
9721
+ i = InterpSpecifier.parse(check.not_none(cmd.spec))
9722
+ o = check.not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
9723
+ return InterpCommand.Output(
9724
+ exe=o.exe,
9725
+ version=str(o.version.version),
9726
+ opts=o.version.opts,
9727
+ )
9728
+
9729
+
9640
9730
  ########################################
9641
9731
  # ../deploy/inject.py
9642
9732