ominfra 0.0.0.dev158__py3-none-any.whl → 0.0.0.dev160__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
ominfra/scripts/manage.py CHANGED
@@ -24,6 +24,7 @@ import decimal
24
24
  import enum
25
25
  import fractions
26
26
  import functools
27
+ import hashlib
27
28
  import inspect
28
29
  import itertools
29
30
  import json
@@ -41,6 +42,7 @@ import string
41
42
  import struct
42
43
  import subprocess
43
44
  import sys
45
+ import tempfile
44
46
  import threading
45
47
  import time
46
48
  import traceback
@@ -98,6 +100,10 @@ CallableVersionOperator = ta.Callable[['Version', str], bool]
98
100
  CommandT = ta.TypeVar('CommandT', bound='Command')
99
101
  CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
100
102
 
103
+ # deploy/atomics.py
104
+ DeployAtomicPathSwapKind = ta.Literal['dir', 'file']
105
+ DeployAtomicPathSwapState = ta.Literal['open', 'committed', 'aborted'] # ta.TypeAlias
106
+
101
107
  # deploy/paths.py
102
108
  DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
103
109
  DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
@@ -1382,6 +1388,7 @@ DeployHome = ta.NewType('DeployHome', str)
1382
1388
  DeployApp = ta.NewType('DeployApp', str)
1383
1389
  DeployTag = ta.NewType('DeployTag', str)
1384
1390
  DeployRev = ta.NewType('DeployRev', str)
1391
+ DeployKey = ta.NewType('DeployKey', str)
1385
1392
 
1386
1393
 
1387
1394
  class DeployAppTag(ta.NamedTuple):
@@ -2685,6 +2692,10 @@ def is_new_type(spec: ta.Any) -> bool:
2685
2692
  return isinstance(spec, types.FunctionType) and spec.__code__ is ta.NewType.__code__.co_consts[1] # type: ignore # noqa
2686
2693
 
2687
2694
 
2695
+ def get_new_type_supertype(spec: ta.Any) -> ta.Any:
2696
+ return spec.__supertype__
2697
+
2698
+
2688
2699
  def deep_subclasses(cls: ta.Type[T]) -> ta.Iterator[ta.Type[T]]:
2689
2700
  seen = set()
2690
2701
  todo = list(reversed(cls.__subclasses__()))
@@ -4054,41 +4065,212 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
4054
4065
 
4055
4066
 
4056
4067
  ########################################
4057
- # ../deploy/paths.py
4058
- """
4059
- ~deploy
4060
- deploy.pid (flock)
4061
- /app
4062
- /<appplaceholder> - shallow clone
4063
- /conf
4064
- /env
4065
- <appplaceholder>.env
4066
- /nginx
4067
- <appplaceholder>.conf
4068
- /supervisor
4069
- <appplaceholder>.conf
4070
- /venv
4071
- /<appplaceholder>
4072
-
4073
- ?
4074
- /logs
4075
- /wrmsr--omlish--<placeholder>
4076
-
4077
- placeholder = <name>--<rev>--<when>
4078
-
4079
- ==
4080
-
4081
- for dn in [
4082
- 'app',
4083
- 'conf',
4084
- 'conf/env',
4085
- 'conf/nginx',
4086
- 'conf/supervisor',
4087
- 'venv',
4088
- ]:
4068
+ # ../deploy/atomics.py
4069
+
4070
+
4071
+ ##
4072
+
4073
+
4074
+ class DeployAtomicPathSwap(abc.ABC):
4075
+ def __init__(
4076
+ self,
4077
+ kind: DeployAtomicPathSwapKind,
4078
+ dst_path: str,
4079
+ *,
4080
+ auto_commit: bool = False,
4081
+ ) -> None:
4082
+ super().__init__()
4083
+
4084
+ self._kind = kind
4085
+ self._dst_path = dst_path
4086
+ self._auto_commit = auto_commit
4087
+
4088
+ self._state: DeployAtomicPathSwapState = 'open'
4089
+
4090
+ def __repr__(self) -> str:
4091
+ return attr_repr(self, 'kind', 'dst_path', 'tmp_path')
4092
+
4093
+ @property
4094
+ def kind(self) -> DeployAtomicPathSwapKind:
4095
+ return self._kind
4096
+
4097
+ @property
4098
+ def dst_path(self) -> str:
4099
+ return self._dst_path
4100
+
4101
+ @property
4102
+ @abc.abstractmethod
4103
+ def tmp_path(self) -> str:
4104
+ raise NotImplementedError
4105
+
4106
+ #
4107
+
4108
+ @property
4109
+ def state(self) -> DeployAtomicPathSwapState:
4110
+ return self._state
4111
+
4112
+ def _check_state(self, *states: DeployAtomicPathSwapState) -> None:
4113
+ if self._state not in states:
4114
+ raise RuntimeError(f'Atomic path swap not in correct state: {self._state}, {states}')
4115
+
4116
+ #
4117
+
4118
+ @abc.abstractmethod
4119
+ def _commit(self) -> None:
4120
+ raise NotImplementedError
4121
+
4122
+ def commit(self) -> None:
4123
+ if self._state == 'committed':
4124
+ return
4125
+ self._check_state('open')
4126
+ try:
4127
+ self._commit()
4128
+ except Exception: # noqa
4129
+ self._abort()
4130
+ raise
4131
+ else:
4132
+ self._state = 'committed'
4133
+
4134
+ #
4135
+
4136
+ @abc.abstractmethod
4137
+ def _abort(self) -> None:
4138
+ raise NotImplementedError
4139
+
4140
+ def abort(self) -> None:
4141
+ if self._state == 'aborted':
4142
+ return
4143
+ self._abort()
4144
+ self._state = 'aborted'
4145
+
4146
+ #
4147
+
4148
+ def __enter__(self) -> 'DeployAtomicPathSwap':
4149
+ return self
4150
+
4151
+ def __exit__(self, exc_type, exc_val, exc_tb):
4152
+ if (
4153
+ exc_type is None and
4154
+ self._auto_commit and
4155
+ self._state == 'open'
4156
+ ):
4157
+ self.commit()
4158
+ else:
4159
+ self.abort()
4160
+
4161
+
4162
+ #
4163
+
4164
+
4165
+ class DeployAtomicPathSwapping(abc.ABC):
4166
+ @abc.abstractmethod
4167
+ def begin_atomic_path_swap(
4168
+ self,
4169
+ kind: DeployAtomicPathSwapKind,
4170
+ dst_path: str,
4171
+ *,
4172
+ name_hint: ta.Optional[str] = None,
4173
+ make_dirs: bool = False,
4174
+ **kwargs: ta.Any,
4175
+ ) -> DeployAtomicPathSwap:
4176
+ raise NotImplementedError
4177
+
4178
+
4179
+ ##
4180
+
4181
+
4182
+ class OsRenameDeployAtomicPathSwap(DeployAtomicPathSwap):
4183
+ def __init__(
4184
+ self,
4185
+ kind: DeployAtomicPathSwapKind,
4186
+ dst_path: str,
4187
+ tmp_path: str,
4188
+ **kwargs: ta.Any,
4189
+ ) -> None:
4190
+ if kind == 'dir':
4191
+ check.state(os.path.isdir(tmp_path))
4192
+ elif kind == 'file':
4193
+ check.state(os.path.isfile(tmp_path))
4194
+ else:
4195
+ raise TypeError(kind)
4196
+
4197
+ super().__init__(
4198
+ kind,
4199
+ dst_path,
4200
+ **kwargs,
4201
+ )
4202
+
4203
+ self._tmp_path = tmp_path
4204
+
4205
+ @property
4206
+ def tmp_path(self) -> str:
4207
+ return self._tmp_path
4208
+
4209
+ def _commit(self) -> None:
4210
+ os.rename(self._tmp_path, self._dst_path)
4211
+
4212
+ def _abort(self) -> None:
4213
+ shutil.rmtree(self._tmp_path, ignore_errors=True)
4214
+
4215
+
4216
+ class TempDirDeployAtomicPathSwapping(DeployAtomicPathSwapping):
4217
+ def __init__(
4218
+ self,
4219
+ *,
4220
+ temp_dir: ta.Optional[str] = None,
4221
+ root_dir: ta.Optional[str] = None,
4222
+ ) -> None:
4223
+ super().__init__()
4224
+
4225
+ if root_dir is not None:
4226
+ root_dir = os.path.abspath(root_dir)
4227
+ self._root_dir = root_dir
4228
+ self._temp_dir = temp_dir
4229
+
4230
+ def begin_atomic_path_swap(
4231
+ self,
4232
+ kind: DeployAtomicPathSwapKind,
4233
+ dst_path: str,
4234
+ *,
4235
+ name_hint: ta.Optional[str] = None,
4236
+ make_dirs: bool = False,
4237
+ **kwargs: ta.Any,
4238
+ ) -> DeployAtomicPathSwap:
4239
+ dst_path = os.path.abspath(dst_path)
4240
+ if self._root_dir is not None and not dst_path.startswith(check.non_empty_str(self._root_dir)):
4241
+ raise RuntimeError(f'Atomic path swap dst must be in root dir: {dst_path}, {self._root_dir}')
4242
+
4243
+ dst_dir = os.path.dirname(dst_path)
4244
+ if make_dirs:
4245
+ os.makedirs(dst_dir, exist_ok=True)
4246
+ if not os.path.isdir(dst_dir):
4247
+ raise RuntimeError(f'Atomic path swap dst dir does not exist: {dst_dir}')
4248
+
4249
+ if kind == 'dir':
4250
+ tmp_path = tempfile.mkdtemp(prefix=name_hint, dir=self._temp_dir)
4251
+ elif kind == 'file':
4252
+ fd, tmp_path = tempfile.mkstemp(prefix=name_hint, dir=self._temp_dir)
4253
+ os.close(fd)
4254
+ else:
4255
+ raise TypeError(kind)
4089
4256
 
4090
- ==
4257
+ return OsRenameDeployAtomicPathSwap(
4258
+ kind,
4259
+ dst_path,
4260
+ tmp_path,
4261
+ **kwargs,
4262
+ )
4091
4263
 
4264
+
4265
+ ########################################
4266
+ # ../deploy/paths.py
4267
+ """
4268
+ TODO:
4269
+ - run/pidfile
4270
+ - logs/...
4271
+ - current symlink
4272
+ - conf/{nginx,supervisor}
4273
+ - env/?
4092
4274
  """
4093
4275
 
4094
4276
 
@@ -4100,7 +4282,7 @@ DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
4100
4282
 
4101
4283
  DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
4102
4284
  'app',
4103
- 'tag', # <rev>-<dt>
4285
+ 'tag',
4104
4286
  ])
4105
4287
 
4106
4288
 
@@ -4227,6 +4409,8 @@ class DeployPath:
4227
4409
  parts: ta.Sequence[DeployPathPart]
4228
4410
 
4229
4411
  def __post_init__(self) -> None:
4412
+ hash(self)
4413
+
4230
4414
  check.not_empty(self.parts)
4231
4415
  for p in self.parts[:-1]:
4232
4416
  check.equal(p.kind, 'dir')
@@ -4261,10 +4445,10 @@ class DeployPath:
4261
4445
  else:
4262
4446
  tail_parse = FileDeployPathPart.parse
4263
4447
  ps = check.non_empty_str(s).split('/')
4264
- return cls([
4448
+ return cls((
4265
4449
  *([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
4266
4450
  tail_parse(ps[-1]),
4267
- ])
4451
+ ))
4268
4452
 
4269
4453
 
4270
4454
  ##
@@ -4272,10 +4456,41 @@ class DeployPath:
4272
4456
 
4273
4457
  class DeployPathOwner(abc.ABC):
4274
4458
  @abc.abstractmethod
4275
- def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
4459
+ def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
4276
4460
  raise NotImplementedError
4277
4461
 
4278
4462
 
4463
+ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
4464
+ def __init__(
4465
+ self,
4466
+ *args: ta.Any,
4467
+ owned_dir: str,
4468
+ deploy_home: ta.Optional[DeployHome],
4469
+ **kwargs: ta.Any,
4470
+ ) -> None:
4471
+ super().__init__(*args, **kwargs)
4472
+
4473
+ check.not_in('/', owned_dir)
4474
+ self._owned_dir: str = check.non_empty_str(owned_dir)
4475
+
4476
+ self._deploy_home = deploy_home
4477
+
4478
+ self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
4479
+
4480
+ @cached_nullary
4481
+ def _dir(self) -> str:
4482
+ return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
4483
+
4484
+ @cached_nullary
4485
+ def _make_dir(self) -> str:
4486
+ if not os.path.isdir(d := self._dir()):
4487
+ os.makedirs(d, exist_ok=True)
4488
+ return d
4489
+
4490
+ def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
4491
+ return self._owned_deploy_paths
4492
+
4493
+
4279
4494
  ########################################
4280
4495
  # ../deploy/specs.py
4281
4496
 
@@ -4294,14 +4509,35 @@ class DeployGitRepo:
4294
4509
  check.not_in('.', check.non_empty_str(self.path))
4295
4510
 
4296
4511
 
4512
+ @dc.dataclass(frozen=True)
4513
+ class DeployGitCheckout:
4514
+ repo: DeployGitRepo
4515
+ rev: DeployRev
4516
+
4517
+ subtrees: ta.Optional[ta.Sequence[str]] = None
4518
+
4519
+ def __post_init__(self) -> None:
4520
+ hash(self)
4521
+ check.non_empty_str(self.rev)
4522
+ if self.subtrees is not None:
4523
+ for st in self.subtrees:
4524
+ check.non_empty_str(st)
4525
+
4526
+
4297
4527
  ##
4298
4528
 
4299
4529
 
4300
4530
  @dc.dataclass(frozen=True)
4301
4531
  class DeploySpec:
4302
4532
  app: DeployApp
4303
- repo: DeployGitRepo
4304
- rev: DeployRev
4533
+ checkout: DeployGitCheckout
4534
+
4535
+ def __post_init__(self) -> None:
4536
+ hash(self)
4537
+
4538
+ @cached_nullary
4539
+ def key(self) -> DeployKey:
4540
+ return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])
4305
4541
 
4306
4542
 
4307
4543
  ########################################
@@ -5761,9 +5997,7 @@ inj = Injection
5761
5997
  """
5762
5998
  TODO:
5763
5999
  - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
5764
- - namedtuple
5765
6000
  - literals
5766
- - newtypes?
5767
6001
  """
5768
6002
 
5769
6003
 
@@ -5773,7 +6007,7 @@ TODO:
5773
6007
  @dc.dataclass(frozen=True)
5774
6008
  class ObjMarshalOptions:
5775
6009
  raw_bytes: bool = False
5776
- nonstrict_dataclasses: bool = False
6010
+ non_strict_fields: bool = False
5777
6011
 
5778
6012
 
5779
6013
  class ObjMarshaler(abc.ABC):
@@ -5902,10 +6136,10 @@ class IterableObjMarshaler(ObjMarshaler):
5902
6136
 
5903
6137
 
5904
6138
  @dc.dataclass(frozen=True)
5905
- class DataclassObjMarshaler(ObjMarshaler):
6139
+ class FieldsObjMarshaler(ObjMarshaler):
5906
6140
  ty: type
5907
6141
  fs: ta.Mapping[str, ObjMarshaler]
5908
- nonstrict: bool = False
6142
+ non_strict: bool = False
5909
6143
 
5910
6144
  def marshal(self, o: ta.Any, ctx: 'ObjMarshalContext') -> ta.Any:
5911
6145
  return {
@@ -5917,7 +6151,7 @@ class DataclassObjMarshaler(ObjMarshaler):
5917
6151
  return self.ty(**{
5918
6152
  k: self.fs[k].unmarshal(v, ctx)
5919
6153
  for k, v in o.items()
5920
- if not (self.nonstrict or ctx.options.nonstrict_dataclasses) or k in self.fs
6154
+ if not (self.non_strict or ctx.options.non_strict_fields) or k in self.fs
5921
6155
  })
5922
6156
 
5923
6157
 
@@ -6049,7 +6283,7 @@ class ObjMarshalerManager:
6049
6283
  ty: ta.Any,
6050
6284
  rec: ta.Callable[[ta.Any], ObjMarshaler],
6051
6285
  *,
6052
- nonstrict_dataclasses: bool = False,
6286
+ non_strict_fields: bool = False,
6053
6287
  ) -> ObjMarshaler:
6054
6288
  if isinstance(ty, type):
6055
6289
  if abc.ABC in ty.__bases__:
@@ -6071,12 +6305,22 @@ class ObjMarshalerManager:
6071
6305
  return EnumObjMarshaler(ty)
6072
6306
 
6073
6307
  if dc.is_dataclass(ty):
6074
- return DataclassObjMarshaler(
6308
+ return FieldsObjMarshaler(
6075
6309
  ty,
6076
6310
  {f.name: rec(f.type) for f in dc.fields(ty)},
6077
- nonstrict=nonstrict_dataclasses,
6311
+ non_strict=non_strict_fields,
6078
6312
  )
6079
6313
 
6314
+ if issubclass(ty, tuple) and hasattr(ty, '_fields'):
6315
+ return FieldsObjMarshaler(
6316
+ ty,
6317
+ {p.name: rec(p.annotation) for p in inspect.signature(ty).parameters.values()},
6318
+ non_strict=non_strict_fields,
6319
+ )
6320
+
6321
+ if is_new_type(ty):
6322
+ return rec(get_new_type_supertype(ty))
6323
+
6080
6324
  if is_generic_alias(ty):
6081
6325
  try:
6082
6326
  mt = self._generic_mapping_types[ta.get_origin(ty)]
@@ -6210,12 +6454,12 @@ def is_debugger_attached() -> bool:
6210
6454
  return any(frame[1].endswith('pydevd.py') for frame in inspect.stack())
6211
6455
 
6212
6456
 
6213
- REQUIRED_PYTHON_VERSION = (3, 8)
6457
+ LITE_REQUIRED_PYTHON_VERSION = (3, 8)
6214
6458
 
6215
6459
 
6216
- def check_runtime_version() -> None:
6217
- if sys.version_info < REQUIRED_PYTHON_VERSION:
6218
- raise OSError(f'Requires python {REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
6460
+ def check_lite_runtime_version() -> None:
6461
+ if sys.version_info < LITE_REQUIRED_PYTHON_VERSION:
6462
+ raise OSError(f'Requires python {LITE_REQUIRED_PYTHON_VERSION}, got {sys.version_info} from {sys.executable}') # noqa
6219
6463
 
6220
6464
 
6221
6465
  ########################################
@@ -6503,24 +6747,41 @@ CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command
6503
6747
 
6504
6748
 
6505
6749
  ########################################
6506
- # ../deploy/commands.py
6507
-
6508
-
6509
- ##
6750
+ # ../deploy/tmp.py
6510
6751
 
6511
6752
 
6512
- @dc.dataclass(frozen=True)
6513
- class DeployCommand(Command['DeployCommand.Output']):
6514
- @dc.dataclass(frozen=True)
6515
- class Output(Command.Output):
6516
- pass
6517
-
6753
+ class DeployTmpManager(
6754
+ SingleDirDeployPathOwner,
6755
+ DeployAtomicPathSwapping,
6756
+ ):
6757
+ def __init__(
6758
+ self,
6759
+ *,
6760
+ deploy_home: ta.Optional[DeployHome] = None,
6761
+ ) -> None:
6762
+ super().__init__(
6763
+ owned_dir='tmp',
6764
+ deploy_home=deploy_home,
6765
+ )
6518
6766
 
6519
- class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
6520
- async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
6521
- log.info('Deploying!')
6767
+ @cached_nullary
6768
+ def _swapping(self) -> DeployAtomicPathSwapping:
6769
+ return TempDirDeployAtomicPathSwapping(
6770
+ temp_dir=self._make_dir(),
6771
+ root_dir=check.non_empty_str(self._deploy_home),
6772
+ )
6522
6773
 
6523
- return DeployCommand.Output()
6774
+ def begin_atomic_path_swap(
6775
+ self,
6776
+ kind: DeployAtomicPathSwapKind,
6777
+ dst_path: str,
6778
+ **kwargs: ta.Any,
6779
+ ) -> DeployAtomicPathSwap:
6780
+ return self._swapping().begin_atomic_path_swap(
6781
+ kind,
6782
+ dst_path,
6783
+ **kwargs,
6784
+ )
6524
6785
 
6525
6786
 
6526
6787
  ########################################
@@ -6630,6 +6891,7 @@ TODO:
6630
6891
  - structured
6631
6892
  - prefixed
6632
6893
  - debug
6894
+ - optional noisy? noisy will never be lite - some kinda configure_standard callback mechanism?
6633
6895
  """
6634
6896
 
6635
6897
 
@@ -6666,8 +6928,9 @@ class StandardLogFormatter(logging.Formatter):
6666
6928
  ##
6667
6929
 
6668
6930
 
6669
- class StandardLogHandler(ProxyLogHandler):
6670
- pass
6931
+ class StandardConfiguredLogHandler(ProxyLogHandler):
6932
+ def __init_subclass__(cls, **kwargs):
6933
+ raise TypeError('This class serves only as a marker and should not be subclassed.')
6671
6934
 
6672
6935
 
6673
6936
  ##
@@ -6698,7 +6961,7 @@ def configure_standard_logging(
6698
6961
  target: ta.Optional[logging.Logger] = None,
6699
6962
  force: bool = False,
6700
6963
  handler_factory: ta.Optional[ta.Callable[[], logging.Handler]] = None,
6701
- ) -> ta.Optional[StandardLogHandler]:
6964
+ ) -> ta.Optional[StandardConfiguredLogHandler]:
6702
6965
  with _locking_logging_module_lock():
6703
6966
  if target is None:
6704
6967
  target = logging.root
@@ -6706,7 +6969,7 @@ def configure_standard_logging(
6706
6969
  #
6707
6970
 
6708
6971
  if not force:
6709
- if any(isinstance(h, StandardLogHandler) for h in list(target.handlers)):
6972
+ if any(isinstance(h, StandardConfiguredLogHandler) for h in list(target.handlers)):
6710
6973
  return None
6711
6974
 
6712
6975
  #
@@ -6740,7 +7003,7 @@ def configure_standard_logging(
6740
7003
 
6741
7004
  #
6742
7005
 
6743
- return StandardLogHandler(handler)
7006
+ return StandardConfiguredLogHandler(handler)
6744
7007
 
6745
7008
 
6746
7009
  ########################################
@@ -7907,27 +8170,22 @@ github.com/wrmsr/omlish@rev
7907
8170
  ##
7908
8171
 
7909
8172
 
7910
- class DeployGitManager(DeployPathOwner):
8173
+ class DeployGitManager(SingleDirDeployPathOwner):
7911
8174
  def __init__(
7912
8175
  self,
7913
8176
  *,
7914
8177
  deploy_home: ta.Optional[DeployHome] = None,
8178
+ atomics: DeployAtomicPathSwapping,
7915
8179
  ) -> None:
7916
- super().__init__()
8180
+ super().__init__(
8181
+ owned_dir='git',
8182
+ deploy_home=deploy_home,
8183
+ )
7917
8184
 
7918
- self._deploy_home = deploy_home
8185
+ self._atomics = atomics
7919
8186
 
7920
8187
  self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
7921
8188
 
7922
- @cached_nullary
7923
- def _dir(self) -> str:
7924
- return os.path.join(check.non_empty_str(self._deploy_home), 'git')
7925
-
7926
- def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
7927
- return {
7928
- DeployPath.parse('git'),
7929
- }
7930
-
7931
8189
  class RepoDir:
7932
8190
  def __init__(
7933
8191
  self,
@@ -7939,7 +8197,7 @@ class DeployGitManager(DeployPathOwner):
7939
8197
  self._git = git
7940
8198
  self._repo = repo
7941
8199
  self._dir = os.path.join(
7942
- self._git._dir(), # noqa
8200
+ self._git._make_dir(), # noqa
7943
8201
  check.non_empty_str(repo.host),
7944
8202
  check.non_empty_str(repo.path),
7945
8203
  )
@@ -7955,12 +8213,16 @@ class DeployGitManager(DeployPathOwner):
7955
8213
  else:
7956
8214
  return f'https://{self._repo.host}/{self._repo.path}'
7957
8215
 
8216
+ #
8217
+
7958
8218
  async def _call(self, *cmd: str) -> None:
7959
8219
  await asyncio_subprocesses.check_call(
7960
8220
  *cmd,
7961
8221
  cwd=self._dir,
7962
8222
  )
7963
8223
 
8224
+ #
8225
+
7964
8226
  @async_cached_nullary
7965
8227
  async def init(self) -> None:
7966
8228
  os.makedirs(self._dir, exist_ok=True)
@@ -7974,20 +8236,24 @@ class DeployGitManager(DeployPathOwner):
7974
8236
  await self.init()
7975
8237
  await self._call('git', 'fetch', '--depth=1', 'origin', rev)
7976
8238
 
7977
- async def checkout(self, rev: DeployRev, dst_dir: str) -> None:
7978
- check.state(not os.path.exists(dst_dir))
7979
-
7980
- await self.fetch(rev)
8239
+ #
7981
8240
 
7982
- # FIXME: temp dir swap
7983
- os.makedirs(dst_dir)
8241
+ async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
8242
+ check.state(not os.path.exists(dst_dir))
8243
+ with self._git._atomics.begin_atomic_path_swap( # noqa
8244
+ 'dir',
8245
+ dst_dir,
8246
+ auto_commit=True,
8247
+ make_dirs=True,
8248
+ ) as dst_swap:
8249
+ await self.fetch(checkout.rev)
7984
8250
 
7985
- dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_dir)
7986
- await dst_call('git', 'init')
8251
+ dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_swap.tmp_path)
8252
+ await dst_call('git', 'init')
7987
8253
 
7988
- await dst_call('git', 'remote', 'add', 'local', self._dir)
7989
- await dst_call('git', 'fetch', '--depth=1', 'local', rev)
7990
- await dst_call('git', 'checkout', rev)
8254
+ await dst_call('git', 'remote', 'add', 'local', self._dir)
8255
+ await dst_call('git', 'fetch', '--depth=1', 'local', checkout.rev)
8256
+ await dst_call('git', 'checkout', checkout.rev, *(checkout.subtrees or []))
7991
8257
 
7992
8258
  def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
7993
8259
  try:
@@ -7996,8 +8262,8 @@ class DeployGitManager(DeployPathOwner):
7996
8262
  repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
7997
8263
  return repo_dir
7998
8264
 
7999
- async def checkout(self, repo: DeployGitRepo, rev: DeployRev, dst_dir: str) -> None:
8000
- await self.get_repo_dir(repo).checkout(rev, dst_dir)
8265
+ async def checkout(self, checkout: DeployGitCheckout, dst_dir: str) -> None:
8266
+ await self.get_repo_dir(checkout.repo).checkout(checkout, dst_dir)
8001
8267
 
8002
8268
 
8003
8269
  ########################################
@@ -8014,16 +8280,18 @@ class DeployVenvManager(DeployPathOwner):
8014
8280
  self,
8015
8281
  *,
8016
8282
  deploy_home: ta.Optional[DeployHome] = None,
8283
+ atomics: DeployAtomicPathSwapping,
8017
8284
  ) -> None:
8018
8285
  super().__init__()
8019
8286
 
8020
8287
  self._deploy_home = deploy_home
8288
+ self._atomics = atomics
8021
8289
 
8022
8290
  @cached_nullary
8023
8291
  def _dir(self) -> str:
8024
8292
  return os.path.join(check.non_empty_str(self._deploy_home), 'venvs')
8025
8293
 
8026
- def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
8294
+ def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
8027
8295
  return {
8028
8296
  DeployPath.parse('venvs/@app/@tag/'),
8029
8297
  }
@@ -8037,6 +8305,8 @@ class DeployVenvManager(DeployPathOwner):
8037
8305
  ) -> None:
8038
8306
  sys_exe = 'python3'
8039
8307
 
8308
+ # !! NOTE: (most) venvs cannot be relocated, so an atomic swap can't be used. it's up to the path manager to
8309
+ # garbage collect orphaned dirs.
8040
8310
  await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
8041
8311
 
8042
8312
  #
@@ -8594,13 +8864,15 @@ def bind_commands(
8594
8864
 
8595
8865
  def make_deploy_tag(
8596
8866
  rev: DeployRev,
8597
- now: ta.Optional[datetime.datetime] = None,
8867
+ key: DeployKey,
8868
+ *,
8869
+ utcnow: ta.Optional[datetime.datetime] = None,
8598
8870
  ) -> DeployTag:
8599
- if now is None:
8600
- now = datetime.datetime.utcnow() # noqa
8601
- now_fmt = '%Y%m%dT%H%M%S'
8602
- now_str = now.strftime(now_fmt)
8603
- return DeployTag('-'.join([now_str, rev]))
8871
+ if utcnow is None:
8872
+ utcnow = datetime.datetime.now(tz=datetime.timezone.utc) # noqa
8873
+ now_fmt = '%Y%m%dT%H%M%SZ'
8874
+ now_str = utcnow.strftime(now_fmt)
8875
+ return DeployTag('-'.join([now_str, rev, key]))
8604
8876
 
8605
8877
 
8606
8878
  class DeployAppManager(DeployPathOwner):
@@ -8621,7 +8893,7 @@ class DeployAppManager(DeployPathOwner):
8621
8893
  def _dir(self) -> str:
8622
8894
  return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
8623
8895
 
8624
- def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
8896
+ def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
8625
8897
  return {
8626
8898
  DeployPath.parse('apps/@app/@tag'),
8627
8899
  }
@@ -8629,15 +8901,14 @@ class DeployAppManager(DeployPathOwner):
8629
8901
  async def prepare_app(
8630
8902
  self,
8631
8903
  spec: DeploySpec,
8632
- ):
8633
- app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.rev))
8904
+ ) -> None:
8905
+ app_tag = DeployAppTag(spec.app, make_deploy_tag(spec.checkout.rev, spec.key()))
8634
8906
  app_dir = os.path.join(self._dir(), spec.app, app_tag.tag)
8635
8907
 
8636
8908
  #
8637
8909
 
8638
8910
  await self._git.checkout(
8639
- spec.repo,
8640
- spec.rev,
8911
+ spec.checkout,
8641
8912
  app_dir,
8642
8913
  )
8643
8914
 
@@ -9360,6 +9631,34 @@ class SystemInterpProvider(InterpProvider):
9360
9631
  raise KeyError(version)
9361
9632
 
9362
9633
 
9634
+ ########################################
9635
+ # ../deploy/commands.py
9636
+
9637
+
9638
+ ##
9639
+
9640
+
9641
+ @dc.dataclass(frozen=True)
9642
+ class DeployCommand(Command['DeployCommand.Output']):
9643
+ spec: DeploySpec
9644
+
9645
+ @dc.dataclass(frozen=True)
9646
+ class Output(Command.Output):
9647
+ pass
9648
+
9649
+
9650
+ @dc.dataclass(frozen=True)
9651
+ class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
9652
+ _apps: DeployAppManager
9653
+
9654
+ async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
9655
+ log.info('Deploying! %r', cmd.spec)
9656
+
9657
+ await self._apps.prepare_app(cmd.spec)
9658
+
9659
+ return DeployCommand.Output()
9660
+
9661
+
9363
9662
  ########################################
9364
9663
  # ../remote/inject.py
9365
9664
 
@@ -9738,10 +10037,19 @@ def bind_deploy(
9738
10037
  lst: ta.List[InjectorBindingOrBindings] = [
9739
10038
  inj.bind(deploy_config),
9740
10039
 
10040
+ #
10041
+
9741
10042
  inj.bind(DeployAppManager, singleton=True),
10043
+
9742
10044
  inj.bind(DeployGitManager, singleton=True),
10045
+
10046
+ inj.bind(DeployTmpManager, singleton=True),
10047
+ inj.bind(DeployAtomicPathSwapping, to_key=DeployTmpManager),
10048
+
9743
10049
  inj.bind(DeployVenvManager, singleton=True),
9744
10050
 
10051
+ #
10052
+
9745
10053
  bind_command(DeployCommand, DeployCommandExecutor),
9746
10054
  bind_command(InterpCommand, InterpCommandExecutor),
9747
10055
  ]