ominfra 0.0.0.dev167__py3-none-any.whl → 0.0.0.dev169__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
ominfra/scripts/manage.py CHANGED
@@ -100,9 +100,9 @@ CallableVersionOperator = ta.Callable[['Version', str], bool]
100
100
  CommandT = ta.TypeVar('CommandT', bound='Command')
101
101
  CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
102
102
 
103
- # deploy/paths.py
103
+ # deploy/types.py
104
104
  DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
105
- DeployPathPlaceholder = ta.Literal['app', 'tag'] # ta.TypeAlias
105
+ DeployPathPlaceholder = ta.Literal['app', 'tag', 'conf'] # ta.TypeAlias
106
106
 
107
107
  # ../../omlish/argparse/cli.py
108
108
  ArgparseCommandFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
@@ -1379,23 +1379,6 @@ class DeployConfig:
1379
1379
  deploy_home: ta.Optional[str] = None
1380
1380
 
1381
1381
 
1382
- ########################################
1383
- # ../deploy/types.py
1384
-
1385
-
1386
- DeployHome = ta.NewType('DeployHome', str)
1387
-
1388
- DeployApp = ta.NewType('DeployApp', str)
1389
- DeployTag = ta.NewType('DeployTag', str)
1390
- DeployRev = ta.NewType('DeployRev', str)
1391
- DeployKey = ta.NewType('DeployKey', str)
1392
-
1393
-
1394
- class DeployAppTag(ta.NamedTuple):
1395
- app: DeployApp
1396
- tag: DeployTag
1397
-
1398
-
1399
1382
  ########################################
1400
1383
  # ../../pyremote.py
1401
1384
  """
@@ -2074,6 +2057,8 @@ def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
2074
2057
  """
2075
2058
  TODO:
2076
2059
  - def maybe(v: lang.Maybe[T])
2060
+ - def not_ ?
2061
+ - ** class @dataclass Raise - user message should be able to be an exception type or instance or factory
2077
2062
  """
2078
2063
 
2079
2064
 
@@ -2738,6 +2723,28 @@ def strip_with_newline(s: str) -> str:
2738
2723
  return s.strip() + '\n'
2739
2724
 
2740
2725
 
2726
+ @ta.overload
2727
+ def split_keep_delimiter(s: str, d: str) -> str:
2728
+ ...
2729
+
2730
+
2731
+ @ta.overload
2732
+ def split_keep_delimiter(s: bytes, d: bytes) -> bytes:
2733
+ ...
2734
+
2735
+
2736
+ def split_keep_delimiter(s, d):
2737
+ ps = []
2738
+ i = 0
2739
+ while i < len(s):
2740
+ if (n := s.find(d, i)) < i:
2741
+ ps.append(s[i:])
2742
+ break
2743
+ ps.append(s[i:n + 1])
2744
+ i = n + 1
2745
+ return ps
2746
+
2747
+
2741
2748
  ##
2742
2749
 
2743
2750
 
@@ -3422,8 +3429,12 @@ def relative_symlink(
3422
3429
  *,
3423
3430
  target_is_directory: bool = False,
3424
3431
  dir_fd: ta.Optional[int] = None,
3432
+ make_dirs: bool = False,
3425
3433
  **kwargs: ta.Any,
3426
3434
  ) -> None:
3435
+ if make_dirs:
3436
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
3437
+
3427
3438
  os.symlink(
3428
3439
  os.path.relpath(src, os.path.dirname(dst)),
3429
3440
  dst,
@@ -4106,531 +4117,201 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
4106
4117
 
4107
4118
 
4108
4119
  ########################################
4109
- # ../deploy/paths.py
4110
- """
4111
- TODO:
4112
- - run/{.pid,.sock}
4113
- - logs/...
4114
- - current symlink
4115
- - conf/{nginx,supervisor}
4116
- - env/?
4117
- - apps/<app>/shared
4118
- """
4120
+ # ../deploy/types.py
4119
4121
 
4120
4122
 
4121
4123
  ##
4122
4124
 
4123
4125
 
4124
- DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER = '@'
4125
- DEPLOY_PATH_PLACEHOLDER_SEPARATORS = '-.'
4126
+ DeployHome = ta.NewType('DeployHome', str)
4126
4127
 
4127
- DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
4128
- 'app',
4129
- 'tag',
4130
- ])
4128
+ DeployApp = ta.NewType('DeployApp', str)
4129
+ DeployTag = ta.NewType('DeployTag', str)
4130
+ DeployRev = ta.NewType('DeployRev', str)
4131
+ DeployKey = ta.NewType('DeployKey', str)
4131
4132
 
4132
4133
 
4133
- class DeployPathError(Exception):
4134
- pass
4134
+ ##
4135
4135
 
4136
4136
 
4137
4137
  @dc.dataclass(frozen=True)
4138
- class DeployPathPart(abc.ABC): # noqa
4139
- @property
4140
- @abc.abstractmethod
4141
- def kind(self) -> DeployPathKind:
4142
- raise NotImplementedError
4138
+ class DeployAppTag:
4139
+ app: DeployApp
4140
+ tag: DeployTag
4143
4141
 
4144
- @abc.abstractmethod
4145
- def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4146
- raise NotImplementedError
4142
+ def __post_init__(self) -> None:
4143
+ for s in [self.app, self.tag]:
4144
+ check.non_empty_str(s)
4145
+ check.equal(s, s.strip())
4147
4146
 
4147
+ def placeholders(self) -> ta.Mapping[DeployPathPlaceholder, str]:
4148
+ return {
4149
+ 'app': self.app,
4150
+ 'tag': self.tag,
4151
+ }
4148
4152
 
4149
- #
4150
4153
 
4154
+ ########################################
4155
+ # ../remote/config.py
4151
4156
 
4152
- class DirDeployPathPart(DeployPathPart, abc.ABC):
4153
- @property
4154
- def kind(self) -> DeployPathKind:
4155
- return 'dir'
4156
4157
 
4157
- @classmethod
4158
- def parse(cls, s: str) -> 'DirDeployPathPart':
4159
- if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
4160
- check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
4161
- return PlaceholderDirDeployPathPart(s[1:])
4162
- else:
4163
- return ConstDirDeployPathPart(s)
4158
+ @dc.dataclass(frozen=True)
4159
+ class RemoteConfig:
4160
+ payload_file: ta.Optional[str] = None
4164
4161
 
4162
+ set_pgid: bool = True
4165
4163
 
4166
- class FileDeployPathPart(DeployPathPart, abc.ABC):
4167
- @property
4168
- def kind(self) -> DeployPathKind:
4169
- return 'file'
4164
+ deathsig: ta.Optional[str] = 'KILL'
4170
4165
 
4171
- @classmethod
4172
- def parse(cls, s: str) -> 'FileDeployPathPart':
4173
- if DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER in s:
4174
- check.equal(s[0], DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER)
4175
- if not any(c in s for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS):
4176
- return PlaceholderFileDeployPathPart(s[1:], '')
4177
- else:
4178
- p = min(f for c in DEPLOY_PATH_PLACEHOLDER_SEPARATORS if (f := s.find(c)) > 0)
4179
- return PlaceholderFileDeployPathPart(s[1:p], s[p:])
4180
- else:
4181
- return ConstFileDeployPathPart(s)
4166
+ pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
4182
4167
 
4168
+ forward_logging: bool = True
4183
4169
 
4184
- #
4170
+ timebomb_delay_s: ta.Optional[float] = 60 * 60.
4185
4171
 
4172
+ heartbeat_interval_s: float = 3.
4186
4173
 
4187
- @dc.dataclass(frozen=True)
4188
- class ConstDeployPathPart(DeployPathPart, abc.ABC):
4189
- name: str
4190
4174
 
4191
- def __post_init__(self) -> None:
4192
- check.non_empty_str(self.name)
4193
- check.not_in('/', self.name)
4194
- check.not_in(DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, self.name)
4175
+ ########################################
4176
+ # ../remote/payload.py
4195
4177
 
4196
- def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4197
- return self.name
4198
4178
 
4179
+ RemoteExecutionPayloadFile = ta.NewType('RemoteExecutionPayloadFile', str)
4199
4180
 
4200
- class ConstDirDeployPathPart(ConstDeployPathPart, DirDeployPathPart):
4201
- pass
4202
4181
 
4182
+ @cached_nullary
4183
+ def _get_self_src() -> str:
4184
+ return inspect.getsource(sys.modules[__name__])
4203
4185
 
4204
- class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
4205
- pass
4206
4186
 
4187
+ def _is_src_amalg(src: str) -> bool:
4188
+ for l in src.splitlines(): # noqa
4189
+ if l.startswith('# @omlish-amalg-output '):
4190
+ return True
4191
+ return False
4207
4192
 
4208
- #
4209
4193
 
4194
+ @cached_nullary
4195
+ def _is_self_amalg() -> bool:
4196
+ return _is_src_amalg(_get_self_src())
4210
4197
 
4211
- @dc.dataclass(frozen=True)
4212
- class PlaceholderDeployPathPart(DeployPathPart, abc.ABC):
4213
- placeholder: str # DeployPathPlaceholder
4214
4198
 
4215
- def __post_init__(self) -> None:
4216
- check.non_empty_str(self.placeholder)
4217
- for c in [*DEPLOY_PATH_PLACEHOLDER_SEPARATORS, DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
4218
- check.not_in(c, self.placeholder)
4219
- check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
4199
+ def get_remote_payload_src(
4200
+ *,
4201
+ file: ta.Optional[RemoteExecutionPayloadFile],
4202
+ ) -> str:
4203
+ if file is not None:
4204
+ with open(file) as f:
4205
+ return f.read()
4220
4206
 
4221
- def _render_placeholder(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4222
- if placeholders is not None:
4223
- return placeholders[self.placeholder] # type: ignore
4224
- else:
4225
- return DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER + self.placeholder
4207
+ if _is_self_amalg():
4208
+ return _get_self_src()
4226
4209
 
4210
+ import importlib.resources
4211
+ return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
4227
4212
 
4228
- @dc.dataclass(frozen=True)
4229
- class PlaceholderDirDeployPathPart(PlaceholderDeployPathPart, DirDeployPathPart):
4230
- def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4231
- return self._render_placeholder(placeholders)
4232
4213
 
4214
+ ########################################
4215
+ # ../system/platforms.py
4233
4216
 
4234
- @dc.dataclass(frozen=True)
4235
- class PlaceholderFileDeployPathPart(PlaceholderDeployPathPart, FileDeployPathPart):
4236
- suffix: str
4237
4217
 
4238
- def __post_init__(self) -> None:
4239
- super().__post_init__()
4240
- if self.suffix:
4241
- for c in [DEPLOY_PATH_PLACEHOLDER_PLACEHOLDER, '/']:
4242
- check.not_in(c, self.suffix)
4218
+ ##
4243
4219
 
4244
- def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4245
- return self._render_placeholder(placeholders) + self.suffix
4246
4220
 
4221
+ @dc.dataclass(frozen=True)
4222
+ class Platform(abc.ABC): # noqa
4223
+ pass
4247
4224
 
4248
- ##
4249
4225
 
4226
+ class LinuxPlatform(Platform, abc.ABC):
4227
+ pass
4250
4228
 
4251
- @dc.dataclass(frozen=True)
4252
- class DeployPath:
4253
- parts: ta.Sequence[DeployPathPart]
4254
4229
 
4255
- def __post_init__(self) -> None:
4256
- hash(self)
4230
+ class UbuntuPlatform(LinuxPlatform):
4231
+ pass
4257
4232
 
4258
- check.not_empty(self.parts)
4259
- for p in self.parts[:-1]:
4260
- check.equal(p.kind, 'dir')
4261
4233
 
4262
- pd = {}
4263
- for i, p in enumerate(self.parts):
4264
- if isinstance(p, PlaceholderDeployPathPart):
4265
- if p.placeholder in pd:
4266
- raise DeployPathError('Duplicate placeholders in path', self)
4267
- pd[p.placeholder] = i
4234
+ class AmazonLinuxPlatform(LinuxPlatform):
4235
+ pass
4268
4236
 
4269
- if 'tag' in pd:
4270
- if 'app' not in pd or pd['app'] >= pd['tag']:
4271
- raise DeployPathError('Tag placeholder in path without preceding app', self)
4272
4237
 
4273
- @property
4274
- def kind(self) -> ta.Literal['file', 'dir']:
4275
- return self.parts[-1].kind
4238
+ class GenericLinuxPlatform(LinuxPlatform):
4239
+ pass
4276
4240
 
4277
- def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
4278
- return os.path.join( # noqa
4279
- *[p.render(placeholders) for p in self.parts],
4280
- *([''] if self.kind == 'dir' else []),
4281
- )
4282
4241
 
4283
- @classmethod
4284
- def parse(cls, s: str) -> 'DeployPath':
4285
- tail_parse: ta.Callable[[str], DeployPathPart]
4286
- if s.endswith('/'):
4287
- tail_parse = DirDeployPathPart.parse
4288
- s = s[:-1]
4289
- else:
4290
- tail_parse = FileDeployPathPart.parse
4291
- ps = check.non_empty_str(s).split('/')
4292
- return cls((
4293
- *([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
4294
- tail_parse(ps[-1]),
4295
- ))
4242
+ class DarwinPlatform(Platform):
4243
+ pass
4296
4244
 
4297
4245
 
4298
- ##
4246
+ class UnknownPlatform(Platform):
4247
+ pass
4299
4248
 
4300
4249
 
4301
- class DeployPathOwner(abc.ABC):
4302
- @abc.abstractmethod
4303
- def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
4304
- raise NotImplementedError
4250
+ ##
4305
4251
 
4306
4252
 
4307
- DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
4253
+ def _detect_system_platform() -> Platform:
4254
+ plat = sys.platform
4308
4255
 
4256
+ if plat == 'linux':
4257
+ if (osr := LinuxOsRelease.read()) is None:
4258
+ return GenericLinuxPlatform()
4309
4259
 
4310
- class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
4311
- def __init__(
4312
- self,
4313
- *args: ta.Any,
4314
- owned_dir: str,
4315
- deploy_home: ta.Optional[DeployHome],
4316
- **kwargs: ta.Any,
4317
- ) -> None:
4318
- super().__init__(*args, **kwargs)
4260
+ if osr.id == 'amzn':
4261
+ return AmazonLinuxPlatform()
4319
4262
 
4320
- check.not_in('/', owned_dir)
4321
- self._owned_dir: str = check.non_empty_str(owned_dir)
4263
+ elif osr.id == 'ubuntu':
4264
+ return UbuntuPlatform()
4322
4265
 
4323
- self._deploy_home = deploy_home
4266
+ else:
4267
+ return GenericLinuxPlatform()
4324
4268
 
4325
- self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
4269
+ elif plat == 'darwin':
4270
+ return DarwinPlatform()
4326
4271
 
4327
- @cached_nullary
4328
- def _dir(self) -> str:
4329
- return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
4272
+ else:
4273
+ return UnknownPlatform()
4330
4274
 
4331
- @cached_nullary
4332
- def _make_dir(self) -> str:
4333
- if not os.path.isdir(d := self._dir()):
4334
- os.makedirs(d, exist_ok=True)
4335
- return d
4336
4275
 
4337
- def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
4338
- return self._owned_deploy_paths
4276
+ @cached_nullary
4277
+ def detect_system_platform() -> Platform:
4278
+ platform = _detect_system_platform()
4279
+ log.info('Detected platform: %r', platform)
4280
+ return platform
4339
4281
 
4340
4282
 
4341
4283
  ########################################
4342
- # ../deploy/specs.py
4284
+ # ../targets/targets.py
4285
+ """
4286
+ It's desugaring. Subprocess and locals are only leafs. Retain an origin?
4287
+ ** TWO LAYERS ** - ManageTarget is user-facing, ConnectorTarget is bound, internal
4288
+ """
4343
4289
 
4344
4290
 
4345
4291
  ##
4346
4292
 
4347
4293
 
4348
- def check_valid_deploy_spec_path(s: str) -> str:
4349
- check.non_empty_str(s)
4350
- for c in ['..', '//']:
4351
- check.not_in(c, s)
4352
- check.arg(not s.startswith('/'))
4353
- return s
4294
+ class ManageTarget(abc.ABC): # noqa
4295
+ def __init_subclass__(cls, **kwargs: ta.Any) -> None:
4296
+ super().__init_subclass__(**kwargs)
4354
4297
 
4298
+ check.state(cls.__name__.endswith('ManageTarget'))
4355
4299
 
4356
- ##
4300
+
4301
+ #
4357
4302
 
4358
4303
 
4359
4304
  @dc.dataclass(frozen=True)
4360
- class DeployGitRepo:
4361
- host: ta.Optional[str] = None
4362
- username: ta.Optional[str] = None
4363
- path: ta.Optional[str] = None
4305
+ class PythonRemoteManageTarget:
4306
+ DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
4307
+ python: str = DEFAULT_PYTHON
4364
4308
 
4365
- def __post_init__(self) -> None:
4366
- check.not_in('..', check.non_empty_str(self.host))
4367
- check.not_in('.', check.non_empty_str(self.path))
4368
4309
 
4310
+ #
4369
4311
 
4370
- @dc.dataclass(frozen=True)
4371
- class DeployGitSpec:
4372
- repo: DeployGitRepo
4373
- rev: DeployRev
4374
4312
 
4375
- subtrees: ta.Optional[ta.Sequence[str]] = None
4376
-
4377
- def __post_init__(self) -> None:
4378
- hash(self)
4379
- check.non_empty_str(self.rev)
4380
- if self.subtrees is not None:
4381
- for st in self.subtrees:
4382
- check.non_empty_str(st)
4383
-
4384
-
4385
- ##
4386
-
4387
-
4388
- @dc.dataclass(frozen=True)
4389
- class DeployVenvSpec:
4390
- interp: ta.Optional[str] = None
4391
-
4392
- requirements_files: ta.Optional[ta.Sequence[str]] = None
4393
- extra_dependencies: ta.Optional[ta.Sequence[str]] = None
4394
-
4395
- use_uv: bool = False
4396
-
4397
-
4398
- ##
4399
-
4400
-
4401
- @dc.dataclass(frozen=True)
4402
- class DeployConfFile:
4403
- path: str
4404
- body: str
4405
-
4406
- def __post_init__(self) -> None:
4407
- check_valid_deploy_spec_path(self.path)
4408
-
4409
-
4410
- #
4411
-
4412
-
4413
- @dc.dataclass(frozen=True)
4414
- class DeployConfLink(abc.ABC): # noqa
4415
- """
4416
- May be either:
4417
- - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
4418
- - @conf/file - links a single file in a single subdir to conf/@conf/@dst-file
4419
- - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
4420
- """
4421
-
4422
- src: str
4423
-
4424
- def __post_init__(self) -> None:
4425
- check_valid_deploy_spec_path(self.src)
4426
- if '/' in self.src:
4427
- check.equal(self.src.count('/'), 1)
4428
-
4429
-
4430
- class AppDeployConfLink(DeployConfLink):
4431
- pass
4432
-
4433
-
4434
- class TagDeployConfLink(DeployConfLink):
4435
- pass
4436
-
4437
-
4438
- #
4439
-
4440
-
4441
- @dc.dataclass(frozen=True)
4442
- class DeployConfSpec:
4443
- files: ta.Optional[ta.Sequence[DeployConfFile]] = None
4444
-
4445
- links: ta.Optional[ta.Sequence[DeployConfLink]] = None
4446
-
4447
- def __post_init__(self) -> None:
4448
- if self.files:
4449
- seen: ta.Set[str] = set()
4450
- for f in self.files:
4451
- check.not_in(f.path, seen)
4452
- seen.add(f.path)
4453
-
4454
-
4455
- ##
4456
-
4457
-
4458
- @dc.dataclass(frozen=True)
4459
- class DeploySpec:
4460
- app: DeployApp
4461
-
4462
- git: DeployGitSpec
4463
-
4464
- venv: ta.Optional[DeployVenvSpec] = None
4465
-
4466
- conf: ta.Optional[DeployConfSpec] = None
4467
-
4468
- @cached_nullary
4469
- def key(self) -> DeployKey:
4470
- return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])
4471
-
4472
-
4473
- ########################################
4474
- # ../remote/config.py
4475
-
4476
-
4477
- @dc.dataclass(frozen=True)
4478
- class RemoteConfig:
4479
- payload_file: ta.Optional[str] = None
4480
-
4481
- set_pgid: bool = True
4482
-
4483
- deathsig: ta.Optional[str] = 'KILL'
4484
-
4485
- pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
4486
-
4487
- forward_logging: bool = True
4488
-
4489
- timebomb_delay_s: ta.Optional[float] = 60 * 60.
4490
-
4491
- heartbeat_interval_s: float = 3.
4492
-
4493
-
4494
- ########################################
4495
- # ../remote/payload.py
4496
-
4497
-
4498
- RemoteExecutionPayloadFile = ta.NewType('RemoteExecutionPayloadFile', str)
4499
-
4500
-
4501
- @cached_nullary
4502
- def _get_self_src() -> str:
4503
- return inspect.getsource(sys.modules[__name__])
4504
-
4505
-
4506
- def _is_src_amalg(src: str) -> bool:
4507
- for l in src.splitlines(): # noqa
4508
- if l.startswith('# @omlish-amalg-output '):
4509
- return True
4510
- return False
4511
-
4512
-
4513
- @cached_nullary
4514
- def _is_self_amalg() -> bool:
4515
- return _is_src_amalg(_get_self_src())
4516
-
4517
-
4518
- def get_remote_payload_src(
4519
- *,
4520
- file: ta.Optional[RemoteExecutionPayloadFile],
4521
- ) -> str:
4522
- if file is not None:
4523
- with open(file) as f:
4524
- return f.read()
4525
-
4526
- if _is_self_amalg():
4527
- return _get_self_src()
4528
-
4529
- import importlib.resources
4530
- return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
4531
-
4532
-
4533
- ########################################
4534
- # ../system/platforms.py
4535
-
4536
-
4537
- ##
4538
-
4539
-
4540
- @dc.dataclass(frozen=True)
4541
- class Platform(abc.ABC): # noqa
4542
- pass
4543
-
4544
-
4545
- class LinuxPlatform(Platform, abc.ABC):
4546
- pass
4547
-
4548
-
4549
- class UbuntuPlatform(LinuxPlatform):
4550
- pass
4551
-
4552
-
4553
- class AmazonLinuxPlatform(LinuxPlatform):
4554
- pass
4555
-
4556
-
4557
- class GenericLinuxPlatform(LinuxPlatform):
4558
- pass
4559
-
4560
-
4561
- class DarwinPlatform(Platform):
4562
- pass
4563
-
4564
-
4565
- class UnknownPlatform(Platform):
4566
- pass
4567
-
4568
-
4569
- ##
4570
-
4571
-
4572
- def _detect_system_platform() -> Platform:
4573
- plat = sys.platform
4574
-
4575
- if plat == 'linux':
4576
- if (osr := LinuxOsRelease.read()) is None:
4577
- return GenericLinuxPlatform()
4578
-
4579
- if osr.id == 'amzn':
4580
- return AmazonLinuxPlatform()
4581
-
4582
- elif osr.id == 'ubuntu':
4583
- return UbuntuPlatform()
4584
-
4585
- else:
4586
- return GenericLinuxPlatform()
4587
-
4588
- elif plat == 'darwin':
4589
- return DarwinPlatform()
4590
-
4591
- else:
4592
- return UnknownPlatform()
4593
-
4594
-
4595
- @cached_nullary
4596
- def detect_system_platform() -> Platform:
4597
- platform = _detect_system_platform()
4598
- log.info('Detected platform: %r', platform)
4599
- return platform
4600
-
4601
-
4602
- ########################################
4603
- # ../targets/targets.py
4604
- """
4605
- It's desugaring. Subprocess and locals are only leafs. Retain an origin?
4606
- ** TWO LAYERS ** - ManageTarget is user-facing, ConnectorTarget is bound, internal
4607
- """
4608
-
4609
-
4610
- ##
4611
-
4612
-
4613
- class ManageTarget(abc.ABC): # noqa
4614
- def __init_subclass__(cls, **kwargs: ta.Any) -> None:
4615
- super().__init_subclass__(**kwargs)
4616
-
4617
- check.state(cls.__name__.endswith('ManageTarget'))
4618
-
4619
-
4620
- #
4621
-
4622
-
4623
- @dc.dataclass(frozen=True)
4624
- class PythonRemoteManageTarget:
4625
- DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
4626
- python: str = DEFAULT_PYTHON
4627
-
4628
-
4629
- #
4630
-
4631
-
4632
- class RemoteManageTarget(ManageTarget, abc.ABC):
4633
- pass
4313
+ class RemoteManageTarget(ManageTarget, abc.ABC):
4314
+ pass
4634
4315
 
4635
4316
 
4636
4317
  class PhysicallyRemoteManageTarget(RemoteManageTarget, abc.ABC):
@@ -6859,172 +6540,343 @@ CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command
6859
6540
 
6860
6541
 
6861
6542
  ########################################
6862
- # ../deploy/conf.py
6543
+ # ../deploy/paths/paths.py
6863
6544
  """
6864
6545
  TODO:
6865
- - @conf DeployPathPlaceholder? :|
6866
- - post-deploy: remove any dir_links not present in new spec
6867
- - * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
6868
- - no such thing as 'previously present'.. build a 'deploy state' and pass it back?
6869
- - ** whole thing can be atomic **
6870
- - 1) new atomic temp dir
6871
- - 2) for each subdir not needing modification, hardlink into temp dir
6872
- - 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
6873
- - 4) write (or if deleting, omit) new files
6874
- - 5) swap top level
6875
- - ** whole deploy can be atomic(-ish) - do this for everything **
6876
- - just a '/deploy/current' dir
6877
- - some things (venvs) cannot be moved, thus the /deploy/venvs dir
6878
- - ** ensure (enforce) equivalent relpath nesting
6546
+ - run/{.pid,.sock}
6547
+ - logs/...
6548
+ - current symlink
6549
+ - conf/{nginx,supervisor}
6550
+ - env/?
6551
+ - apps/<app>/shared
6879
6552
  """
6880
6553
 
6881
6554
 
6882
- class DeployConfManager(SingleDirDeployPathOwner):
6883
- def __init__(
6884
- self,
6885
- *,
6886
- deploy_home: ta.Optional[DeployHome] = None,
6887
- ) -> None:
6888
- super().__init__(
6889
- owned_dir='conf',
6890
- deploy_home=deploy_home,
6891
- )
6555
+ ##
6892
6556
 
6893
- async def _write_conf_file(
6894
- self,
6895
- cf: DeployConfFile,
6896
- conf_dir: str,
6897
- ) -> None:
6898
- conf_file = os.path.join(conf_dir, cf.path)
6899
- check.arg(is_path_in_dir(conf_dir, conf_file))
6900
6557
 
6901
- os.makedirs(os.path.dirname(conf_file), exist_ok=True)
6558
+ DEPLOY_PATH_PLACEHOLDER_SIGIL = '@'
6559
+ DEPLOY_PATH_PLACEHOLDER_SEPARATOR = '--'
6902
6560
 
6903
- with open(conf_file, 'w') as f: # noqa
6904
- f.write(cf.body)
6561
+ DEPLOY_PATH_PLACEHOLDER_DELIMITERS: ta.AbstractSet[str] = frozenset([
6562
+ DEPLOY_PATH_PLACEHOLDER_SEPARATOR,
6563
+ '.',
6564
+ ])
6905
6565
 
6906
- async def _make_conf_link(
6907
- self,
6908
- link: DeployConfLink,
6909
- conf_dir: str,
6910
- app_tag: DeployAppTag,
6911
- link_dir: str,
6912
- ) -> None:
6913
- link_src = os.path.join(conf_dir, link.src)
6914
- check.arg(is_path_in_dir(conf_dir, link_src))
6566
+ DEPLOY_PATH_PLACEHOLDERS: ta.FrozenSet[str] = frozenset([
6567
+ 'app',
6568
+ 'tag',
6569
+ 'conf',
6570
+ ])
6915
6571
 
6916
- is_link_dir = link.src.endswith('/')
6917
- if is_link_dir:
6918
- check.arg(link.src.count('/') == 1)
6919
- check.arg(os.path.isdir(link_src))
6920
- link_dst_pfx = link.src
6921
- link_dst_sfx = ''
6922
6572
 
6923
- elif '/' in link.src:
6924
- check.arg(os.path.isfile(link_src))
6925
- d, f = os.path.split(link.src)
6926
- # TODO: check filename :|
6927
- link_dst_pfx = d + '/'
6928
- link_dst_sfx = '-' + f
6573
+ class DeployPathError(Exception):
6574
+ pass
6575
+
6576
+
6577
+ class DeployPathRenderable(abc.ABC):
6578
+ @abc.abstractmethod
6579
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6580
+ raise NotImplementedError
6581
+
6582
+
6583
+ ##
6929
6584
 
6585
+
6586
+ class DeployPathNamePart(DeployPathRenderable, abc.ABC):
6587
+ @classmethod
6588
+ def parse(cls, s: str) -> 'DeployPathNamePart':
6589
+ check.non_empty_str(s)
6590
+ if s.startswith(DEPLOY_PATH_PLACEHOLDER_SIGIL):
6591
+ return PlaceholderDeployPathNamePart(s[1:])
6592
+ elif s in DEPLOY_PATH_PLACEHOLDER_DELIMITERS:
6593
+ return DelimiterDeployPathNamePart(s)
6930
6594
  else:
6931
- check.arg(os.path.isfile(link_src))
6932
- if '.' in link.src:
6933
- l, _, r = link.src.partition('.')
6934
- link_dst_pfx = l + '/'
6935
- link_dst_sfx = '.' + r
6936
- else:
6937
- link_dst_pfx = link.src + '/'
6938
- link_dst_sfx = ''
6595
+ return ConstDeployPathNamePart(s)
6596
+
6597
+
6598
+ @dc.dataclass(frozen=True)
6599
+ class PlaceholderDeployPathNamePart(DeployPathNamePart):
6600
+ placeholder: str # DeployPathPlaceholder
6601
+
6602
+ def __post_init__(self) -> None:
6603
+ check.in_(self.placeholder, DEPLOY_PATH_PLACEHOLDERS)
6604
+
6605
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6606
+ if placeholders is not None:
6607
+ return placeholders[self.placeholder] # type: ignore
6608
+ else:
6609
+ return DEPLOY_PATH_PLACEHOLDER_SIGIL + self.placeholder
6610
+
6611
+
6612
+ @dc.dataclass(frozen=True)
6613
+ class DelimiterDeployPathNamePart(DeployPathNamePart):
6614
+ delimiter: str
6615
+
6616
+ def __post_init__(self) -> None:
6617
+ check.in_(self.delimiter, DEPLOY_PATH_PLACEHOLDER_DELIMITERS)
6618
+
6619
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6620
+ return self.delimiter
6621
+
6622
+
6623
+ @dc.dataclass(frozen=True)
6624
+ class ConstDeployPathNamePart(DeployPathNamePart):
6625
+ const: str
6626
+
6627
+ def __post_init__(self) -> None:
6628
+ check.non_empty_str(self.const)
6629
+ for c in [*DEPLOY_PATH_PLACEHOLDER_DELIMITERS, DEPLOY_PATH_PLACEHOLDER_SIGIL, '/']:
6630
+ check.not_in(c, self.const)
6631
+
6632
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6633
+ return self.const
6634
+
6635
+
6636
+ @dc.dataclass(frozen=True)
6637
+ class DeployPathName(DeployPathRenderable):
6638
+ parts: ta.Sequence[DeployPathNamePart]
6639
+
6640
+ def __post_init__(self) -> None:
6641
+ hash(self)
6642
+ check.not_empty(self.parts)
6643
+ for k, g in itertools.groupby(self.parts, type):
6644
+ if len(gl := list(g)) > 1:
6645
+ raise DeployPathError(f'May not have consecutive path name part types: {k} {gl}')
6646
+
6647
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6648
+ return ''.join(p.render(placeholders) for p in self.parts)
6649
+
6650
+ @classmethod
6651
+ def parse(cls, s: str) -> 'DeployPathName':
6652
+ check.non_empty_str(s)
6653
+ check.not_in('/', s)
6654
+
6655
+ i = 0
6656
+ ps = []
6657
+ while i < len(s):
6658
+ ns = [(n, d) for d in DEPLOY_PATH_PLACEHOLDER_DELIMITERS if (n := s.find(d, i)) >= 0]
6659
+ if not ns:
6660
+ ps.append(s[i:])
6661
+ break
6662
+ n, d = min(ns)
6663
+ ps.append(check.non_empty_str(s[i:n]))
6664
+ ps.append(s[n:n + len(d)])
6665
+ i = n + len(d)
6666
+
6667
+ return cls(tuple(DeployPathNamePart.parse(p) for p in ps))
6668
+
6669
+
6670
+ ##
6671
+
6672
+
6673
+ @dc.dataclass(frozen=True)
6674
+ class DeployPathPart(DeployPathRenderable, abc.ABC): # noqa
6675
+ name: DeployPathName
6676
+
6677
+ @property
6678
+ @abc.abstractmethod
6679
+ def kind(self) -> DeployPathKind:
6680
+ raise NotImplementedError
6681
+
6682
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6683
+ return self.name.render(placeholders) + ('/' if self.kind == 'dir' else '')
6684
+
6685
+ @classmethod
6686
+ def parse(cls, s: str) -> 'DeployPathPart':
6687
+ if (is_dir := s.endswith('/')):
6688
+ s = s[:-1]
6689
+ check.non_empty_str(s)
6690
+ check.not_in('/', s)
6691
+
6692
+ n = DeployPathName.parse(s)
6693
+ if is_dir:
6694
+ return DirDeployPathPart(n)
6695
+ else:
6696
+ return FileDeployPathPart(n)
6697
+
6698
+
6699
+ class DirDeployPathPart(DeployPathPart):
6700
+ @property
6701
+ def kind(self) -> DeployPathKind:
6702
+ return 'dir'
6703
+
6704
+
6705
+ class FileDeployPathPart(DeployPathPart):
6706
+ @property
6707
+ def kind(self) -> DeployPathKind:
6708
+ return 'file'
6709
+
6710
+
6711
+ #
6712
+
6713
+
6714
+ @dc.dataclass(frozen=True)
6715
+ class DeployPath:
6716
+ parts: ta.Sequence[DeployPathPart]
6717
+
6718
+ @property
6719
+ def name_parts(self) -> ta.Iterator[DeployPathNamePart]:
6720
+ for p in self.parts:
6721
+ yield from p.name.parts
6722
+
6723
+ def __post_init__(self) -> None:
6724
+ hash(self)
6725
+ check.not_empty(self.parts)
6726
+ for p in self.parts[:-1]:
6727
+ check.equal(p.kind, 'dir')
6728
+
6729
+ pd: ta.Dict[DeployPathPlaceholder, ta.List[int]] = {}
6730
+ for i, np in enumerate(self.name_parts):
6731
+ if isinstance(np, PlaceholderDeployPathNamePart):
6732
+ pd.setdefault(ta.cast(DeployPathPlaceholder, np.placeholder), []).append(i)
6733
+
6734
+ # if 'tag' in pd and 'app' not in pd:
6735
+ # raise DeployPathError('Tag placeholder in path without app', self)
6736
+
6737
+ @property
6738
+ def kind(self) -> ta.Literal['file', 'dir']:
6739
+ return self.parts[-1].kind
6740
+
6741
+ def render(self, placeholders: ta.Optional[ta.Mapping[DeployPathPlaceholder, str]] = None) -> str:
6742
+ return ''.join([p.render(placeholders) for p in self.parts])
6743
+
6744
+ @classmethod
6745
+ def parse(cls, s: str) -> 'DeployPath':
6746
+ check.non_empty_str(s)
6747
+ ps = split_keep_delimiter(s, '/')
6748
+ return cls(tuple(DeployPathPart.parse(p) for p in ps))
6749
+
6750
+
6751
+ ########################################
6752
+ # ../deploy/specs.py
6753
+
6754
+
6755
+ ##
6756
+
6757
+
6758
+ def check_valid_deploy_spec_path(s: str) -> str:
6759
+ check.non_empty_str(s)
6760
+ for c in ['..', '//']:
6761
+ check.not_in(c, s)
6762
+ check.arg(not s.startswith('/'))
6763
+ return s
6764
+
6765
+
6766
+ ##
6767
+
6768
+
6769
+ @dc.dataclass(frozen=True)
6770
+ class DeployGitRepo:
6771
+ host: ta.Optional[str] = None
6772
+ username: ta.Optional[str] = None
6773
+ path: ta.Optional[str] = None
6774
+
6775
+ def __post_init__(self) -> None:
6776
+ check.not_in('..', check.non_empty_str(self.host))
6777
+ check.not_in('.', check.non_empty_str(self.path))
6778
+
6779
+
6780
+ @dc.dataclass(frozen=True)
6781
+ class DeployGitSpec:
6782
+ repo: DeployGitRepo
6783
+ rev: DeployRev
6784
+
6785
+ subtrees: ta.Optional[ta.Sequence[str]] = None
6786
+
6787
+ def __post_init__(self) -> None:
6788
+ check.non_empty_str(self.rev)
6789
+ if self.subtrees is not None:
6790
+ for st in self.subtrees:
6791
+ check.non_empty_str(st)
6792
+
6793
+
6794
+ ##
6795
+
6796
+
6797
+ @dc.dataclass(frozen=True)
6798
+ class DeployVenvSpec:
6799
+ interp: ta.Optional[str] = None
6800
+
6801
+ requirements_files: ta.Optional[ta.Sequence[str]] = None
6802
+ extra_dependencies: ta.Optional[ta.Sequence[str]] = None
6803
+
6804
+ use_uv: bool = False
6805
+
6806
+
6807
+ ##
6808
+
6809
+
6810
+ @dc.dataclass(frozen=True)
6811
+ class DeployConfFile:
6812
+ path: str
6813
+ body: str
6814
+
6815
+ def __post_init__(self) -> None:
6816
+ check_valid_deploy_spec_path(self.path)
6817
+
6818
+
6819
+ #
6820
+
6821
+
6822
+ @dc.dataclass(frozen=True)
6823
+ class DeployConfLink(abc.ABC): # noqa
6824
+ """
6825
+ May be either:
6826
+ - @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
6827
+ - @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
6828
+ - @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
6829
+ """
6830
+
6831
+ src: str
6832
+
6833
+ def __post_init__(self) -> None:
6834
+ check_valid_deploy_spec_path(self.src)
6835
+ if '/' in self.src:
6836
+ check.equal(self.src.count('/'), 1)
6837
+
6838
+
6839
+ class AppDeployConfLink(DeployConfLink):
6840
+ pass
6841
+
6842
+
6843
+ class TagDeployConfLink(DeployConfLink):
6844
+ pass
6939
6845
 
6940
- if isinstance(link, AppDeployConfLink):
6941
- link_dst_mid = str(app_tag.app)
6942
- sym_root = link_dir
6943
- elif isinstance(link, TagDeployConfLink):
6944
- link_dst_mid = '-'.join([app_tag.app, app_tag.tag])
6945
- sym_root = conf_dir
6946
- else:
6947
- raise TypeError(link)
6948
6846
 
6949
- link_dst = ''.join([
6950
- link_dst_pfx,
6951
- link_dst_mid,
6952
- link_dst_sfx,
6953
- ])
6847
+ #
6954
6848
 
6955
- root_conf_dir = self._make_dir()
6956
- sym_src = os.path.join(sym_root, link.src)
6957
- sym_dst = os.path.join(root_conf_dir, link_dst)
6958
- check.arg(is_path_in_dir(root_conf_dir, sym_dst))
6959
6849
 
6960
- os.makedirs(os.path.dirname(sym_dst), exist_ok=True)
6961
- relative_symlink(sym_src, sym_dst, target_is_directory=is_link_dir)
6850
+ @dc.dataclass(frozen=True)
6851
+ class DeployConfSpec:
6852
+ files: ta.Optional[ta.Sequence[DeployConfFile]] = None
6962
6853
 
6963
- async def write_conf(
6964
- self,
6965
- spec: DeployConfSpec,
6966
- conf_dir: str,
6967
- app_tag: DeployAppTag,
6968
- link_dir: str,
6969
- ) -> None:
6970
- conf_dir = os.path.abspath(conf_dir)
6971
- os.makedirs(conf_dir)
6854
+ links: ta.Optional[ta.Sequence[DeployConfLink]] = None
6972
6855
 
6973
- #
6856
+ def __post_init__(self) -> None:
6857
+ if self.files:
6858
+ seen: ta.Set[str] = set()
6859
+ for f in self.files:
6860
+ check.not_in(f.path, seen)
6861
+ seen.add(f.path)
6974
6862
 
6975
- for cf in spec.files or []:
6976
- await self._write_conf_file(
6977
- cf,
6978
- conf_dir,
6979
- )
6980
6863
 
6981
- #
6864
+ ##
6982
6865
 
6983
- for link in spec.links or []:
6984
- await self._make_conf_link(
6985
- link,
6986
- conf_dir,
6987
- app_tag,
6988
- link_dir,
6989
- )
6990
6866
 
6867
+ @dc.dataclass(frozen=True)
6868
+ class DeploySpec:
6869
+ app: DeployApp
6991
6870
 
6992
- ########################################
6993
- # ../deploy/tmp.py
6871
+ git: DeployGitSpec
6994
6872
 
6873
+ venv: ta.Optional[DeployVenvSpec] = None
6995
6874
 
6996
- class DeployTmpManager(
6997
- SingleDirDeployPathOwner,
6998
- AtomicPathSwapping,
6999
- ):
7000
- def __init__(
7001
- self,
7002
- *,
7003
- deploy_home: ta.Optional[DeployHome] = None,
7004
- ) -> None:
7005
- super().__init__(
7006
- owned_dir='tmp',
7007
- deploy_home=deploy_home,
7008
- )
6875
+ conf: ta.Optional[DeployConfSpec] = None
7009
6876
 
7010
6877
  @cached_nullary
7011
- def _swapping(self) -> AtomicPathSwapping:
7012
- return TempDirAtomicPathSwapping(
7013
- temp_dir=self._make_dir(),
7014
- root_dir=check.non_empty_str(self._deploy_home),
7015
- )
7016
-
7017
- def begin_atomic_path_swap(
7018
- self,
7019
- kind: AtomicPathSwapKind,
7020
- dst_path: str,
7021
- **kwargs: ta.Any,
7022
- ) -> AtomicPathSwap:
7023
- return self._swapping().begin_atomic_path_swap(
7024
- kind,
7025
- dst_path,
7026
- **kwargs,
7027
- )
6878
+ def key(self) -> DeployKey:
6879
+ return DeployKey(hashlib.sha256(repr(self).encode('utf-8')).hexdigest()[:8])
7028
6880
 
7029
6881
 
7030
6882
  ########################################
@@ -7537,78 +7389,293 @@ class AbstractAsyncSubprocesses(BaseSubprocesses):
7537
7389
 
7538
7390
  #
7539
7391
 
7540
- async def check_output_str(
7541
- self,
7542
- *cmd: str,
7543
- **kwargs: ta.Any,
7544
- ) -> str:
7545
- return (await self.check_output(*cmd, **kwargs)).decode().strip()
7392
+ async def check_output_str(
7393
+ self,
7394
+ *cmd: str,
7395
+ **kwargs: ta.Any,
7396
+ ) -> str:
7397
+ return (await self.check_output(*cmd, **kwargs)).decode().strip()
7398
+
7399
+ #
7400
+
7401
+ async def try_call(
7402
+ self,
7403
+ *cmd: str,
7404
+ **kwargs: ta.Any,
7405
+ ) -> bool:
7406
+ if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
7407
+ return False
7408
+ else:
7409
+ return True
7410
+
7411
+ async def try_output(
7412
+ self,
7413
+ *cmd: str,
7414
+ **kwargs: ta.Any,
7415
+ ) -> ta.Optional[bytes]:
7416
+ if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
7417
+ return None
7418
+ else:
7419
+ return ret
7420
+
7421
+ async def try_output_str(
7422
+ self,
7423
+ *cmd: str,
7424
+ **kwargs: ta.Any,
7425
+ ) -> ta.Optional[str]:
7426
+ if (ret := await self.try_output(*cmd, **kwargs)) is None:
7427
+ return None
7428
+ else:
7429
+ return ret.decode().strip()
7430
+
7431
+
7432
+ ########################################
7433
+ # ../bootstrap.py
7434
+
7435
+
7436
+ @dc.dataclass(frozen=True)
7437
+ class MainBootstrap:
7438
+ main_config: MainConfig = MainConfig()
7439
+
7440
+ deploy_config: DeployConfig = DeployConfig()
7441
+
7442
+ remote_config: RemoteConfig = RemoteConfig()
7443
+
7444
+ system_config: SystemConfig = SystemConfig()
7445
+
7446
+
7447
+ ########################################
7448
+ # ../commands/local.py
7449
+
7450
+
7451
+ class LocalCommandExecutor(CommandExecutor):
7452
+ def __init__(
7453
+ self,
7454
+ *,
7455
+ command_executors: CommandExecutorMap,
7456
+ ) -> None:
7457
+ super().__init__()
7458
+
7459
+ self._command_executors = command_executors
7460
+
7461
+ async def execute(self, cmd: Command) -> Command.Output:
7462
+ ce: CommandExecutor = self._command_executors[type(cmd)]
7463
+ return await ce.execute(cmd)
7464
+
7465
+
7466
+ ########################################
7467
+ # ../deploy/conf.py
7468
+ """
7469
+ TODO:
7470
+ - @conf DeployPathPlaceholder? :|
7471
+ - post-deploy: remove any dir_links not present in new spec
7472
+ - * only if succeeded * - otherwise, remove any dir_links present in new spec but not previously present?
7473
+ - no such thing as 'previously present'.. build a 'deploy state' and pass it back?
7474
+ - ** whole thing can be atomic **
7475
+ - 1) new atomic temp dir
7476
+ - 2) for each subdir not needing modification, hardlink into temp dir
7477
+ - 3) for each subdir needing modification, new subdir, hardlink all files not needing modification
7478
+ - 4) write (or if deleting, omit) new files
7479
+ - 5) swap top level
7480
+ - ** whole deploy can be atomic(-ish) - do this for everything **
7481
+ - just a '/deploy/current' dir
7482
+ - some things (venvs) cannot be moved, thus the /deploy/venvs dir
7483
+ - ** ensure (enforce) equivalent relpath nesting
7484
+ """
7485
+
7486
+
7487
+ class DeployConfManager:
7488
+ def __init__(
7489
+ self,
7490
+ *,
7491
+ deploy_home: ta.Optional[DeployHome] = None,
7492
+ ) -> None:
7493
+ super().__init__()
7494
+
7495
+ self._deploy_home = deploy_home
7496
+
7497
+ #
7498
+
7499
+ async def _write_conf_file(
7500
+ self,
7501
+ cf: DeployConfFile,
7502
+ conf_dir: str,
7503
+ ) -> None:
7504
+ conf_file = os.path.join(conf_dir, cf.path)
7505
+ check.arg(is_path_in_dir(conf_dir, conf_file))
7506
+
7507
+ os.makedirs(os.path.dirname(conf_file), exist_ok=True)
7508
+
7509
+ with open(conf_file, 'w') as f: # noqa
7510
+ f.write(cf.body)
7511
+
7512
+ #
7513
+
7514
+ class _ComputedConfLink(ta.NamedTuple):
7515
+ is_dir: bool
7516
+ link_src: str
7517
+ link_dst: str
7518
+
7519
+ def _compute_conf_link_dst(
7520
+ self,
7521
+ link: DeployConfLink,
7522
+ app_tag: DeployAppTag,
7523
+ conf_dir: str,
7524
+ link_dir: str,
7525
+ ) -> _ComputedConfLink:
7526
+ link_src = os.path.join(conf_dir, link.src)
7527
+ check.arg(is_path_in_dir(conf_dir, link_src))
7528
+
7529
+ #
7530
+
7531
+ if (is_dir := link.src.endswith('/')):
7532
+ # @conf/ - links a directory in root of app conf dir to conf/@conf/@dst/
7533
+ check.arg(link.src.count('/') == 1)
7534
+ link_dst_pfx = link.src
7535
+ link_dst_sfx = ''
7536
+
7537
+ elif '/' in link.src:
7538
+ # @conf/file - links a single file in a single subdir to conf/@conf/@dst--file
7539
+ d, f = os.path.split(link.src)
7540
+ # TODO: check filename :|
7541
+ link_dst_pfx = d + '/'
7542
+ link_dst_sfx = DEPLOY_PATH_PLACEHOLDER_SEPARATOR + f
7543
+
7544
+ else: # noqa
7545
+ # @conf(.ext)* - links a single file in root of app conf dir to conf/@conf/@dst(.ext)*
7546
+ if '.' in link.src:
7547
+ l, _, r = link.src.partition('.')
7548
+ link_dst_pfx = l + '/'
7549
+ link_dst_sfx = '.' + r
7550
+ else:
7551
+ link_dst_pfx = link.src + '/'
7552
+ link_dst_sfx = ''
7553
+
7554
+ #
7555
+
7556
+ if isinstance(link, AppDeployConfLink):
7557
+ link_dst_mid = str(app_tag.app)
7558
+ elif isinstance(link, TagDeployConfLink):
7559
+ link_dst_mid = DEPLOY_PATH_PLACEHOLDER_SEPARATOR.join([app_tag.app, app_tag.tag])
7560
+ else:
7561
+ raise TypeError(link)
7562
+
7563
+ #
7564
+
7565
+ link_dst_name = ''.join([
7566
+ link_dst_pfx,
7567
+ link_dst_mid,
7568
+ link_dst_sfx,
7569
+ ])
7570
+ link_dst = os.path.join(link_dir, link_dst_name)
7571
+
7572
+ return DeployConfManager._ComputedConfLink(
7573
+ is_dir=is_dir,
7574
+ link_src=link_src,
7575
+ link_dst=link_dst,
7576
+ )
7577
+
7578
+ async def _make_conf_link(
7579
+ self,
7580
+ link: DeployConfLink,
7581
+ app_tag: DeployAppTag,
7582
+ conf_dir: str,
7583
+ link_dir: str,
7584
+ ) -> None:
7585
+ comp = self._compute_conf_link_dst(
7586
+ link,
7587
+ app_tag,
7588
+ conf_dir,
7589
+ link_dir,
7590
+ )
7591
+
7592
+ #
7546
7593
 
7547
- #
7594
+ check.arg(is_path_in_dir(conf_dir, comp.link_src))
7595
+ check.arg(is_path_in_dir(link_dir, comp.link_dst))
7548
7596
 
7549
- async def try_call(
7550
- self,
7551
- *cmd: str,
7552
- **kwargs: ta.Any,
7553
- ) -> bool:
7554
- if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
7555
- return False
7597
+ if comp.is_dir:
7598
+ check.arg(os.path.isdir(comp.link_src))
7556
7599
  else:
7557
- return True
7600
+ check.arg(os.path.isfile(comp.link_src))
7558
7601
 
7559
- async def try_output(
7560
- self,
7561
- *cmd: str,
7562
- **kwargs: ta.Any,
7563
- ) -> ta.Optional[bytes]:
7564
- if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
7565
- return None
7566
- else:
7567
- return ret
7602
+ #
7568
7603
 
7569
- async def try_output_str(
7570
- self,
7571
- *cmd: str,
7572
- **kwargs: ta.Any,
7573
- ) -> ta.Optional[str]:
7574
- if (ret := await self.try_output(*cmd, **kwargs)) is None:
7575
- return None
7576
- else:
7577
- return ret.decode().strip()
7604
+ relative_symlink( # noqa
7605
+ comp.link_src,
7606
+ comp.link_dst,
7607
+ target_is_directory=comp.is_dir,
7608
+ make_dirs=True,
7609
+ )
7578
7610
 
7611
+ #
7579
7612
 
7580
- ########################################
7581
- # ../bootstrap.py
7613
+ async def write_conf(
7614
+ self,
7615
+ spec: DeployConfSpec,
7616
+ app_tag: DeployAppTag,
7617
+ conf_dir: str,
7618
+ link_dir: str,
7619
+ ) -> None:
7620
+ for cf in spec.files or []:
7621
+ await self._write_conf_file(
7622
+ cf,
7623
+ conf_dir,
7624
+ )
7582
7625
 
7626
+ #
7583
7627
 
7584
- @dc.dataclass(frozen=True)
7585
- class MainBootstrap:
7586
- main_config: MainConfig = MainConfig()
7628
+ for link in spec.links or []:
7629
+ await self._make_conf_link(
7630
+ link,
7631
+ app_tag,
7632
+ conf_dir,
7633
+ link_dir,
7634
+ )
7587
7635
 
7588
- deploy_config: DeployConfig = DeployConfig()
7589
7636
 
7590
- remote_config: RemoteConfig = RemoteConfig()
7637
+ ########################################
7638
+ # ../deploy/paths/owners.py
7591
7639
 
7592
- system_config: SystemConfig = SystemConfig()
7640
+
7641
+ class DeployPathOwner(abc.ABC):
7642
+ @abc.abstractmethod
7643
+ def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
7644
+ raise NotImplementedError
7593
7645
 
7594
7646
 
7595
- ########################################
7596
- # ../commands/local.py
7647
+ DeployPathOwners = ta.NewType('DeployPathOwners', ta.Sequence[DeployPathOwner])
7597
7648
 
7598
7649
 
7599
- class LocalCommandExecutor(CommandExecutor):
7650
+ class SingleDirDeployPathOwner(DeployPathOwner, abc.ABC):
7600
7651
  def __init__(
7601
7652
  self,
7602
- *,
7603
- command_executors: CommandExecutorMap,
7653
+ *args: ta.Any,
7654
+ owned_dir: str,
7655
+ deploy_home: ta.Optional[DeployHome],
7656
+ **kwargs: ta.Any,
7604
7657
  ) -> None:
7605
- super().__init__()
7658
+ super().__init__(*args, **kwargs)
7606
7659
 
7607
- self._command_executors = command_executors
7660
+ check.not_in('/', owned_dir)
7661
+ self._owned_dir: str = check.non_empty_str(owned_dir)
7608
7662
 
7609
- async def execute(self, cmd: Command) -> Command.Output:
7610
- ce: CommandExecutor = self._command_executors[type(cmd)]
7611
- return await ce.execute(cmd)
7663
+ self._deploy_home = deploy_home
7664
+
7665
+ self._owned_deploy_paths = frozenset([DeployPath.parse(self._owned_dir + '/')])
7666
+
7667
+ @cached_nullary
7668
+ def _dir(self) -> str:
7669
+ return os.path.join(check.non_empty_str(self._deploy_home), self._owned_dir)
7670
+
7671
+ @cached_nullary
7672
+ def _make_dir(self) -> str:
7673
+ if not os.path.isdir(d := self._dir()):
7674
+ os.makedirs(d, exist_ok=True)
7675
+ return d
7676
+
7677
+ def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
7678
+ return self._owned_deploy_paths
7612
7679
 
7613
7680
 
7614
7681
  ########################################
@@ -8513,6 +8580,74 @@ class DeployGitManager(SingleDirDeployPathOwner):
8513
8580
  await self.get_repo_dir(spec.repo).checkout(spec, dst_dir)
8514
8581
 
8515
8582
 
8583
+ ########################################
8584
+ # ../deploy/paths/manager.py
8585
+
8586
+
8587
+ class DeployPathsManager:
8588
+ def __init__(
8589
+ self,
8590
+ *,
8591
+ deploy_home: ta.Optional[DeployHome],
8592
+ deploy_path_owners: DeployPathOwners,
8593
+ ) -> None:
8594
+ super().__init__()
8595
+
8596
+ self._deploy_home = deploy_home
8597
+ self._deploy_path_owners = deploy_path_owners
8598
+
8599
+ @cached_nullary
8600
+ def owners_by_path(self) -> ta.Mapping[DeployPath, DeployPathOwner]:
8601
+ dct: ta.Dict[DeployPath, DeployPathOwner] = {}
8602
+ for o in self._deploy_path_owners:
8603
+ for p in o.get_owned_deploy_paths():
8604
+ if p in dct:
8605
+ raise DeployPathError(f'Duplicate deploy path owner: {p}')
8606
+ dct[p] = o
8607
+ return dct
8608
+
8609
+ def validate_deploy_paths(self) -> None:
8610
+ self.owners_by_path()
8611
+
8612
+
8613
+ ########################################
8614
+ # ../deploy/tmp.py
8615
+
8616
+
8617
+ class DeployTmpManager(
8618
+ SingleDirDeployPathOwner,
8619
+ AtomicPathSwapping,
8620
+ ):
8621
+ def __init__(
8622
+ self,
8623
+ *,
8624
+ deploy_home: ta.Optional[DeployHome] = None,
8625
+ ) -> None:
8626
+ super().__init__(
8627
+ owned_dir='tmp',
8628
+ deploy_home=deploy_home,
8629
+ )
8630
+
8631
+ @cached_nullary
8632
+ def _swapping(self) -> AtomicPathSwapping:
8633
+ return TempDirAtomicPathSwapping(
8634
+ temp_dir=self._make_dir(),
8635
+ root_dir=check.non_empty_str(self._deploy_home),
8636
+ )
8637
+
8638
+ def begin_atomic_path_swap(
8639
+ self,
8640
+ kind: AtomicPathSwapKind,
8641
+ dst_path: str,
8642
+ **kwargs: ta.Any,
8643
+ ) -> AtomicPathSwap:
8644
+ return self._swapping().begin_atomic_path_swap(
8645
+ kind,
8646
+ dst_path,
8647
+ **kwargs,
8648
+ )
8649
+
8650
+
8516
8651
  ########################################
8517
8652
  # ../deploy/venvs.py
8518
8653
  """
@@ -9122,20 +9257,44 @@ class DeployAppManager(DeployPathOwner):
9122
9257
  self._git = git
9123
9258
  self._venvs = venvs
9124
9259
 
9125
- @cached_nullary
9126
- def _dir(self) -> str:
9127
- return os.path.join(check.non_empty_str(self._deploy_home), 'apps')
9260
+ #
9261
+
9262
+ _APP_TAG_DIR_STR = 'tags/apps/@app/@tag/'
9263
+ _APP_TAG_DIR = DeployPath.parse(_APP_TAG_DIR_STR)
9264
+
9265
+ _CONF_TAG_DIR_STR = 'tags/conf/@tag--@app/'
9266
+ _CONF_TAG_DIR = DeployPath.parse(_CONF_TAG_DIR_STR)
9128
9267
 
9268
+ _DEPLOY_DIR_STR = 'deploys/@tag--@app/'
9269
+ _DEPLOY_DIR = DeployPath.parse(_DEPLOY_DIR_STR)
9270
+
9271
+ _APP_DEPLOY_LINK = DeployPath.parse(f'{_DEPLOY_DIR_STR}apps/@app')
9272
+ _CONF_DEPLOY_LINK = DeployPath.parse(f'{_DEPLOY_DIR_STR}conf')
9273
+
9274
+ @cached_nullary
9129
9275
  def get_owned_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
9130
9276
  return {
9131
- DeployPath.parse('apps/@app/current'),
9132
- DeployPath.parse('apps/@app/deploying'),
9277
+ self._APP_TAG_DIR,
9278
+
9279
+ self._CONF_TAG_DIR,
9133
9280
 
9134
- DeployPath.parse('apps/@app/tags/@tag/conf/'),
9135
- DeployPath.parse('apps/@app/tags/@tag/git/'),
9136
- DeployPath.parse('apps/@app/tags/@tag/venv/'),
9281
+ self._DEPLOY_DIR,
9282
+
9283
+ self._APP_DEPLOY_LINK,
9284
+ self._CONF_DEPLOY_LINK,
9285
+
9286
+ *[
9287
+ DeployPath.parse(f'{self._APP_TAG_DIR_STR}{sfx}/')
9288
+ for sfx in [
9289
+ 'conf',
9290
+ 'git',
9291
+ 'venv',
9292
+ ]
9293
+ ],
9137
9294
  }
9138
9295
 
9296
+ #
9297
+
9139
9298
  async def prepare_app(
9140
9299
  self,
9141
9300
  spec: DeploySpec,
@@ -9144,26 +9303,52 @@ class DeployAppManager(DeployPathOwner):
9144
9303
 
9145
9304
  #
9146
9305
 
9147
- app_dir = os.path.join(self._dir(), spec.app)
9148
- os.makedirs(app_dir, exist_ok=True)
9306
+ deploy_home = check.non_empty_str(self._deploy_home)
9307
+
9308
+ def build_path(pth: DeployPath) -> str:
9309
+ return os.path.join(deploy_home, pth.render(app_tag.placeholders()))
9310
+
9311
+ app_tag_dir = build_path(self._APP_TAG_DIR)
9312
+ conf_tag_dir = build_path(self._CONF_TAG_DIR)
9313
+ deploy_dir = build_path(self._DEPLOY_DIR)
9314
+ app_deploy_link = build_path(self._APP_DEPLOY_LINK)
9315
+ conf_deploy_link_file = build_path(self._CONF_DEPLOY_LINK)
9149
9316
 
9150
9317
  #
9151
9318
 
9152
- tag_dir = os.path.join(app_dir, 'tags', app_tag.tag)
9153
- os.makedirs(tag_dir)
9319
+ os.makedirs(deploy_dir)
9320
+
9321
+ deploying_link = os.path.join(deploy_home, 'deploys/deploying')
9322
+ relative_symlink(
9323
+ deploy_dir,
9324
+ deploying_link,
9325
+ target_is_directory=True,
9326
+ make_dirs=True,
9327
+ )
9154
9328
 
9155
9329
  #
9156
9330
 
9157
- deploying_file = os.path.join(app_dir, 'deploying')
9158
- current_file = os.path.join(app_dir, 'current')
9331
+ os.makedirs(app_tag_dir)
9332
+ relative_symlink(
9333
+ app_tag_dir,
9334
+ app_deploy_link,
9335
+ target_is_directory=True,
9336
+ make_dirs=True,
9337
+ )
9338
+
9339
+ #
9159
9340
 
9160
- if os.path.exists(deploying_file):
9161
- os.unlink(deploying_file)
9162
- relative_symlink(tag_dir, deploying_file, target_is_directory=True)
9341
+ os.makedirs(conf_tag_dir)
9342
+ relative_symlink(
9343
+ conf_tag_dir,
9344
+ conf_deploy_link_file,
9345
+ target_is_directory=True,
9346
+ make_dirs=True,
9347
+ )
9163
9348
 
9164
9349
  #
9165
9350
 
9166
- git_dir = os.path.join(tag_dir, 'git')
9351
+ git_dir = os.path.join(app_tag_dir, 'git')
9167
9352
  await self._git.checkout(
9168
9353
  spec.git,
9169
9354
  git_dir,
@@ -9172,7 +9357,7 @@ class DeployAppManager(DeployPathOwner):
9172
9357
  #
9173
9358
 
9174
9359
  if spec.venv is not None:
9175
- venv_dir = os.path.join(tag_dir, 'venv')
9360
+ venv_dir = os.path.join(app_tag_dir, 'venv')
9176
9361
  await self._venvs.setup_venv(
9177
9362
  spec.venv,
9178
9363
  git_dir,
@@ -9182,18 +9367,33 @@ class DeployAppManager(DeployPathOwner):
9182
9367
  #
9183
9368
 
9184
9369
  if spec.conf is not None:
9185
- conf_dir = os.path.join(tag_dir, 'conf')
9186
- conf_link_dir = os.path.join(current_file, 'conf')
9370
+ conf_dir = os.path.join(app_tag_dir, 'conf')
9187
9371
  await self._conf.write_conf(
9188
9372
  spec.conf,
9189
- conf_dir,
9190
9373
  app_tag,
9191
- conf_link_dir,
9374
+ conf_dir,
9375
+ conf_tag_dir,
9192
9376
  )
9193
9377
 
9194
9378
  #
9195
9379
 
9196
- os.replace(deploying_file, current_file)
9380
+ current_link = os.path.join(deploy_home, 'deploys/current')
9381
+ os.replace(deploying_link, current_link)
9382
+
9383
+
9384
+ ########################################
9385
+ # ../deploy/paths/inject.py
9386
+
9387
+
9388
+ def bind_deploy_paths() -> InjectorBindings:
9389
+ lst: ta.List[InjectorBindingOrBindings] = [
9390
+ inj.bind_array(DeployPathOwner),
9391
+ inj.bind_array_type(DeployPathOwner, DeployPathOwners),
9392
+
9393
+ inj.bind(DeployPathsManager, singleton=True),
9394
+ ]
9395
+
9396
+ return inj.as_bindings(*lst)
9197
9397
 
9198
9398
 
9199
9399
  ########################################
@@ -9911,31 +10111,30 @@ class SystemInterpProvider(InterpProvider):
9911
10111
 
9912
10112
 
9913
10113
  ########################################
9914
- # ../deploy/commands.py
9915
-
10114
+ # ../deploy/deploy.py
9916
10115
 
9917
- ##
9918
-
9919
-
9920
- @dc.dataclass(frozen=True)
9921
- class DeployCommand(Command['DeployCommand.Output']):
9922
- spec: DeploySpec
9923
-
9924
- @dc.dataclass(frozen=True)
9925
- class Output(Command.Output):
9926
- pass
9927
10116
 
10117
+ class DeployManager:
10118
+ def __init__(
10119
+ self,
10120
+ *,
10121
+ apps: DeployAppManager,
10122
+ paths: DeployPathsManager,
10123
+ ):
10124
+ super().__init__()
9928
10125
 
9929
- @dc.dataclass(frozen=True)
9930
- class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
9931
- _apps: DeployAppManager
10126
+ self._apps = apps
10127
+ self._paths = paths
9932
10128
 
9933
- async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
9934
- log.info('Deploying! %r', cmd.spec)
10129
+ async def run_deploy(
10130
+ self,
10131
+ spec: DeploySpec,
10132
+ ) -> None:
10133
+ self._paths.validate_deploy_paths()
9935
10134
 
9936
- await self._apps.prepare_app(cmd.spec)
10135
+ #
9937
10136
 
9938
- return DeployCommand.Output()
10137
+ await self._apps.prepare_app(spec)
9939
10138
 
9940
10139
 
9941
10140
  ########################################
@@ -10246,6 +10445,34 @@ DEFAULT_INTERP_RESOLVER = InterpResolver([(p.name, p) for p in [
10246
10445
  ]])
10247
10446
 
10248
10447
 
10448
+ ########################################
10449
+ # ../deploy/commands.py
10450
+
10451
+
10452
+ ##
10453
+
10454
+
10455
+ @dc.dataclass(frozen=True)
10456
+ class DeployCommand(Command['DeployCommand.Output']):
10457
+ spec: DeploySpec
10458
+
10459
+ @dc.dataclass(frozen=True)
10460
+ class Output(Command.Output):
10461
+ pass
10462
+
10463
+
10464
+ @dc.dataclass(frozen=True)
10465
+ class DeployCommandExecutor(CommandExecutor[DeployCommand, DeployCommand.Output]):
10466
+ _deploy: DeployManager
10467
+
10468
+ async def execute(self, cmd: DeployCommand) -> DeployCommand.Output:
10469
+ log.info('Deploying! %r', cmd.spec)
10470
+
10471
+ await self._deploy.run_deploy(cmd.spec)
10472
+
10473
+ return DeployCommand.Output()
10474
+
10475
+
10249
10476
  ########################################
10250
10477
  # ../targets/inject.py
10251
10478
 
@@ -10315,6 +10542,8 @@ def bind_deploy(
10315
10542
  ) -> InjectorBindings:
10316
10543
  lst: ta.List[InjectorBindingOrBindings] = [
10317
10544
  inj.bind(deploy_config),
10545
+
10546
+ bind_deploy_paths(),
10318
10547
  ]
10319
10548
 
10320
10549
  #
@@ -10326,11 +10555,6 @@ def bind_deploy(
10326
10555
  *([inj.bind(DeployPathOwner, to_key=cls, array=True)] if issubclass(cls, DeployPathOwner) else []),
10327
10556
  )
10328
10557
 
10329
- lst.extend([
10330
- inj.bind_array(DeployPathOwner),
10331
- inj.bind_array_type(DeployPathOwner, DeployPathOwners),
10332
- ])
10333
-
10334
10558
  #
10335
10559
 
10336
10560
  lst.extend([
@@ -10340,6 +10564,8 @@ def bind_deploy(
10340
10564
 
10341
10565
  bind_manager(DeployGitManager),
10342
10566
 
10567
+ bind_manager(DeployManager),
10568
+
10343
10569
  bind_manager(DeployTmpManager),
10344
10570
  inj.bind(AtomicPathSwapping, to_key=DeployTmpManager),
10345
10571