ominfra 0.0.0.dev154__py3-none-any.whl → 0.0.0.dev156__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (43) hide show
  1. ominfra/manage/bootstrap.py +4 -0
  2. ominfra/manage/bootstrap_.py +5 -0
  3. ominfra/manage/commands/inject.py +8 -11
  4. ominfra/manage/commands/{execution.py → local.py} +1 -5
  5. ominfra/manage/commands/ping.py +23 -0
  6. ominfra/manage/commands/subprocess.py +3 -4
  7. ominfra/manage/commands/types.py +8 -0
  8. ominfra/manage/deploy/apps.py +72 -0
  9. ominfra/manage/deploy/config.py +8 -0
  10. ominfra/manage/deploy/git.py +136 -0
  11. ominfra/manage/deploy/inject.py +21 -0
  12. ominfra/manage/deploy/paths.py +81 -28
  13. ominfra/manage/deploy/types.py +13 -0
  14. ominfra/manage/deploy/venvs.py +66 -0
  15. ominfra/manage/inject.py +20 -4
  16. ominfra/manage/main.py +15 -27
  17. ominfra/manage/remote/_main.py +1 -1
  18. ominfra/manage/remote/config.py +0 -2
  19. ominfra/manage/remote/connection.py +7 -24
  20. ominfra/manage/remote/execution.py +1 -1
  21. ominfra/manage/remote/inject.py +3 -14
  22. ominfra/manage/remote/spawning.py +2 -2
  23. ominfra/manage/system/commands.py +22 -2
  24. ominfra/manage/system/config.py +3 -1
  25. ominfra/manage/system/inject.py +16 -6
  26. ominfra/manage/system/packages.py +38 -14
  27. ominfra/manage/system/platforms.py +72 -0
  28. ominfra/manage/targets/__init__.py +0 -0
  29. ominfra/manage/targets/connection.py +150 -0
  30. ominfra/manage/targets/inject.py +42 -0
  31. ominfra/manage/targets/targets.py +87 -0
  32. ominfra/scripts/journald2aws.py +205 -134
  33. ominfra/scripts/manage.py +2192 -734
  34. ominfra/scripts/supervisor.py +187 -25
  35. ominfra/supervisor/configs.py +163 -18
  36. {ominfra-0.0.0.dev154.dist-info → ominfra-0.0.0.dev156.dist-info}/METADATA +3 -3
  37. {ominfra-0.0.0.dev154.dist-info → ominfra-0.0.0.dev156.dist-info}/RECORD +42 -31
  38. ominfra/manage/system/types.py +0 -5
  39. /ominfra/manage/{commands → deploy}/interp.py +0 -0
  40. {ominfra-0.0.0.dev154.dist-info → ominfra-0.0.0.dev156.dist-info}/LICENSE +0 -0
  41. {ominfra-0.0.0.dev154.dist-info → ominfra-0.0.0.dev156.dist-info}/WHEEL +0 -0
  42. {ominfra-0.0.0.dev154.dist-info → ominfra-0.0.0.dev156.dist-info}/entry_points.txt +0 -0
  43. {ominfra-0.0.0.dev154.dist-info → ominfra-0.0.0.dev156.dist-info}/top_level.txt +0 -0
ominfra/scripts/manage.py CHANGED
@@ -92,9 +92,16 @@ CallableVersionOperator = ta.Callable[['Version', str], bool]
92
92
  CommandT = ta.TypeVar('CommandT', bound='Command')
93
93
  CommandOutputT = ta.TypeVar('CommandOutputT', bound='Command.Output')
94
94
 
95
+ # deploy/paths.py
96
+ DeployPathKind = ta.Literal['dir', 'file'] # ta.TypeAlias
97
+ DeployPathSpec = ta.Literal['app', 'tag'] # ta.TypeAlias
98
+
95
99
  # ../../omlish/argparse/cli.py
96
100
  ArgparseCommandFn = ta.Callable[[], ta.Optional[int]] # ta.TypeAlias
97
101
 
102
+ # ../../omlish/lite/contextmanagers.py
103
+ ExitStackedT = ta.TypeVar('ExitStackedT', bound='ExitStacked')
104
+
98
105
  # ../../omlish/lite/inject.py
99
106
  U = ta.TypeVar('U')
100
107
  InjectorKeyCls = ta.Union[type, ta.NewType]
@@ -528,19 +535,28 @@ class MainConfig:
528
535
 
529
536
 
530
537
  ########################################
531
- # ../system/config.py
538
+ # ../deploy/config.py
532
539
 
533
540
 
534
541
  @dc.dataclass(frozen=True)
535
- class SystemConfig:
536
- platform: ta.Optional[str] = None
542
+ class DeployConfig:
543
+ deploy_home: ta.Optional[str] = None
537
544
 
538
545
 
539
546
  ########################################
540
- # ../system/types.py
547
+ # ../deploy/types.py
541
548
 
542
549
 
543
- SystemPlatform = ta.NewType('SystemPlatform', str)
550
+ DeployHome = ta.NewType('DeployHome', str)
551
+
552
+ DeployApp = ta.NewType('DeployApp', str)
553
+ DeployTag = ta.NewType('DeployTag', str)
554
+ DeployRev = ta.NewType('DeployRev', str)
555
+
556
+
557
+ class DeployAppTag(ta.NamedTuple):
558
+ app: DeployApp
559
+ tag: DeployTag
544
560
 
545
561
 
546
562
  ########################################
@@ -1221,8 +1237,6 @@ def async_cached_nullary(fn): # ta.Callable[..., T]) -> ta.Callable[..., T]:
1221
1237
  """
1222
1238
  TODO:
1223
1239
  - def maybe(v: lang.Maybe[T])
1224
- - patch / override lite.check ?
1225
- - checker interface?
1226
1240
  """
1227
1241
 
1228
1242
 
@@ -1937,6 +1951,489 @@ def set_process_deathsig(sig: int) -> bool:
1937
1951
  return False
1938
1952
 
1939
1953
 
1954
+ ########################################
1955
+ # ../../../omlish/os/linux.py
1956
+ """
1957
+ ➜ ~ cat /etc/os-release
1958
+ NAME="Amazon Linux"
1959
+ VERSION="2"
1960
+ ID="amzn"
1961
+ ID_LIKE="centos rhel fedora"
1962
+ VERSION_ID="2"
1963
+ PRETTY_NAME="Amazon Linux 2"
1964
+
1965
+ ➜ ~ cat /etc/os-release
1966
+ PRETTY_NAME="Ubuntu 22.04.5 LTS"
1967
+ NAME="Ubuntu"
1968
+ VERSION_ID="22.04"
1969
+ VERSION="22.04.5 LTS (Jammy Jellyfish)"
1970
+ VERSION_CODENAME=jammy
1971
+ ID=ubuntu
1972
+ ID_LIKE=debian
1973
+ UBUNTU_CODENAME=jammy
1974
+
1975
+ ➜ omlish git:(master) docker run -i python:3.12 cat /etc/os-release
1976
+ PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
1977
+ NAME="Debian GNU/Linux"
1978
+ VERSION_ID="12"
1979
+ VERSION="12 (bookworm)"
1980
+ VERSION_CODENAME=bookworm
1981
+ ID=debian
1982
+ """
1983
+
1984
+
1985
+ @dc.dataclass(frozen=True)
1986
+ class LinuxOsRelease:
1987
+ """
1988
+ https://man7.org/linux/man-pages/man5/os-release.5.html
1989
+ """
1990
+
1991
+ raw: ta.Mapping[str, str]
1992
+
1993
+ # General information identifying the operating system
1994
+
1995
+ @property
1996
+ def name(self) -> str:
1997
+ """
1998
+ A string identifying the operating system, without a version component, and suitable for presentation to the
1999
+ user. If not set, a default of "NAME=Linux" may be used.
2000
+
2001
+ Examples: "NAME=Fedora", "NAME="Debian GNU/Linux"".
2002
+ """
2003
+
2004
+ return self.raw['NAME']
2005
+
2006
+ @property
2007
+ def id(self) -> str:
2008
+ """
2009
+ A lower-case string (no spaces or other characters outside of 0-9, a-z, ".", "_" and "-") identifying the
2010
+ operating system, excluding any version information and suitable for processing by scripts or usage in generated
2011
+ filenames. If not set, a default of "ID=linux" may be used. Note that even though this string may not include
2012
+ characters that require shell quoting, quoting may nevertheless be used.
2013
+
2014
+ Examples: "ID=fedora", "ID=debian".
2015
+ """
2016
+
2017
+ return self.raw['ID']
2018
+
2019
+ @property
2020
+ def id_like(self) -> str:
2021
+ """
2022
+ A space-separated list of operating system identifiers in the same syntax as the ID= setting. It should list
2023
+ identifiers of operating systems that are closely related to the local operating system in regards to packaging
2024
+ and programming interfaces, for example listing one or more OS identifiers the local OS is a derivative from. An
2025
+ OS should generally only list other OS identifiers it itself is a derivative of, and not any OSes that are
2026
+ derived from it, though symmetric relationships are possible. Build scripts and similar should check this
2027
+ variable if they need to identify the local operating system and the value of ID= is not recognized. Operating
2028
+ systems should be listed in order of how closely the local operating system relates to the listed ones, starting
2029
+ with the closest. This field is optional.
2030
+
2031
+ Examples: for an operating system with "ID=centos", an assignment of "ID_LIKE="rhel fedora"" would be
2032
+ appropriate. For an operating system with "ID=ubuntu", an assignment of "ID_LIKE=debian" is appropriate.
2033
+ """
2034
+
2035
+ return self.raw['ID_LIKE']
2036
+
2037
+ @property
2038
+ def pretty_name(self) -> str:
2039
+ """
2040
+ A pretty operating system name in a format suitable for presentation to the user. May or may not contain a
2041
+ release code name or OS version of some kind, as suitable. If not set, a default of "PRETTY_NAME="Linux"" may be
2042
+ used
2043
+
2044
+ Example: "PRETTY_NAME="Fedora 17 (Beefy Miracle)"".
2045
+ """
2046
+
2047
+ return self.raw['PRETTY_NAME']
2048
+
2049
+ @property
2050
+ def cpe_name(self) -> str:
2051
+ """
2052
+ A CPE name for the operating system, in URI binding syntax, following the Common Platform Enumeration
2053
+ Specification[4] as proposed by the NIST. This field is optional.
2054
+
2055
+ Example: "CPE_NAME="cpe:/o:fedoraproject:fedora:17""
2056
+ """
2057
+
2058
+ return self.raw['CPE_NAME']
2059
+
2060
+ @property
2061
+ def variant(self) -> str:
2062
+ """
2063
+ A string identifying a specific variant or edition of the operating system suitable for presentation to the
2064
+ user. This field may be used to inform the user that the configuration of this system is subject to a specific
2065
+ divergent set of rules or default configuration settings. This field is optional and may not be implemented on
2066
+ all systems.
2067
+
2068
+ Examples: "VARIANT="Server Edition"", "VARIANT="Smart Refrigerator Edition"".
2069
+
2070
+ Note: this field is for display purposes only. The VARIANT_ID field should be used for making programmatic
2071
+ decisions.
2072
+
2073
+ Added in version 220.
2074
+ """
2075
+
2076
+ return self.raw['VARIANT']
2077
+
2078
+ @property
2079
+ def variant_id(self) -> str:
2080
+ """
2081
+ A lower-case string (no spaces or other characters outside of 0-9, a-z, ".", "_" and "-"), identifying a
2082
+ specific variant or edition of the operating system. This may be interpreted by other packages in order to
2083
+ determine a divergent default configuration. This field is optional and may not be implemented on all systems.
2084
+
2085
+ Examples: "VARIANT_ID=server", "VARIANT_ID=embedded".
2086
+
2087
+ Added in version 220.
2088
+ """
2089
+
2090
+ return self.raw['variant_id']
2091
+
2092
+ # Information about the version of the operating system
2093
+
2094
+ @property
2095
+ def version(self) -> str:
2096
+ """
2097
+ A string identifying the operating system version, excluding any OS name information, possibly including a
2098
+ release code name, and suitable for presentation to the user. This field is optional.
2099
+
2100
+ Examples: "VERSION=17", "VERSION="17 (Beefy Miracle)"".
2101
+ """
2102
+
2103
+ return self.raw['VERSION']
2104
+
2105
+ @property
2106
+ def version_id(self) -> str:
2107
+ """
2108
+ A lower-case string (mostly numeric, no spaces or other characters outside of 0-9, a-z, ".", "_" and "-")
2109
+ identifying the operating system version, excluding any OS name information or release code name, and suitable
2110
+ for processing by scripts or usage in generated filenames. This field is optional.
2111
+
2112
+ Examples: "VERSION_ID=17", "VERSION_ID=11.04".
2113
+ """
2114
+
2115
+ return self.raw['VERSION_ID']
2116
+
2117
+ @property
2118
+ def version_codename(self) -> str:
2119
+ """
2120
+ A lower-case string (no spaces or other characters outside of 0-9, a-z, ".", "_" and "-") identifying the
2121
+ operating system release code name, excluding any OS name information or release version, and suitable for
2122
+ processing by scripts or usage in generated filenames. This field is optional and may not be implemented on all
2123
+ systems.
2124
+
2125
+ Examples: "VERSION_CODENAME=buster", "VERSION_CODENAME=xenial".
2126
+
2127
+ Added in version 231.
2128
+ """
2129
+
2130
+ return self.raw['VERSION_CODENAME']
2131
+
2132
+ @property
2133
+ def build_id(self) -> str:
2134
+ """
2135
+ A string uniquely identifying the system image originally used as the installation base. In most cases,
2136
+ VERSION_ID or IMAGE_ID+IMAGE_VERSION are updated when the entire system image is replaced during an update.
2137
+ BUILD_ID may be used in distributions where the original installation image version is important: VERSION_ID
2138
+ would change during incremental system updates, but BUILD_ID would not. This field is optional.
2139
+
2140
+ Examples: "BUILD_ID="2013-03-20.3"", "BUILD_ID=201303203".
2141
+
2142
+ Added in version 200.
2143
+ """
2144
+
2145
+ return self.raw['BUILD_ID']
2146
+
2147
+ @property
2148
+ def image_id(self) -> str:
2149
+ """
2150
+ A lower-case string (no spaces or other characters outside of 0-9, a-z, ".", "_" and "-"), identifying a
2151
+ specific image of the operating system. This is supposed to be used for environments where OS images are
2152
+ prepared, built, shipped and updated as comprehensive, consistent OS images. This field is optional and may not
2153
+ be implemented on all systems, in particularly not on those that are not managed via images but put together and
2154
+ updated from individual packages and on the local system.
2155
+
2156
+ Examples: "IMAGE_ID=vendorx-cashier-system", "IMAGE_ID=netbook-image".
2157
+
2158
+ Added in version 249.
2159
+ """
2160
+
2161
+ return self.raw['IMAGE_ID']
2162
+
2163
+ @property
2164
+ def image_version(self) -> str:
2165
+ """
2166
+ A lower-case string (mostly numeric, no spaces or other characters outside of 0-9, a-z, ".", "_" and "-")
2167
+ identifying the OS image version. This is supposed to be used together with IMAGE_ID described above, to discern
2168
+ different versions of the same image.
2169
+
2170
+ Examples: "IMAGE_VERSION=33", "IMAGE_VERSION=47.1rc1".
2171
+
2172
+ Added in version 249.
2173
+ """
2174
+
2175
+ return self.raw['IMAGE_VERSION']
2176
+
2177
+ # To summarize: if the image updates are built and shipped as comprehensive units, IMAGE_ID+IMAGE_VERSION is the
2178
+ # best fit. Otherwise, if updates eventually completely replace previously installed contents, as in a typical
2179
+ # binary distribution, VERSION_ID should be used to identify major releases of the operating system. BUILD_ID may
2180
+ # be used instead or in addition to VERSION_ID when the original system image version is important.
2181
+
2182
+ #
2183
+
2184
+ # Presentation information and links
2185
+
2186
+ # Links to resources on the Internet related to the operating system. HOME_URL= should refer to the homepage of the
2187
+ # operating system, or alternatively some homepage of the specific version of the operating system.
2188
+ # DOCUMENTATION_URL= should refer to the main documentation page for this operating system. SUPPORT_URL= should
2189
+ # refer to the main support page for the operating system, if there is any. This is primarily intended for operating
2190
+ # systems which vendors provide support for. BUG_REPORT_URL= should refer to the main bug reporting page for the
2191
+ # operating system, if there is any. This is primarily intended for operating systems that rely on community QA.
2192
+ # PRIVACY_POLICY_URL= should refer to the main privacy policy page for the operating system, if there is any. These
2193
+ # settings are optional, and providing only some of these settings is common. These URLs are intended to be exposed
2194
+ # in "About this system" UIs behind links with captions such as "About this Operating System", "Obtain Support",
2195
+ # "Report a Bug", or "Privacy Policy". The values should be in RFC3986 format[5], and should be "http:" or "https:"
2196
+ # URLs, and possibly "mailto:" or "tel:". Only one URL shall be listed in each setting. If multiple resources need
2197
+ # to be referenced, it is recommended to provide an online landing page linking all available resources.
2198
+
2199
+ # Examples: "HOME_URL="https://fedoraproject.org/"", "BUG_REPORT_URL="https://bugzilla.redhat.com/"".
2200
+
2201
+ @property
2202
+ def home_url(self) -> str:
2203
+ return self.raw['HOME_URL']
2204
+
2205
+ @property
2206
+ def documentation_url(self) -> str:
2207
+ return self.raw['DOCUMENTATION_URL']
2208
+
2209
+ @property
2210
+ def support_url(self) -> str:
2211
+ return self.raw['SUPPORT_URL']
2212
+
2213
+ @property
2214
+ def bug_report_url(self) -> str:
2215
+ return self.raw['BUG_REPORT_URL']
2216
+
2217
+ @property
2218
+ def privacy_policy_url(self) -> str:
2219
+ return self.raw['PRIVACY_POLICY_URL']
2220
+
2221
+ @property
2222
+ def support_end(self) -> str:
2223
+ """
2224
+ The date at which support for this version of the OS ends. (What exactly "lack of support" means varies between
2225
+ vendors, but generally users should assume that updates, including security fixes, will not be provided.) The
2226
+ value is a date in the ISO 8601 format "YYYY-MM-DD", and specifies the first day on which support is not
2227
+ provided.
2228
+
2229
+ For example, "SUPPORT_END=2001-01-01" means that the system was supported until the end of the last day of the
2230
+ previous millennium.
2231
+
2232
+ Added in version 252.
2233
+ """
2234
+
2235
+ return self.raw['SUPPORT_END']
2236
+
2237
+ @property
2238
+ def logo(self) -> str:
2239
+ """
2240
+ A string, specifying the name of an icon as defined by freedesktop.org Icon Theme Specification[6]. This can be
2241
+ used by graphical applications to display an operating system's or distributor's logo. This field is optional
2242
+ and may not necessarily be implemented on all systems.
2243
+
2244
+ Examples: "LOGO=fedora-logo", "LOGO=distributor-logo-opensuse"
2245
+
2246
+ Added in version 240.
2247
+ """
2248
+
2249
+ return self.raw['LOGO']
2250
+
2251
+ @property
2252
+ def ansi_color(self) -> str:
2253
+ """
2254
+ A suggested presentation color when showing the OS name on the console. This should be specified as string
2255
+ suitable for inclusion in the ESC [ m ANSI/ECMA-48 escape code for setting graphical rendition. This field is
2256
+ optional.
2257
+
2258
+ Examples: "ANSI_COLOR="0;31"" for red, "ANSI_COLOR="1;34"" for light blue, or "ANSI_COLOR="0;38;2;60;110;180""
2259
+ for Fedora blue.
2260
+ """
2261
+
2262
+ return self.raw['ANSI_COLOR']
2263
+
2264
+ @property
2265
+ def vendor_name(self) -> str:
2266
+ """
2267
+ The name of the OS vendor. This is the name of the organization or company which produces the OS. This field is
2268
+ optional.
2269
+
2270
+ This name is intended to be exposed in "About this system" UIs or software update UIs when needed to distinguish
2271
+ the OS vendor from the OS itself. It is intended to be human readable.
2272
+
2273
+ Examples: "VENDOR_NAME="Fedora Project"" for Fedora Linux, "VENDOR_NAME="Canonical"" for Ubuntu.
2274
+
2275
+ Added in version 254.
2276
+ """
2277
+
2278
+ return self.raw['VENDOR_NAME']
2279
+
2280
+ @property
2281
+ def vendor_url(self) -> str:
2282
+ """
2283
+ The homepage of the OS vendor. This field is optional. The VENDOR_NAME= field should be set if this one is,
2284
+ although clients must be robust against either field not being set.
2285
+
2286
+ The value should be in RFC3986 format[5], and should be "http:" or "https:" URLs. Only one URL shall be listed
2287
+ in the setting.
2288
+
2289
+ Examples: "VENDOR_URL="https://fedoraproject.org/"", "VENDOR_URL="https://canonical.com/"".
2290
+
2291
+ Added in version 254.
2292
+ """
2293
+
2294
+ return self.raw['VENDOR_URL']
2295
+
2296
+ # Distribution-level defaults and metadata
2297
+
2298
+ @property
2299
+ def default_hostname(self) -> str:
2300
+ """
2301
+ A string specifying the hostname if hostname(5) is not present and no other configuration source specifies the
2302
+ hostname. Must be either a single DNS label (a string composed of 7-bit ASCII lower-case characters and no
2303
+ spaces or dots, limited to the format allowed for DNS domain name labels), or a sequence of such labels
2304
+ separated by single dots that forms a valid DNS FQDN. The hostname must be at most 64 characters, which is a
2305
+ Linux limitation (DNS allows longer names).
2306
+
2307
+ See org.freedesktop.hostname1(5) for a description of how systemd-hostnamed.service(8) determines the fallback
2308
+ hostname.
2309
+
2310
+ Added in version 248.
2311
+ """
2312
+
2313
+ return self.raw['DEFAULT_HOSTNAME']
2314
+
2315
+ @property
2316
+ def architecture(self) -> str:
2317
+ """
2318
+ A string that specifies which CPU architecture the userspace binaries require. The architecture identifiers are
2319
+ the same as for ConditionArchitecture= described in systemd.unit(5). The field is optional and should only be
2320
+ used when just single architecture is supported. It may provide redundant information when used in a GPT
2321
+ partition with a GUID type that already encodes the architecture. If this is not the case, the architecture
2322
+ should be specified in e.g., an extension image, to prevent an incompatible host from loading it.
2323
+
2324
+ Added in version 252.
2325
+ """
2326
+
2327
+ return self.raw['ARCHITECTURE']
2328
+
2329
+ @property
2330
+ def sysext_level(self) -> str:
2331
+ """
2332
+ A lower-case string (mostly numeric, no spaces or other characters outside of 0-9, a-z, ".", "_" and "-")
2333
+ identifying the operating system extensions support level, to indicate which extension images are supported. See
2334
+ /usr/lib/extension-release.d/extension-release.IMAGE, initrd[2] and systemd-sysext(8)) for more information.
2335
+
2336
+ Examples: "SYSEXT_LEVEL=2", "SYSEXT_LEVEL=15.14".
2337
+
2338
+ Added in version 248.
2339
+ """
2340
+
2341
+ return self.raw['SYSEXT_LEVEL']
2342
+
2343
+ @property
2344
+ def confext_level(self) -> str:
2345
+ """
2346
+ Semantically the same as SYSEXT_LEVEL= but for confext images. See
2347
+ /etc/extension-release.d/extension-release.IMAGE for more information.
2348
+
2349
+ Examples: "CONFEXT_LEVEL=2", "CONFEXT_LEVEL=15.14".
2350
+
2351
+ Added in version 254.
2352
+ """
2353
+
2354
+ return self.raw['CONFEXT_LEVEL']
2355
+
2356
+ @property
2357
+ def sysext_scope(self) -> str:
2358
+ """
2359
+ Takes a space-separated list of one or more of the strings "system", "initrd" and "portable". This field is only
2360
+ supported in extension-release.d/ files and indicates what environments the system extension is applicable to:
2361
+ i.e. to regular systems, to initrds, or to portable service images. If unspecified, "SYSEXT_SCOPE=system
2362
+ portable" is implied, i.e. any system extension without this field is applicable to regular systems and to
2363
+ portable service environments, but not to initrd environments.
2364
+
2365
+ Added in version 250.
2366
+ """
2367
+
2368
+ return self.raw['SYSEXT_SCOPE']
2369
+
2370
+ @property
2371
+ def confext_scope(self) -> str:
2372
+ """
2373
+ Semantically the same as SYSEXT_SCOPE= but for confext images.
2374
+
2375
+ Added in version 254.
2376
+ """
2377
+
2378
+ return self.raw['CONFEXT_SCOPE']
2379
+
2380
+ @property
2381
+ def portable_prefixes(self) -> str:
2382
+ """
2383
+ Takes a space-separated list of one or more valid prefix match strings for the Portable Services[3] logic. This
2384
+ field serves two purposes: it is informational, identifying portable service images as such (and thus allowing
2385
+ them to be distinguished from other OS images, such as bootable system images). It is also used when a portable
2386
+ service image is attached: the specified or implied portable service prefix is checked against the list
2387
+ specified here, to enforce restrictions how images may be attached to a system.
2388
+
2389
+ Added in version 250.
2390
+ """
2391
+
2392
+ return self.raw['PORTABLE_PREFIXES']
2393
+
2394
+ #
2395
+
2396
+ DEFAULT_PATHS: ta.ClassVar[ta.Sequence[str]] = [
2397
+ '/etc/os-release',
2398
+ '/usr/lib/os-release',
2399
+ ]
2400
+
2401
+ @classmethod
2402
+ def read(cls, *paths: str) -> ta.Optional['LinuxOsRelease']:
2403
+ for fp in (paths or cls.DEFAULT_PATHS):
2404
+ if not os.path.isfile(fp):
2405
+ continue
2406
+ with open(fp) as f:
2407
+ src = f.read()
2408
+ break
2409
+ else:
2410
+ return None
2411
+
2412
+ raw = cls._parse_os_release(src)
2413
+
2414
+ return cls(raw)
2415
+
2416
+ @classmethod
2417
+ def _parse_os_release(cls, src: str) -> ta.Mapping[str, str]:
2418
+ dct: ta.Dict[str, str] = {}
2419
+
2420
+ for l in src.splitlines():
2421
+ if not (l := l.strip()):
2422
+ continue
2423
+ if l.startswith('#') or '=' not in l:
2424
+ continue
2425
+
2426
+ k, _, v = l.partition('=')
2427
+ if k.startswith('"'):
2428
+ k = k[1:-1]
2429
+ if v.startswith('"'):
2430
+ v = v[1:-1]
2431
+
2432
+ dct[k] = v
2433
+
2434
+ return dct
2435
+
2436
+
1940
2437
  ########################################
1941
2438
  # ../../../omdev/packaging/specifiers.py
1942
2439
  # Copyright (c) Donald Stufft and individual contributors.
@@ -2610,43 +3107,264 @@ def build_command_name_map(crs: CommandRegistrations) -> CommandNameMap:
2610
3107
 
2611
3108
 
2612
3109
  ########################################
2613
- # ../remote/config.py
3110
+ # ../deploy/paths.py
3111
+ """
3112
+ ~deploy
3113
+ deploy.pid (flock)
3114
+ /app
3115
+ /<appspec> - shallow clone
3116
+ /conf
3117
+ /env
3118
+ <appspec>.env
3119
+ /nginx
3120
+ <appspec>.conf
3121
+ /supervisor
3122
+ <appspec>.conf
3123
+ /venv
3124
+ /<appspec>
3125
+
3126
+ ?
3127
+ /logs
3128
+ /wrmsr--omlish--<spec>
3129
+
3130
+ spec = <name>--<rev>--<when>
3131
+
3132
+ ==
3133
+
3134
+ for dn in [
3135
+ 'app',
3136
+ 'conf',
3137
+ 'conf/env',
3138
+ 'conf/nginx',
3139
+ 'conf/supervisor',
3140
+ 'venv',
3141
+ ]:
3142
+
3143
+ ==
2614
3144
 
3145
+ """
2615
3146
 
2616
- @dc.dataclass(frozen=True)
2617
- class RemoteConfig:
2618
- payload_file: ta.Optional[str] = None
2619
3147
 
2620
- set_pgid: bool = True
3148
+ ##
2621
3149
 
2622
- deathsig: ta.Optional[str] = 'KILL'
2623
3150
 
2624
- pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
3151
+ DEPLOY_PATH_SPEC_PLACEHOLDER = '@'
3152
+ DEPLOY_PATH_SPEC_SEPARATORS = '-.'
2625
3153
 
2626
- forward_logging: bool = True
3154
+ DEPLOY_PATH_SPECS: ta.FrozenSet[str] = frozenset([
3155
+ 'app',
3156
+ 'tag', # <rev>-<dt>
3157
+ ])
2627
3158
 
2628
- timebomb_delay_s: ta.Optional[float] = 60 * 60.
2629
3159
 
2630
- heartbeat_interval_s: float = 3.
3160
+ class DeployPathError(Exception):
3161
+ pass
2631
3162
 
2632
- use_in_process_remote_executor: bool = False
2633
3163
 
3164
+ @dc.dataclass(frozen=True)
3165
+ class DeployPathPart(abc.ABC): # noqa
3166
+ @property
3167
+ @abc.abstractmethod
3168
+ def kind(self) -> DeployPathKind:
3169
+ raise NotImplementedError
2634
3170
 
2635
- ########################################
2636
- # ../remote/payload.py
3171
+ @abc.abstractmethod
3172
+ def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
3173
+ raise NotImplementedError
2637
3174
 
2638
3175
 
2639
- RemoteExecutionPayloadFile = ta.NewType('RemoteExecutionPayloadFile', str)
3176
+ #
2640
3177
 
2641
3178
 
2642
- @cached_nullary
2643
- def _get_self_src() -> str:
2644
- return inspect.getsource(sys.modules[__name__])
3179
+ class DirDeployPathPart(DeployPathPart, abc.ABC):
3180
+ @property
3181
+ def kind(self) -> DeployPathKind:
3182
+ return 'dir'
2645
3183
 
3184
+ @classmethod
3185
+ def parse(cls, s: str) -> 'DirDeployPathPart':
3186
+ if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
3187
+ check.equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
3188
+ return SpecDirDeployPathPart(s[1:])
3189
+ else:
3190
+ return ConstDirDeployPathPart(s)
2646
3191
 
2647
- def _is_src_amalg(src: str) -> bool:
2648
- for l in src.splitlines(): # noqa
2649
- if l.startswith('# @omlish-amalg-output '):
3192
+
3193
+ class FileDeployPathPart(DeployPathPart, abc.ABC):
3194
+ @property
3195
+ def kind(self) -> DeployPathKind:
3196
+ return 'file'
3197
+
3198
+ @classmethod
3199
+ def parse(cls, s: str) -> 'FileDeployPathPart':
3200
+ if DEPLOY_PATH_SPEC_PLACEHOLDER in s:
3201
+ check.equal(s[0], DEPLOY_PATH_SPEC_PLACEHOLDER)
3202
+ if not any(c in s for c in DEPLOY_PATH_SPEC_SEPARATORS):
3203
+ return SpecFileDeployPathPart(s[1:], '')
3204
+ else:
3205
+ p = min(f for c in DEPLOY_PATH_SPEC_SEPARATORS if (f := s.find(c)) > 0)
3206
+ return SpecFileDeployPathPart(s[1:p], s[p:])
3207
+ else:
3208
+ return ConstFileDeployPathPart(s)
3209
+
3210
+
3211
+ #
3212
+
3213
+
3214
+ @dc.dataclass(frozen=True)
3215
+ class ConstDeployPathPart(DeployPathPart, abc.ABC):
3216
+ name: str
3217
+
3218
+ def __post_init__(self) -> None:
3219
+ check.non_empty_str(self.name)
3220
+ check.not_in('/', self.name)
3221
+ check.not_in(DEPLOY_PATH_SPEC_PLACEHOLDER, self.name)
3222
+
3223
+ def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
3224
+ return self.name
3225
+
3226
+
3227
+ class ConstDirDeployPathPart(ConstDeployPathPart, DirDeployPathPart):
3228
+ pass
3229
+
3230
+
3231
+ class ConstFileDeployPathPart(ConstDeployPathPart, FileDeployPathPart):
3232
+ pass
3233
+
3234
+
3235
+ #
3236
+
3237
+
3238
+ @dc.dataclass(frozen=True)
3239
+ class SpecDeployPathPart(DeployPathPart, abc.ABC):
3240
+ spec: str # DeployPathSpec
3241
+
3242
+ def __post_init__(self) -> None:
3243
+ check.non_empty_str(self.spec)
3244
+ for c in [*DEPLOY_PATH_SPEC_SEPARATORS, DEPLOY_PATH_SPEC_PLACEHOLDER, '/']:
3245
+ check.not_in(c, self.spec)
3246
+ check.in_(self.spec, DEPLOY_PATH_SPECS)
3247
+
3248
+ def _render_spec(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
3249
+ if specs is not None:
3250
+ return specs[self.spec] # type: ignore
3251
+ else:
3252
+ return DEPLOY_PATH_SPEC_PLACEHOLDER + self.spec
3253
+
3254
+
3255
+ @dc.dataclass(frozen=True)
3256
+ class SpecDirDeployPathPart(SpecDeployPathPart, DirDeployPathPart):
3257
+ def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
3258
+ return self._render_spec(specs)
3259
+
3260
+
3261
+ @dc.dataclass(frozen=True)
3262
+ class SpecFileDeployPathPart(SpecDeployPathPart, FileDeployPathPart):
3263
+ suffix: str
3264
+
3265
+ def __post_init__(self) -> None:
3266
+ super().__post_init__()
3267
+ if self.suffix:
3268
+ for c in [DEPLOY_PATH_SPEC_PLACEHOLDER, '/']:
3269
+ check.not_in(c, self.suffix)
3270
+
3271
+ def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
3272
+ return self._render_spec(specs) + self.suffix
3273
+
3274
+
3275
+ ##
3276
+
3277
+
3278
+ @dc.dataclass(frozen=True)
3279
+ class DeployPath:
3280
+ parts: ta.Sequence[DeployPathPart]
3281
+
3282
+ def __post_init__(self) -> None:
3283
+ check.not_empty(self.parts)
3284
+ for p in self.parts[:-1]:
3285
+ check.equal(p.kind, 'dir')
3286
+
3287
+ pd = {}
3288
+ for i, p in enumerate(self.parts):
3289
+ if isinstance(p, SpecDeployPathPart):
3290
+ if p.spec in pd:
3291
+ raise DeployPathError('Duplicate specs in path', self)
3292
+ pd[p.spec] = i
3293
+
3294
+ if 'tag' in pd:
3295
+ if 'app' not in pd or pd['app'] >= pd['tag']:
3296
+ raise DeployPathError('Tag spec in path without preceding app', self)
3297
+
3298
+ @property
3299
+ def kind(self) -> ta.Literal['file', 'dir']:
3300
+ return self.parts[-1].kind
3301
+
3302
+ def render(self, specs: ta.Optional[ta.Mapping[DeployPathSpec, str]] = None) -> str:
3303
+ return os.path.join( # noqa
3304
+ *[p.render(specs) for p in self.parts],
3305
+ *([''] if self.kind == 'dir' else []),
3306
+ )
3307
+
3308
+ @classmethod
3309
+ def parse(cls, s: str) -> 'DeployPath':
3310
+ tail_parse: ta.Callable[[str], DeployPathPart]
3311
+ if s.endswith('/'):
3312
+ tail_parse = DirDeployPathPart.parse
3313
+ s = s[:-1]
3314
+ else:
3315
+ tail_parse = FileDeployPathPart.parse
3316
+ ps = check.non_empty_str(s).split('/')
3317
+ return cls([
3318
+ *([DirDeployPathPart.parse(p) for p in ps[:-1]] if len(ps) > 1 else []),
3319
+ tail_parse(ps[-1]),
3320
+ ])
3321
+
3322
+
3323
+ ##
3324
+
3325
+
3326
+ class DeployPathOwner(abc.ABC):
3327
+ @abc.abstractmethod
3328
+ def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
3329
+ raise NotImplementedError
3330
+
3331
+
3332
+ ########################################
3333
+ # ../remote/config.py
3334
+
3335
+
3336
+ @dc.dataclass(frozen=True)
3337
+ class RemoteConfig:
3338
+ payload_file: ta.Optional[str] = None
3339
+
3340
+ set_pgid: bool = True
3341
+
3342
+ deathsig: ta.Optional[str] = 'KILL'
3343
+
3344
+ pycharm_remote_debug: ta.Optional[PycharmRemoteDebug] = None
3345
+
3346
+ forward_logging: bool = True
3347
+
3348
+ timebomb_delay_s: ta.Optional[float] = 60 * 60.
3349
+
3350
+ heartbeat_interval_s: float = 3.
3351
+
3352
+
3353
+ ########################################
3354
+ # ../remote/payload.py
3355
+
3356
+
3357
+ RemoteExecutionPayloadFile = ta.NewType('RemoteExecutionPayloadFile', str)
3358
+
3359
+
3360
+ @cached_nullary
3361
+ def _get_self_src() -> str:
3362
+ return inspect.getsource(sys.modules[__name__])
3363
+
3364
+
3365
+ def _is_src_amalg(src: str) -> bool:
3366
+ for l in src.splitlines(): # noqa
3367
+ if l.startswith('# @omlish-amalg-output '):
2650
3368
  return True
2651
3369
  return False
2652
3370
 
@@ -2671,6 +3389,90 @@ def get_remote_payload_src(
2671
3389
  return importlib.resources.files(__package__.split('.')[0] + '.scripts').joinpath('manage.py').read_text()
2672
3390
 
2673
3391
 
3392
+ ########################################
3393
+ # ../targets/targets.py
3394
+ """
3395
+ It's desugaring. Subprocess and locals are only leafs. Retain an origin?
3396
+ ** TWO LAYERS ** - ManageTarget is user-facing, ConnectorTarget is bound, internal
3397
+ """
3398
+
3399
+
3400
+ ##
3401
+
3402
+
3403
+ class ManageTarget(abc.ABC): # noqa
3404
+ def __init_subclass__(cls, **kwargs: ta.Any) -> None:
3405
+ super().__init_subclass__(**kwargs)
3406
+
3407
+ check.state(cls.__name__.endswith('ManageTarget'))
3408
+
3409
+
3410
+ #
3411
+
3412
+
3413
+ @dc.dataclass(frozen=True)
3414
+ class PythonRemoteManageTarget:
3415
+ DEFAULT_PYTHON: ta.ClassVar[str] = 'python3'
3416
+ python: str = DEFAULT_PYTHON
3417
+
3418
+
3419
+ #
3420
+
3421
+
3422
+ class RemoteManageTarget(ManageTarget, abc.ABC):
3423
+ pass
3424
+
3425
+
3426
+ class PhysicallyRemoteManageTarget(RemoteManageTarget, abc.ABC):
3427
+ pass
3428
+
3429
+
3430
+ class LocalManageTarget(ManageTarget, abc.ABC):
3431
+ pass
3432
+
3433
+
3434
+ ##
3435
+
3436
+
3437
+ @dc.dataclass(frozen=True)
3438
+ class SshManageTarget(PhysicallyRemoteManageTarget, PythonRemoteManageTarget):
3439
+ host: ta.Optional[str] = None
3440
+ username: ta.Optional[str] = None
3441
+ key_file: ta.Optional[str] = None
3442
+
3443
+ def __post_init__(self) -> None:
3444
+ check.non_empty_str(self.host)
3445
+
3446
+
3447
+ ##
3448
+
3449
+
3450
+ @dc.dataclass(frozen=True)
3451
+ class DockerManageTarget(RemoteManageTarget, PythonRemoteManageTarget): # noqa
3452
+ image: ta.Optional[str] = None
3453
+ container_id: ta.Optional[str] = None
3454
+
3455
+ def __post_init__(self) -> None:
3456
+ check.arg(bool(self.image) ^ bool(self.container_id))
3457
+
3458
+
3459
+ ##
3460
+
3461
+
3462
+ @dc.dataclass(frozen=True)
3463
+ class InProcessManageTarget(LocalManageTarget):
3464
+ class Mode(enum.Enum):
3465
+ DIRECT = enum.auto()
3466
+ FAKE_REMOTE = enum.auto()
3467
+
3468
+ mode: Mode = Mode.DIRECT
3469
+
3470
+
3471
+ @dc.dataclass(frozen=True)
3472
+ class SubprocessManageTarget(LocalManageTarget, PythonRemoteManageTarget):
3473
+ pass
3474
+
3475
+
2674
3476
  ########################################
2675
3477
  # ../../../omlish/argparse/cli.py
2676
3478
  """
@@ -2943,6 +3745,78 @@ class ArgparseCli:
2943
3745
  return await fn()
2944
3746
 
2945
3747
 
3748
+ ########################################
3749
+ # ../../../omlish/lite/contextmanagers.py
3750
+
3751
+
3752
+ ##
3753
+
3754
+
3755
+ class ExitStacked:
3756
+ _exit_stack: ta.Optional[contextlib.ExitStack] = None
3757
+
3758
+ def __enter__(self: ExitStackedT) -> ExitStackedT:
3759
+ check.state(self._exit_stack is None)
3760
+ es = self._exit_stack = contextlib.ExitStack()
3761
+ es.__enter__()
3762
+ return self
3763
+
3764
+ def __exit__(self, exc_type, exc_val, exc_tb):
3765
+ if (es := self._exit_stack) is None:
3766
+ return None
3767
+ self._exit_contexts()
3768
+ return es.__exit__(exc_type, exc_val, exc_tb)
3769
+
3770
+ def _exit_contexts(self) -> None:
3771
+ pass
3772
+
3773
+ def _enter_context(self, cm: ta.ContextManager[T]) -> T:
3774
+ es = check.not_none(self._exit_stack)
3775
+ return es.enter_context(cm)
3776
+
3777
+
3778
+ ##
3779
+
3780
+
3781
+ @contextlib.contextmanager
3782
+ def defer(fn: ta.Callable) -> ta.Generator[ta.Callable, None, None]:
3783
+ try:
3784
+ yield fn
3785
+ finally:
3786
+ fn()
3787
+
3788
+
3789
+ @contextlib.contextmanager
3790
+ def attr_setting(obj, attr, val, *, default=None): # noqa
3791
+ not_set = object()
3792
+ orig = getattr(obj, attr, not_set)
3793
+ try:
3794
+ setattr(obj, attr, val)
3795
+ if orig is not not_set:
3796
+ yield orig
3797
+ else:
3798
+ yield default
3799
+ finally:
3800
+ if orig is not_set:
3801
+ delattr(obj, attr)
3802
+ else:
3803
+ setattr(obj, attr, orig)
3804
+
3805
+
3806
+ ##
3807
+
3808
+
3809
+ class aclosing(contextlib.AbstractAsyncContextManager): # noqa
3810
+ def __init__(self, thing):
3811
+ self.thing = thing
3812
+
3813
+ async def __aenter__(self):
3814
+ return self.thing
3815
+
3816
+ async def __aexit__(self, *exc_info):
3817
+ await self.thing.aclose()
3818
+
3819
+
2946
3820
  ########################################
2947
3821
  # ../../../omlish/lite/inject.py
2948
3822
 
@@ -4111,7 +4985,8 @@ def configure_standard_logging(
4111
4985
  """
4112
4986
  TODO:
4113
4987
  - pickle stdlib objs? have to pin to 3.8 pickle protocol, will be cross-version
4114
- - nonstrict toggle
4988
+ - namedtuple
4989
+ - literals
4115
4990
  """
4116
4991
 
4117
4992
 
@@ -4401,14 +5276,18 @@ class ObjMarshalerManager:
4401
5276
  ) -> ObjMarshaler:
4402
5277
  if isinstance(ty, type):
4403
5278
  if abc.ABC in ty.__bases__:
4404
- return PolymorphicObjMarshaler.of([ # type: ignore
5279
+ impls = [ity for ity in deep_subclasses(ty) if abc.ABC not in ity.__bases__] # type: ignore
5280
+ if all(ity.__qualname__.endswith(ty.__name__) for ity in impls):
5281
+ ins = {ity: snake_case(ity.__qualname__[:-len(ty.__name__)]) for ity in impls}
5282
+ else:
5283
+ ins = {ity: ity.__qualname__ for ity in impls}
5284
+ return PolymorphicObjMarshaler.of([
4405
5285
  PolymorphicObjMarshaler.Impl(
4406
5286
  ity,
4407
- ity.__qualname__,
5287
+ itn,
4408
5288
  rec(ity),
4409
5289
  )
4410
- for ity in deep_subclasses(ty)
4411
- if abc.ABC not in ity.__bases__
5290
+ for ity, itn in ins.items()
4412
5291
  ])
4413
5292
 
4414
5293
  if issubclass(ty, enum.Enum):
@@ -4655,41 +5534,6 @@ class Interp:
4655
5534
  version: InterpVersion
4656
5535
 
4657
5536
 
4658
- ########################################
4659
- # ../bootstrap.py
4660
-
4661
-
4662
- @dc.dataclass(frozen=True)
4663
- class MainBootstrap:
4664
- main_config: MainConfig = MainConfig()
4665
-
4666
- remote_config: RemoteConfig = RemoteConfig()
4667
-
4668
- system_config: SystemConfig = SystemConfig()
4669
-
4670
-
4671
- ########################################
4672
- # ../commands/execution.py
4673
-
4674
-
4675
- CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
4676
-
4677
-
4678
- class LocalCommandExecutor(CommandExecutor):
4679
- def __init__(
4680
- self,
4681
- *,
4682
- command_executors: CommandExecutorMap,
4683
- ) -> None:
4684
- super().__init__()
4685
-
4686
- self._command_executors = command_executors
4687
-
4688
- async def execute(self, cmd: Command) -> Command.Output:
4689
- ce: CommandExecutor = self._command_executors[type(cmd)]
4690
- return await ce.execute(cmd)
4691
-
4692
-
4693
5537
  ########################################
4694
5538
  # ../commands/marshal.py
4695
5539
 
@@ -4715,6 +5559,34 @@ def install_command_marshaling(
4715
5559
  )
4716
5560
 
4717
5561
 
5562
+ ########################################
5563
+ # ../commands/ping.py
5564
+
5565
+
5566
+ ##
5567
+
5568
+
5569
+ @dc.dataclass(frozen=True)
5570
+ class PingCommand(Command['PingCommand.Output']):
5571
+ time: float = dc.field(default_factory=time.time)
5572
+
5573
+ @dc.dataclass(frozen=True)
5574
+ class Output(Command.Output):
5575
+ time: float
5576
+
5577
+
5578
+ class PingCommandExecutor(CommandExecutor[PingCommand, PingCommand.Output]):
5579
+ async def execute(self, cmd: PingCommand) -> PingCommand.Output:
5580
+ return PingCommand.Output(cmd.time)
5581
+
5582
+
5583
+ ########################################
5584
+ # ../commands/types.py
5585
+
5586
+
5587
+ CommandExecutorMap = ta.NewType('CommandExecutorMap', ta.Mapping[ta.Type[Command], CommandExecutor])
5588
+
5589
+
4718
5590
  ########################################
4719
5591
  # ../deploy/commands.py
4720
5592
 
@@ -4828,24 +5700,72 @@ class RemoteChannelImpl(RemoteChannel):
4828
5700
 
4829
5701
 
4830
5702
  ########################################
4831
- # ../system/commands.py
5703
+ # ../system/platforms.py
4832
5704
 
4833
5705
 
4834
5706
  ##
4835
5707
 
4836
5708
 
4837
5709
  @dc.dataclass(frozen=True)
4838
- class CheckSystemPackageCommand(Command['CheckSystemPackageCommand.Output']):
4839
- @dc.dataclass(frozen=True)
4840
- class Output(Command.Output):
4841
- pass
5710
+ class Platform(abc.ABC): # noqa
5711
+ pass
4842
5712
 
4843
5713
 
4844
- class CheckSystemPackageCommandExecutor(CommandExecutor[CheckSystemPackageCommand, CheckSystemPackageCommand.Output]):
4845
- async def execute(self, cmd: CheckSystemPackageCommand) -> CheckSystemPackageCommand.Output:
4846
- log.info('Checking system package!')
5714
+ class LinuxPlatform(Platform, abc.ABC):
5715
+ pass
5716
+
5717
+
5718
+ class UbuntuPlatform(LinuxPlatform):
5719
+ pass
5720
+
5721
+
5722
+ class AmazonLinuxPlatform(LinuxPlatform):
5723
+ pass
5724
+
5725
+
5726
+ class GenericLinuxPlatform(LinuxPlatform):
5727
+ pass
5728
+
5729
+
5730
+ class DarwinPlatform(Platform):
5731
+ pass
5732
+
5733
+
5734
+ class UnknownPlatform(Platform):
5735
+ pass
5736
+
5737
+
5738
+ ##
5739
+
5740
+
5741
+ def _detect_system_platform() -> Platform:
5742
+ plat = sys.platform
5743
+
5744
+ if plat == 'linux':
5745
+ if (osr := LinuxOsRelease.read()) is None:
5746
+ return GenericLinuxPlatform()
5747
+
5748
+ if osr.id == 'amzn':
5749
+ return AmazonLinuxPlatform()
5750
+
5751
+ elif osr.id == 'ubuntu':
5752
+ return UbuntuPlatform()
5753
+
5754
+ else:
5755
+ return GenericLinuxPlatform()
4847
5756
 
4848
- return CheckSystemPackageCommand.Output()
5757
+ elif plat == 'darwin':
5758
+ return DarwinPlatform()
5759
+
5760
+ else:
5761
+ return UnknownPlatform()
5762
+
5763
+
5764
+ @cached_nullary
5765
+ def detect_system_platform() -> Platform:
5766
+ platform = _detect_system_platform()
5767
+ log.info('Detected platform: %r', platform)
5768
+ return platform
4849
5769
 
4850
5770
 
4851
5771
  ########################################
@@ -4868,168 +5788,241 @@ SUBPROCESS_CHANNEL_OPTION_VALUES: ta.Mapping[SubprocessChannelOption, int] = {
4868
5788
  _SUBPROCESS_SHELL_WRAP_EXECS = False
4869
5789
 
4870
5790
 
4871
- def subprocess_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
4872
- return ('sh', '-c', ' '.join(map(shlex.quote, args)))
5791
+ def subprocess_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
5792
+ return ('sh', '-c', ' '.join(map(shlex.quote, cmd)))
4873
5793
 
4874
5794
 
4875
- def subprocess_maybe_shell_wrap_exec(*args: str) -> ta.Tuple[str, ...]:
5795
+ def subprocess_maybe_shell_wrap_exec(*cmd: str) -> ta.Tuple[str, ...]:
4876
5796
  if _SUBPROCESS_SHELL_WRAP_EXECS or is_debugger_attached():
4877
- return subprocess_shell_wrap_exec(*args)
5797
+ return subprocess_shell_wrap_exec(*cmd)
4878
5798
  else:
4879
- return args
4880
-
4881
-
4882
- def prepare_subprocess_invocation(
4883
- *args: str,
4884
- env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
4885
- extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
4886
- quiet: bool = False,
4887
- shell: bool = False,
4888
- **kwargs: ta.Any,
4889
- ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
4890
- log.debug('prepare_subprocess_invocation: args=%r', args)
4891
- if extra_env:
4892
- log.debug('prepare_subprocess_invocation: extra_env=%r', extra_env)
4893
-
4894
- if extra_env:
4895
- env = {**(env if env is not None else os.environ), **extra_env}
4896
-
4897
- if quiet and 'stderr' not in kwargs:
4898
- if not log.isEnabledFor(logging.DEBUG):
4899
- kwargs['stderr'] = subprocess.DEVNULL
4900
-
4901
- if not shell:
4902
- args = subprocess_maybe_shell_wrap_exec(*args)
4903
-
4904
- return args, dict(
4905
- env=env,
4906
- shell=shell,
4907
- **kwargs,
4908
- )
5799
+ return cmd
4909
5800
 
4910
5801
 
4911
5802
  ##
4912
5803
 
4913
5804
 
4914
- @contextlib.contextmanager
4915
- def subprocess_common_context(*args: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
4916
- start_time = time.time()
4917
- try:
4918
- log.debug('subprocess_common_context.try: args=%r', args)
4919
- yield
4920
-
4921
- except Exception as exc: # noqa
4922
- log.debug('subprocess_common_context.except: exc=%r', exc)
4923
- raise
5805
+ def subprocess_close(
5806
+ proc: subprocess.Popen,
5807
+ timeout: ta.Optional[float] = None,
5808
+ ) -> None:
5809
+ # TODO: terminate, sleep, kill
5810
+ if proc.stdout:
5811
+ proc.stdout.close()
5812
+ if proc.stderr:
5813
+ proc.stderr.close()
5814
+ if proc.stdin:
5815
+ proc.stdin.close()
4924
5816
 
4925
- finally:
4926
- end_time = time.time()
4927
- elapsed_s = end_time - start_time
4928
- log.debug('subprocess_common_context.finally: elapsed_s=%f args=%r', elapsed_s, args)
5817
+ proc.wait(timeout)
4929
5818
 
4930
5819
 
4931
5820
  ##
4932
5821
 
4933
5822
 
4934
- def subprocess_check_call(
4935
- *args: str,
4936
- stdout: ta.Any = sys.stderr,
4937
- **kwargs: ta.Any,
4938
- ) -> None:
4939
- args, kwargs = prepare_subprocess_invocation(*args, stdout=stdout, **kwargs)
4940
- with subprocess_common_context(*args, **kwargs):
4941
- return subprocess.check_call(args, **kwargs) # type: ignore
5823
+ class AbstractSubprocesses(abc.ABC): # noqa
5824
+ DEFAULT_LOGGER: ta.ClassVar[ta.Optional[logging.Logger]] = log
5825
+
5826
+ def __init__(
5827
+ self,
5828
+ *,
5829
+ log: ta.Optional[logging.Logger] = None,
5830
+ try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
5831
+ ) -> None:
5832
+ super().__init__()
5833
+
5834
+ self._log = log if log is not None else self.DEFAULT_LOGGER
5835
+ self._try_exceptions = try_exceptions if try_exceptions is not None else self.DEFAULT_TRY_EXCEPTIONS
5836
+
5837
+ #
4942
5838
 
5839
+ def prepare_args(
5840
+ self,
5841
+ *cmd: str,
5842
+ env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
5843
+ extra_env: ta.Optional[ta.Mapping[str, ta.Any]] = None,
5844
+ quiet: bool = False,
5845
+ shell: bool = False,
5846
+ **kwargs: ta.Any,
5847
+ ) -> ta.Tuple[ta.Tuple[ta.Any, ...], ta.Dict[str, ta.Any]]:
5848
+ if self._log:
5849
+ self._log.debug('Subprocesses.prepare_args: cmd=%r', cmd)
5850
+ if extra_env:
5851
+ self._log.debug('Subprocesses.prepare_args: extra_env=%r', extra_env)
4943
5852
 
4944
- def subprocess_check_output(
4945
- *args: str,
4946
- **kwargs: ta.Any,
4947
- ) -> bytes:
4948
- args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
4949
- with subprocess_common_context(*args, **kwargs):
4950
- return subprocess.check_output(args, **kwargs)
5853
+ if extra_env:
5854
+ env = {**(env if env is not None else os.environ), **extra_env}
4951
5855
 
5856
+ if quiet and 'stderr' not in kwargs:
5857
+ if self._log and not self._log.isEnabledFor(logging.DEBUG):
5858
+ kwargs['stderr'] = subprocess.DEVNULL
4952
5859
 
4953
- def subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
4954
- return subprocess_check_output(*args, **kwargs).decode().strip()
5860
+ if not shell:
5861
+ cmd = subprocess_maybe_shell_wrap_exec(*cmd)
4955
5862
 
5863
+ return cmd, dict(
5864
+ env=env,
5865
+ shell=shell,
5866
+ **kwargs,
5867
+ )
4956
5868
 
4957
- ##
5869
+ @contextlib.contextmanager
5870
+ def wrap_call(self, *cmd: ta.Any, **kwargs: ta.Any) -> ta.Iterator[None]:
5871
+ start_time = time.time()
5872
+ try:
5873
+ if self._log:
5874
+ self._log.debug('Subprocesses.wrap_call.try: cmd=%r', cmd)
5875
+ yield
4958
5876
 
5877
+ except Exception as exc: # noqa
5878
+ if self._log:
5879
+ self._log.debug('Subprocesses.wrap_call.except: exc=%r', exc)
5880
+ raise
4959
5881
 
4960
- DEFAULT_SUBPROCESS_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
4961
- FileNotFoundError,
4962
- subprocess.CalledProcessError,
4963
- )
5882
+ finally:
5883
+ end_time = time.time()
5884
+ elapsed_s = end_time - start_time
5885
+ if self._log:
5886
+ self._log.debug('sSubprocesses.wrap_call.finally: elapsed_s=%f cmd=%r', elapsed_s, cmd)
4964
5887
 
5888
+ @contextlib.contextmanager
5889
+ def prepare_and_wrap(
5890
+ self,
5891
+ *cmd: ta.Any,
5892
+ **kwargs: ta.Any,
5893
+ ) -> ta.Iterator[ta.Tuple[
5894
+ ta.Tuple[ta.Any, ...],
5895
+ ta.Dict[str, ta.Any],
5896
+ ]]:
5897
+ cmd, kwargs = self.prepare_args(*cmd, **kwargs)
5898
+ with self.wrap_call(*cmd, **kwargs):
5899
+ yield cmd, kwargs
4965
5900
 
4966
- def _subprocess_try_run(
4967
- fn: ta.Callable[..., T],
4968
- *args: ta.Any,
4969
- try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4970
- **kwargs: ta.Any,
4971
- ) -> ta.Union[T, Exception]:
4972
- try:
4973
- return fn(*args, **kwargs)
4974
- except try_exceptions as e: # noqa
4975
- if log.isEnabledFor(logging.DEBUG):
4976
- log.exception('command failed')
4977
- return e
4978
-
4979
-
4980
- def subprocess_try_call(
4981
- *args: str,
4982
- try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4983
- **kwargs: ta.Any,
4984
- ) -> bool:
4985
- if isinstance(_subprocess_try_run(
4986
- subprocess_check_call,
4987
- *args,
4988
- try_exceptions=try_exceptions,
4989
- **kwargs,
4990
- ), Exception):
4991
- return False
4992
- else:
4993
- return True
5901
+ #
4994
5902
 
5903
+ DEFAULT_TRY_EXCEPTIONS: ta.Tuple[ta.Type[Exception], ...] = (
5904
+ FileNotFoundError,
5905
+ subprocess.CalledProcessError,
5906
+ )
4995
5907
 
4996
- def subprocess_try_output(
4997
- *args: str,
4998
- try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
4999
- **kwargs: ta.Any,
5000
- ) -> ta.Optional[bytes]:
5001
- if isinstance(ret := _subprocess_try_run(
5002
- subprocess_check_output,
5003
- *args,
5004
- try_exceptions=try_exceptions,
5005
- **kwargs,
5006
- ), Exception):
5007
- return None
5008
- else:
5009
- return ret
5908
+ def try_fn(
5909
+ self,
5910
+ fn: ta.Callable[..., T],
5911
+ *cmd: str,
5912
+ try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
5913
+ **kwargs: ta.Any,
5914
+ ) -> ta.Union[T, Exception]:
5915
+ if try_exceptions is None:
5916
+ try_exceptions = self._try_exceptions
5917
+
5918
+ try:
5919
+ return fn(*cmd, **kwargs)
5920
+
5921
+ except try_exceptions as e: # noqa
5922
+ if self._log and self._log.isEnabledFor(logging.DEBUG):
5923
+ self._log.exception('command failed')
5924
+ return e
5010
5925
 
5926
+ async def async_try_fn(
5927
+ self,
5928
+ fn: ta.Callable[..., ta.Awaitable[T]],
5929
+ *cmd: ta.Any,
5930
+ try_exceptions: ta.Optional[ta.Tuple[ta.Type[Exception], ...]] = None,
5931
+ **kwargs: ta.Any,
5932
+ ) -> ta.Union[T, Exception]:
5933
+ if try_exceptions is None:
5934
+ try_exceptions = self._try_exceptions
5935
+
5936
+ try:
5937
+ return await fn(*cmd, **kwargs)
5011
5938
 
5012
- def subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
5013
- out = subprocess_try_output(*args, **kwargs)
5014
- return out.decode().strip() if out is not None else None
5939
+ except try_exceptions as e: # noqa
5940
+ if self._log and self._log.isEnabledFor(logging.DEBUG):
5941
+ self._log.exception('command failed')
5942
+ return e
5015
5943
 
5016
5944
 
5017
5945
  ##
5018
5946
 
5019
5947
 
5020
- def subprocess_close(
5021
- proc: subprocess.Popen,
5022
- timeout: ta.Optional[float] = None,
5023
- ) -> None:
5024
- # TODO: terminate, sleep, kill
5025
- if proc.stdout:
5026
- proc.stdout.close()
5027
- if proc.stderr:
5028
- proc.stderr.close()
5029
- if proc.stdin:
5030
- proc.stdin.close()
5948
+ class Subprocesses(AbstractSubprocesses):
5949
+ def check_call(
5950
+ self,
5951
+ *cmd: str,
5952
+ stdout: ta.Any = sys.stderr,
5953
+ **kwargs: ta.Any,
5954
+ ) -> None:
5955
+ with self.prepare_and_wrap(*cmd, stdout=stdout, **kwargs) as (cmd, kwargs): # noqa
5956
+ subprocess.check_call(cmd, **kwargs)
5031
5957
 
5032
- proc.wait(timeout)
5958
+ def check_output(
5959
+ self,
5960
+ *cmd: str,
5961
+ **kwargs: ta.Any,
5962
+ ) -> bytes:
5963
+ with self.prepare_and_wrap(*cmd, **kwargs) as (cmd, kwargs): # noqa
5964
+ return subprocess.check_output(cmd, **kwargs)
5965
+
5966
+ def check_output_str(
5967
+ self,
5968
+ *cmd: str,
5969
+ **kwargs: ta.Any,
5970
+ ) -> str:
5971
+ return self.check_output(*cmd, **kwargs).decode().strip()
5972
+
5973
+ #
5974
+
5975
+ def try_call(
5976
+ self,
5977
+ *cmd: str,
5978
+ **kwargs: ta.Any,
5979
+ ) -> bool:
5980
+ if isinstance(self.try_fn(self.check_call, *cmd, **kwargs), Exception):
5981
+ return False
5982
+ else:
5983
+ return True
5984
+
5985
+ def try_output(
5986
+ self,
5987
+ *cmd: str,
5988
+ **kwargs: ta.Any,
5989
+ ) -> ta.Optional[bytes]:
5990
+ if isinstance(ret := self.try_fn(self.check_output, *cmd, **kwargs), Exception):
5991
+ return None
5992
+ else:
5993
+ return ret
5994
+
5995
+ def try_output_str(
5996
+ self,
5997
+ *cmd: str,
5998
+ **kwargs: ta.Any,
5999
+ ) -> ta.Optional[str]:
6000
+ if (ret := self.try_output(*cmd, **kwargs)) is None:
6001
+ return None
6002
+ else:
6003
+ return ret.decode().strip()
6004
+
6005
+
6006
+ subprocesses = Subprocesses()
6007
+
6008
+
6009
+ ########################################
6010
+ # ../commands/local.py
6011
+
6012
+
6013
+ class LocalCommandExecutor(CommandExecutor):
6014
+ def __init__(
6015
+ self,
6016
+ *,
6017
+ command_executors: CommandExecutorMap,
6018
+ ) -> None:
6019
+ super().__init__()
6020
+
6021
+ self._command_executors = command_executors
6022
+
6023
+ async def execute(self, cmd: Command) -> Command.Output:
6024
+ ce: CommandExecutor = self._command_executors[type(cmd)]
6025
+ return await ce.execute(cmd)
5033
6026
 
5034
6027
 
5035
6028
  ########################################
@@ -5409,7 +6402,7 @@ class RemoteCommandExecutor(CommandExecutor):
5409
6402
  self,
5410
6403
  cmd: Command,
5411
6404
  *,
5412
- log: ta.Optional[logging.Logger] = None,
6405
+ log: ta.Optional[logging.Logger] = None, # noqa
5413
6406
  omit_exc_object: bool = False,
5414
6407
  ) -> CommandOutputOrException:
5415
6408
  try:
@@ -5430,44 +6423,16 @@ class RemoteCommandExecutor(CommandExecutor):
5430
6423
 
5431
6424
 
5432
6425
  ########################################
5433
- # ../../../omlish/lite/asyncio/subprocesses.py
5434
-
5435
-
5436
- ##
6426
+ # ../system/config.py
5437
6427
 
5438
6428
 
5439
- @contextlib.asynccontextmanager
5440
- async def asyncio_subprocess_popen(
5441
- *cmd: str,
5442
- shell: bool = False,
5443
- timeout: ta.Optional[float] = None,
5444
- **kwargs: ta.Any,
5445
- ) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
5446
- fac: ta.Any
5447
- if shell:
5448
- fac = functools.partial(
5449
- asyncio.create_subprocess_shell,
5450
- check.single(cmd),
5451
- )
5452
- else:
5453
- fac = functools.partial(
5454
- asyncio.create_subprocess_exec,
5455
- *cmd,
5456
- )
6429
+ @dc.dataclass(frozen=True)
6430
+ class SystemConfig:
6431
+ platform: ta.Optional[Platform] = None
5457
6432
 
5458
- with subprocess_common_context(
5459
- *cmd,
5460
- shell=shell,
5461
- timeout=timeout,
5462
- **kwargs,
5463
- ):
5464
- proc: asyncio.subprocess.Process
5465
- proc = await fac(**kwargs)
5466
- try:
5467
- yield proc
5468
6433
 
5469
- finally:
5470
- await asyncio_maybe_timeout(proc.wait(), timeout)
6434
+ ########################################
6435
+ # ../../../omlish/lite/asyncio/subprocesses.py
5471
6436
 
5472
6437
 
5473
6438
  ##
@@ -5583,146 +6548,156 @@ class AsyncioProcessCommunicator:
5583
6548
  return await asyncio_maybe_timeout(self._communicate(input), timeout)
5584
6549
 
5585
6550
 
5586
- async def asyncio_subprocess_communicate(
5587
- proc: asyncio.subprocess.Process,
5588
- input: ta.Any = None, # noqa
5589
- timeout: ta.Optional[float] = None,
5590
- ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
5591
- return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
5592
-
5593
-
5594
- async def asyncio_subprocess_run(
5595
- *args: str,
5596
- input: ta.Any = None, # noqa
5597
- timeout: ta.Optional[float] = None,
5598
- check: bool = False, # noqa
5599
- capture_output: ta.Optional[bool] = None,
5600
- **kwargs: ta.Any,
5601
- ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
5602
- if capture_output:
5603
- kwargs.setdefault('stdout', subprocess.PIPE)
5604
- kwargs.setdefault('stderr', subprocess.PIPE)
5605
-
5606
- args, kwargs = prepare_subprocess_invocation(*args, **kwargs)
5607
-
5608
- proc: asyncio.subprocess.Process
5609
- async with asyncio_subprocess_popen(*args, **kwargs) as proc:
5610
- stdout, stderr = await asyncio_subprocess_communicate(proc, input, timeout)
5611
-
5612
- if check and proc.returncode:
5613
- raise subprocess.CalledProcessError(
5614
- proc.returncode,
5615
- args,
5616
- output=stdout,
5617
- stderr=stderr,
5618
- )
5619
-
5620
- return stdout, stderr
5621
-
5622
-
5623
6551
  ##
5624
6552
 
5625
6553
 
5626
- async def asyncio_subprocess_check_call(
5627
- *args: str,
5628
- stdout: ta.Any = sys.stderr,
5629
- input: ta.Any = None, # noqa
5630
- timeout: ta.Optional[float] = None,
5631
- **kwargs: ta.Any,
5632
- ) -> None:
5633
- _, _ = await asyncio_subprocess_run(
5634
- *args,
5635
- stdout=stdout,
5636
- input=input,
5637
- timeout=timeout,
5638
- check=True,
5639
- **kwargs,
5640
- )
5641
-
5642
-
5643
- async def asyncio_subprocess_check_output(
5644
- *args: str,
5645
- input: ta.Any = None, # noqa
5646
- timeout: ta.Optional[float] = None,
5647
- **kwargs: ta.Any,
5648
- ) -> bytes:
5649
- stdout, stderr = await asyncio_subprocess_run(
5650
- *args,
5651
- stdout=asyncio.subprocess.PIPE,
5652
- input=input,
5653
- timeout=timeout,
5654
- check=True,
5655
- **kwargs,
5656
- )
6554
+ class AsyncioSubprocesses(AbstractSubprocesses):
6555
+ async def communicate(
6556
+ self,
6557
+ proc: asyncio.subprocess.Process,
6558
+ input: ta.Any = None, # noqa
6559
+ timeout: ta.Optional[float] = None,
6560
+ ) -> ta.Tuple[ta.Optional[bytes], ta.Optional[bytes]]:
6561
+ return await AsyncioProcessCommunicator(proc).communicate(input, timeout) # noqa
5657
6562
 
5658
- return check.not_none(stdout)
6563
+ #
5659
6564
 
6565
+ @contextlib.asynccontextmanager
6566
+ async def popen(
6567
+ self,
6568
+ *cmd: str,
6569
+ shell: bool = False,
6570
+ timeout: ta.Optional[float] = None,
6571
+ **kwargs: ta.Any,
6572
+ ) -> ta.AsyncGenerator[asyncio.subprocess.Process, None]:
6573
+ fac: ta.Any
6574
+ if shell:
6575
+ fac = functools.partial(
6576
+ asyncio.create_subprocess_shell,
6577
+ check.single(cmd),
6578
+ )
6579
+ else:
6580
+ fac = functools.partial(
6581
+ asyncio.create_subprocess_exec,
6582
+ *cmd,
6583
+ )
5660
6584
 
5661
- async def asyncio_subprocess_check_output_str(*args: str, **kwargs: ta.Any) -> str:
5662
- return (await asyncio_subprocess_check_output(*args, **kwargs)).decode().strip()
6585
+ with self.prepare_and_wrap( *cmd, shell=shell, **kwargs) as (cmd, kwargs): # noqa
6586
+ proc: asyncio.subprocess.Process = await fac(**kwargs)
6587
+ try:
6588
+ yield proc
5663
6589
 
6590
+ finally:
6591
+ await asyncio_maybe_timeout(proc.wait(), timeout)
5664
6592
 
5665
- ##
6593
+ #
5666
6594
 
6595
+ @dc.dataclass(frozen=True)
6596
+ class RunOutput:
6597
+ proc: asyncio.subprocess.Process
6598
+ stdout: ta.Optional[bytes]
6599
+ stderr: ta.Optional[bytes]
5667
6600
 
5668
- async def _asyncio_subprocess_try_run(
5669
- fn: ta.Callable[..., ta.Awaitable[T]],
5670
- *args: ta.Any,
5671
- try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
5672
- **kwargs: ta.Any,
5673
- ) -> ta.Union[T, Exception]:
5674
- try:
5675
- return await fn(*args, **kwargs)
5676
- except try_exceptions as e: # noqa
5677
- if log.isEnabledFor(logging.DEBUG):
5678
- log.exception('command failed')
5679
- return e
5680
-
5681
-
5682
- async def asyncio_subprocess_try_call(
5683
- *args: str,
5684
- try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
5685
- **kwargs: ta.Any,
5686
- ) -> bool:
5687
- if isinstance(await _asyncio_subprocess_try_run(
5688
- asyncio_subprocess_check_call,
5689
- *args,
5690
- try_exceptions=try_exceptions,
5691
- **kwargs,
5692
- ), Exception):
5693
- return False
5694
- else:
5695
- return True
6601
+ async def run(
6602
+ self,
6603
+ *cmd: str,
6604
+ input: ta.Any = None, # noqa
6605
+ timeout: ta.Optional[float] = None,
6606
+ check: bool = False, # noqa
6607
+ capture_output: ta.Optional[bool] = None,
6608
+ **kwargs: ta.Any,
6609
+ ) -> RunOutput:
6610
+ if capture_output:
6611
+ kwargs.setdefault('stdout', subprocess.PIPE)
6612
+ kwargs.setdefault('stderr', subprocess.PIPE)
5696
6613
 
6614
+ proc: asyncio.subprocess.Process
6615
+ async with self.popen(*cmd, **kwargs) as proc:
6616
+ stdout, stderr = await self.communicate(proc, input, timeout)
6617
+
6618
+ if check and proc.returncode:
6619
+ raise subprocess.CalledProcessError(
6620
+ proc.returncode,
6621
+ cmd,
6622
+ output=stdout,
6623
+ stderr=stderr,
6624
+ )
5697
6625
 
5698
- async def asyncio_subprocess_try_output(
5699
- *args: str,
5700
- try_exceptions: ta.Tuple[ta.Type[Exception], ...] = DEFAULT_SUBPROCESS_TRY_EXCEPTIONS,
5701
- **kwargs: ta.Any,
5702
- ) -> ta.Optional[bytes]:
5703
- if isinstance(ret := await _asyncio_subprocess_try_run(
5704
- asyncio_subprocess_check_output,
5705
- *args,
5706
- try_exceptions=try_exceptions,
5707
- **kwargs,
5708
- ), Exception):
5709
- return None
5710
- else:
5711
- return ret
6626
+ return self.RunOutput(
6627
+ proc,
6628
+ stdout,
6629
+ stderr,
6630
+ )
5712
6631
 
6632
+ #
5713
6633
 
5714
- async def asyncio_subprocess_try_output_str(*args: str, **kwargs: ta.Any) -> ta.Optional[str]:
5715
- out = await asyncio_subprocess_try_output(*args, **kwargs)
5716
- return out.decode().strip() if out is not None else None
6634
+ async def check_call(
6635
+ self,
6636
+ *cmd: str,
6637
+ stdout: ta.Any = sys.stderr,
6638
+ **kwargs: ta.Any,
6639
+ ) -> None:
6640
+ with self.prepare_and_wrap(*cmd, stdout=stdout, check=True, **kwargs) as (cmd, kwargs): # noqa
6641
+ await self.run(*cmd, **kwargs)
5717
6642
 
6643
+ async def check_output(
6644
+ self,
6645
+ *cmd: str,
6646
+ **kwargs: ta.Any,
6647
+ ) -> bytes:
6648
+ with self.prepare_and_wrap(*cmd, stdout=subprocess.PIPE, check=True, **kwargs) as (cmd, kwargs): # noqa
6649
+ return check.not_none((await self.run(*cmd, **kwargs)).stdout)
5718
6650
 
5719
- ########################################
5720
- # ../../../omdev/interp/inspect.py
6651
+ async def check_output_str(
6652
+ self,
6653
+ *cmd: str,
6654
+ **kwargs: ta.Any,
6655
+ ) -> str:
6656
+ return (await self.check_output(*cmd, **kwargs)).decode().strip()
5721
6657
 
6658
+ #
5722
6659
 
5723
- @dc.dataclass(frozen=True)
5724
- class InterpInspection:
5725
- exe: str
6660
+ async def try_call(
6661
+ self,
6662
+ *cmd: str,
6663
+ **kwargs: ta.Any,
6664
+ ) -> bool:
6665
+ if isinstance(await self.async_try_fn(self.check_call, *cmd, **kwargs), Exception):
6666
+ return False
6667
+ else:
6668
+ return True
6669
+
6670
+ async def try_output(
6671
+ self,
6672
+ *cmd: str,
6673
+ **kwargs: ta.Any,
6674
+ ) -> ta.Optional[bytes]:
6675
+ if isinstance(ret := await self.async_try_fn(self.check_output, *cmd, **kwargs), Exception):
6676
+ return None
6677
+ else:
6678
+ return ret
6679
+
6680
+ async def try_output_str(
6681
+ self,
6682
+ *cmd: str,
6683
+ **kwargs: ta.Any,
6684
+ ) -> ta.Optional[str]:
6685
+ if (ret := await self.try_output(*cmd, **kwargs)) is None:
6686
+ return None
6687
+ else:
6688
+ return ret.decode().strip()
6689
+
6690
+
6691
+ asyncio_subprocesses = AsyncioSubprocesses()
6692
+
6693
+
6694
+ ########################################
6695
+ # ../../../omdev/interp/inspect.py
6696
+
6697
+
6698
+ @dc.dataclass(frozen=True)
6699
+ class InterpInspection:
6700
+ exe: str
5726
6701
  version: Version
5727
6702
 
5728
6703
  version_str: str
@@ -5790,7 +6765,7 @@ class InterpInspector:
5790
6765
  return cls._build_inspection(sys.executable, eval(cls._INSPECTION_CODE)) # noqa
5791
6766
 
5792
6767
  async def _inspect(self, exe: str) -> InterpInspection:
5793
- output = await asyncio_subprocess_check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
6768
+ output = await asyncio_subprocesses.check_output(exe, '-c', f'print({self._INSPECTION_CODE})', quiet=True)
5794
6769
  return self._build_inspection(exe, output.decode())
5795
6770
 
5796
6771
  async def inspect(self, exe: str) -> ta.Optional[InterpInspection]:
@@ -5811,6 +6786,21 @@ class InterpInspector:
5811
6786
  INTERP_INSPECTOR = InterpInspector()
5812
6787
 
5813
6788
 
6789
+ ########################################
6790
+ # ../bootstrap.py
6791
+
6792
+
6793
+ @dc.dataclass(frozen=True)
6794
+ class MainBootstrap:
6795
+ main_config: MainConfig = MainConfig()
6796
+
6797
+ deploy_config: DeployConfig = DeployConfig()
6798
+
6799
+ remote_config: RemoteConfig = RemoteConfig()
6800
+
6801
+ system_config: SystemConfig = SystemConfig()
6802
+
6803
+
5814
6804
  ########################################
5815
6805
  # ../commands/subprocess.py
5816
6806
 
@@ -5849,7 +6839,7 @@ class SubprocessCommand(Command['SubprocessCommand.Output']):
5849
6839
  class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCommand.Output]):
5850
6840
  async def execute(self, cmd: SubprocessCommand) -> SubprocessCommand.Output:
5851
6841
  proc: asyncio.subprocess.Process
5852
- async with asyncio_subprocess_popen(
6842
+ async with asyncio_subprocesses.popen(
5853
6843
  *subprocess_maybe_shell_wrap_exec(*cmd.cmd),
5854
6844
 
5855
6845
  shell=cmd.shell,
@@ -5863,7 +6853,7 @@ class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCom
5863
6853
  timeout=cmd.timeout,
5864
6854
  ) as proc:
5865
6855
  start_time = time.time()
5866
- stdout, stderr = await asyncio_subprocess_communicate(
6856
+ stdout, stderr = await asyncio_subprocesses.communicate(
5867
6857
  proc,
5868
6858
  input=cmd.input,
5869
6859
  timeout=cmd.timeout,
@@ -5882,146 +6872,190 @@ class SubprocessCommandExecutor(CommandExecutor[SubprocessCommand, SubprocessCom
5882
6872
 
5883
6873
 
5884
6874
  ########################################
5885
- # ../remote/_main.py
6875
+ # ../deploy/git.py
6876
+ """
6877
+ TODO:
6878
+ - 'repos'?
6879
+
6880
+ git/github.com/wrmsr/omlish <- bootstrap repo
6881
+ - shallow clone off bootstrap into /apps
6882
+
6883
+ github.com/wrmsr/omlish@rev
6884
+ """
5886
6885
 
5887
6886
 
5888
6887
  ##
5889
6888
 
5890
6889
 
5891
- class _RemoteExecutionLogHandler(logging.Handler):
5892
- def __init__(self, fn: ta.Callable[[str], None]) -> None:
5893
- super().__init__()
5894
- self._fn = fn
6890
+ @dc.dataclass(frozen=True)
6891
+ class DeployGitRepo:
6892
+ host: ta.Optional[str] = None
6893
+ username: ta.Optional[str] = None
6894
+ path: ta.Optional[str] = None
5895
6895
 
5896
- def emit(self, record):
5897
- msg = self.format(record)
5898
- self._fn(msg)
6896
+ def __post_init__(self) -> None:
6897
+ check.not_in('..', check.non_empty_str(self.host))
6898
+ check.not_in('.', check.non_empty_str(self.path))
6899
+
6900
+
6901
+ @dc.dataclass(frozen=True)
6902
+ class DeployGitSpec:
6903
+ repo: DeployGitRepo
6904
+ rev: DeployRev
5899
6905
 
5900
6906
 
5901
6907
  ##
5902
6908
 
5903
6909
 
5904
- class _RemoteExecutionMain:
6910
+ class DeployGitManager(DeployPathOwner):
5905
6911
  def __init__(
5906
6912
  self,
5907
- chan: RemoteChannel,
6913
+ *,
6914
+ deploy_home: DeployHome,
5908
6915
  ) -> None:
5909
6916
  super().__init__()
5910
6917
 
5911
- self._chan = chan
5912
-
5913
- self.__bootstrap: ta.Optional[MainBootstrap] = None
5914
- self.__injector: ta.Optional[Injector] = None
5915
-
5916
- @property
5917
- def _bootstrap(self) -> MainBootstrap:
5918
- return check.not_none(self.__bootstrap)
6918
+ self._deploy_home = deploy_home
6919
+ self._dir = os.path.join(deploy_home, 'git')
5919
6920
 
5920
- @property
5921
- def _injector(self) -> Injector:
5922
- return check.not_none(self.__injector)
5923
-
5924
- #
6921
+ self._repo_dirs: ta.Dict[DeployGitRepo, DeployGitManager.RepoDir] = {}
5925
6922
 
5926
- def _timebomb_main(
5927
- self,
5928
- delay_s: float,
5929
- *,
5930
- sig: int = signal.SIGINT,
5931
- code: int = 1,
5932
- ) -> None:
5933
- time.sleep(delay_s)
6923
+ def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
6924
+ return {
6925
+ DeployPath.parse('git'),
6926
+ }
5934
6927
 
5935
- if (pgid := os.getpgid(0)) == os.getpid():
5936
- os.killpg(pgid, sig)
6928
+ class RepoDir:
6929
+ def __init__(
6930
+ self,
6931
+ git: 'DeployGitManager',
6932
+ repo: DeployGitRepo,
6933
+ ) -> None:
6934
+ super().__init__()
5937
6935
 
5938
- os._exit(code) # noqa
6936
+ self._git = git
6937
+ self._repo = repo
6938
+ self._dir = os.path.join(
6939
+ self._git._dir, # noqa
6940
+ check.non_empty_str(repo.host),
6941
+ check.non_empty_str(repo.path),
6942
+ )
5939
6943
 
5940
- @cached_nullary
5941
- def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
5942
- if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
5943
- return None
6944
+ @property
6945
+ def repo(self) -> DeployGitRepo:
6946
+ return self._repo
5944
6947
 
5945
- thr = threading.Thread(
5946
- target=functools.partial(self._timebomb_main, tbd),
5947
- name=f'{self.__class__.__name__}.timebomb',
5948
- daemon=True,
5949
- )
6948
+ @property
6949
+ def url(self) -> str:
6950
+ if self._repo.username is not None:
6951
+ return f'{self._repo.username}@{self._repo.host}:{self._repo.path}'
6952
+ else:
6953
+ return f'https://{self._repo.host}/{self._repo.path}'
5950
6954
 
5951
- thr.start()
6955
+ async def _call(self, *cmd: str) -> None:
6956
+ await asyncio_subprocesses.check_call(
6957
+ *cmd,
6958
+ cwd=self._dir,
6959
+ )
5952
6960
 
5953
- log.debug('Started timebomb thread: %r', thr)
6961
+ @async_cached_nullary
6962
+ async def init(self) -> None:
6963
+ os.makedirs(self._dir, exist_ok=True)
6964
+ if os.path.exists(os.path.join(self._dir, '.git')):
6965
+ return
5954
6966
 
5955
- return thr
6967
+ await self._call('git', 'init')
6968
+ await self._call('git', 'remote', 'add', 'origin', self.url)
5956
6969
 
5957
- #
6970
+ async def fetch(self, rev: DeployRev) -> None:
6971
+ await self.init()
6972
+ await self._call('git', 'fetch', '--depth=1', 'origin', rev)
5958
6973
 
5959
- @cached_nullary
5960
- def _log_handler(self) -> _RemoteLogHandler:
5961
- return _RemoteLogHandler(self._chan)
6974
+ async def checkout(self, rev: DeployRev, dst_dir: str) -> None:
6975
+ check.state(not os.path.exists(dst_dir))
5962
6976
 
5963
- #
6977
+ await self.fetch(rev)
5964
6978
 
5965
- async def _setup(self) -> None:
5966
- check.none(self.__bootstrap)
5967
- check.none(self.__injector)
6979
+ # FIXME: temp dir swap
6980
+ os.makedirs(dst_dir)
5968
6981
 
5969
- # Bootstrap
6982
+ dst_call = functools.partial(asyncio_subprocesses.check_call, cwd=dst_dir)
6983
+ await dst_call('git', 'init')
5970
6984
 
5971
- self.__bootstrap = check.not_none(await self._chan.recv_obj(MainBootstrap))
6985
+ await dst_call('git', 'remote', 'add', 'local', self._dir)
6986
+ await dst_call('git', 'fetch', '--depth=1', 'local', rev)
6987
+ await dst_call('git', 'checkout', rev)
5972
6988
 
5973
- if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
5974
- pycharm_debug_connect(prd)
6989
+ def get_repo_dir(self, repo: DeployGitRepo) -> RepoDir:
6990
+ try:
6991
+ return self._repo_dirs[repo]
6992
+ except KeyError:
6993
+ repo_dir = self._repo_dirs[repo] = DeployGitManager.RepoDir(self, repo)
6994
+ return repo_dir
5975
6995
 
5976
- self.__injector = main_bootstrap(self._bootstrap)
6996
+ async def checkout(self, spec: DeployGitSpec, dst_dir: str) -> None:
6997
+ await self.get_repo_dir(spec.repo).checkout(spec.rev, dst_dir)
5977
6998
 
5978
- self._chan.set_marshaler(self._injector[ObjMarshalerManager])
5979
6999
 
5980
- # Post-bootstrap
7000
+ ########################################
7001
+ # ../deploy/venvs.py
7002
+ """
7003
+ TODO:
7004
+ - interp
7005
+ - share more code with pyproject?
7006
+ """
5981
7007
 
5982
- if self._bootstrap.remote_config.set_pgid:
5983
- if os.getpgid(0) != os.getpid():
5984
- log.debug('Setting pgid')
5985
- os.setpgid(0, 0)
5986
7008
 
5987
- if (ds := self._bootstrap.remote_config.deathsig) is not None:
5988
- log.debug('Setting deathsig: %s', ds)
5989
- set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
7009
+ class DeployVenvManager(DeployPathOwner):
7010
+ def __init__(
7011
+ self,
7012
+ *,
7013
+ deploy_home: DeployHome,
7014
+ ) -> None:
7015
+ super().__init__()
5990
7016
 
5991
- self._timebomb_thread()
7017
+ self._deploy_home = deploy_home
7018
+ self._dir = os.path.join(deploy_home, 'venvs')
5992
7019
 
5993
- if self._bootstrap.remote_config.forward_logging:
5994
- log.debug('Installing log forwarder')
5995
- logging.root.addHandler(self._log_handler())
7020
+ def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
7021
+ return {
7022
+ DeployPath.parse('venvs/@app/@tag/'),
7023
+ }
5996
7024
 
5997
- #
7025
+ async def setup_venv(
7026
+ self,
7027
+ app_dir: str,
7028
+ venv_dir: str,
7029
+ *,
7030
+ use_uv: bool = True,
7031
+ ) -> None:
7032
+ sys_exe = 'python3'
5998
7033
 
5999
- async def run(self) -> None:
6000
- await self._setup()
7034
+ await asyncio_subprocesses.check_call(sys_exe, '-m', 'venv', venv_dir)
6001
7035
 
6002
- executor = self._injector[LocalCommandExecutor]
7036
+ #
6003
7037
 
6004
- handler = _RemoteCommandHandler(self._chan, executor)
7038
+ venv_exe = os.path.join(venv_dir, 'bin', 'python3')
6005
7039
 
6006
- await handler.run()
7040
+ #
6007
7041
 
7042
+ reqs_txt = os.path.join(app_dir, 'requirements.txt')
6008
7043
 
6009
- def _remote_execution_main() -> None:
6010
- rt = pyremote_bootstrap_finalize() # noqa
7044
+ if os.path.isfile(reqs_txt):
7045
+ if use_uv:
7046
+ await asyncio_subprocesses.check_call(venv_exe, '-m', 'pip', 'install', 'uv')
7047
+ pip_cmd = ['-m', 'uv', 'pip']
7048
+ else:
7049
+ pip_cmd = ['-m', 'pip']
6011
7050
 
6012
- async def inner() -> None:
6013
- input = await asyncio_open_stream_reader(rt.input) # noqa
6014
- output = await asyncio_open_stream_writer(rt.output)
7051
+ await asyncio_subprocesses.check_call(venv_exe, *pip_cmd,'install', '-r', reqs_txt)
6015
7052
 
6016
- chan = RemoteChannelImpl(
6017
- input,
6018
- output,
7053
+ async def setup_app_venv(self, app_tag: DeployAppTag) -> None:
7054
+ await self.setup_venv(
7055
+ os.path.join(self._deploy_home, 'apps', app_tag.app, app_tag.tag),
7056
+ os.path.join(self._deploy_home, 'venvs', app_tag.app, app_tag.tag),
6019
7057
  )
6020
7058
 
6021
- await _RemoteExecutionMain(chan).run()
6022
-
6023
- asyncio.run(inner())
6024
-
6025
7059
 
6026
7060
  ########################################
6027
7061
  # ../remote/spawning.py
@@ -6099,7 +7133,7 @@ class SubprocessRemoteSpawning(RemoteSpawning):
6099
7133
  if not debug:
6100
7134
  cmd = subprocess_maybe_shell_wrap_exec(*cmd)
6101
7135
 
6102
- async with asyncio_subprocess_popen(
7136
+ async with asyncio_subprocesses.popen(
6103
7137
  *cmd,
6104
7138
  shell=pc.shell,
6105
7139
  stdin=subprocess.PIPE,
@@ -6161,10 +7195,10 @@ class SystemPackageManager(abc.ABC):
6161
7195
 
6162
7196
  class BrewSystemPackageManager(SystemPackageManager):
6163
7197
  async def update(self) -> None:
6164
- await asyncio_subprocess_check_call('brew', 'update')
7198
+ await asyncio_subprocesses.check_call('brew', 'update')
6165
7199
 
6166
7200
  async def upgrade(self) -> None:
6167
- await asyncio_subprocess_check_call('brew', 'upgrade')
7201
+ await asyncio_subprocesses.check_call('brew', 'upgrade')
6168
7202
 
6169
7203
  async def install(self, *packages: SystemPackageOrStr) -> None:
6170
7204
  es: ta.List[str] = []
@@ -6173,11 +7207,11 @@ class BrewSystemPackageManager(SystemPackageManager):
6173
7207
  es.append(p.name + (f'@{p.version}' if p.version is not None else ''))
6174
7208
  else:
6175
7209
  es.append(p)
6176
- await asyncio_subprocess_check_call('brew', 'install', *es)
7210
+ await asyncio_subprocesses.check_call('brew', 'install', *es)
6177
7211
 
6178
7212
  async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
6179
7213
  pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
6180
- o = await asyncio_subprocess_check_output('brew', 'info', '--json', *pns)
7214
+ o = await asyncio_subprocesses.check_output('brew', 'info', '--json', *pns)
6181
7215
  j = json.loads(o.decode())
6182
7216
  d: ta.Dict[str, SystemPackage] = {}
6183
7217
  for e in j:
@@ -6196,25 +7230,24 @@ class AptSystemPackageManager(SystemPackageManager):
6196
7230
  }
6197
7231
 
6198
7232
  async def update(self) -> None:
6199
- await asyncio_subprocess_check_call('apt', 'update', env={**os.environ, **self._APT_ENV})
7233
+ await asyncio_subprocesses.check_call('sudo', 'apt', 'update', env={**os.environ, **self._APT_ENV})
6200
7234
 
6201
7235
  async def upgrade(self) -> None:
6202
- await asyncio_subprocess_check_call('apt', 'upgrade', '-y', env={**os.environ, **self._APT_ENV})
7236
+ await asyncio_subprocesses.check_call('sudo', 'apt', 'upgrade', '-y', env={**os.environ, **self._APT_ENV})
6203
7237
 
6204
7238
  async def install(self, *packages: SystemPackageOrStr) -> None:
6205
7239
  pns = [p.name if isinstance(p, SystemPackage) else p for p in packages] # FIXME: versions
6206
- await asyncio_subprocess_check_call('apt', 'install', '-y', *pns, env={**os.environ, **self._APT_ENV})
7240
+ await asyncio_subprocesses.check_call('sudo', 'apt', 'install', '-y', *pns, env={**os.environ, **self._APT_ENV})
6207
7241
 
6208
7242
  async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
6209
7243
  pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
6210
- cmd = ['dpkg-query', '-W', '-f=${Package}=${Version}\n', *pns]
6211
- stdout, stderr = await asyncio_subprocess_run(
6212
- *cmd,
7244
+ out = await asyncio_subprocesses.run(
7245
+ 'dpkg-query', '-W', '-f=${Package}=${Version}\n', *pns,
6213
7246
  capture_output=True,
6214
7247
  check=False,
6215
7248
  )
6216
7249
  d: ta.Dict[str, SystemPackage] = {}
6217
- for l in check.not_none(stdout).decode('utf-8').strip().splitlines():
7250
+ for l in check.not_none(out.stdout).decode('utf-8').strip().splitlines():
6218
7251
  n, v = l.split('=', 1)
6219
7252
  d[n] = SystemPackage(
6220
7253
  name=n,
@@ -6223,6 +7256,33 @@ class AptSystemPackageManager(SystemPackageManager):
6223
7256
  return d
6224
7257
 
6225
7258
 
7259
+ class YumSystemPackageManager(SystemPackageManager):
7260
+ async def update(self) -> None:
7261
+ await asyncio_subprocesses.check_call('sudo', 'yum', 'check-update')
7262
+
7263
+ async def upgrade(self) -> None:
7264
+ await asyncio_subprocesses.check_call('sudo', 'yum', 'update')
7265
+
7266
+ async def install(self, *packages: SystemPackageOrStr) -> None:
7267
+ pns = [p.name if isinstance(p, SystemPackage) else p for p in packages] # FIXME: versions
7268
+ await asyncio_subprocesses.check_call('sudo', 'yum', 'install', *pns)
7269
+
7270
+ async def query(self, *packages: SystemPackageOrStr) -> ta.Mapping[str, SystemPackage]:
7271
+ pns = [p.name if isinstance(p, SystemPackage) else p for p in packages]
7272
+ d: ta.Dict[str, SystemPackage] = {}
7273
+ for pn in pns:
7274
+ out = await asyncio_subprocesses.run(
7275
+ 'rpm', '-q', pn,
7276
+ capture_output=True,
7277
+ )
7278
+ if not out.proc.returncode:
7279
+ d[pn] = SystemPackage(
7280
+ pn,
7281
+ check.not_none(out.stdout).decode().strip(),
7282
+ )
7283
+ return d
7284
+
7285
+
6226
7286
  ########################################
6227
7287
  # ../../../omdev/interp/providers.py
6228
7288
  """
@@ -6285,139 +7345,342 @@ class RunningInterpProvider(InterpProvider):
6285
7345
 
6286
7346
 
6287
7347
  ########################################
6288
- # ../remote/connection.py
7348
+ # ../commands/inject.py
6289
7349
 
6290
7350
 
6291
7351
  ##
6292
7352
 
6293
7353
 
6294
- class RemoteExecutionConnector(abc.ABC):
6295
- @abc.abstractmethod
6296
- def connect(
6297
- self,
6298
- tgt: RemoteSpawning.Target,
6299
- bs: MainBootstrap,
6300
- ) -> ta.AsyncContextManager[RemoteCommandExecutor]:
6301
- raise NotImplementedError
7354
+ def bind_command(
7355
+ command_cls: ta.Type[Command],
7356
+ executor_cls: ta.Optional[ta.Type[CommandExecutor]],
7357
+ ) -> InjectorBindings:
7358
+ lst: ta.List[InjectorBindingOrBindings] = [
7359
+ inj.bind(CommandRegistration(command_cls), array=True),
7360
+ ]
7361
+
7362
+ if executor_cls is not None:
7363
+ lst.extend([
7364
+ inj.bind(executor_cls, singleton=True),
7365
+ inj.bind(CommandExecutorRegistration(command_cls, executor_cls), array=True),
7366
+ ])
7367
+
7368
+ return inj.as_bindings(*lst)
6302
7369
 
6303
7370
 
6304
7371
  ##
6305
7372
 
6306
7373
 
6307
- class PyremoteRemoteExecutionConnector(RemoteExecutionConnector):
6308
- def __init__(
6309
- self,
6310
- *,
6311
- spawning: RemoteSpawning,
6312
- msh: ObjMarshalerManager,
6313
- payload_file: ta.Optional[RemoteExecutionPayloadFile] = None,
6314
- ) -> None:
6315
- super().__init__()
7374
+ @dc.dataclass(frozen=True)
7375
+ class _FactoryCommandExecutor(CommandExecutor):
7376
+ factory: ta.Callable[[], CommandExecutor]
6316
7377
 
6317
- self._spawning = spawning
6318
- self._msh = msh
6319
- self._payload_file = payload_file
7378
+ def execute(self, i: Command) -> ta.Awaitable[Command.Output]:
7379
+ return self.factory().execute(i)
6320
7380
 
6321
- #
6322
7381
 
6323
- @cached_nullary
6324
- def _payload_src(self) -> str:
6325
- return get_remote_payload_src(file=self._payload_file)
7382
+ ##
6326
7383
 
6327
- @cached_nullary
6328
- def _remote_src(self) -> ta.Sequence[str]:
6329
- return [
6330
- self._payload_src(),
6331
- '_remote_execution_main()',
6332
- ]
6333
7384
 
6334
- @cached_nullary
6335
- def _spawn_src(self) -> str:
6336
- return pyremote_build_bootstrap_cmd(__package__ or 'manage')
7385
+ def bind_commands(
7386
+ *,
7387
+ main_config: MainConfig,
7388
+ ) -> InjectorBindings:
7389
+ lst: ta.List[InjectorBindingOrBindings] = [
7390
+ inj.bind_array(CommandRegistration),
7391
+ inj.bind_array_type(CommandRegistration, CommandRegistrations),
7392
+
7393
+ inj.bind_array(CommandExecutorRegistration),
7394
+ inj.bind_array_type(CommandExecutorRegistration, CommandExecutorRegistrations),
7395
+
7396
+ inj.bind(build_command_name_map, singleton=True),
7397
+ ]
6337
7398
 
6338
7399
  #
6339
7400
 
6340
- @contextlib.asynccontextmanager
6341
- async def connect(
6342
- self,
6343
- tgt: RemoteSpawning.Target,
6344
- bs: MainBootstrap,
6345
- ) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
6346
- spawn_src = self._spawn_src()
6347
- remote_src = self._remote_src()
7401
+ def provide_obj_marshaler_installer(cmds: CommandNameMap) -> ObjMarshalerInstaller:
7402
+ return ObjMarshalerInstaller(functools.partial(install_command_marshaling, cmds))
6348
7403
 
6349
- async with self._spawning.spawn(
6350
- tgt,
6351
- spawn_src,
6352
- debug=bs.main_config.debug,
6353
- ) as proc:
6354
- res = await PyremoteBootstrapDriver( # noqa
6355
- remote_src,
6356
- PyremoteBootstrapOptions(
6357
- debug=bs.main_config.debug,
6358
- ),
6359
- ).async_run(
6360
- proc.stdout,
6361
- proc.stdin,
6362
- )
7404
+ lst.append(inj.bind(provide_obj_marshaler_installer, array=True))
6363
7405
 
6364
- chan = RemoteChannelImpl(
6365
- proc.stdout,
6366
- proc.stdin,
6367
- msh=self._msh,
6368
- )
7406
+ #
6369
7407
 
6370
- await chan.send_obj(bs)
7408
+ def provide_command_executor_map(
7409
+ injector: Injector,
7410
+ crs: CommandExecutorRegistrations,
7411
+ ) -> CommandExecutorMap:
7412
+ dct: ta.Dict[ta.Type[Command], CommandExecutor] = {}
6371
7413
 
6372
- rce: RemoteCommandExecutor
6373
- async with contextlib.aclosing(RemoteCommandExecutor(chan)) as rce:
6374
- await rce.start()
7414
+ cr: CommandExecutorRegistration
7415
+ for cr in crs:
7416
+ if cr.command_cls in dct:
7417
+ raise KeyError(cr.command_cls)
6375
7418
 
6376
- yield rce
7419
+ factory = functools.partial(injector.provide, cr.executor_cls)
7420
+ if main_config.debug:
7421
+ ce = factory()
7422
+ else:
7423
+ ce = _FactoryCommandExecutor(factory)
6377
7424
 
7425
+ dct[cr.command_cls] = ce
6378
7426
 
6379
- ##
7427
+ return CommandExecutorMap(dct)
6380
7428
 
7429
+ lst.extend([
7430
+ inj.bind(provide_command_executor_map, singleton=True),
7431
+
7432
+ inj.bind(LocalCommandExecutor, singleton=True, eager=main_config.debug),
7433
+ ])
7434
+
7435
+ #
7436
+
7437
+ lst.extend([
7438
+ bind_command(PingCommand, PingCommandExecutor),
7439
+ bind_command(SubprocessCommand, SubprocessCommandExecutor),
7440
+ ])
7441
+
7442
+ #
7443
+
7444
+ return inj.as_bindings(*lst)
7445
+
7446
+
7447
+ ########################################
7448
+ # ../deploy/apps.py
7449
+
7450
+
7451
+ def make_deploy_tag(
7452
+ rev: DeployRev,
7453
+ now: ta.Optional[datetime.datetime] = None,
7454
+ ) -> DeployTag:
7455
+ if now is None:
7456
+ now = datetime.datetime.utcnow() # noqa
7457
+ now_fmt = '%Y%m%dT%H%M%S'
7458
+ now_str = now.strftime(now_fmt)
7459
+ return DeployTag('-'.join([rev, now_str]))
6381
7460
 
6382
- class InProcessRemoteExecutionConnector(RemoteExecutionConnector):
7461
+
7462
+ class DeployAppManager(DeployPathOwner):
6383
7463
  def __init__(
6384
7464
  self,
6385
7465
  *,
6386
- msh: ObjMarshalerManager,
6387
- local_executor: LocalCommandExecutor,
7466
+ deploy_home: DeployHome,
7467
+ git: DeployGitManager,
7468
+ venvs: DeployVenvManager,
6388
7469
  ) -> None:
6389
7470
  super().__init__()
6390
7471
 
6391
- self._msh = msh
6392
- self._local_executor = local_executor
7472
+ self._deploy_home = deploy_home
7473
+ self._git = git
7474
+ self._venvs = venvs
6393
7475
 
6394
- @contextlib.asynccontextmanager
6395
- async def connect(
7476
+ self._dir = os.path.join(deploy_home, 'apps')
7477
+
7478
+ def get_deploy_paths(self) -> ta.AbstractSet[DeployPath]:
7479
+ return {
7480
+ DeployPath.parse('apps/@app/@tag'),
7481
+ }
7482
+
7483
+ async def prepare_app(
6396
7484
  self,
6397
- tgt: RemoteSpawning.Target,
6398
- bs: MainBootstrap,
6399
- ) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
6400
- r0, w0 = asyncio_create_bytes_channel()
6401
- r1, w1 = asyncio_create_bytes_channel()
7485
+ app: DeployApp,
7486
+ rev: DeployRev,
7487
+ repo: DeployGitRepo,
7488
+ ):
7489
+ app_tag = DeployAppTag(app, make_deploy_tag(rev))
7490
+ app_dir = os.path.join(self._dir, app, app_tag.tag)
6402
7491
 
6403
- remote_chan = RemoteChannelImpl(r0, w1, msh=self._msh)
6404
- local_chan = RemoteChannelImpl(r1, w0, msh=self._msh)
7492
+ #
6405
7493
 
6406
- rch = _RemoteCommandHandler(
6407
- remote_chan,
6408
- self._local_executor,
7494
+ await self._git.checkout(
7495
+ DeployGitSpec(
7496
+ repo=repo,
7497
+ rev=rev,
7498
+ ),
7499
+ app_dir,
6409
7500
  )
6410
- rch_task = asyncio.create_task(rch.run()) # noqa
6411
- try:
6412
- rce: RemoteCommandExecutor
6413
- async with contextlib.aclosing(RemoteCommandExecutor(local_chan)) as rce:
6414
- await rce.start()
6415
7501
 
6416
- yield rce
7502
+ #
6417
7503
 
6418
- finally:
6419
- rch.stop()
6420
- await rch_task
7504
+ await self._venvs.setup_app_venv(app_tag)
7505
+
7506
+
7507
+ ########################################
7508
+ # ../remote/_main.py
7509
+
7510
+
7511
+ ##
7512
+
7513
+
7514
+ class _RemoteExecutionLogHandler(logging.Handler):
7515
+ def __init__(self, fn: ta.Callable[[str], None]) -> None:
7516
+ super().__init__()
7517
+ self._fn = fn
7518
+
7519
+ def emit(self, record):
7520
+ msg = self.format(record)
7521
+ self._fn(msg)
7522
+
7523
+
7524
+ ##
7525
+
7526
+
7527
+ class _RemoteExecutionMain:
7528
+ def __init__(
7529
+ self,
7530
+ chan: RemoteChannel,
7531
+ ) -> None:
7532
+ super().__init__()
7533
+
7534
+ self._chan = chan
7535
+
7536
+ self.__bootstrap: ta.Optional[MainBootstrap] = None
7537
+ self.__injector: ta.Optional[Injector] = None
7538
+
7539
+ @property
7540
+ def _bootstrap(self) -> MainBootstrap:
7541
+ return check.not_none(self.__bootstrap)
7542
+
7543
+ @property
7544
+ def _injector(self) -> Injector:
7545
+ return check.not_none(self.__injector)
7546
+
7547
+ #
7548
+
7549
+ def _timebomb_main(
7550
+ self,
7551
+ delay_s: float,
7552
+ *,
7553
+ sig: int = signal.SIGINT,
7554
+ code: int = 1,
7555
+ ) -> None:
7556
+ time.sleep(delay_s)
7557
+
7558
+ if (pgid := os.getpgid(0)) == os.getpid():
7559
+ os.killpg(pgid, sig)
7560
+
7561
+ os._exit(code) # noqa
7562
+
7563
+ @cached_nullary
7564
+ def _timebomb_thread(self) -> ta.Optional[threading.Thread]:
7565
+ if (tbd := self._bootstrap.remote_config.timebomb_delay_s) is None:
7566
+ return None
7567
+
7568
+ thr = threading.Thread(
7569
+ target=functools.partial(self._timebomb_main, tbd),
7570
+ name=f'{self.__class__.__name__}.timebomb',
7571
+ daemon=True,
7572
+ )
7573
+
7574
+ thr.start()
7575
+
7576
+ log.debug('Started timebomb thread: %r', thr)
7577
+
7578
+ return thr
7579
+
7580
+ #
7581
+
7582
+ @cached_nullary
7583
+ def _log_handler(self) -> _RemoteLogHandler:
7584
+ return _RemoteLogHandler(self._chan)
7585
+
7586
+ #
7587
+
7588
+ async def _setup(self) -> None:
7589
+ check.none(self.__bootstrap)
7590
+ check.none(self.__injector)
7591
+
7592
+ # Bootstrap
7593
+
7594
+ self.__bootstrap = check.not_none(await self._chan.recv_obj(MainBootstrap))
7595
+
7596
+ if (prd := self._bootstrap.remote_config.pycharm_remote_debug) is not None:
7597
+ pycharm_debug_connect(prd)
7598
+
7599
+ self.__injector = main_bootstrap(self._bootstrap)
7600
+
7601
+ self._chan.set_marshaler(self._injector[ObjMarshalerManager])
7602
+
7603
+ # Post-bootstrap
7604
+
7605
+ if self._bootstrap.remote_config.set_pgid:
7606
+ if os.getpgid(0) != os.getpid():
7607
+ log.debug('Setting pgid')
7608
+ os.setpgid(0, 0)
7609
+
7610
+ if (ds := self._bootstrap.remote_config.deathsig) is not None:
7611
+ log.debug('Setting deathsig: %s', ds)
7612
+ set_process_deathsig(int(signal.Signals[f'SIG{ds.upper()}']))
7613
+
7614
+ self._timebomb_thread()
7615
+
7616
+ if self._bootstrap.remote_config.forward_logging:
7617
+ log.debug('Installing log forwarder')
7618
+ logging.root.addHandler(self._log_handler())
7619
+
7620
+ #
7621
+
7622
+ async def run(self) -> None:
7623
+ await self._setup()
7624
+
7625
+ executor = self._injector[LocalCommandExecutor]
7626
+
7627
+ handler = _RemoteCommandHandler(self._chan, executor)
7628
+
7629
+ await handler.run()
7630
+
7631
+
7632
+ def _remote_execution_main() -> None:
7633
+ rt = pyremote_bootstrap_finalize() # noqa
7634
+
7635
+ async def inner() -> None:
7636
+ input = await asyncio_open_stream_reader(rt.input) # noqa
7637
+ output = await asyncio_open_stream_writer(rt.output)
7638
+
7639
+ chan = RemoteChannelImpl(
7640
+ input,
7641
+ output,
7642
+ )
7643
+
7644
+ await _RemoteExecutionMain(chan).run()
7645
+
7646
+ asyncio.run(inner())
7647
+
7648
+
7649
+ ########################################
7650
+ # ../system/commands.py
7651
+
7652
+
7653
+ ##
7654
+
7655
+
7656
+ @dc.dataclass(frozen=True)
7657
+ class CheckSystemPackageCommand(Command['CheckSystemPackageCommand.Output']):
7658
+ pkgs: ta.Sequence[str] = ()
7659
+
7660
+ def __post_init__(self) -> None:
7661
+ check.not_isinstance(self.pkgs, str)
7662
+
7663
+ @dc.dataclass(frozen=True)
7664
+ class Output(Command.Output):
7665
+ pkgs: ta.Sequence[SystemPackage]
7666
+
7667
+
7668
+ class CheckSystemPackageCommandExecutor(CommandExecutor[CheckSystemPackageCommand, CheckSystemPackageCommand.Output]):
7669
+ def __init__(
7670
+ self,
7671
+ *,
7672
+ mgr: SystemPackageManager,
7673
+ ) -> None:
7674
+ super().__init__()
7675
+
7676
+ self._mgr = mgr
7677
+
7678
+ async def execute(self, cmd: CheckSystemPackageCommand) -> CheckSystemPackageCommand.Output:
7679
+ log.info('Checking system package!')
7680
+
7681
+ ret = await self._mgr.query(*cmd.pkgs)
7682
+
7683
+ return CheckSystemPackageCommand.Output(list(ret.values()))
6421
7684
 
6422
7685
 
6423
7686
  ########################################
@@ -6457,7 +7720,7 @@ class Pyenv:
6457
7720
  return self._root_kw
6458
7721
 
6459
7722
  if shutil.which('pyenv'):
6460
- return await asyncio_subprocess_check_output_str('pyenv', 'root')
7723
+ return await asyncio_subprocesses.check_output_str('pyenv', 'root')
6461
7724
 
6462
7725
  d = os.path.expanduser('~/.pyenv')
6463
7726
  if os.path.isdir(d) and os.path.isfile(os.path.join(d, 'bin', 'pyenv')):
@@ -6486,7 +7749,7 @@ class Pyenv:
6486
7749
  if await self.root() is None:
6487
7750
  return []
6488
7751
  ret = []
6489
- s = await asyncio_subprocess_check_output_str(await self.exe(), 'install', '--list')
7752
+ s = await asyncio_subprocesses.check_output_str(await self.exe(), 'install', '--list')
6490
7753
  for l in s.splitlines():
6491
7754
  if not l.startswith(' '):
6492
7755
  continue
@@ -6501,7 +7764,7 @@ class Pyenv:
6501
7764
  return False
6502
7765
  if not os.path.isdir(os.path.join(root, '.git')):
6503
7766
  return False
6504
- await asyncio_subprocess_check_call('git', 'pull', cwd=root)
7767
+ await asyncio_subprocesses.check_call('git', 'pull', cwd=root)
6505
7768
  return True
6506
7769
 
6507
7770
 
@@ -6592,7 +7855,7 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
6592
7855
  cflags = []
6593
7856
  ldflags = []
6594
7857
  for dep in self.BREW_DEPS:
6595
- dep_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', dep)
7858
+ dep_prefix = await asyncio_subprocesses.check_output_str('brew', '--prefix', dep)
6596
7859
  cflags.append(f'-I{dep_prefix}/include')
6597
7860
  ldflags.append(f'-L{dep_prefix}/lib')
6598
7861
  return PyenvInstallOpts(
@@ -6602,11 +7865,11 @@ class DarwinPyenvInstallOpts(PyenvInstallOptsProvider):
6602
7865
 
6603
7866
  @async_cached_nullary
6604
7867
  async def brew_tcl_opts(self) -> PyenvInstallOpts:
6605
- if await asyncio_subprocess_try_output('brew', '--prefix', 'tcl-tk') is None:
7868
+ if await asyncio_subprocesses.try_output('brew', '--prefix', 'tcl-tk') is None:
6606
7869
  return PyenvInstallOpts()
6607
7870
 
6608
- tcl_tk_prefix = await asyncio_subprocess_check_output_str('brew', '--prefix', 'tcl-tk')
6609
- tcl_tk_ver_str = await asyncio_subprocess_check_output_str('brew', 'ls', '--versions', 'tcl-tk')
7871
+ tcl_tk_prefix = await asyncio_subprocesses.check_output_str('brew', '--prefix', 'tcl-tk')
7872
+ tcl_tk_ver_str = await asyncio_subprocesses.check_output_str('brew', 'ls', '--versions', 'tcl-tk')
6610
7873
  tcl_tk_ver = '.'.join(tcl_tk_ver_str.split()[1].split('.')[:2])
6611
7874
 
6612
7875
  return PyenvInstallOpts(conf_opts=[
@@ -6727,7 +7990,7 @@ class PyenvVersionInstaller:
6727
7990
  *conf_args,
6728
7991
  ]
6729
7992
 
6730
- await asyncio_subprocess_check_call(
7993
+ await asyncio_subprocesses.check_call(
6731
7994
  *full_args,
6732
7995
  env=env,
6733
7996
  )
@@ -6961,54 +8224,183 @@ class SystemInterpProvider(InterpProvider):
6961
8224
  lst.append((e, ev))
6962
8225
  return lst
6963
8226
 
6964
- #
8227
+ #
8228
+
8229
+ async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
8230
+ return [ev for e, ev in await self.exe_versions()]
8231
+
8232
+ async def get_installed_version(self, version: InterpVersion) -> Interp:
8233
+ for e, ev in await self.exe_versions():
8234
+ if ev != version:
8235
+ continue
8236
+ return Interp(
8237
+ exe=e,
8238
+ version=ev,
8239
+ )
8240
+ raise KeyError(version)
8241
+
8242
+
8243
+ ########################################
8244
+ # ../remote/connection.py
8245
+
8246
+
8247
+ ##
8248
+
8249
+
8250
+ class PyremoteRemoteExecutionConnector:
8251
+ def __init__(
8252
+ self,
8253
+ *,
8254
+ spawning: RemoteSpawning,
8255
+ msh: ObjMarshalerManager,
8256
+ payload_file: ta.Optional[RemoteExecutionPayloadFile] = None,
8257
+ ) -> None:
8258
+ super().__init__()
8259
+
8260
+ self._spawning = spawning
8261
+ self._msh = msh
8262
+ self._payload_file = payload_file
8263
+
8264
+ #
8265
+
8266
+ @cached_nullary
8267
+ def _payload_src(self) -> str:
8268
+ return get_remote_payload_src(file=self._payload_file)
8269
+
8270
+ @cached_nullary
8271
+ def _remote_src(self) -> ta.Sequence[str]:
8272
+ return [
8273
+ self._payload_src(),
8274
+ '_remote_execution_main()',
8275
+ ]
8276
+
8277
+ @cached_nullary
8278
+ def _spawn_src(self) -> str:
8279
+ return pyremote_build_bootstrap_cmd(__package__ or 'manage')
8280
+
8281
+ #
8282
+
8283
+ @contextlib.asynccontextmanager
8284
+ async def connect(
8285
+ self,
8286
+ tgt: RemoteSpawning.Target,
8287
+ bs: MainBootstrap,
8288
+ ) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
8289
+ spawn_src = self._spawn_src()
8290
+ remote_src = self._remote_src()
8291
+
8292
+ async with self._spawning.spawn(
8293
+ tgt,
8294
+ spawn_src,
8295
+ debug=bs.main_config.debug,
8296
+ ) as proc:
8297
+ res = await PyremoteBootstrapDriver( # noqa
8298
+ remote_src,
8299
+ PyremoteBootstrapOptions(
8300
+ debug=bs.main_config.debug,
8301
+ ),
8302
+ ).async_run(
8303
+ proc.stdout,
8304
+ proc.stdin,
8305
+ )
8306
+
8307
+ chan = RemoteChannelImpl(
8308
+ proc.stdout,
8309
+ proc.stdin,
8310
+ msh=self._msh,
8311
+ )
8312
+
8313
+ await chan.send_obj(bs)
8314
+
8315
+ rce: RemoteCommandExecutor
8316
+ async with aclosing(RemoteCommandExecutor(chan)) as rce:
8317
+ await rce.start()
8318
+
8319
+ yield rce
8320
+
8321
+
8322
+ ##
8323
+
8324
+
8325
+ class InProcessRemoteExecutionConnector:
8326
+ def __init__(
8327
+ self,
8328
+ *,
8329
+ msh: ObjMarshalerManager,
8330
+ local_executor: LocalCommandExecutor,
8331
+ ) -> None:
8332
+ super().__init__()
8333
+
8334
+ self._msh = msh
8335
+ self._local_executor = local_executor
8336
+
8337
+ @contextlib.asynccontextmanager
8338
+ async def connect(self) -> ta.AsyncGenerator[RemoteCommandExecutor, None]:
8339
+ r0, w0 = asyncio_create_bytes_channel()
8340
+ r1, w1 = asyncio_create_bytes_channel()
8341
+
8342
+ remote_chan = RemoteChannelImpl(r0, w1, msh=self._msh)
8343
+ local_chan = RemoteChannelImpl(r1, w0, msh=self._msh)
6965
8344
 
6966
- async def get_installed_versions(self, spec: InterpSpecifier) -> ta.Sequence[InterpVersion]:
6967
- return [ev for e, ev in await self.exe_versions()]
8345
+ rch = _RemoteCommandHandler(
8346
+ remote_chan,
8347
+ self._local_executor,
8348
+ )
8349
+ rch_task = asyncio.create_task(rch.run()) # noqa
8350
+ try:
8351
+ rce: RemoteCommandExecutor
8352
+ async with aclosing(RemoteCommandExecutor(local_chan)) as rce:
8353
+ await rce.start()
6968
8354
 
6969
- async def get_installed_version(self, version: InterpVersion) -> Interp:
6970
- for e, ev in await self.exe_versions():
6971
- if ev != version:
6972
- continue
6973
- return Interp(
6974
- exe=e,
6975
- version=ev,
6976
- )
6977
- raise KeyError(version)
8355
+ yield rce
8356
+
8357
+ finally:
8358
+ rch.stop()
8359
+ await rch_task
6978
8360
 
6979
8361
 
6980
8362
  ########################################
6981
- # ../remote/inject.py
8363
+ # ../system/inject.py
6982
8364
 
6983
8365
 
6984
- def bind_remote(
8366
+ def bind_system(
6985
8367
  *,
6986
- remote_config: RemoteConfig,
8368
+ system_config: SystemConfig,
6987
8369
  ) -> InjectorBindings:
6988
8370
  lst: ta.List[InjectorBindingOrBindings] = [
6989
- inj.bind(remote_config),
6990
-
6991
- inj.bind(SubprocessRemoteSpawning, singleton=True),
6992
- inj.bind(RemoteSpawning, to_key=SubprocessRemoteSpawning),
8371
+ inj.bind(system_config),
6993
8372
  ]
6994
8373
 
6995
8374
  #
6996
8375
 
6997
- if remote_config.use_in_process_remote_executor:
8376
+ platform = system_config.platform or detect_system_platform()
8377
+ lst.append(inj.bind(platform, key=Platform))
8378
+
8379
+ #
8380
+
8381
+ if isinstance(platform, AmazonLinuxPlatform):
6998
8382
  lst.extend([
6999
- inj.bind(InProcessRemoteExecutionConnector, singleton=True),
7000
- inj.bind(RemoteExecutionConnector, to_key=InProcessRemoteExecutionConnector),
8383
+ inj.bind(YumSystemPackageManager, singleton=True),
8384
+ inj.bind(SystemPackageManager, to_key=YumSystemPackageManager),
7001
8385
  ])
7002
- else:
8386
+
8387
+ elif isinstance(platform, LinuxPlatform):
8388
+ lst.extend([
8389
+ inj.bind(AptSystemPackageManager, singleton=True),
8390
+ inj.bind(SystemPackageManager, to_key=AptSystemPackageManager),
8391
+ ])
8392
+
8393
+ elif isinstance(platform, DarwinPlatform):
7003
8394
  lst.extend([
7004
- inj.bind(PyremoteRemoteExecutionConnector, singleton=True),
7005
- inj.bind(RemoteExecutionConnector, to_key=PyremoteRemoteExecutionConnector),
8395
+ inj.bind(BrewSystemPackageManager, singleton=True),
8396
+ inj.bind(SystemPackageManager, to_key=BrewSystemPackageManager),
7006
8397
  ])
7007
8398
 
7008
8399
  #
7009
8400
 
7010
- if (pf := remote_config.payload_file) is not None:
7011
- lst.append(inj.bind(pf, key=RemoteExecutionPayloadFile))
8401
+ lst.extend([
8402
+ bind_command(CheckSystemPackageCommand, CheckSystemPackageCommandExecutor),
8403
+ ])
7012
8404
 
7013
8405
  #
7014
8406
 
@@ -7114,189 +8506,248 @@ DEFAULT_INTERP_RESOLVER = InterpResolver([(p.name, p) for p in [
7114
8506
 
7115
8507
 
7116
8508
  ########################################
7117
- # ../commands/interp.py
8509
+ # ../remote/inject.py
7118
8510
 
7119
8511
 
7120
- ##
8512
+ def bind_remote(
8513
+ *,
8514
+ remote_config: RemoteConfig,
8515
+ ) -> InjectorBindings:
8516
+ lst: ta.List[InjectorBindingOrBindings] = [
8517
+ inj.bind(remote_config),
7121
8518
 
8519
+ inj.bind(SubprocessRemoteSpawning, singleton=True),
8520
+ inj.bind(RemoteSpawning, to_key=SubprocessRemoteSpawning),
7122
8521
 
7123
- @dc.dataclass(frozen=True)
7124
- class InterpCommand(Command['InterpCommand.Output']):
7125
- spec: str
7126
- install: bool = False
8522
+ inj.bind(PyremoteRemoteExecutionConnector, singleton=True),
8523
+ inj.bind(InProcessRemoteExecutionConnector, singleton=True),
8524
+ ]
7127
8525
 
7128
- @dc.dataclass(frozen=True)
7129
- class Output(Command.Output):
7130
- exe: str
7131
- version: str
7132
- opts: InterpOpts
8526
+ #
8527
+
8528
+ if (pf := remote_config.payload_file) is not None:
8529
+ lst.append(inj.bind(pf, key=RemoteExecutionPayloadFile))
7133
8530
 
8531
+ #
7134
8532
 
7135
- class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
7136
- async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
7137
- i = InterpSpecifier.parse(check.not_none(cmd.spec))
7138
- o = check.not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
7139
- return InterpCommand.Output(
7140
- exe=o.exe,
7141
- version=str(o.version.version),
7142
- opts=o.version.opts,
7143
- )
8533
+ return inj.as_bindings(*lst)
7144
8534
 
7145
8535
 
7146
8536
  ########################################
7147
- # ../commands/inject.py
8537
+ # ../targets/connection.py
7148
8538
 
7149
8539
 
7150
8540
  ##
7151
8541
 
7152
8542
 
7153
- def bind_command(
7154
- command_cls: ta.Type[Command],
7155
- executor_cls: ta.Optional[ta.Type[CommandExecutor]],
7156
- ) -> InjectorBindings:
7157
- lst: ta.List[InjectorBindingOrBindings] = [
7158
- inj.bind(CommandRegistration(command_cls), array=True),
7159
- ]
8543
+ class ManageTargetConnector(abc.ABC):
8544
+ @abc.abstractmethod
8545
+ def connect(self, tgt: ManageTarget) -> ta.AsyncContextManager[CommandExecutor]:
8546
+ raise NotImplementedError
7160
8547
 
7161
- if executor_cls is not None:
7162
- lst.extend([
7163
- inj.bind(executor_cls, singleton=True),
7164
- inj.bind(CommandExecutorRegistration(command_cls, executor_cls), array=True),
7165
- ])
7166
8548
 
7167
- return inj.as_bindings(*lst)
8549
+ ##
7168
8550
 
7169
8551
 
7170
- ##
8552
+ ManageTargetConnectorMap = ta.NewType('ManageTargetConnectorMap', ta.Mapping[ta.Type[ManageTarget], ManageTargetConnector]) # noqa
7171
8553
 
7172
8554
 
7173
8555
  @dc.dataclass(frozen=True)
7174
- class _FactoryCommandExecutor(CommandExecutor):
7175
- factory: ta.Callable[[], CommandExecutor]
8556
+ class TypeSwitchedManageTargetConnector(ManageTargetConnector):
8557
+ connectors: ManageTargetConnectorMap
7176
8558
 
7177
- def execute(self, i: Command) -> ta.Awaitable[Command.Output]:
7178
- return self.factory().execute(i)
8559
+ def get_connector(self, ty: ta.Type[ManageTarget]) -> ManageTargetConnector:
8560
+ for k, v in self.connectors.items():
8561
+ if issubclass(ty, k):
8562
+ return v
8563
+ raise KeyError(ty)
8564
+
8565
+ def connect(self, tgt: ManageTarget) -> ta.AsyncContextManager[CommandExecutor]:
8566
+ return self.get_connector(type(tgt)).connect(tgt)
7179
8567
 
7180
8568
 
7181
8569
  ##
7182
8570
 
7183
8571
 
7184
- def bind_commands(
7185
- *,
7186
- main_config: MainConfig,
7187
- ) -> InjectorBindings:
7188
- lst: ta.List[InjectorBindingOrBindings] = [
7189
- inj.bind_array(CommandRegistration),
7190
- inj.bind_array_type(CommandRegistration, CommandRegistrations),
8572
+ @dc.dataclass(frozen=True)
8573
+ class LocalManageTargetConnector(ManageTargetConnector):
8574
+ _local_executor: LocalCommandExecutor
8575
+ _in_process_connector: InProcessRemoteExecutionConnector
8576
+ _pyremote_connector: PyremoteRemoteExecutionConnector
8577
+ _bootstrap: MainBootstrap
7191
8578
 
7192
- inj.bind_array(CommandExecutorRegistration),
7193
- inj.bind_array_type(CommandExecutorRegistration, CommandExecutorRegistrations),
8579
+ @contextlib.asynccontextmanager
8580
+ async def connect(self, tgt: ManageTarget) -> ta.AsyncGenerator[CommandExecutor, None]:
8581
+ lmt = check.isinstance(tgt, LocalManageTarget)
7194
8582
 
7195
- inj.bind(build_command_name_map, singleton=True),
7196
- ]
8583
+ if isinstance(lmt, InProcessManageTarget):
8584
+ imt = check.isinstance(lmt, InProcessManageTarget)
7197
8585
 
7198
- #
8586
+ if imt.mode == InProcessManageTarget.Mode.DIRECT:
8587
+ yield self._local_executor
7199
8588
 
7200
- def provide_obj_marshaler_installer(cmds: CommandNameMap) -> ObjMarshalerInstaller:
7201
- return ObjMarshalerInstaller(functools.partial(install_command_marshaling, cmds))
8589
+ elif imt.mode == InProcessManageTarget.Mode.FAKE_REMOTE:
8590
+ async with self._in_process_connector.connect() as rce:
8591
+ yield rce
7202
8592
 
7203
- lst.append(inj.bind(provide_obj_marshaler_installer, array=True))
8593
+ else:
8594
+ raise TypeError(imt.mode)
8595
+
8596
+ elif isinstance(lmt, SubprocessManageTarget):
8597
+ async with self._pyremote_connector.connect(
8598
+ RemoteSpawning.Target(
8599
+ python=lmt.python,
8600
+ ),
8601
+ self._bootstrap,
8602
+ ) as rce:
8603
+ yield rce
7204
8604
 
7205
- #
8605
+ else:
8606
+ raise TypeError(lmt)
7206
8607
 
7207
- def provide_command_executor_map(
7208
- injector: Injector,
7209
- crs: CommandExecutorRegistrations,
7210
- ) -> CommandExecutorMap:
7211
- dct: ta.Dict[ta.Type[Command], CommandExecutor] = {}
7212
8608
 
7213
- cr: CommandExecutorRegistration
7214
- for cr in crs:
7215
- if cr.command_cls in dct:
7216
- raise KeyError(cr.command_cls)
8609
+ ##
7217
8610
 
7218
- factory = functools.partial(injector.provide, cr.executor_cls)
7219
- if main_config.debug:
7220
- ce = factory()
7221
- else:
7222
- ce = _FactoryCommandExecutor(factory)
7223
8611
 
7224
- dct[cr.command_cls] = ce
8612
+ @dc.dataclass(frozen=True)
8613
+ class DockerManageTargetConnector(ManageTargetConnector):
8614
+ _pyremote_connector: PyremoteRemoteExecutionConnector
8615
+ _bootstrap: MainBootstrap
7225
8616
 
7226
- return CommandExecutorMap(dct)
8617
+ @contextlib.asynccontextmanager
8618
+ async def connect(self, tgt: ManageTarget) -> ta.AsyncGenerator[CommandExecutor, None]:
8619
+ dmt = check.isinstance(tgt, DockerManageTarget)
8620
+
8621
+ sh_parts: ta.List[str] = ['docker']
8622
+ if dmt.image is not None:
8623
+ sh_parts.extend(['run', '-i', dmt.image])
8624
+ elif dmt.container_id is not None:
8625
+ sh_parts.extend(['exec', dmt.container_id])
8626
+ else:
8627
+ raise ValueError(dmt)
7227
8628
 
7228
- lst.extend([
7229
- inj.bind(provide_command_executor_map, singleton=True),
8629
+ async with self._pyremote_connector.connect(
8630
+ RemoteSpawning.Target(
8631
+ shell=' '.join(sh_parts),
8632
+ python=dmt.python,
8633
+ ),
8634
+ self._bootstrap,
8635
+ ) as rce:
8636
+ yield rce
7230
8637
 
7231
- inj.bind(LocalCommandExecutor, singleton=True, eager=main_config.debug),
7232
- ])
7233
8638
 
7234
- #
8639
+ ##
7235
8640
 
7236
- command_cls: ta.Any
7237
- executor_cls: ta.Any
7238
- for command_cls, executor_cls in [
7239
- (SubprocessCommand, SubprocessCommandExecutor),
7240
- (InterpCommand, InterpCommandExecutor),
7241
- ]:
7242
- lst.append(bind_command(command_cls, executor_cls))
7243
8641
 
7244
- #
8642
+ @dc.dataclass(frozen=True)
8643
+ class SshManageTargetConnector(ManageTargetConnector):
8644
+ _pyremote_connector: PyremoteRemoteExecutionConnector
8645
+ _bootstrap: MainBootstrap
7245
8646
 
7246
- return inj.as_bindings(*lst)
8647
+ @contextlib.asynccontextmanager
8648
+ async def connect(self, tgt: ManageTarget) -> ta.AsyncGenerator[CommandExecutor, None]:
8649
+ smt = check.isinstance(tgt, SshManageTarget)
8650
+
8651
+ sh_parts: ta.List[str] = ['ssh']
8652
+ if smt.key_file is not None:
8653
+ sh_parts.extend(['-i', smt.key_file])
8654
+ addr = check.not_none(smt.host)
8655
+ if smt.username is not None:
8656
+ addr = f'{smt.username}@{addr}'
8657
+ sh_parts.append(addr)
8658
+
8659
+ async with self._pyremote_connector.connect(
8660
+ RemoteSpawning.Target(
8661
+ shell=' '.join(sh_parts),
8662
+ shell_quote=True,
8663
+ python=smt.python,
8664
+ ),
8665
+ self._bootstrap,
8666
+ ) as rce:
8667
+ yield rce
7247
8668
 
7248
8669
 
7249
8670
  ########################################
7250
- # ../deploy/inject.py
8671
+ # ../deploy/interp.py
7251
8672
 
7252
8673
 
7253
- def bind_deploy(
7254
- ) -> InjectorBindings:
7255
- lst: ta.List[InjectorBindingOrBindings] = [
7256
- bind_command(DeployCommand, DeployCommandExecutor),
7257
- ]
8674
+ ##
7258
8675
 
7259
- return inj.as_bindings(*lst)
8676
+
8677
+ @dc.dataclass(frozen=True)
8678
+ class InterpCommand(Command['InterpCommand.Output']):
8679
+ spec: str
8680
+ install: bool = False
8681
+
8682
+ @dc.dataclass(frozen=True)
8683
+ class Output(Command.Output):
8684
+ exe: str
8685
+ version: str
8686
+ opts: InterpOpts
8687
+
8688
+
8689
+ class InterpCommandExecutor(CommandExecutor[InterpCommand, InterpCommand.Output]):
8690
+ async def execute(self, cmd: InterpCommand) -> InterpCommand.Output:
8691
+ i = InterpSpecifier.parse(check.not_none(cmd.spec))
8692
+ o = check.not_none(await DEFAULT_INTERP_RESOLVER.resolve(i, install=cmd.install))
8693
+ return InterpCommand.Output(
8694
+ exe=o.exe,
8695
+ version=str(o.version.version),
8696
+ opts=o.version.opts,
8697
+ )
7260
8698
 
7261
8699
 
7262
8700
  ########################################
7263
- # ../system/inject.py
8701
+ # ../targets/inject.py
7264
8702
 
7265
8703
 
7266
- def bind_system(
7267
- *,
7268
- system_config: SystemConfig,
7269
- ) -> InjectorBindings:
8704
+ def bind_targets() -> InjectorBindings:
7270
8705
  lst: ta.List[InjectorBindingOrBindings] = [
7271
- inj.bind(system_config),
8706
+ inj.bind(LocalManageTargetConnector, singleton=True),
8707
+ inj.bind(DockerManageTargetConnector, singleton=True),
8708
+ inj.bind(SshManageTargetConnector, singleton=True),
8709
+
8710
+ inj.bind(TypeSwitchedManageTargetConnector, singleton=True),
8711
+ inj.bind(ManageTargetConnector, to_key=TypeSwitchedManageTargetConnector),
7272
8712
  ]
7273
8713
 
7274
8714
  #
7275
8715
 
7276
- platform = system_config.platform or sys.platform
7277
- lst.append(inj.bind(platform, key=SystemPlatform))
8716
+ def provide_manage_target_connector_map(injector: Injector) -> ManageTargetConnectorMap:
8717
+ return ManageTargetConnectorMap({
8718
+ LocalManageTarget: injector[LocalManageTargetConnector],
8719
+ DockerManageTarget: injector[DockerManageTargetConnector],
8720
+ SshManageTarget: injector[SshManageTargetConnector],
8721
+ })
8722
+ lst.append(inj.bind(provide_manage_target_connector_map, singleton=True))
7278
8723
 
7279
8724
  #
7280
8725
 
7281
- if platform == 'linux':
7282
- lst.extend([
7283
- inj.bind(AptSystemPackageManager, singleton=True),
7284
- inj.bind(SystemPackageManager, to_key=AptSystemPackageManager),
7285
- ])
8726
+ return inj.as_bindings(*lst)
7286
8727
 
7287
- elif platform == 'darwin':
7288
- lst.extend([
7289
- inj.bind(BrewSystemPackageManager, singleton=True),
7290
- inj.bind(SystemPackageManager, to_key=BrewSystemPackageManager),
7291
- ])
7292
8728
 
7293
- #
8729
+ ########################################
8730
+ # ../deploy/inject.py
7294
8731
 
7295
- lst.extend([
7296
- bind_command(CheckSystemPackageCommand, CheckSystemPackageCommandExecutor),
7297
- ])
7298
8732
 
7299
- #
8733
+ def bind_deploy(
8734
+ *,
8735
+ deploy_config: DeployConfig,
8736
+ ) -> InjectorBindings:
8737
+ lst: ta.List[InjectorBindingOrBindings] = [
8738
+ inj.bind(deploy_config),
8739
+
8740
+ inj.bind(DeployAppManager, singleton=True),
8741
+ inj.bind(DeployGitManager, singleton=True),
8742
+ inj.bind(DeployVenvManager, singleton=True),
8743
+
8744
+ bind_command(DeployCommand, DeployCommandExecutor),
8745
+ bind_command(InterpCommand, InterpCommandExecutor),
8746
+ ]
8747
+
8748
+ if (dh := deploy_config.deploy_home) is not None:
8749
+ dh = os.path.abspath(os.path.expanduser(dh))
8750
+ lst.append(inj.bind(dh, key=DeployHome))
7300
8751
 
7301
8752
  return inj.as_bindings(*lst)
7302
8753
 
@@ -7310,9 +8761,13 @@ def bind_system(
7310
8761
 
7311
8762
  def bind_main(
7312
8763
  *,
7313
- main_config: MainConfig,
7314
- remote_config: RemoteConfig,
7315
- system_config: SystemConfig,
8764
+ main_config: MainConfig = MainConfig(),
8765
+
8766
+ deploy_config: DeployConfig = DeployConfig(),
8767
+ remote_config: RemoteConfig = RemoteConfig(),
8768
+ system_config: SystemConfig = SystemConfig(),
8769
+
8770
+ main_bootstrap: ta.Optional[MainBootstrap] = None,
7316
8771
  ) -> InjectorBindings:
7317
8772
  lst: ta.List[InjectorBindingOrBindings] = [
7318
8773
  inj.bind(main_config),
@@ -7321,7 +8776,9 @@ def bind_main(
7321
8776
  main_config=main_config,
7322
8777
  ),
7323
8778
 
7324
- bind_deploy(),
8779
+ bind_deploy(
8780
+ deploy_config=deploy_config,
8781
+ ),
7325
8782
 
7326
8783
  bind_remote(
7327
8784
  remote_config=remote_config,
@@ -7330,10 +8787,17 @@ def bind_main(
7330
8787
  bind_system(
7331
8788
  system_config=system_config,
7332
8789
  ),
8790
+
8791
+ bind_targets(),
7333
8792
  ]
7334
8793
 
7335
8794
  #
7336
8795
 
8796
+ if main_bootstrap is not None:
8797
+ lst.append(inj.bind(main_bootstrap))
8798
+
8799
+ #
8800
+
7337
8801
  def build_obj_marshaler_manager(insts: ObjMarshalerInstallers) -> ObjMarshalerManager:
7338
8802
  msh = ObjMarshalerManager()
7339
8803
  inst: ObjMarshalerInstaller
@@ -7363,8 +8827,12 @@ def main_bootstrap(bs: MainBootstrap) -> Injector:
7363
8827
 
7364
8828
  injector = inj.create_injector(bind_main( # noqa
7365
8829
  main_config=bs.main_config,
8830
+
8831
+ deploy_config=bs.deploy_config,
7366
8832
  remote_config=bs.remote_config,
7367
8833
  system_config=bs.system_config,
8834
+
8835
+ main_bootstrap=bs,
7368
8836
  ))
7369
8837
 
7370
8838
  return injector
@@ -7378,10 +8846,6 @@ class MainCli(ArgparseCli):
7378
8846
  @argparse_command(
7379
8847
  argparse_arg('--_payload-file'),
7380
8848
 
7381
- argparse_arg('-s', '--shell'),
7382
- argparse_arg('-q', '--shell-quote', action='store_true'),
7383
- argparse_arg('--python', default='python3'),
7384
-
7385
8849
  argparse_arg('--pycharm-debug-port', type=int),
7386
8850
  argparse_arg('--pycharm-debug-host'),
7387
8851
  argparse_arg('--pycharm-debug-version'),
@@ -7390,8 +8854,9 @@ class MainCli(ArgparseCli):
7390
8854
 
7391
8855
  argparse_arg('--debug', action='store_true'),
7392
8856
 
7393
- argparse_arg('--local', action='store_true'),
8857
+ argparse_arg('--deploy-home'),
7394
8858
 
8859
+ argparse_arg('target'),
7395
8860
  argparse_arg('command', nargs='+'),
7396
8861
  )
7397
8862
  async def run(self) -> None:
@@ -7402,6 +8867,10 @@ class MainCli(ArgparseCli):
7402
8867
  debug=bool(self.args.debug),
7403
8868
  ),
7404
8869
 
8870
+ deploy_config=DeployConfig(
8871
+ deploy_home=self.args.deploy_home,
8872
+ ),
8873
+
7405
8874
  remote_config=RemoteConfig(
7406
8875
  payload_file=self.args._payload_file, # noqa
7407
8876
 
@@ -7412,8 +8881,6 @@ class MainCli(ArgparseCli):
7412
8881
  ) if self.args.pycharm_debug_port is not None else None,
7413
8882
 
7414
8883
  timebomb_delay_s=self.args.remote_timebomb_delay_s,
7415
-
7416
- use_in_process_remote_executor=True,
7417
8884
  ),
7418
8885
  )
7419
8886
 
@@ -7427,6 +8894,11 @@ class MainCli(ArgparseCli):
7427
8894
 
7428
8895
  msh = injector[ObjMarshalerManager]
7429
8896
 
8897
+ ts = self.args.target
8898
+ if not ts.startswith('{'):
8899
+ ts = json.dumps({ts: {}})
8900
+ tgt: ManageTarget = msh.unmarshal_obj(json.loads(ts), ManageTarget)
8901
+
7430
8902
  cmds: ta.List[Command] = []
7431
8903
  cmd: Command
7432
8904
  for c in self.args.command:
@@ -7437,21 +8909,7 @@ class MainCli(ArgparseCli):
7437
8909
 
7438
8910
  #
7439
8911
 
7440
- async with contextlib.AsyncExitStack() as es:
7441
- ce: CommandExecutor
7442
-
7443
- if self.args.local:
7444
- ce = injector[LocalCommandExecutor]
7445
-
7446
- else:
7447
- tgt = RemoteSpawning.Target(
7448
- shell=self.args.shell,
7449
- shell_quote=self.args.shell_quote,
7450
- python=self.args.python,
7451
- )
7452
-
7453
- ce = await es.enter_async_context(injector[RemoteExecutionConnector].connect(tgt, bs)) # noqa
7454
-
8912
+ async with injector[ManageTargetConnector].connect(tgt) as ce:
7455
8913
  async def run_command(cmd: Command) -> None:
7456
8914
  res = await ce.try_execute(
7457
8915
  cmd,