omdev 0.0.0.dev213__py3-none-any.whl → 0.0.0.dev215__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omdev/.manifests.json +1 -1
- omdev/ci/__init__.py +1 -0
- omdev/ci/cache.py +100 -121
- omdev/ci/ci.py +120 -118
- omdev/ci/cli.py +50 -24
- omdev/ci/compose.py +1 -8
- omdev/ci/consts.py +1 -0
- omdev/ci/docker.py +4 -6
- omdev/ci/github/{cacheapi.py → api.py} +0 -1
- omdev/ci/github/bootstrap.py +8 -1
- omdev/ci/github/cache.py +36 -289
- omdev/ci/github/cli.py +9 -5
- omdev/ci/github/client.py +492 -0
- omdev/ci/github/env.py +21 -0
- omdev/ci/requirements.py +0 -1
- omdev/ci/shell.py +0 -1
- omdev/ci/utils.py +2 -14
- omdev/scripts/ci.py +1149 -922
- omdev/scripts/pyproject.py +79 -12
- omdev/tools/docker.py +6 -0
- {omdev-0.0.0.dev213.dist-info → omdev-0.0.0.dev215.dist-info}/METADATA +2 -2
- {omdev-0.0.0.dev213.dist-info → omdev-0.0.0.dev215.dist-info}/RECORD +26 -24
- omdev/ci/github/curl.py +0 -209
- {omdev-0.0.0.dev213.dist-info → omdev-0.0.0.dev215.dist-info}/LICENSE +0 -0
- {omdev-0.0.0.dev213.dist-info → omdev-0.0.0.dev215.dist-info}/WHEEL +0 -0
- {omdev-0.0.0.dev213.dist-info → omdev-0.0.0.dev215.dist-info}/entry_points.txt +0 -0
- {omdev-0.0.0.dev213.dist-info → omdev-0.0.0.dev215.dist-info}/top_level.txt +0 -0
omdev/ci/cli.py
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# @omlish-amalg ../scripts/ci.py
|
2
2
|
# ruff: noqa: UP006 UP007
|
3
|
-
# @omlish-lite
|
4
3
|
"""
|
5
4
|
Inputs:
|
6
5
|
- requirements.txt
|
@@ -9,9 +8,11 @@ Inputs:
|
|
9
8
|
|
10
9
|
==
|
11
10
|
|
12
|
-
./python -m ci run --cache-dir ci/cache ci/project omlish-ci
|
11
|
+
./python -m omdev.ci run --cache-dir omdev/ci/tests/cache omdev/ci/tests/project omlish-ci
|
13
12
|
"""
|
13
|
+
import argparse
|
14
14
|
import asyncio
|
15
|
+
import itertools
|
15
16
|
import os.path
|
16
17
|
import sys
|
17
18
|
import typing as ta
|
@@ -20,15 +21,15 @@ from omlish.argparse.cli import ArgparseCli
|
|
20
21
|
from omlish.argparse.cli import argparse_arg
|
21
22
|
from omlish.argparse.cli import argparse_cmd
|
22
23
|
from omlish.lite.check import check
|
24
|
+
from omlish.lite.logs import log
|
23
25
|
from omlish.logs.standard import configure_standard_logging
|
24
26
|
|
25
27
|
from .cache import DirectoryFileCache
|
26
|
-
from .cache import DirectoryShellCache
|
27
28
|
from .cache import FileCache
|
28
|
-
from .cache import ShellCache
|
29
29
|
from .ci import Ci
|
30
30
|
from .compose import get_compose_service_dependencies
|
31
|
-
from .github.
|
31
|
+
from .github.bootstrap import is_in_github_actions
|
32
|
+
from .github.cache import GithubFileCache
|
32
33
|
from .github.cli import GithubCli
|
33
34
|
from .requirements import build_requirements_hash
|
34
35
|
from .shell import ShellCmd
|
@@ -65,8 +66,8 @@ class CiCli(ArgparseCli):
|
|
65
66
|
@argparse_cmd(
|
66
67
|
accepts_unknown=True,
|
67
68
|
)
|
68
|
-
def github(self) -> ta.Optional[int]:
|
69
|
-
return GithubCli(self.unknown_args).
|
69
|
+
async def github(self) -> ta.Optional[int]:
|
70
|
+
return await GithubCli(self.unknown_args).async_cli_run()
|
70
71
|
|
71
72
|
#
|
72
73
|
|
@@ -77,13 +78,20 @@ class CiCli(ArgparseCli):
|
|
77
78
|
argparse_arg('--compose-file'),
|
78
79
|
argparse_arg('-r', '--requirements-txt', action='append'),
|
79
80
|
|
80
|
-
argparse_arg('--github-cache', action='store_true'),
|
81
81
|
argparse_arg('--cache-dir'),
|
82
82
|
|
83
|
+
argparse_arg('--github', action='store_true'),
|
84
|
+
argparse_arg('--github-detect', action='store_true'),
|
85
|
+
|
83
86
|
argparse_arg('--always-pull', action='store_true'),
|
84
87
|
argparse_arg('--always-build', action='store_true'),
|
85
88
|
|
86
89
|
argparse_arg('--no-dependencies', action='store_true'),
|
90
|
+
|
91
|
+
argparse_arg('-e', '--env', action='append'),
|
92
|
+
argparse_arg('-v', '--volume', action='append'),
|
93
|
+
|
94
|
+
argparse_arg('cmd', nargs=argparse.REMAINDER),
|
87
95
|
)
|
88
96
|
async def run(self) -> None:
|
89
97
|
project_dir = self.args.project_dir
|
@@ -94,6 +102,11 @@ class CiCli(ArgparseCli):
|
|
94
102
|
|
95
103
|
#
|
96
104
|
|
105
|
+
cmd = ' '.join(self.args.cmd)
|
106
|
+
check.non_empty_str(cmd)
|
107
|
+
|
108
|
+
#
|
109
|
+
|
97
110
|
check.state(os.path.isdir(project_dir))
|
98
111
|
|
99
112
|
#
|
@@ -102,6 +115,7 @@ class CiCli(ArgparseCli):
|
|
102
115
|
for alt in alts:
|
103
116
|
alt_file = os.path.abspath(os.path.join(project_dir, alt))
|
104
117
|
if os.path.isfile(alt_file):
|
118
|
+
log.debug('Using %s', alt_file)
|
105
119
|
return alt_file
|
106
120
|
return None
|
107
121
|
|
@@ -135,6 +149,7 @@ class CiCli(ArgparseCli):
|
|
135
149
|
'requirements-ci.txt',
|
136
150
|
]:
|
137
151
|
if os.path.exists(os.path.join(project_dir, rf)):
|
152
|
+
log.debug('Using %s', rf)
|
138
153
|
requirements_txts.append(rf)
|
139
154
|
else:
|
140
155
|
for rf in requirements_txts:
|
@@ -142,21 +157,34 @@ class CiCli(ArgparseCli):
|
|
142
157
|
|
143
158
|
#
|
144
159
|
|
145
|
-
|
160
|
+
github = self.args.github
|
161
|
+
if not github and self.args.github_detect:
|
162
|
+
github = is_in_github_actions()
|
163
|
+
if github:
|
164
|
+
log.debug('Github detected')
|
165
|
+
|
166
|
+
#
|
167
|
+
|
146
168
|
file_cache: ta.Optional[FileCache] = None
|
147
169
|
if cache_dir is not None:
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
170
|
+
cache_dir = os.path.abspath(cache_dir)
|
171
|
+
log.debug('Using cache dir %s', cache_dir)
|
172
|
+
if github:
|
173
|
+
file_cache = GithubFileCache(cache_dir)
|
174
|
+
else:
|
175
|
+
file_cache = DirectoryFileCache(cache_dir)
|
153
176
|
|
154
|
-
|
177
|
+
#
|
155
178
|
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
179
|
+
run_options: ta.List[str] = []
|
180
|
+
for run_arg, run_arg_vals in [
|
181
|
+
('-e', self.args.env or []),
|
182
|
+
('-v', self.args.volume or []),
|
183
|
+
]:
|
184
|
+
run_options.extend(itertools.chain.from_iterable(
|
185
|
+
[run_arg, run_arg_val]
|
186
|
+
for run_arg_val in run_arg_vals
|
187
|
+
))
|
160
188
|
|
161
189
|
#
|
162
190
|
|
@@ -171,18 +199,16 @@ class CiCli(ArgparseCli):
|
|
171
199
|
|
172
200
|
requirements_txts=requirements_txts,
|
173
201
|
|
174
|
-
cmd=ShellCmd(
|
175
|
-
'cd /project',
|
176
|
-
'python3 -m pytest -svv test.py',
|
177
|
-
])),
|
202
|
+
cmd=ShellCmd(cmd),
|
178
203
|
|
179
204
|
always_pull=self.args.always_pull,
|
180
205
|
always_build=self.args.always_build,
|
181
206
|
|
182
207
|
no_dependencies=self.args.no_dependencies,
|
208
|
+
|
209
|
+
run_options=run_options,
|
183
210
|
),
|
184
211
|
file_cache=file_cache,
|
185
|
-
shell_cache=shell_cache,
|
186
212
|
) as ci:
|
187
213
|
await ci.run()
|
188
214
|
|
omdev/ci/compose.py
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
-
# @omlish-lite
|
3
2
|
"""
|
4
3
|
TODO:
|
5
4
|
- fix rmi - only when not referenced anymore
|
@@ -18,9 +17,9 @@ from omlish.lite.contextmanagers import AsyncExitStacked
|
|
18
17
|
from omlish.lite.contextmanagers import adefer
|
19
18
|
from omlish.lite.contextmanagers import defer
|
20
19
|
from omlish.lite.json import json_dumps_pretty
|
20
|
+
from omlish.os.temp import make_temp_file
|
21
21
|
|
22
22
|
from .shell import ShellCmd
|
23
|
-
from .utils import make_temp_file
|
24
23
|
from .utils import read_yaml_file
|
25
24
|
|
26
25
|
|
@@ -103,11 +102,6 @@ class DockerComposeRun(AsyncExitStacked):
|
|
103
102
|
if k in out_service:
|
104
103
|
del out_service[k]
|
105
104
|
|
106
|
-
out_service['links'] = [
|
107
|
-
f'{l}:{l}' if ':' not in l else l
|
108
|
-
for l in out_service.get('links', [])
|
109
|
-
]
|
110
|
-
|
111
105
|
#
|
112
106
|
|
113
107
|
if not self._cfg.no_dependencies:
|
@@ -124,7 +118,6 @@ class DockerComposeRun(AsyncExitStacked):
|
|
124
118
|
|
125
119
|
else:
|
126
120
|
out_service['depends_on'] = []
|
127
|
-
out_service['links'] = []
|
128
121
|
|
129
122
|
#
|
130
123
|
|
omdev/ci/consts.py
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
CI_CACHE_VERSION = 1
|
omdev/ci/docker.py
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
-
# @omlish-lite
|
3
2
|
"""
|
4
3
|
TODO:
|
5
4
|
- some less stupid Dockerfile hash
|
@@ -15,10 +14,9 @@ import typing as ta
|
|
15
14
|
|
16
15
|
from omlish.asyncs.asyncio.subprocesses import asyncio_subprocesses
|
17
16
|
from omlish.lite.check import check
|
18
|
-
from omlish.
|
17
|
+
from omlish.os.temp import temp_file_context
|
19
18
|
|
20
19
|
from .shell import ShellCmd
|
21
|
-
from .utils import make_temp_file
|
22
20
|
from .utils import sha256_str
|
23
21
|
|
24
22
|
|
@@ -91,16 +89,16 @@ async def build_docker_image(
|
|
91
89
|
*,
|
92
90
|
tag: ta.Optional[str] = None,
|
93
91
|
cwd: ta.Optional[str] = None,
|
92
|
+
run_options: ta.Optional[ta.Sequence[str]] = None,
|
94
93
|
) -> str:
|
95
|
-
|
96
|
-
with defer(lambda: os.unlink(id_file)):
|
94
|
+
with temp_file_context() as id_file:
|
97
95
|
await asyncio_subprocesses.check_call(
|
98
96
|
'docker',
|
99
97
|
'build',
|
100
98
|
'-f', os.path.abspath(docker_file),
|
101
99
|
'--iidfile', id_file,
|
102
|
-
'--squash',
|
103
100
|
*(['--tag', tag] if tag is not None else []),
|
101
|
+
*(run_options or []),
|
104
102
|
'.',
|
105
103
|
**(dict(cwd=cwd) if cwd is not None else {}),
|
106
104
|
)
|
omdev/ci/github/bootstrap.py
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
-
# @omlish-lite
|
3
2
|
"""
|
4
3
|
sudo rm -rf \
|
5
4
|
/usr/local/.ghcup \
|
@@ -9,3 +8,11 @@ sudo rm -rf \
|
|
9
8
|
/opt/hostedtoolcache 8.0G, 14843980 files
|
10
9
|
/usr/local/lib/android 6.4G, 17251667 files
|
11
10
|
"""
|
11
|
+
from .env import register_github_env_var
|
12
|
+
|
13
|
+
|
14
|
+
GITHUB_ACTIONS_ENV_VAR = register_github_env_var('GITHUB_ACTIONS')
|
15
|
+
|
16
|
+
|
17
|
+
def is_in_github_actions() -> bool:
|
18
|
+
return GITHUB_ACTIONS_ENV_VAR() is not None
|
omdev/ci/github/cache.py
CHANGED
@@ -1,324 +1,71 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
-
|
3
|
-
import abc
|
4
|
-
import dataclasses as dc
|
5
|
-
import os
|
6
|
-
import shlex
|
2
|
+
import os.path
|
7
3
|
import typing as ta
|
8
|
-
import urllib.parse
|
9
4
|
|
10
5
|
from omlish.lite.check import check
|
11
|
-
from omlish.
|
6
|
+
from omlish.os.files import unlinking_if_exists
|
12
7
|
|
13
8
|
from ..cache import DirectoryFileCache
|
14
|
-
from ..cache import
|
15
|
-
from
|
16
|
-
from
|
17
|
-
from .cacheapi import GithubCacheServiceV1
|
18
|
-
from .curl import GithubServiceCurlClient
|
9
|
+
from ..cache import FileCache
|
10
|
+
from .client import GithubCacheClient
|
11
|
+
from .client import GithubCacheServiceV1Client
|
19
12
|
|
20
13
|
|
21
14
|
##
|
22
15
|
|
23
16
|
|
24
|
-
class
|
25
|
-
class Entry(abc.ABC): # noqa
|
26
|
-
pass
|
27
|
-
|
28
|
-
@abc.abstractmethod
|
29
|
-
def run_get_entry(self, key: str) -> ta.Optional[Entry]:
|
30
|
-
raise NotImplementedError
|
31
|
-
|
32
|
-
@abc.abstractmethod
|
33
|
-
def download_get_entry(self, entry: Entry, out_file: str) -> None:
|
34
|
-
raise NotImplementedError
|
35
|
-
|
36
|
-
@abc.abstractmethod
|
37
|
-
def upload_cache_entry(self, key: str, in_file: str) -> None:
|
38
|
-
raise NotImplementedError
|
39
|
-
|
40
|
-
|
41
|
-
#
|
42
|
-
|
43
|
-
|
44
|
-
class GithubCacheServiceV1ShellClient(GithubCacheShellClient):
|
45
|
-
BASE_URL_ENV_KEY = 'ACTIONS_CACHE_URL'
|
46
|
-
AUTH_TOKEN_ENV_KEY = 'ACTIONS_RUNTIME_TOKEN' # noqa
|
47
|
-
|
48
|
-
KEY_SUFFIX_ENV_KEY = 'GITHUB_RUN_ID'
|
49
|
-
|
50
|
-
CACHE_VERSION: ta.ClassVar[int] = 1
|
51
|
-
|
52
|
-
#
|
53
|
-
|
54
|
-
def __init__(
|
55
|
-
self,
|
56
|
-
*,
|
57
|
-
base_url: ta.Optional[str] = None,
|
58
|
-
auth_token: ta.Optional[str] = None,
|
59
|
-
|
60
|
-
key_prefix: ta.Optional[str] = None,
|
61
|
-
key_suffix: ta.Optional[str] = None,
|
62
|
-
) -> None:
|
63
|
-
super().__init__()
|
64
|
-
|
65
|
-
#
|
66
|
-
|
67
|
-
if base_url is None:
|
68
|
-
base_url = os.environ[self.BASE_URL_ENV_KEY]
|
69
|
-
service_url = GithubCacheServiceV1.get_service_url(base_url)
|
70
|
-
|
71
|
-
if auth_token is None:
|
72
|
-
auth_token = os.environ.get(self.AUTH_TOKEN_ENV_KEY)
|
73
|
-
|
74
|
-
self._curl = GithubServiceCurlClient(
|
75
|
-
service_url,
|
76
|
-
auth_token,
|
77
|
-
api_version=GithubCacheServiceV1.API_VERSION,
|
78
|
-
)
|
79
|
-
|
80
|
-
#
|
81
|
-
|
82
|
-
self._key_prefix = key_prefix
|
83
|
-
|
84
|
-
if key_suffix is None:
|
85
|
-
key_suffix = os.environ[self.KEY_SUFFIX_ENV_KEY]
|
86
|
-
self._key_suffix = check.non_empty_str(key_suffix)
|
87
|
-
|
88
|
-
#
|
89
|
-
|
90
|
-
KEY_PART_SEPARATOR = '--'
|
91
|
-
|
92
|
-
def fix_key(self, s: str) -> str:
|
93
|
-
return self.KEY_PART_SEPARATOR.join([
|
94
|
-
*([self._key_prefix] if self._key_prefix else []),
|
95
|
-
s,
|
96
|
-
self._key_suffix,
|
97
|
-
])
|
98
|
-
|
99
|
-
#
|
100
|
-
|
101
|
-
@dc.dataclass(frozen=True)
|
102
|
-
class Entry(GithubCacheShellClient.Entry):
|
103
|
-
artifact: GithubCacheServiceV1.ArtifactCacheEntry
|
104
|
-
|
105
|
-
#
|
106
|
-
|
107
|
-
def build_get_entry_curl_cmd(self, key: str) -> ShellCmd:
|
108
|
-
fixed_key = self.fix_key(key)
|
109
|
-
|
110
|
-
qp = dict(
|
111
|
-
keys=fixed_key,
|
112
|
-
version=str(self.CACHE_VERSION),
|
113
|
-
)
|
114
|
-
|
115
|
-
return self._curl.build_cmd(
|
116
|
-
'GET',
|
117
|
-
shlex.quote('?'.join([
|
118
|
-
'cache',
|
119
|
-
'&'.join([
|
120
|
-
f'{k}={urllib.parse.quote_plus(v)}'
|
121
|
-
for k, v in qp.items()
|
122
|
-
]),
|
123
|
-
])),
|
124
|
-
)
|
125
|
-
|
126
|
-
def run_get_entry(self, key: str) -> ta.Optional[Entry]:
|
127
|
-
fixed_key = self.fix_key(key)
|
128
|
-
curl_cmd = self.build_get_entry_curl_cmd(fixed_key)
|
129
|
-
|
130
|
-
obj = self._curl.run_json_cmd(
|
131
|
-
curl_cmd,
|
132
|
-
success_status_codes=[200, 204],
|
133
|
-
)
|
134
|
-
if obj is None:
|
135
|
-
return None
|
136
|
-
|
137
|
-
return self.Entry(GithubCacheServiceV1.dataclass_from_json(
|
138
|
-
GithubCacheServiceV1.ArtifactCacheEntry,
|
139
|
-
obj,
|
140
|
-
))
|
141
|
-
|
142
|
-
#
|
143
|
-
|
144
|
-
def build_download_get_entry_cmd(self, entry: Entry, out_file: str) -> ShellCmd:
|
145
|
-
return ShellCmd(' '.join([
|
146
|
-
'aria2c',
|
147
|
-
'-x', '4',
|
148
|
-
'-o', out_file,
|
149
|
-
check.non_empty_str(entry.artifact.archive_location),
|
150
|
-
]))
|
151
|
-
|
152
|
-
def download_get_entry(self, entry: GithubCacheShellClient.Entry, out_file: str) -> None:
|
153
|
-
dl_cmd = self.build_download_get_entry_cmd(
|
154
|
-
check.isinstance(entry, GithubCacheServiceV1ShellClient.Entry),
|
155
|
-
out_file,
|
156
|
-
)
|
157
|
-
dl_cmd.run(subprocesses.check_call)
|
158
|
-
|
159
|
-
#
|
160
|
-
|
161
|
-
def upload_cache_entry(self, key: str, in_file: str) -> None:
|
162
|
-
fixed_key = self.fix_key(key)
|
163
|
-
|
164
|
-
check.state(os.path.isfile(in_file))
|
165
|
-
|
166
|
-
file_size = os.stat(in_file).st_size
|
167
|
-
|
168
|
-
#
|
169
|
-
|
170
|
-
reserve_req = GithubCacheServiceV1.ReserveCacheRequest(
|
171
|
-
key=fixed_key,
|
172
|
-
cache_size=file_size,
|
173
|
-
version=str(self.CACHE_VERSION),
|
174
|
-
)
|
175
|
-
reserve_cmd = self._curl.build_post_json_cmd(
|
176
|
-
'caches',
|
177
|
-
GithubCacheServiceV1.dataclass_to_json(reserve_req),
|
178
|
-
)
|
179
|
-
reserve_resp_obj: ta.Any = check.not_none(self._curl.run_json_cmd(
|
180
|
-
reserve_cmd,
|
181
|
-
success_status_codes=[201],
|
182
|
-
))
|
183
|
-
reserve_resp = GithubCacheServiceV1.dataclass_from_json( # noqa
|
184
|
-
GithubCacheServiceV1.ReserveCacheResponse,
|
185
|
-
reserve_resp_obj,
|
186
|
-
)
|
187
|
-
cache_id = check.isinstance(reserve_resp.cache_id, int)
|
188
|
-
|
189
|
-
#
|
190
|
-
|
191
|
-
tmp_file = make_temp_file()
|
192
|
-
|
193
|
-
print(f'{file_size=}')
|
194
|
-
num_written = 0
|
195
|
-
chunk_size = 32 * 1024 * 1024
|
196
|
-
for i in range((file_size // chunk_size) + (1 if file_size % chunk_size else 0)):
|
197
|
-
ofs = i * chunk_size
|
198
|
-
sz = min(chunk_size, file_size - ofs)
|
199
|
-
|
200
|
-
patch_cmd = self._curl.build_cmd(
|
201
|
-
'PATCH',
|
202
|
-
f'caches/{cache_id}',
|
203
|
-
content_type='application/octet-stream',
|
204
|
-
headers={
|
205
|
-
'Content-Range': f'bytes {ofs}-{ofs + sz - 1}/*',
|
206
|
-
},
|
207
|
-
)
|
208
|
-
|
209
|
-
#
|
210
|
-
|
211
|
-
# patch_data_cmd = dc.replace(patch_cmd, s=' | '.join([
|
212
|
-
# f'dd if={in_file} bs={chunk_size} skip={i} count=1 status=none',
|
213
|
-
# f'{patch_cmd.s} --data-binary -',
|
214
|
-
# ]))
|
215
|
-
# print(f'{patch_data_cmd.s=}')
|
216
|
-
# patch_result = self._curl.run_cmd(patch_data_cmd, raise_=True)
|
217
|
-
|
218
|
-
#
|
219
|
-
|
220
|
-
with open(in_file, 'rb') as f:
|
221
|
-
f.seek(ofs)
|
222
|
-
buf = f.read(sz)
|
223
|
-
with open(tmp_file, 'wb') as f:
|
224
|
-
f.write(buf)
|
225
|
-
num_written += len(buf)
|
226
|
-
print(f'{num_written=}')
|
227
|
-
patch_data_cmd = dc.replace(patch_cmd, s=f'{patch_cmd.s} --data-binary @{tmp_file}')
|
228
|
-
print(f'{patch_data_cmd.s=}')
|
229
|
-
patch_result = self._curl.run_cmd(patch_data_cmd, raise_=True)
|
230
|
-
|
231
|
-
#
|
232
|
-
|
233
|
-
check.equal(patch_result.status_code, 204)
|
234
|
-
ofs += sz
|
235
|
-
|
236
|
-
#
|
237
|
-
|
238
|
-
commit_req = GithubCacheServiceV1.CommitCacheRequest(
|
239
|
-
size=file_size,
|
240
|
-
)
|
241
|
-
commit_cmd = self._curl.build_post_json_cmd(
|
242
|
-
f'caches/{cache_id}',
|
243
|
-
GithubCacheServiceV1.dataclass_to_json(commit_req),
|
244
|
-
)
|
245
|
-
commit_result = self._curl.run_cmd(commit_cmd, raise_=True)
|
246
|
-
check.equal(commit_result.status_code, 204)
|
247
|
-
|
248
|
-
|
249
|
-
##
|
250
|
-
|
251
|
-
|
252
|
-
class GithubShellCache(ShellCache):
|
17
|
+
class GithubFileCache(FileCache):
|
253
18
|
def __init__(
|
254
19
|
self,
|
255
20
|
dir: str, # noqa
|
256
21
|
*,
|
257
|
-
client: ta.Optional[
|
22
|
+
client: ta.Optional[GithubCacheClient] = None,
|
23
|
+
**kwargs: ta.Any,
|
258
24
|
) -> None:
|
259
|
-
super().__init__()
|
25
|
+
super().__init__(**kwargs)
|
260
26
|
|
261
27
|
self._dir = check.not_none(dir)
|
262
28
|
|
263
29
|
if client is None:
|
264
|
-
client =
|
265
|
-
|
30
|
+
client = GithubCacheServiceV1Client(
|
31
|
+
cache_version=self._version,
|
32
|
+
)
|
33
|
+
self._client: GithubCacheClient = client
|
266
34
|
|
267
|
-
self._local = DirectoryFileCache(
|
35
|
+
self._local = DirectoryFileCache(
|
36
|
+
self._dir,
|
37
|
+
version=self._version,
|
38
|
+
)
|
268
39
|
|
269
|
-
def
|
40
|
+
async def get_file(self, key: str) -> ta.Optional[str]:
|
270
41
|
local_file = self._local.get_cache_file_path(key)
|
271
42
|
if os.path.exists(local_file):
|
272
|
-
return
|
43
|
+
return local_file
|
273
44
|
|
274
|
-
if (entry := self._client.
|
45
|
+
if (entry := await self._client.get_entry(key)) is None:
|
275
46
|
return None
|
276
47
|
|
277
48
|
tmp_file = self._local.format_incomplete_file(local_file)
|
278
|
-
|
279
|
-
self._client.
|
49
|
+
with unlinking_if_exists(tmp_file):
|
50
|
+
await self._client.download_file(entry, tmp_file)
|
280
51
|
|
281
52
|
os.replace(tmp_file, local_file)
|
282
53
|
|
283
|
-
|
284
|
-
os.unlink(tmp_file)
|
285
|
-
|
286
|
-
raise
|
54
|
+
return local_file
|
287
55
|
|
288
|
-
|
289
|
-
|
290
|
-
class _PutFileCmdContext(ShellCache.PutFileCmdContext): # noqa
|
291
|
-
def __init__(
|
292
|
-
self,
|
293
|
-
owner: 'GithubShellCache',
|
294
|
-
key: str,
|
295
|
-
tmp_file: str,
|
296
|
-
local_file: str,
|
297
|
-
) -> None:
|
298
|
-
super().__init__()
|
299
|
-
|
300
|
-
self._owner = owner
|
301
|
-
self._key = key
|
302
|
-
self._tmp_file = tmp_file
|
303
|
-
self._local_file = local_file
|
304
|
-
|
305
|
-
@property
|
306
|
-
def cmd(self) -> ShellCmd:
|
307
|
-
return ShellCmd(f'cat > {shlex.quote(self._tmp_file)}')
|
308
|
-
|
309
|
-
def _commit(self) -> None:
|
310
|
-
os.replace(self._tmp_file, self._local_file)
|
311
|
-
|
312
|
-
self._owner._client.upload_cache_entry(self._key, self._local_file) # noqa
|
313
|
-
|
314
|
-
def _abort(self) -> None:
|
315
|
-
os.unlink(self._tmp_file)
|
316
|
-
|
317
|
-
def put_file_cmd(self, key: str) -> ShellCache.PutFileCmdContext:
|
318
|
-
local_file = self._local.get_cache_file_path(key, make_dirs=True)
|
319
|
-
return self._PutFileCmdContext(
|
56
|
+
async def put_file(
|
320
57
|
self,
|
58
|
+
key: str,
|
59
|
+
file_path: str,
|
60
|
+
*,
|
61
|
+
steal: bool = False,
|
62
|
+
) -> str:
|
63
|
+
cache_file_path = await self._local.put_file(
|
321
64
|
key,
|
322
|
-
|
323
|
-
|
65
|
+
file_path,
|
66
|
+
steal=steal,
|
324
67
|
)
|
68
|
+
|
69
|
+
await self._client.upload_file(key, cache_file_path)
|
70
|
+
|
71
|
+
return cache_file_path
|
omdev/ci/github/cli.py
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
|
-
# @omlish-lite
|
3
2
|
"""
|
4
3
|
See:
|
5
4
|
- https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28
|
@@ -11,16 +10,21 @@ from omlish.argparse.cli import argparse_arg
|
|
11
10
|
from omlish.argparse.cli import argparse_cmd
|
12
11
|
from omlish.lite.json import json_dumps_pretty
|
13
12
|
|
14
|
-
from .
|
13
|
+
from .client import GithubCacheServiceV1Client
|
14
|
+
from .env import GITHUB_ENV_VARS
|
15
15
|
|
16
16
|
|
17
17
|
class GithubCli(ArgparseCli):
|
18
|
+
@argparse_cmd()
|
19
|
+
def list_referenced_env_vars(self) -> None:
|
20
|
+
print('\n'.join(sorted(ev.k for ev in GITHUB_ENV_VARS)))
|
21
|
+
|
18
22
|
@argparse_cmd(
|
19
23
|
argparse_arg('key'),
|
20
24
|
)
|
21
|
-
def get_cache_entry(self) -> None:
|
22
|
-
|
23
|
-
entry =
|
25
|
+
async def get_cache_entry(self) -> None:
|
26
|
+
client = GithubCacheServiceV1Client()
|
27
|
+
entry = await client.get_entry(self.args.key)
|
24
28
|
if entry is None:
|
25
29
|
return
|
26
30
|
print(json_dumps_pretty(dc.asdict(entry))) # noqa
|