omdev 0.0.0.dev223__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omdev/ci/cache.py +108 -0
- omdev/ci/ci.py +1 -1
- omdev/ci/docker/cacheserved.py +262 -0
- omdev/ci/docker/dataserver.py +204 -0
- omdev/ci/docker/imagepulling.py +2 -1
- omdev/ci/docker/packing.py +72 -0
- omdev/ci/docker/repositories.py +40 -0
- omdev/ci/github/cache.py +20 -1
- omdev/ci/github/client.py +9 -2
- omdev/ci/github/inject.py +4 -4
- omdev/ci/utils.py +0 -49
- omdev/dataserver/targets.py +32 -0
- omdev/oci/data.py +19 -0
- omdev/oci/dataserver.py +4 -1
- omdev/oci/pack/__init__.py +0 -0
- omdev/oci/pack/packing.py +185 -0
- omdev/oci/pack/repositories.py +162 -0
- omdev/oci/{packing.py → pack/unpacking.py} +0 -177
- omdev/oci/repositories.py +6 -0
- omdev/scripts/ci.py +423 -255
- omdev/scripts/interp.py +19 -0
- omdev/scripts/pyproject.py +19 -0
- {omdev-0.0.0.dev223.dist-info → omdev-0.0.0.dev224.dist-info}/METADATA +2 -2
- {omdev-0.0.0.dev223.dist-info → omdev-0.0.0.dev224.dist-info}/RECORD +28 -21
- {omdev-0.0.0.dev223.dist-info → omdev-0.0.0.dev224.dist-info}/LICENSE +0 -0
- {omdev-0.0.0.dev223.dist-info → omdev-0.0.0.dev224.dist-info}/WHEEL +0 -0
- {omdev-0.0.0.dev223.dist-info → omdev-0.0.0.dev224.dist-info}/entry_points.txt +0 -0
- {omdev-0.0.0.dev223.dist-info → omdev-0.0.0.dev224.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,6 @@
|
|
1
1
|
# ruff: noqa: UP006 UP007
|
2
2
|
# @omlish-lite
|
3
3
|
import contextlib
|
4
|
-
import heapq
|
5
4
|
import os.path
|
6
5
|
import tarfile
|
7
6
|
import typing as ta
|
@@ -10,10 +9,6 @@ from omlish.lite.cached import cached_nullary
|
|
10
9
|
from omlish.lite.check import check
|
11
10
|
from omlish.lite.contextmanagers import ExitStacked
|
12
11
|
|
13
|
-
from .compression import OciCompression
|
14
|
-
from .tars import OciDataTarWriter
|
15
|
-
from .tars import WrittenOciDataTarFileInfo
|
16
|
-
|
17
12
|
|
18
13
|
##
|
19
14
|
|
@@ -207,175 +202,3 @@ class OciLayerUnpacker(ExitStacked):
|
|
207
202
|
def write(self) -> None:
|
208
203
|
for input_file in self._input_files:
|
209
204
|
self._unpack_file(input_file)
|
210
|
-
|
211
|
-
|
212
|
-
#
|
213
|
-
|
214
|
-
|
215
|
-
class OciLayerPacker(ExitStacked):
|
216
|
-
def __init__(
|
217
|
-
self,
|
218
|
-
input_file_path: str,
|
219
|
-
output_file_paths: ta.Sequence[str],
|
220
|
-
*,
|
221
|
-
compression: ta.Optional[OciCompression] = None,
|
222
|
-
) -> None:
|
223
|
-
super().__init__()
|
224
|
-
|
225
|
-
self._input_file_path = input_file_path
|
226
|
-
self._output_file_paths = list(output_file_paths)
|
227
|
-
self._compression = compression
|
228
|
-
|
229
|
-
self._output_file_indexes_by_name: ta.Dict[str, int] = {}
|
230
|
-
|
231
|
-
#
|
232
|
-
|
233
|
-
@cached_nullary
|
234
|
-
def _input_tar_file(self) -> tarfile.TarFile:
|
235
|
-
# FIXME: check uncompressed
|
236
|
-
return self._enter_context(tarfile.open(self._input_file_path))
|
237
|
-
|
238
|
-
#
|
239
|
-
|
240
|
-
@cached_nullary
|
241
|
-
def _entries_by_name(self) -> ta.Mapping[str, tarfile.TarInfo]:
|
242
|
-
return {
|
243
|
-
info.name: info
|
244
|
-
for info in self._input_tar_file().getmembers()
|
245
|
-
}
|
246
|
-
|
247
|
-
#
|
248
|
-
|
249
|
-
class _CategorizedEntries(ta.NamedTuple):
|
250
|
-
files_by_name: ta.Mapping[str, tarfile.TarInfo]
|
251
|
-
non_files_by_name: ta.Mapping[str, tarfile.TarInfo]
|
252
|
-
links_by_name: ta.Mapping[str, tarfile.TarInfo]
|
253
|
-
|
254
|
-
@cached_nullary
|
255
|
-
def _categorized_entries(self) -> _CategorizedEntries:
|
256
|
-
files_by_name: ta.Dict[str, tarfile.TarInfo] = {}
|
257
|
-
non_files_by_name: ta.Dict[str, tarfile.TarInfo] = {}
|
258
|
-
links_by_name: ta.Dict[str, tarfile.TarInfo] = {}
|
259
|
-
|
260
|
-
for name, info in self._entries_by_name().items():
|
261
|
-
if info.type in tarfile.REGULAR_TYPES:
|
262
|
-
files_by_name[name] = info
|
263
|
-
elif info.type in (tarfile.LNKTYPE, tarfile.GNUTYPE_LONGLINK):
|
264
|
-
links_by_name[name] = info
|
265
|
-
else:
|
266
|
-
non_files_by_name[name] = info
|
267
|
-
|
268
|
-
return self._CategorizedEntries(
|
269
|
-
files_by_name=files_by_name,
|
270
|
-
non_files_by_name=non_files_by_name,
|
271
|
-
links_by_name=links_by_name,
|
272
|
-
)
|
273
|
-
|
274
|
-
#
|
275
|
-
|
276
|
-
@cached_nullary
|
277
|
-
def _non_files_sorted_by_name(self) -> ta.Sequence[tarfile.TarInfo]:
|
278
|
-
return sorted(
|
279
|
-
self._categorized_entries().non_files_by_name.values(),
|
280
|
-
key=lambda info: info.name,
|
281
|
-
)
|
282
|
-
|
283
|
-
@cached_nullary
|
284
|
-
def _files_descending_by_size(self) -> ta.Sequence[tarfile.TarInfo]:
|
285
|
-
return sorted(
|
286
|
-
self._categorized_entries().files_by_name.values(),
|
287
|
-
key=lambda info: -check.isinstance(info.size, int),
|
288
|
-
)
|
289
|
-
|
290
|
-
#
|
291
|
-
|
292
|
-
@cached_nullary
|
293
|
-
def _output_files(self) -> ta.Sequence[ta.BinaryIO]:
|
294
|
-
return [
|
295
|
-
self._enter_context(open(output_file_path, 'wb'))
|
296
|
-
for output_file_path in self._output_file_paths
|
297
|
-
]
|
298
|
-
|
299
|
-
@cached_nullary
|
300
|
-
def _output_tar_writers(self) -> ta.Sequence[OciDataTarWriter]:
|
301
|
-
return [
|
302
|
-
self._enter_context(
|
303
|
-
OciDataTarWriter(
|
304
|
-
output_file,
|
305
|
-
compression=self._compression,
|
306
|
-
),
|
307
|
-
)
|
308
|
-
for output_file in self._output_files()
|
309
|
-
]
|
310
|
-
|
311
|
-
#
|
312
|
-
|
313
|
-
def _write_entry(
|
314
|
-
self,
|
315
|
-
info: tarfile.TarInfo,
|
316
|
-
output_file_idx: int,
|
317
|
-
) -> None:
|
318
|
-
check.not_in(info.name, self._output_file_indexes_by_name)
|
319
|
-
|
320
|
-
writer = self._output_tar_writers()[output_file_idx]
|
321
|
-
|
322
|
-
if info.type in tarfile.REGULAR_TYPES:
|
323
|
-
with check.not_none(self._input_tar_file().extractfile(info)) as f:
|
324
|
-
writer.add_file(info, f) # type: ignore
|
325
|
-
|
326
|
-
else:
|
327
|
-
writer.add_file(info)
|
328
|
-
|
329
|
-
self._output_file_indexes_by_name[info.name] = output_file_idx
|
330
|
-
|
331
|
-
@cached_nullary
|
332
|
-
def _write_non_files(self) -> None:
|
333
|
-
for non_file in self._non_files_sorted_by_name():
|
334
|
-
self._write_entry(non_file, 0)
|
335
|
-
|
336
|
-
@cached_nullary
|
337
|
-
def _write_files(self) -> None:
|
338
|
-
writers = self._output_tar_writers()
|
339
|
-
|
340
|
-
bins = [
|
341
|
-
(writer.info().compressed_sz, i)
|
342
|
-
for i, writer in enumerate(writers)
|
343
|
-
]
|
344
|
-
|
345
|
-
heapq.heapify(bins)
|
346
|
-
|
347
|
-
for file in self._files_descending_by_size():
|
348
|
-
_, bin_index = heapq.heappop(bins)
|
349
|
-
|
350
|
-
writer = writers[bin_index]
|
351
|
-
|
352
|
-
self._write_entry(file, bin_index)
|
353
|
-
|
354
|
-
bin_size = writer.info().compressed_sz
|
355
|
-
|
356
|
-
heapq.heappush(bins, (bin_size, bin_index))
|
357
|
-
|
358
|
-
@cached_nullary
|
359
|
-
def _write_links(self) -> None:
|
360
|
-
for link in self._categorized_entries().links_by_name.values():
|
361
|
-
link_name = check.non_empty_str(link.linkname)
|
362
|
-
|
363
|
-
output_file_idx = self._output_file_indexes_by_name[link_name]
|
364
|
-
|
365
|
-
self._write_entry(link, output_file_idx)
|
366
|
-
|
367
|
-
@cached_nullary
|
368
|
-
def write(self) -> ta.Mapping[str, WrittenOciDataTarFileInfo]:
|
369
|
-
writers = self._output_tar_writers()
|
370
|
-
|
371
|
-
self._write_non_files()
|
372
|
-
self._write_files()
|
373
|
-
self._write_links()
|
374
|
-
|
375
|
-
for output_tar_writer in writers:
|
376
|
-
output_tar_writer.tar_file().close()
|
377
|
-
|
378
|
-
return {
|
379
|
-
output_file_path: output_tar_writer.info()
|
380
|
-
for output_file_path, output_tar_writer in zip(self._output_file_paths, writers)
|
381
|
-
}
|
omdev/oci/repositories.py
CHANGED
@@ -72,6 +72,9 @@ class DirectoryOciRepository(FileOciRepository):
|
|
72
72
|
|
73
73
|
self._data_dir = check.non_empty_str(data_dir)
|
74
74
|
|
75
|
+
def __repr__(self) -> str:
|
76
|
+
return f'{self.__class__.__name__}({self._data_dir!r})'
|
77
|
+
|
75
78
|
def read_file(self, path: str) -> bytes:
|
76
79
|
full_path = os.path.join(self._data_dir, path)
|
77
80
|
check.arg(is_path_in_dir(self._data_dir, full_path))
|
@@ -109,6 +112,9 @@ class TarFileOciRepository(FileOciRepository):
|
|
109
112
|
|
110
113
|
self._tar_file = tar_file
|
111
114
|
|
115
|
+
def __repr__(self) -> str:
|
116
|
+
return f'{self.__class__.__name__}({self._tar_file!r})'
|
117
|
+
|
112
118
|
def read_file(self, path: str) -> bytes:
|
113
119
|
if (ti := self._tar_file.getmember(path)) is None:
|
114
120
|
raise FileNotFoundError(path)
|