omdev 0.0.0.dev222__py3-none-any.whl → 0.0.0.dev224__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. omdev/ci/cache.py +148 -23
  2. omdev/ci/ci.py +50 -110
  3. omdev/ci/cli.py +24 -23
  4. omdev/ci/docker/__init__.py +0 -0
  5. omdev/ci/docker/buildcaching.py +69 -0
  6. omdev/ci/docker/cache.py +57 -0
  7. omdev/ci/docker/cacheserved.py +262 -0
  8. omdev/ci/{docker.py → docker/cmds.py} +1 -44
  9. omdev/ci/docker/dataserver.py +204 -0
  10. omdev/ci/docker/imagepulling.py +65 -0
  11. omdev/ci/docker/inject.py +37 -0
  12. omdev/ci/docker/packing.py +72 -0
  13. omdev/ci/docker/repositories.py +40 -0
  14. omdev/ci/docker/utils.py +48 -0
  15. omdev/ci/github/cache.py +35 -6
  16. omdev/ci/github/client.py +9 -2
  17. omdev/ci/github/inject.py +30 -0
  18. omdev/ci/inject.py +61 -0
  19. omdev/ci/utils.py +0 -49
  20. omdev/dataserver/__init__.py +1 -0
  21. omdev/dataserver/handlers.py +198 -0
  22. omdev/dataserver/http.py +69 -0
  23. omdev/dataserver/routes.py +49 -0
  24. omdev/dataserver/server.py +90 -0
  25. omdev/dataserver/targets.py +121 -0
  26. omdev/oci/building.py +107 -9
  27. omdev/oci/compression.py +8 -0
  28. omdev/oci/data.py +43 -0
  29. omdev/oci/datarefs.py +90 -50
  30. omdev/oci/dataserver.py +64 -0
  31. omdev/oci/loading.py +20 -0
  32. omdev/oci/media.py +20 -0
  33. omdev/oci/pack/__init__.py +0 -0
  34. omdev/oci/pack/packing.py +185 -0
  35. omdev/oci/pack/repositories.py +162 -0
  36. omdev/oci/pack/unpacking.py +204 -0
  37. omdev/oci/repositories.py +84 -2
  38. omdev/oci/tars.py +144 -0
  39. omdev/pyproject/resources/python.sh +1 -1
  40. omdev/scripts/ci.py +2137 -512
  41. omdev/scripts/interp.py +119 -22
  42. omdev/scripts/pyproject.py +141 -28
  43. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/METADATA +2 -2
  44. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/RECORD +48 -23
  45. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/LICENSE +0 -0
  46. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/WHEEL +0 -0
  47. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/entry_points.txt +0 -0
  48. {omdev-0.0.0.dev222.dist-info → omdev-0.0.0.dev224.dist-info}/top_level.txt +0 -0
omdev/oci/repositories.py CHANGED
@@ -2,6 +2,7 @@
2
2
  # @omlish-lite
3
3
  import abc
4
4
  import os.path
5
+ import tarfile
5
6
  import typing as ta
6
7
 
7
8
  from omlish.lite.check import check
@@ -10,6 +11,7 @@ from omlish.os.paths import is_path_in_dir
10
11
  from .datarefs import BytesOciDataRef
11
12
  from .datarefs import FileOciDataRef
12
13
  from .datarefs import OciDataRef
14
+ from .datarefs import TarFileOciDataRef
13
15
 
14
16
 
15
17
  ##
@@ -28,16 +30,51 @@ class OciRepository(abc.ABC):
28
30
  def ref_blob(self, digest: str) -> OciDataRef:
29
31
  raise NotImplementedError
30
32
 
33
+ @classmethod
34
+ def of(
35
+ cls,
36
+ obj: ta.Union[
37
+ 'OciRepository',
38
+ str,
39
+ tarfile.TarFile,
40
+ ta.Mapping[str, bytes],
41
+ ],
42
+ ) -> 'OciRepository':
43
+ if isinstance(obj, OciRepository):
44
+ return obj
45
+
46
+ elif isinstance(obj, str):
47
+ check.arg(os.path.isdir(obj))
48
+ return DirectoryOciRepository(obj)
49
+
50
+ elif isinstance(obj, tarfile.TarFile):
51
+ return TarFileOciRepository(obj)
52
+
53
+ elif isinstance(obj, ta.Mapping):
54
+ return DictOciRepository(obj)
55
+
56
+ else:
57
+ raise TypeError(obj)
58
+
59
+
60
+ class FileOciRepository(OciRepository, abc.ABC):
61
+ @abc.abstractmethod
62
+ def read_file(self, path: str) -> bytes:
63
+ raise NotImplementedError
64
+
31
65
 
32
66
  #
33
67
 
34
68
 
35
- class DirectoryOciRepository(OciRepository):
69
+ class DirectoryOciRepository(FileOciRepository):
36
70
  def __init__(self, data_dir: str) -> None:
37
71
  super().__init__()
38
72
 
39
73
  self._data_dir = check.non_empty_str(data_dir)
40
74
 
75
+ def __repr__(self) -> str:
76
+ return f'{self.__class__.__name__}({self._data_dir!r})'
77
+
41
78
  def read_file(self, path: str) -> bytes:
42
79
  full_path = os.path.join(self._data_dir, path)
43
80
  check.arg(is_path_in_dir(self._data_dir, full_path))
@@ -67,7 +104,52 @@ class DirectoryOciRepository(OciRepository):
67
104
  #
68
105
 
69
106
 
70
- class DictionaryOciRepository(OciRepository):
107
+ class TarFileOciRepository(FileOciRepository):
108
+ def __init__(self, tar_file: tarfile.TarFile) -> None:
109
+ super().__init__()
110
+
111
+ check.arg('r' in tar_file.mode)
112
+
113
+ self._tar_file = tar_file
114
+
115
+ def __repr__(self) -> str:
116
+ return f'{self.__class__.__name__}({self._tar_file!r})'
117
+
118
+ def read_file(self, path: str) -> bytes:
119
+ if (ti := self._tar_file.getmember(path)) is None:
120
+ raise FileNotFoundError(path)
121
+ with check.not_none(self._tar_file.extractfile(ti)) as f:
122
+ return f.read()
123
+
124
+ def blob_name(self, digest: str) -> str:
125
+ scheme, value = digest.split(':')
126
+ return os.path.join('blobs', scheme, value)
127
+
128
+ def contains_blob(self, digest: str) -> bool:
129
+ try:
130
+ self._tar_file.getmember(self.blob_name(digest))
131
+ except KeyError:
132
+ return False
133
+ else:
134
+ return True
135
+
136
+ def read_blob(self, digest: str) -> bytes:
137
+ if (ti := self._tar_file.getmember(self.blob_name(digest))) is None:
138
+ raise KeyError(digest)
139
+ with check.not_none(self._tar_file.extractfile(ti)) as f:
140
+ return f.read()
141
+
142
+ def ref_blob(self, digest: str) -> OciDataRef:
143
+ return TarFileOciDataRef(
144
+ tar_file=self._tar_file,
145
+ tar_info=self._tar_file.getmember(self.blob_name(digest)),
146
+ )
147
+
148
+
149
+ #
150
+
151
+
152
+ class DictOciRepository(OciRepository):
71
153
  def __init__(self, blobs: ta.Mapping[str, bytes]) -> None:
72
154
  super().__init__()
73
155
 
omdev/oci/tars.py ADDED
@@ -0,0 +1,144 @@
1
+ # ruff: noqa: UP006 UP007
2
+ # @omlish-lite
3
+ import gzip
4
+ import hashlib
5
+ import tarfile
6
+ import typing as ta
7
+
8
+ from omlish.lite.contextmanagers import ExitStacked
9
+
10
+ from .compression import OciCompression
11
+ from .datarefs import OciDataRef
12
+ from .datarefs import OciDataRefInfo
13
+ from .datarefs import open_oci_data_ref
14
+
15
+
16
+ ##
17
+
18
+
19
+ class WrittenOciDataTarFileInfo(ta.NamedTuple):
20
+ compressed_sz: int
21
+ compressed_sha256: str
22
+
23
+ tar_sz: int
24
+ tar_sha256: str
25
+
26
+
27
+ class OciDataTarWriter(ExitStacked):
28
+ def __init__(
29
+ self,
30
+ f: ta.BinaryIO,
31
+ compression: ta.Optional[OciCompression] = None,
32
+ *,
33
+ gzip_level: int = 1,
34
+ zstd_level: int = 10,
35
+ ) -> None:
36
+ super().__init__()
37
+
38
+ self._f = f
39
+ self._compression = compression
40
+
41
+ self._gzip_level = gzip_level
42
+ self._zstd_level = zstd_level
43
+
44
+ class _FileWrapper:
45
+ def __init__(self, f):
46
+ super().__init__()
47
+
48
+ self._f = f
49
+ self._c = 0
50
+ self._h = hashlib.sha256()
51
+
52
+ @property
53
+ def size(self) -> int:
54
+ return self._c
55
+
56
+ def sha256(self) -> str:
57
+ return self._h.hexdigest()
58
+
59
+ def write(self, d):
60
+ self._c += len(d)
61
+ self._h.update(d)
62
+ self._f.write(d)
63
+
64
+ def tell(self) -> int:
65
+ return self._f.tell()
66
+
67
+ _cw: _FileWrapper
68
+ _cf: ta.BinaryIO
69
+
70
+ _tw: _FileWrapper
71
+ _tf: tarfile.TarFile
72
+
73
+ def info(self) -> WrittenOciDataTarFileInfo:
74
+ return WrittenOciDataTarFileInfo(
75
+ compressed_sz=self._cw.size,
76
+ compressed_sha256=self._cw.sha256(),
77
+
78
+ tar_sz=self._tw.size,
79
+ tar_sha256=self._tw.sha256(),
80
+ )
81
+
82
+ def __enter__(self) -> 'OciDataTarWriter':
83
+ super().__enter__()
84
+
85
+ #
86
+
87
+ self._cw = self._FileWrapper(self._f)
88
+
89
+ if self._compression is OciCompression.GZIP:
90
+ self._cf = self._enter_context(
91
+ gzip.GzipFile( # type: ignore
92
+ fileobj=self._cw,
93
+ mode='wb',
94
+ compresslevel=self._gzip_level,
95
+ ),
96
+ )
97
+
98
+ elif self._compression is OciCompression.ZSTD:
99
+ zc = __import__('zstandard').ZstdCompressor(
100
+ level=self._zstd_level,
101
+ )
102
+ self._cf = self._enter_context(zc.stream_writer(self._cw))
103
+
104
+ elif self._compression is None:
105
+ self._cf = self._cw # type: ignore
106
+
107
+ else:
108
+ raise ValueError(self._compression)
109
+
110
+ #
111
+
112
+ self._tw = self._FileWrapper(self._cf)
113
+
114
+ self._tf = self._enter_context(
115
+ tarfile.open( # type: ignore
116
+ fileobj=self._tw,
117
+ mode='w',
118
+ ),
119
+ )
120
+
121
+ #
122
+
123
+ return self
124
+
125
+ def tar_file(self) -> tarfile.TarFile:
126
+ return self._tf
127
+
128
+ def add_file(self, ti: tarfile.TarInfo, f: ta.Optional[ta.BinaryIO] = None) -> None:
129
+ self._tf.addfile(ti, f)
130
+
131
+
132
+ def write_oci_data_tar_file(
133
+ f: ta.BinaryIO,
134
+ data: ta.Mapping[str, OciDataRef],
135
+ ) -> WrittenOciDataTarFileInfo:
136
+ with OciDataTarWriter(f) as tgw:
137
+ for n, dr in data.items():
138
+ ti = tarfile.TarInfo(name=n)
139
+ ri = OciDataRefInfo(dr)
140
+ ti.size = ri.size()
141
+ with open_oci_data_ref(dr) as df:
142
+ tgw.add_file(ti, df)
143
+
144
+ return tgw.info()
@@ -2,7 +2,7 @@
2
2
  set -e
3
3
 
4
4
  if [ -z "${VENV}" ] ; then
5
- if [ $(uname) = "Linux" ] && (cat /proc/mounts | grep -E '^overlay / .*/(docker|desktop-containerd)/' > /dev/null) ; then
5
+ if [ $(uname) = "Linux" ] && grep -E '^overlay / .*/(docker|desktop-containerd)/' /proc/mounts > /dev/null ; then
6
6
  VENV=docker
7
7
  else
8
8
  VENV=default