ScriptCollection 4.3.64__py3-none-any.whl → 4.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/AnionBuildPlatform.py +205 -205
- ScriptCollection/CertificateUpdater.py +157 -157
- ScriptCollection/Executables.py +4 -2
- ScriptCollection/GeneralUtilities.py +1696 -1696
- ScriptCollection/OCIImages/OCIImageManager.py +278 -278
- ScriptCollection/ProcessesRunner.py +43 -43
- ScriptCollection/ProgramRunnerSudo.py +110 -110
- ScriptCollection/Resources/CultureChooser/CultureChooser.js +0 -0
- ScriptCollection/Resources/CultureChooser/index.html +0 -0
- ScriptCollection/Resources/Dependencies/JRE/Version.txt +0 -0
- ScriptCollection/Resources/Dependencies/OpenAPIGenerator/Version.txt +0 -0
- ScriptCollection/Resources/Dependencies/PlantUML/Version.txt +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-Bold.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-BoldOblique.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans-Oblique.ttf +0 -0
- ScriptCollection/Resources/Fonts/DejaVuSans.ttf +0 -0
- ScriptCollection/Resources/Fonts/LICENSE.txt +0 -0
- ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html +0 -0
- ScriptCollection/SCLog.py +115 -115
- ScriptCollection/ScriptCollectionCore.py +4372 -4351
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +716 -709
- ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +235 -235
- ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +258 -258
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +524 -524
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +157 -155
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +691 -681
- ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -98
- ScriptCollection/TFCPS/TFCPS_Generic.py +44 -44
- ScriptCollection/TFCPS/TFCPS_MergeToMain.py +132 -132
- ScriptCollection/TFCPS/TFCPS_MergeToStable.py +358 -358
- ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -48
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +30 -4
- {scriptcollection-4.3.64.dist-info → scriptcollection-4.4.0.dist-info}/METADATA +1 -1
- {scriptcollection-4.3.64.dist-info → scriptcollection-4.4.0.dist-info}/RECORD +26 -26
- {scriptcollection-4.3.64.dist-info → scriptcollection-4.4.0.dist-info}/WHEEL +0 -0
- {scriptcollection-4.3.64.dist-info → scriptcollection-4.4.0.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.3.64.dist-info → scriptcollection-4.4.0.dist-info}/top_level.txt +0 -0
|
@@ -1,278 +1,278 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import yaml
|
|
3
|
-
from packaging.version import Version
|
|
4
|
-
from ..GeneralUtilities import GeneralUtilities
|
|
5
|
-
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
6
|
-
from ..GeneralUtilities import VersionEcholon
|
|
7
|
-
from ..SCLog import LogLevel
|
|
8
|
-
from .AbstractImageHandler import AbstractImageHandler
|
|
9
|
-
from .ConcreteImageHandlers.ImageHandlerDebian import ImageHandlerDebian
|
|
10
|
-
from .ConcreteImageHandlers.ImageHandlerDebianSlim import ImageHandlerDebianSlim
|
|
11
|
-
from .ConcreteImageHandlers.ImageHandlerGeneric import ImageHandlerGeneric
|
|
12
|
-
from .ConcreteImageHandlers.ImageHandlerGenericV import ImageHandlerGenericV
|
|
13
|
-
from .ConcreteImageHandlers.ImageHandlerGitlabCE import ImageHandlerGitlabCE
|
|
14
|
-
from .ConcreteImageHandlers.ImageHandlerGitlabEE import ImageHandlerGitlabEE
|
|
15
|
-
|
|
16
|
-
class OCIImageManager:
|
|
17
|
-
|
|
18
|
-
__sc:ScriptCollectionCore=None
|
|
19
|
-
image_handler:list[AbstractImageHandler] = None
|
|
20
|
-
|
|
21
|
-
def __init__(self,sc:ScriptCollectionCore):
|
|
22
|
-
if sc is None:
|
|
23
|
-
sc=ScriptCollectionCore()
|
|
24
|
-
self.__sc=sc
|
|
25
|
-
self.image_handler=[
|
|
26
|
-
ImageHandlerDebian(),
|
|
27
|
-
ImageHandlerDebianSlim(),
|
|
28
|
-
ImageHandlerGeneric(),
|
|
29
|
-
ImageHandlerGenericV(),
|
|
30
|
-
ImageHandlerGitlabCE(),
|
|
31
|
-
ImageHandlerGitlabEE(),
|
|
32
|
-
]
|
|
33
|
-
|
|
34
|
-
@GeneralUtilities.check_arguments
|
|
35
|
-
def get_image_handler(self,image_name:str)->AbstractImageHandler:
|
|
36
|
-
for image_handler in self.image_handler:
|
|
37
|
-
if image_handler.can_handle(image_name):
|
|
38
|
-
return image_handler
|
|
39
|
-
raise ValueError(f"No image-handler available for image \"{image_name}\".")
|
|
40
|
-
|
|
41
|
-
@GeneralUtilities.check_arguments
|
|
42
|
-
def get_repository_image_definition_file(self,repository:str)->str:
|
|
43
|
-
self.__sc.assert_is_git_repository(repository)
|
|
44
|
-
return os.path.join(repository,".ScriptCollection","OCIImages","ImageDefinition.csv")
|
|
45
|
-
|
|
46
|
-
@GeneralUtilities.check_arguments
|
|
47
|
-
def get_global_docker_image_registries_file(self)->str:
|
|
48
|
-
folder=os.path.join(self.__sc.get_global_cache_folder(),"OCIImages")
|
|
49
|
-
GeneralUtilities.ensure_directory_exists(folder)
|
|
50
|
-
result=os.path.join(folder,"ImageRegistries.csv")
|
|
51
|
-
if not os.path.isfile(result):
|
|
52
|
-
GeneralUtilities.ensure_file_exists(result)
|
|
53
|
-
GeneralUtilities.write_lines_to_file(result,["ImageName;RegistryAddress"])
|
|
54
|
-
return result
|
|
55
|
-
|
|
56
|
-
@GeneralUtilities.check_arguments
|
|
57
|
-
def get_used_images_in_repository(self,repository:str)->list[str]:
|
|
58
|
-
result:list[str]=[]
|
|
59
|
-
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
60
|
-
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
61
|
-
result.append(line[0])
|
|
62
|
-
return result
|
|
63
|
-
|
|
64
|
-
@GeneralUtilities.check_arguments
|
|
65
|
-
def custom_registry_is_defined(self,image_name:str)->bool:
|
|
66
|
-
global_docker_image_registries_file=self.get_global_docker_image_registries_file()
|
|
67
|
-
for line in GeneralUtilities.read_nonempty_lines_from_file(global_docker_image_registries_file)[1:]:
|
|
68
|
-
splitted_line=line.split(";")
|
|
69
|
-
if image_name==splitted_line[0]:
|
|
70
|
-
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(splitted_line[1]),f"No registry defined for image {image_name}.")
|
|
71
|
-
return True
|
|
72
|
-
return False
|
|
73
|
-
|
|
74
|
-
@GeneralUtilities.check_arguments
|
|
75
|
-
def get_tag_for_image(self,repository:str,image_name:str)->str:
|
|
76
|
-
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
77
|
-
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
78
|
-
if image_name==line[0]:
|
|
79
|
-
return line[2]
|
|
80
|
-
raise ValueError(f"No tag defined for image \"{image_name}\".")
|
|
81
|
-
|
|
82
|
-
@GeneralUtilities.check_arguments
|
|
83
|
-
def get_registry_address_for_image(self,repository:str,image_name:str)->str:
|
|
84
|
-
"""Example: if image_name==Debian this function returns something like "myregistry.example.com/debian", always without tag."""
|
|
85
|
-
if self.custom_registry_is_defined(image_name):
|
|
86
|
-
#return image from custom registry-address
|
|
87
|
-
global_docker_image_registries_file=self.get_global_docker_image_registries_file()
|
|
88
|
-
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(global_docker_image_registries_file)[1:]]:
|
|
89
|
-
if image_name==line[0]:
|
|
90
|
-
return line[1]
|
|
91
|
-
else:
|
|
92
|
-
#return fallback-registry-address
|
|
93
|
-
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
94
|
-
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
95
|
-
if image_name==line[0]:
|
|
96
|
-
return line[1]
|
|
97
|
-
raise ValueError(f"No registry defined for image \"{image_name}\".")
|
|
98
|
-
|
|
99
|
-
@GeneralUtilities.check_arguments
|
|
100
|
-
def get_registry_address_for_image_with_default_tag(self,repository:str,image_name:str)->str:
|
|
101
|
-
return f"{self.get_registry_address_for_image(repository,image_name)}:{self.get_tag_for_image(repository,image_name)}"
|
|
102
|
-
|
|
103
|
-
@GeneralUtilities.check_arguments
|
|
104
|
-
def update_default_tag_for_images_in_image_definitions_file(self,repository:str,search_in_custom_registry_only_if_available:bool)->None:
|
|
105
|
-
file=f"{repository}/.ScriptCollection/OCIImages/ImageDefinition.csv"
|
|
106
|
-
GeneralUtilities.assert_file_exists(file)
|
|
107
|
-
lines=GeneralUtilities.read_nonempty_lines_from_file(file)
|
|
108
|
-
new_lines:list[str]=[]
|
|
109
|
-
#file looks like:
|
|
110
|
-
#ImageName;UpstreamRegistryAddress;DefaultTag
|
|
111
|
-
#Debian;docker.io/library/debian;13.4-slim
|
|
112
|
-
for line in lines:
|
|
113
|
-
if line.startswith("ImageName;"): #header line
|
|
114
|
-
new_lines.append(line)
|
|
115
|
-
continue
|
|
116
|
-
splitted_line=line.split(";")
|
|
117
|
-
image_name=splitted_line[0]
|
|
118
|
-
registry_address=splitted_line[1]
|
|
119
|
-
default_tag=splitted_line[2]
|
|
120
|
-
tag=default_tag
|
|
121
|
-
try:
|
|
122
|
-
addresses_to_check=[]
|
|
123
|
-
if self.custom_registry_is_defined(image_name):
|
|
124
|
-
addresses_to_check.append(self.get_registry_address_for_image(repository,image_name))
|
|
125
|
-
else:
|
|
126
|
-
if search_in_custom_registry_only_if_available:
|
|
127
|
-
raise ValueError(f"No custom registry defined for image {image_name}.")
|
|
128
|
-
if not search_in_custom_registry_only_if_available:
|
|
129
|
-
addresses_to_check.append(registry_address)
|
|
130
|
-
newest_versions:set[Version]={default_tag}
|
|
131
|
-
current_version=Version(default_tag)
|
|
132
|
-
for address in addresses_to_check:
|
|
133
|
-
newest_versions_for_address=self.get_latest_of_image(image_name,address,current_version,VersionEcholon.LatestVersion)
|
|
134
|
-
if newest_versions_for_address is not None:
|
|
135
|
-
newest_versions.update(newest_versions_for_address)
|
|
136
|
-
GeneralUtilities.assert_condition(len(newest_versions)>0,f"Could not find any version for image {image_name} in registry {registry_address}.")
|
|
137
|
-
newest_version=max(newest_versions)
|
|
138
|
-
tag=self.version_to_tag(image_name,newest_version)
|
|
139
|
-
except Exception as e:
|
|
140
|
-
self.__sc.log.log(f"Could not get tag for image {image_name} from registry {registry_address}. Reason: {str(e)}",LogLevel.Warning)
|
|
141
|
-
new_lines.append(f"{image_name};{registry_address};{tag}")
|
|
142
|
-
GeneralUtilities.write_lines_to_file(file,new_lines)
|
|
143
|
-
|
|
144
|
-
@GeneralUtilities.check_arguments
|
|
145
|
-
def get_latest_of_image(self,image_name:str,registry_address:str,current_version:Version,echolon: VersionEcholon)->Version:
|
|
146
|
-
result= None
|
|
147
|
-
image_handler=self.get_image_handler(image_name)
|
|
148
|
-
if echolon not in (VersionEcholon.NoUpdate, VersionEcholon.CustomAlgorithm):
|
|
149
|
-
available_tags:list[str]=self.get_available_tags_of_image(image_name,registry_address)
|
|
150
|
-
available_versions:set[Version]={image_handler.tag_to_version(image_name,tag) for tag in available_tags}
|
|
151
|
-
#the current version must always be a candidate so that the result is never a downgrade
|
|
152
|
-
available_versions.add(current_version)
|
|
153
|
-
match echolon:
|
|
154
|
-
case VersionEcholon.LatestPatch:#caution: this never updates the minor or major version
|
|
155
|
-
#take the latest available patch-version for the current minor-version of the image.
|
|
156
|
-
result=self.__get_latest_patch(available_versions,current_version.major,current_version.minor)
|
|
157
|
-
case VersionEcholon.LatestPatchOrLatestMinor:#caution: this never updates the major version
|
|
158
|
-
#take the latest available patch-version for the current minor-version of the image if there is a newer patch version available.
|
|
159
|
-
#otherwise take the latest available patch-version of the next available minor-version within the current major-version.
|
|
160
|
-
result=self.__get_latest_patch_or_next_minor(available_versions,current_version)
|
|
161
|
-
case VersionEcholon.LatestPatchOrLatestMinorOrNextMajor:#usual case for images which have complex migrations sometimes, like GitLab or NextCloud
|
|
162
|
-
#like LatestPatchOrLatestMinor, but if there is neither a newer patch nor a newer minor-version available within the current major-version
|
|
163
|
-
#then take the latest available patch-version of the lowest minor-version of the next available major-version.
|
|
164
|
-
result=self.__get_latest_patch_or_next_minor(available_versions,current_version)
|
|
165
|
-
if result==current_version:
|
|
166
|
-
next_major=self.__get_next_higher_value({v.major for v in available_versions},current_version.major)
|
|
167
|
-
if next_major is not None:
|
|
168
|
-
lowest_minor=min({v.minor for v in available_versions if v.major==next_major})
|
|
169
|
-
result=self.__get_latest_patch(available_versions,next_major,lowest_minor)
|
|
170
|
-
case VersionEcholon.LatestVersion:#usual case for most images
|
|
171
|
-
#take the latest available version of the image.
|
|
172
|
-
result=max(available_versions)
|
|
173
|
-
case VersionEcholon.CustomAlgorithm:
|
|
174
|
-
#the image-handler decides which version to use for the given current version.
|
|
175
|
-
result=image_handler.get_latest_version_custom(image_name,registry_address,current_version)
|
|
176
|
-
case VersionEcholon.NoUpdate:
|
|
177
|
-
result = current_version
|
|
178
|
-
return result
|
|
179
|
-
|
|
180
|
-
@GeneralUtilities.check_arguments
|
|
181
|
-
def __get_latest_patch(self,available_versions:set[Version],major:int,minor:int)->Version:
|
|
182
|
-
"""Returns the latest available patch-version for the given major- and minor-version or None if no such version is available."""
|
|
183
|
-
matching_versions={v for v in available_versions if v.major==major and v.minor==minor}
|
|
184
|
-
if len(matching_versions)==0:
|
|
185
|
-
return None
|
|
186
|
-
return max(matching_versions)
|
|
187
|
-
|
|
188
|
-
@GeneralUtilities.check_arguments
|
|
189
|
-
def __get_latest_patch_or_next_minor(self,available_versions:set[Version],current_version:Version)->Version:
|
|
190
|
-
latest_patch=self.__get_latest_patch(available_versions,current_version.major,current_version.minor)
|
|
191
|
-
if latest_patch is not None and latest_patch>current_version:
|
|
192
|
-
return latest_patch
|
|
193
|
-
next_minor=self.__get_next_higher_value({v.minor for v in available_versions if v.major==current_version.major},current_version.minor)
|
|
194
|
-
if next_minor is not None:
|
|
195
|
-
return self.__get_latest_patch(available_versions,current_version.major,next_minor)
|
|
196
|
-
return current_version
|
|
197
|
-
|
|
198
|
-
@GeneralUtilities.check_arguments
|
|
199
|
-
def __get_next_higher_value(self,values:set[int],current_value:int)->int:
|
|
200
|
-
"""Returns the smallest value in values which is greater than current_value or None if no such value exists."""
|
|
201
|
-
higher_values={v for v in values if v>current_value}
|
|
202
|
-
if len(higher_values)==0:
|
|
203
|
-
return None
|
|
204
|
-
return min(higher_values)
|
|
205
|
-
|
|
206
|
-
@GeneralUtilities.check_arguments
|
|
207
|
-
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
208
|
-
"""registry_address must have one of theese formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian".
|
|
209
|
-
returns something like ["13.2-slim", "13.2", "13.3-slim", "13.3"]."""
|
|
210
|
-
return self.get_image_handler(image_name).get_available_tags_of_image(image_name,registry_address)
|
|
211
|
-
|
|
212
|
-
@GeneralUtilities.check_arguments
|
|
213
|
-
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
214
|
-
"""registry_address must have one of theese formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian"."""
|
|
215
|
-
return self.get_image_handler(image_name).tag_to_version(image_name, tag)
|
|
216
|
-
|
|
217
|
-
@GeneralUtilities.check_arguments
|
|
218
|
-
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
219
|
-
"""registry_address must have one of theese formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian".
|
|
220
|
-
returns something like "13.3-slim".
|
|
221
|
-
If there are multiple tags available for a certain version then the image-handler decides which one will be returned."""
|
|
222
|
-
return self.get_image_handler(image_name).version_to_tag(image_name,version)
|
|
223
|
-
|
|
224
|
-
@GeneralUtilities.check_arguments
|
|
225
|
-
def get_images_used_in_docker_compose_file(self,docker_compose_file:str)->dict[str,tuple[str,str,str]]:#returns dict[service_name,[image_name,image_address,current_tag]]
|
|
226
|
-
GeneralUtilities.assert_file_exists(docker_compose_file)
|
|
227
|
-
result: dict[str, tuple[str, str, str]] = {}
|
|
228
|
-
with open(docker_compose_file, "r", encoding="utf-8") as f:
|
|
229
|
-
compose_data = yaml.safe_load(f)
|
|
230
|
-
services = (compose_data or {}).get("services", {}) or {}
|
|
231
|
-
for service_name, service_definition in services.items():
|
|
232
|
-
if not isinstance(service_definition, dict):
|
|
233
|
-
continue
|
|
234
|
-
image_value = service_definition.get("image")
|
|
235
|
-
if not isinstance(image_value, str) or not image_value:
|
|
236
|
-
continue
|
|
237
|
-
last_colon = image_value.rfind(":")
|
|
238
|
-
last_slash = image_value.rfind("/")
|
|
239
|
-
if last_colon == -1 or last_colon < last_slash:
|
|
240
|
-
image_address = image_value
|
|
241
|
-
current_tag = "latest"
|
|
242
|
-
else:
|
|
243
|
-
image_address = image_value[:last_colon]
|
|
244
|
-
current_tag = image_value[last_colon + 1:]
|
|
245
|
-
image_name = image_address.rsplit("/", 1)[-1]
|
|
246
|
-
result[service_name] = (image_name, image_address, current_tag)
|
|
247
|
-
return result
|
|
248
|
-
|
|
249
|
-
@GeneralUtilities.check_arguments
|
|
250
|
-
def update_image_in_docker_compose_file(self,docker_compose_file:str,default_echolon:VersionEcholon=None,per_image_echolons:dict[str,VersionEcholon]=None,verbose:bool=False)->None:
|
|
251
|
-
if per_image_echolons is None:
|
|
252
|
-
per_image_echolons = {}
|
|
253
|
-
if verbose:
|
|
254
|
-
self.__sc.log.log(f"Checking images in docker-compose-file \"{docker_compose_file}\" for updates.",LogLevel.Information)
|
|
255
|
-
for service,service_information in self.get_images_used_in_docker_compose_file(docker_compose_file).items():
|
|
256
|
-
image_name=service_information[0]
|
|
257
|
-
image_address=service_information[1]
|
|
258
|
-
current_tag=service_information[2]
|
|
259
|
-
image_handler=self.get_image_handler(image_name)
|
|
260
|
-
effective_echolon=per_image_echolons.get(image_name,default_echolon)
|
|
261
|
-
if effective_echolon is None:
|
|
262
|
-
effective_echolon=image_handler.get_default_echolon_for_update()
|
|
263
|
-
if verbose:
|
|
264
|
-
self.__sc.log.log(f"Service \"{service}\": checking image \"{image_address}:{current_tag}\" using echolon \"{effective_echolon.name}\".",LogLevel.Information)
|
|
265
|
-
new_versions_for_address=self.get_latest_of_image(image_name,image_address,self.tag_to_version(image_name,current_tag),effective_echolon)
|
|
266
|
-
new_tag=self.version_to_tag(image_name,new_versions_for_address)
|
|
267
|
-
if new_tag is not None and new_tag != current_tag:
|
|
268
|
-
new_image_reference = f"{image_address}:{new_tag}"
|
|
269
|
-
if verbose:
|
|
270
|
-
self.__sc.log.log(f"Service \"{service}\": updating image \"{image_address}\" from tag \"{current_tag}\" to tag \"{new_tag}\".",LogLevel.Information)
|
|
271
|
-
with open(docker_compose_file, "r", encoding="utf-8") as f:
|
|
272
|
-
compose_data = yaml.safe_load(f)
|
|
273
|
-
compose_data["services"][service]["image"] = new_image_reference
|
|
274
|
-
with open(docker_compose_file, "w", encoding="utf-8") as f:
|
|
275
|
-
yaml.safe_dump(compose_data, f, default_flow_style=False, sort_keys=False)
|
|
276
|
-
else:
|
|
277
|
-
if verbose:
|
|
278
|
-
self.__sc.log.log(f"Service \"{service}\": image \"{image_address}\" is already up-to-date at tag \"{current_tag}\".",LogLevel.Information)
|
|
1
|
+
import os
|
|
2
|
+
import yaml
|
|
3
|
+
from packaging.version import Version
|
|
4
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
5
|
+
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
6
|
+
from ..GeneralUtilities import VersionEcholon
|
|
7
|
+
from ..SCLog import LogLevel
|
|
8
|
+
from .AbstractImageHandler import AbstractImageHandler
|
|
9
|
+
from .ConcreteImageHandlers.ImageHandlerDebian import ImageHandlerDebian
|
|
10
|
+
from .ConcreteImageHandlers.ImageHandlerDebianSlim import ImageHandlerDebianSlim
|
|
11
|
+
from .ConcreteImageHandlers.ImageHandlerGeneric import ImageHandlerGeneric
|
|
12
|
+
from .ConcreteImageHandlers.ImageHandlerGenericV import ImageHandlerGenericV
|
|
13
|
+
from .ConcreteImageHandlers.ImageHandlerGitlabCE import ImageHandlerGitlabCE
|
|
14
|
+
from .ConcreteImageHandlers.ImageHandlerGitlabEE import ImageHandlerGitlabEE
|
|
15
|
+
|
|
16
|
+
class OCIImageManager:
|
|
17
|
+
|
|
18
|
+
__sc:ScriptCollectionCore=None
|
|
19
|
+
image_handler:list[AbstractImageHandler] = None
|
|
20
|
+
|
|
21
|
+
def __init__(self,sc:ScriptCollectionCore):
|
|
22
|
+
if sc is None:
|
|
23
|
+
sc=ScriptCollectionCore()
|
|
24
|
+
self.__sc=sc
|
|
25
|
+
self.image_handler=[
|
|
26
|
+
ImageHandlerDebian(),
|
|
27
|
+
ImageHandlerDebianSlim(),
|
|
28
|
+
ImageHandlerGeneric(),
|
|
29
|
+
ImageHandlerGenericV(),
|
|
30
|
+
ImageHandlerGitlabCE(),
|
|
31
|
+
ImageHandlerGitlabEE(),
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
@GeneralUtilities.check_arguments
|
|
35
|
+
def get_image_handler(self,image_name:str)->AbstractImageHandler:
|
|
36
|
+
for image_handler in self.image_handler:
|
|
37
|
+
if image_handler.can_handle(image_name):
|
|
38
|
+
return image_handler
|
|
39
|
+
raise ValueError(f"No image-handler available for image \"{image_name}\".")
|
|
40
|
+
|
|
41
|
+
@GeneralUtilities.check_arguments
|
|
42
|
+
def get_repository_image_definition_file(self,repository:str)->str:
|
|
43
|
+
self.__sc.assert_is_git_repository(repository)
|
|
44
|
+
return os.path.join(repository,".ScriptCollection","OCIImages","ImageDefinition.csv")
|
|
45
|
+
|
|
46
|
+
@GeneralUtilities.check_arguments
|
|
47
|
+
def get_global_docker_image_registries_file(self)->str:
|
|
48
|
+
folder=os.path.join(self.__sc.get_global_cache_folder(),"OCIImages")
|
|
49
|
+
GeneralUtilities.ensure_directory_exists(folder)
|
|
50
|
+
result=os.path.join(folder,"ImageRegistries.csv")
|
|
51
|
+
if not os.path.isfile(result):
|
|
52
|
+
GeneralUtilities.ensure_file_exists(result)
|
|
53
|
+
GeneralUtilities.write_lines_to_file(result,["ImageName;RegistryAddress"])
|
|
54
|
+
return result
|
|
55
|
+
|
|
56
|
+
@GeneralUtilities.check_arguments
|
|
57
|
+
def get_used_images_in_repository(self,repository:str)->list[str]:
|
|
58
|
+
result:list[str]=[]
|
|
59
|
+
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
60
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
61
|
+
result.append(line[0])
|
|
62
|
+
return result
|
|
63
|
+
|
|
64
|
+
@GeneralUtilities.check_arguments
|
|
65
|
+
def custom_registry_is_defined(self,image_name:str)->bool:
|
|
66
|
+
global_docker_image_registries_file=self.get_global_docker_image_registries_file()
|
|
67
|
+
for line in GeneralUtilities.read_nonempty_lines_from_file(global_docker_image_registries_file)[1:]:
|
|
68
|
+
splitted_line=line.split(";")
|
|
69
|
+
if image_name==splitted_line[0]:
|
|
70
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(splitted_line[1]),f"No registry defined for image {image_name}.")
|
|
71
|
+
return True
|
|
72
|
+
return False
|
|
73
|
+
|
|
74
|
+
@GeneralUtilities.check_arguments
|
|
75
|
+
def get_tag_for_image(self,repository:str,image_name:str)->str:
|
|
76
|
+
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
77
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
78
|
+
if image_name==line[0]:
|
|
79
|
+
return line[2]
|
|
80
|
+
raise ValueError(f"No tag defined for image \"{image_name}\".")
|
|
81
|
+
|
|
82
|
+
@GeneralUtilities.check_arguments
|
|
83
|
+
def get_registry_address_for_image(self,repository:str,image_name:str)->str:
|
|
84
|
+
"""Example: if image_name==Debian this function returns something like "myregistry.example.com/debian", always without tag."""
|
|
85
|
+
if self.custom_registry_is_defined(image_name):
|
|
86
|
+
#return image from custom registry-address
|
|
87
|
+
global_docker_image_registries_file=self.get_global_docker_image_registries_file()
|
|
88
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(global_docker_image_registries_file)[1:]]:
|
|
89
|
+
if image_name==line[0]:
|
|
90
|
+
return line[1]
|
|
91
|
+
else:
|
|
92
|
+
#return fallback-registry-address
|
|
93
|
+
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
94
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
95
|
+
if image_name==line[0]:
|
|
96
|
+
return line[1]
|
|
97
|
+
raise ValueError(f"No registry defined for image \"{image_name}\".")
|
|
98
|
+
|
|
99
|
+
@GeneralUtilities.check_arguments
|
|
100
|
+
def get_registry_address_for_image_with_default_tag(self,repository:str,image_name:str)->str:
|
|
101
|
+
return f"{self.get_registry_address_for_image(repository,image_name)}:{self.get_tag_for_image(repository,image_name)}"
|
|
102
|
+
|
|
103
|
+
@GeneralUtilities.check_arguments
|
|
104
|
+
def update_default_tag_for_images_in_image_definitions_file(self,repository:str,search_in_custom_registry_only_if_available:bool)->None:
|
|
105
|
+
file=f"{repository}/.ScriptCollection/OCIImages/ImageDefinition.csv"
|
|
106
|
+
GeneralUtilities.assert_file_exists(file)
|
|
107
|
+
lines=GeneralUtilities.read_nonempty_lines_from_file(file)
|
|
108
|
+
new_lines:list[str]=[]
|
|
109
|
+
#file looks like:
|
|
110
|
+
#ImageName;UpstreamRegistryAddress;DefaultTag
|
|
111
|
+
#Debian;docker.io/library/debian;13.4-slim
|
|
112
|
+
for line in lines:
|
|
113
|
+
if line.startswith("ImageName;"): #header line
|
|
114
|
+
new_lines.append(line)
|
|
115
|
+
continue
|
|
116
|
+
splitted_line=line.split(";")
|
|
117
|
+
image_name=splitted_line[0]
|
|
118
|
+
registry_address=splitted_line[1]
|
|
119
|
+
default_tag=splitted_line[2]
|
|
120
|
+
tag=default_tag
|
|
121
|
+
try:
|
|
122
|
+
addresses_to_check=[]
|
|
123
|
+
if self.custom_registry_is_defined(image_name):
|
|
124
|
+
addresses_to_check.append(self.get_registry_address_for_image(repository,image_name))
|
|
125
|
+
else:
|
|
126
|
+
if search_in_custom_registry_only_if_available:
|
|
127
|
+
raise ValueError(f"No custom registry defined for image {image_name}.")
|
|
128
|
+
if not search_in_custom_registry_only_if_available:
|
|
129
|
+
addresses_to_check.append(registry_address)
|
|
130
|
+
newest_versions:set[Version]={default_tag}
|
|
131
|
+
current_version=Version(default_tag)
|
|
132
|
+
for address in addresses_to_check:
|
|
133
|
+
newest_versions_for_address=self.get_latest_of_image(image_name,address,current_version,VersionEcholon.LatestVersion)
|
|
134
|
+
if newest_versions_for_address is not None:
|
|
135
|
+
newest_versions.update(newest_versions_for_address)
|
|
136
|
+
GeneralUtilities.assert_condition(len(newest_versions)>0,f"Could not find any version for image {image_name} in registry {registry_address}.")
|
|
137
|
+
newest_version=max(newest_versions)
|
|
138
|
+
tag=self.version_to_tag(image_name,newest_version)
|
|
139
|
+
except Exception as e:
|
|
140
|
+
self.__sc.log.log(f"Could not get tag for image {image_name} from registry {registry_address}. Reason: {str(e)}",LogLevel.Warning)
|
|
141
|
+
new_lines.append(f"{image_name};{registry_address};{tag}")
|
|
142
|
+
GeneralUtilities.write_lines_to_file(file,new_lines)
|
|
143
|
+
|
|
144
|
+
@GeneralUtilities.check_arguments
|
|
145
|
+
def get_latest_of_image(self,image_name:str,registry_address:str,current_version:Version,echolon: VersionEcholon)->Version:
|
|
146
|
+
result= None
|
|
147
|
+
image_handler=self.get_image_handler(image_name)
|
|
148
|
+
if echolon not in (VersionEcholon.NoUpdate, VersionEcholon.CustomAlgorithm):
|
|
149
|
+
available_tags:list[str]=self.get_available_tags_of_image(image_name,registry_address)
|
|
150
|
+
available_versions:set[Version]={image_handler.tag_to_version(image_name,tag) for tag in available_tags}
|
|
151
|
+
#the current version must always be a candidate so that the result is never a downgrade
|
|
152
|
+
available_versions.add(current_version)
|
|
153
|
+
match echolon:
|
|
154
|
+
case VersionEcholon.LatestPatch:#caution: this never updates the minor or major version
|
|
155
|
+
#take the latest available patch-version for the current minor-version of the image.
|
|
156
|
+
result=self.__get_latest_patch(available_versions,current_version.major,current_version.minor)
|
|
157
|
+
case VersionEcholon.LatestPatchOrLatestMinor:#caution: this never updates the major version
|
|
158
|
+
#take the latest available patch-version for the current minor-version of the image if there is a newer patch version available.
|
|
159
|
+
#otherwise take the latest available patch-version of the next available minor-version within the current major-version.
|
|
160
|
+
result=self.__get_latest_patch_or_next_minor(available_versions,current_version)
|
|
161
|
+
case VersionEcholon.LatestPatchOrLatestMinorOrNextMajor:#usual case for images which have complex migrations sometimes, like GitLab or NextCloud
|
|
162
|
+
#like LatestPatchOrLatestMinor, but if there is neither a newer patch nor a newer minor-version available within the current major-version
|
|
163
|
+
#then take the latest available patch-version of the lowest minor-version of the next available major-version.
|
|
164
|
+
result=self.__get_latest_patch_or_next_minor(available_versions,current_version)
|
|
165
|
+
if result==current_version:
|
|
166
|
+
next_major=self.__get_next_higher_value({v.major for v in available_versions},current_version.major)
|
|
167
|
+
if next_major is not None:
|
|
168
|
+
lowest_minor=min({v.minor for v in available_versions if v.major==next_major})
|
|
169
|
+
result=self.__get_latest_patch(available_versions,next_major,lowest_minor)
|
|
170
|
+
case VersionEcholon.LatestVersion:#usual case for most images
|
|
171
|
+
#take the latest available version of the image.
|
|
172
|
+
result=max(available_versions)
|
|
173
|
+
case VersionEcholon.CustomAlgorithm:
|
|
174
|
+
#the image-handler decides which version to use for the given current version.
|
|
175
|
+
result=image_handler.get_latest_version_custom(image_name,registry_address,current_version)
|
|
176
|
+
case VersionEcholon.NoUpdate:
|
|
177
|
+
result = current_version
|
|
178
|
+
return result
|
|
179
|
+
|
|
180
|
+
@GeneralUtilities.check_arguments
|
|
181
|
+
def __get_latest_patch(self,available_versions:set[Version],major:int,minor:int)->Version:
|
|
182
|
+
"""Returns the latest available patch-version for the given major- and minor-version or None if no such version is available."""
|
|
183
|
+
matching_versions={v for v in available_versions if v.major==major and v.minor==minor}
|
|
184
|
+
if len(matching_versions)==0:
|
|
185
|
+
return None
|
|
186
|
+
return max(matching_versions)
|
|
187
|
+
|
|
188
|
+
@GeneralUtilities.check_arguments
|
|
189
|
+
def __get_latest_patch_or_next_minor(self,available_versions:set[Version],current_version:Version)->Version:
|
|
190
|
+
latest_patch=self.__get_latest_patch(available_versions,current_version.major,current_version.minor)
|
|
191
|
+
if latest_patch is not None and latest_patch>current_version:
|
|
192
|
+
return latest_patch
|
|
193
|
+
next_minor=self.__get_next_higher_value({v.minor for v in available_versions if v.major==current_version.major},current_version.minor)
|
|
194
|
+
if next_minor is not None:
|
|
195
|
+
return self.__get_latest_patch(available_versions,current_version.major,next_minor)
|
|
196
|
+
return current_version
|
|
197
|
+
|
|
198
|
+
@GeneralUtilities.check_arguments
|
|
199
|
+
def __get_next_higher_value(self,values:set[int],current_value:int)->int:
|
|
200
|
+
"""Returns the smallest value in values which is greater than current_value or None if no such value exists."""
|
|
201
|
+
higher_values={v for v in values if v>current_value}
|
|
202
|
+
if len(higher_values)==0:
|
|
203
|
+
return None
|
|
204
|
+
return min(higher_values)
|
|
205
|
+
|
|
206
|
+
@GeneralUtilities.check_arguments
|
|
207
|
+
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
208
|
+
"""registry_address must have one of theese formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian".
|
|
209
|
+
returns something like ["13.2-slim", "13.2", "13.3-slim", "13.3"]."""
|
|
210
|
+
return self.get_image_handler(image_name).get_available_tags_of_image(image_name,registry_address)
|
|
211
|
+
|
|
212
|
+
@GeneralUtilities.check_arguments
|
|
213
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
214
|
+
"""registry_address must have one of theese formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian"."""
|
|
215
|
+
return self.get_image_handler(image_name).tag_to_version(image_name, tag)
|
|
216
|
+
|
|
217
|
+
@GeneralUtilities.check_arguments
|
|
218
|
+
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
219
|
+
"""registry_address must have one of theese formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian".
|
|
220
|
+
returns something like "13.3-slim".
|
|
221
|
+
If there are multiple tags available for a certain version then the image-handler decides which one will be returned."""
|
|
222
|
+
return self.get_image_handler(image_name).version_to_tag(image_name,version)
|
|
223
|
+
|
|
224
|
+
@GeneralUtilities.check_arguments
|
|
225
|
+
def get_images_used_in_docker_compose_file(self,docker_compose_file:str)->dict[str,tuple[str,str,str]]:#returns dict[service_name,[image_name,image_address,current_tag]]
|
|
226
|
+
GeneralUtilities.assert_file_exists(docker_compose_file)
|
|
227
|
+
result: dict[str, tuple[str, str, str]] = {}
|
|
228
|
+
with open(docker_compose_file, "r", encoding="utf-8") as f:
|
|
229
|
+
compose_data = yaml.safe_load(f)
|
|
230
|
+
services = (compose_data or {}).get("services", {}) or {}
|
|
231
|
+
for service_name, service_definition in services.items():
|
|
232
|
+
if not isinstance(service_definition, dict):
|
|
233
|
+
continue
|
|
234
|
+
image_value = service_definition.get("image")
|
|
235
|
+
if not isinstance(image_value, str) or not image_value:
|
|
236
|
+
continue
|
|
237
|
+
last_colon = image_value.rfind(":")
|
|
238
|
+
last_slash = image_value.rfind("/")
|
|
239
|
+
if last_colon == -1 or last_colon < last_slash:
|
|
240
|
+
image_address = image_value
|
|
241
|
+
current_tag = "latest"
|
|
242
|
+
else:
|
|
243
|
+
image_address = image_value[:last_colon]
|
|
244
|
+
current_tag = image_value[last_colon + 1:]
|
|
245
|
+
image_name = image_address.rsplit("/", 1)[-1]
|
|
246
|
+
result[service_name] = (image_name, image_address, current_tag)
|
|
247
|
+
return result
|
|
248
|
+
|
|
249
|
+
@GeneralUtilities.check_arguments
|
|
250
|
+
def update_image_in_docker_compose_file(self,docker_compose_file:str,default_echolon:VersionEcholon=None,per_image_echolons:dict[str,VersionEcholon]=None,verbose:bool=False)->None:
|
|
251
|
+
if per_image_echolons is None:
|
|
252
|
+
per_image_echolons = {}
|
|
253
|
+
if verbose:
|
|
254
|
+
self.__sc.log.log(f"Checking images in docker-compose-file \"{docker_compose_file}\" for updates.",LogLevel.Information)
|
|
255
|
+
for service,service_information in self.get_images_used_in_docker_compose_file(docker_compose_file).items():
|
|
256
|
+
image_name=service_information[0]
|
|
257
|
+
image_address=service_information[1]
|
|
258
|
+
current_tag=service_information[2]
|
|
259
|
+
image_handler=self.get_image_handler(image_name)
|
|
260
|
+
effective_echolon=per_image_echolons.get(image_name,default_echolon)
|
|
261
|
+
if effective_echolon is None:
|
|
262
|
+
effective_echolon=image_handler.get_default_echolon_for_update()
|
|
263
|
+
if verbose:
|
|
264
|
+
self.__sc.log.log(f"Service \"{service}\": checking image \"{image_address}:{current_tag}\" using echolon \"{effective_echolon.name}\".",LogLevel.Information)
|
|
265
|
+
new_versions_for_address=self.get_latest_of_image(image_name,image_address,self.tag_to_version(image_name,current_tag),effective_echolon)
|
|
266
|
+
new_tag=self.version_to_tag(image_name,new_versions_for_address)
|
|
267
|
+
if new_tag is not None and new_tag != current_tag:
|
|
268
|
+
new_image_reference = f"{image_address}:{new_tag}"
|
|
269
|
+
if verbose:
|
|
270
|
+
self.__sc.log.log(f"Service \"{service}\": updating image \"{image_address}\" from tag \"{current_tag}\" to tag \"{new_tag}\".",LogLevel.Information)
|
|
271
|
+
with open(docker_compose_file, "r", encoding="utf-8") as f:
|
|
272
|
+
compose_data = yaml.safe_load(f)
|
|
273
|
+
compose_data["services"][service]["image"] = new_image_reference
|
|
274
|
+
with open(docker_compose_file, "w", encoding="utf-8") as f:
|
|
275
|
+
yaml.safe_dump(compose_data, f, default_flow_style=False, sort_keys=False)
|
|
276
|
+
else:
|
|
277
|
+
if verbose:
|
|
278
|
+
self.__sc.log.log(f"Service \"{service}\": image \"{image_address}\" is already up-to-date at tag \"{current_tag}\".",LogLevel.Information)
|