ScriptCollection 4.4.71__py3-none-any.whl → 4.4.72__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +3 -3
- ScriptCollection/GeneralUtilities.py +42 -0
- ScriptCollection/OCIImages/AbstractImageHandler.py +14 -26
- ScriptCollection/OCIImages/AbstractTagPatternImageHandler.py +46 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +5 -14
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +5 -14
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +6 -15
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +6 -15
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +7 -8
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +7 -8
- ScriptCollection/OCIImages/OCIImageManager.py +73 -68
- ScriptCollection/OCIImages/OCIRegistryClient.py +117 -0
- ScriptCollection/ScriptCollectionCore.py +9 -4
- ScriptCollection/TFCPS/Android/TFCPS_CodeUnitSpecific_Android.py +375 -0
- ScriptCollection/TFCPS/Android/__init__.py +0 -0
- ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py +5 -4
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +38 -4
- ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +1 -0
- ScriptCollection/TFCPS/Rust/CargoLockFile.py +151 -0
- ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +5 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +1 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +14 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +19 -0
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +15 -2
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.72.dist-info}/METADATA +177 -177
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.72.dist-info}/RECORD +29 -24
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.72.dist-info}/WHEEL +0 -0
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.72.dist-info}/entry_points.txt +0 -0
- {scriptcollection-4.4.71.dist-info → scriptcollection-4.4.72.dist-info}/top_level.txt +0 -0
ScriptCollection/Executables.py
CHANGED
|
@@ -1258,15 +1258,15 @@ def ReclaimSpaceFromDocker()->int:
|
|
|
1258
1258
|
sc = ScriptCollectionCore()
|
|
1259
1259
|
parser = argparse.ArgumentParser()
|
|
1260
1260
|
parser.add_argument('-c', '--removecontainers', action='store_true', default=False)
|
|
1261
|
-
parser.add_argument('-
|
|
1262
|
-
parser.add_argument('-i', '--removeimages', action='store_true', default=False)
|
|
1261
|
+
parser.add_argument('-n', '--removenamedvolumes', action='store_true', default=False, help="Also removes unused named volumes. Without this option only unused anonymous volumes are removed.")
|
|
1262
|
+
parser.add_argument('-i', '--removeimages', action='store_true', default=False, help="Removes all unused images. Without this option only dangling images are removed.")
|
|
1263
1263
|
verbosity_values = ", ".join(f"{lvl.value}={lvl.name}" for lvl in LogLevel)
|
|
1264
1264
|
parser.add_argument('-v', '--verbosity', required=False, default=3, help=f"Sets the loglevel. Possible values: {verbosity_values}")
|
|
1265
1265
|
args = parser.parse_args()
|
|
1266
1266
|
sc:ScriptCollectionCore=ScriptCollectionCore()
|
|
1267
1267
|
verbosity=int(args.verbosity)
|
|
1268
1268
|
sc.log.loglevel=LogLevel(verbosity)
|
|
1269
|
-
sc.reclaim_space_from_docker(args.removecontainers,args.
|
|
1269
|
+
sc.reclaim_space_from_docker(args.removecontainers,args.removenamedvolumes,args.removeimages)
|
|
1270
1270
|
return 0
|
|
1271
1271
|
|
|
1272
1272
|
|
|
@@ -245,6 +245,17 @@ class GeneralUtilities:
|
|
|
245
245
|
case _:
|
|
246
246
|
raise ValueError("Unknown echolon-value: "+str(echolon))
|
|
247
247
|
|
|
248
|
+
@staticmethod
|
|
249
|
+
@check_arguments
|
|
250
|
+
def get_amount_of_major_versions_behind(available_versions:list[str],current_version:str) -> int:
|
|
251
|
+
"""Returns how many major-versions the given current version is behind the latest of the given available versions.
|
|
252
|
+
The difference of the major-parts is used and not the amount of available major-versions, so a skipped major-version (for example 1.x.x and 3.x.x without 2.x.x) still counts as 2.
|
|
253
|
+
Returns 0 if the current version has the latest major-version or if no version is available."""
|
|
254
|
+
if len(available_versions)==0:
|
|
255
|
+
return 0
|
|
256
|
+
latest_major:int=GeneralUtilities.get_major_part_of_version(GeneralUtilities.get_latest_version(available_versions))
|
|
257
|
+
return max(0,latest_major-GeneralUtilities.get_major_part_of_version(current_version))
|
|
258
|
+
|
|
248
259
|
@staticmethod
|
|
249
260
|
@check_arguments
|
|
250
261
|
def get_latest_version(versions:list[str]) -> str:
|
|
@@ -715,6 +726,37 @@ class GeneralUtilities:
|
|
|
715
726
|
GeneralUtilities.__rmtree(folder)
|
|
716
727
|
GeneralUtilities.__rmtree(path)
|
|
717
728
|
|
|
729
|
+
@staticmethod
|
|
730
|
+
@check_arguments
|
|
731
|
+
def replace_symlinks_by_copies(folder: str, link_target_folder: str, excluded_folders: list[str]) -> list[str]:
|
|
732
|
+
"""Replaces every symbolic link inside folder (recursively) which points to a file or folder inside link_target_folder by a copy of
|
|
733
|
+
its target. Symbolic links which point to somewhere else are kept unchanged. Nothing inside excluded_folders (and nothing at all if
|
|
734
|
+
folder itself is one of them) is changed. Returns the paths of the replaced symbolic links.
|
|
735
|
+
A ValueError is raised for a symbolic link which points into link_target_folder but whose target does not exist."""
|
|
736
|
+
result: list[str] = []
|
|
737
|
+
normalized_link_target_folder: str = os.path.normcase(os.path.realpath(link_target_folder))
|
|
738
|
+
# abspath instead of realpath: a symbolic link which points into an excluded folder is not excluded itself.
|
|
739
|
+
normalized_excluded_folders: list[str] = [os.path.normcase(os.path.abspath(excluded_folder)) for excluded_folder in excluded_folders]
|
|
740
|
+
if os.path.normcase(os.path.abspath(folder)) in normalized_excluded_folders:
|
|
741
|
+
return result
|
|
742
|
+
for root, dirs, files in os.walk(folder):
|
|
743
|
+
dirs[:] = [name for name in dirs if os.path.normcase(os.path.abspath(os.path.join(root, name))) not in normalized_excluded_folders]
|
|
744
|
+
for name in dirs+files:
|
|
745
|
+
path: str = os.path.join(root, name)
|
|
746
|
+
if os.path.islink(path):
|
|
747
|
+
target: str = os.path.realpath(path)
|
|
748
|
+
normalized_target: str = os.path.normcase(target)
|
|
749
|
+
if normalized_target == normalized_link_target_folder or normalized_target.startswith(normalized_link_target_folder+os.sep):
|
|
750
|
+
if not os.path.exists(target):
|
|
751
|
+
raise ValueError(f"The target \"{target}\" of the symbolic link \"{path}\" does not exist.")
|
|
752
|
+
os.unlink(path)
|
|
753
|
+
if os.path.isdir(target):
|
|
754
|
+
shutil.copytree(target, path)
|
|
755
|
+
else:
|
|
756
|
+
shutil.copyfile(target, path)
|
|
757
|
+
result.append(path)
|
|
758
|
+
return result
|
|
759
|
+
|
|
718
760
|
@staticmethod
|
|
719
761
|
@check_arguments
|
|
720
762
|
def ensure_folder_exists_and_is_empty(path: str) -> None:
|
|
@@ -1,45 +1,33 @@
|
|
|
1
1
|
from abc import ABC, abstractmethod
|
|
2
2
|
from packaging.version import Version
|
|
3
|
-
from ..GeneralUtilities import
|
|
3
|
+
from ..GeneralUtilities import VersionEcholon
|
|
4
4
|
|
|
5
5
|
class AbstractImageHandler(ABC):
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
raise NotImplementedError()#because it is abstract
|
|
10
|
-
|
|
11
|
-
def _protected_get_credentials_for_registry(self,registry_address:str,username_str:str)->tuple[str,str]:
|
|
12
|
-
"""return (username, password) for basic auth.
|
|
13
|
-
Data will be taken from the registries which are declared in the environment-variables-configuration-file or in the environment
|
|
14
|
-
(see ScriptCollectionCore.get_docker_registry_credentials_from_environment_variables) if available.
|
|
15
|
-
If no credentials are available then None will be returned for the missing values."""
|
|
16
|
-
raise NotImplementedError()
|
|
17
|
-
|
|
18
|
-
def _protected_get_tags_from_images_from_custom_registry(self,registry_address:str)->list[str]:
|
|
19
|
-
raise NotImplementedError()
|
|
20
|
-
|
|
21
|
-
def _protected_get_tags_from_images_from_docker_hub(self,registry_address:str,tag_filter:str)->list[str]:
|
|
22
|
-
GeneralUtilities.assert_condition(registry_address.startswith("docker.io/",f"Image \"{registry_address}\" is not from docker-hub."))
|
|
23
|
-
raise NotImplementedError()
|
|
24
|
-
|
|
6
|
+
"""An image-handler knows the format of the tags of certain images: which tags of a registry are candidates for an update of a
|
|
7
|
+
given tag and which version a tag represents. The tags themselves are read from the registry by the OCIRegistryClient."""
|
|
8
|
+
|
|
25
9
|
@abstractmethod
|
|
26
|
-
def
|
|
10
|
+
def can_handle(self,image_name:str,tag:str)->bool:
|
|
11
|
+
"""Returns whether this handler knows the format of the given tag of the given image."""
|
|
27
12
|
raise NotImplementedError()#because it is abstract
|
|
28
13
|
|
|
29
14
|
@abstractmethod
|
|
30
|
-
def
|
|
15
|
+
def tag_has_same_format(self,image_name:str,tag:str,reference_tag:str)->bool:
|
|
16
|
+
"""Returns whether the given tag has the same format as the reference-tag, which means it is a candidate for an update of the
|
|
17
|
+
reference-tag. Example: for the reference-tag "13.4-slim" the tag "13.5-slim" has the same format but "13.5" and "13-slim" do not."""
|
|
31
18
|
raise NotImplementedError()#because it is abstract
|
|
32
19
|
|
|
33
20
|
@abstractmethod
|
|
34
|
-
def
|
|
21
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
35
22
|
raise NotImplementedError()#because it is abstract
|
|
36
23
|
|
|
37
24
|
@abstractmethod
|
|
38
25
|
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
39
26
|
raise NotImplementedError()#because it is abstract
|
|
40
27
|
|
|
41
|
-
def
|
|
42
|
-
"""Determines the
|
|
28
|
+
def get_latest_tag_custom(self,image_name:str,current_tag:str,candidate_tags:list[str])->str:
|
|
29
|
+
"""Determines the tag the image should be updated to using an image-specific custom algorithm.
|
|
30
|
+
candidate_tags contains the current tag and all available tags which have the same format as the current tag.
|
|
43
31
|
This is used when the echolon VersionEcholon.CustomAlgorithm is selected.
|
|
44
32
|
Image-handlers which want to support a custom algorithm must override this method."""
|
|
45
|
-
raise
|
|
33
|
+
raise ValueError(f"Image-handler \"{type(self).__name__}\" does not define a custom algorithm for determining the latest version of image \"{image_name}\".")
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from abc import abstractmethod
|
|
3
|
+
from packaging.version import Version
|
|
4
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
5
|
+
from .AbstractImageHandler import AbstractImageHandler
|
|
6
|
+
|
|
7
|
+
class AbstractTagPatternImageHandler(AbstractImageHandler):
|
|
8
|
+
"""Base for image-handlers whose tags consist of an optional prefix, a numeric version and an optional suffix, for example
|
|
9
|
+
"13.4-slim" (prefix "", version "13.4", suffix "-slim") or "v1.62.1-noble" (prefix "v", version "1.62.1", suffix "-noble").
|
|
10
|
+
Two tags have the same format if they have the same prefix, the same suffix and the same amount of version-parts. The suffix
|
|
11
|
+
must start with "-", "_" or "+" so that it can be separated from the version unambiguously."""
|
|
12
|
+
|
|
13
|
+
__tag_pattern=re.compile(r"^(?P<prefix>v?)(?P<version>\d+(?:\.\d+)*)(?P<suffix>(?:[-_+].*)?)$")
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def _protected_accepts_tag_parts(self,image_name:str,prefix:str,suffix:str)->bool:
|
|
17
|
+
"""Returns whether this handler is responsible for tags with the given prefix and suffix of the given image."""
|
|
18
|
+
raise NotImplementedError()#because it is abstract
|
|
19
|
+
|
|
20
|
+
def can_handle(self,image_name:str,tag:str)->bool:
|
|
21
|
+
tag_parts=self.__split_tag(tag)
|
|
22
|
+
if tag_parts is None:
|
|
23
|
+
return False
|
|
24
|
+
return self._protected_accepts_tag_parts(image_name,tag_parts[0],tag_parts[2])
|
|
25
|
+
|
|
26
|
+
def tag_has_same_format(self,image_name:str,tag:str,reference_tag:str)->bool:
|
|
27
|
+
tag_parts=self.__split_tag(tag)
|
|
28
|
+
reference_tag_parts=self.__split_tag(reference_tag)
|
|
29
|
+
if tag_parts is None or reference_tag_parts is None:
|
|
30
|
+
return False
|
|
31
|
+
same_prefix=tag_parts[0]==reference_tag_parts[0]
|
|
32
|
+
same_amount_of_version_parts=len(tag_parts[1].split("."))==len(reference_tag_parts[1].split("."))
|
|
33
|
+
same_suffix=tag_parts[2]==reference_tag_parts[2]
|
|
34
|
+
return same_prefix and same_amount_of_version_parts and same_suffix
|
|
35
|
+
|
|
36
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
37
|
+
tag_parts=self.__split_tag(tag)
|
|
38
|
+
GeneralUtilities.assert_condition(tag_parts is not None,f"Tag \"{tag}\" of image \"{image_name}\" does not have the format <prefix><version><suffix>.")
|
|
39
|
+
return Version(tag_parts[1])
|
|
40
|
+
|
|
41
|
+
def __split_tag(self,tag:str)->tuple[str,str,str]:
|
|
42
|
+
"""Returns (prefix, version, suffix) of the tag or None if the tag does not have the format <prefix><version><suffix>."""
|
|
43
|
+
match=AbstractTagPatternImageHandler.__tag_pattern.match(tag)
|
|
44
|
+
if match is None:
|
|
45
|
+
return None
|
|
46
|
+
return (match.group("prefix"),match.group("version"),match.group("suffix"))
|
|
@@ -1,20 +1,11 @@
|
|
|
1
|
-
from packaging.version import Version
|
|
2
1
|
from ...GeneralUtilities import VersionEcholon
|
|
3
|
-
from ..
|
|
2
|
+
from ..AbstractTagPatternImageHandler import AbstractTagPatternImageHandler
|
|
4
3
|
|
|
5
|
-
class ImageHandlerDebian(
|
|
4
|
+
class ImageHandlerDebian(AbstractTagPatternImageHandler):
|
|
5
|
+
"""Handles the tags of debian without variant, for example "13.4"."""
|
|
6
6
|
|
|
7
|
-
def
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
11
|
-
raise NotImplementedError()
|
|
12
|
-
|
|
13
|
-
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
-
raise NotImplementedError()
|
|
15
|
-
|
|
16
|
-
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
-
raise NotImplementedError()
|
|
7
|
+
def _protected_accepts_tag_parts(self,image_name:str,prefix:str,suffix:str)->bool:
|
|
8
|
+
return image_name.lower()=="debian" and prefix=="" and suffix==""
|
|
18
9
|
|
|
19
10
|
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
11
|
return VersionEcholon.LatestPatch
|
|
@@ -1,20 +1,11 @@
|
|
|
1
|
-
from packaging.version import Version
|
|
2
1
|
from ...GeneralUtilities import VersionEcholon
|
|
3
|
-
from ..
|
|
2
|
+
from ..AbstractTagPatternImageHandler import AbstractTagPatternImageHandler
|
|
4
3
|
|
|
5
|
-
class ImageHandlerDebianSlim(
|
|
4
|
+
class ImageHandlerDebianSlim(AbstractTagPatternImageHandler):
|
|
5
|
+
"""Handles the tags of the slim-variant of debian, for example "13.4-slim"."""
|
|
6
6
|
|
|
7
|
-
def
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
11
|
-
raise NotImplementedError()
|
|
12
|
-
|
|
13
|
-
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
-
raise NotImplementedError()
|
|
15
|
-
|
|
16
|
-
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
-
raise NotImplementedError()
|
|
7
|
+
def _protected_accepts_tag_parts(self,image_name:str,prefix:str,suffix:str)->bool:
|
|
8
|
+
return image_name.lower()=="debian" and prefix=="" and suffix=="-slim"
|
|
18
9
|
|
|
19
10
|
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
11
|
return VersionEcholon.LatestPatch
|
|
@@ -1,20 +1,11 @@
|
|
|
1
|
-
from packaging.version import Version
|
|
2
1
|
from ...GeneralUtilities import VersionEcholon
|
|
3
|
-
from ..
|
|
2
|
+
from ..AbstractTagPatternImageHandler import AbstractTagPatternImageHandler
|
|
4
3
|
|
|
5
|
-
class ImageHandlerGeneric(
|
|
4
|
+
class ImageHandlerGeneric(AbstractTagPatternImageHandler):
|
|
5
|
+
"""Handles tags which consist of a numeric version and an optional suffix, for example "12.2.2", "1.2026.8" or "7.2-26.04_edge"."""
|
|
6
6
|
|
|
7
|
-
def
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
11
|
-
raise NotImplementedError()
|
|
12
|
-
|
|
13
|
-
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
-
raise NotImplementedError()
|
|
15
|
-
|
|
16
|
-
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
-
raise NotImplementedError()
|
|
7
|
+
def _protected_accepts_tag_parts(self,image_name:str,prefix:str,suffix:str)->bool:
|
|
8
|
+
return prefix==""
|
|
18
9
|
|
|
19
10
|
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
|
-
|
|
11
|
+
return VersionEcholon.LatestVersion
|
|
@@ -1,20 +1,11 @@
|
|
|
1
|
-
from packaging.version import Version
|
|
2
1
|
from ...GeneralUtilities import VersionEcholon
|
|
3
|
-
from ..
|
|
2
|
+
from ..AbstractTagPatternImageHandler import AbstractTagPatternImageHandler
|
|
4
3
|
|
|
5
|
-
class ImageHandlerGenericV(
|
|
4
|
+
class ImageHandlerGenericV(AbstractTagPatternImageHandler):
|
|
5
|
+
"""Handles tags which consist of the prefix "v", a numeric version and an optional suffix, for example "v1.41.2" or "v1.62.1-noble"."""
|
|
6
6
|
|
|
7
|
-
def
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
11
|
-
raise NotImplementedError()
|
|
12
|
-
|
|
13
|
-
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
-
raise NotImplementedError()
|
|
15
|
-
|
|
16
|
-
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
-
raise NotImplementedError()
|
|
7
|
+
def _protected_accepts_tag_parts(self,image_name:str,prefix:str,suffix:str)->bool:
|
|
8
|
+
return prefix=="v"
|
|
18
9
|
|
|
19
10
|
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
|
-
|
|
11
|
+
return VersionEcholon.LatestVersion
|
|
@@ -3,18 +3,17 @@ from ...GeneralUtilities import VersionEcholon
|
|
|
3
3
|
from ..AbstractImageHandler import AbstractImageHandler
|
|
4
4
|
|
|
5
5
|
class ImageHandlerGitlabCE(AbstractImageHandler):
|
|
6
|
+
"""Recognizes the tags of GitLab ce (for example "18.5.1-ce.0"), so that they are not handled by a generic handler: an update of
|
|
7
|
+
GitLab must respect the required upgrade-stops, which is not implemented yet."""
|
|
6
8
|
|
|
7
|
-
def can_handle(self,image_name:str)->bool:
|
|
8
|
-
|
|
9
|
+
def can_handle(self,image_name:str,tag:str)->bool:
|
|
10
|
+
return "gitlab" in image_name.lower() and tag.endswith("-ce.0")
|
|
9
11
|
|
|
10
|
-
def
|
|
11
|
-
raise NotImplementedError()
|
|
12
|
+
def tag_has_same_format(self,image_name:str,tag:str,reference_tag:str)->bool:
|
|
13
|
+
raise NotImplementedError(f"Updating the tag of image \"{image_name}\" (GitLab ce) is not implemented yet.")
|
|
12
14
|
|
|
13
15
|
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
-
raise NotImplementedError()
|
|
15
|
-
|
|
16
|
-
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
-
raise NotImplementedError()
|
|
16
|
+
raise NotImplementedError(f"Updating the tag of image \"{image_name}\" (GitLab ce) is not implemented yet.")
|
|
18
17
|
|
|
19
18
|
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
19
|
return VersionEcholon.CustomAlgorithm
|
|
@@ -3,18 +3,17 @@ from ...GeneralUtilities import VersionEcholon
|
|
|
3
3
|
from ..AbstractImageHandler import AbstractImageHandler
|
|
4
4
|
|
|
5
5
|
class ImageHandlerGitlabEE(AbstractImageHandler):
|
|
6
|
+
"""Recognizes the tags of GitLab ee (for example "18.5.1-ee.0"), so that they are not handled by a generic handler: an update of
|
|
7
|
+
GitLab must respect the required upgrade-stops, which is not implemented yet."""
|
|
6
8
|
|
|
7
|
-
def can_handle(self,image_name:str)->bool:
|
|
8
|
-
|
|
9
|
+
def can_handle(self,image_name:str,tag:str)->bool:
|
|
10
|
+
return "gitlab" in image_name.lower() and tag.endswith("-ee.0")
|
|
9
11
|
|
|
10
|
-
def
|
|
11
|
-
raise NotImplementedError()
|
|
12
|
+
def tag_has_same_format(self,image_name:str,tag:str,reference_tag:str)->bool:
|
|
13
|
+
raise NotImplementedError(f"Updating the tag of image \"{image_name}\" (GitLab ee) is not implemented yet.")
|
|
12
14
|
|
|
13
15
|
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
-
raise NotImplementedError()
|
|
15
|
-
|
|
16
|
-
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
-
raise NotImplementedError()
|
|
16
|
+
raise NotImplementedError(f"Updating the tag of image \"{image_name}\" (GitLab ee) is not implemented yet.")
|
|
18
17
|
|
|
19
18
|
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
19
|
return VersionEcholon.CustomAlgorithm
|
|
@@ -6,6 +6,7 @@ from ..ScriptCollectionCore import ScriptCollectionCore
|
|
|
6
6
|
from ..GeneralUtilities import VersionEcholon
|
|
7
7
|
from ..SCLog import LogLevel
|
|
8
8
|
from .AbstractImageHandler import AbstractImageHandler
|
|
9
|
+
from .OCIRegistryClient import OCIRegistryClient
|
|
9
10
|
from .ConcreteImageHandlers.ImageHandlerDebian import ImageHandlerDebian
|
|
10
11
|
from .ConcreteImageHandlers.ImageHandlerDebianSlim import ImageHandlerDebianSlim
|
|
11
12
|
from .ConcreteImageHandlers.ImageHandlerGeneric import ImageHandlerGeneric
|
|
@@ -17,6 +18,7 @@ class OCIImageManager:
|
|
|
17
18
|
|
|
18
19
|
__sc:ScriptCollectionCore=None
|
|
19
20
|
image_handler:list[AbstractImageHandler] = None
|
|
21
|
+
registry_client:OCIRegistryClient = None
|
|
20
22
|
#image-names for which it was already logged that the fallback-registry is used. The lookup happens for every image of
|
|
21
23
|
#the repository and often several times per build, so without this the same hint would flood the log.
|
|
22
24
|
__images_with_reported_fallback_registry:set[str] = None
|
|
@@ -30,21 +32,23 @@ class OCIImageManager:
|
|
|
30
32
|
self.__sc=sc
|
|
31
33
|
self.__images_with_reported_fallback_registry=set()
|
|
32
34
|
self.__availability_of_image=dict()
|
|
35
|
+
self.registry_client=OCIRegistryClient(self.__sc)
|
|
36
|
+
#the order matters: the first handler which can handle a tag is used, so the specific handlers have to be listed before the generic ones.
|
|
33
37
|
self.image_handler=[
|
|
34
|
-
ImageHandlerDebian(),
|
|
35
|
-
ImageHandlerDebianSlim(),
|
|
36
|
-
ImageHandlerGeneric(),
|
|
37
|
-
ImageHandlerGenericV(),
|
|
38
38
|
ImageHandlerGitlabCE(),
|
|
39
39
|
ImageHandlerGitlabEE(),
|
|
40
|
+
ImageHandlerDebianSlim(),
|
|
41
|
+
ImageHandlerDebian(),
|
|
42
|
+
ImageHandlerGenericV(),
|
|
43
|
+
ImageHandlerGeneric(),
|
|
40
44
|
]
|
|
41
45
|
|
|
42
46
|
@GeneralUtilities.check_arguments
|
|
43
|
-
def get_image_handler(self,image_name:str)->AbstractImageHandler:
|
|
44
|
-
for image_handler in self.image_handler:
|
|
45
|
-
if image_handler.can_handle(image_name):
|
|
47
|
+
def get_image_handler(self,image_name:str,tag:str)->AbstractImageHandler:
|
|
48
|
+
for image_handler in self.image_handler:
|
|
49
|
+
if image_handler.can_handle(image_name,tag):
|
|
46
50
|
return image_handler
|
|
47
|
-
raise ValueError(f"No image-handler available for image \"{image_name}\".")
|
|
51
|
+
raise ValueError(f"No image-handler available for tag \"{tag}\" of image \"{image_name}\".")
|
|
48
52
|
|
|
49
53
|
@GeneralUtilities.check_arguments
|
|
50
54
|
def get_repository_image_definition_file(self,repository:str)->str:
|
|
@@ -189,7 +193,10 @@ class OCIImageManager:
|
|
|
189
193
|
|
|
190
194
|
@GeneralUtilities.check_arguments
|
|
191
195
|
def update_default_tag_for_images_in_image_definitions_file(self,repository:str,search_in_custom_registry_only_if_available:bool)->None:
|
|
192
|
-
file
|
|
196
|
+
"""Updates the default-tag of every image in the image-definition-file of the repository to the latest available tag with the same format.
|
|
197
|
+
If search_in_custom_registry_only_if_available is True then only the custom registry which is defined for an image on this machine is
|
|
198
|
+
asked for newer tags, so that only tags are chosen which are available there; an image without custom registry keeps its tag then."""
|
|
199
|
+
file=self.get_repository_image_definition_file(repository)
|
|
193
200
|
GeneralUtilities.assert_file_exists(file)
|
|
194
201
|
lines=GeneralUtilities.read_nonempty_lines_from_file(file)
|
|
195
202
|
new_lines:list[str]=[]
|
|
@@ -205,41 +212,61 @@ class OCIImageManager:
|
|
|
205
212
|
registry_address=line_parts[1]
|
|
206
213
|
default_tag=line_parts[2]
|
|
207
214
|
tag=default_tag
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
if not search_in_custom_registry_only_if_available:
|
|
219
|
-
addresses_to_check.append(registry_address)
|
|
220
|
-
newest_versions:set[Version]={default_tag}
|
|
221
|
-
current_version=Version(default_tag)
|
|
222
|
-
for address in addresses_to_check:
|
|
223
|
-
newest_versions_for_address=self.get_latest_of_image(image_name,address,current_version,VersionEcholon.LatestVersion)
|
|
224
|
-
if newest_versions_for_address is not None:
|
|
225
|
-
newest_versions.update(newest_versions_for_address)
|
|
226
|
-
GeneralUtilities.assert_condition(len(newest_versions)>0,f"Could not find any version for image {image_name} in registry {registry_address}.")
|
|
227
|
-
newest_version=max(newest_versions)
|
|
228
|
-
tag=self.version_to_tag(image_name,newest_version)
|
|
229
|
-
except Exception as e:
|
|
230
|
-
self.__sc.log.log(f"Could not get tag for image {image_name} from registry {registry_address}. Reason: {str(e)}",LogLevel.Warning)
|
|
215
|
+
addresses_to_check=self.__get_registry_addresses_to_search_for_newer_tags(image_name,registry_address,search_in_custom_registry_only_if_available)
|
|
216
|
+
if len(addresses_to_check)==0:
|
|
217
|
+
self.__sc.log.log(f"The tag of image \"{image_name}\" is not updated because no custom registry is defined for it in \"{self.get_global_docker_image_registries_file()}\".",LogLevel.Warning)
|
|
218
|
+
else:
|
|
219
|
+
try:
|
|
220
|
+
tag=self.get_latest_tag_of_image(image_name,addresses_to_check,default_tag,VersionEcholon.LatestVersion)
|
|
221
|
+
except Exception as e:
|
|
222
|
+
self.__sc.log.log(f"Could not determine the latest tag of image \"{image_name}\" using the registries {', '.join(addresses_to_check)}. Reason: {type(e).__name__}: {str(e)}",LogLevel.Warning)
|
|
223
|
+
if tag!=default_tag:
|
|
224
|
+
self.__sc.log.log(f"Update tag of image \"{image_name}\" from \"{default_tag}\" to \"{tag}\".",LogLevel.Information)
|
|
231
225
|
new_lines.append(f"{image_name};{registry_address};{tag}")
|
|
232
226
|
GeneralUtilities.write_lines_to_file(file,new_lines)
|
|
233
227
|
|
|
234
228
|
@GeneralUtilities.check_arguments
|
|
235
|
-
def
|
|
236
|
-
result=
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
229
|
+
def __get_registry_addresses_to_search_for_newer_tags(self,image_name:str,upstream_registry_address:str,search_in_custom_registry_only_if_available:bool)->list[str]:
|
|
230
|
+
result:list[str]=[]
|
|
231
|
+
if self.custom_registry_is_defined(image_name):
|
|
232
|
+
#the custom registry-address is used directly (and not get_registry_address_for_image) because this function does not
|
|
233
|
+
#pull an image but asks the registry which tags it has, so it does not matter whether that registry currently provides
|
|
234
|
+
#the image with the tag which is defined at the moment.
|
|
235
|
+
result.append(self.get_custom_registry_address_for_image(image_name))
|
|
236
|
+
if not search_in_custom_registry_only_if_available:
|
|
237
|
+
result.append(upstream_registry_address)
|
|
238
|
+
return result
|
|
239
|
+
|
|
240
|
+
@GeneralUtilities.check_arguments
|
|
241
|
+
def get_latest_tag_of_image(self,image_name:str,registry_addresses:list[str],current_tag:str,echolon:VersionEcholon)->str:
|
|
242
|
+
"""Returns the tag the image should be updated to according to the echolon, considering the tags of all given registry-addresses
|
|
243
|
+
(always without tag, for example "docker.io/library/debian").
|
|
244
|
+
Only tags with the same format as the current tag are candidates (see AbstractImageHandler.tag_has_same_format). The current tag is
|
|
245
|
+
always a candidate, so the result is never a downgrade."""
|
|
246
|
+
image_handler=self.get_image_handler(image_name,current_tag)
|
|
247
|
+
if echolon==VersionEcholon.NoUpdate:
|
|
248
|
+
return current_tag
|
|
249
|
+
candidate_tags:list[str]=[current_tag]
|
|
250
|
+
for registry_address in registry_addresses:
|
|
251
|
+
for tag in self.registry_client.get_tags(registry_address):
|
|
252
|
+
if tag not in candidate_tags and image_handler.tag_has_same_format(image_name,tag,current_tag):
|
|
253
|
+
candidate_tags.append(tag)
|
|
254
|
+
if echolon==VersionEcholon.CustomAlgorithm:
|
|
255
|
+
return image_handler.get_latest_tag_custom(image_name,current_tag,candidate_tags)
|
|
256
|
+
tags_by_version:dict[Version,str]={}
|
|
257
|
+
for tag in candidate_tags:
|
|
258
|
+
#setdefault, because different tags can represent the same version (like "13.4" and "13.04"); then the first one (the current tag if affected) is used.
|
|
259
|
+
tags_by_version.setdefault(image_handler.tag_to_version(image_name,tag),tag)
|
|
260
|
+
current_version=image_handler.tag_to_version(image_name,current_tag)
|
|
261
|
+
return tags_by_version[self.choose_version(set(tags_by_version.keys()),current_version,echolon)]
|
|
262
|
+
|
|
263
|
+
@GeneralUtilities.check_arguments
|
|
264
|
+
def choose_version(self,available_versions:set[Version],current_version:Version,echolon:VersionEcholon)->Version:
|
|
265
|
+
"""Returns the version out of the available versions which should be used according to the echolon.
|
|
266
|
+
available_versions must contain current_version. The echolons NoUpdate and CustomAlgorithm are not handled here because they do
|
|
267
|
+
not choose out of the available versions."""
|
|
268
|
+
GeneralUtilities.assert_condition(current_version in available_versions,f"The available versions do not contain the current version {current_version}.")
|
|
269
|
+
result:Version=None
|
|
243
270
|
match echolon:
|
|
244
271
|
case VersionEcholon.LatestPatch:#caution: this never updates the minor or major version
|
|
245
272
|
#take the latest available patch-version for the current minor-version of the image.
|
|
@@ -260,11 +287,8 @@ class OCIImageManager:
|
|
|
260
287
|
case VersionEcholon.LatestVersion:#usual case for most images
|
|
261
288
|
#take the latest available version of the image.
|
|
262
289
|
result=max(available_versions)
|
|
263
|
-
case
|
|
264
|
-
|
|
265
|
-
result=image_handler.get_latest_version_custom(image_name,registry_address,current_version)
|
|
266
|
-
case VersionEcholon.NoUpdate:
|
|
267
|
-
result = current_version
|
|
290
|
+
case _:
|
|
291
|
+
raise ValueError(f"Echolon \"{echolon.name}\" does not choose a version out of the available versions.")
|
|
268
292
|
return result
|
|
269
293
|
|
|
270
294
|
@GeneralUtilities.check_arguments
|
|
@@ -292,24 +316,6 @@ class OCIImageManager:
|
|
|
292
316
|
if len(higher_values)==0:
|
|
293
317
|
return None
|
|
294
318
|
return min(higher_values)
|
|
295
|
-
|
|
296
|
-
@GeneralUtilities.check_arguments
|
|
297
|
-
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
298
|
-
"""registry_address must have one of these formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian".
|
|
299
|
-
returns something like ["13.2-slim", "13.2", "13.3-slim", "13.3"]."""
|
|
300
|
-
return self.get_image_handler(image_name).get_available_tags_of_image(image_name,registry_address)
|
|
301
|
-
|
|
302
|
-
@GeneralUtilities.check_arguments
|
|
303
|
-
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
304
|
-
"""registry_address must have one of these formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian"."""
|
|
305
|
-
return self.get_image_handler(image_name).tag_to_version(image_name, tag)
|
|
306
|
-
|
|
307
|
-
@GeneralUtilities.check_arguments
|
|
308
|
-
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
309
|
-
"""registry_address must have one of these formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian".
|
|
310
|
-
returns something like "13.3-slim".
|
|
311
|
-
If there are multiple tags available for a certain version then the image-handler decides which one will be returned."""
|
|
312
|
-
return self.get_image_handler(image_name).version_to_tag(image_name,version)
|
|
313
319
|
|
|
314
320
|
@GeneralUtilities.check_arguments
|
|
315
321
|
def get_images_used_in_docker_compose_file(self,docker_compose_file:str)->dict[str,tuple[str,str,str]]:#returns dict[service_name,[image_name,image_address,current_tag]]
|
|
@@ -346,15 +352,14 @@ class OCIImageManager:
|
|
|
346
352
|
image_name=service_information[0]
|
|
347
353
|
image_address=service_information[1]
|
|
348
354
|
current_tag=service_information[2]
|
|
349
|
-
image_handler=self.get_image_handler(image_name)
|
|
355
|
+
image_handler=self.get_image_handler(image_name,current_tag)
|
|
350
356
|
effective_echolon=per_image_echolons.get(image_name,default_echolon)
|
|
351
357
|
if effective_echolon is None:
|
|
352
|
-
effective_echolon=image_handler.get_default_echolon_for_update()
|
|
358
|
+
effective_echolon=image_handler.get_default_echolon_for_update(image_name)
|
|
353
359
|
if verbose:
|
|
354
360
|
self.__sc.log.log(f"Service \"{service}\": checking image \"{image_address}:{current_tag}\" using echolon \"{effective_echolon.name}\".",LogLevel.Information)
|
|
355
|
-
|
|
356
|
-
new_tag
|
|
357
|
-
if new_tag is not None and new_tag != current_tag:
|
|
361
|
+
new_tag=self.get_latest_tag_of_image(image_name,[image_address],current_tag,effective_echolon)
|
|
362
|
+
if new_tag != current_tag:
|
|
358
363
|
new_image_reference = f"{image_address}:{new_tag}"
|
|
359
364
|
if verbose:
|
|
360
365
|
self.__sc.log.log(f"Service \"{service}\": updating image \"{image_address}\" from tag \"{current_tag}\" to tag \"{new_tag}\".",LogLevel.Information)
|