scriptcollection 4.2.81__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/AnionBuildPlatform.py +199 -0
- ScriptCollection/CertificateUpdater.py +149 -0
- ScriptCollection/Executables.py +921 -0
- ScriptCollection/GeneralUtilities.py +1589 -0
- ScriptCollection/HTTPMaintenanceOverheadHelper.py +36 -0
- ScriptCollection/OCIImages/AbstractImageHandler.py +38 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +20 -0
- ScriptCollection/OCIImages/ConcreteImageHandlers/__init__.py +0 -0
- ScriptCollection/OCIImages/OCIImageManager.py +190 -0
- ScriptCollection/OCIImages/__init__.py +0 -0
- ScriptCollection/ProcessesRunner.py +43 -0
- ScriptCollection/ProgramRunnerBase.py +47 -0
- ScriptCollection/ProgramRunnerMock.py +2 -0
- ScriptCollection/ProgramRunnerPopen.py +57 -0
- ScriptCollection/ProgramRunnerSudo.py +108 -0
- ScriptCollection/Resources/CultureChooser/CultureChooser.js +29 -0
- ScriptCollection/Resources/CultureChooser/index.html +15 -0
- ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html +15 -0
- ScriptCollection/SCLog.py +115 -0
- ScriptCollection/ScriptCollectionCore.py +3485 -0
- ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +192 -0
- ScriptCollection/TFCPS/Docker/__init__.py +0 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
- ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
- ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +547 -0
- ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
- ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +137 -0
- ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
- ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -0
- ScriptCollection/TFCPS/Go/__init__.py +0 -0
- ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +42 -0
- ScriptCollection/TFCPS/Maven/__init__.py +0 -0
- ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +232 -0
- ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
- ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +239 -0
- ScriptCollection/TFCPS/Python/__init__.py +0 -0
- ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +42 -0
- ScriptCollection/TFCPS/Rust/__init__.py +0 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +433 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +135 -0
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +301 -0
- ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -0
- ScriptCollection/TFCPS/TFCPS_Generic.py +44 -0
- ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -0
- ScriptCollection/TFCPS/TFCPS_MergeToStable.py +356 -0
- ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -0
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +1565 -0
- ScriptCollection/TFCPS/__init__.py +0 -0
- ScriptCollection/__init__.py +0 -0
- ScriptCollection/__pycache__/GeneralUtilities.cpython-311.pyc +0 -0
- ScriptCollection/__pycache__/__init__.cpython-311.pyc +0 -0
- scriptcollection-4.2.81.dist-info/METADATA +169 -0
- scriptcollection-4.2.81.dist-info/RECORD +62 -0
- scriptcollection-4.2.81.dist-info/WHEEL +5 -0
- scriptcollection-4.2.81.dist-info/entry_points.txt +67 -0
- scriptcollection-4.2.81.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from .GeneralUtilities import GeneralUtilities
|
|
3
|
+
|
|
4
|
+
class HTTPMaintenanceOverheadHelper:
|
|
5
|
+
|
|
6
|
+
__culture_regex = re.compile(r"^[a-zA-Z]+(-[a-zA-Z]+)?$")
|
|
7
|
+
|
|
8
|
+
def is_valid_culture(self,culture_string: str) -> bool:
|
|
9
|
+
"""
|
|
10
|
+
Checks if a string is in a valid language/culture format.
|
|
11
|
+
result:
|
|
12
|
+
de, de-DE, en, en-US, fr, fr-FR, etc. is allowed.
|
|
13
|
+
de3, zh-Hans-CN, en_US, etc. is not allowed.
|
|
14
|
+
"""
|
|
15
|
+
return bool(self.__culture_regex.fullmatch(culture_string))
|
|
16
|
+
|
|
17
|
+
def get_index_html(self,site_title:str) -> str:
|
|
18
|
+
content = GeneralUtilities._internal_load_resource("CultureChooser/index.html")
|
|
19
|
+
content_as_string = content.decode("utf-8")
|
|
20
|
+
result=GeneralUtilities.replace_variable("<!--","title","-->", site_title, content_as_string)
|
|
21
|
+
return result
|
|
22
|
+
|
|
23
|
+
def get_culture_chooser_script(self,supported_languages:list[str]) -> str:
|
|
24
|
+
GeneralUtilities.assert_condition("en" in supported_languages, "The default language 'en' must be included in the list of supported languages.")
|
|
25
|
+
for supported_language in supported_languages:
|
|
26
|
+
GeneralUtilities.assert_condition(self.is_valid_culture(supported_language), f"Invalid language code '{supported_language}'. Supported languages must be in the format 'en' or 'en-US'.")
|
|
27
|
+
content = GeneralUtilities._internal_load_resource("CultureChooser/CultureChooser.js")
|
|
28
|
+
content_as_string = content.decode("utf-8")
|
|
29
|
+
result=GeneralUtilities.replace_variable("/*","supportedCultures","*/", ", ".join([f"\"{supported_language}\"" for supported_language in supported_languages]), content_as_string)
|
|
30
|
+
return result
|
|
31
|
+
|
|
32
|
+
def get_maintenance_file(self,site_title:str) -> str:
|
|
33
|
+
content = GeneralUtilities._internal_load_resource("MaintenanceSite/MaintenanceSite.html")
|
|
34
|
+
content_as_string = content.decode("utf-8")
|
|
35
|
+
result=GeneralUtilities.replace_variable("<!--","title","-->", site_title, content_as_string)
|
|
36
|
+
return result
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from packaging.version import Version
|
|
3
|
+
from ..GeneralUtilities import GeneralUtilities, VersionEcholon
|
|
4
|
+
|
|
5
|
+
class AbstractImageHandler(ABC):
|
|
6
|
+
|
|
7
|
+
@abstractmethod
|
|
8
|
+
def can_handle(self,image_name:str)->bool:
|
|
9
|
+
raise NotImplementedError()#because it is abstract
|
|
10
|
+
|
|
11
|
+
def _protected_get_credentials_for_registry(self,registry_address:str,username_str:str)->tuple[str,str]:
|
|
12
|
+
"""return (username, password) for basic auth.
|
|
13
|
+
Data will be taken from "~/.ScriptCollection/GlobalCache/RegistryCredentials.csv" if available.
|
|
14
|
+
If no credentials are available then None will be returned for the missing values."""
|
|
15
|
+
raise NotImplementedError()
|
|
16
|
+
|
|
17
|
+
def _protected_get_tags_from_images_from_custom_registry(self,registry_address:str)->list[str]:
|
|
18
|
+
raise NotImplementedError()
|
|
19
|
+
|
|
20
|
+
def _protected_get_tags_from_images_from_docker_hub(self,registry_address:str,tag_filter:str)->list[str]:
|
|
21
|
+
GeneralUtilities.assert_condition(registry_address.startswith("docker.io/",f"Image \"{registry_address}\" is not from docker-hub."))
|
|
22
|
+
raise NotImplementedError()
|
|
23
|
+
|
|
24
|
+
@abstractmethod
|
|
25
|
+
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
26
|
+
raise NotImplementedError()#because it is abstract
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
30
|
+
raise NotImplementedError()#because it is abstract
|
|
31
|
+
|
|
32
|
+
@abstractmethod
|
|
33
|
+
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
34
|
+
raise NotImplementedError()#because it is abstract
|
|
35
|
+
|
|
36
|
+
@abstractmethod
|
|
37
|
+
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
38
|
+
raise NotImplementedError()#because it is abstract
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from packaging.version import Version
|
|
2
|
+
from ...GeneralUtilities import VersionEcholon
|
|
3
|
+
from ..AbstractImageHandler import AbstractImageHandler
|
|
4
|
+
|
|
5
|
+
class ImageHandlerDebian(AbstractImageHandler):
|
|
6
|
+
|
|
7
|
+
def can_handle(self,image_name:str)->bool:
|
|
8
|
+
raise NotImplementedError()#TODO
|
|
9
|
+
|
|
10
|
+
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
11
|
+
raise NotImplementedError()
|
|
12
|
+
|
|
13
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
+
raise NotImplementedError()
|
|
15
|
+
|
|
16
|
+
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
+
raise NotImplementedError()
|
|
18
|
+
|
|
19
|
+
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
|
+
return VersionEcholon.LatestPatch
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from packaging.version import Version
|
|
2
|
+
from ...GeneralUtilities import VersionEcholon
|
|
3
|
+
from ..AbstractImageHandler import AbstractImageHandler
|
|
4
|
+
|
|
5
|
+
class ImageHandlerDebianSlim(AbstractImageHandler):
|
|
6
|
+
|
|
7
|
+
def can_handle(self,image_name:str)->bool:
|
|
8
|
+
raise NotImplementedError()#TODO
|
|
9
|
+
|
|
10
|
+
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
11
|
+
raise NotImplementedError()
|
|
12
|
+
|
|
13
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
+
raise NotImplementedError()
|
|
15
|
+
|
|
16
|
+
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
+
raise NotImplementedError()
|
|
18
|
+
|
|
19
|
+
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
|
+
return VersionEcholon.LatestPatch
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from packaging.version import Version
|
|
2
|
+
from ...GeneralUtilities import VersionEcholon
|
|
3
|
+
from ..AbstractImageHandler import AbstractImageHandler
|
|
4
|
+
|
|
5
|
+
class ImageHandlerGeneric(AbstractImageHandler):
|
|
6
|
+
|
|
7
|
+
def can_handle(self,image_name:str)->bool:
|
|
8
|
+
raise NotImplementedError()#TODO
|
|
9
|
+
|
|
10
|
+
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
11
|
+
raise NotImplementedError()
|
|
12
|
+
|
|
13
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
+
raise NotImplementedError()
|
|
15
|
+
|
|
16
|
+
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
+
raise NotImplementedError()
|
|
18
|
+
|
|
19
|
+
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
|
+
raise NotImplementedError()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from packaging.version import Version
|
|
2
|
+
from ...GeneralUtilities import VersionEcholon
|
|
3
|
+
from ..AbstractImageHandler import AbstractImageHandler
|
|
4
|
+
|
|
5
|
+
class ImageHandlerGenericV(AbstractImageHandler):
|
|
6
|
+
|
|
7
|
+
def can_handle(self,image_name:str)->bool:
|
|
8
|
+
raise NotImplementedError()#TODO
|
|
9
|
+
|
|
10
|
+
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
11
|
+
raise NotImplementedError()
|
|
12
|
+
|
|
13
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
+
raise NotImplementedError()
|
|
15
|
+
|
|
16
|
+
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
+
raise NotImplementedError()
|
|
18
|
+
|
|
19
|
+
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
|
+
raise NotImplementedError()
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from packaging.version import Version
|
|
2
|
+
from ...GeneralUtilities import VersionEcholon
|
|
3
|
+
from ..AbstractImageHandler import AbstractImageHandler
|
|
4
|
+
|
|
5
|
+
class ImageHandlerGitlabCE(AbstractImageHandler):
|
|
6
|
+
|
|
7
|
+
def can_handle(self,image_name:str)->bool:
|
|
8
|
+
raise NotImplementedError()#TODO
|
|
9
|
+
|
|
10
|
+
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
11
|
+
raise NotImplementedError()
|
|
12
|
+
|
|
13
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
+
raise NotImplementedError()
|
|
15
|
+
|
|
16
|
+
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
+
raise NotImplementedError()
|
|
18
|
+
|
|
19
|
+
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
|
+
return VersionEcholon.CustomAlgorithm
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
from packaging.version import Version
|
|
2
|
+
from ...GeneralUtilities import VersionEcholon
|
|
3
|
+
from ..AbstractImageHandler import AbstractImageHandler
|
|
4
|
+
|
|
5
|
+
class ImageHandlerGitlabEE(AbstractImageHandler):
|
|
6
|
+
|
|
7
|
+
def can_handle(self,image_name:str)->bool:
|
|
8
|
+
raise NotImplementedError()#TODO
|
|
9
|
+
|
|
10
|
+
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
11
|
+
raise NotImplementedError()
|
|
12
|
+
|
|
13
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
14
|
+
raise NotImplementedError()
|
|
15
|
+
|
|
16
|
+
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
17
|
+
raise NotImplementedError()
|
|
18
|
+
|
|
19
|
+
def get_default_echolon_for_update(self,image_name:str)->VersionEcholon:
|
|
20
|
+
return VersionEcholon.CustomAlgorithm
|
|
File without changes
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from packaging.version import Version
|
|
3
|
+
from ..GeneralUtilities import GeneralUtilities
|
|
4
|
+
from ..ScriptCollectionCore import ScriptCollectionCore
|
|
5
|
+
from ..GeneralUtilities import VersionEcholon
|
|
6
|
+
from ..SCLog import LogLevel
|
|
7
|
+
from .AbstractImageHandler import AbstractImageHandler
|
|
8
|
+
from .ConcreteImageHandlers.ImageHandlerDebian import ImageHandlerDebian
|
|
9
|
+
from .ConcreteImageHandlers.ImageHandlerDebianSlim import ImageHandlerDebianSlim
|
|
10
|
+
from .ConcreteImageHandlers.ImageHandlerGeneric import ImageHandlerGeneric
|
|
11
|
+
from .ConcreteImageHandlers.ImageHandlerGenericV import ImageHandlerGenericV
|
|
12
|
+
from .ConcreteImageHandlers.ImageHandlerGitlabCE import ImageHandlerGitlabCE
|
|
13
|
+
from .ConcreteImageHandlers.ImageHandlerGitlabEE import ImageHandlerGitlabEE
|
|
14
|
+
|
|
15
|
+
class OCIImageManager:
|
|
16
|
+
|
|
17
|
+
__sc:ScriptCollectionCore=None
|
|
18
|
+
image_handler:list[AbstractImageHandler]
|
|
19
|
+
|
|
20
|
+
def __init__(self,sc:ScriptCollectionCore):
|
|
21
|
+
if sc is None:
|
|
22
|
+
sc=ScriptCollectionCore()
|
|
23
|
+
self.__sc=sc
|
|
24
|
+
self.image_handler=[
|
|
25
|
+
ImageHandlerDebian(),
|
|
26
|
+
ImageHandlerDebianSlim(),
|
|
27
|
+
ImageHandlerGeneric(),
|
|
28
|
+
ImageHandlerGenericV(),
|
|
29
|
+
ImageHandlerGitlabCE(),
|
|
30
|
+
ImageHandlerGitlabEE(),
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
@GeneralUtilities.check_arguments
|
|
34
|
+
def get_image_handler(self,image_name:str)->AbstractImageHandler:
|
|
35
|
+
for image_handler in self.image_handler:
|
|
36
|
+
if image_handler.can_handle(image_name):
|
|
37
|
+
return image_handler
|
|
38
|
+
raise ValueError(f"No image-handler available for image \"{image_name}\".")
|
|
39
|
+
|
|
40
|
+
@GeneralUtilities.check_arguments
|
|
41
|
+
def get_repository_image_definition_file(self,repository:str)->str:
|
|
42
|
+
self.__sc.assert_is_git_repository(repository)
|
|
43
|
+
sc_folder_in_repo=os.path.join(repository,".ScriptCollection","OCIImages")
|
|
44
|
+
GeneralUtilities.ensure_directory_exists(sc_folder_in_repo)
|
|
45
|
+
image_definition_file=os.path.join(sc_folder_in_repo,"ImageDefinition.csv")
|
|
46
|
+
if not os.path.isfile(image_definition_file):
|
|
47
|
+
GeneralUtilities.ensure_file_exists(image_definition_file)
|
|
48
|
+
GeneralUtilities.write_text_to_file(image_definition_file,"ImageName;UpstreamRegistryAddress;DefaultTag")
|
|
49
|
+
return image_definition_file
|
|
50
|
+
|
|
51
|
+
@GeneralUtilities.check_arguments
|
|
52
|
+
def get_global_docker_image_registries_file(self)->str:
|
|
53
|
+
folder=os.path.join(self.__sc.get_global_cache_folder(),"OCIImages")
|
|
54
|
+
GeneralUtilities.ensure_directory_exists(folder)
|
|
55
|
+
result=os.path.join(folder,"ImageRegistries.csv")
|
|
56
|
+
if not os.path.isfile(result):
|
|
57
|
+
GeneralUtilities.ensure_file_exists(result)
|
|
58
|
+
GeneralUtilities.write_lines_to_file(result,["ImageName;RegistryAddress"])
|
|
59
|
+
return result
|
|
60
|
+
|
|
61
|
+
@GeneralUtilities.check_arguments
|
|
62
|
+
def get_used_images_in_repository(self,repository:str)->list[str]:
|
|
63
|
+
result:list[str]=[]
|
|
64
|
+
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
65
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
66
|
+
result.append(line[0])
|
|
67
|
+
return result
|
|
68
|
+
|
|
69
|
+
@GeneralUtilities.check_arguments
|
|
70
|
+
def custom_registry_is_defined(self,image_name:str)->bool:
|
|
71
|
+
global_docker_image_registries_file=self.get_global_docker_image_registries_file()
|
|
72
|
+
for line in GeneralUtilities.read_nonempty_lines_from_file(global_docker_image_registries_file)[1:]:
|
|
73
|
+
splitted_line=line.split(";")
|
|
74
|
+
if image_name==splitted_line[0]:
|
|
75
|
+
GeneralUtilities.assert_condition(GeneralUtilities.string_has_content(splitted_line[1]),f"No registry defined for image {image_name}.")
|
|
76
|
+
return True
|
|
77
|
+
return False
|
|
78
|
+
|
|
79
|
+
@GeneralUtilities.check_arguments
|
|
80
|
+
def get_tag_for_image(self,repository:str,image_name:str,strict_mode:bool)->str:
|
|
81
|
+
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
82
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
83
|
+
if image_name==line[0]:
|
|
84
|
+
return line[2]
|
|
85
|
+
raise ValueError(f"No tag defined for image \"{image_name}\".")
|
|
86
|
+
|
|
87
|
+
@GeneralUtilities.check_arguments
|
|
88
|
+
def get_registry_address_for_image(self,repository:str,image_name:str)->str:
|
|
89
|
+
"""if image_name==Debian this function returns something like "myregistry.example.com/debian", always without tag."""
|
|
90
|
+
if self.custom_registry_is_defined(image_name):
|
|
91
|
+
#return image from custom registry-address
|
|
92
|
+
global_docker_image_registries_file=self.get_global_docker_image_registries_file()
|
|
93
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(global_docker_image_registries_file)[1:]]:
|
|
94
|
+
if image_name==line[0]:
|
|
95
|
+
return line[1]
|
|
96
|
+
else:
|
|
97
|
+
#return fallback-registry-address
|
|
98
|
+
repository_image_definition_file=self.get_repository_image_definition_file(repository)
|
|
99
|
+
for line in [f.split(";") for f in GeneralUtilities.read_nonempty_lines_from_file(repository_image_definition_file)[1:]]:
|
|
100
|
+
if image_name==line[0]:
|
|
101
|
+
return line[1]
|
|
102
|
+
raise ValueError(f"No registry defined for image \"{image_name}\".")
|
|
103
|
+
|
|
104
|
+
@GeneralUtilities.check_arguments
|
|
105
|
+
def get_registry_address_for_image_with_default_tag(self,repository:str,image_name:str,strict_mode:bool=True)->str:
|
|
106
|
+
return f"{self.get_registry_address_for_image(repository,image_name)}:{self.get_tag_for_image(repository,image_name,strict_mode)}"
|
|
107
|
+
|
|
108
|
+
@GeneralUtilities.check_arguments
|
|
109
|
+
def update_default_tag_for_images_in_image_definitions_file(self,repository:str,search_in_custom_registry_only_if_available:bool)->None:
|
|
110
|
+
file=f"{repository}/.ScriptCollection/OCIImages/ImageDefinition.csv"
|
|
111
|
+
GeneralUtilities.assert_file_exists(file)
|
|
112
|
+
lines=GeneralUtilities.read_nonempty_lines_from_file(file)
|
|
113
|
+
new_lines:list[str]=[]
|
|
114
|
+
#file looks like:
|
|
115
|
+
#ImageName;UpstreamRegistryAddress;DefaultTag
|
|
116
|
+
#Debian;docker.io/library/debian;13.4-slim
|
|
117
|
+
for line in lines:
|
|
118
|
+
if line.startswith("ImageName;"): #header line
|
|
119
|
+
new_lines.append(line)
|
|
120
|
+
continue
|
|
121
|
+
splitted_line=line.split(";")
|
|
122
|
+
image_name=splitted_line[0]
|
|
123
|
+
registry_address=splitted_line[1]
|
|
124
|
+
default_tag=splitted_line[2]
|
|
125
|
+
tag=default_tag
|
|
126
|
+
try:
|
|
127
|
+
addresses_to_check=[]
|
|
128
|
+
if self.custom_registry_is_defined(image_name):
|
|
129
|
+
addresses_to_check.append(self.get_registry_address_for_image(repository,image_name))
|
|
130
|
+
else:
|
|
131
|
+
if search_in_custom_registry_only_if_available:
|
|
132
|
+
raise ValueError(f"No custom registry defined for image {image_name}.")
|
|
133
|
+
if not search_in_custom_registry_only_if_available:
|
|
134
|
+
addresses_to_check.append(registry_address)
|
|
135
|
+
newest_versions:set[Version]={default_tag}
|
|
136
|
+
current_version=Version(default_tag)
|
|
137
|
+
for address in addresses_to_check:
|
|
138
|
+
newest_versions_for_address=self.get_available_versions_of_image_which_are_newer(image_name,address,current_version,VersionEcholon.LatestVersion)
|
|
139
|
+
if newest_versions_for_address is not None:
|
|
140
|
+
newest_versions.update(newest_versions_for_address)
|
|
141
|
+
GeneralUtilities.assert_condition(len(newest_versions)>0,f"Could not find any version for image {image_name} in registry {registry_address}.")
|
|
142
|
+
newest_version=max(newest_versions)
|
|
143
|
+
tag=self.version_to_tag(image_name,newest_version)
|
|
144
|
+
except Exception as e:
|
|
145
|
+
self.__sc.log.log(f"Could not get tag for image {image_name} from registry {registry_address}. Reason: {str(e)}",LogLevel.Warning)
|
|
146
|
+
new_lines.append(f"{image_name};{registry_address};{tag}")
|
|
147
|
+
GeneralUtilities.write_lines_to_file(file,new_lines)
|
|
148
|
+
|
|
149
|
+
@GeneralUtilities.check_arguments
|
|
150
|
+
def get_available_versions_of_image_which_are_newer(self,image_name:str,registry_address:str,current_version:Version,echolon: VersionEcholon)->Version:
|
|
151
|
+
#image_handler=self.get_image_handler(image_name)
|
|
152
|
+
result= None #TODO calculate result using get_available_tags_of_image.
|
|
153
|
+
#TODO if echolon is not none, then use echolon instead of the default echolon of the image-handler.
|
|
154
|
+
#TODO return the versions sorted.
|
|
155
|
+
#TODO if result is empty: return None
|
|
156
|
+
return result
|
|
157
|
+
|
|
158
|
+
@GeneralUtilities.check_arguments
|
|
159
|
+
def get_available_tags_of_image(self,image_name:str,registry_address:str)->list[str]:
|
|
160
|
+
"""registry_address must have one of theese formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian".
|
|
161
|
+
returns something like ["13.2-slim", "13.2", "13.3-slim", "13.3"]."""
|
|
162
|
+
return self.get_image_handler(image_name).get_available_tags_of_image(image_name,registry_address)
|
|
163
|
+
|
|
164
|
+
@GeneralUtilities.check_arguments
|
|
165
|
+
def tag_to_version(self,image_name:str,tag:str)->Version:
|
|
166
|
+
"""registry_address must have one of theese formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian"."""
|
|
167
|
+
return self.get_image_handler(image_name).tag_to_version(image_name, tag)
|
|
168
|
+
|
|
169
|
+
@GeneralUtilities.check_arguments
|
|
170
|
+
def version_to_tag(self,image_name:str,version:Version)->str:
|
|
171
|
+
"""registry_address must have one of theese formats: "myregistry.example.com/debian" or "docker.io/debian" or "docker.io/myuser/debian".
|
|
172
|
+
returns something like "13.3-slim".
|
|
173
|
+
If there are multiple tags available for a certain version then the image-handler decides which one will be returned."""
|
|
174
|
+
return self.get_image_handler(image_name).version_to_tag(image_name,version)
|
|
175
|
+
|
|
176
|
+
@GeneralUtilities.check_arguments
|
|
177
|
+
def get_images_used_in_docker_compose_file(self,docker_compose_file:str)->dict[str,tuple[str,str,str]]:#returns dict[service_name,[image_name,image_address,current_tag]]
|
|
178
|
+
GeneralUtilities.assert_file_exists(docker_compose_file)
|
|
179
|
+
return {}#TODO implement function
|
|
180
|
+
|
|
181
|
+
@GeneralUtilities.check_arguments
|
|
182
|
+
def update_image_in_docker_compose_file(self,docker_compose_file:str)->None:
|
|
183
|
+
for service,service_information in self.get_images_used_in_docker_compose_file(docker_compose_file).items():#pylint:disable=unused-variable
|
|
184
|
+
image_name=service_information[0]
|
|
185
|
+
image_address=service_information[1]
|
|
186
|
+
current_tag=service_information[2]
|
|
187
|
+
image_handler=self.get_image_handler(image_name)
|
|
188
|
+
new_versions_for_address=self.get_available_versions_of_image_which_are_newer(image_name,image_address,self.tag_to_version(image_name,current_tag),image_handler.get_default_echolon_for_update())
|
|
189
|
+
new_tag=self.version_to_tag(image_name,new_versions_for_address)#pylint:disable=unused-variable
|
|
190
|
+
#TODO update tag for service in docker-compose-file to new_tag
|
|
File without changes
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import psutil
|
|
2
|
+
from .GeneralUtilities import GeneralUtilities
|
|
3
|
+
from .ScriptCollectionCore import ScriptCollectionCore
|
|
4
|
+
|
|
5
|
+
# runs multiple processes in parallel and terminate all if at least one is terminated
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ProcessStartInformation:
|
|
9
|
+
workingdirectory: str = None
|
|
10
|
+
program: str = None
|
|
11
|
+
arguments: str = None
|
|
12
|
+
|
|
13
|
+
def __init__(self, workingdirectory: str, program: str, arguments: str):
|
|
14
|
+
self.workingdirectory = workingdirectory
|
|
15
|
+
self.program = program
|
|
16
|
+
self.arguments = arguments
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class ProcessesRunner:
|
|
20
|
+
sc: ScriptCollectionCore
|
|
21
|
+
processes: list[ProcessStartInformation]
|
|
22
|
+
|
|
23
|
+
def __init__(self, processes: list[ProcessStartInformation]):
|
|
24
|
+
self.sc = ScriptCollectionCore()
|
|
25
|
+
self.processes = processes
|
|
26
|
+
|
|
27
|
+
@GeneralUtilities.check_arguments
|
|
28
|
+
def run(self):
|
|
29
|
+
pids: list[int] = list[int]()
|
|
30
|
+
for processstartinfo in self.processes:
|
|
31
|
+
pids.append(self.sc.run_program_async(processstartinfo.program, processstartinfo.argumentss, processstartinfo.workingdirectory))
|
|
32
|
+
enabled = True
|
|
33
|
+
while enabled:
|
|
34
|
+
for pid in pids:
|
|
35
|
+
if not psutil.pid_exists(pid):
|
|
36
|
+
enabled = False
|
|
37
|
+
# one program terminate so exit and terminate all now
|
|
38
|
+
processes = psutil.process_iter()
|
|
39
|
+
for pid in pids:
|
|
40
|
+
if psutil.pid_exists(pid):
|
|
41
|
+
for proc in processes:
|
|
42
|
+
if proc.pid == pid:
|
|
43
|
+
proc.kill()
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from abc import abstractmethod
|
|
2
|
+
from subprocess import Popen
|
|
3
|
+
from .GeneralUtilities import GeneralUtilities
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ProgramRunnerBase:
|
|
7
|
+
|
|
8
|
+
# Return-values program_runner: Pid
|
|
9
|
+
@abstractmethod
|
|
10
|
+
@GeneralUtilities.check_arguments
|
|
11
|
+
def run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> Popen:
|
|
12
|
+
raise NotImplementedError
|
|
13
|
+
|
|
14
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
15
|
+
@abstractmethod
|
|
16
|
+
@GeneralUtilities.check_arguments
|
|
17
|
+
def wait(self, process: Popen, custom_argument: object) -> tuple[int, str, str, int]:
|
|
18
|
+
raise NotImplementedError
|
|
19
|
+
|
|
20
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
21
|
+
@abstractmethod
|
|
22
|
+
@GeneralUtilities.check_arguments
|
|
23
|
+
def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
|
|
24
|
+
raise NotImplementedError
|
|
25
|
+
|
|
26
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
27
|
+
@abstractmethod
|
|
28
|
+
@GeneralUtilities.check_arguments
|
|
29
|
+
def run_program(self, program: str, arguments: str = "", working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
|
|
30
|
+
raise NotImplementedError
|
|
31
|
+
|
|
32
|
+
# Return-values program_runner: Pid
|
|
33
|
+
@abstractmethod
|
|
34
|
+
@GeneralUtilities.check_arguments
|
|
35
|
+
def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> int:
|
|
36
|
+
raise NotImplementedError
|
|
37
|
+
|
|
38
|
+
# Return-values program_runner: Pid
|
|
39
|
+
@abstractmethod
|
|
40
|
+
@GeneralUtilities.check_arguments
|
|
41
|
+
def run_program_async(self, program: str, arguments: str, working_directory: str, custom_argument: object, interactive: bool = False) -> int:
|
|
42
|
+
raise NotImplementedError
|
|
43
|
+
|
|
44
|
+
@abstractmethod
|
|
45
|
+
@GeneralUtilities.check_arguments
|
|
46
|
+
def will_be_executed_locally(self) -> bool:
|
|
47
|
+
raise NotImplementedError
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
from subprocess import PIPE, Popen
|
|
3
|
+
from .GeneralUtilities import GeneralUtilities
|
|
4
|
+
from .ProgramRunnerBase import ProgramRunnerBase
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ProgramRunnerPopen(ProgramRunnerBase):
|
|
8
|
+
|
|
9
|
+
@GeneralUtilities.check_arguments
|
|
10
|
+
def run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> Popen:
|
|
11
|
+
arguments_for_process = [program]
|
|
12
|
+
arguments_for_process.extend(arguments_as_array)
|
|
13
|
+
# "shell=True" is not allowed because it is not recommended and also something like
|
|
14
|
+
# "ScriptCollectionCore().run_program('curl', 'https://example.com/dataset?id=1&format=json')"
|
|
15
|
+
# would not be possible anymore because the ampersand will be treated as shell-command.
|
|
16
|
+
try:
|
|
17
|
+
if interactive:
|
|
18
|
+
result = Popen(arguments_for_process, cwd=working_directory, stdout=PIPE, stderr=PIPE, shell=False, text=True, stdin=sys.stdin) # pylint: disable=consider-using-with
|
|
19
|
+
else:
|
|
20
|
+
result = Popen(arguments_for_process, cwd=working_directory, stdout=PIPE, stderr=PIPE, shell=False, text=True) # pylint: disable=consider-using-with
|
|
21
|
+
except FileNotFoundError as fileNotFoundError:
|
|
22
|
+
raise FileNotFoundError(f"Starting '{program}' in '{working_directory}' resulted in a FileNotFoundError: '{str(fileNotFoundError)}'")
|
|
23
|
+
except NotADirectoryError as notADirectoryError:
|
|
24
|
+
raise NotADirectoryError(f"Starting '{program}' in '{working_directory}' resulted in a NotADirectoryError: '{str(notADirectoryError)}'")
|
|
25
|
+
return result
|
|
26
|
+
|
|
27
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
28
|
+
@GeneralUtilities.check_arguments
|
|
29
|
+
def wait(self, process: Popen, custom_argument: object) -> tuple[int, str, str, int]:
|
|
30
|
+
pid = process.pid
|
|
31
|
+
stdout, stderr = process.communicate()
|
|
32
|
+
exit_code = process.wait()
|
|
33
|
+
stdout = GeneralUtilities.bytes_to_string(stdout).replace('\r', '')
|
|
34
|
+
stderr = GeneralUtilities.bytes_to_string(stderr).replace('\r', '')
|
|
35
|
+
result = (exit_code, stdout, stderr, pid)
|
|
36
|
+
return result
|
|
37
|
+
|
|
38
|
+
@GeneralUtilities.check_arguments
|
|
39
|
+
def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
|
|
40
|
+
process: Popen = self.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
|
|
41
|
+
return self.wait(process, custom_argument)
|
|
42
|
+
|
|
43
|
+
@GeneralUtilities.check_arguments
|
|
44
|
+
def run_program(self, program: str, arguments: str = "", working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
|
|
45
|
+
return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, custom_argument)
|
|
46
|
+
|
|
47
|
+
@GeneralUtilities.check_arguments
|
|
48
|
+
def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> int:
|
|
49
|
+
return self.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive).pid
|
|
50
|
+
|
|
51
|
+
@GeneralUtilities.check_arguments
|
|
52
|
+
def run_program_async(self, program: str, arguments: str = "", working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> int:
|
|
53
|
+
return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, custom_argument, interactive)
|
|
54
|
+
|
|
55
|
+
@GeneralUtilities.check_arguments
|
|
56
|
+
def will_be_executed_locally(self) -> bool:
|
|
57
|
+
return True
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
from subprocess import Popen
|
|
2
|
+
from .GeneralUtilities import GeneralUtilities
|
|
3
|
+
from .ProgramRunnerBase import ProgramRunnerBase
|
|
4
|
+
from .ScriptCollectionCore import ScriptCollectionCore
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class SudoPopenReader:
|
|
8
|
+
content: bytes = None
|
|
9
|
+
|
|
10
|
+
def __init__(self, content: bytes):
|
|
11
|
+
self.content = content
|
|
12
|
+
|
|
13
|
+
def readable(self) -> bool:
|
|
14
|
+
return True
|
|
15
|
+
|
|
16
|
+
def read(self) -> bytes:
|
|
17
|
+
return self.content
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class SudoPopen:
|
|
21
|
+
returncode: int = None
|
|
22
|
+
stdout_str: str = None
|
|
23
|
+
stderr_str: str = None
|
|
24
|
+
pid: int = None
|
|
25
|
+
stdout: bytes = None
|
|
26
|
+
stderr: bytes = None
|
|
27
|
+
|
|
28
|
+
def __init__(self, exitcode: int, stdout: str, stderr: str, pid: int):
|
|
29
|
+
self.returncode: int = exitcode
|
|
30
|
+
self.stdout_str: str = stdout
|
|
31
|
+
self.stdout = str.encode(self.stdout_str)
|
|
32
|
+
self.stderr_str: str = stderr
|
|
33
|
+
self.stderr = str.encode(self.stderr_str)
|
|
34
|
+
self.pid = pid
|
|
35
|
+
|
|
36
|
+
def communicate(self):
|
|
37
|
+
return (self.stdout, self.stderr)
|
|
38
|
+
|
|
39
|
+
def wait(self):
|
|
40
|
+
return self.returncode
|
|
41
|
+
|
|
42
|
+
def poll(self) -> object:
|
|
43
|
+
return self.returncode
|
|
44
|
+
|
|
45
|
+
def __enter__(self):
|
|
46
|
+
return self
|
|
47
|
+
|
|
48
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
|
49
|
+
pass
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class ProgramRunnerSudo(ProgramRunnerBase):
|
|
53
|
+
__sc: ScriptCollectionCore
|
|
54
|
+
__password: str
|
|
55
|
+
|
|
56
|
+
@GeneralUtilities.check_arguments
|
|
57
|
+
def __init__(self,user_password:str):
|
|
58
|
+
GeneralUtilities.assert_condition(GeneralUtilities.current_system_is_linux(), "SudoRunner can only be only executed on Linux.")
|
|
59
|
+
self.__sc = ScriptCollectionCore()
|
|
60
|
+
self.__password = user_password
|
|
61
|
+
|
|
62
|
+
@GeneralUtilities.check_arguments
|
|
63
|
+
def will_be_executed_locally(self) -> bool:
|
|
64
|
+
return True
|
|
65
|
+
|
|
66
|
+
@GeneralUtilities.check_arguments
|
|
67
|
+
def run_program_internal(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None) -> tuple[int, str, str, int]:
|
|
68
|
+
argument = program+" " + ' '.join(GeneralUtilities.args_array_surround_with_quotes_if_required(arguments_as_array))
|
|
69
|
+
argument = f"echo {self.__password} | sudo -k -S {argument}" # TODO maybe add "exit" somewhere before argument or before sudo to correctly return the exit-code"
|
|
70
|
+
result = self.__sc.run_program_argsasarray("sh", ["-c", argument], working_directory)
|
|
71
|
+
return result
|
|
72
|
+
|
|
73
|
+
@GeneralUtilities.check_arguments
|
|
74
|
+
def run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> Popen:
|
|
75
|
+
if interactive:
|
|
76
|
+
raise ValueError("Interactive execution is not supported in Sudo-runner")
|
|
77
|
+
r: tuple[int, str, str, int] = self.run_program_internal(program, arguments_as_array, working_directory, custom_argument)
|
|
78
|
+
popen: SudoPopen = SudoPopen(r[0], r[1], r[2], r[3])
|
|
79
|
+
return popen
|
|
80
|
+
|
|
81
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
82
|
+
@GeneralUtilities.check_arguments
|
|
83
|
+
def wait(self, process: Popen, custom_argument: object) -> tuple[int, str, str, int]:
|
|
84
|
+
raise ValueError("Wait is not supported in Sudo-runner")
|
|
85
|
+
|
|
86
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
87
|
+
@GeneralUtilities.check_arguments
|
|
88
|
+
def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
|
|
89
|
+
if interactive:
|
|
90
|
+
raise ValueError("Interactive execution is not supported in Sudo-runner")
|
|
91
|
+
return self.run_program_internal(program, arguments_as_array, working_directory, custom_argument)
|
|
92
|
+
|
|
93
|
+
# Return-values program_runner: Exitcode, StdOut, StdErr, Pid
|
|
94
|
+
@GeneralUtilities.check_arguments
|
|
95
|
+
def run_program(self, program: str, arguments: str = "", working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> tuple[int, str, str, int]:
|
|
96
|
+
if interactive:
|
|
97
|
+
raise ValueError("Interactive execution is not supported in Sudo-runner")
|
|
98
|
+
return self.run_program_internal(program, arguments.split(" "), working_directory, custom_argument)
|
|
99
|
+
|
|
100
|
+
# Return-values program_runner: Pid
|
|
101
|
+
@GeneralUtilities.check_arguments
|
|
102
|
+
def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, custom_argument: object = None, interactive: bool = False) -> int:
|
|
103
|
+
raise ValueError("Async execution is not supported in Sudo-runner")
|
|
104
|
+
|
|
105
|
+
# Return-values program_runner: Pid
|
|
106
|
+
@GeneralUtilities.check_arguments
|
|
107
|
+
def run_program_async(self, program: str, arguments: str, working_directory: str, custom_argument: object, interactive: bool = False) -> int:
|
|
108
|
+
raise ValueError("Async execution is not supported in Sudo-runner")
|