scriptcollection 4.2.81__py3-none-any.whl

ScriptCollection/AnionBuildPlatform.py

@@ -0,0 +1,199 @@
+import argparse
+import os
+from .TFCPS.TFCPS_CodeUnit_BuildCodeUnits import TFCPS_CodeUnit_BuildCodeUnits
+from .SCLog import LogLevel
+from .GeneralUtilities import GeneralUtilities
+from .ScriptCollectionCore import ScriptCollectionCore
+
+class AnionBuildPlatformConfiguration:
+    build_repositories_folder:str
+    additional_arguments_file:str
+    verbosity:LogLevel
+    source_branch:str#other/next-release
+    common_remote_name:str
+    update_dependencies:bool
+    lazy_mode:bool
+
+    def __init__(self,
+                 build_repositories_folder:str,
+                 additional_arguments_file:str,
+                 verbosity:LogLevel,
+                 source_branch:str,
+                 common_remote_name:str,
+                 update_dependencies:bool,
+                 lazy_mode:bool):
+        self.build_repositories_folder=build_repositories_folder
+        self.additional_arguments_file=additional_arguments_file
+        self.verbosity=verbosity
+        self.source_branch=source_branch
+        self.common_remote_name=common_remote_name
+        self.update_dependencies=update_dependencies
+        self.lazy_mode=lazy_mode
+
+class AnionBuildPlatform:
+
+    __configuration: AnionBuildPlatformConfiguration
+    __sc:ScriptCollectionCore
+
+    def __init__(self, configuration: AnionBuildPlatformConfiguration):
+        self.__configuration = configuration
+        self.__sc = ScriptCollectionCore()
+        self.__sc.log.loglevel=configuration.verbosity
+
+    @GeneralUtilities.check_arguments
+    def run(self) -> None:
+        #TODO refactor this
+        # ensure that if
+        # - main is up to date and
+        # - all dependencies are up to date and
+        # - other/next-release==main and
+        # - main is buildable and
+        # - the latest main is already merged in stable
+        # then this function does nothing
+
+        # Checkout source branch
+        build_repo_folder:str=self.__configuration.build_repositories_folder
+        GeneralUtilities.assert_condition(build_repo_folder.endswith("Build"),f"buildrepositoriesfolder {build_repo_folder} must end with 'Build'")
+        self.__sc.assert_is_git_repository(build_repo_folder)
+        product_name=os.path.basename(build_repo_folder)[:-len("Build")]
+        repository:str=os.path.join(build_repo_folder,"Submodules",product_name)
+        self.__sc.assert_is_git_repository(repository)
+        reference_repo:str=os.path.join(build_repo_folder,"Submodules",product_name+"Reference")
+        self.__sc.assert_is_git_repository(reference_repo)
+        self.__sc.git_commit(reference_repo,"Updated changes")
+        self.__sc.git_checkout(repository,self.__configuration.source_branch)
+        self.__sc.git_commit(build_repo_folder,"Updated changes")
+
+        # Pull changes from remote
+        self.__sc.git_fetch(repository)
+        self.__sc.git_merge(repository,self.__configuration.common_remote_name+"/"+self.__configuration.source_branch,self.__configuration.source_branch,fastforward=True)#TODO check if is anchestor and throw exception if nor
+        self.__sc.git_commit(build_repo_folder,"Updated changes")
+
+        # Added changelog entry and build to verify buildability and to update versions etc.
+        if self.__configuration.lazy_mode:
+            self.__sc.run_program("sccreatechangelogentry","-m Update.",repository)
+            arguments:str="bb --"
+            if self.__configuration.verbosity == LogLevel.Debug:
+                arguments+=f" --verbosity {self.__configuration.verbosity.value}"
+            self.__sc.run_program("task",arguments,repository)
+            self.__sc.git_commit(repository,"update")
+
+        # Update dependencies
+        if self.__configuration.update_dependencies:
+            self.__update_dependencies(product_name)
+        
+        # Do release
+        scripts_folder:str=os.path.join(build_repo_folder,"Scripts","CreateRelease")
+
+        merge_to_main_arguments=""
+        #if self.__configuration.project_to_build is not None:
+        #    merge_to_main_arguments+=f" --productname {self.__configuration.project_to_build}"
+        if self.__configuration.source_branch is not None:
+            merge_to_main_arguments+=f" --mergesourcebranch {self.__configuration.source_branch}"
+        #if self.__configuration.additional_arguments_file is not None:
+        #    merge_to_main_arguments+=f" --additionalargumentsfile {self.__configuration.additional_arguments_file}"
+        #if self.__configuration.main_branch is not None:
+        #    merge_to_main_arguments+=f" --mainbranch {self.__configuration.main_branch}"
+        #if self.__configuration.common_remote_name is not None:
+        #    merge_to_main_arguments+=f" --commonremotename {self.__configuration.common_remote_name}"
+        if self.__configuration.verbosity is not None:
+            merge_to_main_arguments+=f" --verbosity {self.__configuration.verbosity.value}"
+        self.__sc.run_program(GeneralUtilities.get_python_executable(),f"MergeToMain.py{merge_to_main_arguments}",scripts_folder,print_live_output=True)
+
+        merge_to_stable_arguments=""
+        #if self.__configuration.project_to_build is not None:
+        #    merge_to_stable_arguments+=f" --productname {self.__configuration.project_to_build}"
+        #if self.__configuration.additional_arguments_file is not None:
+        #    merge_to_stable_arguments+=f" --additionalargumentsfile {self.__configuration.additional_arguments_file}"
+        #if self.__configuration.source_branch is not None:
+        #    merge_to_stable_arguments+=f" --sourcebranch {self.__configuration.source_branch}"
+        #if self.__configuration.main_branch is not None:
+        #    merge_to_stable_arguments+=f" --targetbranch {self.__configuration.main_branch}"
+        #if self.__configuration.reference_repo is not None:
+        #    merge_to_stable_arguments+=f" --referencerepo {self.__configuration.referencerepo}"
+        #if self.__configuration.common_remote_name is not None:
+        #    merge_to_stable_arguments+=f" --commonremotename {self.__configuration.common_remote_name}"
+        #if self.__configuration.build_repo_main_branch_name is not None:
+        #    merge_to_stable_arguments+=f" --buildrepomainbranchname {self.__configuration.build_repo_main_branch_name}"
+        #if self.__configuration.reference_repo_main_branch_name is not None:
+        #    merge_to_stable_arguments+=f" --referencerepomainbranchname {self.__configuration.reference_repo_main_branch_name}"
+        #if self.__configuration.reference_remote_name is not None:
+        #    merge_to_stable_arguments+=f" --referenceremotename {self.__configuration.reference_remote_name}"
+        #if self.__configuration.build_repo_remote_name is not None:
+        #    merge_to_stable_arguments+=f" --buildreporemotename {self.__configuration.build_repo_remote_name}"
+        #if self.__configuration.artifacts_target_folder is not None:
+        #    merge_to_stable_arguments+=f" --artifactstargetfolder {self.__configuration.artifacts_target_folder}"
+        #if self.__configuration.common_remote_url is not None:
+        #    merge_to_stable_arguments+=f" --commonremoteurl {self.__configuration.common_remote_url}"
+        if self.__configuration.verbosity == LogLevel.Debug:
+            merge_to_stable_arguments+=f" --verbosity {self.__configuration.verbosity.value}"
+        self.__sc.run_program(GeneralUtilities.get_python_executable(),f"MergeToStable.py{merge_to_stable_arguments}",scripts_folder,print_live_output=True)
+
+        #prepare for next-release
+        self.__sc.git_checkout(repository,self.__configuration.source_branch)
+
+    @GeneralUtilities.check_arguments
+    def __update_dependencies(self,product_name:str) -> None:
+        self.__sc.log.log("Update dependencies...")
+        repository:str=os.path.join(self.__configuration.build_repositories_folder,"Submodules",product_name)
+        t:TFCPS_CodeUnit_BuildCodeUnits=TFCPS_CodeUnit_BuildCodeUnits(repository,self.__sc.log.loglevel,"QualityCheck",None,True,False)
+        t.update_dependencies()
+
+
+
+class TFCPS_AnionBuildPlatform_CLI:
+
+    @staticmethod
+    @GeneralUtilities.check_arguments
+    def get_with_overwritable_defaults(default_project_to_build:str=None,default_loglevel:LogLevel=None,default_additionalargumentsfile:str=None,default_build_repositories_folder:str=None,default_source_branch:str=None,default_main_branch:str=None,default_remote_name:str=None)->AnionBuildPlatform:
+        parser = argparse.ArgumentParser()
+        verbosity_values = ", ".join(f"{lvl.value}={lvl.name}" for lvl in LogLevel)
+        parser.add_argument('-b', '--buildrepositoriesfolder', required=False,default=None)
+        parser.add_argument('-p', '--projecttobuild', required=False, default=None)
+        parser.add_argument('-a', '--additionalargumentsfile', required=False, default=None)
+        parser.add_argument('-v', '--verbosity', required=False,  help=f"Sets the loglevel. Possible values: {verbosity_values}")
+        parser.add_argument('-s', '--sourcebranch', required=False)#other/next-release
+        parser.add_argument('-m', '--mainbranch', required=False)#main
+        parser.add_argument('-r', '--defaultremotename', required=False)#origin
+        parser.add_argument('-u', '--updatedependencies', required=False, action='store_true', default=False)
+        parser.add_argument('-l', '--lazymode', required=False, action='store_true', default=False)
+        args=parser.parse_args()
+
+        if args.projecttobuild is not None: 
+            default_project_to_build=args.projecttobuild
+
+        if args.buildrepositoriesfolder is not None:
+            default_build_repositories_folder=args.buildrepositoriesfolder
+
+        scripts_folder=os.getcwd()
+        
+        if default_build_repositories_folder is None:
+            parent_parent_folder=GeneralUtilities.resolve_relative_path("../..",scripts_folder)
+            if os.path.basename(parent_parent_folder).endswith("Build"):
+                default_build_repositories_folder=os.path.dirname(parent_parent_folder)
+        GeneralUtilities.assert_not_null(default_build_repositories_folder,"buildrepositoriesfolder is not set")
+        
+        if default_project_to_build is None:
+            parent_parent_folder=GeneralUtilities.resolve_relative_path("../..",scripts_folder)
+            if os.path.basename(parent_parent_folder).endswith("Build"):
+                default_project_to_build=os.path.basename(parent_parent_folder)[:-len("Build")]
+        GeneralUtilities.assert_not_null(default_project_to_build,"projecttobuild is not set")
+
+        if args.verbosity is not None:
+            default_loglevel=LogLevel(int( args.verbosity))
+        GeneralUtilities.assert_not_null(default_loglevel,"verbosity is not set")
+
+        if args.additionalargumentsfile is not None:
+            default_additionalargumentsfile=args.additionalargumentsfile
+
+        if args.sourcebranch is not None:
+            default_source_branch=args.sourcebranch
+        GeneralUtilities.assert_not_null(default_source_branch,"sourcebranch is not set")
+
+        if args.defaultremotename is not None:
+            default_remote_name=args.defaultremotename
+        GeneralUtilities.assert_not_null(default_remote_name,"defaultremotename is not set")
+
+        config:AnionBuildPlatformConfiguration=AnionBuildPlatformConfiguration(default_build_repositories_folder,default_additionalargumentsfile,default_loglevel,default_source_branch,default_remote_name,args.updatedependencies,args.lazymode)
+        tFCPS_MergeToMain:AnionBuildPlatform=AnionBuildPlatform(config)
+        return tFCPS_MergeToMain

ScriptCollection/CertificateUpdater.py

@@ -0,0 +1,149 @@
+import os
+from pathlib import Path
+from datetime import datetime, timedelta
+import traceback
+from shutil import copyfile
+import argparse
+from .GeneralUtilities import GeneralUtilities
+from .ScriptCollectionCore import ScriptCollectionCore
+
+
+class CertificateUpdater:
+    maximal_age_of_certificates_in_days: int = None
+    __domains: list[str] = None
+    __email: str = None
+    __current_folder: str = None
+    __last_update_timestamp_file: str = None
+    __repository_folder: str = None
+    __letsencrypt_folder: str = None
+    __letsencrypt_live_folder: str = None
+    __letsencrypt_archive_folder: str = None
+    __log_folder: str = None
+    __sc: ScriptCollectionCore = None
+    __arguments: ScriptCollectionCore = None
+    __certbot_image_address_with_tag:str=None
+
+    def __init__(self, domains: list[str], email: str, current_file: str, arguments: list[str],certbot_image_address_with_tag:str=None):
+        self.__sc = ScriptCollectionCore()
+        self.maximal_age_of_certificates_in_days = 15
+        self.__domains = domains
+        self.__email = email
+        self.__current_folder = current_file
+        self.__arguments = arguments
+        self.__last_update_timestamp_file = GeneralUtilities.resolve_relative_path("./LastCertificateUpdate.csv", self.__current_folder)
+        self.__repository_folder = GeneralUtilities.resolve_relative_path("../..", self.__current_folder)
+        self.__sc.assert_is_git_repository(self.__repository_folder)
+        self.__letsencrypt_folder = f"{ self.__repository_folder}/Configuration/Volumes/letsencrypt"
+        self.__letsencrypt_live_folder = os.path.join(self.__letsencrypt_folder, "live")
+        self.__letsencrypt_archive_folder = os.path.join(self.__letsencrypt_folder, "archive")
+        self.__log_folder = GeneralUtilities.resolve_relative_path("Logs/Overhead", self.__repository_folder)
+        if certbot_image_address_with_tag is None:
+            certbot_image_address_with_tag="docker.io/certbot/certbot:latest"
+        self.__certbot_image_address_with_tag=certbot_image_address_with_tag
+
+    @GeneralUtilities.check_arguments
+    def __get_latest_index_by_domain(self, domain: str) -> int:
+        result = self.__get_latest_index_by_filelist(GeneralUtilities.get_all_files_of_folder(os.path.join(self.__letsencrypt_archive_folder, domain)))
+        GeneralUtilities.write_message_to_stdout(f"Debug: Latest found existing number for domain {domain}: {result}")
+        return result
+
+    @GeneralUtilities.check_arguments
+    def __get_latest_index_by_filelist(self, filenames: list[str]) -> int:
+        filenames = [Path(os.path.basename(file)).stem for file in filenames]
+        filenames = [file for file in filenames if file.startswith("privkey")]
+        numbers = [int(file[len("privkey"):]) for file in filenames]
+        result = max(numbers)
+        return result
+
+    @GeneralUtilities.check_arguments
+    def __replace_symlink_by_file(self, domain: str, filename: str, index: int) -> None:
+        # ".../live/example.com/cert.pem" is a symlink but should replaced by a copy of ".../archive/example.com/cert.42pem"
+        archive_file = os.path.join(self.__letsencrypt_archive_folder, domain, filename+str(index)+".pem")
+        live_folder = os.path.join(self.__letsencrypt_live_folder, domain)
+        live_filename = filename+".pem"
+        live_file = os.path.join(live_folder, live_filename)
+        self.__sc.run_program("rm", live_filename, live_folder, throw_exception_if_exitcode_is_not_zero=True)
+        copyfile(archive_file, live_file)
+
+    @GeneralUtilities.check_arguments
+    def __replace_file_by_symlink(self, domain: str, filename: str, index: int) -> None:
+        # new ".../live/example.com/cert.pem" is a file but should replaced by a symlink which points to ".../archive/example.com/cert42.pem"
+        live_folder = os.path.join(self.__letsencrypt_live_folder, domain)
+        live_filename = filename+".pem"
+        self.__sc.run_program("rm", live_filename, live_folder, throw_exception_if_exitcode_is_not_zero=True)
+        self.__sc.run_program("ln", f"-s ../../archive/{domain}/{filename+str(index)}.pem {live_filename}", live_folder, throw_exception_if_exitcode_is_not_zero=True)
+
+    @GeneralUtilities.check_arguments
+    def __replace_symlinks_by_files(self, domain):
+        index = self.__get_latest_index_by_domain(domain)
+        self.__replace_symlink_by_file(domain, "cert", index)
+        self.__replace_symlink_by_file(domain, "chain", index)
+        self.__replace_symlink_by_file(domain, "fullchain", index)
+        self.__replace_symlink_by_file(domain, "privkey", index)
+
+    @GeneralUtilities.check_arguments
+    def __replace_files_by_symlinks(self, domain):
+        index = self.__get_latest_index_by_domain(domain)
+        self.__replace_file_by_symlink(domain, "cert", index)
+        self.__replace_file_by_symlink(domain, "chain", index)
+        self.__replace_file_by_symlink(domain, "fullchain", index)
+        self.__replace_file_by_symlink(domain, "privkey", index)
+
+    @GeneralUtilities.check_arguments
+    def __update_certificates(self) -> None:
+        self.__sc.git_commit(self.__repository_folder, "Saved current changes")
+        error_occurred = False
+        for domain in self.__domains:
+            certbot_container_name = "certificate_updater"
+            try:
+                GeneralUtilities.write_message_to_stdout(GeneralUtilities.get_line())
+                GeneralUtilities.write_message_to_stdout(f"Process domain {domain}")
+                self.__sc.run_program("docker", f"container rm {certbot_container_name}", self.__current_folder, throw_exception_if_exitcode_is_not_zero=False)
+                certificate_for_domain_already_exists = os.path.isfile(f"{self.__letsencrypt_folder}/renewal/{domain}.conf")
+                if certificate_for_domain_already_exists:
+                    GeneralUtilities.write_message_to_stdout(f"Update certificate for domain {domain}")
+                    self.__replace_files_by_symlinks(domain)
+                else:
+                    GeneralUtilities.write_message_to_stdout(f"Create certificate for domain {domain}")
+                dockerargument = f"run --rm --name {certbot_container_name} --volume {self.__letsencrypt_folder}:/etc/letsencrypt"
+                dockerargument = dockerargument + f" --volume {self.__log_folder}:/var/log/letsencrypt -p 80:80 "+self.__certbot_image_address_with_tag
+                certbotargument = f"--standalone --email {self.__email} --agree-tos --force-renewal --rsa-key-size 4096 --non-interactive --no-eff-email --domain {domain}"
+                if (certificate_for_domain_already_exists):
+                    self.__sc.run_program("docker", f"{dockerargument} certonly --no-random-sleep-on-renew {certbotargument}", self.__current_folder)
+                    self.__replace_symlinks_by_files(domain)
+                    GeneralUtilities.write_message_to_stdout(f"Certificates updated for {domain}.")
+                else:
+                    self.__sc.run_program("docker", f"{dockerargument} certonly --cert-name {domain} {certbotargument}", self.__current_folder)
+            except Exception as exception:
+                error_occurred = True
+                GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback, "Error while updating certificate")
+        self.__sc.git_commit(self.__repository_folder, "Executed certificate-update-process")
+        GeneralUtilities.write_message_to_stdout("Finished certificate-update-process")
+        if error_occurred:
+            raise ValueError("Certificates for at least one domain could not be added/updated.")
+
+    @GeneralUtilities.check_arguments
+    def __get_last_certificate_update_date(self) -> datetime:
+        if os.path.exists(self.__last_update_timestamp_file):
+            filecontent = GeneralUtilities.read_text_from_file(self.__last_update_timestamp_file)
+            return GeneralUtilities.string_to_datetime(filecontent.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string))
+        else:
+            return datetime(year=1970, month=1, day=1)
+
+    @GeneralUtilities.check_arguments
+    def __set_last_certificate_update_date(self, moment: datetime) -> datetime:
+        GeneralUtilities.ensure_file_exists(self.__last_update_timestamp_file)
+        GeneralUtilities.write_text_to_file(self.__last_update_timestamp_file, GeneralUtilities.datetime_to_string(moment))
+
+    @GeneralUtilities.check_arguments
+    def update_certificates_if_required(self) -> None:
+        parser = argparse.ArgumentParser(description="Updated lets-encrypt-certificates")
+        parser.add_argument('-f', '--force', action='store_true', required=False, default=False)
+        args = parser.parse_args(self.__arguments)
+        now = datetime.now()
+        if (self.__get_last_certificate_update_date()+timedelta(days=self.maximal_age_of_certificates_in_days)) < now or args.force:
+            GeneralUtilities.write_message_to_stdout(f"Update certificates...")
+            self.__update_certificates()
+            self.__set_last_certificate_update_date(now)
+        else:
+            GeneralUtilities.write_message_to_stdout(f"Certificates are already up to date.")