ScriptCollection 3.5.16__py3-none-any.whl → 4.0.78__py3-none-any.whl

ScriptCollection/AnionBuildPlatform.py

@@ -0,0 +1,206 @@
+import argparse
+import os
+from .TFCPS.TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
+from .TFCPS.TFCPS_CodeUnit_BuildCodeUnits import TFCPS_CodeUnit_BuildCodeUnits
+from .TFCPS.TFCPS_Tools_General import TFCPS_Tools_General
+from .SCLog import LogLevel
+from .GeneralUtilities import GeneralUtilities
+from .ScriptCollectionCore import ScriptCollectionCore
+
+class AnionBuildPlatformConfiguration:
+    build_repositories_folder:str
+    additional_arguments_file:str
+    verbosity:LogLevel
+    source_branch:str
+    common_remote_name:str
+    update_dependencies:bool
+
+    def __init__(self,
+                 build_repositories_folder:str,
+                 additional_arguments_file:str,
+                 verbosity:LogLevel,
+                 source_branch:str,
+                 common_remote_name:str,
+                 update_dependencies:bool):
+        self.build_repositories_folder=build_repositories_folder
+        self.additional_arguments_file=additional_arguments_file
+        self.verbosity=verbosity
+        self.source_branch=source_branch
+        self.common_remote_name=common_remote_name
+        self.update_dependencies=update_dependencies
+
+class AnionBuildPlatform:
+
+    __configuration: AnionBuildPlatformConfiguration
+    __sc:ScriptCollectionCore
+    __tFCPS_Tools_General:TFCPS_Tools_General
+
+    def __init__(self, configuration: AnionBuildPlatformConfiguration):
+        self.__configuration = configuration
+        self.__sc = ScriptCollectionCore()
+        self.__sc.log.loglevel=configuration.verbosity
+        self.__tFCPS_Tools_General=TFCPS_Tools_General(self.__sc)
+
+    def run(self) -> None:
+        # Checkout source branch
+        build_repo_folder:str=self.__configuration.build_repositories_folder
+        GeneralUtilities.assert_condition(build_repo_folder.endswith("Build"),f"buildrepositoriesfolder {build_repo_folder} must end with 'Build'")
+        self.__sc.assert_is_git_repository(build_repo_folder)
+        product_name=os.path.basename(build_repo_folder)[:-len("Build")]
+        repository:str=os.path.join(build_repo_folder,"Submodules",product_name)
+        self.__sc.assert_is_git_repository(repository)
+        self.__sc.git_commit(build_repo_folder,"Updated changes")
+        self.__sc.git_checkout(repository,self.__configuration.source_branch)
+
+        # Pull changes from remote
+        self.__sc.git_fetch(repository)
+        self.__sc.git_merge(repository,self.__configuration.common_remote_name+"/"+self.__configuration.source_branch,self.__configuration.source_branch,fastforward=True)#TODO check if is anchestor and throw exception if nor
+        self.__sc.git_commit(build_repo_folder,"Updated changes")
+
+        #Update dependencies
+        if self.__configuration.update_dependencies:
+            self.__update_dependencies(product_name)
+        
+        #Do release
+        scripts_folder:str=os.path.join(build_repo_folder,"Scripts","CreateRelease")
+
+        merge_to_main_arguments=""
+        #if self.__configuration.project_to_build is not None:
+        #    merge_to_main_arguments+=f" --productname {self.__configuration.project_to_build}"
+        if self.__configuration.source_branch is not None:
+            merge_to_main_arguments+=f" --mergesourcebranch {self.__configuration.source_branch}"
+        #if self.__configuration.additional_arguments_file is not None:
+        #    merge_to_main_arguments+=f" --additionalargumentsfile {self.__configuration.additional_arguments_file}"
+        #if self.__configuration.main_branch is not None:
+        #    merge_to_main_arguments+=f" --mainbranch {self.__configuration.main_branch}"
+        #if self.__configuration.common_remote_name is not None:
+        #    merge_to_main_arguments+=f" --commonremotename {self.__configuration.common_remote_name}"
+        if self.__configuration.verbosity is not None:
+            merge_to_main_arguments+=f" --verbosity {self.__configuration.verbosity.value}"
+        self.__sc.run_program("python",f"MergeToMain.py{merge_to_main_arguments}",scripts_folder,print_live_output=True)
+
+        merge_to_stable_arguments=""
+        #if self.__configuration.project_to_build is not None:
+        #    merge_to_stable_arguments+=f" --productname {self.__configuration.project_to_build}"
+        #if self.__configuration.additional_arguments_file is not None:
+        #    merge_to_stable_arguments+=f" --additionalargumentsfile {self.__configuration.additional_arguments_file}"
+        #if self.__configuration.source_branch is not None:
+        #    merge_to_stable_arguments+=f" --sourcebranch {self.__configuration.source_branch}"
+        #if self.__configuration.main_branch is not None:
+        #    merge_to_stable_arguments+=f" --targetbranch {self.__configuration.main_branch}"
+        #if self.__configuration.reference_repo is not None:
+        #    merge_to_stable_arguments+=f" --referencerepo {self.__configuration.referencerepo}"
+        #if self.__configuration.common_remote_name is not None:
+        #    merge_to_stable_arguments+=f" --commonremotename {self.__configuration.common_remote_name}"
+        #if self.__configuration.build_repo_main_branch_name is not None:
+        #    merge_to_stable_arguments+=f" --buildrepomainbranchname {self.__configuration.build_repo_main_branch_name}"
+        #if self.__configuration.reference_repo_main_branch_name is not None:
+        #    merge_to_stable_arguments+=f" --referencerepomainbranchname {self.__configuration.reference_repo_main_branch_name}"
+        #if self.__configuration.reference_remote_name is not None:
+        #    merge_to_stable_arguments+=f" --referenceremotename {self.__configuration.reference_remote_name}"
+        #if self.__configuration.build_repo_remote_name is not None:
+        #    merge_to_stable_arguments+=f" --buildreporemotename {self.__configuration.build_repo_remote_name}"
+        #if self.__configuration.artifacts_target_folder is not None:
+        #    merge_to_stable_arguments+=f" --artifactstargetfolder {self.__configuration.artifacts_target_folder}"
+        #if self.__configuration.common_remote_url is not None:
+        #    merge_to_stable_arguments+=f" --commonremoteurl {self.__configuration.common_remote_url}"
+        if self.__configuration.verbosity is not None:
+            merge_to_stable_arguments+=f" --verbosity {self.__configuration.verbosity.value}"
+        self.__sc.run_program("python",f"MergeToStable.py{merge_to_stable_arguments}",scripts_folder,print_live_output=True)
+
+        #prepare for next-release
+        self.__sc.git_checkout(repository,self.__configuration.source_branch)
+
+    def __update_dependencies(self,product_name:str) -> None:
+        self.__sc.log.log("Update dependencies...")
+        repository:str=os.path.join(self.__configuration.build_repositories_folder,"Submodules",product_name)
+        self.__sc.assert_is_git_repository(repository)
+        self.__sc.assert_no_uncommitted_changes(repository)
+        if os.path.isfile(os.path.join(repository,"Other","Scripts","UpdateDependencies.py")):
+            self.__sc.run_program("python","UpdateDependencies.py",os.path.join(repository,"Other","Scripts"))
+        codeunits:list[str]=self.__tFCPS_Tools_General.get_codeunits(repository)   
+        for codeunit_name in codeunits:
+            self.__sc.log.log(f"Update dependencies of codeunit {codeunit_name}...")
+            codeunit_folder=os.path.join(repository,codeunit_name)
+            tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(codeunit_folder,self.__sc.log.loglevel,"QualityCheck",None,True,False)
+            tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#ensure requirements for updating are there (some programming types needs this)
+            if self.__tFCPS_Tools_General.codeunit_has_updatable_dependencies(os.path.join(codeunit_folder,f"{codeunit_name}.codeunit.xml")):
+                self.__sc.run_program("python","UpdateDependencies.py",os.path.join(codeunit_folder,"Other"))
+            tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()#check if codeunit is still buildable
+
+        if self.__sc.git_repository_has_uncommitted_changes(repository):
+            changelog_folder = os.path.join(repository, "Other", "Resources", "Changelog")
+            project_version:str=self.__tFCPS_Tools_General.get_version_of_project(repository)
+            changelog_file = os.path.join(changelog_folder, f"v{project_version}.md")
+            if not os.path.isfile(changelog_file):
+                self.__ensure_changelog_file_is_added(repository, project_version)
+            t=TFCPS_CodeUnit_BuildCodeUnits(repository,self.__sc.log.loglevel,"QualityCheck",None,True,False)
+            t.build_codeunits()#check codeunits are buildable at all
+            self.__sc.git_commit(repository, "Updated dependencies", stage_all_changes=True) 
+
+
+    def __ensure_changelog_file_is_added(self, repository_folder: str, version_of_project: str):
+        changelog_file = os.path.join(repository_folder, "Other", "Resources", "Changelog", f"v{version_of_project}.md")
+        if not os.path.isfile(changelog_file):
+            GeneralUtilities.ensure_file_exists(changelog_file)
+            GeneralUtilities.write_text_to_file(changelog_file, """# Release notes
+
+## Changes
+
+- Updated dependencies.
+""")
+
+class TFCPS_AnionBuildPlatform_CLI:
+
+    @staticmethod
+    def get_with_overwritable_defaults(default_project_to_build:str=None,default_loglevel:LogLevel=None,default_additionalargumentsfile:str=None,default_build_repositories_folder:str=None,default_source_branch:str=None,default_main_branch:str=None,default_remote_name:str=None)->AnionBuildPlatform:
+        parser = argparse.ArgumentParser()
+        verbosity_values = ", ".join(f"{lvl.value}={lvl.name}" for lvl in LogLevel)
+        parser.add_argument('-b', '--buildrepositoriesfolder', required=False,default=None)
+        parser.add_argument('-p', '--projecttobuild', required=False, default=None)
+        parser.add_argument('-a', '--additionalargumentsfile', required=False, default=None)
+        parser.add_argument('-v', '--verbosity', required=False,  help=f"Sets the loglevel. Possible values: {verbosity_values}")
+        parser.add_argument('-s', '--sourcebranch', required=False)#other/next-release
+        parser.add_argument('-m', '--mainbranch', required=False)#main
+        parser.add_argument('-r', '--defaultremotename', required=False)#origin
+        parser.add_argument('-u', '--updatedependencies', required=False, action='store_true', default=False)
+        args=parser.parse_args()
+
+        if args.projecttobuild is not None: 
+            default_project_to_build=args.projecttobuild
+
+        if args.buildrepositoriesfolder is not None:
+            default_build_repositories_folder=args.buildrepositoriesfolder
+
+        scripts_folder=os.getcwd()
+        
+        if default_build_repositories_folder is None:
+            parent_parent_folder=GeneralUtilities.resolve_relative_path("../..",scripts_folder)
+            if os.path.basename(parent_parent_folder).endswith("Build"):
+                default_build_repositories_folder=os.path.dirname(parent_parent_folder)
+        GeneralUtilities.assert_not_null(default_build_repositories_folder,"buildrepositoriesfolder is not set")
+        
+        if default_project_to_build is None:
+            parent_parent_folder=GeneralUtilities.resolve_relative_path("../..",scripts_folder)
+            if os.path.basename(parent_parent_folder).endswith("Build"):
+                default_project_to_build=os.path.basename(parent_parent_folder)[:-len("Build")]
+        GeneralUtilities.assert_not_null(default_project_to_build,"projecttobuild is not set")
+
+        if args.verbosity is not None:
+            default_loglevel=LogLevel(int( args.verbosity))
+        GeneralUtilities.assert_not_null(default_loglevel,"verbosity is not set")
+
+        if args.additionalargumentsfile is not None:
+            default_additionalargumentsfile=args.additionalargumentsfile
+
+        if args.sourcebranch is not None:
+            default_source_branch=args.sourcebranch
+        GeneralUtilities.assert_not_null(default_source_branch,"sourcebranch is not set")
+
+        if args.defaultremotename is not None:
+            default_remote_name=args.defaultremotename
+        GeneralUtilities.assert_not_null(default_remote_name,"defaultremotename is not set")
+
+        config:AnionBuildPlatformConfiguration=AnionBuildPlatformConfiguration(default_build_repositories_folder,default_additionalargumentsfile,default_loglevel,default_source_branch,default_remote_name,args.updatedependencies)
+        tFCPS_MergeToMain:AnionBuildPlatform=AnionBuildPlatform(config)
+        return tFCPS_MergeToMain

ScriptCollection/{UpdateCertificates.py → CertificateUpdater.py}

@@ -1,28 +1,41 @@
 import os
 from pathlib import Path
+from datetime import datetime, timedelta
 import traceback
 from shutil import copyfile
-from .ScriptCollectionCore import ScriptCollectionCore
+import argparse
 from .GeneralUtilities import GeneralUtilities
+from .ScriptCollectionCore import ScriptCollectionCore
 
 
 class CertificateUpdater:
+    maximal_age_of_certificates_in_days: int = None
+    __domains: list[str] = None
+    __email: str = None
+    __current_folder: str = None
+    __last_update_timestamp_file: str = None
+    __repository_folder: str = None
+    __letsencrypt_folder: str = None
+    __letsencrypt_live_folder: str = None
+    __letsencrypt_archive_folder: str = None
+    __log_folder: str = None
+    __sc: ScriptCollectionCore = None
+    __arguments: ScriptCollectionCore = None
 
-    __domains: list[str]
-    __email: str
-
-    __current_folder = os.path.dirname(os.path.abspath(__file__))
-    __repository_folder = GeneralUtilities.resolve_relative_path(f"..{os.path.sep}..{os.path.sep}..{os.path.sep}", __current_folder)
-    __letsencrypt_folder = GeneralUtilities.resolve_relative_path(f"..{os.path.sep}..{os.path.sep}Volumes{os.path.sep}letsencrypt", __current_folder)
-    __letsencrypt_live_folder = os.path.join(__letsencrypt_folder, "live")
-    __letsencrypt_archive_folder = os.path.join(__letsencrypt_folder, "archive")
-    __log_folder = GeneralUtilities.resolve_relative_path(f"Logs{os.path.sep}Overhead", __repository_folder)
-    __sc = ScriptCollectionCore()
-    __line = "___________________________________________________________________"
-
-    def __init__(self, domains: list[str], email: str):
+    def __init__(self, domains: list[str], email: str, current_file: str, arguments: list[str]):
+        self.__sc = ScriptCollectionCore()
+        self.maximal_age_of_certificates_in_days = 15
         self.__domains = domains
         self.__email = email
+        self.__current_folder = current_file
+        self.__arguments = arguments
+        self.__last_update_timestamp_file = GeneralUtilities.resolve_relative_path("./LastCertificateUpdate.csv", self.__current_folder)
+        self.__repository_folder = GeneralUtilities.resolve_relative_path("../..", self.__current_folder)
+        self.__sc.assert_is_git_repository(self.__repository_folder)
+        self.__letsencrypt_folder = f"{ self.__repository_folder}/Configuration/Volumes/letsencrypt"
+        self.__letsencrypt_live_folder = os.path.join(self.__letsencrypt_folder, "live")
+        self.__letsencrypt_archive_folder = os.path.join(self.__letsencrypt_folder, "archive")
+        self.__log_folder = GeneralUtilities.resolve_relative_path("Logs/Overhead", self.__repository_folder)
 
     @GeneralUtilities.check_arguments
     def __get_latest_index_by_domain(self, domain: str) -> int:
@@ -31,16 +44,10 @@ class CertificateUpdater:
         return result
 
     @GeneralUtilities.check_arguments
-    def __get_latest_index_by_filelist(self, filenames: list) -> int:
-        print("files:")
-        print(filenames)
+    def __get_latest_index_by_filelist(self, filenames: list[str]) -> int:
         filenames = [Path(os.path.basename(file)).stem for file in filenames]
-        print(filenames)
         filenames = [file for file in filenames if file.startswith("privkey")]
-        print(filenames)
         numbers = [int(file[len("privkey"):]) for file in filenames]
-        # numbers=[]
-        # print([os.path.basename(file) for file in filenames])
         result = max(numbers)
         return result
 
@@ -51,7 +58,7 @@ class CertificateUpdater:
         live_folder = os.path.join(self.__letsencrypt_live_folder, domain)
         live_filename = filename+".pem"
         live_file = os.path.join(live_folder, live_filename)
-        self.__sc.run_program("rm", live_filename, live_folder)
+        self.__sc.run_program("rm", live_filename, live_folder, throw_exception_if_exitcode_is_not_zero=True)
         copyfile(archive_file, live_file)
 
     @GeneralUtilities.check_arguments
@@ -59,8 +66,8 @@ class CertificateUpdater:
         # new ".../live/example.com/cert.pem" is a file but should replaced by a symlink which points to ".../archive/example.com/cert42.pem"
         live_folder = os.path.join(self.__letsencrypt_live_folder, domain)
         live_filename = filename+".pem"
-        self.__sc.run_program("rm", live_filename, live_folder)
-        self.__sc.run_program("ln", f"-s ../../archive/{domain}/{filename+str(index)}.pem {live_filename}", live_folder)
+        self.__sc.run_program("rm", live_filename, live_folder, throw_exception_if_exitcode_is_not_zero=True)
+        self.__sc.run_program("ln", f"-s ../../archive/{domain}/{filename+str(index)}.pem {live_filename}", live_folder, throw_exception_if_exitcode_is_not_zero=True)
 
     @GeneralUtilities.check_arguments
     def __replace_symlinks_by_files(self, domain):
@@ -79,48 +86,64 @@ class CertificateUpdater:
         self.__replace_file_by_symlink(domain, "privkey", index)
 
     @GeneralUtilities.check_arguments
-    def update_certificate_managed_by_docker_and_letsencrypt(self) -> None:
-        GeneralUtilities.write_message_to_stdout("current_folder:")
-        GeneralUtilities.write_message_to_stdout(self.__current_folder)
-        GeneralUtilities.write_message_to_stdout("letsencrypt_folder:")
-        GeneralUtilities.write_message_to_stdout(self.__letsencrypt_folder)
-        GeneralUtilities.write_message_to_stdout("letsencrypt_live_folder:")
-        GeneralUtilities.write_message_to_stdout(self.__letsencrypt_live_folder)
-        GeneralUtilities.write_message_to_stdout("letsencrypt_archive_folder:")
-        GeneralUtilities.write_message_to_stdout(self.__letsencrypt_archive_folder)
-        GeneralUtilities.write_message_to_stdout("log_folder:")
-        GeneralUtilities.write_message_to_stdout(self.__log_folder)
-
-        GeneralUtilities.write_message_to_stdout(self.__line+self.__line)
-        GeneralUtilities.write_message_to_stdout("Updating certificates")
-        self.__sc.git_commit(self.__current_folder, "Saved current changes")
+    def __update_certificates(self) -> None:
+        self.__sc.git_commit(self.__repository_folder, "Saved current changes")
+        error_occurred = False
         for domain in self.__domains:
+            certbot_container_name = "certificate_updater"
             try:
-                GeneralUtilities.write_message_to_stdout(self.__line)
+                GeneralUtilities.write_message_to_stdout(GeneralUtilities.get_line())
                 GeneralUtilities.write_message_to_stdout(f"Process domain {domain}")
+                self.__sc.run_program("docker", f"container rm {certbot_container_name}", self.__current_folder, throw_exception_if_exitcode_is_not_zero=False)
                 certificate_for_domain_already_exists = os.path.isfile(f"{self.__letsencrypt_folder}/renewal/{domain}.conf")
                 if certificate_for_domain_already_exists:
                     GeneralUtilities.write_message_to_stdout(f"Update certificate for domain {domain}")
                     self.__replace_files_by_symlinks(domain)
                 else:
                     GeneralUtilities.write_message_to_stdout(f"Create certificate for domain {domain}")
-                certbot_container_name = "r2_updatecertificates_certbot"
                 dockerargument = f"run --name {certbot_container_name} --volume {self.__letsencrypt_folder}:/etc/letsencrypt"
-                dockerargument = dockerargument+f" --volume {self.__log_folder}:/var/log/letsencrypt -p 80:80 certbot/certbot:latest"
+                dockerargument = dockerargument + f" --volume {self.__log_folder}:/var/log/letsencrypt -p 80:80 certbot/certbot:latest"
                 certbotargument = f"--standalone --email {self.__email} --agree-tos --force-renewal --rsa-key-size 4096 --non-interactive --no-eff-email --domain {domain}"
                 if (certificate_for_domain_already_exists):
-                    self.__sc.run_program("docker", f"{dockerargument} certonly --no-random-sleep-on-renew {certbotargument}",                                          self.__current_folder, throw_exception_if_exitcode_is_not_zero=True)
+                    self.__sc.run_program("docker", f"{dockerargument} certonly --no-random-sleep-on-renew {certbotargument}", self.__current_folder)
                     self.__replace_symlinks_by_files(domain)
                 else:
-                    self.__sc.run_program("docker", f"{dockerargument} certonly --cert-name {domain} {certbotargument}",                                          self.__current_folder, throw_exception_if_exitcode_is_not_zero=True)
+                    self.__sc.run_program("docker", f"{dockerargument} certonly --cert-name {domain} {certbotargument}", self.__current_folder)
             except Exception as exception:
+                error_occurred = True
                 GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback, "Error while updating certificate")
             finally:
                 try:
                     self.__sc.run_program("docker", f"container rm {certbot_container_name}", self.__current_folder, throw_exception_if_exitcode_is_not_zero=True)
                 except Exception as exception:
                     GeneralUtilities.write_exception_to_stderr_with_traceback(exception, traceback, "Error while removing container")
-
-        GeneralUtilities.write_message_to_stdout("Commit changes...")
         self.__sc.git_commit(self.__repository_folder, "Executed certificate-update-process")
         GeneralUtilities.write_message_to_stdout("Finished certificate-update-process")
+        if error_occurred:
+            raise ValueError("Certificates for at least one domain could not be added/updated.")
+
+    @GeneralUtilities.check_arguments
+    def __get_last_certificate_update_date(self) -> datetime:
+        if os.path.exists(self.__last_update_timestamp_file):
+            filecontent = GeneralUtilities.read_text_from_file(self.__last_update_timestamp_file)
+            return GeneralUtilities.string_to_datetime(filecontent.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string))
+        else:
+            return datetime(year=1970, month=1, day=1)
+
+    @GeneralUtilities.check_arguments
+    def __set_last_certificate_update_date(self, moment: datetime) -> datetime:
+        GeneralUtilities.ensure_file_exists(self.__last_update_timestamp_file)
+        GeneralUtilities.write_text_to_file(self.__last_update_timestamp_file, GeneralUtilities.datetime_to_string(moment))
+
+    @GeneralUtilities.check_arguments
+    def update_certificates_if_required(self) -> None:
+        parser = argparse.ArgumentParser(description="Updated lets-encrypt-certificates")
+        parser.add_argument('-f', '--force', action='store_true', required=False, default=False)
+        args = parser.parse_args(self.__arguments)
+        now = datetime.now()
+        if (self.__get_last_certificate_update_date()+timedelta(days=self.maximal_age_of_certificates_in_days)) < now or args.force:
+            GeneralUtilities.write_message_to_stdout(f"Update certificates...")
+            self.__update_certificates()
+            self.__set_last_certificate_update_date(now)
+        else:
+            GeneralUtilities.write_message_to_stdout(f"Certificates are already up to date.")