ScriptCollection 4.0.43__py3-none-any.whl → 4.0.45__py3-none-any.whl

ScriptCollection/ScriptCollectionCore.py

@@ -37,7 +37,7 @@ from .ProgramRunnerPopen import ProgramRunnerPopen
 from .ProgramRunnerEpew import ProgramRunnerEpew, CustomEpewArgument
 from .SCLog import SCLog, LogLevel
 
-version = "4.0.43"
+version = "4.0.45"
 __version__ = version
 
 
@@ -2473,7 +2473,7 @@ OCR-content:
     def get_lines_of_code(self, repository: str, excluded_pattern: list[str]) -> int:
         self.assert_is_git_repository(repository)
         result: int = 0
-        self.log.log(f"Calculate lines of code in repository '{repository}' with excluded patterns: {', '.join(excluded_pattern)}")
+        self.log.log(f"Calculate lines of code in repository '{repository}' with excluded patterns: {', '.join(excluded_pattern)}",LogLevel.Debug)
         git_response = self.run_program("git", "ls-files", repository)
         files: list[str] = GeneralUtilities.string_to_lines(git_response[1])
         for file in files:

ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py

@@ -427,11 +427,11 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
         GeneralUtilities.ensure_directory_exists(temp_folder)
         runsettings_file = "runsettings.xml"
         codeunit_folder = f"{repository_folder}/{codeunit_name}"
-        arg = f"test . -c {dotnet_build_configuration} -o {temp_folder}"
+        arg = f"test . -c {dotnet_build_configuration}"
         if os.path.isfile(os.path.join(codeunit_folder, runsettings_file)):
             arg = f"{arg} --settings {runsettings_file}"
         arg = f"{arg} /p:CollectCoverage=true /p:CoverletOutput=../Other/Artifacts/TestCoverage/Testcoverage /p:CoverletOutputFormat=cobertura"
-        self._protected_sc.run_program("dotnet", arg, codeunit_folder, print_live_output=True)
+        test_run_result=self._protected_sc.run_program("dotnet", arg, codeunit_folder, print_live_output=True)#pylint:disable=unused-variable
         target_file = os.path.join(coverage_file_folder,  "TestCoverage.xml")
         GeneralUtilities.ensure_file_does_not_exist(target_file)
         os.rename(os.path.join(coverage_file_folder,  "Testcoverage.cobertura.xml"), target_file)

ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py

@@ -28,6 +28,21 @@ class TFCPS_CodeUnit_BuildCodeUnits:
         self.additionalargumentsfile=additionalargumentsfile
         self.__is_pre_merge=is_pre_merge
 
+    @GeneralUtilities.check_arguments
+    def __save_lines_of_code(self, repository_folder: str, project_version: str) -> None:
+        loc = self.sc.get_lines_of_code_with_default_excluded_patterns(repository_folder)
+        loc_metric_folder = os.path.join(repository_folder, "Other", "Metrics")
+        GeneralUtilities.ensure_directory_exists(loc_metric_folder)
+        loc_metric_file = os.path.join(loc_metric_folder, "LinesOfCode.csv")
+        GeneralUtilities.ensure_file_exists(loc_metric_file)
+        old_lines = GeneralUtilities.read_lines_from_file(loc_metric_file)
+        new_lines = []
+        for line in old_lines:
+            if not line.startswith(f"v{project_version};"):
+                new_lines.append(line)
+        new_lines.append(f"v{project_version};{loc}")
+        GeneralUtilities.write_lines_to_file(loc_metric_file, new_lines)
+
     @GeneralUtilities.check_arguments
     def build_codeunits(self) -> None:
         self.sc.log.log(GeneralUtilities.get_line())
@@ -61,9 +76,6 @@ class TFCPS_CodeUnit_BuildCodeUnits:
             from_day = datetime(now.year, now.month, now.day, 0, 0, 0)
             self.tFCPS_Other.mark_current_version_as_supported(self.repository,project_version,from_day,until_day)
 
-        #TODO search for secrets using TruffleHog
-        #TODO run static code analysis tool to search for vulnerabilities
-
         codeunits:list[str]=self.tFCPS_Other.get_codeunits(self.repository)        
         self.sc.log.log("Codeunits will be built in the following order:")
         for codeunit_name in codeunits:
@@ -72,10 +84,34 @@ class TFCPS_CodeUnit_BuildCodeUnits:
             tFCPS_CodeUnit_BuildCodeUnit:TFCPS_CodeUnit_BuildCodeUnit = TFCPS_CodeUnit_BuildCodeUnit(os.path.join(self.repository,codeunit_name),self.sc.log.loglevel,self.target_environment_type,self.additionalargumentsfile,self.use_cache(),self.is_pre_merge())
             self.sc.log.log(GeneralUtilities.get_line())
             tFCPS_CodeUnit_BuildCodeUnit.build_codeunit()
+
+        #TODO run static code analysis tool to search for vulnerabilities
+        self.__search_for_secrets()
+        self.__save_lines_of_code(self.repository,self.tFCPS_Other.get_version_of_project(self.repository))
+
         self.sc.log.log(GeneralUtilities.get_line())
         self.sc.log.log("Finished building codeunits.")
         self.sc.log.log(GeneralUtilities.get_line())
 
+    def __search_for_secrets(self):
+        exe_paths=self.tFCPS_Other.ensure_trufflehog_is_available()
+        exe_path:str=None
+        if GeneralUtilities.current_system_is_windows():
+            exe_path=exe_paths["Windows"]
+        elif GeneralUtilities.current_system_is_linux():
+            exe_path=exe_paths["Linux"]
+        else:
+            raise ValueError("unsupported")#TODO check for macos
+        result=self.sc.run_program(exe_path,"filesystem . --json",self.repository)
+
+        enabled:bool=False
+        if enabled:
+            self.sc.log.log("Secret-scan-result:")#TODO replace this by real analysis
+            for line in GeneralUtilities.string_to_lines(result[1]):
+                self.sc.log.log(line)
+            for line in GeneralUtilities.string_to_lines(result[2]):
+                self.sc.log.log(line,LogLevel.Error)
+
     @GeneralUtilities.check_arguments
     def use_cache(self) -> bool:
         return self.__use_cache

ScriptCollection/TFCPS/TFCPS_Tools_General.py

@@ -5,6 +5,7 @@ from pathlib import Path
 import shutil
 import zipfile
 import tarfile
+import time
 import re
 import sys
 import json
@@ -61,15 +62,18 @@ class TFCPS_Tools_General:
         if not file_exists:
             self.__sc.log.log(f"Download Asset \"{githubuser}/{githubprojectname}: {resource_name}\" from GitHub to global cache...", LogLevel.Information)
             GeneralUtilities.ensure_folder_exists_and_is_empty(resource_folder)
-            headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.5845.96 Safari/537.36'}
+            headers = { 'User-Agent': 'Mozilla/5.0'}
             self.__add_github_api_key_if_available(headers)
             url = f"https://api.github.com/repos/{githubuser}/{githubprojectname}/releases/latest"
             self.__sc.log.log(f"Download \"{url}\"...", LogLevel.Debug)
+            time.sleep(2)
             response = requests.get(url, headers=headers, allow_redirects=True, timeout=(10, 10))
-            latest_version = response.json()["tag_name"]
+            response_json=response.json()
+            latest_version = response_json["tag_name"]
             filename_on_github = get_filename_on_github(latest_version)
             link = f"https://github.com/{githubuser}/{githubprojectname}/releases/download/{latest_version}/{filename_on_github}"
-            with requests.get(link, headers=headers, stream=True, allow_redirects=True,  timeout=(5, 300)) as r:
+            time.sleep(2)
+            with requests.get(link, headers=headers, stream=True, allow_redirects=True,  timeout=(5, 600)) as r:
                 r.raise_for_status()
                 total_size = int(r.headers.get("Content-Length", 0))
                 downloaded = 0
@@ -84,7 +88,7 @@ class TFCPS_Tools_General:
                                 sys.stdout.write(f"\rDownload: {percent:.2f}%")
                                 sys.stdout.flush()
             self.__sc.log.log(f"Downloaded \"{url}\".", LogLevel.Diagnostic)
-            GeneralUtilities.assert_file_exists(file)
+        GeneralUtilities.assert_file_exists(file)
         return file
 
     def __add_github_api_key_if_available(self, headers: dict):
@@ -555,6 +559,38 @@ class TFCPS_Tools_General:
         lines.append("@enduml")
 
         GeneralUtilities.write_lines_to_file(target_file, lines)
+        
+    @GeneralUtilities.check_arguments
+    def ensure_trufflehog_is_available(self,enforce_update:bool=False) -> dict[str,str]:
+        def download_and_extract(osname: str, osname_in_github_asset: str, extension: str):
+            resource_name: str = f"TruffleHog_{osname}"
+            zip_filename: str = f"{resource_name}.{extension}"
+            target_folder_unextracted = os.path.join(self.get_global_cache_folder(),"Tools",resource_name+"_Unextracted")
+            target_folder_extracted = os.path.join(self.get_global_cache_folder(),"Tools",resource_name)
+            update:bool=not os.path.isdir(target_folder_extracted) or GeneralUtilities.folder_is_empty(target_folder_extracted) or enforce_update
+            if update:
+                downloaded_file=self.ensure_file_from_github_assets_is_available_with_retry(target_folder_unextracted, "trufflesecurity", "trufflehog", resource_name+"_Unextracted", zip_filename, lambda latest_version: f"trufflehog_{latest_version[1:]}_{osname_in_github_asset}_amd64.tar.gz",enforce_update=enforce_update)
+                #TODO add option to also download arm-version
+                local_zip_file: str = downloaded_file
+                GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder_extracted)
+                if extension == "zip":
+                    with zipfile.ZipFile(local_zip_file, 'r') as zip_ref:
+                        zip_ref.extractall(target_folder_extracted)
+                elif extension == "tar.gz": 
+                    with tarfile.open(local_zip_file, "r:gz") as tar:
+                        tar.extractall(path=target_folder_extracted)
+                else:
+                    raise ValueError(f"Unknown extension: \"{extension}\"")
+                GeneralUtilities.ensure_directory_does_not_exist(target_folder_unextracted)
+            GeneralUtilities.assert_folder_exists(target_folder_extracted)
+            executable=[f for f in GeneralUtilities.get_all_files_of_folder(target_folder_extracted) if os.path.basename(f).startswith("trufflehog")][0]
+            return executable
+
+        result=dict[str,str]()
+        result["Windows"]=download_and_extract("Windows", "windows", "tar.gz")
+        result["Linux"]=download_and_extract("Linux", "linux", "tar.gz")
+        result["MacOS"]=download_and_extract("MacOS", "darwin", "tar.gz")
+        return result
 
     @GeneralUtilities.check_arguments
     def generate_tasksfile_from_workspace_file(self, repository_folder: str, append_cli_args_at_end: bool = False) -> None:
@@ -671,9 +707,9 @@ class TFCPS_Tools_General:
             if repository_subname is not None:
                 target_folder = f"{resrepo_data_folder}/{repository_subname}"
             
-            update:bool=GeneralUtilities.folder_is_empty(target_folder) or not use_cache
+            update:bool=not os.path.isdir(target_folder) or GeneralUtilities.folder_is_empty(target_folder) or not use_cache
             if update:
-                self.__sc.run_program(f"Clone {remote_repository_link} as resource", LogLevel.Information)
+                self.__sc.log.log(f"Clone {remote_repository_link} as resource...", LogLevel.Information)
                 GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder)
                 self.__sc.run_program("git", f"clone --recurse-submodules {remote_repository_link} {target_folder}")
                 self.__sc.run_program("git", f"checkout {latest_version}", target_folder)
@@ -755,7 +791,7 @@ class TFCPS_Tools_General:
     @GeneralUtilities.check_arguments
     def do_npm_install(self, package_json_folder: str, npm_force: bool,use_cache:bool) -> None:
         target_folder:str=os.path.join(package_json_folder,"node_modules")
-        update:bool=GeneralUtilities.folder_is_empty(target_folder) or  GeneralUtilities.folder_is_empty(target_folder) or not use_cache
+        update:bool=not os.path.isdir(target_folder) or  GeneralUtilities.folder_is_empty(target_folder) or not use_cache
         if update:
             self.__sc.log.log("Do npm-install...")
             argument1 = "install"
@@ -966,7 +1002,7 @@ class TFCPS_Tools_General:
     @GeneralUtilities.check_arguments
     def ensure_openapigenerator_is_available(self, codeunit_folder: str,use_cache:bool) -> None:
         self.assert_is_codeunit_folder(codeunit_folder)
-        openapigenerator_folder = os.path.join(codeunit_folder, "Other", "Resources", "OpenAPIGenerator")
+        openapigenerator_folder = os.path.join(self.get_global_cache_folder(), "Tools", "OpenAPIGenerator")
         filename = "open-api-generator.jar"
         jar_file = f"{openapigenerator_folder}/{filename}"
         jar_file_exists = os.path.isfile(jar_file)
@@ -979,6 +1015,8 @@ class TFCPS_Tools_General:
             GeneralUtilities.ensure_directory_does_not_exist(openapigenerator_folder)
             GeneralUtilities.ensure_directory_exists(openapigenerator_folder)
             urllib.request.urlretrieve(download_link, jar_file)
+        GeneralUtilities.assert_file_exists(jar_file)
+        return jar_file
 
     @GeneralUtilities.check_arguments
     def standardized_tasks_update_version_in_docker_examples(self, codeunit_folder:str, codeunit_version:str) -> None:
@@ -1159,3 +1197,14 @@ class TFCPS_Tools_General:
         self.generate_tasksfile_from_workspace_file(repository_folder)
         self.generate_codeunits_overview_diagram(repository_folder)
         self.generate_svg_files_from_plantuml_files_for_repository(repository_folder,use_cache)
+
+    @GeneralUtilities.check_arguments
+    def copy_product_resource_to_codeunit_resource_folder(self, codeunit_folder: str, resourcename: str) -> None:
+        repository_folder = GeneralUtilities.resolve_relative_path(f"..", codeunit_folder)
+        self.__sc.assert_is_git_repository(repository_folder)
+        src_folder = GeneralUtilities.resolve_relative_path(f"Other/Resources/{resourcename}", repository_folder)
+        GeneralUtilities.assert_condition(os.path.isdir(src_folder), f"Required product-resource {resourcename} does not exist. Expected folder: {src_folder}")
+        trg_folder = GeneralUtilities.resolve_relative_path(f"Other/Resources/{resourcename}", codeunit_folder)
+        GeneralUtilities.ensure_directory_does_not_exist(trg_folder)
+        GeneralUtilities.ensure_directory_exists(trg_folder)
+        GeneralUtilities.copy_content_of_folder(src_folder, trg_folder)

{scriptcollection-4.0.43.dist-info → scriptcollection-4.0.45.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ScriptCollection
-Version: 4.0.43
+Version: 4.0.45
 Summary: The ScriptCollection is the place for reusable scripts.
 Home-page: https://github.com/anionDev/ScriptCollection
 Author: Marius Göcke

{scriptcollection-4.0.43.dist-info → scriptcollection-4.0.45.dist-info}/RECORD

@@ -9,25 +9,25 @@ ScriptCollection/ProgramRunnerEpew.py,sha256=TJdDx9zIMSiCaXh8X-ekrMlbXfGtmd0Mmyx
 ScriptCollection/ProgramRunnerMock.py,sha256=uTu-aFle1W_oKjeQEmuPsFPQpvo0kRf2FrRjAPIwT5Y,37
 ScriptCollection/ProgramRunnerPopen.py,sha256=BPY7-ZMIlqT7JOKz8qlB5c0laF2Js-ijzqk09GxZC48,3821
 ScriptCollection/SCLog.py,sha256=dxGOI4E9lG5v9jk_LajXCkM5nghliCDV8YB8Ihn160s,4541
-ScriptCollection/ScriptCollectionCore.py,sha256=hzraZbT2qQrny90RSK55aVMmrQPMPIiHAiB35Coj8U4,142098
+ScriptCollection/ScriptCollectionCore.py,sha256=_qwjaWj7vuU1BSkTgn94r2hw5I4G0Ox0WY58-8k3a5A,142113
 ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=2Ng-GyKWQ6FfTUmogd7g-xF6_XeTMxvkeJddagxFR5o,25853
 ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=4rYKgTAga11NiDx8YUqz3K_Q4eX_n3kC6lvNdXEa24s,7389
-ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=h3PWj7SgJF9huMognPpfNwGb9E9Y9HQ0Ym72yvvJwwg,5676
+ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=EH5yV33emYfbYqdikrPgW9UdF7UYV6rpBS1JGHNxA6M,7483
 ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=bcJlfI062Eoq7MOIhun-_iNG7SdO1ZIuC_cylaoLI1s,6332
 ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=O-0guM_LJCcZmPZJhMgTvXD2RXUJEBWWv6Bt6hDFhvM,1943
 ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=41g219jaBRZ2VQCrWM4-Trvervrt8b5oATPwIIGNpag,7244
 ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=ifB1K6A903vvfH0LvtiFbZgYSgR94thfEI-jjf40LpU,21653
 ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=CxdwUklhZVuJGp0vcokoH_KMXFzmlUlZwj77xFYijho,2242
 ScriptCollection/TFCPS/TFCPS_Tools_Dependencies.py,sha256=o7HI3ki3WWqlAiUsrh3Lky_w6UhYh9hdjYPGOhubQGA,414
-ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=S_uQX8kmjntIe8b_3FVh7ABgPCBcLCCpmewMv2gVvD4,73160
+ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=wqF_szXoB_EMMvpdYdow20v2A2E5jT3P3Op-tgT0y-s,76517
 ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=-g8h2gCf9rh0KJXUBeQD5d0qLJgBU3Q8DNZXM1UXC04,5259
 ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py,sha256=bT6Gd5pQpZCw4OQz6HWkPCSn5z__eUUEisABLDSxd0o,200
 ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py,sha256=QyjOfMY22JWCvKjMelHiDWbJiWqotOfebpJpgDUaoO4,237
 ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py,sha256=i0zEGehj0sttxjjZtoq2KFSKp_ulxVyWp_ZgAhIY_So,241
-ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=myR8hG2RMGg59S6Rf0Ts7R7df2IIVFDkazu4XSmSvrA,31001
+ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py,sha256=ufeWuqtNOPAtEyHayxHGPdjiJsT0lxYR2gAWle9p7T4,31031
 ScriptCollection/TFCPS/DotNet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py,sha256=TIR95f6TVOnW25ieX9q4RUi1FogbYEfrlZOcZ1aE014,6969
 ScriptCollection/TFCPS/Flutter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -35,8 +35,8 @@ ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py,sha256=dnuDlQXThF
 ScriptCollection/TFCPS/NodeJS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py,sha256=q7msAxCb5VIZ-xhFg1MfzUvWomQRKYldqmW42KFhyMU,6868
 ScriptCollection/TFCPS/Python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scriptcollection-4.0.43.dist-info/METADATA,sha256=gUEW02jG627c1YV8_WeShyqVa8XFVr8NWTiyn1UMfYc,7688
-scriptcollection-4.0.43.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-scriptcollection-4.0.43.dist-info/entry_points.txt,sha256=NeU26D6q7d8n2cmKQiOvHK21w1C7D2kxoNRJaKiyZ5w,4295
-scriptcollection-4.0.43.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
-scriptcollection-4.0.43.dist-info/RECORD,,
+scriptcollection-4.0.45.dist-info/METADATA,sha256=qzrk27G_cN2Gw5hfAfffblykLWFYU0CH1TiOtRVaDDs,7688
+scriptcollection-4.0.45.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+scriptcollection-4.0.45.dist-info/entry_points.txt,sha256=RhfJxpWvSMIkdQ_1Pan1uXPTgYCTD1EcP0ZshkA2vtA,4366
+scriptcollection-4.0.45.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
+scriptcollection-4.0.45.dist-info/RECORD,,

{scriptcollection-4.0.43.dist-info → scriptcollection-4.0.45.dist-info}/entry_points.txt

@@ -56,6 +56,7 @@ scsetcontentoffile = ScriptCollection.Executables:SetContentOfFile
 scshow2faasqrcode = ScriptCollection.Executables:Show2FAAsQRCode
 scshowmissingfiles = ScriptCollection.Executables:ShowMissingFiles
 scsigncertificate = ScriptCollection.Executables:SignCertificate
+scupdatedependencies = ScriptCollection.Executables:UpdateDependencies
 scupdateimagesindockercomposefile = ScriptCollection.Executables:UpdateImagesInDockerComposeFile
 scupdatenugetpackagesincsharpproject = ScriptCollection.Executables:UpdateNugetpackagesInCsharpProject
 scupdatetimestampinfile = ScriptCollection.Executables:UpdateTimestampInFile