ScriptCollection 4.2.103__py3-none-any.whl → 4.2.105__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ScriptCollection/Executables.py +10 -0
- ScriptCollection/ScriptCollectionCore.py +1 -1
- ScriptCollection/TFCPS/TFCPS_Tools_General.py +94 -27
- {scriptcollection-4.2.103.dist-info → scriptcollection-4.2.105.dist-info}/METADATA +1 -1
- {scriptcollection-4.2.103.dist-info → scriptcollection-4.2.105.dist-info}/RECORD +8 -8
- {scriptcollection-4.2.103.dist-info → scriptcollection-4.2.105.dist-info}/entry_points.txt +1 -0
- {scriptcollection-4.2.103.dist-info → scriptcollection-4.2.105.dist-info}/WHEEL +0 -0
- {scriptcollection-4.2.103.dist-info → scriptcollection-4.2.105.dist-info}/top_level.txt +0 -0
ScriptCollection/Executables.py
CHANGED
|
@@ -863,6 +863,16 @@ def CleanToolsCache()->int:
|
|
|
863
863
|
return 0
|
|
864
864
|
|
|
865
865
|
|
|
866
|
+
def DownloadCachableTools()->int:
|
|
867
|
+
parser = argparse.ArgumentParser(description="Downloads all tools that are stored in the global ScriptCollection-cache (for example CycloneDX-CLI, PlantUML, MediaMTX, TruffleHog, OpenAPIGenerator and AndroidAppBundleTool). Running this - for example in a build-image - lets repeated pipeline-runs avoid rate-limits and run faster because the tools are already present.")
|
|
868
|
+
parser.add_argument('-v', '--verbose', action='store_true', required=False, default=False, help="Enables verbose (debug) output.")
|
|
869
|
+
parser.add_argument('-e', '--enforceupdate', action='store_true', required=False, default=False, help="Re-download the tools even if they are already present in the cache.")
|
|
870
|
+
args = parser.parse_args()
|
|
871
|
+
sc: ScriptCollectionCore = ScriptCollectionCore()
|
|
872
|
+
TFCPS_Tools_General(sc).download_all_cachable_tools(args.enforceupdate, args.verbose)
|
|
873
|
+
return 0
|
|
874
|
+
|
|
875
|
+
|
|
866
876
|
def EnsureDockerNetworkIsAvailable()->int:
|
|
867
877
|
sc = ScriptCollectionCore()
|
|
868
878
|
parser = argparse.ArgumentParser()
|
|
@@ -66,6 +66,56 @@ class TFCPS_Tools_General:
|
|
|
66
66
|
else:
|
|
67
67
|
raise ValueError("Unsupported operating system for cyclonedx-cli.")
|
|
68
68
|
|
|
69
|
+
@GeneralUtilities.check_arguments
|
|
70
|
+
def download_all_cachable_tools(self, enforce_update: bool = False, verbose: bool = False) -> None:
|
|
71
|
+
"""Downloads all tools that are stored in the global ScriptCollection-cache.
|
|
72
|
+
Running this (for example in a build-image) lets repeated pipeline-runs avoid
|
|
73
|
+
rate-limits and run faster, because the tools are already present when a later
|
|
74
|
+
ensure_*_is_available-call needs them. Each tool is downloaded for all platforms,
|
|
75
|
+
so the warmed cache can be reused independent of the executing platform.
|
|
76
|
+
When verbose is set, the loglevel is set to debug so the per-tool log-output becomes visible."""
|
|
77
|
+
if verbose:
|
|
78
|
+
self.__sc.log.loglevel = LogLevel.Debug
|
|
79
|
+
self.download_cyclonedx(enforce_update)
|
|
80
|
+
self.download_plantuml(enforce_update)
|
|
81
|
+
self.download_mediamtx(enforce_update)
|
|
82
|
+
self.download_trufflehog(enforce_update)
|
|
83
|
+
self.download_openapigenerator(enforce_update)
|
|
84
|
+
self.download_androidappbundletool(enforce_update)
|
|
85
|
+
|
|
86
|
+
@GeneralUtilities.check_arguments
|
|
87
|
+
def download_cyclonedx(self, enforce_update: bool = False) -> None:
|
|
88
|
+
self.__sc.log.log("Download cachable tool \"CycloneDX-CLI\" into global cache...", LogLevel.Debug)
|
|
89
|
+
self.ensure_cyclonedxcli_is_available(enforce_update)
|
|
90
|
+
|
|
91
|
+
@GeneralUtilities.check_arguments
|
|
92
|
+
def download_plantuml(self, enforce_update: bool = False) -> None:
|
|
93
|
+
self.__sc.log.log("Download cachable tool \"PlantUML\" into global cache...", LogLevel.Debug)
|
|
94
|
+
self.ensure_plantuml_is_available(enforce_update)
|
|
95
|
+
|
|
96
|
+
@GeneralUtilities.check_arguments
|
|
97
|
+
def download_mediamtx(self, enforce_update: bool = False) -> None:
|
|
98
|
+
self.__sc.log.log("Download cachable tool \"MediaMTX\" into global cache...", LogLevel.Debug)
|
|
99
|
+
for architecture in [Platform.Windows_AMD64, Platform.Linux_AMD64, Platform.Linux_ARM64, Platform.MacOS_ARM64]:
|
|
100
|
+
self.__ensure_mediamtx_archive_in_global_cache(architecture, enforce_update)
|
|
101
|
+
|
|
102
|
+
@GeneralUtilities.check_arguments
|
|
103
|
+
def download_trufflehog(self, enforce_update: bool = False) -> None:
|
|
104
|
+
self.__sc.log.log("Download cachable tool \"TruffleHog\" into global cache...", LogLevel.Debug)
|
|
105
|
+
self.ensure_trufflehog_is_available(enforce_update)
|
|
106
|
+
|
|
107
|
+
@GeneralUtilities.check_arguments
|
|
108
|
+
def download_openapigenerator(self, enforce_update: bool = False) -> None:
|
|
109
|
+
self.__sc.log.log("Download cachable tool \"OpenAPIGenerator\" into global cache...", LogLevel.Debug)
|
|
110
|
+
self.ensure_openapigenerator_is_available(not enforce_update)
|
|
111
|
+
|
|
112
|
+
@GeneralUtilities.check_arguments
|
|
113
|
+
def download_androidappbundletool(self, enforce_update: bool = False) -> None:
|
|
114
|
+
self.__sc.log.log("Download cachable tool \"AndroidAppBundleTool\" into global cache...", LogLevel.Debug)
|
|
115
|
+
# target_folder is not used by ensure_androidappbundletool_is_available (the jar is
|
|
116
|
+
# always stored in the global cache); pass the cache folder as a harmless value.
|
|
117
|
+
self.ensure_androidappbundletool_is_available(self.__sc.get_global_cache_folder(), enforce_update)
|
|
118
|
+
|
|
69
119
|
@GeneralUtilities.check_arguments
|
|
70
120
|
def ensure_file_from_github_assets_is_available_with_retry(self, githubuser: str, githubprojectname: str, local_resource_name: str, local_filename: str, get_filename_on_github, amount_of_attempts: int = 5,enforce_update:bool=False) -> str:
|
|
71
121
|
return GeneralUtilities.retry_action(lambda: self.ensure_file_from_github_assets_is_available(githubuser, githubprojectname, local_resource_name, local_filename, get_filename_on_github,enforce_update), amount_of_attempts)
|
|
@@ -78,7 +128,11 @@ class TFCPS_Tools_General:
|
|
|
78
128
|
file_exists = os.path.isfile(file)
|
|
79
129
|
if not file_exists:
|
|
80
130
|
self.__sc.log.log(f"Download Asset \"{githubuser}/{githubprojectname}: {local_resource_name}\" from GitHub to global cache...", LogLevel.Information)
|
|
81
|
-
|
|
131
|
+
# Only ensure the folder exists; do NOT empty it. Several assets (e.g. the
|
|
132
|
+
# linux/windows x64/arm64 variants of cyclonedx-cli) share one resource folder
|
|
133
|
+
# and are downloaded one after another, so emptying here would delete the
|
|
134
|
+
# siblings downloaded by previous calls and leave only the last one.
|
|
135
|
+
GeneralUtilities.ensure_directory_exists(resource_folder)
|
|
82
136
|
headers = { 'User-Agent': 'Mozilla/5.0'}
|
|
83
137
|
self.__add_github_api_key_if_available(headers)
|
|
84
138
|
url = f"https://api.github.com/repos/{githubuser}/{githubprojectname}/releases/latest"
|
|
@@ -105,6 +159,10 @@ class TFCPS_Tools_General:
|
|
|
105
159
|
sys.stdout.write(f"\rDownload: {percent:.2f}%")
|
|
106
160
|
sys.stdout.flush()
|
|
107
161
|
self.__sc.log.log(f"Downloaded \"{url}\".", LogLevel.Diagnostic)
|
|
162
|
+
if not GeneralUtilities.current_system_is_windows():
|
|
163
|
+
# Downloaded binaries (e.g. cyclonedx-linux-x64) are written without the
|
|
164
|
+
# executable bit; make them runnable so they can be executed directly.
|
|
165
|
+
os.chmod(file, 0o755)
|
|
108
166
|
GeneralUtilities.assert_file_exists(file)
|
|
109
167
|
return file
|
|
110
168
|
|
|
@@ -675,46 +733,55 @@ class TFCPS_Tools_General:
|
|
|
675
733
|
|
|
676
734
|
@GeneralUtilities.check_arguments
|
|
677
735
|
def ensure_mediamtx_is_available(self, target_folder: str,enforce_update:bool) -> None:
|
|
678
|
-
|
|
736
|
+
for architecture in [Platform.Windows_AMD64, Platform.Linux_AMD64, Platform.Linux_ARM64, Platform.MacOS_ARM64]:
|
|
679
737
|
resource_name: str = f"MediaMTX_{GeneralUtilities.platform_to_dash_str(architecture)}"
|
|
680
738
|
resource_folder: str = os.path.join(target_folder, "Other", "Resources", resource_name)
|
|
681
739
|
target_folder_extracted = os.path.join(resource_folder, "MediaMTX")
|
|
682
740
|
update:bool=not os.path.isdir(target_folder_extracted) or GeneralUtilities.folder_is_empty(target_folder_extracted) or enforce_update
|
|
683
741
|
if update:
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
case Platform.Windows_AMD64:
|
|
687
|
-
platform_str = "windows_amd64"
|
|
688
|
-
case Platform.Linux_ARM64:
|
|
689
|
-
platform_str = "linux_arm64"
|
|
690
|
-
case Platform.Linux_AMD64:
|
|
691
|
-
platform_str = "linux_amd64"
|
|
692
|
-
case Platform.MacOS_ARM64:
|
|
693
|
-
platform_str = "darwin_arm64"
|
|
694
|
-
case _:
|
|
695
|
-
raise ValueError(f"Unknown platform: {str(architecture)}")
|
|
696
|
-
|
|
697
|
-
resource_filename_name_remote:str=f"mediamtx_{platform_str}.{extension}"
|
|
698
|
-
resource_name_local:str=f"MediaMTCX_{platform_str}"
|
|
699
|
-
global_cache_file=os.path.join( self.__sc.get_global_cache_folder(),"Tools",resource_name_local,resource_filename_name_remote)
|
|
700
|
-
if (not os.path.isfile( global_cache_file )) or enforce_update:
|
|
701
|
-
self.ensure_file_from_github_assets_is_available_with_retry( "bluenviron", "mediamtx", resource_name_local, resource_filename_name_remote, lambda latest_version: f"mediamtx_{latest_version}_{platform_str}.{extension}",enforce_update=enforce_update)
|
|
702
|
-
GeneralUtilities.assert_file_exists(global_cache_file)
|
|
703
|
-
GeneralUtilities.assert_file_exists(global_cache_file)
|
|
742
|
+
global_cache_file = self.__ensure_mediamtx_archive_in_global_cache(architecture, enforce_update)
|
|
743
|
+
extension: str = self.__get_mediamtx_archive_extension(architecture)
|
|
704
744
|
GeneralUtilities.ensure_folder_exists_and_is_empty(target_folder_extracted)
|
|
705
745
|
if extension == "zip":
|
|
706
746
|
with zipfile.ZipFile(global_cache_file, 'r') as zip_ref:
|
|
707
747
|
zip_ref.extractall(target_folder_extracted)
|
|
708
|
-
elif extension == "tar.gz":
|
|
748
|
+
elif extension == "tar.gz":
|
|
709
749
|
with tarfile.open(global_cache_file, "r:gz") as tar:
|
|
710
750
|
tar.extractall(path=target_folder_extracted)
|
|
711
751
|
else:
|
|
712
752
|
raise ValueError(f"Unknown extension: \"{extension}\"")
|
|
713
753
|
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
754
|
+
@GeneralUtilities.check_arguments
|
|
755
|
+
def __get_mediamtx_platform_str(self, architecture: Platform) -> str:
|
|
756
|
+
match architecture:
|
|
757
|
+
case Platform.Windows_AMD64:
|
|
758
|
+
return "windows_amd64"
|
|
759
|
+
case Platform.Linux_ARM64:
|
|
760
|
+
return "linux_arm64"
|
|
761
|
+
case Platform.Linux_AMD64:
|
|
762
|
+
return "linux_amd64"
|
|
763
|
+
case Platform.MacOS_ARM64:
|
|
764
|
+
return "darwin_arm64"
|
|
765
|
+
case _:
|
|
766
|
+
raise ValueError(f"Unknown platform: {str(architecture)}")
|
|
767
|
+
|
|
768
|
+
@GeneralUtilities.check_arguments
|
|
769
|
+
def __get_mediamtx_archive_extension(self, architecture: Platform) -> str:
|
|
770
|
+
if architecture == Platform.Windows_AMD64:
|
|
771
|
+
return "zip"
|
|
772
|
+
return "tar.gz"
|
|
773
|
+
|
|
774
|
+
@GeneralUtilities.check_arguments
|
|
775
|
+
def __ensure_mediamtx_archive_in_global_cache(self, architecture: Platform, enforce_update: bool) -> str:
|
|
776
|
+
platform_str: str = self.__get_mediamtx_platform_str(architecture)
|
|
777
|
+
extension: str = self.__get_mediamtx_archive_extension(architecture)
|
|
778
|
+
resource_filename_name_remote: str = f"mediamtx_{platform_str}.{extension}"
|
|
779
|
+
resource_name_local: str = f"MediaMTCX_{platform_str}"
|
|
780
|
+
global_cache_file = os.path.join(self.__sc.get_global_cache_folder(), "Tools", resource_name_local, resource_filename_name_remote)
|
|
781
|
+
if (not os.path.isfile(global_cache_file)) or enforce_update:
|
|
782
|
+
self.ensure_file_from_github_assets_is_available_with_retry("bluenviron", "mediamtx", resource_name_local, resource_filename_name_remote, lambda latest_version: f"mediamtx_{latest_version}_{platform_str}.{extension}", enforce_update=enforce_update)
|
|
783
|
+
GeneralUtilities.assert_file_exists(global_cache_file)
|
|
784
|
+
return global_cache_file
|
|
718
785
|
|
|
719
786
|
@GeneralUtilities.check_arguments
|
|
720
787
|
def clone_repository_as_resource(self, local_repository_folder: str, remote_repository_link: str, resource_name: str, repository_subname: str = None,use_cache:bool=True) -> None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptcollection
|
|
3
|
-
Version: 4.2.
|
|
3
|
+
Version: 4.2.105
|
|
4
4
|
Summary: The ScriptCollection is the place for reusable scripts.
|
|
5
5
|
Author-email: Marius Göcke <marius.goecke@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
|
|
@@ -6,7 +6,7 @@ Other/QualityCheck/RunTestcases.py,sha256=3vJBquOr9q1DhF4648FhY_sXS-DV_l193mE_O9
|
|
|
6
6
|
Other/Reference/GenerateReference.py,sha256=MFzu7U1OHxgecYDi7WSulO8XRGBsgkdqV_r_jkyVF7c,369
|
|
7
7
|
ScriptCollection/AnionBuildPlatform.py,sha256=OnL0V0jFGrT3ZbCRamy4LBmeGUu-cBFMbk1bhs_S64k,11781
|
|
8
8
|
ScriptCollection/CertificateUpdater.py,sha256=dR5JpJDlZdjD4_pgMz7zCrt1IM7VfatmK6pULSczxQw,9693
|
|
9
|
-
ScriptCollection/Executables.py,sha256=
|
|
9
|
+
ScriptCollection/Executables.py,sha256=YtNmvkz1ASf1VLwo6B38BMm49mGV6IpRcVV0zjvfCOo,47711
|
|
10
10
|
ScriptCollection/GeneralUtilities.py,sha256=4BmzJhTPAZmKQd5YlMuX0E5Dl6fhuVkmVaCpST0WzRk,67654
|
|
11
11
|
ScriptCollection/HTTPMaintenanceOverheadHelper.py,sha256=TToNtyO1XzsMbBsTBf3o0xgOK0v4Jf03qw2Z0xb2nCk,2007
|
|
12
12
|
ScriptCollection/ProcessesRunner.py,sha256=o5raxIt3lknNPoPrjNzJ2bprRPJ3SnL0rrR7crraD7E,1523
|
|
@@ -15,7 +15,7 @@ ScriptCollection/ProgramRunnerMock.py,sha256=uTu-aFle1W_oKjeQEmuPsFPQpvo0kRf2FrR
|
|
|
15
15
|
ScriptCollection/ProgramRunnerPopen.py,sha256=BPY7-ZMIlqT7JOKz8qlB5c0laF2Js-ijzqk09GxZC48,3821
|
|
16
16
|
ScriptCollection/ProgramRunnerSudo.py,sha256=_khC3xuTdrPoLluBJZWfldltmmuKltABJPcbjZSFW-4,4835
|
|
17
17
|
ScriptCollection/SCLog.py,sha256=8TRy1LeYMsPOIuWUcnUNNbO5pd-cNBS-3cn-kdzP8FU,4768
|
|
18
|
-
ScriptCollection/ScriptCollectionCore.py,sha256=
|
|
18
|
+
ScriptCollection/ScriptCollectionCore.py,sha256=AO24YBlf78Uz8YdQNt-LgPKxwy6PDmmhaGrjNmzaK28,188793
|
|
19
19
|
ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
20
|
ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=o129UYDtLymCRjMZBP9Puaj19ueMHoNt6UTLspfnaRc,1758
|
|
21
21
|
ScriptCollection/OCIImages/OCIImageManager.py,sha256=2muW4hJ_a7EfavY3WVnafqZ0HC87RcZsoTuNvA3c0To,11175
|
|
@@ -38,7 +38,7 @@ ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=Tpzgiz6m3-cYCkObZOG5Uu7oM-EMoWFzz
|
|
|
38
38
|
ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Vwj7IWmvwIICXiv7I-c6MkxbqlRWCEwzaKW6dgEUjCM,7480
|
|
39
39
|
ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=xVPer8EGH5vhTT3_mpTMJJxWLB3aLxIcyTxKHODMbYg,22201
|
|
40
40
|
ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=f0Uq1cA_4LvmL72cal0crrbKF6PcxL13D9wBKuQ1YBw,2328
|
|
41
|
-
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=
|
|
41
|
+
ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=l42hYiDwcafEGIUCXmeyxr7XMzQ01MUEpElVvB1zhKY,105372
|
|
42
42
|
ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
43
43
|
ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py,sha256=TU9An7feBsSMardrtr0byYijQfYWxA_LB27yVNsOqy8,12762
|
|
44
44
|
ScriptCollection/TFCPS/Docker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -291,8 +291,8 @@ build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Python/TFCPS_Code
|
|
|
291
291
|
build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
292
292
|
build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=WhOsMpJxNGqnQGA3nJ99ruyqymxBeJxCcpK98zJduTo,1680
|
|
293
293
|
build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
294
|
-
scriptcollection-4.2.
|
|
295
|
-
scriptcollection-4.2.
|
|
296
|
-
scriptcollection-4.2.
|
|
297
|
-
scriptcollection-4.2.
|
|
298
|
-
scriptcollection-4.2.
|
|
294
|
+
scriptcollection-4.2.105.dist-info/METADATA,sha256=FwJgtaMYwy4d9adTu8devQa6Cd6SLpLGiauglhE2q4o,7697
|
|
295
|
+
scriptcollection-4.2.105.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
296
|
+
scriptcollection-4.2.105.dist-info/entry_points.txt,sha256=_vnVTrGJc_z-DTPLvaYyyOpDEQLz0bdO2OLN3_ZmeR8,4978
|
|
297
|
+
scriptcollection-4.2.105.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
|
|
298
|
+
scriptcollection-4.2.105.dist-info/RECORD,,
|
|
@@ -18,6 +18,7 @@ sccreateisofilewithobfuscatedfiles = ScriptCollection.Executables:CreateISOFileW
|
|
|
18
18
|
sccreaterelease = ScriptCollection.Executables:CreateRelease
|
|
19
19
|
sccreatesimplemergewithoutrelease = ScriptCollection.Executables:CreateSimpleMergeWithoutRelease
|
|
20
20
|
sccurrentuserhaselevatedprivileges = ScriptCollection.Executables:CurrentUserHasElevatedPrivileges
|
|
21
|
+
scdownloadcachabletools = ScriptCollection.Executables:DownloadCachableTools
|
|
21
22
|
scensuredockernetworkisavailable = ScriptCollection.Executables:EnsureDockerNetworkIsAvailable
|
|
22
23
|
scespoc = ScriptCollection.Executables:Espoc
|
|
23
24
|
scextractpdfpages = ScriptCollection.Executables:ExtractPDFPages
|
|
File without changes
|
|
File without changes
|