ScriptCollection 4.2.84__py3-none-any.whl → 4.2.85__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (300) hide show
  1. Other/Build/Build.py +10 -0
  2. Other/CommonTasks.py +10 -0
  3. Other/QualityCheck/Linting.py +11 -0
  4. Other/QualityCheck/RunTestcases.py +11 -0
  5. Other/Reference/GenerateReference.py +10 -0
  6. Other/UpdateDependencies.py +10 -0
  7. ScriptCollection/AnionBuildPlatform.py +199 -0
  8. ScriptCollection/CertificateUpdater.py +155 -0
  9. ScriptCollection/Executables.py +921 -0
  10. ScriptCollection/GeneralUtilities.py +1573 -0
  11. ScriptCollection/HTTPMaintenanceOverheadHelper.py +36 -0
  12. ScriptCollection/OCIImages/AbstractImageHandler.py +38 -0
  13. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +20 -0
  14. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +20 -0
  15. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +20 -0
  16. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +20 -0
  17. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +20 -0
  18. ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +20 -0
  19. ScriptCollection/OCIImages/ConcreteImageHandlers/__init__.py +0 -0
  20. ScriptCollection/OCIImages/OCIImageManager.py +190 -0
  21. ScriptCollection/OCIImages/__init__.py +0 -0
  22. ScriptCollection/ProcessesRunner.py +43 -0
  23. ScriptCollection/ProgramRunnerBase.py +47 -0
  24. ScriptCollection/ProgramRunnerMock.py +2 -0
  25. ScriptCollection/ProgramRunnerPopen.py +57 -0
  26. ScriptCollection/ProgramRunnerSudo.py +108 -0
  27. ScriptCollection/Resources/CultureChooser/CultureChooser.js +29 -0
  28. ScriptCollection/Resources/CultureChooser/index.html +15 -0
  29. ScriptCollection/Resources/MaintenanceSite/MaintenanceSite.html +15 -0
  30. ScriptCollection/SCLog.py +115 -0
  31. ScriptCollection/ScriptCollectionCore.py +3498 -0
  32. ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +192 -0
  33. ScriptCollection/TFCPS/Docker/__init__.py +0 -0
  34. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
  35. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
  36. ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
  37. ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +547 -0
  38. ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
  39. ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +137 -0
  40. ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
  41. ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -0
  42. ScriptCollection/TFCPS/Go/__init__.py +0 -0
  43. ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +42 -0
  44. ScriptCollection/TFCPS/Maven/__init__.py +0 -0
  45. ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +232 -0
  46. ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
  47. ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +239 -0
  48. ScriptCollection/TFCPS/Python/__init__.py +0 -0
  49. ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +42 -0
  50. ScriptCollection/TFCPS/Rust/__init__.py +0 -0
  51. ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +433 -0
  52. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +135 -0
  53. ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +301 -0
  54. ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -0
  55. ScriptCollection/TFCPS/TFCPS_Generic.py +44 -0
  56. ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -0
  57. ScriptCollection/TFCPS/TFCPS_MergeToStable.py +356 -0
  58. ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -0
  59. ScriptCollection/TFCPS/TFCPS_Tools_General.py +1565 -0
  60. ScriptCollection/TFCPS/__init__.py +0 -0
  61. ScriptCollection/__init__.py +0 -0
  62. build/lib/Other/Build/Build.py +10 -0
  63. build/lib/Other/CommonTasks.py +10 -0
  64. build/lib/Other/QualityCheck/Linting.py +11 -0
  65. build/lib/Other/QualityCheck/RunTestcases.py +11 -0
  66. build/lib/Other/Reference/GenerateReference.py +10 -0
  67. build/lib/Other/UpdateDependencies.py +10 -0
  68. build/lib/ScriptCollection/AnionBuildPlatform.py +199 -0
  69. build/lib/ScriptCollection/CertificateUpdater.py +149 -0
  70. build/lib/ScriptCollection/Executables.py +921 -0
  71. build/lib/ScriptCollection/GeneralUtilities.py +1573 -0
  72. build/lib/ScriptCollection/HTTPMaintenanceOverheadHelper.py +36 -0
  73. build/lib/ScriptCollection/OCIImages/AbstractImageHandler.py +38 -0
  74. build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +20 -0
  75. build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +20 -0
  76. build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +20 -0
  77. build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +20 -0
  78. build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +20 -0
  79. build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +20 -0
  80. build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/__init__.py +0 -0
  81. build/lib/ScriptCollection/OCIImages/OCIImageManager.py +190 -0
  82. build/lib/ScriptCollection/OCIImages/__init__.py +0 -0
  83. build/lib/ScriptCollection/ProcessesRunner.py +43 -0
  84. build/lib/ScriptCollection/ProgramRunnerBase.py +47 -0
  85. build/lib/ScriptCollection/ProgramRunnerMock.py +2 -0
  86. build/lib/ScriptCollection/ProgramRunnerPopen.py +57 -0
  87. build/lib/ScriptCollection/ProgramRunnerSudo.py +108 -0
  88. build/lib/ScriptCollection/SCLog.py +115 -0
  89. build/lib/ScriptCollection/ScriptCollectionCore.py +3498 -0
  90. build/lib/ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +192 -0
  91. build/lib/ScriptCollection/TFCPS/Docker/__init__.py +0 -0
  92. build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
  93. build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
  94. build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
  95. build/lib/ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +547 -0
  96. build/lib/ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
  97. build/lib/ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +137 -0
  98. build/lib/ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
  99. build/lib/ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -0
  100. build/lib/ScriptCollection/TFCPS/Go/__init__.py +0 -0
  101. build/lib/ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +42 -0
  102. build/lib/ScriptCollection/TFCPS/Maven/__init__.py +0 -0
  103. build/lib/ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +232 -0
  104. build/lib/ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
  105. build/lib/ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +239 -0
  106. build/lib/ScriptCollection/TFCPS/Python/__init__.py +0 -0
  107. build/lib/ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +42 -0
  108. build/lib/ScriptCollection/TFCPS/Rust/__init__.py +0 -0
  109. build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +433 -0
  110. build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +135 -0
  111. build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +301 -0
  112. build/lib/ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -0
  113. build/lib/ScriptCollection/TFCPS/TFCPS_Generic.py +44 -0
  114. build/lib/ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -0
  115. build/lib/ScriptCollection/TFCPS/TFCPS_MergeToStable.py +356 -0
  116. build/lib/ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -0
  117. build/lib/ScriptCollection/TFCPS/TFCPS_Tools_General.py +1565 -0
  118. build/lib/ScriptCollection/TFCPS/__init__.py +0 -0
  119. build/lib/ScriptCollection/__init__.py +0 -0
  120. build/lib/build/lib/Other/Build/Build.py +10 -0
  121. build/lib/build/lib/Other/CommonTasks.py +10 -0
  122. build/lib/build/lib/Other/QualityCheck/Linting.py +11 -0
  123. build/lib/build/lib/Other/QualityCheck/RunTestcases.py +11 -0
  124. build/lib/build/lib/Other/Reference/GenerateReference.py +10 -0
  125. build/lib/build/lib/Other/UpdateDependencies.py +10 -0
  126. build/lib/build/lib/ScriptCollection/AnionBuildPlatform.py +199 -0
  127. build/lib/build/lib/ScriptCollection/CertificateUpdater.py +149 -0
  128. build/lib/build/lib/ScriptCollection/Executables.py +921 -0
  129. build/lib/build/lib/ScriptCollection/GeneralUtilities.py +1573 -0
  130. build/lib/build/lib/ScriptCollection/HTTPMaintenanceOverheadHelper.py +36 -0
  131. build/lib/build/lib/ScriptCollection/OCIImages/AbstractImageHandler.py +38 -0
  132. build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +20 -0
  133. build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +20 -0
  134. build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +20 -0
  135. build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +20 -0
  136. build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +20 -0
  137. build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +20 -0
  138. build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/__init__.py +0 -0
  139. build/lib/build/lib/ScriptCollection/OCIImages/OCIImageManager.py +190 -0
  140. build/lib/build/lib/ScriptCollection/OCIImages/__init__.py +0 -0
  141. build/lib/build/lib/ScriptCollection/ProcessesRunner.py +43 -0
  142. build/lib/build/lib/ScriptCollection/ProgramRunnerBase.py +47 -0
  143. build/lib/build/lib/ScriptCollection/ProgramRunnerMock.py +2 -0
  144. build/lib/build/lib/ScriptCollection/ProgramRunnerPopen.py +57 -0
  145. build/lib/build/lib/ScriptCollection/ProgramRunnerSudo.py +108 -0
  146. build/lib/build/lib/ScriptCollection/SCLog.py +115 -0
  147. build/lib/build/lib/ScriptCollection/ScriptCollectionCore.py +3498 -0
  148. build/lib/build/lib/ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +192 -0
  149. build/lib/build/lib/ScriptCollection/TFCPS/Docker/__init__.py +0 -0
  150. build/lib/build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
  151. build/lib/build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
  152. build/lib/build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
  153. build/lib/build/lib/ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +547 -0
  154. build/lib/build/lib/ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
  155. build/lib/build/lib/ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +137 -0
  156. build/lib/build/lib/ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
  157. build/lib/build/lib/ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -0
  158. build/lib/build/lib/ScriptCollection/TFCPS/Go/__init__.py +0 -0
  159. build/lib/build/lib/ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +42 -0
  160. build/lib/build/lib/ScriptCollection/TFCPS/Maven/__init__.py +0 -0
  161. build/lib/build/lib/ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +232 -0
  162. build/lib/build/lib/ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
  163. build/lib/build/lib/ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +239 -0
  164. build/lib/build/lib/ScriptCollection/TFCPS/Python/__init__.py +0 -0
  165. build/lib/build/lib/ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +42 -0
  166. build/lib/build/lib/ScriptCollection/TFCPS/Rust/__init__.py +0 -0
  167. build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +433 -0
  168. build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +135 -0
  169. build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +301 -0
  170. build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -0
  171. build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_Generic.py +44 -0
  172. build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -0
  173. build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_MergeToStable.py +356 -0
  174. build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -0
  175. build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_Tools_General.py +1565 -0
  176. build/lib/build/lib/ScriptCollection/TFCPS/__init__.py +0 -0
  177. build/lib/build/lib/ScriptCollection/__init__.py +0 -0
  178. build/lib/build/lib/build/lib/Other/Build/Build.py +10 -0
  179. build/lib/build/lib/build/lib/Other/CommonTasks.py +10 -0
  180. build/lib/build/lib/build/lib/Other/QualityCheck/Linting.py +11 -0
  181. build/lib/build/lib/build/lib/Other/QualityCheck/RunTestcases.py +11 -0
  182. build/lib/build/lib/build/lib/Other/Reference/GenerateReference.py +10 -0
  183. build/lib/build/lib/build/lib/Other/UpdateDependencies.py +10 -0
  184. build/lib/build/lib/build/lib/ScriptCollection/AnionBuildPlatform.py +199 -0
  185. build/lib/build/lib/build/lib/ScriptCollection/CertificateUpdater.py +149 -0
  186. build/lib/build/lib/build/lib/ScriptCollection/Executables.py +921 -0
  187. build/lib/build/lib/build/lib/ScriptCollection/GeneralUtilities.py +1573 -0
  188. build/lib/build/lib/build/lib/ScriptCollection/HTTPMaintenanceOverheadHelper.py +36 -0
  189. build/lib/build/lib/build/lib/ScriptCollection/OCIImages/AbstractImageHandler.py +38 -0
  190. build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +20 -0
  191. build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +20 -0
  192. build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +20 -0
  193. build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +20 -0
  194. build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +20 -0
  195. build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +20 -0
  196. build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/__init__.py +0 -0
  197. build/lib/build/lib/build/lib/ScriptCollection/OCIImages/OCIImageManager.py +190 -0
  198. build/lib/build/lib/build/lib/ScriptCollection/OCIImages/__init__.py +0 -0
  199. build/lib/build/lib/build/lib/ScriptCollection/ProcessesRunner.py +43 -0
  200. build/lib/build/lib/build/lib/ScriptCollection/ProgramRunnerBase.py +47 -0
  201. build/lib/build/lib/build/lib/ScriptCollection/ProgramRunnerMock.py +2 -0
  202. build/lib/build/lib/build/lib/ScriptCollection/ProgramRunnerPopen.py +57 -0
  203. build/lib/build/lib/build/lib/ScriptCollection/ProgramRunnerSudo.py +108 -0
  204. build/lib/build/lib/build/lib/ScriptCollection/SCLog.py +115 -0
  205. build/lib/build/lib/build/lib/ScriptCollection/ScriptCollectionCore.py +3498 -0
  206. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +192 -0
  207. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Docker/__init__.py +0 -0
  208. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
  209. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
  210. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
  211. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +547 -0
  212. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
  213. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +137 -0
  214. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
  215. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -0
  216. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Go/__init__.py +0 -0
  217. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +42 -0
  218. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Maven/__init__.py +0 -0
  219. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +232 -0
  220. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
  221. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +239 -0
  222. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Python/__init__.py +0 -0
  223. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +42 -0
  224. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Rust/__init__.py +0 -0
  225. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +433 -0
  226. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +135 -0
  227. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +301 -0
  228. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -0
  229. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_Generic.py +44 -0
  230. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -0
  231. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_MergeToStable.py +356 -0
  232. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -0
  233. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_Tools_General.py +1565 -0
  234. build/lib/build/lib/build/lib/ScriptCollection/TFCPS/__init__.py +0 -0
  235. build/lib/build/lib/build/lib/ScriptCollection/__init__.py +0 -0
  236. build/lib/build/lib/build/lib/build/lib/Other/Build/Build.py +10 -0
  237. build/lib/build/lib/build/lib/build/lib/Other/CommonTasks.py +10 -0
  238. build/lib/build/lib/build/lib/build/lib/Other/QualityCheck/Linting.py +11 -0
  239. build/lib/build/lib/build/lib/build/lib/Other/QualityCheck/RunTestcases.py +11 -0
  240. build/lib/build/lib/build/lib/build/lib/Other/Reference/GenerateReference.py +10 -0
  241. build/lib/build/lib/build/lib/build/lib/Other/UpdateDependencies.py +10 -0
  242. build/lib/build/lib/build/lib/build/lib/ScriptCollection/AnionBuildPlatform.py +199 -0
  243. build/lib/build/lib/build/lib/build/lib/ScriptCollection/CertificateUpdater.py +149 -0
  244. build/lib/build/lib/build/lib/build/lib/ScriptCollection/Executables.py +921 -0
  245. build/lib/build/lib/build/lib/build/lib/ScriptCollection/GeneralUtilities.py +1573 -0
  246. build/lib/build/lib/build/lib/build/lib/ScriptCollection/HTTPMaintenanceOverheadHelper.py +36 -0
  247. build/lib/build/lib/build/lib/build/lib/ScriptCollection/OCIImages/AbstractImageHandler.py +38 -0
  248. build/lib/build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebian.py +20 -0
  249. build/lib/build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerDebianSlim.py +20 -0
  250. build/lib/build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGeneric.py +20 -0
  251. build/lib/build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGenericV.py +20 -0
  252. build/lib/build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabCE.py +20 -0
  253. build/lib/build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/ImageHandlerGitlabEE.py +20 -0
  254. build/lib/build/lib/build/lib/build/lib/ScriptCollection/OCIImages/ConcreteImageHandlers/__init__.py +0 -0
  255. build/lib/build/lib/build/lib/build/lib/ScriptCollection/OCIImages/OCIImageManager.py +190 -0
  256. build/lib/build/lib/build/lib/build/lib/ScriptCollection/OCIImages/__init__.py +0 -0
  257. build/lib/build/lib/build/lib/build/lib/ScriptCollection/ProcessesRunner.py +43 -0
  258. build/lib/build/lib/build/lib/build/lib/ScriptCollection/ProgramRunnerBase.py +47 -0
  259. build/lib/build/lib/build/lib/build/lib/ScriptCollection/ProgramRunnerMock.py +2 -0
  260. build/lib/build/lib/build/lib/build/lib/ScriptCollection/ProgramRunnerPopen.py +57 -0
  261. build/lib/build/lib/build/lib/build/lib/ScriptCollection/ProgramRunnerSudo.py +108 -0
  262. build/lib/build/lib/build/lib/build/lib/ScriptCollection/SCLog.py +115 -0
  263. build/lib/build/lib/build/lib/build/lib/ScriptCollection/ScriptCollectionCore.py +3498 -0
  264. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Docker/TFCPS_CodeUnitSpecific_Docker.py +192 -0
  265. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Docker/__init__.py +0 -0
  266. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationBase.py +8 -0
  267. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationGenerate.py +6 -0
  268. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/DotNet/CertificateGeneratorInformationNoGenerate.py +7 -0
  269. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/DotNet/TFCPS_CodeUnitSpecific_DotNet.py +547 -0
  270. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/DotNet/__init__.py +0 -0
  271. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Flutter/TFCPS_CodeUnitSpecific_Flutter.py +137 -0
  272. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Flutter/__init__.py +0 -0
  273. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Go/TFCPS_CodeUnitSpecific_Go.py +72 -0
  274. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Go/__init__.py +0 -0
  275. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Maven/TFCPS_CodeUnitSpecific_Maven.py +42 -0
  276. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Maven/__init__.py +0 -0
  277. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/NodeJS/TFCPS_CodeUnitSpecific_NodeJS.py +232 -0
  278. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/NodeJS/__init__.py +0 -0
  279. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Python/TFCPS_CodeUnitSpecific_Python.py +239 -0
  280. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Python/__init__.py +0 -0
  281. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py +42 -0
  282. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Rust/__init__.py +0 -0
  283. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py +433 -0
  284. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py +135 -0
  285. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py +301 -0
  286. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_CreateRelease.py +98 -0
  287. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_Generic.py +44 -0
  288. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_MergeToMain.py +128 -0
  289. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_MergeToStable.py +356 -0
  290. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py +48 -0
  291. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/TFCPS_Tools_General.py +1565 -0
  292. build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/__init__.py +0 -0
  293. build/lib/build/lib/build/lib/build/lib/ScriptCollection/__init__.py +0 -0
  294. {scriptcollection-4.2.84.dist-info → scriptcollection-4.2.85.dist-info}/METADATA +1 -1
  295. scriptcollection-4.2.85.dist-info/RECORD +298 -0
  296. scriptcollection-4.2.85.dist-info/top_level.txt +1 -0
  297. scriptcollection-4.2.84.dist-info/RECORD +0 -5
  298. scriptcollection-4.2.84.dist-info/top_level.txt +0 -1
  299. {scriptcollection-4.2.84.dist-info → scriptcollection-4.2.85.dist-info}/WHEEL +0 -0
  300. {scriptcollection-4.2.84.dist-info → scriptcollection-4.2.85.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,3498 @@
1
+ from datetime import timedelta, datetime
2
+ from functools import cmp_to_key
3
+ import json
4
+ import binascii
5
+ import filecmp
6
+ import hashlib
7
+ import multiprocessing
8
+ import time
9
+ from io import BytesIO
10
+ import itertools
11
+ import copy
12
+ import zipfile
13
+ import math
14
+ import base64
15
+ import os
16
+ import sys
17
+ from html.parser import HTMLParser
18
+ from queue import Queue, Empty
19
+ from concurrent.futures import ThreadPoolExecutor
20
+ import xml.etree.ElementTree as ET
21
+ from pathlib import Path
22
+ from subprocess import Popen
23
+ import re
24
+ import shutil
25
+ from typing import IO
26
+ import fnmatch
27
+ import uuid
28
+ import tempfile
29
+ import io
30
+ import requests
31
+ import ntplib
32
+ import yaml
33
+ import qrcode
34
+ import pycdlib
35
+ import send2trash
36
+ from pypdf import PdfReader, PdfWriter
37
+ from .GeneralUtilities import GeneralUtilities,Platform
38
+ from .ProgramRunnerBase import ProgramRunnerBase
39
+ from .ProgramRunnerPopen import ProgramRunnerPopen
40
+ from .SCLog import SCLog, LogLevel
41
+
42
+ version = "4.2.85"
43
+ __version__ = version
44
+
45
+ class VSCodeWorkspaceShellTask:
46
+ label:str
47
+ description:str#nullable
48
+ work_dir:str#nullable
49
+ command:str
50
+ aliases:list[str]
51
+ allow_custom_arguments:bool
52
+
53
+ def __init__(self,label:str,description:str,work_dir:str,command:str,aliases:list[str],allow_custom_arguments:bool):
54
+ GeneralUtilities.assert_not_null(label,"label")
55
+ self.label=label
56
+ self.description=description
57
+ self.work_dir=work_dir
58
+ GeneralUtilities.assert_not_null(command,"command")
59
+ self.command=command
60
+ GeneralUtilities.assert_not_null(aliases,"aliases")
61
+ self.aliases=aliases
62
+ GeneralUtilities.assert_not_null(allow_custom_arguments,"allow_custom_arguments")
63
+ self.allow_custom_arguments=allow_custom_arguments
64
+
65
+
66
+ def serialize_for_vscode(self)->str:
67
+ aliases=",".join([f"\"{GeneralUtilities.escape_json_string_value(alias)}\"" for alias in self.aliases])
68
+
69
+ cwd:str=None
70
+ if self.work_dir is None:
71
+ cwd=GeneralUtilities.empty_string
72
+ else:
73
+ cwd=f"\"cwd\": \"{GeneralUtilities.escape_json_string_value(self.work_dir)}\""
74
+
75
+ desc:str=None
76
+ if self.description is None:
77
+ desc=GeneralUtilities.empty_string
78
+ else:
79
+ desc=f"\"description\": \"{GeneralUtilities.escape_json_string_value(self.description)}\","
80
+
81
+ result=f""" {{
82
+ "label": "{GeneralUtilities.escape_json_string_value(self.label)}",
83
+ "command": "{GeneralUtilities.escape_json_string_value(self.command)}",
84
+ "type": "shell",
85
+ "options": {{
86
+ {cwd}
87
+ }},
88
+ "aliases": [
89
+ {aliases}
90
+ ],
91
+ {desc}
92
+ "allowcustomarguments": {str(self.allow_custom_arguments).lower()}
93
+ }}"""
94
+ return result
95
+
96
+ class VSCodeWorkspaceMariaDBConnection:
97
+ name:str
98
+ previewLimit:int=50
99
+ server:str
100
+ port:int
101
+ database:str
102
+ username:str
103
+ password:str
104
+
105
+ def __init__(self,name:str,server,port,database,username,password):
106
+ GeneralUtilities.assert_not_null(name,"name")
107
+ self.name=name
108
+ GeneralUtilities.assert_not_null(server,"server")
109
+ self.server=server
110
+ GeneralUtilities.assert_not_null(port,"port")
111
+ self.port=port
112
+ GeneralUtilities.assert_not_null(database,"database")
113
+ self.database=database
114
+ GeneralUtilities.assert_not_null(username,"username")
115
+ self.username=username
116
+ GeneralUtilities.assert_not_null(password,"password")
117
+ self.password=password
118
+
119
+ def serialize_for_vscode(self)->str:
120
+ result=f""" {{
121
+ "name": "{GeneralUtilities.escape_json_string_value(self.name)}",
122
+ "mysqlOptions": {{
123
+ "authProtocol": "default",
124
+ "enableSsl": "Disabled"
125
+ }},
126
+ "previewLimit": {self.previewLimit},
127
+ "server": "{GeneralUtilities.escape_json_string_value(self.server)}",
128
+ "port": {self.port},
129
+ "driver": "MySQL",
130
+ "database": "{GeneralUtilities.escape_json_string_value(self.database)}",
131
+ "username": "{GeneralUtilities.escape_json_string_value(self.username)}",
132
+ "password": "{GeneralUtilities.escape_json_string_value(self.password)}"
133
+ }}
134
+ """
135
+ return result
136
+
137
+
138
+ class VSCodeWorkspaceMongoDBConnection:
139
+ name:str
140
+ connection_string:str
141
+
142
+ def __init__(self,name:str,connection_string:str):
143
+ GeneralUtilities.assert_not_null(name,"name")
144
+ self.name=name
145
+ GeneralUtilities.assert_not_null(connection_string,"connection_string")
146
+ self.connection_string=connection_string
147
+
148
+ def serialize_for_vscode(self)->str:
149
+ result=f""" {{
150
+ "name": "{GeneralUtilities.escape_json_string_value(self.name)}",
151
+ "connectionString": "{GeneralUtilities.escape_json_string_value(self.connection_string)}"
152
+ }}
153
+ """
154
+ return result
155
+
156
+ class ScriptCollectionCore:
157
+
158
+ # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
159
+ # Do not change this value for productive environments.
160
+ mock_program_calls: bool = False#TODO remove this variable. When someone want to mock program-calls then the ProgramRunnerMock can be used instead
161
+ # The purpose of this property is to use it when testing your code which uses scriptcollection for external program-calls.
162
+ execute_program_really_if_no_mock_call_is_defined: bool = False
163
+ __mocked_program_calls: list = None
164
+ program_runner: ProgramRunnerBase = None
165
+ call_program_runner_directly: bool = None
166
+ log: SCLog = None
167
+
168
+ def __init__(self):
169
+ self.program_runner = ProgramRunnerPopen()
170
+ self.call_program_runner_directly = None
171
+ self.__mocked_program_calls = list[ScriptCollectionCore.__MockProgramCall]()
172
+ self.log = SCLog(None, LogLevel.Warning, False)
173
+
174
+ @staticmethod
175
+ @GeneralUtilities.check_arguments
176
+ def get_scriptcollection_version() -> str:
177
+ return __version__
178
+
179
+ @GeneralUtilities.check_arguments
180
+ def get_scriptcollection_configuration_folder(self)->str:
181
+ user_folder = str(Path.home())
182
+ result = os.path.join(user_folder, ".ScriptCollection")
183
+ result=GeneralUtilities.normalize_path(result)
184
+ GeneralUtilities.ensure_directory_exists(result)
185
+ return result
186
+
187
+ def get_global_cache_folder(self)->str:
188
+ result = os.path.join(self.get_scriptcollection_configuration_folder(), "GlobalCache")
189
+ result=GeneralUtilities.normalize_path(result)
190
+ GeneralUtilities.ensure_directory_exists(result)
191
+ return result
192
+
193
+ def __get_docker_registry_credentials_file(self)->str:
194
+ result=os.path.join(self.get_global_cache_folder(),"RegistryCredentials.csv")
195
+ if not os.path.isfile(result):
196
+ GeneralUtilities.ensure_file_exists(result)
197
+ GeneralUtilities.write_lines_to_file(result,["RegistryName;Username;Password"])
198
+ return result
199
+
200
+ def __load_credentials_if_required_and_available(self,registry_url:str,registry_username:str,registry_password:str)->tuple[str,str]:
201
+ if registry_url.startswith("https://"):
202
+ registry_url=registry_url[len("https://"):]
203
+ if registry_password is None:
204
+ credential_file=self.__get_docker_registry_credentials_file()
205
+ lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
206
+ for line in lines:
207
+ splitted=line.split(";")
208
+ registry=splitted[0]
209
+ username=splitted[1]
210
+ password=splitted[2]
211
+ if registry_url==registry and (registry_username is None or username==registry_username):
212
+ registry_username=username
213
+ registry_password=password
214
+ break
215
+ else:
216
+ GeneralUtilities.assert_not_null(registry_username)
217
+ return (registry_username,registry_password)
218
+
219
+ def __get_docker_registry_credentials(self)->list[tuple[str,str,str]]:
220
+ result=[]
221
+ credential_file=self.__get_docker_registry_credentials_file()
222
+ if os.path.isfile(credential_file):
223
+ lines=GeneralUtilities.read_nonempty_lines_from_file(credential_file)[1:]
224
+ for line in lines:
225
+ splitted=line.split(";")
226
+ registry=splitted[0]
227
+ username=splitted[1]
228
+ password=splitted[2]
229
+ result.append((registry,username,password))
230
+ return result
231
+
232
+ def registry_contains_image(self,registry_url:str,image:str,registry_username:str,registry_password:str)->bool:
233
+ """This function assumes that the registry is a custom deployed docker-registry (see https://hub.docker.com/_/registry )"""
234
+ try:
235
+ if "/" in image:
236
+ image=image.rsplit("/", 1)[-1]
237
+ registry_username,registry_password=self.__load_credentials_if_required_and_available(registry_url,registry_username,registry_password)
238
+ catalog_url = f"{registry_url}/v2/_catalog"
239
+ response = requests.get(catalog_url, auth=(registry_username, registry_password),timeout=20)
240
+ response.raise_for_status() # check if statuscode = 200
241
+ data = response.json()
242
+ # expected: {"repositories": ["nginx", "myapp"]}
243
+ images = data.get("repositories", [])
244
+ if not (image in images):
245
+ return False
246
+
247
+ if self.get_tags_of_images_from_registry(registry_url,image,registry_username,registry_password)<1:
248
+ return False
249
+
250
+ return True
251
+ except Exception:
252
+ return False
253
+
254
+ def docker_platform_to_slug(self,platform_value: Platform) -> str:
255
+ if platform_value == Platform.Linux_AMD64:
256
+ return "linux-amd64"
257
+ elif platform_value == Platform.Linux_ARM64:
258
+ return "linux-arm64"
259
+ raise ValueError(f"Unsupported platform: {platform_value}")
260
+
261
+ @GeneralUtilities.check_arguments
262
+ def add_image_to_custom_docker_image_registry(
263
+ self,
264
+ remote_hub: str,
265
+ imagename_on_remote_hub: str,
266
+ own_registry_address: str,
267
+ imagename_on_own_registry: str,
268
+ tag: str,
269
+ registry_username: str,
270
+ registry_password: str,
271
+ ) -> None:
272
+ registry_username, registry_password = self.__load_credentials_if_required_and_available(remote_hub, registry_username, registry_password)
273
+ source_address = f"{remote_hub}/{imagename_on_remote_hub}:{tag}"
274
+ target_address = f"{own_registry_address}/{imagename_on_own_registry}:{tag}"
275
+ self.run_program("docker", f"buildx imagetools create --tag {target_address} {source_address}")#this does pull and push for each platform
276
+
277
+
278
+ def get_tags_of_images_from_registry(self,registry_base_url:str,image:str,registry_username:str,registry_password:str)->list[str]:
279
+ """registry_base_url must be in the format 'https://myregistry.example.com'
280
+ This function assumes that the registry is a custom deployed docker-registry (see https://hub.docker.com/_/registry )"""
281
+ registry_username,registry_password=self.__load_credentials_if_required_and_available(registry_base_url,registry_username,registry_password)
282
+ if "/" in image:
283
+ image=image.rsplit("/", 1)[-1]
284
+ if not self.registry_contains_image(registry_base_url,image,registry_username,registry_password):
285
+ return []
286
+ tags_url = f"{registry_base_url}/v2/{image}/tags/list"
287
+ response = requests.get(tags_url, auth=(registry_username, registry_password),timeout=20)
288
+ response.raise_for_status() # check if statuscode = 200
289
+ data=response.json()
290
+ # expected: {"name":"myapp","tags":["1.2.22","1.2.21","1.2.20"]}
291
+ tags = data.get("tags", [])
292
+ return tags
293
+
294
+ def registry_contains_image_with_tag(self,registry_url:str,image:str,tag:str,registry_username:str,registry_password:str)->bool:
295
+ """This function assumes that the registry is a custom deployed docker-registry (see https://hub.docker.com/_/registry )"""
296
+ registry_username,registry_password=self.__load_credentials_if_required_and_available(registry_url,registry_username,registry_password)
297
+ if "/" in image:
298
+ image=image.rsplit("/", 1)[-1]
299
+ tags=self.get_tags_of_images_from_registry(registry_url,image,registry_username,registry_password)
300
+ if tags is None:
301
+ return False
302
+ else:
303
+ result = tag in tags
304
+ return result
305
+
306
+ def login_to_defined_docker_registries(self)->None:
307
+ registries=self.__get_docker_registry_credentials()
308
+ if len(registries)==0:
309
+ self.log.log("No docker registry credentials defined. Skipping docker login.",LogLevel.Debug)
310
+ else:
311
+ for registry,username,password in registries:
312
+ arg=f"login {registry} -u {username} -p {password}"
313
+ arg_for_log=f"login {registry} -u {username} -p ***"
314
+ self.run_program("docker",arg,arguments_for_log=arg_for_log,print_live_output=self.log.loglevel==LogLevel.Debug)
315
+
316
+ @GeneralUtilities.check_arguments
317
+ def get_python_executable(self) -> str:
318
+ configured_file = os.path.join(self.get_scriptcollection_configuration_folder(), "PythonExecutable.txt")
319
+ if os.path.isfile(configured_file):
320
+ result = GeneralUtilities.read_text_from_file(configured_file).strip()
321
+ GeneralUtilities.assert_condition(os.path.isfile(result), f"Python executable does not exist: '{result}'")
322
+ elif GeneralUtilities.string_has_content(sys.executable):
323
+ result = sys.executable
324
+ else:
325
+ result = "python"
326
+ return result
327
+
328
+ @GeneralUtilities.check_arguments
329
+ def python_file_has_errors(self, file: str, working_directory: str, treat_warnings_as_errors: bool = True) -> tuple[bool, list[str]]:
330
+ errors = list()
331
+ filename = os.path.relpath(file, working_directory)
332
+ if treat_warnings_as_errors:
333
+ errorsonly_argument = GeneralUtilities.empty_string
334
+ else:
335
+ errorsonly_argument = " --errors-only"
336
+ (exit_code, stdout, stderr, _) = self.run_program("pylint", filename + errorsonly_argument, working_directory, throw_exception_if_exitcode_is_not_zero=False)
337
+ if (exit_code != 0):
338
+ errors.append(f"Linting-issues of {file}:")
339
+ errors.append(f"Pylint-exitcode: {exit_code}")
340
+ for line in GeneralUtilities.string_to_lines(stdout):
341
+ errors.append(line)
342
+ for line in GeneralUtilities.string_to_lines(stderr):
343
+ errors.append(line)
344
+ return (True, errors)
345
+
346
+ return (False, errors)
347
+
348
+ @GeneralUtilities.check_arguments
349
+ def replace_version_in_dockerfile_file(self, dockerfile: str, new_version_value: str) -> None:
350
+ GeneralUtilities.write_text_to_file(dockerfile, re.sub("ARG Version=\"\\d+\\.\\d+\\.\\d+\"", f"ARG Version=\"{new_version_value}\"", GeneralUtilities.read_text_from_file(dockerfile)))
351
+
352
+ @GeneralUtilities.check_arguments
353
+ def replace_version_in_python_file(self, file: str, new_version_value: str):
354
+ GeneralUtilities.write_text_to_file(file, re.sub("version = \"\\d+\\.\\d+\\.\\d+\"", f"version = \"{new_version_value}\"", GeneralUtilities.read_text_from_file(file)))
355
+
356
+ @GeneralUtilities.check_arguments
357
+ def replace_version_in_ini_file(self, file: str, new_version_value: str):
358
+ GeneralUtilities.write_text_to_file(file, re.sub("version = \\\"?\\d+\\.\\d+\\.\\d+\\\"?", f"version = \"{new_version_value}\"", GeneralUtilities.read_text_from_file(file)))
359
+
360
+ @GeneralUtilities.check_arguments
361
+ def replace_version_in_nuspec_file(self, nuspec_file: str, new_version: str) -> None:
362
+ # TODO use XSLT instead
363
+ versionregex = "\\d+\\.\\d+\\.\\d+"
364
+ versiononlyregex = f"^{versionregex}$"
365
+ pattern = re.compile(versiononlyregex)
366
+ if pattern.match(new_version):
367
+ GeneralUtilities.write_text_to_file(nuspec_file, re.sub(f"<version>{versionregex}<\\/version>", f"<version>{new_version}</version>", GeneralUtilities.read_text_from_file(nuspec_file)))
368
+ else:
369
+ raise ValueError(f"Version '{new_version}' does not match version-regex '{versiononlyregex}'")
370
+
371
+ @GeneralUtilities.check_arguments
372
+ def replace_version_in_csproj_file(self, csproj_file: str, current_version: str):
373
+ versionregex = "\\d+\\.\\d+\\.\\d+"
374
+ versiononlyregex = f"^{versionregex}$"
375
+ pattern = re.compile(versiononlyregex)
376
+ if pattern.match(current_version):
377
+ for tag in ["Version", "AssemblyVersion", "FileVersion"]:
378
+ GeneralUtilities.write_text_to_file(csproj_file, re.sub(f"<{tag}>{versionregex}(.\\d+)?<\\/{tag}>", f"<{tag}>{current_version}</{tag}>", GeneralUtilities.read_text_from_file(csproj_file)))
379
+ else:
380
+ raise ValueError(f"Version '{current_version}' does not match version-regex '{versiononlyregex}'")
381
+
382
+ @GeneralUtilities.check_arguments
383
+ def push_nuget_build_artifact(self, nupkg_file: str, registry_address: str, api_key: str = None):
384
+ nupkg_file_name = os.path.basename(nupkg_file)
385
+ nupkg_file_folder = os.path.dirname(nupkg_file)
386
+ argument = f"nuget push {nupkg_file_name} --force-english-output --source {registry_address}"
387
+ if api_key is not None:
388
+ argument = f"{argument} --api-key {api_key}"
389
+ self.run_program("dotnet", argument, nupkg_file_folder)
390
+
391
+ @GeneralUtilities.check_arguments
392
+ def dotnet_build(self, folder: str, projectname: str, configuration: str):
393
+ self.run_program("dotnet", f"clean -c {configuration}", folder)
394
+ self.run_program("dotnet", f"build {projectname}/{projectname}.csproj -c {configuration}", folder)
395
+
396
+ @GeneralUtilities.check_arguments
397
+ def find_file_by_extension(self, folder: str, extension_without_dot: str):
398
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
399
+ result = [file for file in self.list_content(folder, True, False, False) if file.endswith(f".{extension_without_dot}")]
400
+ result_length = len(result)
401
+ if result_length == 0:
402
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
403
+ if result_length == 1:
404
+ return result[0]
405
+ else:
406
+ raise ValueError(f"Multiple values available in folder '{folder}' with extension '{extension_without_dot}'.")
407
+
408
+ @GeneralUtilities.check_arguments
409
+ def find_last_file_by_extension(self, folder: str, extension_without_dot: str) -> str:
410
+ files: list[str] = GeneralUtilities.get_direct_files_of_folder(folder)
411
+ possible_results: list[str] = []
412
+ for file in files:
413
+ if file.endswith(f".{extension_without_dot}"):
414
+ possible_results.append(file)
415
+ result_length = len(possible_results)
416
+ if result_length == 0:
417
+ raise FileNotFoundError(f"No file available in folder '{folder}' with extension '{extension_without_dot}'.")
418
+ else:
419
+ return possible_results[-1]
420
+
421
+ @GeneralUtilities.check_arguments
422
+ def commit_is_signed_by_key(self, repository_folder: str, revision_identifier: str, key: str) -> bool:
423
+ self.is_git_or_bare_git_repository(repository_folder)
424
+ result = self.run_program("git", f"verify-commit {revision_identifier}", repository_folder, throw_exception_if_exitcode_is_not_zero=False)
425
+ if (result[0] != 0):
426
+ return False
427
+ if (not GeneralUtilities.contains_line(result[1].splitlines(), f"gpg\\:\\ using\\ [A-Za-z0-9]+\\ key\\ [A-Za-z0-9]+{key}")):
428
+ # TODO check whether this works on machines where gpg is installed in another langauge than english
429
+ return False
430
+ if (not GeneralUtilities.contains_line(result[1].splitlines(), "gpg\\:\\ Good\\ signature\\ from")):
431
+ # TODO check whether this works on machines where gpg is installed in another langauge than english
432
+ return False
433
+ return True
434
+
435
+ @GeneralUtilities.check_arguments
436
+ def get_parent_commit_ids_of_commit(self, repository_folder: str, commit_id: str) -> str:
437
+ self.is_git_or_bare_git_repository(repository_folder)
438
+ return self.run_program("git", f'log --pretty=%P -n 1 "{commit_id}"', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string).split(" ")
439
+
440
+
441
+ @GeneralUtilities.check_arguments
442
+ def get_commit_ids_between_dates(self, repository_folder: str, since: datetime, until: datetime, ignore_commits_which_are_not_in_history_of_head: bool = True) -> None:
443
+ self.is_git_or_bare_git_repository(repository_folder)
444
+ since_as_string = self.__datetime_to_string_for_git(since)
445
+ until_as_string = self.__datetime_to_string_for_git(until)
446
+ result = filter(lambda line: not GeneralUtilities.string_is_none_or_whitespace(line), self.run_program("git", f'log --since "{since_as_string}" --until "{until_as_string}" --pretty=format:"%H" --no-patch', repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1].split("\n").replace("\r", GeneralUtilities.empty_string))
447
+ if ignore_commits_which_are_not_in_history_of_head:
448
+ result = [commit_id for commit_id in result if self.git_commit_is_ancestor(repository_folder, commit_id)]
449
+ return result
450
+
451
+ @GeneralUtilities.check_arguments
452
+ def __datetime_to_string_for_git(self, datetime_object: datetime) -> str:
453
+ return datetime_object.strftime('%Y-%m-%d %H:%M:%S')
454
+
455
+ @GeneralUtilities.check_arguments
456
+ def git_commit_is_ancestor(self, repository_folder: str, ancestor: str, descendant: str = "HEAD") -> bool:
457
+ self.is_git_or_bare_git_repository(repository_folder)
458
+ result = self.run_program_argsasarray("git", ["merge-base", "--is-ancestor", ancestor, descendant], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
459
+ exit_code = result[0]
460
+ if exit_code == 0:
461
+ return True
462
+ elif exit_code == 1:
463
+ return False
464
+ else:
465
+ raise ValueError(f'Can not calculate if {ancestor} is an ancestor of {descendant} in repository {repository_folder}. Outout of "{repository_folder}> git merge-base --is-ancestor {ancestor} {descendant}": Exitcode: {exit_code}; StdOut: {result[1]}; StdErr: {result[2]}.')
466
+
467
+ @GeneralUtilities.check_arguments
468
+ def __git_changes_helper(self, repository_folder: str, arguments_as_array: list[str]) -> bool:
469
+ self.assert_is_git_repository(repository_folder)
470
+ lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", arguments_as_array, repository_folder, throw_exception_if_exitcode_is_not_zero=True)[1], False)
471
+ for line in lines:
472
+ if GeneralUtilities.string_has_content(line):
473
+ return True
474
+ return False
475
+
476
+ @GeneralUtilities.check_arguments
477
+ def git_repository_has_new_untracked_files(self, repository_folder: str):
478
+ self.assert_is_git_repository(repository_folder)
479
+ return self.__git_changes_helper(repository_folder, ["ls-files", "--exclude-standard", "--others"])
480
+
481
+ @GeneralUtilities.check_arguments
482
+ def git_repository_has_unstaged_changes_of_tracked_files(self, repository_folder: str):
483
+ self.assert_is_git_repository(repository_folder)
484
+ return self.__git_changes_helper(repository_folder, ["--no-pager", "diff"])
485
+
486
+ @GeneralUtilities.check_arguments
487
+ def git_repository_has_staged_changes(self, repository_folder: str):
488
+ self.assert_is_git_repository(repository_folder)
489
+ return self.__git_changes_helper(repository_folder, ["--no-pager", "diff", "--cached"])
490
+
491
+ @GeneralUtilities.check_arguments
492
+ def git_repository_has_uncommitted_changes(self, repository_folder: str) -> bool:
493
+ self.assert_is_git_repository(repository_folder)
494
+ if (self.git_repository_has_unstaged_changes(repository_folder)):
495
+ return True
496
+ if (self.git_repository_has_staged_changes(repository_folder)):
497
+ return True
498
+ return False
499
+
500
+ @GeneralUtilities.check_arguments
501
+ def git_repository_has_unstaged_changes(self, repository_folder: str) -> bool:
502
+ self.assert_is_git_repository(repository_folder)
503
+ if (self.git_repository_has_unstaged_changes_of_tracked_files(repository_folder)):
504
+ return True
505
+ if (self.git_repository_has_new_untracked_files(repository_folder)):
506
+ return True
507
+ return False
508
+
509
+ @GeneralUtilities.check_arguments
510
+ def git_get_commit_id(self, repository_folder: str, rev: str = "HEAD") -> str:
511
+ self.is_git_or_bare_git_repository(repository_folder)
512
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["rev-parse", "--verify", rev], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
513
+ return result[1].replace('\n', '')
514
+
515
+ @GeneralUtilities.check_arguments
516
+ def git_get_commit_date(self, repository_folder: str, rev: str = "HEAD") -> datetime:
517
+ self.is_git_or_bare_git_repository(repository_folder)
518
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", ["log","-1","--format=%ci", rev], repository_folder, throw_exception_if_exitcode_is_not_zero=True)
519
+ date_as_string = result[1].replace('\n', '')
520
+ result = datetime.strptime(date_as_string, '%Y-%m-%d %H:%M:%S %z')
521
+ return result
522
+
523
+ @GeneralUtilities.check_arguments
524
+ def git_fetch_with_retry(self, folder: str, remotename: str = "--all", amount_of_attempts: int = 5) -> None:
525
+ GeneralUtilities.retry_action(lambda: self.git_fetch(folder, remotename), amount_of_attempts)
526
+
527
+ @GeneralUtilities.check_arguments
528
+ def git_fetch(self, folder: str, remotename: str = "--all") -> None:
529
+ self.assert_is_git_repository(folder)
530
+ self.run_program_argsasarray("git", ["fetch", remotename, "--tags", "--prune"], folder, throw_exception_if_exitcode_is_not_zero=True)
531
+
532
+ @GeneralUtilities.check_arguments
533
+ def git_fetch_in_bare_repository(self, folder: str, remotename, localbranch: str, remotebranch: str) -> None:
534
+ self.assert_is_git_repository(folder)
535
+ self.run_program_argsasarray("git", ["fetch", remotename, f"{remotebranch}:{localbranch}"], folder, throw_exception_if_exitcode_is_not_zero=True)
536
+
537
+ def branch_exists(self, folder: str, branchname: str) -> bool:
538
+ self.assert_is_git_repository(folder)
539
+ result = self.run_program_argsasarray("git", ["rev-parse", "--verify", branchname], folder, throw_exception_if_exitcode_is_not_zero=False)
540
+ return result[0] == 0
541
+
542
+ @GeneralUtilities.check_arguments
543
+ def git_remove_branch(self, folder: str, branchname: str) -> None:
544
+ self.assert_is_git_repository(folder)
545
+ self.run_program("git", f"branch -D {branchname}", folder, throw_exception_if_exitcode_is_not_zero=True)
546
+
547
+ @GeneralUtilities.check_arguments
548
+ def git_push_with_retry(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet, amount_of_attempts: int = 5) -> None:
549
+ GeneralUtilities.retry_action(lambda: self.git_push(folder, remotename, localbranchname, remotebranchname, forcepush, pushalltags, verbosity), amount_of_attempts)
550
+
551
+ @GeneralUtilities.check_arguments
552
+ def git_push(self, folder: str, remotename: str, localbranchname: str, remotebranchname: str, forcepush: bool = False, pushalltags: bool = True, verbosity: LogLevel = LogLevel.Quiet,resurse_submodules:bool=False) -> None:
553
+ self.is_git_or_bare_git_repository(folder)
554
+ argument = ["push"]
555
+ if resurse_submodules:
556
+ argument = argument + ["--recurse-submodules=on-demand"]
557
+ argument = argument + [remotename, f"{localbranchname}:{remotebranchname}"]
558
+ if (forcepush):
559
+ argument.append("--force")
560
+ if (pushalltags):
561
+ argument.append("--tags")
562
+ result: tuple[int, str, str, int] = self.run_program_argsasarray("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True, print_errors_as_information=True)
563
+ return result[1].replace('\r', '').replace('\n', '')
564
+
565
+ @GeneralUtilities.check_arguments
566
+ def git_pull_with_retry(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False, amount_of_attempts: int = 5) -> None:
567
+ GeneralUtilities.retry_action(lambda: self.git_pull(folder, remote, localbranchname, remotebranchname), amount_of_attempts)
568
+
569
+ @GeneralUtilities.check_arguments
570
+ def git_pull(self, folder: str, remote: str, localbranchname: str, remotebranchname: str, force: bool = False) -> None:
571
+ self.is_git_or_bare_git_repository(folder)
572
+ argument = f"pull {remote} {remotebranchname}:{localbranchname}"
573
+ if force:
574
+ argument = f"{argument} --force"
575
+ self.run_program("git", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
576
+
577
+ @GeneralUtilities.check_arguments
578
+ def git_list_remote_branches(self, folder: str, remote: str, fetch: bool) -> list[str]:
579
+ self.is_git_or_bare_git_repository(folder)
580
+ if fetch:
581
+ self.git_fetch(folder, remote)
582
+ run_program_result = self.run_program("git", f"branch -rl {remote}/*", folder, throw_exception_if_exitcode_is_not_zero=True)
583
+ output = GeneralUtilities.string_to_lines(run_program_result[1])
584
+ result = list[str]()
585
+ for item in output:
586
+ striped_item = item.strip()
587
+ if GeneralUtilities.string_has_content(striped_item):
588
+ branch: str = None
589
+ if " " in striped_item:
590
+ branch = striped_item.split(" ")[0]
591
+ else:
592
+ branch = striped_item
593
+ branchname = branch[len(remote)+1:]
594
+ if branchname != "HEAD":
595
+ result.append(branchname)
596
+ return result
597
+
598
+ @GeneralUtilities.check_arguments
599
+ def git_clone(self, clone_target_folder: str, remote_repository_path: str, include_submodules: bool = True, mirror: bool = False) -> None:
600
+ if (os.path.isdir(clone_target_folder)):
601
+ pass # TODO throw error
602
+ else:
603
+ args = ["clone", remote_repository_path, clone_target_folder]
604
+ if include_submodules:
605
+ args.append("--recurse-submodules")
606
+ args.append("--remote-submodules")
607
+ if mirror:
608
+ args.append("--mirror")
609
+ self.run_program_argsasarray("git", args, os.getcwd(), throw_exception_if_exitcode_is_not_zero=True)
610
+
611
+ @GeneralUtilities.check_arguments
612
+ def git_get_all_remote_names(self, directory: str) -> list[str]:
613
+ self.is_git_or_bare_git_repository(directory)
614
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote"], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
615
+ return result
616
+
617
+ @GeneralUtilities.check_arguments
618
+ def git_get_remote_url(self, directory: str, remote_name: str) -> str:
619
+ self.is_git_or_bare_git_repository(directory)
620
+ result = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["remote", "get-url", remote_name], directory, throw_exception_if_exitcode_is_not_zero=True)[1], False)
621
+ return result[0].replace('\n', '')
622
+
623
+ @GeneralUtilities.check_arguments
624
+ def repository_has_remote_with_specific_name(self, directory: str, remote_name: str) -> bool:
625
+ self.is_git_or_bare_git_repository(directory)
626
+ return remote_name in self.git_get_all_remote_names(directory)
627
+
628
+ @GeneralUtilities.check_arguments
629
+ def git_add_or_set_remote_address(self, directory: str, remote_name: str, remote_address: str) -> None:
630
+ self.assert_is_git_repository(directory)
631
+ if (self.repository_has_remote_with_specific_name(directory, remote_name)):
632
+ self.run_program_argsasarray("git", ['remote', 'set-url', 'remote_name', remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
633
+ else:
634
+ self.run_program_argsasarray("git", ['remote', 'add', remote_name, remote_address], directory, throw_exception_if_exitcode_is_not_zero=True)
635
+
636
+ @GeneralUtilities.check_arguments
637
+ def git_stage_all_changes(self, directory: str) -> None:
638
+ self.assert_is_git_repository(directory)
639
+ self.run_program_argsasarray("git", ["add", "-A"], directory, throw_exception_if_exitcode_is_not_zero=True)
640
+
641
+ @GeneralUtilities.check_arguments
642
+ def git_unstage_all_changes(self, directory: str) -> None:
643
+ self.assert_is_git_repository(directory)
644
+ self.run_program_argsasarray("git", ["reset"], directory, throw_exception_if_exitcode_is_not_zero=True)
645
+ # TODO check if this will also be done for submodules
646
+
647
+ @GeneralUtilities.check_arguments
648
+ def git_stage_file(self, directory: str, file: str) -> None:
649
+ self.assert_is_git_repository(directory)
650
+ self.run_program_argsasarray("git", ['stage', file], directory, throw_exception_if_exitcode_is_not_zero=True)
651
+
652
+ @GeneralUtilities.check_arguments
653
+ def git_unstage_file(self, directory: str, file: str) -> None:
654
+ self.assert_is_git_repository(directory)
655
+ self.run_program_argsasarray("git", ['reset', file], directory, throw_exception_if_exitcode_is_not_zero=True)
656
+
657
+ @GeneralUtilities.check_arguments
658
+ def git_discard_unstaged_changes_of_file(self, directory: str, file: str) -> None:
659
+ """Caution: This method works really only for 'changed' files yet. So this method does not work properly for new or renamed files."""
660
+ self.assert_is_git_repository(directory)
661
+ self.run_program_argsasarray("git", ['checkout', file], directory, throw_exception_if_exitcode_is_not_zero=True)
662
+
663
+ @GeneralUtilities.check_arguments
664
+ def git_discard_all_unstaged_changes(self, directory: str) -> None:
665
+ """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
666
+ self.assert_is_git_repository(directory)
667
+ self.run_program_argsasarray("git", ['clean', '-df'], directory, throw_exception_if_exitcode_is_not_zero=True)
668
+ self.run_program_argsasarray("git", ['checkout', '.'], directory, throw_exception_if_exitcode_is_not_zero=True)
669
+ # TODO check if this will also be done for submodules
670
+
671
+ @GeneralUtilities.check_arguments
672
+ def git_commit(self, directory: str, message: str = "Saved changes.", author_name: str = None, author_email: str = None, stage_all_changes: bool = True, no_changes_behavior: int = 0,commit_message_body:str=None) -> str:
673
+ """no_changes_behavior=0 => No commit; no_changes_behavior=1 => Commit anyway; no_changes_behavior=2 => Exception"""
674
+ self.assert_is_git_repository(directory)
675
+ author_name = GeneralUtilities.str_none_safe(author_name).strip()
676
+ author_email = GeneralUtilities.str_none_safe(author_email).strip()
677
+ argument = ['commit', '--quiet', '--allow-empty', '--message', message]
678
+ if commit_message_body is not None:
679
+ argument.extend(['--message', commit_message_body])
680
+ if (GeneralUtilities.string_has_content(author_name)):
681
+ argument.append(f'--author="{author_name} <{author_email}>"')
682
+ git_repository_has_uncommitted_changes = self.git_repository_has_uncommitted_changes(directory)
683
+
684
+ if git_repository_has_uncommitted_changes:
685
+ do_commit = True
686
+ if stage_all_changes:
687
+ self.git_stage_all_changes(directory)
688
+ else:
689
+ if no_changes_behavior == 0:
690
+ self.log.log(f"Commit '{message}' will not be done because there are no changes to commit in repository '{directory}'", LogLevel.Debug)
691
+ do_commit = False
692
+ elif no_changes_behavior == 1:
693
+ self.log.log(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will be done anyway.", LogLevel.Debug)
694
+ do_commit = True
695
+ elif no_changes_behavior == 2:
696
+ raise RuntimeError(f"There are no changes to commit in repository '{directory}'. Commit '{message}' will not be done.")
697
+ else:
698
+ raise ValueError(f"Unknown value for no_changes_behavior: {GeneralUtilities.str_none_safe(no_changes_behavior)}")
699
+
700
+ if do_commit:
701
+ self.log.log(f"Commit changes in '{directory}'", LogLevel.Information)
702
+ self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
703
+
704
+ return self.git_get_commit_id(directory)
705
+
706
+ def search_repository_folder(self,some_file_in_repository:str)->str:
707
+ current_path:str=os.path.dirname(some_file_in_repository)
708
+ enabled:bool=True
709
+ while enabled:
710
+ try:
711
+ current_path=GeneralUtilities.resolve_relative_path("..",current_path)
712
+ if self.is_git_repository(current_path):
713
+ return current_path
714
+ except:
715
+ enabled=False
716
+ raise ValueError(f"Can not find git-repository for folder \"{some_file_in_repository}\".")
717
+
718
+
719
+ @GeneralUtilities.check_arguments
720
+ def git_create_tag(self, directory: str, target_for_tag: str, tag: str, sign: bool = False, message: str = None) -> None:
721
+ self.is_git_or_bare_git_repository(directory)
722
+ argument = ["tag", tag, target_for_tag]
723
+ if sign:
724
+ if message is None:
725
+ message = f"Created {target_for_tag}"
726
+ argument.extend(["-s", '-m', message])
727
+ self.run_program_argsasarray("git", argument, directory, throw_exception_if_exitcode_is_not_zero=True)
728
+
729
+ @GeneralUtilities.check_arguments
730
+ def git_delete_tag(self, directory: str, tag: str) -> None:
731
+ self.is_git_or_bare_git_repository(directory)
732
+ self.run_program_argsasarray("git", ["tag", "--delete", tag], directory, throw_exception_if_exitcode_is_not_zero=True)
733
+
734
+ @GeneralUtilities.check_arguments
735
+ def git_checkout(self, directory: str, rev: str, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> None:
736
+ self.assert_is_git_repository(directory)
737
+ if assert_no_uncommitted_changes:
738
+ GeneralUtilities.assert_condition(not self.git_repository_has_uncommitted_changes(directory), f"Repository \"{directory}\" has uncommitted changes.")
739
+ self.run_program_argsasarray("git", ["checkout", rev], directory, throw_exception_if_exitcode_is_not_zero=True)
740
+ self.run_program_argsasarray("git", ["submodule", "update", "--recursive"], directory, throw_exception_if_exitcode_is_not_zero=True)
741
+ commit_id=self.git_get_commit_id(directory,"HEAD")
742
+ self.log.log(f"Checked out {commit_id} in \"{directory}\".", LogLevel.Debug)
743
+ if undo_all_changes_after_checkout:
744
+ self.git_undo_all_changes(directory)
745
+
746
+ @GeneralUtilities.check_arguments
747
+ def merge_repository(self, repository_folder: str, remote: str, branch: str):
748
+ GeneralUtilities.assert_condition(not self.git_repository_has_uncommitted_changes(repository_folder),f"Can not merge. There are uncommitted changes in \"{repository_folder}\".")
749
+ is_pullable: bool = self.git_commit_is_ancestor(repository_folder, branch, f"{remote}/{branch}")
750
+ if is_pullable:
751
+ self.git_pull(repository_folder, remote, branch, branch)
752
+ uncommitted_changes = self.git_repository_has_uncommitted_changes(repository_folder)
753
+ GeneralUtilities.assert_condition(not uncommitted_changes, f"Pulling remote \"{remote}\" in \"{repository_folder}\" caused new uncommitted files.")
754
+ self.git_checkout(repository_folder, branch)
755
+ self.git_fetch(repository_folder, remote)
756
+ self.git_merge(repository_folder, f"{remote}/{branch}", branch)
757
+ self.git_push_with_retry(repository_folder, remote, branch, branch)
758
+ self.git_checkout(repository_folder, branch)
759
+ #TODO opeional: checkfor merge conflicts and if there is one merge conflict print a warning
760
+
761
+ @GeneralUtilities.check_arguments
762
+ def git_merge_abort(self, directory: str) -> None:
763
+ self.assert_is_git_repository(directory)
764
+ self.run_program_argsasarray("git", ["merge", "--abort"], directory, throw_exception_if_exitcode_is_not_zero=True)
765
+
766
+ @GeneralUtilities.check_arguments
767
+ def git_merge(self, directory: str, sourcebranch: str, targetbranch: str, fastforward: bool = True, commit: bool = True, commit_message: str = None, undo_all_changes_after_checkout: bool = True, assert_no_uncommitted_changes: bool = True) -> str:
768
+ self.assert_is_git_repository(directory)
769
+ self.git_checkout(directory, targetbranch, undo_all_changes_after_checkout, assert_no_uncommitted_changes)
770
+ args = ["merge"]
771
+ if not commit:
772
+ args.append("--no-commit")
773
+ if not fastforward:
774
+ args.append("--no-ff")
775
+ if commit_message is not None:
776
+ args.append("-m")
777
+ args.append(commit_message)
778
+ args.append(sourcebranch)
779
+ self.run_program_argsasarray("git", args, directory, throw_exception_if_exitcode_is_not_zero=True)
780
+ self.run_program_argsasarray("git", ["submodule", "update"], directory, throw_exception_if_exitcode_is_not_zero=True)
781
+ return self.git_get_commit_id(directory)
782
+
783
+ @GeneralUtilities.check_arguments
784
+ def git_undo_all_changes(self, directory: str) -> None:
785
+ """Caution: This function executes 'git clean -df'. This can delete files which maybe should not be deleted. Be aware of that."""
786
+ self.assert_is_git_repository(directory)
787
+ self.git_unstage_all_changes(directory)
788
+ self.git_discard_all_unstaged_changes(directory)
789
+
790
+ @GeneralUtilities.check_arguments
791
+ def git_fetch_or_clone_all_in_directory(self, source_directory: str, target_directory: str) -> None:
792
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(source_directory):
793
+ foldername = os.path.basename(subfolder)
794
+ if self.is_git_repository(subfolder):
795
+ source_repository = subfolder
796
+ target_repository = os.path.join(target_directory, foldername)
797
+ if os.path.isdir(target_directory):
798
+ # fetch
799
+ self.git_fetch(target_directory)
800
+ else:
801
+ # clone
802
+ self.git_clone(target_repository, source_repository, include_submodules=True, mirror=True)
803
+
804
+ def get_git_submodules(self, directory: str) -> list[str]:
805
+ self.is_git_or_bare_git_repository(directory)
806
+ e = self.run_program("git", "submodule status", directory)
807
+ result = []
808
+ for submodule_line in GeneralUtilities.string_to_lines(e[1], False, True):
809
+ result.append(submodule_line.split(' ')[1])
810
+ return result
811
+
812
+ @GeneralUtilities.check_arguments
813
+ def file_is_git_ignored(self, file_in_repository: str, repositorybasefolder: str) -> None:
814
+ self.is_git_or_bare_git_repository(repositorybasefolder)
815
+ exit_code = self.run_program_argsasarray("git", ['check-ignore', file_in_repository], repositorybasefolder, throw_exception_if_exitcode_is_not_zero=False)[0]
816
+ if (exit_code == 0):
817
+ return True
818
+ if (exit_code == 1):
819
+ return False
820
+ raise ValueError(f"Unable to calculate whether '{file_in_repository}' in repository '{repositorybasefolder}' is ignored due to git-exitcode {exit_code}.")
821
+
822
+ @GeneralUtilities.check_arguments
823
+ def git_discard_all_changes(self, repository: str) -> None:
824
+ self.assert_is_git_repository(repository)
825
+ self.run_program_argsasarray("git", ["reset", "HEAD", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
826
+ self.run_program_argsasarray("git", ["checkout", "."], repository, throw_exception_if_exitcode_is_not_zero=True)
827
+
828
+ @GeneralUtilities.check_arguments
829
+ def git_get_current_branch_name(self, repository: str) -> str:
830
+ self.assert_is_git_repository(repository)
831
+ result = self.run_program_argsasarray("git", ["rev-parse", "--abbrev-ref", "HEAD"], repository, throw_exception_if_exitcode_is_not_zero=True)
832
+ return result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
833
+
834
+ @GeneralUtilities.check_arguments
835
+ def git_get_commitid_of_tag(self, repository: str, tag: str) -> str:
836
+ self.is_git_or_bare_git_repository(repository)
837
+ stdout = self.run_program_argsasarray("git", ["rev-list", "-n", "1", tag], repository)
838
+ result = stdout[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
839
+ return result
840
+
841
+ @GeneralUtilities.check_arguments
842
+ def git_get_tags(self, repository: str) -> list[str]:
843
+ self.is_git_or_bare_git_repository(repository)
844
+ tags = [line.replace("\r", GeneralUtilities.empty_string) for line in self.run_program_argsasarray(
845
+ "git", ["tag"], repository)[1].split("\n") if len(line) > 0]
846
+ return tags
847
+
848
+ @GeneralUtilities.check_arguments
849
+ def git_move_tags_to_another_branch(self, repository: str, tag_source_branch: str, tag_target_branch: str, sign: bool = False, message: str = None) -> None:
850
+ self.is_git_or_bare_git_repository(repository)
851
+ tags = self.git_get_tags(repository)
852
+ tags_count = len(tags)
853
+ counter = 0
854
+ for tag in tags:
855
+ counter = counter+1
856
+ self.log.log(f"Process tag {counter}/{tags_count}.", LogLevel.Information)
857
+ # tag is on source-branch
858
+ if self.git_commit_is_ancestor(repository, tag, tag_source_branch):
859
+ commit_id_old = self.git_get_commit_id(repository, tag)
860
+ commit_date: datetime = self.git_get_commit_date(repository, commit_id_old)
861
+ date_as_string = self.__datetime_to_string_for_git(commit_date)
862
+ search_commit_result = self.run_program_argsasarray("git", ["log", f'--after="{date_as_string}"', f'--before="{date_as_string}"', "--pretty=format:%H", tag_target_branch], repository, throw_exception_if_exitcode_is_not_zero=False)
863
+ if search_commit_result[0] != 0 or not GeneralUtilities.string_has_nonwhitespace_content(search_commit_result[1]):
864
+ raise ValueError(f"Can not calculate corresponding commit for tag '{tag}'.")
865
+ commit_id_new = search_commit_result[1]
866
+ self.git_delete_tag(repository, tag)
867
+ self.git_create_tag(repository, commit_id_new, tag, sign, message)
868
+
869
+ @GeneralUtilities.check_arguments
870
+ def get_current_git_branch_has_tag(self, repository_folder: str) -> bool:
871
+ self.is_git_or_bare_git_repository(repository_folder)
872
+ result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)
873
+ return result[0] == 0
874
+
875
+ @GeneralUtilities.check_arguments
876
+ def get_latest_git_tag(self, repository_folder: str) -> str:
877
+ self.is_git_or_bare_git_repository(repository_folder)
878
+ result = self.run_program_argsasarray("git", ["describe", "--tags", "--abbrev=0"], repository_folder)
879
+ result = result[1].replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
880
+ return result
881
+
882
+ @GeneralUtilities.check_arguments
883
+ def get_staged_or_committed_git_ignored_files(self, repository_folder: str) -> list[str]:
884
+ self.assert_is_git_repository(repository_folder)
885
+ temp_result = self.run_program_argsasarray("git", ["ls-files", "-i", "-c", "--exclude-standard"], repository_folder)
886
+ temp_result = temp_result[1].replace("\r", GeneralUtilities.empty_string)
887
+ result = [line for line in temp_result.split("\n") if len(line) > 0]
888
+ return result
889
+
890
+ @GeneralUtilities.check_arguments
891
+ def git_repository_has_commits(self, repository_folder: str) -> bool:
892
+ self.assert_is_git_repository(repository_folder)
893
+ return self.run_program_argsasarray("git", ["rev-parse", "--verify", "HEAD"], repository_folder, throw_exception_if_exitcode_is_not_zero=False)[0] == 0
894
+
895
+ @GeneralUtilities.check_arguments
896
+ def run_git_command_in_repository_and_submodules(self, repository_folder: str, arguments: list[str],print_live_output:bool) -> None:
897
+ GeneralUtilities.assert_condition(self.is_git_or_bare_git_repository(repository_folder),f"\"{repository_folder}\" is not a git-repository.")
898
+ self.log.log("Run \"git "+" ".join(arguments)+f"\" in {repository_folder} and its submodules...",LogLevel.Debug)
899
+ self.run_program_argsasarray("git", arguments, repository_folder,print_live_output=print_live_output)
900
+ if not self.is_bare_git_repository(repository_folder) and 0<len(self.get_git_submodules(repository_folder)):
901
+ self.run_program_argsasarray("git", ["submodule", "foreach", "--recursive", "git"]+arguments, repository_folder,print_live_output=print_live_output)
902
+
903
+ @GeneralUtilities.check_arguments
904
+ def export_filemetadata(self, folder: str, target_file: str, encoding: str = "utf-8", filter_function=None) -> None:
905
+ folder = GeneralUtilities.resolve_relative_path_from_current_working_directory(folder)
906
+ lines = list()
907
+ path_prefix = len(folder)+1
908
+ items = dict()
909
+ for item in GeneralUtilities.get_all_folders_of_folder(folder):
910
+ items[item] = "d"
911
+ for item in GeneralUtilities.get_all_files_of_folder(folder):
912
+ items[item] = "f"
913
+ for file_or_folder, item_type in items.items():
914
+ truncated_file = file_or_folder[path_prefix:]
915
+ if (filter_function is None or filter_function(folder, truncated_file)):
916
+ owner_and_permisssion = self.get_file_owner_and_file_permission(file_or_folder)
917
+ user = owner_and_permisssion[0]
918
+ permissions = owner_and_permisssion[1]
919
+ lines.append(f"{truncated_file};{item_type};{user};{permissions}")
920
+ lines = sorted(lines, key=str.casefold)
921
+ with open(target_file, "w", encoding=encoding) as file_object:
922
+ file_object.write("\n".join(lines))
923
+
924
+ @GeneralUtilities.check_arguments
925
+ def escape_git_repositories_in_folder(self, folder: str) -> dict[str, str]:
926
+ return self.__escape_git_repositories_in_folder_internal(folder, dict[str, str]())
927
+
928
+ @GeneralUtilities.check_arguments
929
+ def __escape_git_repositories_in_folder_internal(self, folder: str, renamed_items: dict[str, str]) -> dict[str, str]:
930
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
931
+ filename = os.path.basename(file)
932
+ if ".git" in filename:
933
+ new_name = filename.replace(".git", ".gitx")
934
+ target = os.path.join(folder, new_name)
935
+ os.rename(file, target)
936
+ renamed_items[target] = file
937
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
938
+ foldername = os.path.basename(subfolder)
939
+ if ".git" in foldername:
940
+ new_name = foldername.replace(".git", ".gitx")
941
+ subfolder2 = os.path.join(str(Path(subfolder).parent), new_name)
942
+ os.rename(subfolder, subfolder2)
943
+ renamed_items[subfolder2] = subfolder
944
+ else:
945
+ subfolder2 = subfolder
946
+ self.__escape_git_repositories_in_folder_internal(subfolder2, renamed_items)
947
+ return renamed_items
948
+
949
+ @GeneralUtilities.check_arguments
950
+ def deescape_git_repositories_in_folder(self, renamed_items: dict[str, str]):
951
+ for renamed_item, original_name in renamed_items.items():
952
+ os.rename(renamed_item, original_name)
953
+
954
+ @GeneralUtilities.check_arguments
955
+ def is_git_repository(self, folder: str) -> bool:
956
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
957
+ folder=folder.replace("\\","/")
958
+ if folder.endswith("/"):
959
+ folder = folder[:-1]
960
+ if not self.is_folder(folder):
961
+ raise ValueError(f"Folder '{folder}' does not exist.")
962
+ git_folder_path = f"{folder}/.git"
963
+ return self.is_folder(git_folder_path) or self.is_file(git_folder_path)
964
+
965
+ @GeneralUtilities.check_arguments
966
+ def is_bare_git_repository(self, folder: str) -> bool:
967
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
968
+ if folder.endswith("/") or folder.endswith("\\"):
969
+ folder = folder[:-1]
970
+ if not self.is_folder(folder):
971
+ raise ValueError(f"Folder '{folder}' does not exist.")
972
+ return folder.endswith(".git")
973
+
974
+ @GeneralUtilities.check_arguments
975
+ def is_git_or_bare_git_repository(self, folder: str) -> bool:
976
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
977
+ return self.is_git_repository(folder) or self.is_bare_git_repository(folder)
978
+
979
+ @GeneralUtilities.check_arguments
980
+ def assert_is_git_repository(self, folder: str) -> str:
981
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
982
+ GeneralUtilities.assert_condition(self.is_git_repository(folder), f"'{folder}' is not a git-repository.")
983
+
984
+ @GeneralUtilities.check_arguments
985
+ def convert_git_repository_to_bare_repository(self, repository_folder: str):
986
+ repository_folder = repository_folder.replace("\\", "/")
987
+ self.assert_is_git_repository(repository_folder)
988
+ git_folder = repository_folder + "/.git"
989
+ if not self.is_folder(git_folder):
990
+ raise ValueError(f"Converting '{repository_folder}' to a bare repository not possible. The folder '{git_folder}' does not exist. Converting is currently only supported when the git-folder is a direct folder in a repository and not a reference to another location.")
991
+ target_folder: str = repository_folder + ".git"
992
+ GeneralUtilities.ensure_directory_exists(target_folder)
993
+ GeneralUtilities.move_content_of_folder(git_folder, target_folder)
994
+ GeneralUtilities.ensure_directory_does_not_exist(repository_folder)
995
+ self.run_program_argsasarray("git", ["config", "--bool", "core.bare", "true"], target_folder)
996
+
997
+ @GeneralUtilities.check_arguments
998
+ def assert_no_uncommitted_changes(self, repository_folder: str):
999
+ if self.git_repository_has_uncommitted_changes(repository_folder):
1000
+ raise ValueError(f"Repository '{repository_folder}' has uncommitted changes.")
1001
+
1002
+ @GeneralUtilities.check_arguments
1003
+ def list_content(self, path: str, include_files: bool, include_folder: bool, printonlynamewithoutpath: bool) -> list[str]:
1004
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1005
+ result: list[str] = []
1006
+ if self.program_runner.will_be_executed_locally():
1007
+ if include_files:
1008
+ result = result + GeneralUtilities.get_direct_files_of_folder(path)
1009
+ if include_folder:
1010
+ result = result + GeneralUtilities.get_direct_folders_of_folder(path)
1011
+ else:
1012
+ arguments = ["--path", path]
1013
+ if not include_files:
1014
+ arguments = arguments+["--excludefiles"]
1015
+ if not include_folder:
1016
+ arguments = arguments+["--excludedirectories"]
1017
+ if printonlynamewithoutpath:
1018
+ arguments = arguments+["--printonlynamewithoutpath"]
1019
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sclistfoldercontent", arguments)
1020
+ if exit_code == 0:
1021
+ for line in stdout.split("\n"):
1022
+ normalized_line = line.replace("\r", "")
1023
+ result.append(normalized_line)
1024
+ else:
1025
+ raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
1026
+ result = [item for item in result if GeneralUtilities.string_has_nonwhitespace_content(item)]
1027
+ return result
1028
+
1029
+ @GeneralUtilities.check_arguments
1030
+ def is_file(self, path: str) -> bool:
1031
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1032
+ if self.program_runner.will_be_executed_locally():
1033
+ return os.path.isfile(path) # works only locally, but much more performant than always running an external program
1034
+ else:
1035
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scfileexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1036
+ if exit_code == 0:
1037
+ return True
1038
+ elif exit_code == 1:
1039
+ raise ValueError(f"Not calculatable whether file '{path}' exists. StdErr: '{stderr}'")
1040
+ elif exit_code == 2:
1041
+ return False
1042
+ raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
1043
+
1044
+ @GeneralUtilities.check_arguments
1045
+ def get_size(self, path: str) -> int:
1046
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1047
+ if self.program_runner.will_be_executed_locally():
1048
+ return os.path.getsize(path) # works only locally, but much more performant than always running an external program
1049
+ else:
1050
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("scgetsize", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1051
+ if exit_code == 0:
1052
+ return int(stdout.replace("\r","").replace("\n","").strip())
1053
+ else:
1054
+ raise ValueError(f"Fatal error occurrs while checking whether file '{path}' exists. StdErr: '{stderr}'")
1055
+
1056
+ @GeneralUtilities.check_arguments
1057
+ def is_folder(self, path: str) -> bool:
1058
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1059
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
1060
+ return os.path.isdir(path)
1061
+ else:
1062
+ exit_code, _, stderr, _ = self.run_program_argsasarray("scfolderexists", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1063
+ if exit_code == 0:
1064
+ return True
1065
+ elif exit_code == 1:
1066
+ raise ValueError(f"Not calculatable whether folder '{path}' exists. StdErr: '{stderr}'")
1067
+ elif exit_code == 2:
1068
+ return False
1069
+ raise ValueError(f"Fatal error occurrs while checking whether folder '{path}' exists. StdErr: '{stderr}'")
1070
+
1071
+ @GeneralUtilities.check_arguments
1072
+ def get_file_content(self, path: str, encoding: str = "utf-8") -> str:
1073
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1074
+ if self.program_runner.will_be_executed_locally():
1075
+ return GeneralUtilities.read_text_from_file(path, encoding)
1076
+ else:
1077
+ result = self.run_program_argsasarray("scprintfilecontent", ["--path", path, "--encofing", encoding]) # works platform-indepent
1078
+ return result[1].replace("\\n", "\n")
1079
+
1080
+ @GeneralUtilities.check_arguments
1081
+ def set_file_content(self, path: str, content: str, encoding: str = "utf-8") -> None:
1082
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1083
+ if self.program_runner.will_be_executed_locally():
1084
+ GeneralUtilities.write_text_to_file(path, content, encoding)
1085
+ else:
1086
+ content_bytes = content.encode('utf-8')
1087
+ base64_bytes = base64.b64encode(content_bytes)
1088
+ base64_string = base64_bytes.decode('utf-8')
1089
+ self.run_program_argsasarray("scsetfilecontent", ["--path", path, "--argumentisinbase64", "--content", base64_string]) # works platform-indepent
1090
+
1091
+ @GeneralUtilities.check_arguments
1092
+ def remove(self, path: str) -> None:
1093
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1094
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
1095
+ if os.path.isdir(path):
1096
+ GeneralUtilities.ensure_directory_does_not_exist(path)
1097
+ if os.path.isfile(path):
1098
+ GeneralUtilities.ensure_file_does_not_exist(path)
1099
+ else:
1100
+ if self.is_file(path):
1101
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("scremovefile", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1102
+ if exit_code != 0:
1103
+ raise ValueError(f"Fatal error occurrs while removing file '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1104
+ if self.is_folder(path):
1105
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("scremovefolder", ["--path", path], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1106
+ if exit_code != 0:
1107
+ raise ValueError(f"Fatal error occurrs while removing folder '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1108
+
1109
+ @GeneralUtilities.check_arguments
1110
+ def rename(self, source: str, target: str) -> None:
1111
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1112
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
1113
+ os.rename(source, target)
1114
+ else:
1115
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("screname", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1116
+ if exit_code != 0:
1117
+ raise ValueError(f"Fatal error occurrs while renaming '{source}' to '{target}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1118
+
1119
+ @GeneralUtilities.check_arguments
1120
+ def copy(self, source: str, target: str) -> None:
1121
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1122
+ if self.program_runner.will_be_executed_locally(): # works only locally, but much more performant than always running an external program
1123
+ if os.path.isfile(target) or os.path.isdir(target):
1124
+ raise ValueError(f"Can not copy to '{target}' because the target already exists.")
1125
+ if os.path.isfile(source):
1126
+ shutil.copyfile(source, target)
1127
+ elif os.path.isdir(source):
1128
+ GeneralUtilities.ensure_directory_exists(target)
1129
+ GeneralUtilities.copy_content_of_folder(source, target)
1130
+ else:
1131
+ raise ValueError(f"'{source}' can not be copied because the path does not exist.")
1132
+ else:
1133
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sccopy", ["--source", source, "--target", target], throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1134
+ if exit_code != 0:
1135
+ raise ValueError(f"Fatal error occurrs while copying '{source}' to '{target}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1136
+
1137
+ @GeneralUtilities.check_arguments
1138
+ def create_file(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
1139
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1140
+ if self.program_runner.will_be_executed_locally():
1141
+ if not os.path.isabs(path):
1142
+ path = os.path.join(os.getcwd(), path)
1143
+
1144
+ if os.path.isfile(path) and error_if_already_exists:
1145
+ raise ValueError(f"File '{path}' already exists.")
1146
+
1147
+ # TODO maybe it should be checked if there is a folder with the same path which already exists.
1148
+
1149
+ folder = os.path.dirname(path)
1150
+
1151
+ if not os.path.isdir(folder):
1152
+ if create_necessary_folder:
1153
+ GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
1154
+ else:
1155
+ raise ValueError(f"Folder '{folder}' does not exist.")
1156
+
1157
+ GeneralUtilities.ensure_file_exists(path)
1158
+ else:
1159
+ arguments = ["--path", path]
1160
+
1161
+ if error_if_already_exists:
1162
+ arguments = arguments+["--errorwhenexists"]
1163
+
1164
+ if create_necessary_folder:
1165
+ arguments = arguments+["--createnecessaryfolder"]
1166
+
1167
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sccreatefile", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1168
+ if exit_code != 0:
1169
+ raise ValueError(f"Fatal error occurrs while create file '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1170
+
1171
+ @GeneralUtilities.check_arguments
1172
+ def create_folder(self, path: str, error_if_already_exists: bool, create_necessary_folder: bool) -> None:
1173
+ """This function works platform-independent also for non-local-executions if the ScriptCollection commandline-commands are available as global command on the target-system."""
1174
+ if self.program_runner.will_be_executed_locally():
1175
+ if not os.path.isabs(path):
1176
+ path = os.path.join(os.getcwd(), path)
1177
+
1178
+ if os.path.isdir(path) and error_if_already_exists:
1179
+ raise ValueError(f"Folder '{path}' already exists.")
1180
+
1181
+ # TODO maybe it should be checked if there is a file with the same path which already exists.
1182
+
1183
+ folder = os.path.dirname(path)
1184
+
1185
+ if not os.path.isdir(folder):
1186
+ if create_necessary_folder:
1187
+ GeneralUtilities.ensure_directory_exists(folder) # TODO check if this also create nested folders if required
1188
+ else:
1189
+ raise ValueError(f"Folder '{folder}' does not exist.")
1190
+
1191
+ GeneralUtilities.ensure_directory_exists(path)
1192
+ else:
1193
+ arguments = ["--path", path]
1194
+
1195
+ if error_if_already_exists:
1196
+ arguments = arguments+["--errorwhenexists"]
1197
+
1198
+ if create_necessary_folder:
1199
+ arguments = arguments+["--createnecessaryfolder"]
1200
+
1201
+ exit_code, stdout, stderr, _ = self.run_program_argsasarray("sccreatefolder", arguments, throw_exception_if_exitcode_is_not_zero=False) # works platform-indepent
1202
+ if exit_code != 0:
1203
+ raise ValueError(f"Fatal error occurrs while create folder '{path}'; Exitcode: '{exit_code}'; StdOut: '{stdout}'. StdErr: '{stderr}'")
1204
+
1205
+ @GeneralUtilities.check_arguments
1206
+ def __sort_fmd(self, line: str):
1207
+ splitted: list = line.split(";")
1208
+ filetype: str = splitted[1]
1209
+ if filetype == "d":
1210
+ return -1
1211
+ if filetype == "f":
1212
+ return 1
1213
+ return 0
1214
+
1215
+ @GeneralUtilities.check_arguments
1216
+ def restore_filemetadata(self, folder: str, source_file: str, strict=False, encoding: str = "utf-8", create_folder_is_not_exist: bool = True) -> None:
1217
+ lines = GeneralUtilities.read_lines_from_file(source_file, encoding)
1218
+ lines.sort(key=self.__sort_fmd)
1219
+ for line in lines:
1220
+ splitted: list = line.split(";")
1221
+ full_path_of_file_or_folder: str = os.path.join(folder, splitted[0])
1222
+ filetype: str = splitted[1]
1223
+ user: str = splitted[2]
1224
+ permissions: str = splitted[3]
1225
+ if filetype == "d" and create_folder_is_not_exist and not os.path.isdir(full_path_of_file_or_folder):
1226
+ GeneralUtilities.ensure_directory_exists(full_path_of_file_or_folder)
1227
+ if (filetype == "f" and os.path.isfile(full_path_of_file_or_folder)) or (filetype == "d" and os.path.isdir(full_path_of_file_or_folder)):
1228
+ self.set_owner(full_path_of_file_or_folder, user, os.name != 'nt')
1229
+ self.set_permission(full_path_of_file_or_folder, permissions)
1230
+ else:
1231
+ if strict:
1232
+ if filetype == "f":
1233
+ filetype_full = "File"
1234
+ elif filetype == "d":
1235
+ filetype_full = "Directory"
1236
+ else:
1237
+ raise ValueError(f"Unknown filetype: {GeneralUtilities.str_none_safe(filetype)}")
1238
+ raise ValueError(f"{filetype_full} '{full_path_of_file_or_folder}' does not exist")
1239
+
1240
+ @GeneralUtilities.check_arguments
1241
+ def __calculate_lengh_in_seconds(self, filename: str, folder: str) -> float:
1242
+ argument = ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename]
1243
+ result = self.run_program_argsasarray("ffprobe", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
1244
+ return float(result[1].replace('\n', ''))
1245
+
1246
+ @GeneralUtilities.check_arguments
1247
+ def __create_thumbnails(self, filename: str, fps: str, folder: str, tempname_for_thumbnails: str) -> list[str]:
1248
+ argument = ['-i', filename, '-r', fps, '-vf', 'scale=-1:120', '-vcodec', 'png', f'{tempname_for_thumbnails}-%002d.png']
1249
+ self.run_program_argsasarray("ffmpeg", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
1250
+ files = GeneralUtilities.get_direct_files_of_folder(folder)
1251
+ result: list[str] = []
1252
+ regex = "^"+re.escape(tempname_for_thumbnails)+"\\-\\d+\\.png$"
1253
+ regex_for_files = re.compile(regex)
1254
+ for file in files:
1255
+ filename = os.path.basename(file)
1256
+ if regex_for_files.match(filename):
1257
+ result.append(file)
1258
+ GeneralUtilities.assert_condition(0 < len(result), "No thumbnail-files found.")
1259
+ return result
1260
+
1261
+ @GeneralUtilities.check_arguments
1262
+ def __create_thumbnail(self, outputfilename: str, folder: str, length_in_seconds: float, tempname_for_thumbnails: str, amount_of_images: int) -> None:
1263
+ duration = timedelta(seconds=length_in_seconds)
1264
+ info = GeneralUtilities.timedelta_to_simple_string(duration)
1265
+ next_square_number = GeneralUtilities.get_next_square_number(amount_of_images)
1266
+ root = math.sqrt(next_square_number)
1267
+ rows: int = root # 5
1268
+ columns: int = root # math.ceil(amount_of_images/rows)
1269
+ argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
1270
+ self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
1271
+
1272
+ @GeneralUtilities.check_arguments
1273
+ def __create_thumbnail2(self, outputfilename: str, folder: str, length_in_seconds: float, rows: int, columns: int, tempname_for_thumbnails: str, amount_of_images: int) -> None:
1274
+ duration = timedelta(seconds=length_in_seconds)
1275
+ info = GeneralUtilities.timedelta_to_simple_string(duration)
1276
+ argument = ['-title', f'"{outputfilename} ({info})"', '-tile', f'{rows}x{columns}', f'{tempname_for_thumbnails}*.png', f'{outputfilename}.png']
1277
+ self.run_program_argsasarray("montage", argument, folder, throw_exception_if_exitcode_is_not_zero=True)
1278
+
1279
+ @GeneralUtilities.check_arguments
1280
+ def __roundup(self, x: float, places: int) -> int:
1281
+ d = 10 ** places
1282
+ if x < 0:
1283
+ return math.floor(x * d) / d
1284
+ else:
1285
+ return math.ceil(x * d) / d
1286
+
1287
+ @GeneralUtilities.check_arguments
1288
+ def generate_thumbnail(self, file: str, frames_per_second: float, tempname_for_thumbnails: str = None, hook=None) -> None:
1289
+ if tempname_for_thumbnails is None:
1290
+ tempname_for_thumbnails = "t_"+str(uuid.uuid4())
1291
+
1292
+ file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
1293
+ filename = os.path.basename(file)
1294
+ folder = os.path.dirname(file)
1295
+ filename_without_extension = Path(file).stem
1296
+ preview_files: list[str] = []
1297
+ try:
1298
+ length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
1299
+ # frames per second, example: frames_per_second="20fps" => 20 frames per second
1300
+ frames_per_second = self.__roundup(float(frames_per_second[:-3]), 2)
1301
+ frames_per_second_as_string = str(frames_per_second)
1302
+ preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
1303
+ if hook is not None:
1304
+ hook(file, preview_files)
1305
+ actual_amounf_of_previewframes = len(preview_files)
1306
+ self.__create_thumbnail(filename_without_extension, folder, length_in_seconds, tempname_for_thumbnails, actual_amounf_of_previewframes)
1307
+ finally:
1308
+ for thumbnail_to_delete in preview_files:
1309
+ os.remove(thumbnail_to_delete)
1310
+
1311
+ @GeneralUtilities.check_arguments
1312
+ def generate_thumbnail_by_amount_of_pictures(self, file: str, amount_of_columns: int, amount_of_rows: int, tempname_for_thumbnails: str = None, hook=None) -> None:
1313
+ if tempname_for_thumbnails is None:
1314
+ tempname_for_thumbnails = "t_"+str(uuid.uuid4())
1315
+
1316
+ file = GeneralUtilities.resolve_relative_path_from_current_working_directory(file)
1317
+ filename = os.path.basename(file)
1318
+ folder = os.path.dirname(file)
1319
+ filename_without_extension = Path(file).stem
1320
+ preview_files: list[str] = []
1321
+ try:
1322
+ length_in_seconds = self.__calculate_lengh_in_seconds(filename, folder)
1323
+ amounf_of_previewframes = int(amount_of_columns*amount_of_rows)
1324
+ frames_per_second_as_string = f"{amounf_of_previewframes-2}/{length_in_seconds}"
1325
+ preview_files = self.__create_thumbnails(filename, frames_per_second_as_string, folder, tempname_for_thumbnails)
1326
+ if hook is not None:
1327
+ hook(file, preview_files)
1328
+ actual_amounf_of_previewframes = len(preview_files)
1329
+ self.__create_thumbnail2(filename_without_extension, folder, length_in_seconds, amount_of_rows, amount_of_columns, tempname_for_thumbnails, actual_amounf_of_previewframes)
1330
+ finally:
1331
+ for thumbnail_to_delete in preview_files:
1332
+ os.remove(thumbnail_to_delete)
1333
+
1334
+ @GeneralUtilities.check_arguments
1335
+ def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
1336
+ pdf_reader: PdfReader = PdfReader(file)
1337
+ pdf_writer: PdfWriter = PdfWriter()
1338
+ start = from_page
1339
+ end = to_page
1340
+ while start <= end:
1341
+ pdf_writer.add_page(pdf_reader.pages[start-1])
1342
+ start += 1
1343
+ with open(outputfile, 'wb') as out:
1344
+ pdf_writer.write(out)
1345
+
1346
+ @GeneralUtilities.check_arguments
1347
+ def merge_pdf_files(self, files: list[str], outputfile: str) -> None:
1348
+ # TODO add wildcard-option
1349
+ pdfFileMerger: PdfWriter = PdfWriter()
1350
+ for file in files:
1351
+ with open(file, "rb") as f:
1352
+ pdfFileMerger.append(f)
1353
+ with open(outputfile, "wb") as output:
1354
+ pdfFileMerger.write(output)
1355
+ pdfFileMerger.close()
1356
+
1357
+ @GeneralUtilities.check_arguments
1358
+ def show_missing_files(self, folderA: str, folderB: str):
1359
+ for file in GeneralUtilities.get_missing_files(folderA, folderB):
1360
+ GeneralUtilities.write_message_to_stdout(file)
1361
+
1362
+ @GeneralUtilities.check_arguments
1363
+ def SCCreateEmptyFileWithSpecificSize(self, name: str, size_string: str) -> int:
1364
+ if size_string.isdigit():
1365
+ size = int(size_string)
1366
+ else:
1367
+ if len(size_string) >= 3:
1368
+ if (size_string.endswith("kb")):
1369
+ size = int(size_string[:-2]) * pow(10, 3)
1370
+ elif (size_string.endswith("mb")):
1371
+ size = int(size_string[:-2]) * pow(10, 6)
1372
+ elif (size_string.endswith("gb")):
1373
+ size = int(size_string[:-2]) * pow(10, 9)
1374
+ elif (size_string.endswith("kib")):
1375
+ size = int(size_string[:-3]) * pow(2, 10)
1376
+ elif (size_string.endswith("mib")):
1377
+ size = int(size_string[:-3]) * pow(2, 20)
1378
+ elif (size_string.endswith("gib")):
1379
+ size = int(size_string[:-3]) * pow(2, 30)
1380
+ else:
1381
+ self.log.log("Wrong format", LogLevel.Error)
1382
+ return 1
1383
+ else:
1384
+ self.log.log("Wrong format", LogLevel.Error)
1385
+ return 1
1386
+ with open(name, "wb") as f:
1387
+ f.seek(size-1)
1388
+ f.write(b"\0")
1389
+ return 0
1390
+
1391
+ @GeneralUtilities.check_arguments
1392
+ def SCCreateHashOfAllFiles(self, folder: str) -> None:
1393
+ for file in GeneralUtilities.absolute_file_paths(folder):
1394
+ with open(file+".sha256", "w+", encoding="utf-8") as f:
1395
+ f.write(GeneralUtilities.get_sha256_of_file(file))
1396
+
1397
+ @GeneralUtilities.check_arguments
1398
+ def SCCreateSimpleMergeWithoutRelease(self, repository: str, sourcebranch: str, targetbranch: str, remotename: str, remove_source_branch: bool) -> None:
1399
+ commitid = self.git_merge(repository, sourcebranch, targetbranch, False, True)
1400
+ self.git_merge(repository, targetbranch, sourcebranch, True, True)
1401
+ created_version = self.get_semver_version_from_gitversion(repository)
1402
+ self.git_create_tag(repository, commitid, f"v{created_version}", True)
1403
+ self.git_push(repository, remotename, targetbranch, targetbranch, False, True)
1404
+ if (GeneralUtilities.string_has_nonwhitespace_content(remotename)):
1405
+ self.git_push(repository, remotename, sourcebranch, sourcebranch, False, True)
1406
+ if (remove_source_branch):
1407
+ self.git_remove_branch(repository, sourcebranch)
1408
+
1409
+ @GeneralUtilities.check_arguments
1410
+ def sc_organize_lines_in_file(self, file: str, encoding: str="utf-8", sort: bool = False, remove_duplicated_lines: bool = False, ignore_first_line: bool = False, remove_empty_lines: bool = True, ignored_start_character: list = list()):
1411
+ GeneralUtilities.assert_file_exists(file)
1412
+
1413
+ # read file
1414
+ lines = GeneralUtilities.read_lines_from_file(file, encoding)
1415
+ if (len(lines) == 0):
1416
+ return
1417
+
1418
+ # store first line if desired
1419
+ if (ignore_first_line):
1420
+ first_line = lines.pop(0)
1421
+
1422
+ # remove empty lines if desired
1423
+ if remove_empty_lines:
1424
+ temp = lines
1425
+ lines = []
1426
+ for line in temp:
1427
+ if (not (GeneralUtilities.string_is_none_or_whitespace(line))):
1428
+ lines.append(line)
1429
+
1430
+ # remove duplicated lines if desired
1431
+ if remove_duplicated_lines:
1432
+ lines = GeneralUtilities.remove_duplicates(lines)
1433
+
1434
+ # sort lines if desired
1435
+ if sort:
1436
+ lines = sorted(lines, key=lambda singleline: self.__adapt_line_for_sorting(singleline, ignored_start_character))
1437
+
1438
+ # reinsert first line if required
1439
+ if ignore_first_line:
1440
+ lines.insert(0, first_line)
1441
+
1442
+ # write result to file
1443
+ GeneralUtilities.write_lines_to_file(file, lines, encoding)
1444
+
1445
+
1446
+ @GeneralUtilities.check_arguments
1447
+ def __adapt_line_for_sorting(self, line: str, ignored_start_characters: list):
1448
+ result = line.lower()
1449
+ while len(result) > 0 and result[0] in ignored_start_characters:
1450
+ result = result[1:]
1451
+ return result
1452
+
1453
+ @GeneralUtilities.check_arguments
1454
+ def SCGenerateSnkFiles(self, outputfolder, keysize=4096, amountofkeys=10) -> int:
1455
+ GeneralUtilities.ensure_directory_exists(outputfolder)
1456
+ for _ in range(amountofkeys):
1457
+ file = os.path.join(outputfolder, str(uuid.uuid4())+".snk")
1458
+ argument = f"-k {keysize} {file}"
1459
+ self.run_program("sn", argument, outputfolder)
1460
+
1461
+ @GeneralUtilities.check_arguments
1462
+ def __merge_files(self, sourcefile: str, targetfile: str) -> None:
1463
+ with open(sourcefile, "rb") as f:
1464
+ source_data = f.read()
1465
+ with open(targetfile, "ab") as fout:
1466
+ merge_separator = [0x0A]
1467
+ fout.write(bytes(merge_separator))
1468
+ fout.write(source_data)
1469
+
1470
+ @GeneralUtilities.check_arguments
1471
+ def __process_file(self, file: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
1472
+ new_filename = os.path.join(os.path.dirname(file), os.path.basename(file).replace(substringInFilename, newSubstringInFilename))
1473
+ if file != new_filename:
1474
+ if os.path.isfile(new_filename):
1475
+ if filecmp.cmp(file, new_filename):
1476
+ send2trash.send2trash(file)
1477
+ else:
1478
+ if conflictResolveMode == "ignore":
1479
+ pass
1480
+ elif conflictResolveMode == "preservenewest":
1481
+ if (os.path.getmtime(file) - os.path.getmtime(new_filename) > 0):
1482
+ send2trash.send2trash(file)
1483
+ else:
1484
+ send2trash.send2trash(new_filename)
1485
+ os.rename(file, new_filename)
1486
+ elif (conflictResolveMode == "merge"):
1487
+ self.__merge_files(file, new_filename)
1488
+ send2trash.send2trash(file)
1489
+ else:
1490
+ raise ValueError('Unknown conflict resolve mode')
1491
+ else:
1492
+ os.rename(file, new_filename)
1493
+
1494
+ @GeneralUtilities.check_arguments
1495
+ def SCReplaceSubstringsInFilenames(self, folder: str, substringInFilename: str, newSubstringInFilename: str, conflictResolveMode: str) -> None:
1496
+ for file in GeneralUtilities.absolute_file_paths(folder):
1497
+ self.__process_file(file, substringInFilename, newSubstringInFilename, conflictResolveMode)
1498
+
1499
+ @GeneralUtilities.check_arguments
1500
+ def __check_file(self, file: str, searchstring: str) -> None:
1501
+ bytes_ascii = bytes(searchstring, "ascii")
1502
+ # often called "unicode-encoding"
1503
+ bytes_utf16 = bytes(searchstring, "utf-16")
1504
+ bytes_utf8 = bytes(searchstring, "utf-8")
1505
+ with open(file, mode='rb') as file_object:
1506
+ content = file_object.read()
1507
+ if bytes_ascii in content:
1508
+ GeneralUtilities.write_message_to_stdout(file)
1509
+ elif bytes_utf16 in content:
1510
+ GeneralUtilities.write_message_to_stdout(file)
1511
+ elif bytes_utf8 in content:
1512
+ GeneralUtilities.write_message_to_stdout(file)
1513
+
1514
+ @GeneralUtilities.check_arguments
1515
+ def SCSearchInFiles(self, folder: str, searchstring: str) -> None:
1516
+ for file in GeneralUtilities.absolute_file_paths(folder):
1517
+ self.__check_file(file, searchstring)
1518
+
1519
+ @GeneralUtilities.check_arguments
1520
+ def get_string_as_qr_code(self,string: str) -> None:
1521
+ qr = qrcode.QRCode()
1522
+ qr.add_data(string)
1523
+ f = io.StringIO()
1524
+ qr.print_ascii(out=f)
1525
+ f.seek(0)
1526
+ return f.read()
1527
+
1528
+ @GeneralUtilities.check_arguments
1529
+ def __print_qr_code_by_csv_line(self, displayname: str, website: str, emailaddress: str, key: str, period: str) -> None:
1530
+ qrcode_content = f"otpauth://totp/{website}:{emailaddress}?secret={key}&issuer={displayname}&period={period}"
1531
+ GeneralUtilities.write_message_to_stdout(f"{displayname} ({emailaddress}):")
1532
+ GeneralUtilities.write_message_to_stdout(qrcode_content)
1533
+ GeneralUtilities.write_message_to_stdout(self.get_string_as_qr_code(qrcode_content))
1534
+
1535
+ @GeneralUtilities.check_arguments
1536
+ def SCShow2FAAsQRCode(self, csvfile: str) -> None:
1537
+ lines = GeneralUtilities.read_csv_file(csvfile, True)
1538
+ lines.sort(key=lambda items: ''.join(items).lower())
1539
+ for line in lines:
1540
+ self.__print_qr_code_by_csv_line(line[0], line[1], line[2], line[3], line[4])
1541
+ GeneralUtilities.write_message_to_stdout(GeneralUtilities.get_longline())
1542
+
1543
+ @GeneralUtilities.check_arguments
1544
+ def SCCalculateBitcoinBlockHash(self, block_version_number: str, previousblockhash: str, transactionsmerkleroot: str, timestamp: str, target: str, nonce: str) -> str:
1545
+ # Example-values:
1546
+ # block_version_number: "00000020"
1547
+ # previousblockhash: "66720b99e07d284bd4fe67ff8c49a5db1dd8514fcdab61000000000000000000"
1548
+ # transactionsmerkleroot: "7829844f4c3a41a537b3131ca992643eaa9d093b2383e4cdc060ad7dc5481187"
1549
+ # timestamp: "51eb505a"
1550
+ # target: "c1910018"
1551
+ # nonce: "de19b302"
1552
+ header = str(block_version_number + previousblockhash + transactionsmerkleroot + timestamp + target + nonce)
1553
+ return binascii.hexlify(hashlib.sha256(hashlib.sha256(binascii.unhexlify(header)).digest()).digest()[::-1]).decode('utf-8')
1554
+
1555
+ @GeneralUtilities.check_arguments
1556
+ def SCChangeHashOfProgram(self, inputfile: str) -> None:
1557
+ valuetoappend = str(uuid.uuid4())
1558
+
1559
+ outputfile = inputfile + '.modified'
1560
+
1561
+ shutil.copy2(inputfile, outputfile)
1562
+ with open(outputfile, 'a', encoding="utf-8") as file:
1563
+ # TODO use rcedit for .exe-files instead of appending valuetoappend ( https://github.com/electron/rcedit/ )
1564
+ # background: you can retrieve the "original-filename" from the .exe-file like discussed here:
1565
+ # https://security.stackexchange.com/questions/210843/ is-it-possible-to-change-original-filename-of-an-exe
1566
+ # so removing the original filename with rcedit is probably a better way to make it more difficult to detect the programname.
1567
+ # this would obviously also change the hashvalue of the program so appending a whitespace is not required anymore.
1568
+ file.write(valuetoappend)
1569
+
1570
+ @GeneralUtilities.check_arguments
1571
+ def __adjust_folder_name(self, folder: str) -> str:
1572
+ result = os.path.dirname(folder).replace("\\", "/")
1573
+ if result == "/":
1574
+ return GeneralUtilities.empty_string
1575
+ else:
1576
+ return result
1577
+
1578
+ @GeneralUtilities.check_arguments
1579
+ def __create_iso(self, folder, iso_file) -> None:
1580
+ created_directories = []
1581
+ files_directory = "FILES"
1582
+ iso = pycdlib.PyCdlib()
1583
+ iso.new()
1584
+ files_directory = files_directory.upper()
1585
+ iso.add_directory("/" + files_directory)
1586
+ created_directories.append("/" + files_directory)
1587
+ for root, _, files in os.walk(folder):
1588
+ for file in files:
1589
+ full_path = os.path.join(root, file)
1590
+ with (open(full_path, "rb").read()) as text_io_wrapper:
1591
+ content = text_io_wrapper
1592
+ path_in_iso = '/' + files_directory + \
1593
+ self.__adjust_folder_name(full_path[len(folder)::1]).upper()
1594
+ if path_in_iso not in created_directories:
1595
+ iso.add_directory(path_in_iso)
1596
+ created_directories.append(path_in_iso)
1597
+ iso.add_fp(BytesIO(content), len(content), path_in_iso + '/' + file.upper() + ';1')
1598
+ iso.write(iso_file)
1599
+ iso.close()
1600
+
1601
+ @GeneralUtilities.check_arguments
1602
+ def SCCreateISOFileWithObfuscatedFiles(self, inputfolder: str, outputfile: str, printtableheadline, createisofile, extensions) -> None:
1603
+ if (os.path.isdir(inputfolder)):
1604
+ namemappingfile = "name_map.csv"
1605
+ files_directory = inputfolder
1606
+ files_directory_obf = f"{files_directory}_Obfuscated"
1607
+ self.SCObfuscateFilesFolder(
1608
+ inputfolder, printtableheadline, namemappingfile, extensions)
1609
+ os.rename(namemappingfile, os.path.join(
1610
+ files_directory_obf, namemappingfile))
1611
+ if createisofile:
1612
+ self.__create_iso(files_directory_obf, outputfile)
1613
+ shutil.rmtree(files_directory_obf)
1614
+ else:
1615
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1616
+
1617
+ @GeneralUtilities.check_arguments
1618
+ def SCFilenameObfuscator(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1619
+ obfuscate_all_files = extensions == "*"
1620
+ if (obfuscate_all_files):
1621
+ obfuscate_file_extensions = None
1622
+ else:
1623
+ obfuscate_file_extensions = extensions.split(",")
1624
+ if (os.path.isdir(inputfolder)):
1625
+ printtableheadline = GeneralUtilities.string_to_boolean(
1626
+ printtableheadline)
1627
+ files = []
1628
+ if not os.path.isfile(namemappingfile):
1629
+ with open(namemappingfile, "a", encoding="utf-8"):
1630
+ pass
1631
+ if printtableheadline:
1632
+ GeneralUtilities.append_line_to_file(
1633
+ namemappingfile, "Original filename;new filename;SHA2-hash of file")
1634
+ for file in GeneralUtilities.absolute_file_paths(inputfolder):
1635
+ if os.path.isfile(os.path.join(inputfolder, file)):
1636
+ if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1637
+ files.append(file)
1638
+ for file in files:
1639
+ hash_value = GeneralUtilities.get_sha256_of_file(file)
1640
+ extension = Path(file).suffix
1641
+ new_file_name_without_path = str(uuid.uuid4())[0:8] + extension
1642
+ new_file_name = os.path.join(
1643
+ os.path.dirname(file), new_file_name_without_path)
1644
+ os.rename(file, new_file_name)
1645
+ GeneralUtilities.append_line_to_file(namemappingfile, os.path.basename(file) + ";" + new_file_name_without_path + ";" + hash_value)
1646
+ else:
1647
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1648
+
1649
+ @GeneralUtilities.check_arguments
1650
+ def __extension_matchs(self, file: str, obfuscate_file_extensions) -> bool:
1651
+ for extension in obfuscate_file_extensions:
1652
+ if file.lower().endswith("."+extension.lower()):
1653
+ return True
1654
+ return False
1655
+
1656
+ @GeneralUtilities.check_arguments
1657
+ def SCHealthcheck(self, file: str) -> int:
1658
+ lines = GeneralUtilities.read_lines_from_file(file)
1659
+ for line in reversed(lines):
1660
+ if not GeneralUtilities.string_is_none_or_whitespace(line):
1661
+ if "RunningHealthy (" in line: # TODO use regex
1662
+ GeneralUtilities.write_message_to_stderr(f"Healthy running due to line '{line}' in file '{file}'.")
1663
+ return 0
1664
+ else:
1665
+ GeneralUtilities.write_message_to_stderr(f"Not healthy running due to line '{line}' in file '{file}'.")
1666
+ return 1
1667
+ GeneralUtilities.write_message_to_stderr(f"No valid line found for healthycheck in file '{file}'.")
1668
+ return 2
1669
+
1670
+ @GeneralUtilities.check_arguments
1671
+ def SCObfuscateFilesFolder(self, inputfolder: str, printtableheadline, namemappingfile: str, extensions: str) -> None:
1672
+ obfuscate_all_files = extensions == "*"
1673
+ if (obfuscate_all_files):
1674
+ obfuscate_file_extensions = None
1675
+ else:
1676
+ if "," in extensions:
1677
+ obfuscate_file_extensions = extensions.split(",")
1678
+ else:
1679
+ obfuscate_file_extensions = [extensions]
1680
+ newd = inputfolder+"_Obfuscated"
1681
+ shutil.copytree(inputfolder, newd)
1682
+ inputfolder = newd
1683
+ if (os.path.isdir(inputfolder)):
1684
+ for file in GeneralUtilities.absolute_file_paths(inputfolder):
1685
+ if obfuscate_all_files or self.__extension_matchs(file, obfuscate_file_extensions):
1686
+ self.SCChangeHashOfProgram(file)
1687
+ os.remove(file)
1688
+ os.rename(file + ".modified", file)
1689
+ self.SCFilenameObfuscator(inputfolder, printtableheadline, namemappingfile, extensions)
1690
+ else:
1691
+ raise ValueError(f"Directory not found: '{inputfolder}'")
1692
+
1693
+ @GeneralUtilities.check_arguments
1694
+ def get_services_from_yaml_file(self, yaml_file: str) -> list[str]:
1695
+ with open(yaml_file, encoding="utf-8") as stream:
1696
+ loaded = yaml.safe_load(stream)
1697
+ services = loaded["services"]
1698
+ result = list(services.keys())
1699
+ return result
1700
+
1701
+ @GeneralUtilities.check_arguments
1702
+ def kill_docker_container(self, container_name: str) -> None:
1703
+ self.run_program("docker", f"container rm -f {container_name}")
1704
+
1705
+ @GeneralUtilities.check_arguments
1706
+ def get_latest_apt_package_version_in_debian(self, image: str,package:str) -> str:
1707
+ #docker run --rm -it debian bash -c "apt update && apt list -a tor"
1708
+ output=self.run_with_epew("docker", f"run --rm -it {image} bash -c \"apt --color=false update && apt --color=false list -a tor\"",os.getcwd(),encode_argument_in_base64=True)
1709
+ stdout=output[1]
1710
+ version_lines=[line.strip() for line in GeneralUtilities.string_to_lines(stdout) if GeneralUtilities.string_has_nonwhitespace_content(line) and line.startswith(package+"/")]
1711
+ GeneralUtilities.assert_condition(0<len(version_lines), f"No version found for package '{package}' in image '{image}'.")
1712
+ versions = [version_line.split(" ")[1] for version_line in version_lines]
1713
+ def my_comparer(a, b) -> int:
1714
+ # return:
1715
+ # -1 → a < b
1716
+ # 0 → a = b
1717
+ # 1 → a > b
1718
+ # dpkg --compare-versions <a> lt <b> → exit 0 wenn a < b
1719
+ def dpkg_compare(op: str) -> bool:
1720
+ result = self.run_program_argsasarray("docker", [ "run", "--rm",image, "dpkg", "--compare-versions", a, op, b],throw_exception_if_exitcode_is_not_zero=False)
1721
+ GeneralUtilities.assert_condition(result[1]==GeneralUtilities.empty_string)
1722
+ GeneralUtilities.assert_condition(result[2]==GeneralUtilities.empty_string)
1723
+ return result[0] == 0
1724
+
1725
+ if dpkg_compare("lt"): # a < b
1726
+ return -1
1727
+ elif dpkg_compare("gt"): # a > b
1728
+ return 1
1729
+ else:
1730
+ return 0
1731
+ sorted_versions = sorted(versions, key=cmp_to_key(my_comparer))
1732
+ return sorted_versions[-1]
1733
+
1734
+ @GeneralUtilities.check_arguments
1735
+ def run_testcases_for_python_project(self, repository_folder: str):
1736
+ self.assert_is_git_repository(repository_folder)
1737
+ self.run_program("coverage", "run -m pytest", repository_folder)
1738
+ self.run_program("coverage", "xml", repository_folder)
1739
+ GeneralUtilities.ensure_directory_exists(os.path.join(repository_folder, "Other/TestCoverage"))
1740
+ coveragefile = os.path.join(repository_folder, "Other/TestCoverage/TestCoverage.xml")
1741
+ GeneralUtilities.ensure_file_does_not_exist(coveragefile)
1742
+ os.rename(os.path.join(repository_folder, "coverage.xml"), coveragefile)
1743
+
1744
+ @GeneralUtilities.check_arguments
1745
+ def get_file_permission(self, file: str) -> str:
1746
+ """This function returns an usual octet-triple, for example "700"."""
1747
+ ls_output: str = self.run_ls_for_folder(file)
1748
+ return self.__get_file_permission_helper(ls_output)
1749
+
1750
+ @GeneralUtilities.check_arguments
1751
+ def __get_file_permission_helper(self, permissions: str) -> str:
1752
+ return str(self.__to_octet(permissions[0:3])) + str(self.__to_octet(permissions[3:6]))+str(self.__to_octet(permissions[6:9]))
1753
+
1754
+ @GeneralUtilities.check_arguments
1755
+ def __to_octet(self, string: str) -> int:
1756
+ return int(self.__to_octet_helper(string[0]) + self.__to_octet_helper(string[1])+self.__to_octet_helper(string[2]), 2)
1757
+
1758
+ @GeneralUtilities.check_arguments
1759
+ def __to_octet_helper(self, string: str) -> str:
1760
+ if (string == "-"):
1761
+ return "0"
1762
+ else:
1763
+ return "1"
1764
+
1765
+ @GeneralUtilities.check_arguments
1766
+ def get_file_owner(self, file: str) -> str:
1767
+ """This function returns the user and the group in the format "user:group"."""
1768
+ ls_output: str = self.run_ls_for_folder(file)
1769
+ return self.__get_file_owner_helper(ls_output)
1770
+
1771
+ @GeneralUtilities.check_arguments
1772
+ def __get_file_owner_helper(self, ls_output: str) -> str:
1773
+ splitted = ls_output.split()
1774
+ return f"{splitted[2]}:{splitted[3]}"
1775
+
1776
+ @GeneralUtilities.check_arguments
1777
+ def get_file_owner_and_file_permission(self, file: str) -> str:
1778
+ ls_output: str = self.run_ls_for_folder(file)
1779
+ return [self.__get_file_owner_helper(ls_output), self.__get_file_permission_helper(ls_output)]
1780
+
1781
+ @GeneralUtilities.check_arguments
1782
+ def run_ls_for_folder(self, file_or_folder: str) -> str:
1783
+ file_or_folder = file_or_folder.replace("\\", "/")
1784
+ GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -ld' because '{file_or_folder}' does not exist.")
1785
+ ls_result = self.run_program_argsasarray("ls", ["-ld", file_or_folder])
1786
+ GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -ld {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
1787
+ GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -ld' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
1788
+ output = ls_result[1]
1789
+ result = output.replace("\n", GeneralUtilities.empty_string)
1790
+ result = ' '.join(result.split()) # reduce multiple whitespaces to one
1791
+ return result
1792
+
1793
+ @GeneralUtilities.check_arguments
1794
+ def run_ls_for_folder_content(self, file_or_folder: str) -> list[str]:
1795
+ file_or_folder = file_or_folder.replace("\\", "/")
1796
+ GeneralUtilities.assert_condition(os.path.isfile(file_or_folder) or os.path.isdir(file_or_folder), f"Can not execute 'ls -la' because '{file_or_folder}' does not exist.")
1797
+ ls_result = self.run_program_argsasarray("ls", ["-la", file_or_folder])
1798
+ GeneralUtilities.assert_condition(ls_result[0] == 0, f"'ls -la {file_or_folder}' resulted in exitcode {str(ls_result[0])}. StdErr: {ls_result[2]}")
1799
+ GeneralUtilities.assert_condition(not GeneralUtilities.string_is_none_or_whitespace(ls_result[1]), f"'ls -la' of '{file_or_folder}' had an empty output. StdErr: '{ls_result[2]}'")
1800
+ output = ls_result[1]
1801
+ result = output.split("\n")[3:] # skip the lines with "Total", "." and ".."
1802
+ result = [' '.join(line.split()) for line in result] # reduce multiple whitespaces to one
1803
+ return result
1804
+
1805
+ @GeneralUtilities.check_arguments
1806
+ def set_permission(self, file_or_folder: str, permissions: str, recursive: bool = False) -> None:
1807
+ """This function expects an usual octet-triple, for example "700"."""
1808
+ args = []
1809
+ if recursive:
1810
+ args.append("--recursive")
1811
+ args.append(permissions)
1812
+ args.append(file_or_folder)
1813
+ self.run_program_argsasarray("chmod", args)
1814
+
1815
+ @GeneralUtilities.check_arguments
1816
+ def set_owner(self, file_or_folder: str, owner: str, recursive: bool = False, follow_symlinks: bool = False) -> None:
1817
+ """This function expects the user and the group in the format "user:group"."""
1818
+ args = []
1819
+ if recursive:
1820
+ args.append("--recursive")
1821
+ if follow_symlinks:
1822
+ args.append("--no-dereference")
1823
+ args.append(owner)
1824
+ args.append(file_or_folder)
1825
+ self.run_program_argsasarray("chown", args)
1826
+
1827
+ # <run programs>
1828
+
1829
+ @GeneralUtilities.check_arguments
1830
+ def __run_program_argsasarray_async_helper(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> Popen:
1831
+ popen: Popen = self.program_runner.run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, custom_argument, interactive)
1832
+ return popen
1833
+
1834
+ @staticmethod
1835
+ def __enqueue_output(file: IO, queue: Queue):
1836
+ for line in iter(file.readline, ''):
1837
+ queue.put(line)
1838
+ file.close()
1839
+
1840
+ @staticmethod
1841
+ def __continue_process_reading(pid: int, p: Popen, q_stdout: Queue, q_stderr: Queue, reading_stdout_last_time_resulted_in_exception: bool, reading_stderr_last_time_resulted_in_exception: bool):
1842
+ if p.poll() is None:
1843
+ return True
1844
+
1845
+ # if reading_stdout_last_time_resulted_in_exception and reading_stderr_last_time_resulted_in_exception:
1846
+ # return False
1847
+
1848
+ if not q_stdout.empty():
1849
+ return True
1850
+
1851
+ if not q_stderr.empty():
1852
+ return True
1853
+
1854
+ return False
1855
+
1856
+ @staticmethod
1857
+ def __read_popen_pipes(p: Popen, print_live_output: bool, print_errors_as_information: bool, log: SCLog) -> tuple[list[str], list[str]]:
1858
+ p_id = p.pid
1859
+ with ThreadPoolExecutor(2) as pool:
1860
+ q_stdout = Queue()
1861
+ q_stderr = Queue()
1862
+
1863
+ pool.submit(ScriptCollectionCore.__enqueue_output, p.stdout, q_stdout)
1864
+ pool.submit(ScriptCollectionCore.__enqueue_output, p.stderr, q_stderr)
1865
+ reading_stdout_last_time_resulted_in_exception: bool = False
1866
+ reading_stderr_last_time_resulted_in_exception: bool = False
1867
+
1868
+ stdout_result: list[str] = []
1869
+ stderr_result: list[str] = []
1870
+
1871
+ while (ScriptCollectionCore.__continue_process_reading(p_id, p, q_stdout, q_stderr, reading_stdout_last_time_resulted_in_exception, reading_stderr_last_time_resulted_in_exception)):
1872
+ try:
1873
+ while not q_stdout.empty():
1874
+ out_line: str = q_stdout.get_nowait()
1875
+ out_line = out_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
1876
+ if GeneralUtilities.string_has_content(out_line):
1877
+ stdout_result.append(out_line)
1878
+ reading_stdout_last_time_resulted_in_exception = False
1879
+ if print_live_output:
1880
+ loglevel = LogLevel.Information
1881
+ if out_line.startswith("Debug: "):
1882
+ loglevel = LogLevel.Debug
1883
+ out_line = out_line[len("Debug: "):]
1884
+ if out_line.startswith("Diagnostic: "):
1885
+ loglevel = LogLevel.Diagnostic
1886
+ out_line = out_line[len("Diagnostic: "):]
1887
+ log.log(out_line, loglevel)
1888
+ except Empty:
1889
+ reading_stdout_last_time_resulted_in_exception = True
1890
+
1891
+ try:
1892
+ while not q_stderr.empty():
1893
+ err_line: str = q_stderr.get_nowait()
1894
+ err_line = err_line.replace("\r", GeneralUtilities.empty_string).replace("\n", GeneralUtilities.empty_string)
1895
+ if GeneralUtilities.string_has_content(err_line):
1896
+ stderr_result.append(err_line)
1897
+ reading_stderr_last_time_resulted_in_exception = False
1898
+ if print_live_output:
1899
+ loglevel = LogLevel.Error
1900
+ if err_line.startswith("Warning: "):
1901
+ loglevel = LogLevel.Warning
1902
+ err_line = err_line[len("Warning: "):]
1903
+ if print_errors_as_information: # "errors" in "print_errors_as_information" means: all what is written to std-err
1904
+ loglevel = LogLevel.Information
1905
+ log.log(err_line, loglevel)
1906
+ except Empty:
1907
+ reading_stderr_last_time_resulted_in_exception = True
1908
+
1909
+ time.sleep(0.01) # this is required to not finish too early
1910
+
1911
+ return (stdout_result, stderr_result)
1912
+
1913
+ @GeneralUtilities.check_arguments
1914
+ def run_program_argsasarray(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
1915
+ if self.call_program_runner_directly:
1916
+ return self.program_runner.run_program_argsasarray(program, arguments_as_array, working_directory, custom_argument, interactive)
1917
+ try:
1918
+ GeneralUtilities.assert_not_null(arguments_as_array,"arguments_as_array must not be null")
1919
+ arguments_as_str = ' '.join(arguments_as_array)
1920
+ mock_loader_result = self.__try_load_mock(program, arguments_as_str, working_directory)
1921
+ if mock_loader_result[0]:
1922
+ return mock_loader_result[1]
1923
+
1924
+ if self.program_runner.will_be_executed_locally():
1925
+ working_directory = self.__adapt_workingdirectory(working_directory)
1926
+
1927
+ if arguments_for_log is None or len(arguments_for_log)==0:
1928
+ arguments_for_log = arguments_as_array
1929
+
1930
+ cmd = f'{GeneralUtilities.str_none_safe(working_directory)}>{program}'
1931
+ if 0 < len(arguments_for_log):
1932
+ arguments_for_log_as_string: str = ' '.join([f'"{argument_for_log}"' for argument_for_log in arguments_for_log])
1933
+ cmd = f'{cmd} {arguments_for_log_as_string}'
1934
+
1935
+ if GeneralUtilities.string_is_none_or_whitespace(title):
1936
+ info_for_log = cmd
1937
+ else:
1938
+ info_for_log = title
1939
+
1940
+ self.log.log(f"Run '{info_for_log}'.", LogLevel.Debug)
1941
+
1942
+ exit_code: int = None
1943
+ stdout: str = GeneralUtilities.empty_string
1944
+ stderr: str = GeneralUtilities.empty_string
1945
+ pid: int = None
1946
+
1947
+ with self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive) as process:
1948
+
1949
+ if log_file is not None:
1950
+ GeneralUtilities.ensure_file_exists(log_file)
1951
+ pid = process.pid
1952
+
1953
+ outputs: tuple[list[str], list[str]] = ScriptCollectionCore.__read_popen_pipes(process, print_live_output, print_errors_as_information, self.log)
1954
+
1955
+ for out_line_plain in outputs[0]:
1956
+ if out_line_plain is not None:
1957
+ out_line: str = None
1958
+ if isinstance(out_line_plain, str):
1959
+ out_line = out_line_plain
1960
+ elif isinstance(out_line_plain, bytes):
1961
+ out_line = GeneralUtilities.bytes_to_string(out_line_plain)
1962
+ else:
1963
+ raise ValueError(f"Unknown type of output: {str(type(out_line_plain))}")
1964
+
1965
+ if out_line is not None and GeneralUtilities.string_has_content(out_line):
1966
+ if out_line.endswith("\n"):
1967
+ out_line = out_line[:-1]
1968
+ if 0 < len(stdout):
1969
+ stdout = stdout+"\n"
1970
+ stdout = stdout+out_line
1971
+ if log_file is not None:
1972
+ GeneralUtilities.append_line_to_file(log_file, out_line)
1973
+
1974
+ for err_line_plain in outputs[1]:
1975
+ if err_line_plain is not None:
1976
+ err_line: str = None
1977
+ if isinstance(err_line_plain, str):
1978
+ err_line = err_line_plain
1979
+ elif isinstance(err_line_plain, bytes):
1980
+ err_line = GeneralUtilities.bytes_to_string(err_line_plain)
1981
+ else:
1982
+ raise ValueError(f"Unknown type of output: {str(type(err_line_plain))}")
1983
+ if err_line is not None and GeneralUtilities.string_has_content(err_line):
1984
+ if err_line.endswith("\n"):
1985
+ err_line = err_line[:-1]
1986
+ if 0 < len(stderr):
1987
+ stderr = stderr+"\n"
1988
+ stderr = stderr+err_line
1989
+ if log_file is not None:
1990
+ GeneralUtilities.append_line_to_file(log_file, err_line)
1991
+
1992
+ exit_code = process.returncode
1993
+ GeneralUtilities.assert_condition(exit_code is not None, f"Exitcode of program-run of '{info_for_log}' is None.")
1994
+
1995
+ result_message = f"Program '{info_for_log}' resulted in exitcode {exit_code}."
1996
+
1997
+ self.log.log(result_message, LogLevel.Debug)
1998
+
1999
+ if throw_exception_if_exitcode_is_not_zero and exit_code != 0:
2000
+ raise ValueError(f"{result_message} (StdOut: '{stdout}', StdErr: '{stderr}')")
2001
+
2002
+ result = (exit_code, stdout, stderr, pid)
2003
+ return result
2004
+ except Exception as e:#pylint:disable=unused-variable, try-except-raise
2005
+ raise
2006
+
2007
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
2008
+ @GeneralUtilities.check_arguments
2009
+ def run_program_with_retry(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False, amount_of_attempts: int = 5, delay_in_seconds: int = 2) -> tuple[int, str, str, int]:
2010
+ return GeneralUtilities.retry_action(lambda: self.run_program(program, arguments, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace,arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output), amount_of_attempts, delay_in_seconds=delay_in_seconds)
2011
+
2012
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
2013
+ @GeneralUtilities.check_arguments
2014
+ def run_program(self, program: str, arguments: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False) -> tuple[int, str, str, int]:
2015
+ if self.call_program_runner_directly:
2016
+ return self.program_runner.run_program(program, arguments, working_directory, custom_argument, interactive)
2017
+ return self.run_program_argsasarray(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, GeneralUtilities.arguments_to_array(arguments_for_log), throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output)
2018
+
2019
+ # Return-values program_runner: Exitcode, StdOut, StdErr, Pid
2020
+ @GeneralUtilities.check_arguments
2021
+ def run_program_argsasarray_with_retry(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False, print_live_output: bool = False, amount_of_attempts: int = 5, delay_in_seconds: int = 2) -> tuple[int, str, str, int]:
2022
+ return GeneralUtilities.retry_action(lambda: self.run_program_argsasarray(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace,arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output), amount_of_attempts, delay_in_seconds=delay_in_seconds)
2023
+
2024
+
2025
+ # Return-values program_runner: Pid
2026
+ @GeneralUtilities.check_arguments
2027
+ def run_program_argsasarray_async(self, program: str, arguments_as_array: list[str] = [], working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
2028
+ if self.call_program_runner_directly:
2029
+ return self.program_runner.run_program_argsasarray_async(program, arguments_as_array, working_directory, custom_argument, interactive)
2030
+ mock_loader_result = self.__try_load_mock(program, ' '.join(arguments_as_array), working_directory)
2031
+ if mock_loader_result[0]:
2032
+ return mock_loader_result[1]
2033
+ process: Popen = self.__run_program_argsasarray_async_helper(program, arguments_as_array, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
2034
+ return process.pid
2035
+
2036
+ # Return-values program_runner: Pid
2037
+ @GeneralUtilities.check_arguments
2038
+ def run_program_async(self, program: str, arguments: str = "", working_directory: str = None,print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: list[str] = None, custom_argument: object = None, interactive: bool = False) -> int:
2039
+ if self.call_program_runner_directly:
2040
+ return self.program_runner.run_program_argsasarray_async(program, arguments, working_directory, custom_argument, interactive)
2041
+ return self.run_program_argsasarray_async(program, GeneralUtilities.arguments_to_array(arguments), working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, custom_argument, interactive)
2042
+
2043
+ @GeneralUtilities.check_arguments
2044
+ def __try_load_mock(self, program: str, arguments: str, working_directory: str) -> tuple[bool, tuple[int, str, str, int]]:
2045
+ if self.mock_program_calls:
2046
+ try:
2047
+ return [True, self.__get_mock_program_call(program, arguments, working_directory)]
2048
+ except LookupError:
2049
+ if not self.execute_program_really_if_no_mock_call_is_defined:
2050
+ raise
2051
+ return [False, None]
2052
+
2053
+ @GeneralUtilities.check_arguments
2054
+ def __adapt_workingdirectory(self, workingdirectory: str) -> str:
2055
+ result: str = None
2056
+ if workingdirectory is None:
2057
+ result = os.getcwd()
2058
+ else:
2059
+ if os.path.isabs(workingdirectory):
2060
+ result = workingdirectory
2061
+ else:
2062
+ result = GeneralUtilities.resolve_relative_path_from_current_working_directory(workingdirectory)
2063
+ if not os.path.isdir(result):
2064
+ raise ValueError(f"Working-directory '{workingdirectory}' does not exist.")
2065
+ return result
2066
+
2067
+ @GeneralUtilities.check_arguments
2068
+ def verify_no_pending_mock_program_calls(self):
2069
+ if (len(self.__mocked_program_calls) > 0):
2070
+ raise AssertionError("The following mock-calls were not called:\n"+",\n ".join([self.__format_mock_program_call(r) for r in self.__mocked_program_calls]))
2071
+
2072
+ @GeneralUtilities.check_arguments
2073
+ def __format_mock_program_call(self, r) -> str:
2074
+ r: ScriptCollectionCore.__MockProgramCall = r
2075
+ return f"'{r.workingdirectory}>{r.program} {r.argument}' (" \
2076
+ f"exitcode: {GeneralUtilities.str_none_safe(str(r.exit_code))}, " \
2077
+ f"pid: {GeneralUtilities.str_none_safe(str(r.pid))}, "\
2078
+ f"stdout: {GeneralUtilities.str_none_safe(str(r.stdout))}, " \
2079
+ f"stderr: {GeneralUtilities.str_none_safe(str(r.stderr))})"
2080
+
2081
+ @GeneralUtilities.check_arguments
2082
+ def register_mock_program_call(self, program: str, argument: str, workingdirectory: str, result_exit_code: int, result_stdout: str, result_stderr: str, result_pid: int, amount_of_expected_calls=1):
2083
+ "This function is for test-purposes only"
2084
+ for _ in itertools.repeat(None, amount_of_expected_calls):
2085
+ mock_call = ScriptCollectionCore.__MockProgramCall()
2086
+ mock_call.program = program
2087
+ mock_call.argument = argument
2088
+ mock_call.workingdirectory = workingdirectory
2089
+ mock_call.exit_code = result_exit_code
2090
+ mock_call.stdout = result_stdout
2091
+ mock_call.stderr = result_stderr
2092
+ mock_call.pid = result_pid
2093
+ self.__mocked_program_calls.append(mock_call)
2094
+
2095
+ @GeneralUtilities.check_arguments
2096
+ def __get_mock_program_call(self, program: str, argument: str, workingdirectory: str):
2097
+ result: ScriptCollectionCore.__MockProgramCall = None
2098
+ for mock_call in self.__mocked_program_calls:
2099
+ if ((re.match(mock_call.program, program) is not None)
2100
+ and (re.match(mock_call.argument, argument) is not None)
2101
+ and (re.match(mock_call.workingdirectory, workingdirectory) is not None)):
2102
+ result = mock_call
2103
+ break
2104
+ if result is None:
2105
+ raise LookupError(f"Tried to execute mock-call '{workingdirectory}>{program} {argument}' but no mock-call was defined for that execution")
2106
+ else:
2107
+ self.__mocked_program_calls.remove(result)
2108
+ return (result.exit_code, result.stdout, result.stderr, result.pid)
2109
+
2110
+ @GeneralUtilities.check_arguments
2111
+ class __MockProgramCall:
2112
+ program: str
2113
+ argument: str
2114
+ workingdirectory: str
2115
+ exit_code: int
2116
+ stdout: str
2117
+ stderr: str
2118
+ pid: int
2119
+
2120
+ @GeneralUtilities.check_arguments
2121
+ def run_with_epew_with_retry(self, program: str, argument: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str =None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False,print_live_output:bool=False,encode_argument_in_base64:bool=False, amount_of_attempts: int = 3, delay_in_seconds: int = 2) -> tuple[int, str, str, int]:
2122
+ return GeneralUtilities.retry_action(lambda: self.run_with_epew(program, argument, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace,arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive, print_live_output,encode_argument_in_base64), amount_of_attempts, delay_in_seconds=delay_in_seconds)
2123
+
2124
+ @GeneralUtilities.check_arguments
2125
+ def run_with_epew(self, program: str, argument: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = 600, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str =None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False,print_live_output:bool=False,encode_argument_in_base64:bool=False) -> tuple[int, str, str, int]:
2126
+ epew_argument:list[str]=["-p",program ,"-w", working_directory]
2127
+ if encode_argument_in_base64:
2128
+ if arguments_for_log is None:
2129
+ argument_escaped=argument.replace("\"", "\\\"")
2130
+ arguments_for_log=epew_argument+["-a",f"\"{argument_escaped}\""]
2131
+ base64_string = base64.b64encode(argument.encode("utf-8")).decode("utf-8")
2132
+ epew_argument=epew_argument+["-a",base64_string,"-b"]
2133
+ else:
2134
+ epew_argument=epew_argument+["-a",argument]
2135
+ if arguments_for_log is None:
2136
+ arguments_for_log=epew_argument
2137
+ return self.run_program_argsasarray("epew", epew_argument, working_directory, print_errors_as_information, log_file, timeoutInSeconds, addLogOverhead, title, log_namespace, arguments_for_log, throw_exception_if_exitcode_is_not_zero, custom_argument, interactive,print_live_output=print_live_output)
2138
+
2139
+
2140
+ # </run programs>
2141
+
2142
+ @GeneralUtilities.check_arguments
2143
+ def extract_archive_with_7z(self, unzip_program_file: str, zip_file: str, password: str, output_directory: str) -> None:
2144
+ password_set = not password is None
2145
+ file_name = Path(zip_file).name
2146
+ file_folder = os.path.dirname(zip_file)
2147
+ argument = "x"
2148
+ if password_set:
2149
+ argument = f"{argument} -p\"{password}\""
2150
+ argument = f"{argument} -o {output_directory}"
2151
+ argument = f"{argument} {file_name}"
2152
+ return self.run_program(unzip_program_file, argument, file_folder)
2153
+
2154
+ @GeneralUtilities.check_arguments
2155
+ def get_internet_time(self) -> datetime:
2156
+ response = ntplib.NTPClient().request('pool.ntp.org')
2157
+ return datetime.fromtimestamp(response.tx_time)
2158
+
2159
+ @GeneralUtilities.check_arguments
2160
+ def system_time_equals_internet_time(self, maximal_tolerance_difference: timedelta) -> bool:
2161
+ return abs(GeneralUtilities.get_now() - self.get_internet_time()) < maximal_tolerance_difference
2162
+
2163
+ @GeneralUtilities.check_arguments
2164
+ def system_time_equals_internet_time_with_default_tolerance(self) -> bool:
2165
+ return self.system_time_equals_internet_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
2166
+
2167
+ @GeneralUtilities.check_arguments
2168
+ def check_system_time(self, maximal_tolerance_difference: timedelta):
2169
+ if not self.system_time_equals_internet_time(maximal_tolerance_difference):
2170
+ raise ValueError("System time may be wrong")
2171
+
2172
+ @GeneralUtilities.check_arguments
2173
+ def check_system_time_with_default_tolerance(self) -> None:
2174
+ self.check_system_time(self.__get_default_tolerance_for_system_time_equals_internet_time())
2175
+
2176
+ @GeneralUtilities.check_arguments
2177
+ def __get_default_tolerance_for_system_time_equals_internet_time(self) -> timedelta:
2178
+ return timedelta(hours=0, minutes=0, seconds=3)
2179
+
2180
+ @GeneralUtilities.check_arguments
2181
+ def increment_version(self, input_version: str, increment_major: bool, increment_minor: bool, increment_patch: bool) -> str:
2182
+ splitted = input_version.split(".")
2183
+ GeneralUtilities.assert_condition(len(splitted) == 3, f"Version '{input_version}' does not have the 'major.minor.patch'-pattern.")
2184
+ major = int(splitted[0])
2185
+ minor = int(splitted[1])
2186
+ patch = int(splitted[2])
2187
+ if increment_major:
2188
+ major = major+1
2189
+ if increment_minor:
2190
+ minor = minor+1
2191
+ if increment_patch:
2192
+ patch = patch+1
2193
+ return f"{major}.{minor}.{patch}"
2194
+
2195
+ @GeneralUtilities.check_arguments
2196
+ def get_semver_version_from_gitversion(self, repository_folder: str) -> str:
2197
+ self.assert_is_git_repository(repository_folder)
2198
+ if (self.git_repository_has_commits(repository_folder)):
2199
+ result = self.get_version_from_gitversion(repository_folder, "MajorMinorPatch")
2200
+ if self.git_repository_has_uncommitted_changes(repository_folder):
2201
+ if self.get_current_git_branch_has_tag(repository_folder):
2202
+ id_of_latest_tag = self.git_get_commit_id(repository_folder, self.get_latest_git_tag(repository_folder))
2203
+ current_commit = self.git_get_commit_id(repository_folder)
2204
+ current_commit_is_on_latest_tag = id_of_latest_tag == current_commit
2205
+ if current_commit_is_on_latest_tag:
2206
+ result = self.increment_version(result, False, False, True)
2207
+ else:
2208
+ result = "0.1.0"
2209
+ return result
2210
+
2211
+ @staticmethod
2212
+ @GeneralUtilities.check_arguments
2213
+ def is_patch_version(version_string: str) -> bool:
2214
+ return not version_string.endswith(".0")
2215
+
2216
+ @GeneralUtilities.check_arguments
2217
+ def get_version_from_gitversion(self, folder: str, variable: str) -> str:
2218
+ # called twice as workaround for issue 1877 in gitversion ( https://github.com/GitTools/GitVersion/issues/1877 )
2219
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
2220
+ result = self.run_program_argsasarray("gitversion", ["/showVariable", variable], folder)
2221
+ result = GeneralUtilities.strip_new_line_character(result[1])
2222
+
2223
+ return result
2224
+
2225
+ @GeneralUtilities.check_arguments
2226
+ def generate_certificate_authority(self, folder: str, name: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
2227
+ if days_until_expire is None:
2228
+ days_until_expire = 1825
2229
+ if password is None:
2230
+ password = GeneralUtilities.generate_password()
2231
+ GeneralUtilities.ensure_directory_exists(folder)
2232
+ self.run_program_argsasarray("openssl", ['req', '-new', '-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:prime256v1', '-days', str(days_until_expire), '-nodes', '-x509', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={name}/OU={subj_ou}', '-passout', f'pass:{password}', '-keyout', f'{name}.key', '-out', f'{name}.crt'], folder)
2233
+
2234
+ @GeneralUtilities.check_arguments
2235
+ def generate_certificate(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str, days_until_expire: int = None, password: str = None) -> None:
2236
+ if days_until_expire is None:
2237
+ days_until_expire = 397
2238
+ if password is None:
2239
+ password = GeneralUtilities.generate_password()
2240
+ rsa_key_length = 4096
2241
+ self.run_program_argsasarray("openssl", ['genrsa', '-out', f'{filename}.key', f'{rsa_key_length}'], folder)
2242
+ self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-x509', '-key', f'{filename}.key', '-out', f'{filename}.unsigned.crt', '-days', f'{days_until_expire}'], folder)
2243
+ self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.selfsigned.pfx', '-password', f'pass:{password}', '-inkey', f'{filename}.key', '-in', f'{filename}.unsigned.crt'], folder)
2244
+ GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.password"), password)
2245
+ GeneralUtilities.write_text_to_file(os.path.join(folder, f"{filename}.san.conf"), f"""[ req ]
2246
+ default_bits = {rsa_key_length}
2247
+ distinguished_name = req_distinguished_name
2248
+ req_extensions = v3_req
2249
+ default_md = sha256
2250
+ dirstring_type = nombstr
2251
+ prompt = no
2252
+
2253
+ [ req_distinguished_name ]
2254
+ countryName = {subj_c}
2255
+ stateOrProvinceName = {subj_st}
2256
+ localityName = {subj_l}
2257
+ organizationName = {subj_o}
2258
+ organizationUnit = {subj_ou}
2259
+ commonName = {domain}
2260
+
2261
+ [v3_req]
2262
+ subjectAltName = @subject_alt_name
2263
+
2264
+ [ subject_alt_name ]
2265
+ DNS = {domain}
2266
+ """)
2267
+
2268
+ @GeneralUtilities.check_arguments
2269
+ def generate_certificate_sign_request(self, folder: str, domain: str, filename: str, subj_c: str, subj_st: str, subj_l: str, subj_o: str, subj_ou: str) -> None:
2270
+ self.run_program_argsasarray("openssl", ['req', '-new', '-subj', f'/C={subj_c}/ST={subj_st}/L={subj_l}/O={subj_o}/CN={domain}/OU={subj_ou}', '-key', f'{filename}.key', f'-out', f'{filename}.csr', f'-config', f'{filename}.san.conf'], folder)
2271
+
2272
+ @GeneralUtilities.check_arguments
2273
+ def sign_certificate(self, folder: str, ca_folder: str, ca_name: str, domain: str, filename: str, days_until_expire: int = None) -> None:
2274
+ if days_until_expire is None:
2275
+ days_until_expire = 397
2276
+ ca = os.path.join(ca_folder, ca_name)
2277
+ password_file = os.path.join(folder, f"{filename}.password")
2278
+ password = GeneralUtilities.read_text_from_file(password_file)
2279
+ self.run_program_argsasarray("openssl", ['x509', '-req', '-in', f'{filename}.csr', '-CA', f'{ca}.crt', '-CAkey', f'{ca}.key', '-CAcreateserial', '-CAserial', f'{ca}.srl', '-out', f'{filename}.crt', '-days', str(days_until_expire), '-sha256', '-extensions', 'v3_req', '-extfile', f'{filename}.san.conf'], folder)
2280
+ self.run_program_argsasarray("openssl", ['pkcs12', '-export', '-out', f'{filename}.pfx', f'-inkey', f'{filename}.key', '-in', f'{filename}.crt', '-password', f'pass:{password}'], folder)
2281
+
2282
+ @GeneralUtilities.check_arguments
2283
+ def update_dependencies_of_python_in_requirementstxt_file(self, file: str, ignored_dependencies: list[str]):
2284
+ # TODO consider ignored_dependencies
2285
+ lines = GeneralUtilities.read_lines_from_file(file)
2286
+ new_lines = []
2287
+ for line in lines:
2288
+ if GeneralUtilities.string_has_content(line):
2289
+ new_lines.append(self.__get_updated_line_for_python_requirements(line.strip()))
2290
+ GeneralUtilities.write_lines_to_file(file, new_lines)
2291
+
2292
+ @GeneralUtilities.check_arguments
2293
+ def __get_updated_line_for_python_requirements(self, line: str) -> str:
2294
+ if "==" in line or "<" in line:
2295
+ return line
2296
+ elif ">" in line:
2297
+ try:
2298
+ # line is something like "cyclonedx-bom>=2.0.2" and the function must return with the updated version
2299
+ # (something like "cyclonedx-bom>=2.11.0" for example)
2300
+ package = line.split(">")[0]
2301
+ operator = ">=" if ">=" in line else ">"
2302
+ headers = {'Cache-Control': 'no-cache'}
2303
+ response = requests.get(f'https://pypi.org/pypi/{package}/json', timeout=5, headers=headers)
2304
+ latest_version = response.json()['info']['version']
2305
+ # TODO update only minor- and patch-version
2306
+ # TODO print info if there is a new major-version
2307
+ return package+operator+latest_version
2308
+ except:
2309
+ return line
2310
+ else:
2311
+ raise ValueError(f'Unexpected line in requirements-file: "{line}"')
2312
+
2313
+ @GeneralUtilities.check_arguments
2314
+ def update_dependencies_of_python_in_setupcfg_file(self, setup_cfg_file: str, ignored_dependencies: list[str]):
2315
+ # TODO consider ignored_dependencies
2316
+ lines = GeneralUtilities.read_lines_from_file(setup_cfg_file)
2317
+ new_lines = []
2318
+ requirement_parsing_mode = False
2319
+ for line in lines:
2320
+ new_line = line
2321
+ if (requirement_parsing_mode):
2322
+ if ("<" in line or "=" in line or ">" in line):
2323
+ updated_line = f" {self.__get_updated_line_for_python_requirements(line.strip())}"
2324
+ new_line = updated_line
2325
+ else:
2326
+ requirement_parsing_mode = False
2327
+ else:
2328
+ if line.startswith("install_requires ="):
2329
+ requirement_parsing_mode = True
2330
+ new_lines.append(new_line)
2331
+ GeneralUtilities.write_lines_to_file(setup_cfg_file, new_lines)
2332
+
2333
+ @GeneralUtilities.check_arguments
2334
+ def update_dependencies_of_dotnet_project(self, csproj_file: str, ignored_dependencies: list[str]):
2335
+ folder = os.path.dirname(csproj_file)
2336
+ csproj_filename = os.path.basename(csproj_file)
2337
+ self.log.log(f"Check for updates in {csproj_filename}", LogLevel.Information)
2338
+ result = self.run_program_with_retry("dotnet", f"list {csproj_filename} package --outdated", folder, print_errors_as_information=True)
2339
+ for line in result[1].replace("\r", GeneralUtilities.empty_string).split("\n"):
2340
+ # Relevant output-lines are something like " > NJsonSchema 10.7.0 10.7.0 10.9.0"
2341
+ if ">" in line:
2342
+ package_name = line.replace(">", GeneralUtilities.empty_string).strip().split(" ")[0]
2343
+ if not (package_name in ignored_dependencies):
2344
+ self.log.log(f"Update package {package_name}...", LogLevel.Debug)
2345
+ time.sleep(1.1) # attempt to prevent rate-limit
2346
+ self.run_program_with_retry("dotnet", f"add {csproj_filename} package {package_name}", folder, print_errors_as_information=True)
2347
+
2348
+ @GeneralUtilities.check_arguments
2349
+ def create_deb_package(self, toolname: str, binary_folder: str, control_file_content: str, deb_output_folder: str, permission_of_executable_file_as_octet_triple: int) -> None:
2350
+
2351
+ # prepare
2352
+ GeneralUtilities.ensure_directory_exists(deb_output_folder)
2353
+ temp_folder = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
2354
+ GeneralUtilities.ensure_directory_exists(temp_folder)
2355
+ bin_folder = binary_folder
2356
+ tool_content_folder_name = toolname+"Content"
2357
+
2358
+ # create folder
2359
+ GeneralUtilities.ensure_directory_exists(temp_folder)
2360
+ control_content_folder_name = "controlcontent"
2361
+ packagecontent_control_folder = os.path.join(temp_folder, control_content_folder_name)
2362
+ GeneralUtilities.ensure_directory_exists(packagecontent_control_folder)
2363
+ data_content_folder_name = "datacontent"
2364
+ packagecontent_data_folder = os.path.join(temp_folder, data_content_folder_name)
2365
+ GeneralUtilities.ensure_directory_exists(packagecontent_data_folder)
2366
+ entireresult_content_folder_name = "entireresultcontent"
2367
+ packagecontent_entireresult_folder = os.path.join(temp_folder, entireresult_content_folder_name)
2368
+ GeneralUtilities.ensure_directory_exists(packagecontent_entireresult_folder)
2369
+
2370
+ # create "debian-binary"-file
2371
+ debianbinary_file = os.path.join(packagecontent_entireresult_folder, "debian-binary")
2372
+ GeneralUtilities.ensure_file_exists(debianbinary_file)
2373
+ GeneralUtilities.write_text_to_file(debianbinary_file, "2.0\n")
2374
+
2375
+ # create control-content
2376
+
2377
+ # conffiles
2378
+ conffiles_file = os.path.join(packagecontent_control_folder, "conffiles")
2379
+ GeneralUtilities.ensure_file_exists(conffiles_file)
2380
+
2381
+ # postinst-script
2382
+ postinst_file = os.path.join(packagecontent_control_folder, "postinst")
2383
+ GeneralUtilities.ensure_file_exists(postinst_file)
2384
+ exe_file = f"/usr/bin/{tool_content_folder_name}/{toolname}"
2385
+ link_file = f"/usr/bin/{toolname.lower()}"
2386
+ permission = str(permission_of_executable_file_as_octet_triple)
2387
+ GeneralUtilities.write_text_to_file(postinst_file, f"""#!/bin/sh
2388
+ ln -s {exe_file} {link_file}
2389
+ chmod {permission} {exe_file}
2390
+ chmod {permission} {link_file}
2391
+ """)
2392
+
2393
+ # control
2394
+ control_file = os.path.join(packagecontent_control_folder, "control")
2395
+ GeneralUtilities.ensure_file_exists(control_file)
2396
+ GeneralUtilities.write_text_to_file(control_file, control_file_content)
2397
+
2398
+ # md5sums
2399
+ md5sums_file = os.path.join(packagecontent_control_folder, "md5sums")
2400
+ GeneralUtilities.ensure_file_exists(md5sums_file)
2401
+
2402
+ # create data-content
2403
+
2404
+ # copy binaries
2405
+ usr_bin_folder = os.path.join(packagecontent_data_folder, "usr/bin")
2406
+ GeneralUtilities.ensure_directory_exists(usr_bin_folder)
2407
+ usr_bin_content_folder = os.path.join(usr_bin_folder, tool_content_folder_name)
2408
+ GeneralUtilities.copy_content_of_folder(bin_folder, usr_bin_content_folder)
2409
+
2410
+ # create debfile
2411
+ deb_filename = f"{toolname}.deb"
2412
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/control.tar.gz", "*"], packagecontent_control_folder)
2413
+ self.run_program_argsasarray("tar", ["czf", f"../{entireresult_content_folder_name}/data.tar.gz", "*"], packagecontent_data_folder)
2414
+ self.run_program_argsasarray("ar", ["r", deb_filename, "debian-binary", "control.tar.gz", "data.tar.gz"], packagecontent_entireresult_folder)
2415
+ result_file = os.path.join(packagecontent_entireresult_folder, deb_filename)
2416
+ shutil.copy(result_file, os.path.join(deb_output_folder, deb_filename))
2417
+
2418
+ # cleanup
2419
+ GeneralUtilities.ensure_directory_does_not_exist(temp_folder)
2420
+
2421
+ @GeneralUtilities.check_arguments
2422
+ def update_year_in_copyright_tags(self, file: str) -> None:
2423
+ current_year = str(GeneralUtilities.get_now().year)
2424
+ lines = GeneralUtilities.read_lines_from_file(file)
2425
+ lines_result = []
2426
+ for line in lines:
2427
+ if match := re.search("(.*<[Cc]opyright>.*)\\d\\d\\d\\d(.*<\\/[Cc]opyright>.*)", line):
2428
+ part1 = match.group(1)
2429
+ part2 = match.group(2)
2430
+ adapted = part1+current_year+part2
2431
+ else:
2432
+ adapted = line
2433
+ lines_result.append(adapted)
2434
+ GeneralUtilities.write_lines_to_file(file, lines_result)
2435
+
2436
+ @GeneralUtilities.check_arguments
2437
+ def update_year_in_first_line_of_file(self, file: str) -> None:
2438
+ current_year = str(GeneralUtilities.get_now().year)
2439
+ lines = GeneralUtilities.read_lines_from_file(file)
2440
+ lines[0] = re.sub("\\d\\d\\d\\d", current_year, lines[0])
2441
+ GeneralUtilities.write_lines_to_file(file, lines)
2442
+
2443
+ @GeneralUtilities.check_arguments
2444
+ def get_external_ip_address(self) -> str:
2445
+ information = self.get_externalnetworkinformation_as_json_string()
2446
+ parsed = json.loads(information)
2447
+ return parsed["IPAddress"]
2448
+
2449
+ @GeneralUtilities.check_arguments
2450
+ def get_country_of_external_ip_address(self) -> str:
2451
+ information = self.get_externalnetworkinformation_as_json_string()
2452
+ parsed = json.loads(information)
2453
+ return parsed["Country"]
2454
+
2455
+ @GeneralUtilities.check_arguments
2456
+ def get_externalnetworkinformation_as_json_string(self,clientinformation_link:str='https://clientinformation.anion327.de') -> str:
2457
+ headers = {'Cache-Control': 'no-cache'}
2458
+ response = requests.get(clientinformation_link, timeout=5, headers=headers)
2459
+ network_information_as_json_string = GeneralUtilities.bytes_to_string(response.content)
2460
+ return network_information_as_json_string
2461
+
2462
+ @GeneralUtilities.check_arguments
2463
+ def change_file_extensions(self, folder: str, from_extension: str, to_extension: str, recursive: bool, ignore_case: bool) -> None:
2464
+ extension_to_compare: str = None
2465
+ if ignore_case:
2466
+ extension_to_compare = from_extension.lower()
2467
+ else:
2468
+ extension_to_compare = from_extension
2469
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
2470
+ if (ignore_case and file.lower().endswith(f".{extension_to_compare}") or not ignore_case and file.endswith(f".{extension_to_compare}")):
2471
+ p = Path(file)
2472
+ p.rename(p.with_suffix('.'+to_extension))
2473
+ if recursive:
2474
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
2475
+ self.change_file_extensions(subfolder, from_extension, to_extension, recursive, ignore_case)
2476
+
2477
+ @GeneralUtilities.check_arguments
2478
+ def __add_chapter(self, main_reference_file, reference_content_folder, number: int, chaptertitle: str, content: str = None):
2479
+ if content is None:
2480
+ content = "TXDX add content here"
2481
+ filename = str(number).zfill(2)+"_"+chaptertitle.replace(' ', '-')
2482
+ file = f"{reference_content_folder}/{filename}.md"
2483
+ full_title = f"{number}. {chaptertitle}"
2484
+
2485
+ GeneralUtilities.append_line_to_file(main_reference_file, f"- [{full_title}](./{filename}.md)")
2486
+
2487
+ GeneralUtilities.ensure_file_exists(file)
2488
+ GeneralUtilities.write_text_to_file(file, f"""# {full_title}
2489
+
2490
+ {content}
2491
+ """.replace("XDX", "ODO"))
2492
+
2493
+ @GeneralUtilities.check_arguments
2494
+ def generate_arc42_reference_template(self, repository: str, productname: str = None, subfolder: str = None):
2495
+ productname: str
2496
+ if productname is None:
2497
+ productname = os.path.basename(repository)
2498
+ if subfolder is None:
2499
+ subfolder = "Other/Reference"
2500
+ reference_root_folder = f"{repository}/{subfolder}"
2501
+ reference_content_folder = reference_root_folder + "/Technical"
2502
+ if os.path.isdir(reference_root_folder):
2503
+ raise ValueError(f"The folder '{reference_root_folder}' does already exist.")
2504
+ GeneralUtilities.ensure_directory_exists(reference_root_folder)
2505
+ GeneralUtilities.ensure_directory_exists(reference_content_folder)
2506
+ main_reference_file = f"{reference_root_folder}/Reference.md"
2507
+ GeneralUtilities.ensure_file_exists(main_reference_file)
2508
+ GeneralUtilities.write_text_to_file(main_reference_file, f"""# {productname}
2509
+
2510
+ TXDX add minimal service-description here.
2511
+
2512
+ ## Technical documentation
2513
+
2514
+ """.replace("XDX", "ODO"))
2515
+ self.__add_chapter(main_reference_file, reference_content_folder, 1, 'Introduction and Goals', """## Overview
2516
+
2517
+ TXDX
2518
+
2519
+ ## Quality goals
2520
+
2521
+ TXDX
2522
+
2523
+ ## Stakeholder
2524
+
2525
+ | Name | How to contact | Reason |
2526
+ | ---- | -------------- | ------ |""")
2527
+ self.__add_chapter(main_reference_file, reference_content_folder, 2, 'Constraints', """## Technical constraints
2528
+
2529
+ | Constraint-identifier | Constraint | Reason |
2530
+ | --------------------- | ---------- | ------ |
2531
+
2532
+ ## Organizational constraints
2533
+
2534
+ | Constraint-identifier | Constraint | Reason |
2535
+ | --------------------- | ---------- | ------ |""")
2536
+ self.__add_chapter(main_reference_file, reference_content_folder, 3, 'Context and Scope', """## Context
2537
+
2538
+ TXDX
2539
+
2540
+ ## Scope
2541
+
2542
+ TXDX""")
2543
+ self.__add_chapter(main_reference_file, reference_content_folder, 4, 'Solution Strategy', """TXDX""")
2544
+ self.__add_chapter(main_reference_file, reference_content_folder, 5, 'Building Block View', """TXDX""")
2545
+ self.__add_chapter(main_reference_file, reference_content_folder, 6, 'Runtime View', """TXDX""")
2546
+ self.__add_chapter(main_reference_file, reference_content_folder, 7, 'Deployment View', """## Infrastructure-overview
2547
+
2548
+ TXDX
2549
+
2550
+ ## Infrastructure-requirements
2551
+
2552
+ TXDX
2553
+
2554
+ ## Deployment-proecsses
2555
+
2556
+ TXDX""")
2557
+ self.__add_chapter(main_reference_file, reference_content_folder, 8, 'Crosscutting Concepts', """TXDX""")
2558
+ self.__add_chapter(main_reference_file, reference_content_folder, 9, 'Architectural Decisions', """## Decision-board
2559
+
2560
+ | Decision-identifier | Date | Decision | Reason and notes |
2561
+ | ------------------- | ---- | -------- | ---------------- |""") # empty because there are no decsions yet
2562
+ self.__add_chapter(main_reference_file, reference_content_folder, 10, 'Quality Requirements', """TXDX""")
2563
+ self.__add_chapter(main_reference_file, reference_content_folder, 11, 'Risks and Technical Debt', """## Risks
2564
+
2565
+ Currently there are no known risks.
2566
+
2567
+ ## Technical debts
2568
+
2569
+ Currently there are no technical depts.""")
2570
+ self.__add_chapter(main_reference_file, reference_content_folder, 12, 'Glossary', """## Terms
2571
+
2572
+ | Term | Meaning |
2573
+ | ---- | ------- |
2574
+
2575
+ ## Abbreviations
2576
+
2577
+ | Abbreviation | Meaning |
2578
+ | ------------ | ------- |""")
2579
+
2580
+ GeneralUtilities.append_to_file(main_reference_file, """
2581
+
2582
+ ## Responsibilities
2583
+
2584
+ | Responsibility | Name and contact-information |
2585
+ | --------------- | ---------------------------- |
2586
+ | Pdocut-owner | TXDX |
2587
+ | Product-manager | TXDX |
2588
+ | Support | TXDX |
2589
+
2590
+ ## License & Pricing
2591
+
2592
+ TXDX
2593
+
2594
+ ## External resources
2595
+
2596
+ - [Repository](TXDX)
2597
+ - [Productive-System](TXDX)
2598
+ - [QualityCheck-system](TXDX)
2599
+ """.replace("XDX", "ODO"))
2600
+
2601
+ @GeneralUtilities.check_arguments
2602
+ def run_with_timeout(self, method, timeout_in_seconds: float) -> bool:
2603
+ # Returns true if the method was terminated due to a timeout
2604
+ # Returns false if the method terminates in the given time
2605
+ p = multiprocessing.Process(target=method)
2606
+ p.start()
2607
+ p.join(timeout_in_seconds)
2608
+ if p.is_alive():
2609
+ p.kill()
2610
+ p.join()
2611
+ return True
2612
+ else:
2613
+ return False
2614
+
2615
+ @GeneralUtilities.check_arguments
2616
+ def ensure_local_docker_network_exists(self, network_name: str) -> None:
2617
+ if not self.local_docker_network_exists(network_name):
2618
+ self.create_local_docker_network(network_name)
2619
+
2620
+ @GeneralUtilities.check_arguments
2621
+ def ensure_local_docker_network_does_not_exist(self, network_name: str) -> None:
2622
+ if self.local_docker_network_exists(network_name):
2623
+ self.remove_local_docker_network(network_name)
2624
+
2625
+ @GeneralUtilities.check_arguments
2626
+ def local_docker_network_exists(self, network_name: str) -> bool:
2627
+ return network_name in self.get_all_local_existing_docker_networks()
2628
+
2629
+ @GeneralUtilities.check_arguments
2630
+ def get_all_local_existing_docker_networks(self) -> list[str]:
2631
+ program_call_result = self.run_program("docker", "network list")
2632
+ std_out = program_call_result[1]
2633
+ std_out_lines = std_out.split("\n")[1:]
2634
+ result: list[str] = []
2635
+ for std_out_line in std_out_lines:
2636
+ normalized_line = ';'.join(std_out_line.split())
2637
+ splitted = normalized_line.split(";")
2638
+ result.append(splitted[1])
2639
+ return result
2640
+
2641
+ @GeneralUtilities.check_arguments
2642
+ def remove_local_docker_network(self, network_name: str) -> None:
2643
+ self.run_program("docker", f"network remove {network_name}")
2644
+
2645
+ @GeneralUtilities.check_arguments
2646
+ def create_local_docker_network(self, network_name: str) -> None:
2647
+ self.run_program("docker", f"network create {network_name}")
2648
+
2649
+ @GeneralUtilities.check_arguments
2650
+ def format_xml_file(self, file: str,add_xml_declaration:bool=True) -> None:
2651
+ encoding = "utf-8"
2652
+ element = ET.XML(GeneralUtilities.read_text_from_file(file, encoding))
2653
+ def trim_texts(elem: ET.Element):
2654
+ if elem.text:
2655
+ elem.text = elem.text.strip()
2656
+ if elem.tail:
2657
+ elem.tail = elem.tail.strip()
2658
+ for child in elem:
2659
+ trim_texts(child)
2660
+ trim_texts(element)
2661
+ ET.indent(element)
2662
+ content = ET.tostring(element, xml_declaration=add_xml_declaration, encoding="unicode")
2663
+ GeneralUtilities.write_text_to_file(file, content.rstrip("\n") + "\n", encoding)
2664
+
2665
+ @GeneralUtilities.check_arguments
2666
+ def format_html_file(self, file: str, add_html_declaration: bool = False) -> None:
2667
+ encoding = "utf-8"
2668
+ content = GeneralUtilities.read_text_from_file(file, encoding)
2669
+ content=self.format_html_content(content, add_html_declaration)
2670
+ GeneralUtilities.write_text_to_file(file, content, encoding)
2671
+
2672
+ @GeneralUtilities.check_arguments
2673
+ def format_html_content(self, content: str, add_html_declaration: bool = False) -> str:
2674
+
2675
+ VOID_ELEMENTS = {"area", "base", "br", "col", "embed", "hr", "img", "input",
2676
+ "link", "meta", "param", "source", "track", "wbr"}
2677
+
2678
+ class _Node:
2679
+ __slots__ = ("tag", "attrs", "children", "text", "is_void", "raw")
2680
+ def __init__(self, tag=None, attrs=(), text=None, is_void=False, raw=None):
2681
+ self.tag = tag
2682
+ self.attrs = list(attrs)
2683
+ self.children = []
2684
+ self.text = text
2685
+ self.is_void = is_void
2686
+ self.raw = raw
2687
+
2688
+ class _Builder(HTMLParser):
2689
+ def __init__(self):
2690
+ super().__init__(convert_charrefs=False)
2691
+ self.root = _Node()
2692
+ self.stack = [self.root]
2693
+
2694
+ def _top(self):
2695
+ return self.stack[-1]
2696
+
2697
+ def handle_starttag(self, tag, attrs):
2698
+ raw = self.get_starttag_text()
2699
+ raw_lower = raw.lower()
2700
+ original_attrs = []
2701
+ pos = 1 + len(tag)
2702
+ for lc_name, value in attrs:
2703
+ idx = raw_lower.index(lc_name, pos)
2704
+ original_attrs.append((raw[idx:idx + len(lc_name)], value))
2705
+ pos = idx + len(lc_name)
2706
+ node = _Node(tag=tag, attrs=original_attrs, is_void=tag.lower() in VOID_ELEMENTS)
2707
+ self._top().children.append(node)
2708
+ if not node.is_void:
2709
+ self.stack.append(node)
2710
+
2711
+ def handle_endtag(self, tag):
2712
+ if len(self.stack) > 1 and self.stack[-1].tag == tag:
2713
+ self.stack.pop()
2714
+
2715
+ def handle_data(self, data):
2716
+ t = " ".join(data.split())
2717
+ if t:
2718
+ self._top().children.append(_Node(text=t))
2719
+
2720
+ def handle_entityref(self, name):
2721
+ self._top().children.append(_Node(text=f"&{name};"))
2722
+
2723
+ def handle_charref(self, name):
2724
+ self._top().children.append(_Node(text=f"&#{name};"))
2725
+
2726
+ def handle_comment(self, data):
2727
+ self._top().children.append(_Node(raw=f"<!--{data}-->"))
2728
+
2729
+ def handle_decl(self, decl):
2730
+ self._top().children.append(_Node(raw=f"<!{decl}>"))
2731
+
2732
+ def handle_pi(self, data):
2733
+ self._top().children.append(_Node(raw=f"<?{data}>"))
2734
+
2735
+ _angular_exprs: list[str] = []
2736
+
2737
+ def _protect_angular(m: re.Match) -> str:
2738
+ idx = len(_angular_exprs)
2739
+ _angular_exprs.append(m.group(0))
2740
+ return f"__ANGEXPR{idx}__"
2741
+
2742
+ protected = re.sub(r'\{\{[\s\S]*?\}\}', _protect_angular, content)
2743
+ builder = _Builder()
2744
+ builder.feed(protected)
2745
+ ind = " "
2746
+
2747
+ def _serialize(node: _Node, depth: int) -> list:
2748
+ prefix = ind * depth
2749
+ if node.raw is not None:
2750
+ return [prefix + node.raw]
2751
+ if node.text is not None:
2752
+ return [node.text]
2753
+ if node.tag is None:
2754
+ out = []
2755
+ for c in node.children:
2756
+ out.extend(_serialize(c, depth))
2757
+ return out
2758
+ attr_str = "".join(f" {n}" if v is None else f' {n}="{v}"' for n, v in node.attrs)
2759
+ if node.is_void:
2760
+ return [f"{prefix}<{node.tag}{attr_str}>"]
2761
+ has_elem_children = any(c.tag is not None for c in node.children)
2762
+ if not has_elem_children:
2763
+ inner = "".join(c.text or "" for c in node.children)
2764
+ return [f"{prefix}<{node.tag}{attr_str}>{inner}</{node.tag}>"]
2765
+ lines = [f"{prefix}<{node.tag}{attr_str}>"]
2766
+ for c in node.children:
2767
+ child_lines = _serialize(c, depth + 1)
2768
+ if c.text is not None:
2769
+ lines.append(ind * (depth + 1) + child_lines[0])
2770
+ else:
2771
+ lines.extend(child_lines)
2772
+ lines.append(f"{prefix}</{node.tag}>")
2773
+ return lines
2774
+
2775
+ result = "\n".join(line for line in _serialize(builder.root, 0) if line.strip())
2776
+ for i, expr in enumerate(_angular_exprs):
2777
+ result = result.replace(f"__ANGEXPR{i}__", expr)
2778
+ if add_html_declaration and not result.lstrip().startswith("<!DOCTYPE"):
2779
+ result = "<!DOCTYPE html>\n" + result
2780
+ return result
2781
+
2782
+ @GeneralUtilities.check_arguments
2783
+ def get_pip_index_url_arguments_from_local_cache(self)->list[str]:
2784
+ arguments=[]
2785
+ pip_folder=GeneralUtilities.normalize_path(self.get_global_cache_folder()+"/Pip")
2786
+ if os.path.isdir(pip_folder):
2787
+ main_index_file=GeneralUtilities.normalize_path(os.path.join(pip_folder, "MainIndex.txt"))
2788
+ if os.path.isfile(main_index_file):
2789
+ lines=GeneralUtilities.read_nonempty_lines_from_file(main_index_file)
2790
+ url=[line for line in lines if line.startswith("IndexURL: ")][0].split(":")[1].strip()
2791
+ arguments.append("--index-url")
2792
+ arguments.append(url)
2793
+ extra_index_folder=GeneralUtilities.normalize_path(os.path.join(pip_folder, "ExtraIndexURLs"))
2794
+ if os.path.isdir(extra_index_folder):
2795
+ index_files=GeneralUtilities.get_direct_files_of_folder(extra_index_folder)
2796
+ if len(index_files) > 0:
2797
+ for indexurl_file in index_files:
2798
+ lines=GeneralUtilities.read_nonempty_lines_from_file(indexurl_file)
2799
+ url=[line for line in lines if line.startswith("IndexURL: ")][0].split(":")[1].strip()
2800
+ arguments.append("--extra-index-url")
2801
+ arguments.append(url)
2802
+ return arguments
2803
+
2804
+ @GeneralUtilities.check_arguments
2805
+ def install_requirementstxt_file(self, requirements_txt_file: str):
2806
+ folder: str = os.path.dirname(requirements_txt_file)
2807
+ filename: str = os.path.basename(requirements_txt_file)
2808
+ arguments= ["install", "-r", filename]
2809
+ for argument in self.get_pip_index_url_arguments_from_local_cache():
2810
+ arguments.append(argument)
2811
+ self.run_program_argsasarray("pip", arguments, folder,print_live_output=self.log.loglevel==LogLevel.Debug)
2812
+
2813
+ @GeneralUtilities.check_arguments
2814
+ def ocr_analysis_of_folder_using_local_docker_image(self, folder: str, extensions: list[str], languages: list[str],base_folder_for_entry: str,ignore_pattern:list[str] ) -> list[str]: # Returns a list of changed files due to ocr-analysis.
2815
+ #TODO start docker server
2816
+ serviceaddress:str=None#TODO
2817
+ self.ocr_analysis_of_folder(folder, serviceaddress, extensions, languages, base_folder_for_entry,ignore_pattern)
2818
+ #TODO stop docker server
2819
+
2820
+ @GeneralUtilities.check_arguments
2821
+ def ocr_analysis_of_folder(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str],base_folder_for_entry: str,ignore_pattern:list[str] ) -> list[str]: # Returns a list of changed files due to ocr-analysis.
2822
+ supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'gif', 'pdf', 'docx', 'doc', 'xlsx', 'xls', 'pptx', 'ppt']
2823
+ changes_files: list[str] = []
2824
+ if base_folder_for_entry is None:
2825
+ base_folder_for_entry=folder
2826
+ if extensions is None:
2827
+ extensions = supported_extensions
2828
+ for file in GeneralUtilities.get_direct_files_of_folder(folder):
2829
+ if GeneralUtilities.is_ignored_by_glob_pattern(os.path.dirname(file),file,ignore_pattern):
2830
+ continue
2831
+ file_lower = file.lower()
2832
+ for extension in extensions:
2833
+ if file_lower.endswith("."+extension):
2834
+ if self.ocr_analysis_of_file(file, serviceaddress, languages,base_folder_for_entry):
2835
+ changes_files.append(file)
2836
+ break
2837
+ for subfolder in GeneralUtilities.get_direct_folders_of_folder(folder):
2838
+ if GeneralUtilities.is_ignored_by_glob_pattern(os.path.dirname(subfolder),subfolder,ignore_pattern):
2839
+ continue
2840
+ for file in self.ocr_analysis_of_folder(subfolder, serviceaddress, extensions, languages,base_folder_for_entry+"/"+os.path.basename(subfolder), ignore_pattern):
2841
+ changes_files.append(file)
2842
+ return changes_files
2843
+
2844
+
2845
+ @GeneralUtilities.check_arguments
2846
+ def __it_supported_extension(self, file: str, supported_extensions: list[str]) -> bool:
2847
+ file_lower = file.lower()
2848
+ for extension in supported_extensions:
2849
+ if file_lower.endswith("."+extension):
2850
+ return True
2851
+ return False
2852
+
2853
+ @GeneralUtilities.check_arguments
2854
+ def ocr_analysis_of_file(self, file: str, serviceaddress: str, languages: list[str], readable_folder_entry:str ) -> bool: # Returns true if the ocr-file was generated or updated. Returns false if the existing ocr-file was not changed.
2855
+ supported_extensions = ['png', 'jpg', 'jpeg', 'tiff', 'bmp', 'webp', 'gif', 'pdf', 'rtf', 'docx', 'doc', 'odt', 'xlsx', 'xls', 'ods', 'pptx', 'ppt', 'odp']
2856
+ if not self.__it_supported_extension(file, supported_extensions):
2857
+ raise ValueError(f"File '{file}' is not supported due to unsupported extension. Supported extensions are: {', '.join(supported_extensions)}")
2858
+ target_file = file+".ocr.txt"
2859
+ hash_of_current_file: str = GeneralUtilities.get_sha256_of_file(file)
2860
+ try:
2861
+ if os.path.isfile(target_file):
2862
+ lines = GeneralUtilities.read_lines_from_file(target_file)
2863
+ previous_hash_of_current_file: str = lines[1].split(":")[1].strip()
2864
+ if hash_of_current_file == previous_hash_of_current_file:
2865
+ return False
2866
+ except:
2867
+ pass
2868
+ GeneralUtilities.write_message_to_stdout(f"Starting OCR-analysis of file \"{file}\"...")
2869
+ ocr_content = self.get_ocr_content_of_file(file, serviceaddress, languages)
2870
+ GeneralUtilities.ensure_file_exists(target_file)
2871
+ if readable_folder_entry is None:
2872
+ readable_folder_entry="."
2873
+ readable_folder_entry=readable_folder_entry.replace("\\", "/")
2874
+ GeneralUtilities.write_text_to_file(target_file, f"""Name of file: \"{readable_folder_entry}/{os.path.basename(file)}\""
2875
+ Hash of file: {hash_of_current_file}
2876
+ OCR-content:
2877
+ {ocr_content}""")
2878
+ return True
2879
+
2880
+ @GeneralUtilities.check_arguments
2881
+ def get_ocr_content_of_file(self, file: str, serviceaddress: str, languages: list[str]) -> str:
2882
+ result: str = None
2883
+ extension = Path(file).suffix[1:]
2884
+ mime_types = {
2885
+ "pdf": "application/pdf",
2886
+ "png": "image/png",
2887
+ "jpg": "image/jpeg",
2888
+ "jpeg": "image/jpeg",
2889
+ "txt": "text/plain",
2890
+ "json": "application/json",
2891
+ "doc": "application/msword",
2892
+ "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
2893
+ "xls": "application/vnd.ms-excel",
2894
+ "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
2895
+ }
2896
+ if serviceaddress is None:
2897
+ server_url_file:str= GeneralUtilities.normalize_path(f"{str(Path.home())}/.ScriptCollection/OCR/ServiceURL.txt")
2898
+ if os.path.isfile(server_url_file):
2899
+ for line in GeneralUtilities.read_nonempty_lines_from_file(server_url_file):
2900
+ if not line.startswith("#"):
2901
+ serviceaddress = line.strip()
2902
+ break
2903
+ GeneralUtilities.assert_not_null(serviceaddress, "ocr-service-address must not be null.")
2904
+ mime_type = mime_types.get(extension.lower(), "application/octet-stream")
2905
+ service_url: str = f"{serviceaddress}/API/v1/SimpleOCR/GetOCRContent?mimeType={mime_type}"
2906
+ for language in languages:
2907
+ service_url = service_url + f"&languages={language}"
2908
+ headers = {'Cache-Control': 'no-cache'}
2909
+ with open(file, "rb") as f:
2910
+ files_to_analyse = {
2911
+ "fileContent": (os.path.basename(file), f, mime_type)
2912
+ }
2913
+ r = requests.put(service_url, timeout=3600, headers=headers, files=files_to_analyse,verify=True)
2914
+ if r.status_code != 200:
2915
+ if r.status_code == 400:
2916
+ return f"Could not calculate ocr-content for file \"{file}\". File may be broken."
2917
+ else:
2918
+ raise ValueError(f"Retrieving ocr-content for file \"{file}\" resulted in HTTP-response-code {r.status_code}.")
2919
+
2920
+ result = GeneralUtilities.bytes_to_string(r.content)
2921
+ return result
2922
+
2923
+ @GeneralUtilities.check_arguments
2924
+ def ocr_analysis_of_repository(self, folder: str, serviceaddress: str, extensions: list[str], languages: list[str]) -> None:
2925
+ self.assert_is_git_repository(folder)
2926
+ self.ocr_analysis_of_folder(folder, serviceaddress, extensions, languages,".",[".git"])
2927
+
2928
+ @GeneralUtilities.check_arguments
2929
+ def update_timestamp_in_file(self, target_file: str) -> None:
2930
+ lines = GeneralUtilities.read_lines_from_file(target_file)
2931
+ new_lines = []
2932
+ prefix: str = "# last update: "
2933
+ for line in lines:
2934
+ if line.startswith(prefix):
2935
+ new_lines.append(prefix+GeneralUtilities.datetime_to_string_for_readable_entry(GeneralUtilities.get_now(),False))
2936
+ else:
2937
+ new_lines.append(line)
2938
+ GeneralUtilities.write_lines_to_file(target_file, new_lines)
2939
+
2940
+ @GeneralUtilities.check_arguments
2941
+ def do_and_log_task(self, name_of_task: str, task,log_end_of_Task:bool=True)->int:
2942
+ try:
2943
+ self.log.log(f"Start action \"{name_of_task}\".", LogLevel.Information)
2944
+ result = task()
2945
+ if result is None:
2946
+ result = 0
2947
+ return result
2948
+ except Exception as e:
2949
+ self.log.log_exception(f"Error while running action \"{name_of_task}\".", e, LogLevel.Error)
2950
+ return 1
2951
+ finally:
2952
+ if log_end_of_Task:
2953
+ self.log.log(f"Finished action \"{name_of_task}\".", LogLevel.Information)
2954
+
2955
+
2956
+ default_excluded_patterns_for_loc: list[str] = ["**.txt", "**.md", "**.svg", "**.xlf", "**.vscode", "**/Resources/**", "**/Reference/**", ".gitignore", ".gitattributes", "Other/Metrics/**"]
2957
+
2958
+ @GeneralUtilities.check_arguments
2959
+ def get_lines_of_code_with_default_excluded_patterns(self, repository: str) -> int:
2960
+ return self.get_lines_of_code(repository, self.default_excluded_patterns_for_loc)
2961
+
2962
+ @GeneralUtilities.check_arguments
2963
+ def get_lines_of_code(self, repository: str, excluded_pattern: list[str]) -> int:
2964
+ self.assert_is_git_repository(repository)
2965
+ result: int = 0
2966
+ self.log.log(f"Calculate lines of code in repository '{repository}' with excluded patterns: {', '.join(excluded_pattern)}",LogLevel.Debug)
2967
+ git_response = self.run_program("git", "ls-files", repository)
2968
+ files: list[str] = GeneralUtilities.string_to_lines(git_response[1])
2969
+ for file in files:
2970
+ if os.path.isfile(os.path.join(repository, file)):
2971
+ if self.__is_excluded_by_glob_pattern(file, excluded_pattern):
2972
+ self.log.log(f"File '{file}' is ignored because it matches an excluded pattern.",LogLevel.Diagnostic)
2973
+ else:
2974
+ full_file: str = os.path.join(repository, file)
2975
+ if GeneralUtilities.is_binary_file(full_file):
2976
+ self.log.log(f"File '{file}' is ignored because it is a binary-file.",LogLevel.Diagnostic)
2977
+ else:
2978
+ self.log.log(f"Count lines of file '{file}'.",LogLevel.Diagnostic)
2979
+ length = len(GeneralUtilities.read_nonempty_lines_from_file(full_file))
2980
+ result = result+length
2981
+ else:
2982
+ self.log.log(f"File '{file}' is ignored because it does not exist.",LogLevel.Diagnostic)
2983
+ return result
2984
+
2985
+ @GeneralUtilities.check_arguments
2986
+ def __is_excluded_by_glob_pattern(self, file: str, excluded_patterns: list[str]) -> bool:
2987
+ for pattern in excluded_patterns:
2988
+ if fnmatch.fnmatch(file, pattern):
2989
+ return True
2990
+ return False
2991
+
2992
+ @GeneralUtilities.check_arguments
2993
+ def create_zip_archive(self, folder:str,zip_file:str) -> None:
2994
+ GeneralUtilities.assert_folder_exists(folder)
2995
+ GeneralUtilities.assert_file_does_not_exist(zip_file)
2996
+ folder = os.path.abspath(folder)
2997
+ with zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) as zipf:
2998
+ for root, _, files in os.walk(folder):
2999
+ for file in files:
3000
+ file_path = os.path.join(root, file)
3001
+ arcname = os.path.relpath(file_path, start=folder)
3002
+ zipf.write(file_path, arcname)
3003
+
3004
+ @GeneralUtilities.check_arguments
3005
+ def start_local_test_service(self, file: str):
3006
+ example_folder = os.path.dirname(file)
3007
+ docker_compose_file = os.path.join(example_folder, "docker-compose.yml")
3008
+ for service in self.get_services_from_yaml_file(docker_compose_file):
3009
+ self.kill_docker_container(service)
3010
+ example_name = os.path.basename(example_folder)
3011
+ title = f"Test{example_name}"
3012
+ argument=f"compose -p {title.lower()}"
3013
+ if os.path.isfile(os.path.join(example_folder,"Parameters.env")):
3014
+ argument=argument+" --env-file Parameters.env"
3015
+ argument=argument+" up --detach"
3016
+ self.run_program("docker", argument, example_folder, title=title,print_live_output=True)
3017
+
3018
+ @GeneralUtilities.check_arguments
3019
+ def stop_local_test_service(self, file: str):
3020
+ example_folder = os.path.dirname(file)
3021
+ example_name = os.path.basename(example_folder)
3022
+ title = f"Test{example_name}"
3023
+ self.run_program("docker", f"compose -p {title.lower()} down", example_folder, title=title,print_live_output=True)
3024
+
3025
+ @GeneralUtilities.check_arguments
3026
+ def generate_chart_diagram(self,source_file:str,target_file:str):
3027
+ workingfolder=os.path.dirname(source_file)
3028
+ argument=f"\"{source_file}\" \"{target_file}\""
3029
+ loglevelMap = {
3030
+ LogLevel.Error: "error",
3031
+ LogLevel.Warning: "warn",
3032
+ LogLevel.Information: "info",
3033
+ LogLevel.Debug: "debug",
3034
+ }
3035
+ if self.log.loglevel==LogLevel.Debug:
3036
+ argument=f"-l {loglevelMap[self.log.loglevel]} {argument}"
3037
+ self.run_with_epew("vl2svg",argument,workingfolder,encode_argument_in_base64=True)
3038
+ #this uses vega-light. to use vega "vg2svg" should be used instead.
3039
+
3040
+ @GeneralUtilities.check_arguments
3041
+ def inspect_container(self, container_name: str) :
3042
+ program_result = self.run_program(
3043
+ "docker",
3044
+ f"inspect {container_name}",
3045
+ throw_exception_if_exitcode_is_not_zero=True
3046
+ )
3047
+ stdout=program_result[1]
3048
+
3049
+ data = json.loads(stdout)
3050
+ GeneralUtilities.assert_condition(len(data)==1,f"Unexpected array-length of docker-inspect-output for container \"{container_name}\".")
3051
+ return data[0]
3052
+
3053
+ @GeneralUtilities.check_arguments
3054
+ def container_is_exists(self,container_name:str)->bool:
3055
+ program_result = self.run_program(
3056
+ "docker",
3057
+ f"inspect {container_name}",
3058
+ throw_exception_if_exitcode_is_not_zero=False
3059
+ )
3060
+ return program_result[0]==0
3061
+
3062
+ @GeneralUtilities.check_arguments
3063
+ def container_is_running(self,container_name:str)->bool:
3064
+ data = self.inspect_container( container_name)
3065
+ if data is None:
3066
+ return False
3067
+
3068
+ return data["State"]["Status"] == "running"
3069
+
3070
+ @GeneralUtilities.check_arguments
3071
+ def container_is_healthy(self,container_name:str)->bool:
3072
+ data = self.inspect_container( container_name)
3073
+ if data is None:
3074
+ return False
3075
+
3076
+ state = data["State"]
3077
+ health = state.get("Health")
3078
+
3079
+ if health is None:
3080
+ return False # kein HEALTHCHECK definiert
3081
+
3082
+ return health["Status"] == "healthy"
3083
+
3084
+ @GeneralUtilities.check_arguments
3085
+ def get_output_of_container(self,container_name:str)->str:
3086
+
3087
+ program_result= self.run_program_argsasarray(
3088
+ "docker",
3089
+ ["logs",container_name],
3090
+ throw_exception_if_exitcode_is_not_zero=False
3091
+ )
3092
+ exit_code=program_result[0]
3093
+ stdout=program_result[1]
3094
+ stderr=program_result[2]
3095
+ if exit_code != 0:
3096
+ return ""
3097
+
3098
+ return stdout+"\n"+stderr
3099
+
3100
+ @GeneralUtilities.check_arguments
3101
+ def container_is_running_and_healthy(self,container_name:str)->bool:
3102
+ if not self.container_is_exists(container_name):
3103
+ return False
3104
+ if not self.container_is_running(container_name):
3105
+ return False
3106
+ if not self.container_is_healthy(container_name):
3107
+ return False
3108
+ return True
3109
+
3110
+ def reclaim_space_from_docker(self,remove_containers:bool,remove_volumes:bool,remove_images:bool, amount_of_attempts: int = 5):
3111
+ self.log.log("Reclaim disk space from docker...",LogLevel.Debug)
3112
+ if remove_containers:
3113
+ self.run_program_with_retry("docker","container prune -f",amount_of_attempts=amount_of_attempts)
3114
+ if remove_volumes:
3115
+ self.run_program_with_retry("docker","volume prune -f",amount_of_attempts=amount_of_attempts)
3116
+ if remove_images:
3117
+ self.run_program_with_retry("docker","image prune -a -f",amount_of_attempts=amount_of_attempts)
3118
+ self.run_program_with_retry("docker","builder prune -a -f",amount_of_attempts=amount_of_attempts)
3119
+ self.run_program_with_retry("docker","buildx prune -a -f",amount_of_attempts=amount_of_attempts,throw_exception_if_exitcode_is_not_zero=False) # buildx prune is not available on every machine.
3120
+ self.run_program_with_retry("docker","system df",print_live_output=self.log.loglevel==LogLevel.Debug,amount_of_attempts=amount_of_attempts)
3121
+
3122
+ @GeneralUtilities.check_arguments
3123
+ def get_docker_networks(self)->list[str]:
3124
+ program_result=self.run_program("docker","network list")
3125
+ result=[]
3126
+ lines=program_result[1].split("\n")[1:]
3127
+ for line in lines:
3128
+ splitted=[item for item in line.split(' ') if GeneralUtilities.string_has_content(item)]
3129
+ result.append(splitted[1].replace("\n","").replace("\r","").strip())
3130
+ return result
3131
+
3132
+ @GeneralUtilities.check_arguments
3133
+ def ensure_docker_network_is_available(self,network_name:str):
3134
+ #TODO add cli-script to call this function
3135
+ if not (network_name in self.get_docker_networks()):
3136
+ self.run_program("docker",f"network create {network_name}")
3137
+
3138
+ @GeneralUtilities.check_arguments
3139
+ def ensure_docker_network_is_not_available(self,network_name:str):
3140
+ #TODO add cli-script to call this function
3141
+ if network_name in self.get_docker_networks():
3142
+ self.run_program("docker",f"network rm {network_name}")
3143
+
3144
+ @GeneralUtilities.check_arguments
3145
+ def get_available_cultures_for_angular_app(self,angular_json_file:str)->list[str]:
3146
+ languages = ["en"]
3147
+ with open(angular_json_file, "r", encoding="utf-8") as f:
3148
+ data = json.load(f)
3149
+ for project in data.get("projects", {}).values():
3150
+ i18n = project.get("i18n", {})
3151
+ locales = i18n.get("locales", {})
3152
+ languages.extend(locales.keys())
3153
+
3154
+ languages=list(languages)
3155
+ return languages
3156
+
3157
+ @GeneralUtilities.check_arguments
3158
+ def parse_tasks_from_codeworkspace_file(self,code_workspace_file:str)->list[VSCodeWorkspaceShellTask]:
3159
+ result=[]
3160
+ jsoncontent = json.loads(GeneralUtilities.read_text_from_file(code_workspace_file))
3161
+ tasks = jsoncontent["tasks"]["tasks"]
3162
+ for task in tasks:
3163
+ if task["type"] == "shell":
3164
+ label: str = task["label"]
3165
+ name: str = GeneralUtilities.to_pascal_case(label)
3166
+ command:str= task["command"]
3167
+ work_dir:str = None
3168
+
3169
+ if "options" in task:
3170
+ options = task["options"]
3171
+ if "cwd" in options:
3172
+ work_dir = options["cwd"]
3173
+ work_dir = work_dir.replace("${workspaceFolder}", ".")
3174
+
3175
+ command_with_args = command
3176
+ if "args" in task:
3177
+ args = task["args"]
3178
+ if len(args) > 1:
3179
+ command_with_args = f"{command_with_args} {' '.join(args)}"
3180
+
3181
+ description: str =None
3182
+ if "description" in task:
3183
+ description = f'{label} ({task["description"]})'
3184
+ else:
3185
+ description = label
3186
+
3187
+
3188
+ alias_list:list[str]=[]
3189
+ name_lower=name.lower()
3190
+ if name!=name.lower():
3191
+ alias_list.append(name_lower)
3192
+
3193
+ if "aliases" in task:
3194
+ aliases = task["aliases"]
3195
+ for alias in aliases:
3196
+ alias_list.append(alias)
3197
+
3198
+ allow_custom_arguments:bool=False
3199
+ if "allowcustomarguments" in task:
3200
+ allow_custom_arguments = task["allowcustomarguments"]
3201
+
3202
+ result.append(VSCodeWorkspaceShellTask(name, description, work_dir, command_with_args, alias_list, allow_custom_arguments))
3203
+ return result
3204
+
3205
+ @GeneralUtilities.check_arguments
3206
+ def parse_mongodbconnection_from_codeworkspace_file(self,code_workspace_file:str)->list[VSCodeWorkspaceMongoDBConnection]:
3207
+ result=[]
3208
+ jsoncontent = json.loads(GeneralUtilities.read_text_from_file(code_workspace_file))
3209
+ settings=jsoncontent["settings"]
3210
+ if "mdb.presetConnections" in settings:
3211
+ connections = settings["mdb.presetConnections"]
3212
+ for connection in connections:
3213
+ result.append(VSCodeWorkspaceMongoDBConnection(connection["name"],connection["connectionString"]))
3214
+ return result
3215
+
3216
+ @GeneralUtilities.check_arguments
3217
+ def parse_sqlconnection_from_codeworkspace_file(self,code_workspace_file:str)->list[VSCodeWorkspaceMariaDBConnection]:
3218
+ result=[]
3219
+ jsoncontent = json.loads(GeneralUtilities.read_text_from_file(code_workspace_file))
3220
+ settings=jsoncontent["settings"]
3221
+ if "sqltools.connections" in settings:
3222
+ connections = settings["sqltools.connections"]
3223
+ for connection in connections:
3224
+ result.append(VSCodeWorkspaceMariaDBConnection(connection["name"],connection["server"],connection["port"],connection["database"],connection["username"],connection["password"]))
3225
+ return result
3226
+
3227
+ @GeneralUtilities.check_arguments
3228
+ def is_xliff2_file(self,file: str) -> bool:
3229
+ tree = ET.parse(file)
3230
+ root = tree.getroot()
3231
+
3232
+ tag = root.tag # "{urn:oasis:names:tc:xliff:document:2.0}xliff"
3233
+
3234
+ if tag.startswith("{"):
3235
+ namespace, localname = tag[1:].split("}", 1)
3236
+ else:
3237
+ namespace = None
3238
+ localname = tag
3239
+
3240
+ if localname != "xliff":
3241
+ return False
3242
+
3243
+ if namespace != "urn:oasis:names:tc:xliff:document:2.0":
3244
+ return False
3245
+
3246
+ if root.get("version") != "2.0":
3247
+ return False
3248
+
3249
+ return True
3250
+
3251
+ @GeneralUtilities.check_arguments
3252
+ def __sync_xlf2_files(self,base_file:ET.ElementTree, language_files:dict [
3253
+ str,#filepath
3254
+ ET.ElementTree#parsed file
3255
+ ]):
3256
+ """This function assumes that all files are valid xliff2 files and that the base file is the reference for syncing.
3257
+ This function adds new entries from the base file to the language files if they do not already exist using the value from base_file.
3258
+ This function removes entries from the language files if they do not exist in the base file anymore.
3259
+ In the end the updated language files are written to the disk. The base file is not changed."""
3260
+ #The file which was parsed looks like:
3261
+ #<?xml version="1.0" encoding="UTF-8" ?>
3262
+ #<xliff version="2.0" xmlns="urn:oasis:names:tc:xliff:document:2.0" srcLang="en">
3263
+ # <file id="ngi18n" original="ng.template">
3264
+ # <unit id="logingreeting">
3265
+ # <notes>
3266
+ # <note category="location">src/app/modules/home-page/login-form/login-form.component.html:2,4</note>
3267
+ # </notes>
3268
+ # <segment>
3269
+ # <source>Welcome back, please login</source>
3270
+ # </segment>
3271
+ # </unit>
3272
+ # <unit id="username">
3273
+ # <notes>
3274
+ # <note category="location">src/app/modules/home-page/login-form/login-form.component.html:5,6</note>
3275
+ # </notes>
3276
+ # <segment>
3277
+ # <source>Username</source>
3278
+ # </segment>
3279
+ # </unit>
3280
+ # <unit id="password">
3281
+ # <notes>
3282
+ # <note category="location">src/app/modules/home-page/login-form/login-form.component.html:12,13</note>
3283
+ # </notes>
3284
+ # <segment>
3285
+ # <source>Password</source>
3286
+ # </segment>
3287
+ # </unit>
3288
+ # </file>
3289
+ #</xliff>
3290
+
3291
+ NS = "urn:oasis:names:tc:xliff:document:2.0"
3292
+ NSMAP = {"x": NS}
3293
+ base_root = base_file.getroot()
3294
+ base_file_element = base_root.find("x:file", namespaces=NSMAP)
3295
+ if base_file_element is None:
3296
+ raise ValueError("Invalid XLIFF base file: <file> element not found")
3297
+
3298
+ # Collect base units
3299
+ base_units = {
3300
+ unit.get("id"): unit
3301
+ for unit in base_file_element.findall("x:unit", namespaces=NSMAP)
3302
+ }
3303
+ base_ids = set(base_units.keys())
3304
+ for filepath, lang_tree in language_files.items():
3305
+ lang_root = lang_tree.getroot()
3306
+ lang_file_element = lang_root.find("x:file", namespaces=NSMAP)
3307
+ if lang_file_element is None:
3308
+ raise ValueError(f"{filepath}: <file> element not found")
3309
+
3310
+ # Collect language units
3311
+ lang_units = {
3312
+ unit.get("id"): unit
3313
+ for unit in lang_file_element.findall("x:unit", namespaces=NSMAP)
3314
+ }
3315
+ lang_ids = set(lang_units.keys())
3316
+
3317
+ # Remove obsolete units
3318
+ obsolete_ids = lang_ids - base_ids
3319
+ for unit_id in obsolete_ids:
3320
+ lang_file_element.remove(lang_units[unit_id])
3321
+
3322
+ # Add missing units
3323
+ missing_ids = base_ids - lang_ids
3324
+ for unit_id in missing_ids:
3325
+ new_unit = copy.deepcopy(base_units[unit_id])
3326
+ lang_file_element.append(new_unit)
3327
+
3328
+ # Reorder units to match base order
3329
+ current_units = {
3330
+ unit.get("id"): unit
3331
+ for unit in lang_file_element.findall("x:unit", namespaces=NSMAP)
3332
+ }
3333
+ for unit in list(lang_file_element.findall("x:unit", namespaces=NSMAP)):
3334
+ lang_file_element.remove(unit)
3335
+ for unit_id in base_units.keys():
3336
+ if unit_id in current_units:
3337
+ lang_file_element.append(current_units[unit_id])
3338
+
3339
+ #TODO if a translation-unit has the "new"-attribute: set its value from the fallback-language. (if the culture contains a "-": e. g. take value from "de" as fallback-value for "de-AT"; or else: take value from base_file as fallback-value)
3340
+
3341
+ # Write file back to disk
3342
+ ET.register_namespace("", NS) # Ensure default namespace is declared without prefix
3343
+ Path(filepath).write_bytes(
3344
+ ET.tostring(
3345
+ lang_tree.getroot(),
3346
+ xml_declaration=True,
3347
+ encoding="UTF-8"
3348
+ )
3349
+ )
3350
+ ScriptCollectionCore().format_xml_file(filepath)
3351
+
3352
+ @GeneralUtilities.check_arguments
3353
+ def sync_xlf2_files(self,prefix:str, languages:list[str], folder:str):
3354
+ #languages=["de", "fr"] for example. the default-language (usually english) must not be included.
3355
+ base_file=os.path.join(folder, f"{prefix}.xlf")
3356
+ base_file_xml:ET.ElementTree=ET.parse(base_file)
3357
+ GeneralUtilities.assert_condition(self.is_xliff2_file(base_file), f"The base file '{base_file}' is not a valid XLIFF 2.0 file.")
3358
+ GeneralUtilities.assert_file_exists(base_file)
3359
+ if len(languages)==0:
3360
+ raise ValueError("No files provided for syncing.")
3361
+ if len(languages)==1:
3362
+ return
3363
+ language_files_list=[os.path.join(folder, f"{prefix}.{language}.xlf") for language in languages]
3364
+ language_files_with_content:dict[str,ET.ElementTree]=dict()
3365
+ for language_file in language_files_list:
3366
+ GeneralUtilities.assert_file_exists(language_file)
3367
+ GeneralUtilities.assert_condition(self.is_xliff2_file(language_file), f"The base file '{base_file}' is not a valid XLIFF 2.0 file.")
3368
+ language_files_with_content[language_file]=ET.parse(language_file)
3369
+
3370
+ #sync existing files
3371
+ self.__sync_xlf2_files(base_file_xml, language_files_with_content)
3372
+
3373
+
3374
+ @GeneralUtilities.check_arguments
3375
+ def translate_xlf_files_in_folder(self, folder: str, base_language: str, libre_translate_api_server: str):
3376
+ """Translates all .xlf files directly in the given folder (non-recursive)."""
3377
+ pattern = re.compile(r'^.+\.[a-z]{2,3}\.xlf$')
3378
+ for filename in os.listdir(folder):
3379
+ if not pattern.match(filename):
3380
+ continue
3381
+ file_path = os.path.join(folder, filename)
3382
+ self.translate_xlf_file(file_path, base_language, libre_translate_api_server)
3383
+
3384
+ @GeneralUtilities.check_arguments
3385
+ def translate_xlf_file(self, file: str, base_language: str, libre_translate_api_server: str):
3386
+ """
3387
+ Translates all segments with state='initial' in a XLIFF 2.0 file.
3388
+ The target language is extracted from the filename (e.g. 'messages.es.xlf' -> 'es').
3389
+ """
3390
+ ns_uri = "urn:oasis:names:tc:xliff:document:2.0"
3391
+ ns = {"xliff": ns_uri}
3392
+
3393
+ filename = os.path.basename(file)
3394
+ parts = filename.split(".")
3395
+ if len(parts) < 3:
3396
+ raise ValueError(f"Cannot extract language from filename: {filename}")
3397
+ target_language = parts[-2]
3398
+
3399
+ tree = ET.parse(file)
3400
+ root = tree.getroot()
3401
+
3402
+ for segment in root.findall(".//xliff:segment", ns):
3403
+ if segment.get("state", "initial") != "initial":
3404
+ continue
3405
+ source_el = segment.find("xliff:source", ns)
3406
+ if source_el is None or not source_el.text:
3407
+ continue
3408
+
3409
+ translated_text = self.translate(source_el.text, base_language, target_language, libre_translate_api_server)
3410
+
3411
+ target_el = segment.find("xliff:target", ns)
3412
+ if target_el is None:
3413
+ target_el = ET.SubElement(segment, f"{{{ns_uri}}}target")
3414
+
3415
+ target_el.text = translated_text
3416
+ segment.set("state", "translated")
3417
+
3418
+ ET.register_namespace("", ns_uri)
3419
+ xml_bytes = ET.tostring(root, encoding="utf-8", xml_declaration=True)
3420
+ with open(file, "wb") as f:
3421
+ f.write(xml_bytes)
3422
+
3423
+ @GeneralUtilities.check_arguments
3424
+ def translate(self, content: str, source_language: str, target_language: str, libre_translate_api_server: str) -> str:
3425
+ """Translates text using the LibreTranslate API."""
3426
+ url = f"{libre_translate_api_server.rstrip('/')}/translate"
3427
+ if "-" in source_language:
3428
+ source_language=source_language.split("-")[0]
3429
+ if "-" in target_language:
3430
+ target_language=target_language.split("-")[0]
3431
+ payload = {
3432
+ "q": content,
3433
+ "source": source_language,
3434
+ "target": target_language,
3435
+ "format": "text"
3436
+ }
3437
+ response = requests.post(url, json=payload, timeout=30)
3438
+ response.raise_for_status()
3439
+ return response.json()["translatedText"]
3440
+
3441
+ @GeneralUtilities.check_arguments
3442
+ def detect_language(self, content: str, libre_translate_api_server: str) -> str:
3443
+ """Detects the language of the given text using the LibreTranslate API."""
3444
+ url = f"{libre_translate_api_server.rstrip('/')}/detect"
3445
+ payload = {"q": content}
3446
+ response = requests.post(url, json=payload, timeout=30)
3447
+ response.raise_for_status()
3448
+ results = response.json()
3449
+ if not results:
3450
+ raise ValueError("Language detection returned no results.")
3451
+ return results[0]["language"]
3452
+
3453
+ @GeneralUtilities.check_arguments
3454
+ def get_all_files_in_git_repository(self,repository_folder:str,ignore_ignored_files:bool=True,include_submodules: bool = True) -> list[str]:
3455
+ """Returns a list of all files in a git-repository."""
3456
+ cmd = ["ls-files", "--cached"]
3457
+ if ignore_ignored_files:
3458
+ cmd.append("--exclude-standard")
3459
+ if include_submodules:
3460
+ cmd.append("--recurse-submodules")
3461
+ output=self.run_program_argsasarray("git", cmd,repository_folder)
3462
+ files = [ GeneralUtilities.normalize_path("./" + line) for line in output[1].splitlines()if line.strip() ]
3463
+ return files
3464
+
3465
+ @GeneralUtilities.check_arguments
3466
+ def write_file_list_for_repository(self,repository_folder:str,target_file:str="./FileList.txt",ignore_ignored_files:bool=True,include_submodules: bool = True) -> None:
3467
+ if os.path.isabs(target_file):
3468
+ target_file=GeneralUtilities.resolve_relative_path(target_file,repository_folder)
3469
+ target_file=GeneralUtilities.normalize_path(target_file)
3470
+ files=[path.replace("\\","/") for path in self.get_all_files_in_git_repository(repository_folder,ignore_ignored_files,include_submodules)]
3471
+ GeneralUtilities.ensure_file_exists(target_file)
3472
+ GeneralUtilities.write_lines_to_file(target_file, files)
3473
+
3474
+ @GeneralUtilities.check_arguments
3475
+ def get_all_commits_in_git_repository(self,repository_folder:str,include_all_heads:bool=False) -> list[str]:
3476
+ """Returns a textual visualization of all commits in a git-repository."""
3477
+ #do 'git log --reverse --all --pretty=format:"%ci | %H | %cn <%ce> | %d | %s"'
3478
+ args = ["log", "--reverse", "--pretty=format:%ci | %H | %cn <%ce> | %D | %s"]
3479
+ if include_all_heads:
3480
+ args.append("--all")
3481
+ result=self.run_program_argsasarray("git", args, repository_folder, throw_exception_if_exitcode_is_not_zero=True)
3482
+ output= result[1]
3483
+ result=output.splitlines()
3484
+ return result
3485
+
3486
+ @GeneralUtilities.check_arguments
3487
+ def write_commit_list_for_repository(self,repository_folder:str,target_file:str,include_all_heads:bool=False) -> None:
3488
+ if os.path.isabs(target_file):
3489
+ target_file=GeneralUtilities.resolve_relative_path(target_file,repository_folder)
3490
+ target_file=GeneralUtilities.normalize_path(target_file)
3491
+ commits=self.get_all_commits_in_git_repository(repository_folder, include_all_heads)
3492
+ GeneralUtilities.ensure_file_exists(target_file)
3493
+ GeneralUtilities.write_lines_to_file(target_file, commits)
3494
+
3495
+ @GeneralUtilities.check_arguments
3496
+ def is_runnning_in_container(self) ->bool:
3497
+ """this function is based on a convention and does not do a real check."""
3498
+ return os.environ.get("ISRUNNINGINCONTAINER") == "true"