ScriptCollection 4.4.2__py3-none-any.whl → 4.4.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -312,14 +312,46 @@ def UpdateDependencies() -> int:
312
312
  parser.add_argument('-v', '--verbosity', required=False, default=3, help=f"Sets the loglevel. Possible values: {verbosity_values}")
313
313
  parser.add_argument('--targetenvironment', required=False, default="QualityCheck")
314
314
  parser.add_argument('--additionalargumentsfile', required=False, default=None)
315
- parser.add_argument("-c",'--nocache', required=False, default=False, action='store_true')
315
+ parser.add_argument("-n",'--nocache', required=False, default=False, action='store_true')
316
+ parser.add_argument('-c','--runincontainer', required=False, default=False, action='store_true')
317
+ parser.add_argument('-b','--basemountfolder', required=False, default=None, help="Only used if -c is set.")
316
318
  args = parser.parse_args()
317
319
 
318
320
  verbosity=LogLevel(int(args.verbosity))
319
321
  repo:str=GeneralUtilities.resolve_relative_path(args.repositoryfolder,os.getcwd())
320
- t:TFCPS_CodeUnit_BuildCodeUnits=TFCPS_CodeUnit_BuildCodeUnits(repo,verbosity,args.targetenvironment,args.additionalargumentsfile,not args.nocache,False,False)
321
- t.update_dependencies()
322
- return 0
322
+ t:TFCPS_CodeUnit_BuildCodeUnits=TFCPS_CodeUnit_BuildCodeUnits(repo,verbosity,args.targetenvironment,args.additionalargumentsfile,not args.nocache,False,False)
323
+ if args.runincontainer:
324
+ base_mount_folder:str=args.basemountfolder
325
+ if base_mount_folder is None:
326
+ base_mount_folder=repo
327
+ base_mount_folder:str=GeneralUtilities.resolve_relative_path(base_mount_folder,os.getcwd())
328
+ success, _ = t.update_dependencies_in_container(base_mount_folder)
329
+ return 0 if success else 1
330
+ else:
331
+ t.update_dependencies()
332
+ return 0
333
+
334
+ def UpdateDependenciesC() -> int:
335
+ parser = argparse.ArgumentParser()
336
+ parser.add_argument('-r','--repositoryfolder', required=False, default=".")
337
+ verbosity_values = ", ".join(f"{lvl.value}={lvl.name}" for lvl in LogLevel)
338
+ parser.add_argument('-v', '--verbosity', required=False, default=3, help=f"Sets the loglevel. Possible values: {verbosity_values}")
339
+ parser.add_argument('--targetenvironment', required=False, default="QualityCheck")
340
+ parser.add_argument('--additionalargumentsfile', required=False, default=None)
341
+ parser.add_argument("-n",'--nocache', required=False, default=False, action='store_true')
342
+ parser.add_argument('-b','--basemountfolder', required=False, default=None)
343
+ args = parser.parse_args()
344
+
345
+ GeneralUtilities.reconfigure_standard_input_and_outputs()
346
+ repo:str=GeneralUtilities.resolve_relative_path(args.repositoryfolder,os.getcwd())
347
+ verbosity=LogLevel(int(args.verbosity))
348
+ base_mount_folder:str=args.basemountfolder
349
+ if base_mount_folder is None:
350
+ base_mount_folder=repo
351
+ base_mount_folder:str=GeneralUtilities.resolve_relative_path(base_mount_folder,os.getcwd())
352
+ t:TFCPS_CodeUnit_BuildCodeUnits=TFCPS_CodeUnit_BuildCodeUnits(repo,verbosity,args.targetenvironment,args.additionalargumentsfile,not args.nocache,False,False)
353
+ success, _ = t.update_dependencies_in_container(base_mount_folder)
354
+ return 0 if success else 1
323
355
 
324
356
 
325
357
  def GenerateCertificateAuthority() -> int:
@@ -575,7 +607,7 @@ def NormalizeLineEndings() -> int:
575
607
  if not os.path.isfile(args.path):
576
608
  GeneralUtilities.write_message_to_stderr(f"File '{args.path}' does not exist.")
577
609
  return 1
578
- ScriptCollectionCore().normalize_line_endings(args.path)
610
+ ScriptCollectionCore().normalize_invisible_characters(args.path)
579
611
  return 0
580
612
 
581
613
 
@@ -72,12 +72,24 @@ class GeneralUtilities:
72
72
  @staticmethod
73
73
  def check_arguments(function):
74
74
  def __check_function(*args, **named_args):
75
- parameters: list = inspect.getfullargspec(function)[0].copy()
75
+ # "signature" is used instead of "getfullargspec", because it follows the "__wrapped__"-chain which
76
+ # "functools.wraps" sets. Without that this decorator would only see the wrapper when it is applied
77
+ # above another decorator (for example above "deprecated"), and a wrapper has no named parameters,
78
+ # so no argument could be checked at all.
79
+ try:
80
+ parameters: list = list(inspect.signature(function).parameters)
81
+ except (ValueError, TypeError):
82
+ parameters: list = inspect.getfullargspec(function)[0].copy()
76
83
  arguments: list = list(tuple(args)).copy()
77
84
  if "self" in parameters:
78
85
  parameters.remove("self")
79
86
  arguments.pop(0)
80
87
  for index, argument in enumerate(arguments):
88
+ if len(parameters) <= index:
89
+ # More arguments were passed than the function has parameters. That is a wrong call, but it
90
+ # is not this check which has to report it: the call of the function below raises the
91
+ # regular TypeError of python, whose message names the expected and the given amount.
92
+ break
81
93
  if argument is not None: # Check type of None is not possible. None is always a valid argument-value
82
94
  if parameters[index] in function.__annotations__: # Check if a type-hint for parameter exist. If not, no parameter-type available for argument-type-check
83
95
  # Check type of arguments if the type is a generic type seems to be impossible.
ScriptCollection/SCLog.py CHANGED
@@ -17,7 +17,7 @@ class LogLevel(Enum):
17
17
 
18
18
 
19
19
  class SCLog:
20
- loglevel: LogLevel = None #minimum loglevel
20
+ loglevel: LogLevel = None #minimum loglevel
21
21
  log_file: str = None
22
22
  add_overhead_to_console: bool = None
23
23
  add_overhead_to_logfile: bool = None
@@ -39,7 +39,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
39
39
  from .ProgramRunnerPopen import ProgramRunnerPopen
40
40
  from .SCLog import SCLog, LogLevel
41
41
 
42
- version = "4.4.2"
42
+ version = "4.4.6"
43
43
  __version__ = version
44
44
 
45
45
  class VSCodeWorkspaceShellTask:
@@ -201,6 +201,8 @@ class ScriptCollectionCore:
201
201
  log: SCLog = None
202
202
  # Magic string which can be used inside the arguments of run_command_in_folder. Every occurrence of it will be replaced by the (resolved) actual_folder.
203
203
  run_command_in_folder_actual_folder_placeholder: str = "{actual_folder}"
204
+ # The bytes with which a file starts when it was written with a utf8-byte-order-mark. See normalize_invisible_characters.
205
+ __utf8_byte_order_mark: bytes = b"\xef\xbb\xbf"
204
206
  # Base-url of the GitHub-REST-API. This is an internal constant on purpose and deliberately not exposed as a parameter of the GitHub-functions.
205
207
  __github_api_base_url: str = "https://api.github.com"
206
208
 
@@ -1634,6 +1636,101 @@ class ScriptCollectionCore:
1634
1636
  for thumbnail_to_delete in preview_files:
1635
1637
  os.remove(thumbnail_to_delete)
1636
1638
 
1639
+ @GeneralUtilities.check_arguments
1640
+ def start_rtsp_test_stream(self, target_address: str, additional_text: str = None, width: int = 640, height: int = 480, frames_per_second: int = 10, font_file: str = None, working_directory: str = None) -> int:
1641
+ """Starts a video-stream which shows a number that is counted up once per second and publishes it to the
1642
+ given rtsp-address. Returns the process-id of the running ffmpeg, which can be passed to
1643
+ GeneralUtilities.kill_process to stop the stream again.
1644
+
1645
+ The purpose of this stream is to be a reproducible replacement for a real camera in testcases: the picture
1646
+ depends only on how long the stream is running, so a screenshot which is taken after 1.5 seconds always shows
1647
+ the same number as a screenshot which is taken after 1.5 seconds in another run.
1648
+
1649
+ The address has to point to a running rtsp-server (mediamtx, for example) which accepts the publishing of a
1650
+ stream; ffmpeg does not open a port by itself.
1651
+
1652
+ :param target_address: The rtsp-address to publish to, for example "rtsp://localhost:8554/teststream".
1653
+ :param additional_text: An optional text which is shown below the number. Useful to tell two streams apart.
1654
+ Only letters, digits and spaces are allowed: everything else would have to be escaped
1655
+ for the filter-syntax of ffmpeg, and a stream which only exists for testcases does not
1656
+ need a text which requires that.
1657
+ :param font_file: The font which is used. When it is not given then a font of the operating-system is used.
1658
+ """
1659
+ GeneralUtilities.assert_condition(target_address.startswith("rtsp://"), f"'{target_address}' is not a rtsp-address.")
1660
+ argument: list[str] = self.__get_arguments_for_test_stream(additional_text, width, height, frames_per_second, font_file) + [
1661
+ "-f", "rtsp",
1662
+ # Tcp instead of udp, because udp loses pictures under load, which would make a screenshot show
1663
+ # something else than the number which belongs to the moment it was taken.
1664
+ "-rtsp_transport", "tcp",
1665
+ target_address]
1666
+ return self.run_program_argsasarray_async("ffmpeg", argument, working_directory, title=f"RTSP-test-stream ({target_address})")
1667
+
1668
+ @GeneralUtilities.check_arguments
1669
+ def create_test_stream_video(self, target_file: str, duration_in_seconds: int, additional_text: str = None, width: int = 640, height: int = 480, frames_per_second: int = 10, font_file: str = None, working_directory: str = None) -> None:
1670
+ """Writes a video-file which shows the same picture as start_rtsp_test_stream, but into a file instead of
1671
+ to a rtsp-address and as fast as possible instead of in realtime.
1672
+
1673
+ This exists to be able to create the expected pictures of a testcase without recording a stream: the pictures
1674
+ are the same, so a picture of this file can be used as the baseline for a picture of a recorded stream.
1675
+
1676
+ :param target_file: The video-file which is written.
1677
+ :param duration_in_seconds: How long the video is. The number is counted up once per second, so a video of ten
1678
+ seconds contains the numbers 0 to 9.
1679
+ """
1680
+ argument: list[str] = self.__get_arguments_for_test_stream(additional_text, width, height, frames_per_second, font_file, realtime=False) + [
1681
+ "-t", str(duration_in_seconds),
1682
+ "-y", target_file]
1683
+ self.run_program_argsasarray("ffmpeg", argument, working_directory, throw_exception_if_exitcode_is_not_zero=True)
1684
+
1685
+ @GeneralUtilities.check_arguments
1686
+ def __get_arguments_for_test_stream(self, additional_text: str, width: int, height: int, frames_per_second: int, font_file: str, realtime: bool = True) -> list[str]:
1687
+ """Returns the ffmpeg-arguments which produce the picture of a test-stream. The picture is defined here
1688
+ only once, so that the video which is used as the expected result really shows the same as the stream."""
1689
+ if additional_text is not None:
1690
+ GeneralUtilities.assert_condition(re.fullmatch("[a-zA-Z0-9 ]+", additional_text) is not None, f"The additional text must consist of letters, digits and spaces only, but it is '{additional_text}'.")
1691
+ if font_file is None:
1692
+ font_file = self.__get_font_file_for_drawing_text()
1693
+ # "%{eif:t:d}" is the amount of seconds the stream is running, rounded down to an integer, so the number
1694
+ # changes exactly once per second. The colons of it have to be escaped, because the filter-syntax of
1695
+ # ffmpeg uses the colon as the separator between two options.
1696
+ escaped_font_file = font_file.replace("\\", "/").replace(":", "\\:")
1697
+ text_filters: list[str] = [f"drawtext=fontfile='{escaped_font_file}':text='%{{eif\\:t\\:d}}':fontsize={int(height/4)}:fontcolor=white:x=(w-text_w)/2:y=(h-text_h)/2"]
1698
+ if additional_text is not None:
1699
+ text_filters.append(f"drawtext=fontfile='{escaped_font_file}':text='{additional_text}':fontsize={int(height/12)}:fontcolor=white:x=(w-text_w)/2:y=h*3/4")
1700
+ result: list[str] = []
1701
+ if realtime:
1702
+ # "-re" makes ffmpeg send the pictures in realtime instead of as fast as it can, which is what a
1703
+ # camera does and what makes the number correspond to the elapsed time.
1704
+ result.append("-re")
1705
+ result = result + [
1706
+ "-f", "lavfi",
1707
+ "-i", f"color=c=black:s={width}x{height}:r={frames_per_second}",
1708
+ "-vf", ",".join(text_filters),
1709
+ "-c:v", "libx264",
1710
+ "-preset", "ultrafast",
1711
+ "-tune", "zerolatency",
1712
+ # Without this the stream is not playable by every client, because they expect a chroma-subsampling
1713
+ # which the default of libx264 does not use for this input.
1714
+ "-pix_fmt", "yuv420p",
1715
+ # One keyframe per second, so a client which connects in between shows a picture immediately.
1716
+ "-g", str(frames_per_second)]
1717
+ return result
1718
+
1719
+ @GeneralUtilities.check_arguments
1720
+ def __get_font_file_for_drawing_text(self) -> str:
1721
+ """Returns a font-file of the operating-system. A font-file is used instead of a font-name because
1722
+ resolving a name requires fontconfig, which does not exist on every system ffmpeg runs on."""
1723
+ candidates: list[str] = [
1724
+ "C:/Windows/Fonts/arial.ttf",
1725
+ "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
1726
+ "/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf",
1727
+ "/usr/share/fonts/TTF/DejaVuSans.ttf",
1728
+ "/System/Library/Fonts/Supplemental/Arial.ttf"]
1729
+ for candidate in candidates:
1730
+ if os.path.isfile(candidate):
1731
+ return candidate
1732
+ raise ValueError(f"No font-file was found. Pass one explicitly. Checked: {', '.join(candidates)}")
1733
+
1637
1734
  @GeneralUtilities.check_arguments
1638
1735
  def extract_pdf_pages(self, file: str, from_page: int, to_page: int, outputfile: str) -> None:
1639
1736
  pdf_reader: PdfReader = PdfReader(file)
@@ -2478,6 +2575,7 @@ class ScriptCollectionCore:
2478
2575
 
2479
2576
  @GeneralUtilities.check_arguments
2480
2577
  def run_with_epew(self, program: str, argument: str = "", working_directory: str = None, print_errors_as_information: bool = False, log_file: str = None, timeoutInSeconds: int = None, addLogOverhead: bool = False, title: str = None, log_namespace: str = "", arguments_for_log: str =None, throw_exception_if_exitcode_is_not_zero: bool = True, custom_argument: object = None, interactive: bool = False,print_live_output:bool=False,encode_argument_in_base64:bool=False) -> tuple[int, str, str, int]:
2578
+ GeneralUtilities.assert_condition(GeneralUtilities.epew_is_available(), "Epew is not available.")
2481
2579
  epew_argument:list[str]=["-p",program ,"-w", working_directory]
2482
2580
  if encode_argument_in_base64:
2483
2581
  if arguments_for_log is None:
@@ -3144,7 +3242,7 @@ TXDX
3144
3242
  content = json.loads(GeneralUtilities.read_text_from_file(file, encoding))
3145
3243
  formatted_content = json.dumps(content, indent=indentation, sort_keys=False, ensure_ascii=False)
3146
3244
  GeneralUtilities.write_text_to_file(file, formatted_content.rstrip("\n") + "\n", encoding)
3147
- self.normalize_line_endings(file)
3245
+ self.normalize_invisible_characters(file)
3148
3246
 
3149
3247
  @GeneralUtilities.check_arguments
3150
3248
  def format_xml_file(self, file: str,add_xml_declaration:bool=True) -> None:
@@ -3161,7 +3259,7 @@ TXDX
3161
3259
  ET.indent(element)
3162
3260
  content = self.__serialize_xml_element(element, add_xml_declaration)
3163
3261
  GeneralUtilities.write_text_to_file(file, content.rstrip("\n") + "\n", encoding)
3164
- self.normalize_line_endings(file)
3262
+ self.normalize_invisible_characters(file)
3165
3263
 
3166
3264
  @GeneralUtilities.check_arguments
3167
3265
  def __serialize_xml_element(self, element: ET.Element, add_xml_declaration: bool) -> str:
@@ -3192,25 +3290,33 @@ TXDX
3192
3290
  content = GeneralUtilities.read_text_from_file(file, encoding)
3193
3291
  content=self.format_html_content(content, add_html_declaration)
3194
3292
  GeneralUtilities.write_text_to_file(file, content, encoding)
3195
- self.normalize_line_endings(file)
3293
+ self.normalize_invisible_characters(file)
3196
3294
 
3197
3295
  @GeneralUtilities.check_arguments
3198
- def normalize_line_endings(self, file: str) -> None:
3199
- # Normalizes all physical line-endings of the given file to LF (replaces CRLF and lone CR by LF).
3200
- # Operates on the raw bytes so no character-encoding is assumed and only the line-ending-bytes are
3201
- # touched; the file is only rewritten when its content actually changes.
3296
+ def normalize_invisible_characters(self, file: str) -> None:
3297
+ # Normalizes all physical line-endings of the given file to LF (replaces CRLF and lone CR by LF) and removes
3298
+ # a utf8-byte-order-mark at the begin of the file.
3299
+ # Operates on the raw bytes so no character-encoding is assumed and only the line-ending-bytes and the
3300
+ # byte-order-mark are touched; the file is only rewritten when its content actually changes.
3202
3301
  content = GeneralUtilities.read_binary_from_file(file)
3203
- normalized_content = content.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
3302
+ normalized_content = content
3303
+ # Utf8 has no byte-order which could be marked, so the mark carries no information. But it is part of the
3304
+ # content of the file for everything which does not know it, which is why it breaks tools which expect the
3305
+ # file to start with its actual content (a shebang-line, an xml-declaration or the name of a column of a
3306
+ # csv-file, for example).
3307
+ if normalized_content.startswith(ScriptCollectionCore.__utf8_byte_order_mark):
3308
+ normalized_content = normalized_content[len(ScriptCollectionCore.__utf8_byte_order_mark):]
3309
+ normalized_content = normalized_content.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
3204
3310
  if normalized_content != content:
3205
3311
  GeneralUtilities.write_binary_to_file(file, normalized_content)
3206
3312
 
3207
3313
  @GeneralUtilities.check_arguments
3208
- def normalize_line_endings_of_files_in_folder(self, folder: str, file_extensions: list[str]) -> None:
3314
+ def normalize_invisible_characters_of_files_in_folder(self, folder: str, file_extensions: list[str]) -> None:
3209
3315
  """Normalizes the line-endings of all not-git-ignored files inside 'folder' which have one of the given
3210
3316
  file-extensions. The file-extensions must be given without a leading dot (example: ["ts", "js"])."""
3211
3317
  for file_extension in file_extensions:
3212
3318
  for file in self.get_not_git_ignored_files_of_folder(folder, f".{file_extension}"):
3213
- self.normalize_line_endings(file)
3319
+ self.normalize_invisible_characters(file)
3214
3320
 
3215
3321
  @GeneralUtilities.check_arguments
3216
3322
  def remove_trailing_linebreak(self, file: str) -> None:
@@ -17,6 +17,8 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
17
17
  csproj_file:bool = None
18
18
  #the output of the dotnet-cli is localized. The language is set explicitly for the commands whose output gets parsed, because otherwise the parsing would only work on machines which are configured to use english.
19
19
  __dotnet_cli_environment_variables:dict = {"DOTNET_CLI_UI_LANGUAGE": "en-US"}
20
+ #the only accepted form of a version of a package-reference: an exact version in square brackets. Everything else is a version-range.
21
+ __pinned_version_regex = re.compile(r"^\[[^,\[\]]+\]$")
20
22
 
21
23
  def __init__(self,current_file:str,verbosity:LogLevel,targetenvironmenttype:str,use_cache:bool,is_pre_merge:bool):
22
24
  super().__init__(current_file, verbosity,targetenvironmenttype,use_cache,is_pre_merge)
@@ -161,6 +163,33 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
161
163
  return [self.__remove_carriage_returns_recursively(item) for item in value]
162
164
  return value
163
165
 
166
+ @GeneralUtilities.check_arguments
167
+ def __get_lock_file_name(self, runtime: str) -> str:
168
+ """Returns the name of the lock-file which belongs to the given runtime.
169
+
170
+ Every runtime needs its own lock-file: the lock-file records the set of runtime-identifiers it was created for, and a
171
+ restore in locked mode fails with NU1004 when that set differs from the one which is currently restored (the restore
172
+ gets exactly one runtime via "--runtime"). The name is relative, so NuGet resolves it per project - a restore over the
173
+ solution therefore creates one lock-file next to each of the two project-files instead of letting them collide."""
174
+ return f"packages.{runtime}.lock.json"
175
+
176
+ @GeneralUtilities.check_arguments
177
+ def __get_lock_file_arguments(self, codeunit_folder: str, codeunit_name: str, runtime: str) -> list[str]:
178
+ """Returns the restore-arguments which pin the dependencies of the given runtime to the content of its lock-files.
179
+
180
+ Locked mode is only requested when the lock-file of every project of the codeunit already exists, because a restore in
181
+ locked mode can not create one. A codeunit which does not have them yet is restored without it and gets a warning, so
182
+ introducing the lock-files does not break the build of a codeunit which was not migrated yet."""
183
+ lock_file_name: str = self.__get_lock_file_name(runtime)
184
+ expected_lock_files: list[str] = [os.path.join(codeunit_folder, project_name, lock_file_name) for project_name in [codeunit_name, codeunit_name+"Tests"]]
185
+ missing_lock_files: list[str] = [lock_file for lock_file in expected_lock_files if not os.path.isfile(lock_file)]
186
+ result: list[str] = [f"-p:NuGetLockFilePath={lock_file_name}"]
187
+ if len(missing_lock_files) == 0:
188
+ result.append("--locked-mode")
189
+ else:
190
+ self._protected_sc.log.log(f"The dependencies of codeunit \"{codeunit_name}\" are not restored in locked mode for the runtime \"{runtime}\", because the following lock-file(s) do not exist yet: {', '.join(missing_lock_files)}. The restore creates them now; commit them so that the dependencies of this codeunit are pinned.", LogLevel.Warning)
191
+ return result
192
+
164
193
  @GeneralUtilities.check_arguments
165
194
  def __standardized_tasks_build_for_dotnet_build(self, csproj_file: str, originaloutputfolder: str, files_to_sign: dict[str, str], commitid: str, runtimes: list[str], target_environmenttype_mapping: dict[str, str], copy_license_file_to_target_folder: bool, repository_folder: str, codeunit_name: str) -> None:
166
195
  self._protected_sc.assert_is_git_repository(repository_folder)
@@ -181,7 +210,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
181
210
  GeneralUtilities.ensure_directory_does_not_exist(outputfolder)
182
211
  self._protected_sc.run_program("dotnet", "clean", csproj_file_folder)
183
212
  GeneralUtilities.ensure_directory_exists(outputfolder)
184
- self._protected_sc.run_program_argsasarray("dotnet", ["restore", "--runtime", runtime], codeunit_folder,print_live_output=self.get_verbosity()==LogLevel.Debug)
213
+ self._protected_sc.run_program_argsasarray("dotnet", ["restore", "--runtime", runtime]+self.__get_lock_file_arguments(codeunit_folder, codeunit_name, runtime), codeunit_folder,print_live_output=self.get_verbosity()==LogLevel.Debug)
185
214
  self._protected_sc.run_program_argsasarray("dotnet", ["build", "--no-restore", csproj_file_name, "-c", dotnet_build_configuration, "-o", outputfolder, "--runtime", runtime], csproj_file_folder,print_live_output=self.get_verbosity()==LogLevel.Debug)
186
215
  if copy_license_file_to_target_folder:
187
216
  license_file = os.path.join(repository_folder, "License.txt")
@@ -347,7 +376,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
347
376
  def linting(self) -> None:
348
377
  codeunit_name = self.get_codeunit_name()
349
378
  codeunit_folder = self.get_codeunit_folder()
350
- self._protected_sc.normalize_line_endings_of_files_in_folder(codeunit_folder, ["cs"])
379
+ self._protected_sc.normalize_invisible_characters_of_files_in_folder(codeunit_folder, ["cs"])
351
380
  nuspec_file = os.path.join(codeunit_folder, "Other", "Build", f"{codeunit_name}.nuspec")
352
381
  if os.path.isfile(nuspec_file):
353
382
  self._protected_sc.format_xml_file(nuspec_file)
@@ -393,6 +422,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
393
422
  hints: str = "\n".join(result1[2])
394
423
  raise ValueError(f"'{csproj_file}' with content '{GeneralUtilities.read_text_from_file(csproj_file)}' does not match the standardized .csproj-file-format which is defined by the regex '{result1[1]}'.\n{hints}")
395
424
  self.__check_csproj_urls(csproj_file)
425
+ self.__verify_that_all_package_references_are_pinned(csproj_file)
396
426
 
397
427
  test_csproj_project_name = csproj_project_name+"Tests"
398
428
  test_csproj_file = os.path.join(codeunit_folder, test_csproj_project_name, test_csproj_project_name+".csproj")
@@ -401,6 +431,35 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
401
431
  hints: str = "\n".join(result2[2])
402
432
  raise ValueError(f"'{test_csproj_file}' with content '{GeneralUtilities.read_text_from_file(test_csproj_file)}' does not match the standardized .csproj-file-format which is defined by the regex '{result2[1]}'.\n{hints}")
403
433
  self.__check_csproj_urls(test_csproj_file)
434
+ self.__verify_that_all_package_references_are_pinned(test_csproj_file)
435
+
436
+ @GeneralUtilities.check_arguments
437
+ def __verify_that_all_package_references_are_pinned(self, csproj_file: str) -> None:
438
+ """Ensures that every package-reference of the given project is pinned to exactly one version.
439
+
440
+ The version-attribute of a package-reference is a version-*range*, not a version: "1.2.3" means "at least 1.2.3".
441
+ NuGet resolves such a range to the lowest version which satisfies it - or, when exactly that version is not available
442
+ on the feed (unlisted, removed, incomplete mirror), silently to the next higher one. "[1.2.3]" means exactly 1.2.3,
443
+ so a version which is not available fails loudly instead of becoming a different version than the one which is
444
+ written in the project-file. This keeps the restore deterministic and makes the version which is deployed readable
445
+ from the source-code."""
446
+ root: etree._ElementTree = etree.parse(csproj_file)
447
+ not_pinned: list[str] = []
448
+ for package_reference in root.xpath("//PackageReference"):
449
+ #the attributes are read case-insensitively because msbuild reads them that way: a "version"-attribute is as
450
+ #effective as a "Version"-attribute, so a case-sensitive check would report a version which exists as missing.
451
+ attributes: dict[str, str] = {name.lower(): value for name, value in package_reference.attrib.items()}
452
+ package_name: str = attributes.get("include") or attributes.get("update") or "(package-reference without a name)"
453
+ if 0 < len(package_reference.xpath("*[translate(local-name(),'VERSION','version')='version']")):
454
+ not_pinned.append(f"{package_name}: the version is set as child-element; it must be set as attribute in the form Version=\"[<version>]\".")
455
+ continue
456
+ version: str = attributes.get("version")
457
+ if version is None:
458
+ not_pinned.append(f"{package_name}: no version is set at all, so the restore decides which version is used.")
459
+ elif self.__pinned_version_regex.match(version.strip()) is None:
460
+ not_pinned.append(f"{package_name}: the version \"{version}\" is a version-range, not a pinned version. Pin it in the form Version=\"[<version>]\".")
461
+ if 0 < len(not_pinned):
462
+ raise ValueError(f"The following package-reference(s) in \"{csproj_file}\" are not pinned to exactly one version:\n" + "\n".join(f" {entry}" for entry in not_pinned))
404
463
 
405
464
  @GeneralUtilities.check_arguments
406
465
  def __check_csproj_urls(self, csproj_file: str) -> None:
@@ -437,6 +496,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
437
496
  <IsPackable>false<\\/IsPackable>
438
497
  <PreserveCompilationContext>false<\\/PreserveCompilationContext>
439
498
  <GenerateRuntimeConfigurationFiles>true<\\/GenerateRuntimeConfigurationFiles>
499
+ <RestorePackagesWithLockFile>true<\\/RestorePackagesWithLockFile>
440
500
  <Copyright>([^<]+)<\\/Copyright>
441
501
  <Description>{codeunit_description_regex}<\\/Description>
442
502
  <PackageProjectUrl>https:\\/\\/([^<]+)<\\/PackageProjectUrl>
@@ -502,6 +562,7 @@ class TFCPS_CodeUnitSpecific_DotNet_Functions(TFCPS_CodeUnitSpecific_Base):
502
562
  <IsPackable>false<\\/IsPackable>
503
563
  <PreserveCompilationContext>false<\\/PreserveCompilationContext>
504
564
  <GenerateRuntimeConfigurationFiles>true<\\/GenerateRuntimeConfigurationFiles>
565
+ <RestorePackagesWithLockFile>true<\\/RestorePackagesWithLockFile>
505
566
  <Copyright>([^<]+)<\\/Copyright>
506
567
  <Description>{codeunit_name_regex}Tests is the test-project for {codeunit_name_regex}\\.<\\/Description>
507
568
  <PackageProjectUrl>https:\\/\\/([^<]+)<\\/PackageProjectUrl>
@@ -86,7 +86,7 @@ class TFCPS_CodeUnitSpecific_Flutter_Functions(TFCPS_CodeUnitSpecific_Base):
86
86
  @GeneralUtilities.check_arguments
87
87
  def linting(self) -> None:
88
88
  codeunit_folder = self.get_codeunit_folder()
89
- self._protected_sc.normalize_line_endings_of_files_in_folder(codeunit_folder, ["java","dart", "kt", "html"])
89
+ self._protected_sc.normalize_invisible_characters_of_files_in_folder(codeunit_folder, ["java","dart", "kt", "html"])
90
90
  #TODO call the flutter-linting here when available
91
91
 
92
92
  @GeneralUtilities.check_arguments
@@ -1,72 +1,153 @@
1
1
  import os
2
- from lxml import etree
3
- from ...GeneralUtilities import GeneralUtilities
4
- from ...ScriptCollectionCore import ScriptCollectionCore
5
- from ...SCLog import LogLevel
6
- from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base,TFCPS_CodeUnitSpecific_Base_CLI
2
+ import re
3
+ import requests
4
+ from ...GeneralUtilities import GeneralUtilities, Dependency
5
+ from ...SCLog import LogLevel
6
+ from ..TFCPS_CodeUnitSpecific_Base import TFCPS_CodeUnitSpecific_Base, TFCPS_CodeUnitSpecific_Base_CLI
7
7
 
8
8
  class TFCPS_CodeUnitSpecific_Go_Functions(TFCPS_CodeUnitSpecific_Base):
9
+ """Implements the tasks of a codeunit which contains a go-module.
9
10
 
10
- def __init__(self,current_file:str,verbosity:LogLevel,targetenvironmenttype:str,use_cache:bool,is_pre_merge:bool):
11
- super().__init__(current_file, verbosity,targetenvironmenttype,use_cache,is_pre_merge)
11
+ The go-module is expected in "<codeunit>/<codeunit>" and it contains its testcases in the same module,
12
+ because that is the layout go defines: the testcases of a file live in a "_test.go"-file next to it."""
13
+
14
+ def __init__(self, current_file: str, verbosity: LogLevel, targetenvironmenttype: str, use_cache: bool, is_pre_merge: bool):
15
+ super().__init__(current_file, verbosity, targetenvironmenttype, use_cache, is_pre_merge)
16
+
17
+ @GeneralUtilities.check_arguments
18
+ def get_go_module_folder(self) -> str:
19
+ """Returns the folder which contains the go.mod-file of this codeunit."""
20
+ return os.path.join(self.get_codeunit_folder(), self.get_codeunit_name())
21
+
22
+ @GeneralUtilities.check_arguments
23
+ def get_go_module_file(self) -> str:
24
+ return os.path.join(self.get_go_module_folder(), "go.mod")
25
+
26
+ @GeneralUtilities.check_arguments
27
+ def get_go_module_name(self) -> str:
28
+ """Returns the name of the go-module as it is declared in the go.mod-file. The paths in the
29
+ coverage-report of go are prefixed with it, so it is required to resolve them."""
30
+ for line in GeneralUtilities.read_lines_from_file(self.get_go_module_file()):
31
+ match = re.match(r"^module\s+(\S+)$", line.strip())
32
+ if match is not None:
33
+ return match.group(1)
34
+ raise ValueError(f'The go-module-file "{self.get_go_module_file()}" does not declare a module-name.')
12
35
 
13
36
  @GeneralUtilities.check_arguments
14
37
  def build(self) -> None:
15
- #TODO
16
- self.__generate_sbom_for_go_image()
38
+ codeunit_name = self.get_codeunit_name()
39
+ target_folder = os.path.join(self.get_artifacts_folder(), "BuildResult_Go")
40
+ GeneralUtilities.ensure_directory_exists(target_folder)
41
+ executable_name = codeunit_name+(".exe" if GeneralUtilities.current_system_is_windows() else GeneralUtilities.empty_string)
42
+ # The package of the module-root is built explicitly ("."). "./..." would build every package of the module,
43
+ # which go refuses when a single output-file is given.
44
+ self.__run_go(f"build -o {os.path.join(target_folder, executable_name)} .")
45
+ self.__generate_bom_for_go_module()
46
+ self.copy_source_files_to_output_directory()
17
47
 
18
48
  @GeneralUtilities.check_arguments
19
- def __generate_sbom_for_go_image(self) -> None:
20
- pass#TODO
49
+ def __generate_bom_for_go_module(self) -> None:
50
+ codeunit_version = self.tfcps_Tools_General.get_version_of_codeunit(self.get_codeunit_file())
51
+ bom_folder = os.path.join(self.get_artifacts_folder(), "BOM")
52
+ GeneralUtilities.ensure_directory_exists(bom_folder)
53
+ target_bom_file = os.path.join(bom_folder, f"{self.get_codeunit_name()}.{codeunit_version}.bom.xml")
54
+ self._protected_sc.run_program("cyclonedx-gomod", f"mod -licenses -output {target_bom_file} -output-version 1.5", self.get_go_module_folder(), print_live_output=self.get_verbosity() == LogLevel.Debug)
55
+ GeneralUtilities.assert_file_exists(target_bom_file)
56
+ self._protected_sc.format_xml_file(target_bom_file)
21
57
 
22
58
  @GeneralUtilities.check_arguments
23
59
  def linting(self) -> None:
24
- pass#TODO
60
+ self.__run_go("vet ./...")
61
+ # "gofmt -l" prints the files which are not formatted as gofmt would format them and exits with 0 either way,
62
+ # so the output has to be evaluated to recognize that there are linting-issues.
63
+ formatting_result = self._protected_sc.run_program("gofmt", "-l .", self.get_go_module_folder())
64
+ unformatted_files = [line.strip() for line in GeneralUtilities.string_to_lines(formatting_result[1]) if GeneralUtilities.string_has_content(line.strip())]
65
+ if 0 < len(unformatted_files):
66
+ for unformatted_file in unformatted_files:
67
+ self._protected_sc.log.log(f'File "{unformatted_file}" is not formatted as gofmt formats it.', LogLevel.Warning)
68
+ raise ValueError(f"{len(unformatted_files)} file(s) are not formatted as gofmt formats them.")
25
69
 
26
70
  @GeneralUtilities.check_arguments
27
- def __update_coverage_file(self,coverage_file:str) -> None:
28
- tree = etree.parse(coverage_file)
29
- root = tree.getroot()
30
- for package in root.findall(".//package"):
31
- package.set("name",self.get_codeunit_name())
32
- for cls in root.findall(".//class"):
33
- filename = cls.get("filename")
34
- if filename:
35
- cls.set("filename", f"./{filename}")
36
- cls.set("name",str(filename).rsplit("/", 1)[-1])
37
- tree.write(coverage_file, encoding="utf-8", xml_declaration=True, pretty_print=True)
71
+ def do_common_tasks(self, current_codeunit_version: str) -> None:
72
+ self.do_common_tasks_base(current_codeunit_version)
73
+ # A go-module does not declare its own version: the version of a go-module is the git-tag it is published with,
74
+ # so there is no version in the go.mod-file which could be set here.
75
+
76
+ @GeneralUtilities.check_arguments
77
+ def generate_reference(self) -> None:
78
+ self.generate_reference_using_docfx()
38
79
 
39
80
  @GeneralUtilities.check_arguments
40
81
  def run_testcases(self) -> None:
41
- test_coverage_folder = os.path.join(self.get_codeunit_folder(), "Other", "Artifacts", "TestCoverage").replace("\\", "/")
42
- GeneralUtilities.ensure_directory_exists(test_coverage_folder)
43
- src_folder = GeneralUtilities.resolve_relative_path(self.get_codeunit_name(), self.get_codeunit_folder())
44
- sc: ScriptCollectionCore = ScriptCollectionCore()
45
- sc.run_program_argsasarray("go", ["install", "github.com/t-yuki/gocover-cobertura@latest"], src_folder)
46
- sc.run_program_argsasarray("go", ["test", "-coverprofile=coverage.out", "./..."], src_folder)
47
- coverage_file:str=f"{test_coverage_folder}/TestCoverage.xml"
48
- sc.run_program_argsasarray("sh", ["-c", f"gocover-cobertura < coverage.out > {coverage_file}"], src_folder)
49
- self.__update_coverage_file(coverage_file)
50
- self.run_testcases_common_post_task(self.get_repository_folder(),self.get_codeunit_name(),True,self.get_type_environment_type())
51
-
52
- def get_dependencies(self)->dict[str,set[str]]:
53
- return dict[str,set[str]]()#TODO
54
-
82
+ codeunit_name = self.get_codeunit_name()
83
+ go_module_folder = self.get_go_module_folder()
84
+ coverprofile_file = os.path.join(GeneralUtilities.get_temp_folder(), f"{codeunit_name}.coverprofile")
85
+ GeneralUtilities.ensure_file_does_not_exist(coverprofile_file)
86
+ coverage_file = os.path.join(self.get_artifacts_folder(), "TestCoverage", "TestCoverage.xml")
87
+ GeneralUtilities.ensure_file_does_not_exist(coverage_file)
88
+ try:
89
+ # The coverprofile is written into the temp-folder and not into the module-folder, so that a file which is
90
+ # not git-ignored is not left behind in the codeunit when the testcases fail.
91
+ self.__run_go(f"test -coverprofile={coverprofile_file} ./...")
92
+ self.tfcps_Tools_General.convert_go_coverprofile_to_cobertura_report(coverprofile_file, coverage_file, codeunit_name, self.get_codeunit_folder(), self.get_go_module_name(), codeunit_name)
93
+ finally:
94
+ GeneralUtilities.ensure_file_does_not_exist(coverprofile_file)
95
+ self.run_testcases_common_post_task(self.get_repository_folder(), codeunit_name, True, self.get_target_environment_type())
96
+
55
97
  @GeneralUtilities.check_arguments
56
- def get_available_versions(self,dependencyname:str)->list[str]:
57
- return []#TODO
58
-
98
+ def __run_go(self, arguments: str) -> None:
99
+ self._protected_sc.run_program("go", arguments, self.get_go_module_folder(), print_live_output=self.get_verbosity() == LogLevel.Debug)
100
+
101
+ @GeneralUtilities.check_arguments
102
+ def get_dependencies(self) -> dict[str, set[str]]:
103
+ return GeneralUtilities.merge_dependency_lists([self.__get_dependencies_from_go_module_file()])
104
+
105
+ @GeneralUtilities.check_arguments
106
+ def __get_dependencies_from_go_module_file(self) -> list[Dependency]:
107
+ result: list[Dependency] = []
108
+ is_in_require_block: bool = False
109
+ for line in GeneralUtilities.read_lines_from_file(self.get_go_module_file()):
110
+ stripped_line = line.strip()
111
+ if stripped_line == "require (":
112
+ is_in_require_block = True
113
+ continue
114
+ if is_in_require_block and stripped_line == ")":
115
+ is_in_require_block = False
116
+ continue
117
+ if stripped_line.startswith("//") or not GeneralUtilities.string_has_content(stripped_line):
118
+ continue
119
+ # A dependency is declared either in a require-block or in a single require-line.
120
+ match = re.match(r"^(?:require\s+)?(\S+)\s+v(\d+\.\d+\.\d+)(?:\s+//.*)?$", stripped_line)
121
+ if match is not None and (is_in_require_block or stripped_line.startswith("require ")):
122
+ result.append(Dependency(match.group(1), match.group(2)))
123
+ return result
124
+
59
125
  @GeneralUtilities.check_arguments
60
- def set_dependency_version(self,name:str,new_version:str)->None:
61
- raise ValueError(f"Operation is not implemented.")
62
-
126
+ def get_available_versions(self, dependencyname: str) -> list[str]:
127
+ # The go-module-proxy answers with the published versions of a module, one version per line.
128
+ response = requests.get(f"https://proxy.golang.org/{dependencyname.lower()}/@v/list", timeout=60)
129
+ response.raise_for_status()
130
+ result: list[str] = []
131
+ for line in GeneralUtilities.string_to_lines(response.text):
132
+ match = re.match(r"^v(\d+\.\d+\.\d+)$", line.strip())
133
+ if match is not None:
134
+ result.append(match.group(1))
135
+ return result
136
+
137
+ @GeneralUtilities.check_arguments
138
+ def set_dependency_version(self, name: str, new_version: str) -> None:
139
+ # "go get" is used instead of rewriting the go.mod-file, because it also updates the go.sum-file, which
140
+ # contains the checksums of the dependencies and would make the build fail if it did not match the go.mod-file.
141
+ self.__run_go(f"get {name}@v{new_version}")
142
+ self.__run_go("mod tidy")
143
+
63
144
  class TFCPS_CodeUnitSpecific_Go_CLI:
64
145
 
65
146
  @staticmethod
66
147
  @GeneralUtilities.check_arguments
67
- def parse(file:str)->TFCPS_CodeUnitSpecific_Go_Functions:
68
- parser=TFCPS_CodeUnitSpecific_Base_CLI.get_base_parser()
148
+ def parse(file: str) -> TFCPS_CodeUnitSpecific_Go_Functions:
149
+ parser = TFCPS_CodeUnitSpecific_Base_CLI.get_base_parser()
69
150
  #add custom parameter if desired
70
- args=parser.parse_args()
71
- result:TFCPS_CodeUnitSpecific_Go_Functions=TFCPS_CodeUnitSpecific_Go_Functions(file,LogLevel(int(args.verbosity)),args.targetenvironmenttype,not args.nocache,args.ispremerge)
151
+ args = parser.parse_args()
152
+ result: TFCPS_CodeUnitSpecific_Go_Functions = TFCPS_CodeUnitSpecific_Go_Functions(file, LogLevel(int(args.verbosity)), args.targetenvironmenttype, not args.nocache, args.ispremerge)
72
153
  return result