ScriptCollection 4.4.48__py3-none-any.whl → 4.4.49__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -40,7 +40,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
40
40
  from .ProgramRunnerPopen import ProgramRunnerPopen
41
41
  from .SCLog import SCLog, LogLevel
42
42
 
43
- version = "4.4.48"
43
+ version = "4.4.49"
44
44
  __version__ = version
45
45
 
46
46
  class VSCodeWorkspaceShellTask:
@@ -1251,6 +1251,8 @@ class ScriptCollectionCore:
1251
1251
  # keeps non-ascii paths unquoted. The resulting paths are relative to 'folder'.
1252
1252
  lines = GeneralUtilities.string_to_lines(self.run_program_argsasarray("git", ["-c", "core.quotePath=false", "ls-files", "--cached", "--others", "--exclude-standard"], folder, throw_exception_if_exitcode_is_not_zero=True)[1], False)
1253
1253
  result: list[str] = []
1254
+ # A set is used for the duplicate-check because a lookup in the result-list would make this quadratic, which is too slow for repositories with a lot of files.
1255
+ already_added: set[str] = set()
1254
1256
  for line in lines:
1255
1257
  if not GeneralUtilities.string_has_content(line):
1256
1258
  continue
@@ -1259,7 +1261,8 @@ class ScriptCollectionCore:
1259
1261
  continue
1260
1262
  if not os.path.isfile(absolute_path): # a tracked-but-deleted file would still be listed by "--cached"
1261
1263
  continue
1262
- if absolute_path not in result:
1264
+ if absolute_path not in already_added:
1265
+ already_added.add(absolute_path)
1263
1266
  result.append(absolute_path)
1264
1267
  return result
1265
1268
 
@@ -0,0 +1,80 @@
1
+ import os
2
+ import sys
3
+ import tempfile
4
+ from ..GeneralUtilities import GeneralUtilities
5
+ from ..SCLog import LogLevel
6
+ from ..ScriptCollectionCore import ScriptCollectionCore
7
+
8
+
9
+ class TFCPS_CodeSpellCheck:
10
+ """Searches for typos in a repository using codespell.
11
+
12
+ codespell does not evaluate ".gitignore", so when it searches a folder recursively it also checks git-ignored content like
13
+ "node_modules", "bin", "obj" or the artifacts-folders of the codeunits, which costs a lot of runtime and only produces findings
14
+ which can not be fixed. Because of that only the files which are not git-ignored are checked.
15
+ A repository can contain a lot of files (for example 100000) and the length of a command-line is limited (on Windows to 32767
16
+ characters), so the files are not passed as command-line-arguments but are written into a file-list which is passed to codespell
17
+ using "@<file-list>" (codespell reads additional arguments from this file, one argument per line). This way codespell is called only once."""
18
+
19
+ sc: ScriptCollectionCore = None
20
+
21
+ codespell_exitcode_for_found_typos: int = 65
22
+
23
+ def __init__(self, sc: ScriptCollectionCore):
24
+ self.sc = sc
25
+
26
+ @GeneralUtilities.check_arguments
27
+ def search_for_typos(self, repository_folder: str) -> list[str]:
28
+ """Returns the findings of codespell for all files of the repository which are not git-ignored.
29
+ codespell runs in the repository-folder, so an optional "<repository>/.codespellrc" is used automatically.
30
+ Folders (relative to the repository-root, one per line) which should be ignored additionally (for example because they contain
31
+ committed files which should not be checked) can be defined in the optional file "<repository>/.ScriptCollection/CodeSpellIgnore.txt"."""
32
+ files: list[str] = self.get_files_to_check(repository_folder)
33
+ if len(files) == 0:
34
+ return []
35
+ self.sc.log.log(f"Check {len(files)} file(s) using codespell...", LogLevel.Debug)
36
+ (file_descriptor, file_list) = tempfile.mkstemp(prefix="CodeSpellFileList_", suffix=".txt")
37
+ try:
38
+ # codespell reads the file-list using the filesystem-encoding, so it is written using the same encoding.
39
+ with os.fdopen(file_descriptor, "w", encoding=sys.getfilesystemencoding(), errors=sys.getfilesystemencodeerrors(), newline="\n") as file_list_stream:
40
+ file_list_stream.write("\n".join(files) + "\n")
41
+ # "--check-hidden" is required because codespell would skip hidden files (like ".gitignore") otherwise, even if they are passed explicitly.
42
+ arguments: list[str] = ["--check-hidden"] + self.get_skip_arguments(repository_folder) + [f"@{file_list}"]
43
+ (exit_code, stdout, stderr, _) = self.sc.run_program_argsasarray("codespell", arguments, repository_folder, throw_exception_if_exitcode_is_not_zero=False)
44
+ finally:
45
+ GeneralUtilities.ensure_file_does_not_exist(file_list)
46
+ if exit_code not in (0, TFCPS_CodeSpellCheck.codespell_exitcode_for_found_typos):
47
+ raise ValueError(f"codespell failed with exit-code {exit_code}: {stderr}")
48
+ return [line for line in GeneralUtilities.string_to_lines(stdout) if GeneralUtilities.string_has_content(line)]
49
+
50
+ @GeneralUtilities.check_arguments
51
+ def get_files_to_check(self, repository_folder: str) -> list[str]:
52
+ """Returns the files (relative to the repository-folder and with the prefix "./") which are not git-ignored.
53
+ Hidden files and files inside hidden folders (for example ".github") are included too. Only the content of ".git"-folders is always excluded.
54
+ The prefix "./" is used because the skip-patterns of codespell (for example in ".codespellrc") are usually written in this form.
55
+ Additionally it ensures that no entry of the file-list starts with "-" or "@", so that codespell does not interpret a file as option or as another file-list."""
56
+ result: list[str] = []
57
+ for absolute_path in self.sc.get_not_git_ignored_files_of_folder(repository_folder):
58
+ relative_path: str = os.path.relpath(absolute_path, repository_folder).replace("\\", "/")
59
+ if ".git" in relative_path.split("/"):
60
+ continue
61
+ result.append(f"./{relative_path}")
62
+ return result
63
+
64
+ @GeneralUtilities.check_arguments
65
+ def get_skip_arguments(self, repository_folder: str) -> list[str]:
66
+ """Returns the "--skip"-argument for codespell based on the optional file "<repository>/.ScriptCollection/CodeSpellIgnore.txt"."""
67
+ ignore_file = os.path.join(repository_folder, ".ScriptCollection", "CodeSpellIgnore.txt")
68
+ if not os.path.isfile(ignore_file):
69
+ return []
70
+ skip_patterns: list[str] = []
71
+ for line in GeneralUtilities.string_to_lines(GeneralUtilities.read_text_from_file(ignore_file)):
72
+ folder = line.strip().replace("\\", "/").strip("/")
73
+ if folder.startswith("./"):
74
+ folder = folder[2:]
75
+ if folder != "" and not folder.startswith("#"):
76
+ skip_patterns.append(f"./{folder}")
77
+ skip_patterns.append(f"./{folder}/*")
78
+ if len(skip_patterns) == 0:
79
+ return []
80
+ return ["--skip=" + ",".join(skip_patterns)]
@@ -9,6 +9,7 @@ from ..ScriptCollectionCore import ScriptCollectionCore
9
9
  from ..SCLog import LogLevel
10
10
  from .TFCPS_BuildCodeUnitsHook import TFCPS_BuildCodeUnitsHook
11
11
  from .TFCPS_CodeUnit_BuildCodeUnit import TFCPS_CodeUnit_BuildCodeUnit
12
+ from .TFCPS_CodeSpellCheck import TFCPS_CodeSpellCheck
12
13
  from .TFCPS_OCIImageSecretScan import TFCPS_OCIImageSecretScan
13
14
  from .TFCPS_Tools_General import TFCPS_Tools_General
14
15
 
@@ -244,29 +245,10 @@ class TFCPS_CodeUnit_BuildCodeUnits:
244
245
  @GeneralUtilities.check_arguments
245
246
  def __do_project_linting_using_codespell(self, repository_folder: str) -> None:
246
247
  """Searches for typos in the repository using codespell. Findings are only reported as warnings and do not let the build fail.
247
- codespell runs in the repository-folder, so an optional "<repository>/.codespellrc" is used automatically, for example to
248
- define words which are no typos in the context of the repository (option "ignore-words-list").
249
- Folders (relative to the repository-root, one per line) which should be ignored can be defined in the optional file "<repository>/.ScriptCollection/CodeSpellIgnore.txt"."""
248
+ Only files which are not git-ignored are checked. See TFCPS_CodeSpellCheck for details."""
250
249
  try:
251
250
  self.sc.log.log("Search for typos using codespell...")
252
- arguments: list[str] = []
253
- ignore_file = os.path.join(repository_folder, ".ScriptCollection", "CodeSpellIgnore.txt")
254
- if os.path.isfile(ignore_file):
255
- skip_patterns: list[str] = []
256
- for line in GeneralUtilities.string_to_lines(GeneralUtilities.read_text_from_file(ignore_file)):
257
- folder = line.strip().replace("\\", "/").strip("/")
258
- if folder.startswith("./"):
259
- folder = folder[2:]
260
- if folder != "" and not folder.startswith("#"):
261
- skip_patterns.append(f"./{folder}")
262
- skip_patterns.append(f"./{folder}/*")
263
- if 0 < len(skip_patterns):
264
- arguments.append("--skip=" + ",".join(skip_patterns))
265
- (exit_code, stdout, stderr, _) = self.sc.run_program_argsasarray("codespell", arguments, repository_folder, throw_exception_if_exitcode_is_not_zero=False)
266
- codespell_exitcode_for_found_typos: int = 65
267
- if exit_code not in (0, codespell_exitcode_for_found_typos):
268
- raise ValueError(f"codespell failed with exit-code {exit_code}: {stderr}")
269
- findings: list[str] = [line for line in GeneralUtilities.string_to_lines(stdout) if GeneralUtilities.string_has_content(line)]
251
+ findings: list[str] = TFCPS_CodeSpellCheck(self.sc).search_for_typos(repository_folder)
270
252
  for finding in findings:
271
253
  self.sc.log.log(f"Possible typo: {finding}", LogLevel.Warning)
272
254
  if 0 < len(findings):
@@ -1258,23 +1258,16 @@ class TFCPS_Tools_General:
1258
1258
  @GeneralUtilities.check_arguments
1259
1259
  def do_npm_install(self, package_json_folder: str, npm_force: bool,use_cache:bool) -> None:
1260
1260
  """Installs the npm-dependencies of the package.json in package_json_folder (using epew, which must be available).
1261
- Side-effects: the package-lock.json is updated and node_modules is recreated; a marker-file ".sc_installed_for_platform" is
1262
- written into node_modules. If use_cache is set and node_modules was installed for the current platform before, nothing is done.
1261
+ Side-effects: the package-lock.json is updated and node_modules is recreated; a marker-file ".sc_install_fingerprint" is
1262
+ written into node_modules. If use_cache is set and node_modules was installed before for the same fingerprint (see
1263
+ get_npm_install_fingerprint), nothing is done.
1263
1264
  npm_force passes "--force" to every npm-call, which also hides dependency-conflicts instead of reporting them."""
1264
1265
  target_folder:str=os.path.join(package_json_folder,"node_modules")
1265
- platform_marker_file:str=os.path.join(target_folder,".sc_installed_for_platform")
1266
- current_platform:str=f"{sys.platform}-{platform.machine().lower()}"
1267
- # node_modules holds platform-specific native binaries (esbuild, rollup, lightningcss, swc, ...) which
1268
- # get installed only for the current host-platform. When node_modules is cached/shared between build-hosts
1269
- # of different platforms (e.g. a Windows-native build via 'scbuildcodeunits' followed by a build inside a
1270
- # mounted Linux-container via 'scbuildcodeunitsc') the cached node_modules contains the binaries for the
1271
- # foreign platform and lacks the ones for the current platform, which breaks the build. Therefore the cache
1272
- # is only reused when it was installed for exactly the current platform; otherwise a fresh install is forced
1273
- # so 'npm clean-install' rebuilds node_modules with the binaries matching the current platform.
1266
+ fingerprint_marker_file:str=os.path.join(target_folder,".sc_install_fingerprint")
1274
1267
  cache_is_reusable:bool=os.path.isdir(target_folder) \
1275
1268
  and not GeneralUtilities.folder_is_empty(target_folder) \
1276
- and os.path.isfile(platform_marker_file) \
1277
- and GeneralUtilities.read_text_from_file(platform_marker_file).strip()==current_platform
1269
+ and os.path.isfile(fingerprint_marker_file) \
1270
+ and GeneralUtilities.read_text_from_file(fingerprint_marker_file).strip()==self.get_npm_install_fingerprint(package_json_folder)
1278
1271
  update:bool=not (use_cache and cache_is_reusable)
1279
1272
  if update:
1280
1273
  self.__sc.log.log("Do npm-install...")
@@ -1293,8 +1286,30 @@ class TFCPS_Tools_General:
1293
1286
  argument3 = f"{argument3} --force"
1294
1287
  self.__sc.run_with_epew("npm", argument3, package_json_folder)
1295
1288
 
1296
- # 'npm clean-install' recreates node_modules, so the platform-marker must be written afterwards.
1297
- GeneralUtilities.write_text_to_file(platform_marker_file, current_platform)
1289
+ # 'npm clean-install' recreates node_modules and the npm-calls above may rewrite package-lock.json,
1290
+ # so the fingerprint-marker must be calculated and written afterwards.
1291
+ GeneralUtilities.write_text_to_file(fingerprint_marker_file, self.get_npm_install_fingerprint(package_json_folder))
1292
+
1293
+ @GeneralUtilities.check_arguments
1294
+ def get_npm_install_fingerprint(self, package_json_folder: str) -> str:
1295
+ """Returns a value which identifies for which state node_modules in package_json_folder would be installed by do_npm_install.
1296
+ It consists of the current platform and the sha256-hashes of package.json and (if it exists) package-lock.json."""
1297
+ # node_modules holds platform-specific native binaries (esbuild, rollup, lightningcss, swc, ...) which
1298
+ # get installed only for the current host-platform. When node_modules is cached/shared between build-hosts
1299
+ # of different platforms (e.g. a Windows-native build via 'scbuildcodeunits' followed by a build inside a
1300
+ # mounted Linux-container via 'scbuildcodeunitsc') the cached node_modules contains the binaries for the
1301
+ # foreign platform and lacks the ones for the current platform, which breaks the build. Therefore the platform
1302
+ # is part of the fingerprint.
1303
+ # The dependency-files are part of the fingerprint too, because otherwise a cached node_modules would still be
1304
+ # reused after the dependencies were changed (e.g. by an update of a package-version), so the build would run
1305
+ # with the outdated packages of the previous install.
1306
+ current_platform:str=f"{sys.platform}-{platform.machine().lower()}"
1307
+ fingerprint_parts:list[str]=[current_platform]
1308
+ for dependency_file_name in ["package.json","package-lock.json"]:
1309
+ dependency_file:str=os.path.join(package_json_folder,dependency_file_name)
1310
+ if os.path.isfile(dependency_file):
1311
+ fingerprint_parts.append(f"{dependency_file_name}:{GeneralUtilities.get_sha256_of_file(dependency_file)}")
1312
+ return ";".join(fingerprint_parts)
1298
1313
 
1299
1314
  @staticmethod
1300
1315
  @GeneralUtilities.check_arguments
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.4.48
3
+ Version: 4.4.49
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  License-Expression: GPL-3.0-only
@@ -11,7 +11,7 @@ ScriptCollection/ProgramRunnerSudo.py,sha256=vwKq-MOcqsgiKhSf-MBUDBC_UyaU8evP3Ro
11
11
  ScriptCollection/RepositoryDiagramGenerator.py,sha256=K_u2NW_p4ozdWD50irJEkkcDVa70TcUJpZN63TeXSLw,7092
12
12
  ScriptCollection/RepositoryHistoryAnalyzer.py,sha256=u9141B9D-Kef-9uo3yR7BXKUMzIKLVsesnYk7H-cWm4,6157
13
13
  ScriptCollection/SCLog.py,sha256=Fjf8Pdi63cXHGMxL_aI6A0dwGm7QIfrv3czjvpSUlaM,4705
14
- ScriptCollection/ScriptCollectionCore.py,sha256=jpq4G12DKlQ4S2aLsy-Ob2RmkwPGNAPMryzWlXU89Co,308791
14
+ ScriptCollection/ScriptCollectionCore.py,sha256=38vEcGi5cTq2rXmXhbdxbPXKU5xeYwWQoiBu96WfdFs,309054
15
15
  ScriptCollection/TestKind.py,sha256=jqLJwNvQeyea7SZf-UFzMC9pM0znv4do7FIC1xq8u9M,1672
16
16
  ScriptCollection/UnassignedVariablesCheck.py,sha256=8xNcp1D5HeOLO1a1NEerq2TsnpFhw7yzYPkaay-oisY,3804
17
17
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -36,9 +36,10 @@ ScriptCollection/TFCPS/PlantUMLRenderer.py,sha256=upnafiGKV-SdRI8iRBkbbWHN2u6v9K
36
36
  ScriptCollection/TFCPS/PubDependencies.py,sha256=hl6BfYjzb0-esO-pqyV_sxBLbHBi55GEgUiub3lma_8,9922
37
37
  ScriptCollection/TFCPS/SCTaskRunnerServer.py,sha256=tub8_1QBu0nWAaGnBWp2nErOW-SOHf_UL6qjOSGbnxU,31026
38
38
  ScriptCollection/TFCPS/TFCPS_BuildCodeUnitsHook.py,sha256=67_DQwoH5bhxxSmwtuAeoW1NOCMiLMzOy48OqlJX-KY,1744
39
+ ScriptCollection/TFCPS/TFCPS_CodeSpellCheck.py,sha256=zV62O7UiGjewx1yQUIWmUJ9D-5ipkV_f28uNhp-JWUs,5306
39
40
  ScriptCollection/TFCPS/TFCPS_CodeUnitSpecific_Base.py,sha256=yosafTfbB0LBMKomZvLnFJpgJpJOMHsGKLirVjG1slk,40501
40
41
  ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnit.py,sha256=Mj9t1SyRpkYZE74cLsGWYv4xB89gXL2ld-RX3xvvGdw,9835
41
- ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=wdhKr_0oUN-cHxFf1FtbIzqJ5_tQRzh7FLYiAFjRZOQ,57822
42
+ ScriptCollection/TFCPS/TFCPS_CodeUnit_BuildCodeUnits.py,sha256=SOuLgq2hnz78Hy2rQF2xH6oDShnqLUitiviblh579w8,56321
42
43
  ScriptCollection/TFCPS/TFCPS_CreateRelease.py,sha256=BVK3fpEyjcXYVHIb6YYLH7bWe6iKRaCtPrJ3tXAemYw,6582
43
44
  ScriptCollection/TFCPS/TFCPS_Generic.py,sha256=mRBcf7XuNWOSXbZYc3NuJow18AMZ30D6Z-BtyNYfNcM,2072
44
45
  ScriptCollection/TFCPS/TFCPS_MergeToMain.py,sha256=Krt5oyGNcRAN3szsgU7LoTh8yVTOdHW-Kq7R9MDDV-Q,7673
@@ -46,7 +47,7 @@ ScriptCollection/TFCPS/TFCPS_MergeToStable.py,sha256=oYEhz5ZWMnbu97_CcYEqdDxakUF
46
47
  ScriptCollection/TFCPS/TFCPS_OCIImageSecretScan.py,sha256=Jb7AaMcJDW-gYEYBUVZAG3QLTTNWmXlvVEW-lZsykws,14821
47
48
  ScriptCollection/TFCPS/TFCPS_PreBuildCodeunitsScript.py,sha256=XLVsFTA1jvfDbZ7wYJgIdEByZhF_bYVbA5jkisuvK4k,2322
48
49
  ScriptCollection/TFCPS/TFCPS_RemoteBuild.py,sha256=2Tx5HLk1ZftBWDCTiGZdP2QiiGNyWsEuTCSt33dNQjc,23891
49
- ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=ueObphFdSZ4cU4Siv8XOdKDEuRWV4GrzD-vJ1qfASXg,156148
50
+ ScriptCollection/TFCPS/TFCPS_Tools_General.py,sha256=5dvRL-QWxLuGbT68qLMQMmnhPNoVbDIU7pAQpPAt9Oo,157277
50
51
  ScriptCollection/TFCPS/TFCPS_VisualRegressionTests.py,sha256=6_63udxHnPlu_TMk_ftLZTnzriJO8MCxVCVAsweZqHs,37073
51
52
  ScriptCollection/TFCPS/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
52
53
  ScriptCollection/TFCPS/CPP/TFCPS_CodeUnitSpecific_CPP.py,sha256=BNZPMjGiWLMBH7QDtP06WEVLMnI38NLX4Dpd13bSbi4,16614
@@ -72,8 +73,8 @@ ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=yyjWPMHQjPLkML
72
73
  ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
73
74
  ScriptCollection/TFCPS/TypeScript/TFCPS_CodeUnitSpecific_TypeScript.py,sha256=ILVhJHvwG5TvPBBqSvyACYb7W9QnWrJpxKgHu8IQu4g,9249
74
75
  ScriptCollection/TFCPS/TypeScript/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
75
- scriptcollection-4.4.48.dist-info/METADATA,sha256=UTri6IXI2G1tMOkn1qHO_hBUC6Brxw2vTKd0crWAbPo,8032
76
- scriptcollection-4.4.48.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
77
- scriptcollection-4.4.48.dist-info/entry_points.txt,sha256=1nA5gUD0Y9L105JGIfTnnqDZhFjzKcFpFaUSrLn-Sw0,5991
78
- scriptcollection-4.4.48.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
79
- scriptcollection-4.4.48.dist-info/RECORD,,
76
+ scriptcollection-4.4.49.dist-info/METADATA,sha256=SoS0zLNqVIlCQGhB9oe61J3sqf9_7ll0NOByqmjrSQo,8032
77
+ scriptcollection-4.4.49.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
78
+ scriptcollection-4.4.49.dist-info/entry_points.txt,sha256=1nA5gUD0Y9L105JGIfTnnqDZhFjzKcFpFaUSrLn-Sw0,5991
79
+ scriptcollection-4.4.49.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
80
+ scriptcollection-4.4.49.dist-info/RECORD,,