ScriptCollection 4.2.113__py3-none-any.whl → 4.2.115__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,7 +5,6 @@ import binascii
5
5
  import filecmp
6
6
  import hashlib
7
7
  import multiprocessing
8
- import socket
9
8
  import time
10
9
  from io import BytesIO
11
10
  import itertools
@@ -39,7 +38,7 @@ from .ProgramRunnerBase import ProgramRunnerBase
39
38
  from .ProgramRunnerPopen import ProgramRunnerPopen
40
39
  from .SCLog import SCLog, LogLevel
41
40
 
42
- version = "4.2.113"
41
+ version = "4.2.115"
43
42
  __version__ = version
44
43
 
45
44
  class VSCodeWorkspaceShellTask:
@@ -3017,10 +3016,16 @@ OCR-content:
3017
3016
  if generated_ports_override_file != GeneralUtilities.empty_string:
3018
3017
  GeneralUtilities.ensure_file_does_not_exist(generated_ports_override_file)
3019
3018
  if in_build_container:
3020
- # Inside the build-container the test-process itself runs in a container. Instead of publishing the container-ports to the host it is
3021
- # attached to the docker-network(s) of the test-services so it can reach them directly by their container-name. This way no ports have
3022
- # to be opened on the host at all and concurrent pipelines can not conflict on the same host-port anymore.
3023
- self.__connect_self_to_compose_networks(docker_compose_file)
3019
+ # Inside the build-container the docker-socket points at a daemon on which the test-service-containers are created as SIBLING-containers
3020
+ # attached to the (external) compose-network. The build-container itself is not attached to that network, so without the following step it
3021
+ # could neither route to the container-IPs nor use the docker-DNS (which results in connection-timeouts even though the container-name can
3022
+ # be resolved via /etc/hosts). Therefore the build-container is attached to every (external) network the compose-file uses. No port is
3023
+ # published to the host, so concurrent pipelines on the same host do not conflict on host-ports.
3024
+ self.__connect_build_container_to_compose_networks(docker_compose_file)
3025
+ # In addition the container-names are mapped to their current container-IPs in the /etc/hosts of this build-container so that the
3026
+ # container-names can be used in the connection-strings even if the docker-DNS is not consulted. The managed block is rewritten on each
3027
+ # start because the containers get a new IP every time.
3028
+ self.__register_test_service_containers_in_etc_hosts(docker_compose_file)
3024
3029
 
3025
3030
  @GeneralUtilities.check_arguments
3026
3031
  def __generate_localhost_ports_override_file(self, docker_compose_file: str) -> str:
@@ -3041,25 +3046,84 @@ OCR-content:
3041
3046
  return override_file
3042
3047
 
3043
3048
  @GeneralUtilities.check_arguments
3044
- def get_external_networks_from_yaml_file(self, yaml_file: str) -> list[str]:
3045
- with open(yaml_file, encoding="utf-8") as stream:
3049
+ def __get_own_container_id(self) -> str:
3050
+ # Determines the id of the container this process currently runs in. The hostname usually equals the short container-id, but a runner can
3051
+ # override the hostname, so the id is primarily derived from the cgroup-/mountinfo-entries (docker mounts files like /etc/hostname from
3052
+ # /var/lib/docker/containers/<full-id>/... so the full 64-char container-id appears in these paths). The hostname is used as fallback only.
3053
+ full_container_id_pattern = re.compile(r"([0-9a-f]{64})")
3054
+ for proc_file in ["/proc/self/mountinfo", "/proc/self/cgroup"]:
3055
+ if not os.path.isfile(proc_file):
3056
+ continue
3057
+ try:
3058
+ content = GeneralUtilities.read_text_from_file(proc_file)
3059
+ except Exception:
3060
+ continue
3061
+ # Prefer lines which clearly reference a container-id-path to avoid matching unrelated 64-hex-strings.
3062
+ for line in content.splitlines():
3063
+ if "docker/containers/" in line or "/docker/" in line or "kubepods" in line:
3064
+ match = full_container_id_pattern.search(line)
3065
+ if match is not None:
3066
+ return match.group(1)
3067
+ match = full_container_id_pattern.search(content)
3068
+ if match is not None:
3069
+ return match.group(1)
3070
+ if os.path.isfile("/etc/hostname"):
3071
+ return GeneralUtilities.read_text_from_file_without_linebreak("/etc/hostname").strip()
3072
+ return GeneralUtilities.empty_string
3073
+
3074
+ @GeneralUtilities.check_arguments
3075
+ def __connect_build_container_to_compose_networks(self, docker_compose_file: str) -> None:
3076
+ # Attaches THIS build-container to every (external) network used by the compose-file so that the test-service-containers (which are
3077
+ # sibling-containers on the same docker-daemon) become routable and resolvable. The docker-socket is shared with the daemon, but the
3078
+ # network-namespace of the build-container is not, so without attaching there is no network-path to the container-IPs.
3079
+ own_container_id = self.__get_own_container_id()
3080
+ if not GeneralUtilities.string_has_content(own_container_id):
3081
+ return
3082
+ with open(docker_compose_file, encoding="utf-8") as stream:
3046
3083
  loaded = yaml.safe_load(stream)
3047
- result: list[str] = []
3048
- networks = loaded.get("networks", None)
3049
- if networks is None:
3050
- return result
3084
+ networks = loaded.get("networks", dict())
3051
3085
  for network_key, network_definition in networks.items():
3052
- if isinstance(network_definition, dict) and network_definition.get("external", False):
3053
- # For external networks the actual docker-network-name is the explicit "name" if given, otherwise the key itself.
3054
- result.append(network_definition.get("name", network_key))
3055
- return result
3056
-
3057
- @GeneralUtilities.check_arguments
3058
- def __connect_self_to_compose_networks(self, docker_compose_file: str) -> None:
3059
- own_container = socket.gethostname() # Inside a container the hostname equals the container-id, which "docker network connect" accepts.
3060
- for network_name in self.get_external_networks_from_yaml_file(docker_compose_file):
3061
- # Connecting an already-connected container returns a non-zero exitcode; this is ignored on purpose to keep this operation idempotent.
3062
- self.run_program("docker", f"network connect {network_name} {own_container}", throw_exception_if_exitcode_is_not_zero=False)
3086
+ network_definition = network_definition or dict()
3087
+ network_name = network_definition.get("name", network_key)
3088
+ inspect_result = self.run_program_argsasarray("docker", ["network", "inspect", "-f", "{{range $id, $c := .Containers}}{{$id}}\n{{end}}", network_name], throw_exception_if_exitcode_is_not_zero=False)
3089
+ if inspect_result[0] != 0:
3090
+ continue # The network does not exist (yet); nothing to attach to.
3091
+ # The keys of .Containers are full container-ids; the hostname (own_container_id) is the corresponding short-id (a prefix of it).
3092
+ connected_container_ids = [line.strip() for line in inspect_result[1].splitlines() if GeneralUtilities.string_has_content(line)]
3093
+ already_connected = any(connected_container_id.startswith(own_container_id) for connected_container_id in connected_container_ids)
3094
+ if not already_connected:
3095
+ self.run_program_argsasarray("docker", ["network", "connect", network_name, own_container_id], throw_exception_if_exitcode_is_not_zero=False)
3096
+
3097
+ @GeneralUtilities.check_arguments
3098
+ def __register_test_service_containers_in_etc_hosts(self, docker_compose_file: str) -> None:
3099
+ # Maps the container-names of the test-services to their current container-IPs in the /etc/hosts of THIS build-container (neither the runner-host
3100
+ # nor the test-service-containers are touched). The managed block is rewritten on each start because the containers get a new IP every time.
3101
+ hosts_file = "/etc/hosts"
3102
+ if not os.path.isfile(hosts_file):
3103
+ return
3104
+ with open(docker_compose_file, encoding="utf-8") as stream:
3105
+ loaded = yaml.safe_load(stream)
3106
+ services = loaded.get("services", dict())
3107
+ entries: list[str] = []
3108
+ for service_name, service_definition in services.items():
3109
+ container_name = service_definition.get("container_name", service_name)
3110
+ inspect_result = self.run_program_argsasarray("docker", ["inspect", "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}", container_name], throw_exception_if_exitcode_is_not_zero=False)
3111
+ if inspect_result[0] != 0:
3112
+ continue
3113
+ ip_addresses = [ip_address for ip_address in inspect_result[1].split() if GeneralUtilities.string_has_content(ip_address)]
3114
+ if 0 < len(ip_addresses):
3115
+ entries.append(f"{ip_addresses[0]}\t{container_name}")
3116
+ start_marker = "# >>> scriptcollection-test-service-hosts (managed automatically; do not edit)"
3117
+ end_marker = "# <<< scriptcollection-test-service-hosts"
3118
+ with open(hosts_file, "r", encoding="utf-8") as stream:
3119
+ content = stream.read()
3120
+ # Remove a previously written block first so stale IPs of already-recreated containers do not shadow the current ones.
3121
+ content = re.sub(re.escape(start_marker)+".*?"+re.escape(end_marker)+r"\n?", GeneralUtilities.empty_string, content, flags=re.DOTALL)
3122
+ content = content.rstrip("\n")+"\n"
3123
+ if 0 < len(entries):
3124
+ content = content+start_marker+"\n"+"\n".join(entries)+"\n"+end_marker+"\n"
3125
+ with open(hosts_file, "w", encoding="utf-8") as stream:
3126
+ stream.write(content)
3063
3127
 
3064
3128
  @GeneralUtilities.check_arguments
3065
3129
  def stop_local_test_service(self, file: str):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: scriptcollection
3
- Version: 4.2.113
3
+ Version: 4.2.115
4
4
  Summary: The ScriptCollection is the place for reusable scripts.
5
5
  Author-email: Marius Göcke <marius.goecke@gmail.com>
6
6
  Project-URL: Homepage, https://github.com/anionDev/ScriptCollection
@@ -15,7 +15,7 @@ ScriptCollection/ProgramRunnerMock.py,sha256=uTu-aFle1W_oKjeQEmuPsFPQpvo0kRf2FrR
15
15
  ScriptCollection/ProgramRunnerPopen.py,sha256=8tP0xWKeRYoGlaNB3iUh4o61O-tKtcxrS7IUqtU85C4,4091
16
16
  ScriptCollection/ProgramRunnerSudo.py,sha256=4lxbUFMIWMbthM5Q_QaFjvZfAsNpKlXeqfEbiofg_DI,5109
17
17
  ScriptCollection/SCLog.py,sha256=8TRy1LeYMsPOIuWUcnUNNbO5pd-cNBS-3cn-kdzP8FU,4768
18
- ScriptCollection/ScriptCollectionCore.py,sha256=_KX3pp5W8h25IqccC4LViEM6a_OglAqA7e4MeQ7Hj3w,193204
18
+ ScriptCollection/ScriptCollectionCore.py,sha256=dG2gsn8DMLy9DJg5QzFkcvmQbk4p8j7VCoRLiK6Gfx0,198356
19
19
  ScriptCollection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
20
20
  ScriptCollection/OCIImages/AbstractImageHandler.py,sha256=o129UYDtLymCRjMZBP9Puaj19ueMHoNt6UTLspfnaRc,1758
21
21
  ScriptCollection/OCIImages/OCIImageManager.py,sha256=2muW4hJ_a7EfavY3WVnafqZ0HC87RcZsoTuNvA3c0To,11175
@@ -291,8 +291,8 @@ build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Python/TFCPS_Code
291
291
  build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Python/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
292
292
  build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Rust/TFCPS_CodeUnitSpecific_Rust.py,sha256=WhOsMpJxNGqnQGA3nJ99ruyqymxBeJxCcpK98zJduTo,1680
293
293
  build/lib/build/lib/build/lib/build/lib/ScriptCollection/TFCPS/Rust/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
294
- scriptcollection-4.2.113.dist-info/METADATA,sha256=w4GginBlfMBrxZ6JL2KfvnHN7je22MQ0kvJwhKWGRZI,7697
295
- scriptcollection-4.2.113.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
296
- scriptcollection-4.2.113.dist-info/entry_points.txt,sha256=_vnVTrGJc_z-DTPLvaYyyOpDEQLz0bdO2OLN3_ZmeR8,4978
297
- scriptcollection-4.2.113.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
298
- scriptcollection-4.2.113.dist-info/RECORD,,
294
+ scriptcollection-4.2.115.dist-info/METADATA,sha256=gNAKE0IY-WDdqxXYcI12HIl4a2cANKJ2oFscoNUxzoU,7697
295
+ scriptcollection-4.2.115.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
296
+ scriptcollection-4.2.115.dist-info/entry_points.txt,sha256=_vnVTrGJc_z-DTPLvaYyyOpDEQLz0bdO2OLN3_ZmeR8,4978
297
+ scriptcollection-4.2.115.dist-info/top_level.txt,sha256=hY2hOVH0V0Ce51WB76zKkIWTUNwMUdHo4XDkR2vYVwg,17
298
+ scriptcollection-4.2.115.dist-info/RECORD,,