checkmk-dev-tools 2.2.0__tar.gz → 2.4.0__tar.gz

{checkmk_dev_tools-2.2.0 → checkmk_dev_tools-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: checkmk-dev-tools
-Version: 2.2.0
+Version: 2.4.0
 Summary: Checkmk DevOps tools
 Author: Frans Fürst
 Author-email: frans.fuerst@checkmk.com
@@ -14,7 +14,7 @@ Requires-Dist: pydantic (>=2,<3)
 Requires-Dist: python-jenkins (>=1.8.3,<1.9.0)
 Requires-Dist: rich
 Requires-Dist: trickkiste (>=0.3.4,<0.4.0)
-Project-URL: Changelog, https://github.com/Checkmk/checkmk-dev-tools/blob/release/2.2.0/changelog.md
+Project-URL: Changelog, https://github.com/Checkmk/checkmk-dev-tools/blob/release/2.4.0/changelog.md
 Project-URL: Repository, https://github.com/Checkmk/checkmk-dev-tools
 Description-Content-Type: text/markdown
 

{checkmk_dev_tools-2.2.0 → checkmk_dev_tools-2.4.0}/cmk_dev/ci_artifacts.py

@@ -14,8 +14,10 @@ import asyncio
 import json
 import logging
 import os
+import shutil
 import sys
 import time
+import zipfile
 from argparse import ArgumentParser
 from argparse import Namespace as Args
 from collections.abc import Mapping, MutableMapping, Sequence
@@ -24,7 +26,7 @@ from datetime import datetime
 from itertools import chain
 from pathlib import Path
 from subprocess import check_output
-from typing import Any, List, Literal, cast
+from typing import Any, List, Literal, Set, TypedDict, cast
 
 import requests
 from influxdb_client import InfluxDBClient  # type: ignore[attr-defined]
@@ -56,7 +58,14 @@ from .version import __version__
 
 PathHashes = Mapping[str, str]
 shared_build_info: asyncio.Queue[str] = asyncio.Queue(maxsize=1)
+
+
+class FetchResult(TypedDict):
+    result: None | JobResult
+    artifacts: list[str]
+
 MAX_RETRY_ATTEMPTS = 3
+MAX_SINGLE_FILE_DOWNLOADS = 10  # max number of single files to be downloaded before using archive.zip + decompression
 
 
 def parse_args() -> Args:
@@ -311,13 +320,11 @@ def download_artifacts(
     total_download_timeout: int = 240,
     no_remove_others: bool = False,
     no_raise: bool = False,
-) -> tuple[Sequence[str | None], Sequence[str | None]]:
+) -> tuple[Sequence[str], Sequence[str]]:
     """Downloads all artifacts listed for given job/build to @out_dir"""
     # pylint: disable=protected-access
     # pylint: disable=too-many-locals
 
-    downloaded_artifacts, skipped_artifacts = [], []
-
     # https://bugs.launchpad.net/python-jenkins/+bug/1973243
     # https://bugs.launchpad.net/python-jenkins/+bug/2018576
 
@@ -358,6 +365,99 @@ def download_artifacts(
         p.relative_to(out_dir).as_posix() for p in out_dir.glob("**/*") if p.is_file()
     )
 
+    skipped_artifacts: List[str] = []
+    downloaded_artifacts: List[str] = []
+    if len(artifact_hashes) > MAX_SINGLE_FILE_DOWNLOADS:
+        downloaded_artifact = _download_compressed_artifacs(
+            client=client,
+            build=build,
+            existing_files=existing_files,
+            artifact_hashes=artifact_hashes,
+            out_dir=out_dir,
+            total_download_timeout=total_download_timeout,
+        )
+        downloaded_artifacts.append(downloaded_artifact)
+        downloaded_artifacts.extend(_decompress_artifacts(
+            artifact=out_dir / downloaded_artifact,
+            out_dir=out_dir,
+        ))
+    else:
+        downloaded_artifacts, skipped_artifacts, existing_files = _download_individual_artifacts(
+            client=client,
+            build=build,
+            existing_files=existing_files,
+            artifact_hashes=artifact_hashes,
+            out_dir=out_dir,
+            total_download_timeout=total_download_timeout,
+        )
+
+    if not no_remove_others:
+        for path in existing_files - set(downloaded_artifacts) - set(skipped_artifacts):
+            log().debug("Remove superfluous file %s", path)
+            with suppress(FileNotFoundError):
+                (out_dir / path).unlink()
+    log().info(
+        "%d artifacts available in '%s' (%d skipped, because they were up to date locally)",
+        len(downloaded_artifacts) + len(skipped_artifacts),
+        out_dir,
+        len(skipped_artifacts),
+    )
+
+    return downloaded_artifacts, skipped_artifacts
+
+def _download_compressed_artifacs(
+    client: Jenkins,
+    build: Build,
+    existing_files: Set[str],
+    artifact_hashes: Mapping[str, str],
+    out_dir: Path,
+    total_download_timeout: int = 240,
+) -> str:
+
+    # https://ci.lan.tribe29.com/job/checkmk/job/master/job/cv/job/test-gerrit-single-k8s/710903/artifact/*zip*/archive.zip
+    artifact = "archive.zip"
+    artifact_filename = out_dir / artifact
+
+    # ignore the returned value of the function as it would be like "/*zip*/archive.zip"
+    _download_single_file(
+        client=client,
+        build=build,
+        artifact=f"/*zip*/{artifact}",
+        artifact_filename=artifact_filename,
+        total_download_timeout=total_download_timeout,
+    )
+
+    return artifact
+
+def _decompress_artifacts(artifact: Path, out_dir: Path, decompressed_folder_name: str = "archive") -> List[str]:
+    with zipfile.ZipFile(artifact, "r") as zip_ref:
+        zip_ref.extractall(out_dir)
+
+    # The extracted folder is always named "archive"
+    archive_dir = out_dir / decompressed_folder_name
+
+    downloaded_artifacts = []
+
+    # Move all contents of the decompressed archive folder up one level
+    if archive_dir.exists() and archive_dir.is_dir():
+        for item in archive_dir.iterdir():
+            shutil.copytree(str(item), str(out_dir / item.name), dirs_exist_ok=True)
+            downloaded_artifacts += [str(f.relative_to(out_dir)) for f in (out_dir / item.name).rglob("*") if f.is_file()]
+
+        shutil.rmtree(archive_dir)
+
+    return downloaded_artifacts
+
+def _download_individual_artifacts(
+    client: Jenkins,
+    build: Build,
+    existing_files: Set[str],
+    artifact_hashes: Mapping[str, str],
+    out_dir: Path,
+    total_download_timeout: int = 240,
+) -> tuple[List[str], List[str], Set[str]]:
+    downloaded_artifacts, skipped_artifacts = [], []
+
     for artifact in build.artifacts:
         existing_files -= {artifact}
         fp_hash = artifact_hashes[artifact]
@@ -378,54 +478,62 @@ def download_artifacts(
                 fp_hash,
             )
 
-        for attempts_left in reversed(range(MAX_RETRY_ATTEMPTS + 1)):
-            time_start = time.time()
-            try:
-                with client._session.get(f"{build.url}artifact/{artifact}", stream=True) as reply:
-                    log().debug("download: %s", artifact)
-                    reply.raise_for_status()
-                    artifact_filename.parent.mkdir(parents=True, exist_ok=True)
-                    current_dl_duration = 0.0
-                    with open(artifact_filename, "wb") as out_file:
-                        for chunk in reply.iter_content(chunk_size=8192):
-                            if (
-                                current_dl_duration := (time.time() - time_start)
-                            ) > total_download_timeout:
-                                raise TimeoutError(
-                                    f"Downloading of {reply.url} took longer than {total_download_timeout}s"
-                                )
-                            if chunk:  # Filter out keep-alive chunks
-                                out_file.write(chunk)
-                    log().debug(
-                        "download: %s - successful (took %.2fs)", artifact, current_dl_duration
-                    )
-                    downloaded_artifacts.append(artifact)
-                break
-            except (
-                requests.exceptions.ChunkedEncodingError,
-                requests.exceptions.ConnectionError,
-                TimeoutError,
-            ) as exc:
-                if not attempts_left:
-                    raise
-                log().warning(
-                    "download_artifacts() caught %r (%s retries left)", exc, attempts_left
-                )
+        downloaded_artifacts.append(
+            _download_single_file(
+                client=client,
+                build=build,
+                artifact=artifact,
+                artifact_filename=artifact_filename,
+                total_download_timeout=total_download_timeout,
+            )
+        )
 
-    if not no_remove_others:
-        for path in existing_files - set(downloaded_artifacts) - set(skipped_artifacts):
-            log().debug("Remove superfluous file %s", path)
-            with suppress(FileNotFoundError):
-                (out_dir / path).unlink()
-    log().info(
-        "%d artifacts available in '%s' (%d skipped, because they were up to date locally)",
-        len(downloaded_artifacts) + len(skipped_artifacts),
-        out_dir,
-        len(skipped_artifacts),
-    )
+    return downloaded_artifacts, skipped_artifacts, existing_files
 
-    return downloaded_artifacts, skipped_artifacts
+def _download_single_file(
+    client: Jenkins,
+    build: Build,
+    artifact: str,
+    artifact_filename: Path,
+    total_download_timeout: int = 240,
+) -> str:
+    downloaded_artifact: str = ""
+
+    for attempts_left in reversed(range(MAX_RETRY_ATTEMPTS + 1)):
+        time_start = time.time()
+        try:
+            with client._session.get(f"{build.url}artifact/{artifact}", stream=True) as reply:
+                log().debug("download: %s", artifact)
+                reply.raise_for_status()
+                artifact_filename.parent.mkdir(parents=True, exist_ok=True)
+                current_dl_duration = 0.0
+                with open(artifact_filename, "wb") as out_file:
+                    for chunk in reply.iter_content(chunk_size=8192):
+                        if (
+                            current_dl_duration := (time.time() - time_start)
+                        ) > total_download_timeout:
+                            raise TimeoutError(
+                                f"Downloading of {reply.url} took longer than {total_download_timeout}s"
+                            )
+                        if chunk:  # Filter out keep-alive chunks
+                            out_file.write(chunk)
+                log().debug(
+                    "download: %s - successful (took %.2fs)", artifact, current_dl_duration
+                )
+                downloaded_artifact = artifact
+            break
+        except (
+            requests.exceptions.ChunkedEncodingError,
+            requests.exceptions.ConnectionError,
+            TimeoutError,
+        ) as exc:
+            if not attempts_left:
+                raise
+            log().warning(
+                "download_artifacts() caught %r (%s retries left)", exc, attempts_left
+            )
 
+    return downloaded_artifact
 
 def path_hashes_match(actual: PathHashes, required: PathHashes) -> bool:
     """Returns True if two given path hash mappings are semantically equal, i.e. at least one hash
@@ -733,6 +841,7 @@ async def _fn_request_build(args: Args) -> None:
                 path_hashes=compose_path_hashes(args.base_dir, args.dependency_paths),
                 time_constraints=args.time_constraints,
                 next_check_sleep=args.poll_queue_sleep,
+                ignore_build_queue=args.ignore_build_queue,
                 args=args,
             )
         ):
@@ -901,54 +1010,72 @@ def query_matching_builds(
         return []
 
 
-async def _fn_fetch(args: Args) -> None:
-    """Entry point for fetching (request and download combined) artifacts"""
-    out_dir = compose_out_dir(args.base_dir, args.out_dir)
-    path_hashes = compose_path_hashes(args.base_dir, args.dependency_paths)
+async def fetch_artifacts(
+    job: str,
+    params: None | JobParams = None,
+    *,
+    out_dir: Path = Path("out"),
+    base_dir: Path = Path("."),
+    params_no_check: None | JobParams = None,
+    dependency_paths: None | Sequence[str] = None,
+    time_constraints: None | str = None,
+    force_new_build: bool = False,
+    omit_new_build: bool = False,
+    download: bool = True,
+    no_remove_others: bool = False,
+    no_raise: bool = False,
+    poll_queue_sleep: int = 30,
+    poll_sleep: int = 60,
+    timeout: int = 120,
+    total_download_timeout: int = 240,
+    credentials: None | Mapping[str, str] = None,
+    credentials_file: str = "~/.config/jenkins_jobs/jenkins_jobs.ini",
+) -> FetchResult:
+    """Find or trigger a matching build, wait for it, download artifacts, return result dict."""
+    resolved_out_dir = compose_out_dir(base_dir, out_dir)
+    path_hashes = compose_path_hashes(base_dir, dependency_paths or [])
     async with AugmentedJenkinsClient(
-        **extract_credentials(args.credentials), timeout=args.timeout
+        **extract_credentials(credentials, credentials_file=credentials_file), timeout=timeout
     ) as jenkins_client:
-        if not (job := await jenkins_client.job_info(args.job)).type == "WorkflowJob":
-            raise Fatal(f"{args.job} is not a WorkflowJob")
-        # In case we force a new build anyway we don't have to look for an existing one
+        if not (job_info := await jenkins_client.job_info(job)).type == "WorkflowJob":
+            raise Fatal(f"{job} is not a WorkflowJob")
         matching_build = (
             None
-            if args.force_new_build
+            if force_new_build
             else await identify_matching_build(
-                job,
+                job_info,
                 jenkins_client=jenkins_client,
-                params=flatten(args.params),
-                path_hashes=compose_path_hashes(args.base_dir, args.dependency_paths),
-                time_constraints=args.time_constraints,
-                next_check_sleep=args.poll_queue_sleep,
-                args=args,
+                params=params,
+                path_hashes=path_hashes,
+                time_constraints=time_constraints,
+                next_check_sleep=poll_queue_sleep,
             )
         )
-        if args.omit_new_build and not matching_build:
-            raise Fatal(f"No matching build found for job '{job.name}' but new builds are omitted.")
-
+        if omit_new_build and not matching_build:
+            raise Fatal(
+                f"No matching build found for job '{job_info.name}' but new builds are omitted."
+            )
         build_candidate = matching_build or await trigger_build(
             jenkins_client=jenkins_client,
-            job=job,
+            job=job_info,
             params=compose_build_params(
-                params=flatten(args.params),
-                params_no_check=flatten(args.params_no_check),
-                path_hashes=compose_path_hashes(args.base_dir, args.dependency_paths),
+                params=params,
+                params_no_check=params_no_check,
+                path_hashes=path_hashes,
             ),
-            next_check_sleep=args.poll_queue_sleep,
+            next_check_sleep=poll_queue_sleep,
         )
-
         for key, value in build_candidate.__dict__.items():
             log().debug("  %s: %s", key, value)
-
         completed_build = await await_build(
-            job.path,
+            job_info.path,
             build_candidate.number,
             jenkins_client=jenkins_client,
             check_result=True,
             path_hashes=path_hashes,
-            next_check_sleep=args.poll_sleep,
-            no_raise=args.no_raise,
+            next_check_sleep=poll_sleep,
+            no_raise=no_raise,
+            download=download,
         )
         downloaded_artifacts = (
             list(
@@ -956,25 +1083,44 @@ async def _fn_fetch(args: Args) -> None:
                     *download_artifacts(
                         jenkins_client.client,
                         completed_build,
-                        out_dir,
-                        args.total_download_timeout,
-                        args.no_remove_others,
-                        args.no_raise,
+                        resolved_out_dir,
+                        total_download_timeout,
+                        no_remove_others,
+                        no_raise,
                     )
                 )
             )
-            if args.download
+            if download
             else []
         )
+        return {"result": completed_build.result, "artifacts": downloaded_artifacts}
 
-        print(
-            json.dumps(
-                {
-                    "result": completed_build.result,
-                    "artifacts": downloaded_artifacts,
-                }
+
+async def _fn_fetch(args: Args) -> None:
+    """Entry point for fetching (request and download combined) artifacts"""
+    print(
+        json.dumps(
+            await fetch_artifacts(
+                job=args.job,
+                params=flatten(args.params),
+                out_dir=args.out_dir,
+                base_dir=args.base_dir,
+                params_no_check=flatten(args.params_no_check),
+                dependency_paths=args.dependency_paths,
+                time_constraints=args.time_constraints,
+                force_new_build=args.force_new_build,
+                omit_new_build=args.omit_new_build,
+                download=args.download,
+                no_remove_others=args.no_remove_others,
+                no_raise=args.no_raise,
+                poll_queue_sleep=args.poll_queue_sleep,
+                poll_sleep=args.poll_sleep,
+                timeout=args.timeout,
+                total_download_timeout=args.total_download_timeout,
+                credentials=args.credentials,
             )
         )
+    )
 
 
 async def identify_matching_build(
@@ -985,6 +1131,7 @@ async def identify_matching_build(
     path_hashes: PathHashes,
     time_constraints: None | str,
     next_check_sleep: int = 30,
+    ignore_build_queue: bool = False,
     args: None | Args = None,
 ) -> None | Build:
     """Find an existing build (finished, still running or queued) which matches our
@@ -1102,7 +1249,7 @@ async def identify_matching_build(
         log().info("No matching builds found in the InfluxDB")
 
         log().info("Check for matching queue items via Jenkins API")
-        if hasattr(args, "ignore_build_queue") and not args.ignore_build_queue:
+        if not ignore_build_queue:
             if matching_item := await find_matching_queue_item(
                 jenkins_client=jenkins_client,
                 job=job,
@@ -1143,7 +1290,7 @@ async def identify_matching_build(
     log().info("No matching non finished builds found via the Jenkins API")
 
     log().info("Check for matching queue items via Jenkins API")
-    if args and not args.ignore_build_queue:
+    if not ignore_build_queue:
         if matching_item := await find_matching_queue_item(
             jenkins_client=jenkins_client,
             job=job,

{checkmk_dev_tools-2.2.0 → checkmk_dev_tools-2.4.0}/cmk_dev/version.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
 # -*- coding: UTF-8 -*-
 
-__version_info__ = ("2", "2", "0")
+__version_info__ = ("2", "4", "0")
 __version__ = '.'.join(__version_info__)

{checkmk_dev_tools-2.2.0 → checkmk_dev_tools-2.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "checkmk-dev-tools"
-version = "2.2.0" # will-be-updated-automatically
+version = "2.4.0" # will-be-updated-automatically
 description = "Checkmk DevOps tools"
 authors = [
     "Frans Fürst <frans.fuerst@checkmk.com>",
@@ -14,7 +14,7 @@ exclude = ["cmk_dev/out"]
 
 [tool.poetry.urls]
 Repository = "https://github.com/Checkmk/checkmk-dev-tools"
-Changelog = "https://github.com/Checkmk/checkmk-dev-tools/blob/release/2.2.0/changelog.md"
+Changelog = "https://github.com/Checkmk/checkmk-dev-tools/blob/release/2.4.0/changelog.md"
 
 [tool.poetry.scripts]
 activity-from-fs = 'cmk_dev.activity_from_fs:main'