qontract-reconcile 0.10.1rc868__py3-none-any.whl → 0.10.1rc870__py3-none-any.whl

{qontract_reconcile-0.10.1rc868.dist-info → qontract_reconcile-0.10.1rc870.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc868
+Version: 0.10.1rc870
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc868.dist-info → qontract_reconcile-0.10.1rc870.dist-info}/RECORD

@@ -39,11 +39,11 @@ reconcile/gitlab_permissions.py,sha256=1tFZws0-prcJ7vhFF3N7BxKocc16SjJuoqCkzRFES
 reconcile/gitlab_projects.py,sha256=K3tFf_aD1W4Ijp5q-9Qek3kwFGEWPcZ1kd7tzFJ4GyQ,1781
 reconcile/integrations_manager.py,sha256=J_VV-HINI7YNav2NPIolePZkll-7VBuBXWAyMNhsM_Q,9535
 reconcile/jenkins_base.py,sha256=0Gocu3fU2YTltaxBlbDQOUvP-7CP2OSQV1ZRwtWeVXw,875
-reconcile/jenkins_job_builder.py,sha256=8mcvvkPvcgw0hZnYTcuh9P-ltSHwqg8zihXh7oxGuzI,3957
+reconcile/jenkins_job_builder.py,sha256=XNwEkC688eAUQg8Yd69hZSByvSTmUaIXnZ-1R8LEF94,3521
 reconcile/jenkins_job_builds_cleaner.py,sha256=ksO5TXHxIMV_SF8kO86Wz7qGnYwbdt10wdhpb4aaEyQ,3851
 reconcile/jenkins_job_cleaner.py,sha256=dQGInds5RV-s9caec0212GveZ32xlCi2HiPyrIkVyFM,1761
 reconcile/jenkins_roles.py,sha256=f8ELpZY36UjoaCpR_9LijQuIMuB6a7sVLFf_H1ct9Hc,4460
-reconcile/jenkins_webhooks.py,sha256=j8vhJMWcRhOdc9XzRSm0CPj84jsF3e4Syjm7r1BIsDE,1978
+reconcile/jenkins_webhooks.py,sha256=K5h0OlCghoHlpot40IRq4BuezfKB1rj4Xk0dCUvqp3o,1952
 reconcile/jenkins_webhooks_cleaner.py,sha256=JsN_NVPfZJwv1JtSzZXDIHUqGiefL-DRffFnDGau9aY,1539
 reconcile/jenkins_worker_fleets.py,sha256=PMNGOX0krubFjInPiFT0za0KCiWBLEcVDuXdKRd1BrE,5378
 reconcile/jira_permissions_validator.py,sha256=iDsFdGqB8Zv9cjIVgYFq_N3xtRCrcR5uAZmcc51D-2o,13240
@@ -94,7 +94,7 @@ reconcile/quay_mirror.py,sha256=9NzbNoxl-NdD8CwImcXNG5xTdHmUJxBfeVk5XHH41J8,1488
 reconcile/quay_mirror_org.py,sha256=Oq-t3kSkgfeSAOUDjLCDRBeEvOIEBacfX38qrX_s0oc,10801
 reconcile/quay_permissions.py,sha256=9KOutS1w4RFQqkvMSy54VtsKNx56-phzP6yI_rEW-B8,4244
 reconcile/quay_repos.py,sha256=cuEYG0HUe0ut5yvLdEwOF5-CmccpXQHRb_wDazvDrvQ,6895
-reconcile/queries.py,sha256=dzg1HlbYYbV-UIpEuHVRpgm65q4B5zkF2EK4qhHWqrI,50839
+reconcile/queries.py,sha256=CAHgFEUMSLOX_fZcHja10D53VKHZdWSUL9sUgNsFmYo,51272
 reconcile/query_validator.py,sha256=BAjGrU8_VhzTOv5k0-uz0hY9ziZyconv8VAhgre1Auc,1497
 reconcile/requests_sender.py,sha256=914iluuF4UVgG3VyxxtnHOu4yf6YKS2fIy6PViSsFTQ,3875
 reconcile/resource_scraper.py,sha256=vo1N9vLJCYWvXlTwFRIpEuWjx_39ZV9zxJlpoPq4g3U,2330
@@ -476,7 +476,7 @@ reconcile/terraform_vpc_resources/integration.py,sha256=bqEq3qHaUVQkbGt100Fa7ZwN
 reconcile/terraform_vpc_resources/merge_request.py,sha256=loRymUigCIvaaT0s_NzktZchh-DGRQnCICdBSCAcFPY,1503
 reconcile/terraform_vpc_resources/merge_request_manager.py,sha256=Vj2nuQbQyrL4q_il1My-bLxYNh_r3YXqX45P8fwtP6Q,3259
 reconcile/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/test/conftest.py,sha256=0pO4UxFeBALKbL9gwemyap0VkbPR8n5TtZbf5c9pSv0,4303
+reconcile/test/conftest.py,sha256=0GXXH9DKIL9DlvY4tavLqXPaK8UXoC5tOfxkGfGy4Bs,4365
 reconcile/test/fixtures.py,sha256=9SDWAUlSd1rCx7z3GhULHcpr-I6FyCsXxaFAZIqYQsQ,591
 reconcile/test/test_acs_notifiers.py,sha256=xf3WL6q6V7KQdTVSx6YI-pa4yzOX3mkvIJomgPUc3Mw,12746
 reconcile/test/test_acs_policies.py,sha256=8pwnXpAO-0OI-6oubjf_oPPlpZjVldeZfJJ9uhsNMWM,17579
@@ -686,7 +686,7 @@ reconcile/utils/pagerduty_api.py,sha256=fcSAUez6w51woDvbm0plJW2qSw6_NXQs1Fit_KTN
 reconcile/utils/parse_dhms_duration.py,sha256=TONpLnec5gHeF7k815YNJpQyDjXhkxZIcv9s8ffbTSY,1840
 reconcile/utils/password_validator.py,sha256=XwuWg-8CPlcuG7dl_oQ1G1h2gSVSnfMym_VkuprpWVg,2183
 reconcile/utils/prometheus.py,sha256=i5aCQ_I4WOg76iEjglVxxO9-OKby2N80ScErAbDtHE8,3848
-reconcile/utils/promotion_state.py,sha256=Gf58uvBWqGDZtNsnGbi6-K4LC94Mm_bJDs_Ztq1FPSs,2603
+reconcile/utils/promotion_state.py,sha256=RswY0iMBw92ktyoorb-PFefaTAVlxyiKduVIoPyHpgU,3647
 reconcile/utils/promtool.py,sha256=kT2rFZSBaRqW7SSHAuYzGZzQxM5Dzk8KW1NnEUYZU_s,2896
 reconcile/utils/quay_api.py,sha256=EuOegpb-7ntEjkKLFwM2Oo4Nw7SyFtmyl3sQ9aXMtrM,8152
 reconcile/utils/raw_github_api.py,sha256=ZHC-SZuAyRe1zaMoOU7Krt1-zecDxENd9c_NzQYqK9g,2968
@@ -789,7 +789,7 @@ reconcile/utils/runtime/sharding.py,sha256=roCdbnBklhTK_g34zbgQYqzpKPaNQ8J6Xd9XL
 reconcile/utils/saasherder/__init__.py,sha256=J3MBZBFa5YmhqYm08QsjBXz8mFcVOCiOCkyIcw41t7E,343
 reconcile/utils/saasherder/interfaces.py,sha256=Tte-BAJ71FZF1J_ADay1UVIxLCJZcbefq4SRua4mn5w,9141
 reconcile/utils/saasherder/models.py,sha256=XiAb9pSmTxaNFa3XqNNfe1JxlGgTqsmd1nLi17iIV_g,5566
-reconcile/utils/saasherder/saasherder.py,sha256=4S3Q4tb11hmBI3rux5ajk-BUs-6Lfpg47uHZJj2uNKk,86787
+reconcile/utils/saasherder/saasherder.py,sha256=kPYklRU9hPcdg573QQ0OoaXYG26t_v5COAz62QVPS3c,86883
 reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
 reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
 reconcile/utils/terraform/config_client.py,sha256=py-Ree-QUYD6Hvng6bM40VgSuttteehIKNgwOSoJO1o,4706
@@ -809,7 +809,7 @@ tools/app_interface_metrics_exporter.py,sha256=zkwkxdAUAxjdc-pzx2_oJXG25fo0Fnyd5
 tools/app_interface_reporter.py,sha256=upA-J-n-HXHKVDINRuMR7vTt-iJvQORKUVi9D3leQto,17738
 tools/glitchtip_access_reporter.py,sha256=oPBnk_YoDuljU3v0FaChzOwwnk4vap1xEE67QEjzdqs,2948
 tools/glitchtip_access_revalidation.py,sha256=8kbBJk04mkq28kWoRDDkfCGIF3GRg3pJrFAh1sW0dbk,2821
-tools/qontract_cli.py,sha256=y6Okq0KPRKXx8glu4g3GeDavtoosH8iqwy5YZMV8cwc,118517
+tools/qontract_cli.py,sha256=GsODNjSs-1TP7U0JjHxYhF-9XWZe3cCk-rj7osApLr4,120445
 tools/sd_app_sre_alert_report.py,sha256=e9vAdyenUz2f5c8-z-5WY0wv-SJ9aePKDH2r4IwB6pc,5063
 tools/template_validation.py,sha256=-U-lTGeLaci8yWPEblCJeev2DOlY1jM9QOOh-O1zts8,3376
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -837,8 +837,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jrss,4941
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc868.dist-info/METADATA,sha256=2Ic4FAzZN1tZWidUXUc0YXjGK1fFGxGkQa2rdANjHZ8,2273
-qontract_reconcile-0.10.1rc868.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc868.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc868.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc868.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc870.dist-info/METADATA,sha256=MdcpkzNM0kwEg9ZL_cY9KKyDGzleE4-buVLHbDT-1pM,2273
+qontract_reconcile-0.10.1rc870.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc870.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc870.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc870.dist-info/RECORD,,

reconcile/jenkins_job_builder.py

@@ -7,7 +7,6 @@ from typing import (
 )
 
 from reconcile import queries
-from reconcile.utils import gql
 from reconcile.utils.defer import defer
 from reconcile.utils.jjb_client import JJB
 from reconcile.utils.secret_reader import (
@@ -16,46 +15,14 @@ from reconcile.utils.secret_reader import (
 )
 from reconcile.utils.state import init_state
 
-QUERY = """
-{
-  jenkins_configs: jenkins_configs_v1 {
-    name
-    app {
-      name
-    }
-    instance {
-      name
-      serverUrl
-      token {
-        path
-        field
-        version
-        format
-      }
-      deleteMethod
-    }
-    type
-    config
-    config_path {
-      content
-    }
-  }
-}
-"""
-
 QONTRACT_INTEGRATION = "jenkins-job-builder"
 GENERATE_TYPE = ["jobs", "views"]
 
 
-def get_jenkins_configs():
-    gqlapi = gql.get_api()
-    return gqlapi.query(QUERY)["jenkins_configs"]
-
-
 def collect_configs(
     instance_name: Optional[str], config_name: Optional[str]
 ) -> list[dict[str, Any]]:
-    configs = get_jenkins_configs()
+    configs = queries.get_jenkins_configs()
     if instance_name is not None:
         configs = [n for n in configs if n["instance"]["name"] == instance_name]
         if not configs:

reconcile/jenkins_webhooks.py

@@ -3,10 +3,7 @@ import logging
 from typing import Any
 
 from reconcile import queries
-from reconcile.jenkins_job_builder import (
-    get_jenkins_configs,
-    init_jjb,
-)
+from reconcile.jenkins_job_builder import init_jjb
 from reconcile.utils.defer import defer
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.jjb_client import JJB
@@ -62,5 +59,5 @@ def run(dry_run, defer=None):
 
 def early_exit_desired_state(*args, **kwargs) -> dict[str, Any]:
     return {
-        "jenkins_configs": get_jenkins_configs(),
+        "jenkins_configs": queries.get_jenkins_configs(),
     }

reconcile/queries.py

@@ -2793,3 +2793,36 @@ BLACKBOX_EXPORTER_MONITORING_PROVIDER = """
 def get_blackbox_exporter_monitoring_provider() -> dict:
     gqlapi = gql.get_api()
     return gqlapi.query(BLACKBOX_EXPORTER_MONITORING_PROVIDER)["providers"]
+
+
+JENKINS_CONFIGS = """
+{
+  jenkins_configs: jenkins_configs_v1 {
+    name
+    app {
+      name
+    }
+    instance {
+      name
+      serverUrl
+      token {
+        path
+        field
+        version
+        format
+      }
+      deleteMethod
+    }
+    type
+    config
+    config_path {
+      content
+    }
+  }
+}
+"""
+
+
+def get_jenkins_configs():
+    gqlapi = gql.get_api()
+    return gqlapi.query(JENKINS_CONFIGS)["jenkins_configs"]

reconcile/test/conftest.py

@@ -10,7 +10,7 @@ from typing import (
     Any,
     Optional,
 )
-from unittest.mock import create_autospec
+from unittest.mock import MagicMock, create_autospec
 
 import pytest
 from pydantic import BaseModel
@@ -74,7 +74,8 @@ def s3_state_builder() -> Callable[[Mapping], State]:
             return get(key)
 
         state = create_autospec(spec=State)
-        state.get = get
+        mock_get = MagicMock(side_effect=get)
+        state.get = mock_get
         state.__getitem__ = __getitem__
         state.ls.side_effect = [data.get("ls", [])]
         return state

reconcile/utils/promotion_state.py

@@ -34,11 +34,17 @@ class PromotionState:
     A wrapper around a reconcile.utils.state.State object.
     This is dedicated to storing and retrieving information
     about promotions on S3.
+
+    Note, that PromotionsState holds 2 caches.
+    One cache for the promotion data that has already been fetched.
+    Another cache for commit sha lookup, i.e., checking if a commit sha
+    exists in S3 before making any API calls to it.
     """
 
     def __init__(self, state: State):
         self._state = state
         self._commits_by_channel: dict[str, set[str]] = defaultdict(set)
+        self._promotion_data_cache: dict[str, PromotionData | None] = {}
 
     def _target_key(self, channel: str, target_uid: str) -> str:
         return f"{channel}/{target_uid}"
@@ -59,19 +65,36 @@ class PromotionState:
             self._commits_by_channel[key].add(commit_sha)
 
     def get_promotion_data(
-        self, sha: str, channel: str, target_uid: str = "", local_lookup: bool = True
+        self,
+        sha: str,
+        channel: str,
+        target_uid: str = "",
+        pre_check_sha_exists: bool = True,
+        use_cache: bool = False,
     ) -> Optional[PromotionData]:
+        """
+        Fetch promotion data from S3.
+
+        @param use_cache: Each fetched promotion data is cached locally. Setting this
+        flag to True will use the cache if the data is already fetched.
+
+        @param pre_check_sha_exists: If set to True, we will check if the commit sha exists
+        in local cache and if not will exit before making any API calls. Note, that this requires
+        a prior call to cache_commit_shas_from_s3 to populate the local commit cache.
+        """
         cache_key_v2 = self._target_key(channel=channel, target_uid=target_uid)
-        if local_lookup and sha not in self._commits_by_channel[cache_key_v2]:
+        if pre_check_sha_exists and sha not in self._commits_by_channel[cache_key_v2]:
             # Lets reduce unecessary calls to S3
             return None
 
         path_v2 = f"promotions_v2/{channel}/{target_uid}/{sha}"
-        try:
-            data = self._state.get(path_v2)
-            return PromotionData(**data)
-        except KeyError:
-            return None
+        if use_cache and path_v2 in self._promotion_data_cache:
+            return self._promotion_data_cache[path_v2]
+
+        data = self._state.get(path_v2)
+        promotion_data = PromotionData(**data)
+        self._promotion_data_cache[path_v2] = promotion_data
+        return promotion_data
 
     def publish_promotion_data(
         self, sha: str, channel: str, target_uid: str, data: PromotionData

reconcile/utils/saasherder/saasherder.py

@@ -1899,7 +1899,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                     sha=promotion.commit_sha,
                     channel=channel.name,
                     target_uid=target_uid,
-                    local_lookup=False,
+                    pre_check_sha_exists=False,
                 )
                 if not (deployment and deployment.success):
                     logging.error(
@@ -2007,6 +2007,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                             saas_file=promotion.saas_file,
                             success=success,
                             target_config_hash=promotion.target_config_hash,
+                            # TODO: do not override - check if timestamp already exists
                             check_in=str(now),
                         ),
                     )

tools/qontract_cli.py

@@ -2712,6 +2712,76 @@ def systems_and_tools(ctx):
     print_output(ctx.obj["options"], inventory.data, inventory.columns)
 
 
+@get.command
+@click.pass_context
+def jenkins_jobs(ctx):
+    jenkins_configs = queries.get_jenkins_configs()
+
+    # stats dicts
+    apps = {}
+    totals = {"rhel8": 0, "other": 0}
+
+    for jc in jenkins_configs:
+        app_name = jc["app"]["name"]
+
+        if app_name not in apps:
+            apps[app_name] = {"rhel8": 0, "other": 0}
+
+        config = json.loads(jc["config"]) if jc["config"] else []
+        for c in config:
+            if "project" not in c:
+                continue
+
+            project = c["project"]
+            root_node = project.get("node") or ""
+            if "jobs" not in project:
+                continue
+
+            for pj in project["jobs"]:
+                for job in pj.values():
+                    node = job["node"] if "node" in job else root_node
+                    if node == "rhel8":
+                        apps[app_name]["rhel8"] += 1
+                        totals["rhel8"] += 1
+                    else:
+                        apps[app_name]["other"] += 1
+                        totals["other"] += 1
+
+    results = [
+        {"app": app} | stats
+        for app, stats in sorted(apps.items(), key=lambda i: i[0].lower())
+        if not (stats["other"] == 0 and stats["rhel8"] == 0)
+    ]
+    results.append({"app": "TOTALS"} | totals)
+
+    if ctx.obj["options"]["output"] == "md":
+        json_table = {
+            "filter": True,
+            "fields": [
+                {"key": "app"},
+                {"key": "other"},
+                {"key": "rhel8"},
+            ],
+            "items": results,
+        }
+
+        print(
+            f"""
+You can view the source of this Markdown to extract the JSON data.
+
+{len(results)} apps with Jenkins jobs
+
+```json:table
+{json.dumps(json_table)}
+```
+            """
+        )
+    else:
+        columns = ["app", "other", "rhel8"]
+        ctx.obj["options"]["sort"] = False
+        print_output(ctx.obj["options"], results, columns)
+
+
 @root.group(name="set")
 @output
 @click.pass_context