qontract-reconcile 0.10.1rc867__py3-none-any.whl → 0.10.1rc869__py3-none-any.whl

{qontract_reconcile-0.10.1rc867.dist-info → qontract_reconcile-0.10.1rc869.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc867
+Version: 0.10.1rc869
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc867.dist-info → qontract_reconcile-0.10.1rc869.dist-info}/RECORD

@@ -35,7 +35,7 @@ reconcile/gitlab_labeler.py,sha256=IxE1XM5o4rDOFuR4cM2yAHTy4Uzdg3Nyz2mp7b8Fx1g,4
 reconcile/gitlab_members.py,sha256=M6LwFOrwgvl1NNdOJa1mrQFUon-bEVv1AyhGeLed454,8443
 reconcile/gitlab_mr_sqs_consumer.py,sha256=O46mdziPgGOndbU-0_UJKJVUaiEoVzJPEgKm4_UvYoI,2571
 reconcile/gitlab_owners.py,sha256=sn9njaKOtqcvnhi2qtm-faAfAR4zNqflbSuusA9RUuI,13456
-reconcile/gitlab_permissions.py,sha256=YaLiMqrn9YOM5e7o2NJVg6vtILyBXTryTScA4RhTyPo,3279
+reconcile/gitlab_permissions.py,sha256=1tFZws0-prcJ7vhFF3N7BxKocc16SjJuoqCkzRFESbE,3276
 reconcile/gitlab_projects.py,sha256=K3tFf_aD1W4Ijp5q-9Qek3kwFGEWPcZ1kd7tzFJ4GyQ,1781
 reconcile/integrations_manager.py,sha256=J_VV-HINI7YNav2NPIolePZkll-7VBuBXWAyMNhsM_Q,9535
 reconcile/jenkins_base.py,sha256=0Gocu3fU2YTltaxBlbDQOUvP-7CP2OSQV1ZRwtWeVXw,875
@@ -476,7 +476,7 @@ reconcile/terraform_vpc_resources/integration.py,sha256=bqEq3qHaUVQkbGt100Fa7ZwN
 reconcile/terraform_vpc_resources/merge_request.py,sha256=loRymUigCIvaaT0s_NzktZchh-DGRQnCICdBSCAcFPY,1503
 reconcile/terraform_vpc_resources/merge_request_manager.py,sha256=Vj2nuQbQyrL4q_il1My-bLxYNh_r3YXqX45P8fwtP6Q,3259
 reconcile/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/test/conftest.py,sha256=0pO4UxFeBALKbL9gwemyap0VkbPR8n5TtZbf5c9pSv0,4303
+reconcile/test/conftest.py,sha256=0GXXH9DKIL9DlvY4tavLqXPaK8UXoC5tOfxkGfGy4Bs,4365
 reconcile/test/fixtures.py,sha256=9SDWAUlSd1rCx7z3GhULHcpr-I6FyCsXxaFAZIqYQsQ,591
 reconcile/test/test_acs_notifiers.py,sha256=xf3WL6q6V7KQdTVSx6YI-pa4yzOX3mkvIJomgPUc3Mw,12746
 reconcile/test/test_acs_policies.py,sha256=8pwnXpAO-0OI-6oubjf_oPPlpZjVldeZfJJ9uhsNMWM,17579
@@ -502,7 +502,7 @@ reconcile/test/test_github_repo_invites.py,sha256=UVaDlxSxi5iooyUbz8F11d7cvINHLK
 reconcile/test/test_gitlab_housekeeping.py,sha256=Sn5rERCp28sMiBx5vJaQ5yy80y37GouMClejmXocsT8,10068
 reconcile/test/test_gitlab_labeler.py,sha256=vFLUJXSIaCduj6wSqgw7Fg7FhlopaDnYI5SLzNHtLoY,4362
 reconcile/test/test_gitlab_members.py,sha256=kaCOA02eZSrSMkzHmaLwWW3LY6Af0ciLSEP4PlMLvOU,9949
-reconcile/test/test_gitlab_permissions.py,sha256=FAKT7UuNAjxmke90P2cA-924_CGZ7Kp3JLTb6rmTseM,1965
+reconcile/test/test_gitlab_permissions.py,sha256=5QFy6InIHYM7zph5dPsDPo2hobrI5-eSGRTOzHOhF74,2005
 reconcile/test/test_instrumented_wrappers.py,sha256=CZzhnQH0c4i7-Rxjg7-0dfFMvVPegLHL46z5NHOOCwo,608
 reconcile/test/test_integrations_manager.py,sha256=l6KwSFT0NS9VSR-b_9z_ZEGXDWH3EMitUEMC_1h8Xkk,38184
 reconcile/test/test_jenkins_worker_fleets.py,sha256=o1jlT7OBBSgu0M3iI4xMdz_x6SciF7yhNBpLk5gTJfg,2361
@@ -654,7 +654,7 @@ reconcile/utils/filtering.py,sha256=zZnHH0u0SaTDyzuFXZ_mREURGLvjEqQIQy4z-7QBVlc,
 reconcile/utils/git.py,sha256=BdxXFgQ1XOZpS-4qb3qMsKTCFDG8MlE26rv1jAhvCkM,1560
 reconcile/utils/git_secrets.py,sha256=0wGNL5mvDtVPRuu3vEQgld1Am64gIDJHtmu1_ZKxMAI,1973
 reconcile/utils/github_api.py,sha256=_bttNxYKeam_tLVe27L7O4gKqSn6CeyuFnJn8tSaUVY,2488
-reconcile/utils/gitlab_api.py,sha256=kGYNyx6pbF43rd6R3a3wcIGmATiqNUFJUUHP4sELgNI,29027
+reconcile/utils/gitlab_api.py,sha256=thEdQ4AX96R4F3XhpOpugWmbueIgzDc_9b56DW-6ecY,29116
 reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
 reconcile/utils/gql.py,sha256=o8TLe9zW_zf_vOYC4yAy7mP5UvSR4eq3JKa4rxy1KjI,14277
 reconcile/utils/grouping.py,sha256=kWKivD14eAkiDneH_VIl_XyUdcVVQgiaKA9sLsuD2dw,441
@@ -686,7 +686,7 @@ reconcile/utils/pagerduty_api.py,sha256=fcSAUez6w51woDvbm0plJW2qSw6_NXQs1Fit_KTN
 reconcile/utils/parse_dhms_duration.py,sha256=TONpLnec5gHeF7k815YNJpQyDjXhkxZIcv9s8ffbTSY,1840
 reconcile/utils/password_validator.py,sha256=XwuWg-8CPlcuG7dl_oQ1G1h2gSVSnfMym_VkuprpWVg,2183
 reconcile/utils/prometheus.py,sha256=i5aCQ_I4WOg76iEjglVxxO9-OKby2N80ScErAbDtHE8,3848
-reconcile/utils/promotion_state.py,sha256=Gf58uvBWqGDZtNsnGbi6-K4LC94Mm_bJDs_Ztq1FPSs,2603
+reconcile/utils/promotion_state.py,sha256=RswY0iMBw92ktyoorb-PFefaTAVlxyiKduVIoPyHpgU,3647
 reconcile/utils/promtool.py,sha256=kT2rFZSBaRqW7SSHAuYzGZzQxM5Dzk8KW1NnEUYZU_s,2896
 reconcile/utils/quay_api.py,sha256=EuOegpb-7ntEjkKLFwM2Oo4Nw7SyFtmyl3sQ9aXMtrM,8152
 reconcile/utils/raw_github_api.py,sha256=ZHC-SZuAyRe1zaMoOU7Krt1-zecDxENd9c_NzQYqK9g,2968
@@ -789,7 +789,7 @@ reconcile/utils/runtime/sharding.py,sha256=roCdbnBklhTK_g34zbgQYqzpKPaNQ8J6Xd9XL
 reconcile/utils/saasherder/__init__.py,sha256=J3MBZBFa5YmhqYm08QsjBXz8mFcVOCiOCkyIcw41t7E,343
 reconcile/utils/saasherder/interfaces.py,sha256=Tte-BAJ71FZF1J_ADay1UVIxLCJZcbefq4SRua4mn5w,9141
 reconcile/utils/saasherder/models.py,sha256=XiAb9pSmTxaNFa3XqNNfe1JxlGgTqsmd1nLi17iIV_g,5566
-reconcile/utils/saasherder/saasherder.py,sha256=4S3Q4tb11hmBI3rux5ajk-BUs-6Lfpg47uHZJj2uNKk,86787
+reconcile/utils/saasherder/saasherder.py,sha256=kPYklRU9hPcdg573QQ0OoaXYG26t_v5COAz62QVPS3c,86883
 reconcile/utils/terraform/__init__.py,sha256=zNbiyTWo35AT1sFTElL2j_AA0jJ_yWE_bfFn-nD2xik,250
 reconcile/utils/terraform/config.py,sha256=5UVrd563TMcvi4ooa5JvWVDW1I3bIWg484u79evfV_8,164
 reconcile/utils/terraform/config_client.py,sha256=py-Ree-QUYD6Hvng6bM40VgSuttteehIKNgwOSoJO1o,4706
@@ -837,8 +837,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jrss,4941
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc867.dist-info/METADATA,sha256=kW9F6ZoBWCOD-GG7iiqXdukYen14M_t-HJ5FeYmfSxk,2273
-qontract_reconcile-0.10.1rc867.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc867.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc867.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc867.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc869.dist-info/METADATA,sha256=geTr8-qfS2A6C1-lBsxjSlUZLA5SGrxsm8RgJkSCnsY,2273
+qontract_reconcile-0.10.1rc869.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc869.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc869.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc869.dist-info/RECORD,,

reconcile/gitlab_permissions.py

@@ -80,7 +80,7 @@ def share_project_with_group(gl: GitLabApi, repos: list[str], dry_run: bool) ->
     # get repos not owned by app-sre
     non_app_sre_projects = {repo for repo in repos if "/app-sre/" not in repo}
     group_id, shared_projects = gl.get_group_id_and_shared_projects(APP_SRE_GROUP_NAME)
-    shared_project_repos = {project["web_url"] for project in shared_projects}
+    shared_project_repos = {project.web_url for project in shared_projects}
     repos_to_share = non_app_sre_projects - shared_project_repos
     for repo in repos_to_share:
         gl.share_project_with_group(repo_url=repo, group_id=group_id, dry_run=dry_run)

reconcile/test/conftest.py

@@ -10,7 +10,7 @@ from typing import (
     Any,
     Optional,
 )
-from unittest.mock import create_autospec
+from unittest.mock import MagicMock, create_autospec
 
 import pytest
 from pydantic import BaseModel
@@ -74,7 +74,8 @@ def s3_state_builder() -> Callable[[Mapping], State]:
             return get(key)
 
         state = create_autospec(spec=State)
-        state.get = get
+        mock_get = MagicMock(side_effect=get)
+        state.get = mock_get
         state.__getitem__ = __getitem__
         state.ls.side_effect = [data.get("ls", [])]
         return state

reconcile/test/test_gitlab_permissions.py

@@ -1,7 +1,7 @@
 from unittest.mock import MagicMock, create_autospec
 
 import pytest
-from gitlab.v4.objects import CurrentUser, GroupMember
+from gitlab.v4.objects import CurrentUser, GroupMember, GroupProject
 from pytest_mock import MockerFixture
 
 from reconcile import gitlab_permissions
@@ -51,7 +51,7 @@ def test_run_share_with_group(
     ).return_value = True
     mocked_gl.get_group_id_and_shared_projects.return_value = (
         1234,
-        [{"web_url": "https://test.com"}],
+        [create_autospec(GroupProject, web_url="https://test.com")],
     )
     gitlab_permissions.run(False, thread_pool_size=1)
     mocked_gl.share_project_with_group.assert_called_once()

reconcile/utils/gitlab_api.py

@@ -29,6 +29,7 @@ from gitlab.const import (
 from gitlab.v4.objects import (
     CurrentUser,
     Group,
+    GroupProject,
     Project,
     ProjectIssue,
     ProjectMergeRequest,
@@ -286,13 +287,14 @@ class GitLabApi:  # pylint: disable=too-many-public-methods
 
     def get_group_id_and_shared_projects(
         self, group_name: str
-    ) -> tuple[int, list[dict]]:
+    ) -> tuple[int, list[GroupProject]]:
         gitlab_request.labels(integration=INTEGRATION_NAME).inc()
         group = self.gl.groups.get(group_name)
+        shared_projects = self.get_items(group.projects.list, all=True)
         return group.id, [
             project
-            for project in group.shared_projects
-            for shared_group in project["shared_with_groups"]
+            for project in shared_projects
+            for shared_group in project.shared_with_groups
             if shared_group["group_id"] == group.id
             and shared_group["group_access_level"] >= MAINTAINER_ACCESS
         ]

reconcile/utils/promotion_state.py

@@ -34,11 +34,17 @@ class PromotionState:
     A wrapper around a reconcile.utils.state.State object.
     This is dedicated to storing and retrieving information
     about promotions on S3.
+
+    Note, that PromotionsState holds 2 caches.
+    One cache for the promotion data that has already been fetched.
+    Another cache for commit sha lookup, i.e., checking if a commit sha
+    exists in S3 before making any API calls to it.
     """
 
     def __init__(self, state: State):
         self._state = state
         self._commits_by_channel: dict[str, set[str]] = defaultdict(set)
+        self._promotion_data_cache: dict[str, PromotionData | None] = {}
 
     def _target_key(self, channel: str, target_uid: str) -> str:
         return f"{channel}/{target_uid}"
@@ -59,19 +65,36 @@ class PromotionState:
             self._commits_by_channel[key].add(commit_sha)
 
     def get_promotion_data(
-        self, sha: str, channel: str, target_uid: str = "", local_lookup: bool = True
+        self,
+        sha: str,
+        channel: str,
+        target_uid: str = "",
+        pre_check_sha_exists: bool = True,
+        use_cache: bool = False,
     ) -> Optional[PromotionData]:
+        """
+        Fetch promotion data from S3.
+
+        @param use_cache: Each fetched promotion data is cached locally. Setting this
+        flag to True will use the cache if the data is already fetched.
+
+        @param pre_check_sha_exists: If set to True, we will check if the commit sha exists
+        in local cache and if not will exit before making any API calls. Note, that this requires
+        a prior call to cache_commit_shas_from_s3 to populate the local commit cache.
+        """
         cache_key_v2 = self._target_key(channel=channel, target_uid=target_uid)
-        if local_lookup and sha not in self._commits_by_channel[cache_key_v2]:
+        if pre_check_sha_exists and sha not in self._commits_by_channel[cache_key_v2]:
             # Lets reduce unecessary calls to S3
             return None
 
         path_v2 = f"promotions_v2/{channel}/{target_uid}/{sha}"
-        try:
-            data = self._state.get(path_v2)
-            return PromotionData(**data)
-        except KeyError:
-            return None
+        if use_cache and path_v2 in self._promotion_data_cache:
+            return self._promotion_data_cache[path_v2]
+
+        data = self._state.get(path_v2)
+        promotion_data = PromotionData(**data)
+        self._promotion_data_cache[path_v2] = promotion_data
+        return promotion_data
 
     def publish_promotion_data(
         self, sha: str, channel: str, target_uid: str, data: PromotionData

reconcile/utils/saasherder/saasherder.py

@@ -1899,7 +1899,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                     sha=promotion.commit_sha,
                     channel=channel.name,
                     target_uid=target_uid,
-                    local_lookup=False,
+                    pre_check_sha_exists=False,
                 )
                 if not (deployment and deployment.success):
                     logging.error(
@@ -2007,6 +2007,7 @@ class SaasHerder:  # pylint: disable=too-many-public-methods
                             saas_file=promotion.saas_file,
                             success=success,
                             target_config_hash=promotion.target_config_hash,
+                            # TODO: do not override - check if timestamp already exists
                             check_in=str(now),
                         ),
                     )