qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/external_resources/state.py

@@ -0,0 +1,258 @@
+import logging
+from collections.abc import Mapping
+from datetime import UTC, datetime
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel
+
+from reconcile.external_resources.model import (
+    ExternalResourceKey,
+    ExternalResourceModuleConfiguration,
+    Reconciliation,
+    ReconciliationStatus,
+    ResourceStatus,
+)
+from reconcile.utils.aws_api_typed.api import AWSApi
+
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+
+class StateNotFoundError(Exception):
+    pass
+
+
+class ReconcileStatus(StrEnum):
+    SUCCESS: str = "SUCCESS"
+    ERROR: str = "ERROR"
+    IN_PROGRESS: str = "IN_PROGRESS"
+    NOT_EXISTS: str = "NOT_EXISTS"
+
+
+class ExternalResourceState(BaseModel):
+    key: ExternalResourceKey
+    ts: datetime
+    resource_status: ResourceStatus
+    reconciliation: Reconciliation
+
+    def update_resource_status(
+        self, reconciliation_status: ReconciliationStatus
+    ) -> None:
+        if self.reconciliation_needs_state_update(reconciliation_status):
+            self.ts = datetime.now()
+            self.resource_status = reconciliation_status.resource_status
+
+    def reconciliation_needs_state_update(
+        self, reconciliation_status: ReconciliationStatus
+    ) -> bool:
+        return (
+            self.resource_status.is_in_progress or self.resource_status.does_not_exist
+        ) and not reconciliation_status.resource_status.is_in_progress
+
+
+class DynamoDBStateAdapter:
+    # Table PK
+    ER_KEY_HASH = "external_resource_key_hash"
+
+    RESOURCE_STATUS = "resource_status"
+    TIMESTAMP = "time_stamp"
+
+    ER_KEY = "external_resource_key"
+    ER_KEY_PROVISION_PROVIDER = "provision_provider"
+    ER_KEY_PROVISIONER_NAME = "provisioner_name"
+    ER_KEY_PROVIDER = "provider"
+    ER_KEY_IDENTIFIER = "identifier"
+
+    RECONC = "reconciliation"
+    RECONC_RESOURCE_HASH = "resource_hash"
+    RECONC_INPUT = "input"
+    RECONC_ACTION = "action"
+
+    MODCONF = "module_configuration"
+    MODCONF_IMAGE = "image"
+    MODCONF_VERSION = "version"
+    MODCONF_DRIFT_MINS = "drift_detection_minutes"
+    MODCONF_TIMEOUT_MINS = "timeout_minutes"
+
+    def _get_value(self, item: Mapping[str, Any], key: str, _type: str = "S") -> Any:
+        return item[key][_type]
+
+    def deserialize(
+        self,
+        item: Mapping[str, Any],
+        partial_data: bool = False,
+    ) -> ExternalResourceState:
+        _key = self._get_value(item, self.ER_KEY, _type="M")
+        key = ExternalResourceKey(
+            provision_provider=self._get_value(_key, self.ER_KEY_PROVISION_PROVIDER),
+            provisioner_name=self._get_value(_key, self.ER_KEY_PROVISIONER_NAME),
+            provider=self._get_value(_key, self.ER_KEY_PROVIDER),
+            identifier=self._get_value(_key, self.ER_KEY_IDENTIFIER),
+        )
+        _reconciliation = self._get_value(item, self.RECONC, _type="M")
+
+        if partial_data:
+            r = Reconciliation(
+                key=key,
+                resource_hash=self._get_value(
+                    _reconciliation, self.RECONC_RESOURCE_HASH
+                ),
+            )
+        else:
+            _modconf = self._get_value(_reconciliation, self.MODCONF, _type="M")
+            r = Reconciliation(
+                key=key,
+                resource_hash=self._get_value(
+                    _reconciliation, self.RECONC_RESOURCE_HASH
+                ),
+                input=self._get_value(_reconciliation, self.RECONC_INPUT),
+                action=self._get_value(_reconciliation, self.RECONC_ACTION),
+                module_configuration=ExternalResourceModuleConfiguration(
+                    image=self._get_value(_modconf, self.MODCONF_IMAGE),
+                    version=self._get_value(_modconf, self.MODCONF_VERSION),
+                    reconcile_drift_interval_minutes=self._get_value(
+                        _modconf, self.MODCONF_DRIFT_MINS, _type="N"
+                    ),
+                    reconcile_timeout_minutes=self._get_value(
+                        _modconf, self.MODCONF_TIMEOUT_MINS, _type="N"
+                    ),
+                ),
+            )
+
+        return ExternalResourceState(
+            key=key,
+            ts=self._get_value(item, self.TIMESTAMP),
+            resource_status=self._get_value(item, self.RESOURCE_STATUS),
+            reconciliation=r,
+        )
+
+    def serialize(self, state: ExternalResourceState) -> dict[str, Any]:
+        return {
+            self.ER_KEY_HASH: {"S": state.key.hash()},
+            self.TIMESTAMP: {"S": state.ts.isoformat()},
+            self.RESOURCE_STATUS: {"S": state.resource_status.value},
+            self.ER_KEY: {
+                "M": {
+                    self.ER_KEY_PROVISION_PROVIDER: {"S": state.key.provision_provider},
+                    self.ER_KEY_PROVISIONER_NAME: {"S": state.key.provisioner_name},
+                    self.ER_KEY_PROVIDER: {"S": state.key.provider},
+                    self.ER_KEY_IDENTIFIER: {"S": state.key.identifier},
+                }
+            },
+            self.RECONC: {
+                "M": {
+                    self.RECONC_RESOURCE_HASH: {
+                        "S": state.reconciliation.resource_hash
+                    },
+                    self.RECONC_ACTION: {"S": state.reconciliation.action.value},
+                    self.RECONC_INPUT: {"S": state.reconciliation.input},
+                    self.MODCONF: {
+                        "M": {
+                            self.MODCONF_IMAGE: {
+                                "S": state.reconciliation.module_configuration.image
+                            },
+                            self.MODCONF_VERSION: {
+                                "S": state.reconciliation.module_configuration.version
+                            },
+                            self.MODCONF_DRIFT_MINS: {
+                                "N": str(
+                                    state.reconciliation.module_configuration.reconcile_drift_interval_minutes
+                                )
+                            },
+                            self.MODCONF_TIMEOUT_MINS: {
+                                "N": str(
+                                    state.reconciliation.module_configuration.reconcile_timeout_minutes
+                                )
+                            },
+                        }
+                    },
+                }
+            },
+        }
+
+
+class ExternalResourcesStateDynamoDB:
+    PARTIALS_PROJECTED_VALUES = ",".join([
+        DynamoDBStateAdapter.ER_KEY,
+        DynamoDBStateAdapter.TIMESTAMP,
+        DynamoDBStateAdapter.RESOURCE_STATUS,
+        f"{DynamoDBStateAdapter.RECONC}.{DynamoDBStateAdapter.RECONC_RESOURCE_HASH}",
+    ])
+
+    def __init__(self, aws_api: AWSApi, table_name: str) -> None:
+        self.adapter = DynamoDBStateAdapter()
+        self.aws_api = aws_api
+        self._table = table_name
+        self.partial_resources = self._get_partial_resources()
+
+    def get_external_resource_state(
+        self, key: ExternalResourceKey
+    ) -> ExternalResourceState:
+        data = self.aws_api.dynamodb.boto3_client.get_item(
+            TableName=self._table,
+            ConsistentRead=True,
+            Key={self.adapter.ER_KEY_HASH: {"S": key.hash()}},
+        )
+        if "Item" in data:
+            return self.adapter.deserialize(data["Item"])
+        else:
+            return ExternalResourceState(
+                key=key,
+                ts=datetime.now(UTC),
+                resource_status=ResourceStatus.NOT_EXISTS,
+                reconciliation=Reconciliation(key=key),
+                reconciliation_errors=0,
+            )
+
+    def set_external_resource_state(
+        self,
+        state: ExternalResourceState,
+    ) -> None:
+        self.aws_api.dynamodb.boto3_client.put_item(
+            TableName=self._table, Item=self.adapter.serialize(state)
+        )
+
+    def del_external_resource_state(self, key: ExternalResourceKey) -> None:
+        self.aws_api.dynamodb.boto3_client.delete_item(
+            TableName=self._table,
+            Key={self.adapter.ER_KEY_HASH: {"S": key.hash()}},
+        )
+
+    def _get_partial_resources(
+        self,
+    ) -> dict[ExternalResourceKey, ExternalResourceState]:
+        """A Partial Resoure is the minimum resource data reguired
+        to check if a resource has been removed from the configuration.
+        Getting less data from DynamoDb saves money and the logic does not need it.
+        """
+        logging.debug("Getting Managed resources from DynamoDb")
+        partials = {}
+        for item in self.aws_api.dynamodb.boto3_client.scan(
+            TableName=self._table, ProjectionExpression=self.PARTIALS_PROJECTED_VALUES
+        ).get("Items", []):
+            s = self.adapter.deserialize(item, partial_data=True)
+            partials[s.key] = s
+        return partials
+
+    def get_all_resource_keys(self) -> set[ExternalResourceKey]:
+        return set(self.partial_resources)
+
+    def get_keys_by_status(
+        self, resource_status: ResourceStatus
+    ) -> set[ExternalResourceKey]:
+        return {
+            k
+            for k, v in self.partial_resources.items()
+            if v.resource_status == resource_status
+        }
+
+    def update_resource_status(
+        self, key: ExternalResourceKey, status: ResourceStatus
+    ) -> None:
+        self.aws_api.dynamodb.boto3_client.update_item(
+            TableName=self._table,
+            Key={self.adapter.ER_KEY_HASH: {"S": key.hash()}},
+            UpdateExpression="set resource_status=:new_value",
+            ExpressionAttributeValues={":new_value": {"S": status.value}},
+            ReturnValues="UPDATED_NEW",
+        )

reconcile/gabi_authorized_users.py

@@ -10,16 +10,14 @@ from datetime import (
     date,
     datetime,
 )
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 import reconcile.openshift_base as ob
 from reconcile import queries
 from reconcile.status import ExitCodes
 from reconcile.utils.aggregated_list import RunnerException
 from reconcile.utils.defer import defer
+from reconcile.utils.disabled_integrations import integration_is_enabled
 from reconcile.utils.external_resources import get_external_resource_specs
 from reconcile.utils.openshift_resource import (
     OpenshiftResource,
@@ -29,7 +27,7 @@ from reconcile.utils.semver_helper import make_semver
 
 QONTRACT_INTEGRATION = "gabi-authorized-users"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
-EXPIRATION_DAYS_MAX = 90
+EXPIRATION_DAYS_MAX = 365
 
 
 def construct_gabi_oc_resource(
@@ -76,6 +74,14 @@ def fetch_desired_state(
             namespace = i["namespace"]
             account = i["account"]
             identifier = i["identifier"]
+            cluster = namespace["cluster"]
+            if not integration_is_enabled(QONTRACT_INTEGRATION, cluster):
+                logging.debug(
+                    f"For cluster {cluster['name']} the integration "
+                    f"{QONTRACT_INTEGRATION} is not enabled. Skipping."
+                )
+                continue
+
             specs = get_external_resource_specs(namespace)
             found = False
             for spec in specs:
@@ -86,13 +92,14 @@ def fetch_desired_state(
                     break
             if not found:
                 raise RunnerException(
-                    f"Could not find RDS identifier {identifier} "
-                    f'for account {account} in namespace {namespace["name"]}'
+                    f'[gabi:{g["name"]} (path: {g["path"]})] Could not find RDS identifier {identifier} '
+                    f'for account {account} in namespace {namespace["name"]}. '
+                    "If this is a removed read only instance, consider updating the identifier to the source replica."
                 )
-            users = get_usernames(g["users"], namespace["cluster"])
+            users = get_usernames(g["users"], cluster)
             resource = construct_gabi_oc_resource(g["name"], expiration_date, users)
             ri.add_desired(
-                namespace["cluster"]["name"],
+                cluster["name"],
                 namespace["name"],
                 resource.kind,
                 resource.name,
@@ -104,9 +111,9 @@ def fetch_desired_state(
 def run(
     dry_run: bool,
     thread_pool_size: int = 10,
-    internal: Optional[bool] = None,
+    internal: bool | None = None,
     use_jump_host: bool = True,
-    defer: Optional[Callable] = None,
+    defer: Callable | None = None,
 ) -> None:
     gabi_instances = queries.get_gabi_instances()
     if not gabi_instances:
@@ -127,6 +134,7 @@ def run(
     if defer:
         defer(oc_map.cleanup)
     fetch_desired_state(gabi_instances, ri)
+    ob.publish_metrics(ri, QONTRACT_INTEGRATION)
     ob.realize_data(dry_run, oc_map, ri, thread_pool_size)
 
     if ri.has_error_registered():

reconcile/gcr_mirror.py

@@ -5,7 +5,9 @@ import re
 import tempfile
 import time
 from collections import defaultdict
+from typing import Any, Self
 
+import requests
 from sretoolbox.container import (
     Image,
     Skopeo,
@@ -20,10 +22,10 @@ from reconcile.utils.secret_reader import SecretReader
 _LOG = logging.getLogger(__name__)
 
 QONTRACT_INTEGRATION = "gcr-mirror"
+REQUEST_TIMEOUT = 60
 
 
 class QuayMirror:
-
     GCR_PROJECT_CATALOG_QUERY = """
     {
       projects: gcp_projects_v1 {
@@ -71,6 +73,13 @@ class QuayMirror:
         self.secret_reader = SecretReader(settings=settings)
         self.skopeo_cli = Skopeo(dry_run)
         self.push_creds = self._get_push_creds()
+        self.session = requests.Session()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        self.session.close()
 
     def run(self):
         sync_tasks = self.process_sync_tasks()
@@ -104,25 +113,19 @@ class QuayMirror:
                     if item["mirror"] is None:
                         continue
 
-                    summary[project].append(
-                        {
-                            "name": item["name"],
-                            "mirror": item["mirror"],
-                            "server_url": server_url,
-                        }
-                    )
+                    summary[project].append({
+                        "name": item["name"],
+                        "mirror": item["mirror"],
+                        "server_url": server_url,
+                    })
 
         return summary
 
     @staticmethod
     def sync_tag(tags, tags_exclude, candidate):
         if tags is not None:
-            for tag in tags:
-                if re.match(tag, candidate):
-                    return True
-            # When tags is defined, we don't look at
-            # tags_exclude
-            return False
+            # When tags is defined, we don't look at tags_exclude
+            return any(re.match(tag, candidate) for tag in tags)
 
         if tags_exclude is not None:
             for tag_exclude in tags_exclude:
@@ -143,7 +146,11 @@ class QuayMirror:
         sync_tasks = defaultdict(list)
         for org, data in summary.items():
             for item in data:
-                image = Image(f'{item["server_url"]}/{org}/{item["name"]}')
+                image = Image(
+                    f'{item["server_url"]}/{org}/{item["name"]}',
+                    session=self.session,
+                    timeout=REQUEST_TIMEOUT,
+                )
 
                 mirror_url = item["mirror"]["url"]
 
@@ -157,7 +164,13 @@ class QuayMirror:
                     password = raw_data["token"]
                     mirror_creds = f"{username}:{password}"
 
-                image_mirror = Image(mirror_url, username=username, password=password)
+                image_mirror = Image(
+                    mirror_url,
+                    username=username,
+                    password=password,
+                    session=self.session,
+                    timeout=REQUEST_TIMEOUT,
+                )
 
                 tags = item["mirror"].get("tags")
                 tags_exclude = item["mirror"].get("tagsExclude")
@@ -176,13 +189,11 @@ class QuayMirror:
                             downstream,
                             upstream,
                         )
-                        sync_tasks[org].append(
-                            {
-                                "mirror_url": str(upstream),
-                                "mirror_creds": mirror_creds,
-                                "image_url": str(downstream),
-                            }
-                        )
+                        sync_tasks[org].append({
+                            "mirror_url": str(upstream),
+                            "mirror_creds": mirror_creds,
+                            "image_url": str(downstream),
+                        })
                         continue
 
                     # Deep (slow) check only in non dry-run mode
@@ -214,13 +225,11 @@ class QuayMirror:
                     _LOG.debug(
                         "Image %s and mirror %s are out of sync", downstream, upstream
                     )
-                    sync_tasks[org].append(
-                        {
-                            "mirror_url": str(upstream),
-                            "mirror_creds": mirror_creds,
-                            "image_url": str(downstream),
-                        }
-                    )
+                    sync_tasks[org].append({
+                        "mirror_url": str(upstream),
+                        "mirror_creds": mirror_creds,
+                        "image_url": str(downstream),
+                    })
 
         return sync_tasks
 
@@ -228,7 +237,7 @@ class QuayMirror:
         control_file_name = "qontract-reconcile-gcr-mirror.timestamp"
         control_file_path = os.path.join(tempfile.gettempdir(), control_file_name)
         try:
-            with open(control_file_path, "r") as file_obj:
+            with open(control_file_path, encoding="locale") as file_obj:
                 last_deep_sync = float(file_obj.read())
         except FileNotFoundError:
             self._record_timestamp(control_file_path)
@@ -243,7 +252,7 @@ class QuayMirror:
 
     @staticmethod
     def _record_timestamp(path):
-        with open(path, "w") as file_object:
+        with open(path, "w", encoding="locale") as file_object:
             file_object.write(str(time.time()))
 
     def _get_push_creds(self):
@@ -263,5 +272,5 @@ class QuayMirror:
 
 
 def run(dry_run):
-    gcr_mirror = QuayMirror(dry_run)
-    gcr_mirror.run()
+    with QuayMirror(dry_run) as gcr_mirror:
+        gcr_mirror.run()

reconcile/github_org.py

@@ -128,7 +128,7 @@ def get_config(default=False):
 
 def get_default_config():
     github_config = get_config(default=True)
-    return list(github_config["github"].values())[0]
+    return next(iter(github_config["github"].values()))
 
 
 @retry()
@@ -198,7 +198,7 @@ def fetch_desired_state(infer_clusters=True):
     for role in roles:
         permissions = list(
             filter(
-                lambda p: p.get("service") in ["github-org", "github-org-team"],
+                lambda p: p.get("service") in {"github-org", "github-org-team"},
                 role["permissions"],
             )
         )
@@ -238,7 +238,7 @@ def fetch_desired_state(infer_clusters=True):
     )
     for cluster in clusters:
         for auth in cluster["auth"]:
-            if auth["service"] not in ["github-org", "github-org-team"]:
+            if auth["service"] not in {"github-org", "github-org-team"}:
                 continue
 
             cluster_name = cluster["name"]
@@ -438,9 +438,7 @@ def run(dry_run):
 
     assert (
         current_orgs == desired_orgs
-    ), "Current orgs ({}) don't match desired orgs ({})".format(
-        current_orgs, desired_orgs
-    )
+    ), f"Current orgs ({current_orgs}) don't match desired orgs ({desired_orgs})"
 
     # Calculate diff
     diff = current_state.diff(desired_state)

reconcile/github_owners.py

@@ -48,7 +48,7 @@ def fetch_desired_state():
         permissions = [
             p
             for p in role["permissions"]
-            if p.get("service") in ["github-org", "github-org-team"]
+            if p.get("service") in {"github-org", "github-org-team"}
             and p.get("role") == "owner"
         ]
         if not permissions:

reconcile/github_repo_invites.py

@@ -5,10 +5,7 @@ from collections.abc import (
     Mapping,
 )
 from dataclasses import dataclass
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 from reconcile import queries
 from reconcile.utils import (
@@ -27,7 +24,7 @@ class CodeComponents:
 
 
 def _parse_code_components(
-    raw: Optional[Iterable[Mapping[str, Any]]]
+    raw: Iterable[Mapping[str, Any]] | None,
 ) -> CodeComponents:
     urls = set()
     known_orgs = set()

reconcile/gitlab_fork_compliance.py

@@ -1,14 +1,16 @@
 import logging
 import sys
 
-from gitlab import (
-    MAINTAINER_ACCESS,
-    GitlabGetError,
-)
+from gitlab import GitlabGetError
+from gitlab.const import MAINTAINER_ACCESS
 
 from reconcile import queries
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.labels import BLOCKED_BOT_ACCESS
+from reconcile.utils.secret_reader import create_secret_reader
 
 LOG = logging.getLogger(__name__)
 
@@ -29,7 +31,6 @@ MSG_ACCESS = (
 
 
 class GitlabForkCompliance:
-
     OK = 0x0000
     ERR_MASTER_BRANCH = 0x0001
     ERR_NOT_A_MEMBER = 0x0002
@@ -41,10 +42,11 @@ class GitlabForkCompliance:
         self.maintainers_group = maintainers_group
 
         self.instance = queries.get_gitlab_instance()
-        self.settings = queries.get_app_interface_settings()
+        vault_settings = get_app_interface_vault_settings()
+        self.secret_reader = create_secret_reader(use_vault=vault_settings.vault)
 
         self.gl_cli = GitLabApi(
-            self.instance, project_id=project_id, settings=self.settings
+            self.instance, project_id=project_id, secret_reader=self.secret_reader
         )
         self.mr = self.gl_cli.get_merge_request(mr_id)
 
@@ -67,7 +69,7 @@ class GitlabForkCompliance:
         # who are not
         if self.maintainers_group:
             group = self.gl_cli.gl.groups.get(self.maintainers_group)
-            maintainers = group.members.list()
+            maintainers = group.members.list(iterator=True)
             project_maintainers = self.src.get_project_maintainers()
             for member in maintainers:
                 if member.username in project_maintainers:
@@ -79,9 +81,8 @@ class GitlabForkCompliance:
 
         # Last but not least, we remove the blocked label, in case
         # it is set
-        mr_labels = self.gl_cli.get_merge_request_labels(self.mr.iid)
-        if BLOCKED_BOT_ACCESS in mr_labels:
-            self.gl_cli.remove_label_from_merge_request(self.mr.iid, BLOCKED_BOT_ACCESS)
+        if BLOCKED_BOT_ACCESS in self.mr.labels:
+            self.gl_cli.remove_label(self.mr, BLOCKED_BOT_ACCESS)
 
         sys.exit(self.exit_code)
 
@@ -98,7 +99,7 @@ class GitlabForkCompliance:
             self.src = GitLabApi(
                 self.instance,
                 project_id=self.mr.source_project_id,
-                settings=self.settings,
+                secret_reader=self.secret_reader,
             )
         except GitlabGetError:
             self.handle_error("access denied for user {bot}", MSG_ACCESS)
@@ -114,7 +115,7 @@ class GitlabForkCompliance:
 
     def handle_error(self, log_msg, mr_msg):
         LOG.error([log_msg.format(bot=self.gl_cli.user.username)])
-        self.gl_cli.add_label_to_merge_request(self.mr.iid, BLOCKED_BOT_ACCESS)
+        self.gl_cli.add_label_to_merge_request(self.mr, BLOCKED_BOT_ACCESS)
         comment = mr_msg.format(
             user=self.mr.author["username"],
             bot=self.gl_cli.user.username,