qontract-reconcile 0.10.2.dev13__tar.gz → 0.10.2.dev15__tar.gz

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev13
+Version: 0.10.2.dev15
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/pyproject.toml

@@ -73,19 +73,18 @@ dev = [
     "anymarkup ~=0.8.1",
     # keep in sync project.dependencies.boto3
     "boto3-stubs[account,ec2,s3,rds,iam,route53,organizations,service-quotas,sts,support,dynamodb]==1.34.94",
-    "kubernetes ~=24.0",
     "kubernetes-stubs==22.6.0",
     "MarkupSafe ~=2.1.1",
     "moto ~=2.2",
     "mypy ~=1.13.0",
     "pyfakefs ~=5.1",
-    "pytest ~=7.2",
-    "pytest-cov ~=4.0.0",
+    "pytest ~=8.3",
+    "pytest-cov ~=6.0",
     "pytest-httpserver ~=1.0",
     "pytest-mock ~=3.6",
     "qenerate==0.7.0",
-    "responses ~=0.23.1",
-    "ruff ~=0.6.9",
+    "responses ~=0.25.0",
+    "ruff ~=0.9.0",
     "smtpdfix ~=0.3.3",
     "testslide ~=2.7",
     "typeguard ~=2.13.3",
@@ -184,7 +183,12 @@ extend-select = [
     "FA",
     # ruff special rules
     "RUF",
-
+    # flake8-type-checking
+    "TC",
+    # Perflint
+    "PERF",
+    # refurb
+    "FURB",
 ]
 ignore = [
     "PLR0904", # Too many public methods
@@ -204,6 +208,14 @@ ignore = [
     "B017",    #`pytest.raises(Exception)` should be considered evil
     "RUF005",  # Consider "...."instead of concatenation
     "RUF012",  # Mutable class attributes should be annotated with "typing.ClassVar"
+    "TC001",   # Type-checking block
+    "TC002",   # Type-checking block
+    "TC003",   # Type-checking block
+    "TC004",   # Type-checking block
+    "TC006",   # quoted type expressions in `typing.cast()`
+    # Review later
+    "FURB101", # Replace `open` with `pathlib.Path`
+    "FURB103", # Replace `open` with `pathlib.Path`
 ]
 [tool.ruff.format]
 preview = true
@@ -320,7 +332,6 @@ module = [
     "reconcile.utils.config",
     "reconcile.utils.data_structures",
     "reconcile.utils.defer",
-    "reconcile.utils.dnsutils",
     "reconcile.utils.elasticsearch_exceptions",
     "reconcile.utils.environ",
     "reconcile.utils.exceptions",

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/acs_rbac.py

@@ -1,6 +1,7 @@
 import logging
 from collections import defaultdict
 from collections.abc import Callable
+from itertools import starmap
 from typing import (
     Self,
 )
@@ -151,10 +152,7 @@ class AcsRbacIntegration(QontractReconcileIntegration[NoParams]):
                         permission_usernames[
                             Permission(**permission.dict(by_alias=True))
                         ].append(user.org_username)
-        return [
-            AcsRole.build(permission, usernames)
-            for permission, usernames in permission_usernames.items()
-        ]
+        return list(starmap(AcsRole.build, permission_usernames.items()))
 
     def get_current_state(
         self, auth_provider_id: str, rbac_api_resources: RbacResources

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/aus/base.py

@@ -585,20 +585,20 @@ def fetch_current_state(
             addon_upgrade_policies = addon_service.get_addon_upgrade_policies(
                 ocm_api, spec.cluster.id, addon_id=addon_spec.addon.addon.id
             )
-            for addon_upgrade_policy in addon_upgrade_policies:
-                current_state.append(
-                    AddonUpgradePolicy(
-                        id=addon_upgrade_policy.id,
-                        addon_id=addon_spec.addon.addon.id,
-                        cluster=spec.cluster,
-                        next_run=addon_upgrade_policy.next_run,
-                        schedule=addon_upgrade_policy.schedule,
-                        schedule_type=addon_upgrade_policy.schedule_type,
-                        version=addon_upgrade_policy.version,
-                        state=addon_upgrade_policy.state,
-                        addon_service=addon_service,
-                    )
+            current_state.extend(
+                AddonUpgradePolicy(
+                    id=addon_upgrade_policy.id,
+                    addon_id=addon_spec.addon.addon.id,
+                    cluster=spec.cluster,
+                    next_run=addon_upgrade_policy.next_run,
+                    schedule=addon_upgrade_policy.schedule,
+                    schedule_type=addon_upgrade_policy.schedule_type,
+                    version=addon_upgrade_policy.version,
+                    state=addon_upgrade_policy.state,
+                    addon_service=addon_service,
                 )
+                for addon_upgrade_policy in addon_upgrade_policies
+            )
         elif spec.cluster.is_rosa_hypershift():
             upgrade_policies = get_control_plane_upgrade_policies(
                 ocm_api, spec.cluster.id

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/aws_ami_share.py

@@ -19,8 +19,7 @@ def filter_accounts(accounts: Iterable[dict[str, Any]]) -> list[dict[str, Any]]:
         sharing = a.get("sharing")
         if sharing:
             sharing_account_names.add(a["name"])
-            for s in sharing:
-                sharing_account_names.add(s["account"]["name"])
+            sharing_account_names.update(s["account"]["name"] for s in sharing)
 
     return [a for a in accounts if a["name"] in sharing_account_names]
 

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/aws_cloudwatch_log_retention/integration.py

@@ -35,7 +35,7 @@ class AWSCloudwatchCleanupOption(BaseModel):
 
 
 DEFAULT_AWS_CLOUDWATCH_CLEANUP_OPTION = AWSCloudwatchCleanupOption(
-    regex=re.compile(".*"),
+    regex=re.compile(r".*"),
     retention_in_days=DEFAULT_RETENTION_IN_DAYS,
     delete_empty_log_group=False,
 )

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/aws_saml_idp/integration.py

@@ -177,7 +177,7 @@ class AwsSamlIdpIntegration(QontractReconcileIntegration[AwsSamlIdpIntegrationPa
                 integration_version=QONTRACT_INTEGRATION_VERSION,
                 dry_run=dry_run,
                 cache_source=ts.terraform_configurations(),
-                shard=self.params.account_name if self.params.account_name else "",
+                shard=self.params.account_name or "",
                 ttl_seconds=self.params.extended_early_exit_cache_ttl_seconds,
                 logger=logging.getLogger(),
                 runner=runner,

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/aws_saml_roles/integration.py

@@ -295,7 +295,7 @@ class AwsSamlRolesIntegration(
                 integration_version=QONTRACT_INTEGRATION_VERSION,
                 dry_run=dry_run,
                 cache_source=ts.terraform_configurations(),
-                shard=self.params.account_name if self.params.account_name else "",
+                shard=self.params.account_name or "",
                 ttl_seconds=self.params.extended_early_exit_cache_ttl_seconds,
                 logger=logging.getLogger(),
                 runner=runner,

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/aws_version_sync/integration.py

@@ -273,7 +273,7 @@ class AVSIntegration(QontractReconcileIntegration[AVSIntegrationParams]):
         supported_providers: Iterable[str],
     ) -> list[ExternalResource]:
         external_resources: list[ExternalResource] = []
-        _defaults_cache: dict[str, Any] = {}
+        defaults_cache: dict[str, Any] = {}
         for ns in namespaces:
             for external_resource in ns.external_resources or []:
                 if not isinstance(
@@ -297,12 +297,12 @@ class AVSIntegration(QontractReconcileIntegration[AVSIntegrationParams]):
                     values = {}
                     # get/set the defaults file values from/to cache
                     if resource.defaults:
-                        if not (values := _defaults_cache.get(resource.defaults, {})):
+                        if not (values := defaults_cache.get(resource.defaults, {})):
                             # retrieve the external resource spec values
                             values = get_values(
                                 gql_get_resource_func, resource.defaults
                             )
-                            _defaults_cache[resource.defaults] = values
+                            defaults_cache[resource.defaults] = values
                     values = override_values(values, resource.overrides)
                     if resource.provider.lower() == "elasticache" and str(
                         values["engine_version"]

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/change_owners/change_owners.py

@@ -215,8 +215,8 @@ def write_coverage_report_to_stdout(change_decisions: list[ChangeDecision]) -> N
                 "disabled": False,
             })
         if d.coverage:
-            for ctx in d.coverage:
-                results.append({
+            results.extend(
+                {
                     "file": d.file.path,
                     "schema": d.file.schema,
                     "changed path": d.diff.path,
@@ -228,7 +228,9 @@ def write_coverage_report_to_stdout(change_decisions: list[ChangeDecision]) -> N
                         for ar in ctx.approver_reachability or []
                     ]),
                     "disabled": str(ctx.disabled),
-                })
+                }
+                for ctx in d.coverage
+            )
         else:
             results.append({
                 "file": d.file.path,
@@ -453,8 +455,9 @@ def run(
                 if not co_label.startswith("change-owner/")
             }
             for bc in changes:
-                for label in bc.change_owner_labels:
-                    labels.add(change_owner_label(label))
+                labels.update(
+                    change_owner_label(label) for label in bc.change_owner_labels
+                )
 
             if mr_management_enabled:
                 gl.set_labels_on_merge_request(merge_request, labels)

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/change_owners/change_types.py

@@ -578,15 +578,15 @@ class ChangeTypeProcessor:
 
             # expand context based on change-type composition
             for ce in self._context_expansions:
-                for ec in ce.expand(change, expansion_trail_copy):
-                    # add expanded contexts (derived owned files)
-                    contexts.append(
-                        ResolvedContext(
-                            owned_file_ref=ec.owned_file_ref,
-                            context_file_ref=change.file_ref,
-                            change_type=ec.change_type,
-                        )
+                # add expanded contexts (derived owned files)
+                contexts.extend(
+                    ResolvedContext(
+                        owned_file_ref=ec.owned_file_ref,
+                        context_file_ref=change.file_ref,
+                        change_type=ec.change_type,
                     )
+                    for ec in ce.expand(change, expansion_trail_copy)
+                )
 
         # context detection
         # the context for approver extraction can be found within the changed
@@ -603,17 +603,17 @@ class ChangeTypeProcessor:
                         )
                     )
                     for ce in self._context_expansions:
-                        for ec in ce.expand_from_file_ref(
-                            ctx_file_ref, expansion_trail_copy
-                        ):
-                            # add expanded contexts (derived owned files)
-                            contexts.append(
-                                ResolvedContext(
-                                    owned_file_ref=ec.owned_file_ref,
-                                    context_file_ref=ctx_file_ref,
-                                    change_type=ec.change_type,
-                                )
+                        # add expanded contexts (derived owned files)
+                        contexts.extend(
+                            ResolvedContext(
+                                owned_file_ref=ec.owned_file_ref,
+                                context_file_ref=ctx_file_ref,
+                                change_type=ec.change_type,
+                            )
+                            for ec in ce.expand_from_file_ref(
+                                ctx_file_ref, expansion_trail_copy
                             )
+                        )
 
         return contexts
 

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/change_owners/changes.py

@@ -211,11 +211,11 @@ class BundleFileChange:
         returns all the change-types that are involved in the coverage
         of all changes
         """
-        change_types = []
-        for dc in self.diff_coverage:
-            for ctx in dc.coverage:
-                change_types.append(ctx.change_type_processor)
-        return change_types
+        return [
+            ctx.change_type_processor
+            for dc in self.diff_coverage
+            for ctx in dc.coverage
+        ]
 
 
 def parse_resource_file_content(content: Any | None) -> tuple[Any, str | None]:
@@ -308,10 +308,9 @@ def get_priority_for_changes(
     """
     Finds the lowest priority of all change types involved in the provided bundle file changes.
     """
-    priorities: set[ChangeTypePriority] = set()
-    for bfc in bundle_file_changes:
-        for ct in bfc.involved_change_types():
-            priorities.add(ct.priority)
+    priorities = {
+        ct.priority for bfc in bundle_file_changes for ct in bfc.involved_change_types()
+    }
     # get the lowest priority
     for p in reversed(ChangeTypePriority):
         if p in priorities:

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/change_owners/decision.py

@@ -1,3 +1,4 @@
+import operator
 from collections import defaultdict
 from collections.abc import (
     Iterable,
@@ -35,7 +36,7 @@ def get_approver_decisions_from_mr_comments(
     comments: Iterable[Mapping[str, Any]],
 ) -> list[Decision]:
     decisions: list[Decision] = []
-    for c in sorted(comments, key=lambda k: k["created_at"]):
+    for c in sorted(comments, key=operator.itemgetter("created_at")):
         commenter = c["username"]
         comment_body = c.get("body")
         for line in comment_body.split("\n") if comment_body else []:
@@ -146,20 +147,16 @@ def apply_decisions_to_changes(
     to generate the coverage report and to reason about the approval
     state of the MR.
     """
-
-    diff_decisions = []
-    for c in changes:
-        for d in c.diff_coverage:
-            diff_decisions.append(
-                _apply_decision_to_diff(
-                    c,
-                    d,
-                    approver_decisions,
-                    auto_approver_usernames,
-                )
-            )
-
-    return diff_decisions
+    return [
+        _apply_decision_to_diff(
+            c,
+            d,
+            approver_decisions,
+            auto_approver_usernames,
+        )
+        for c in changes
+        for d in c.diff_coverage
+    ]
 
 
 def _apply_decision_to_diff(

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/change_owners/self_service_roles.py

@@ -139,7 +139,7 @@ def change_type_contexts_for_self_service_roles(
     resolved_approvers = resolve_role_members([r for r in roles if r.self_service])
 
     # match every BundleChange with every relevant ChangeTypeV1
-    change_type_contexts = []
+    change_type_contexts: list[tuple[BundleFileChange, ChangeTypeContext]] = []
     for bc in bundle_changes:
         for ctp in change_type_processors:
             for ownership in ctp.find_context_file_refs(
@@ -177,8 +177,8 @@ def change_type_contexts_for_self_service_roles(
                         else []
                     )
                 })
-                for role in owning_roles.values():
-                    change_type_contexts.append((
+                change_type_contexts.extend(
+                    (
                         bc,
                         ChangeTypeContext(
                             change_type_processor=ctp,
@@ -193,7 +193,9 @@ def change_type_contexts_for_self_service_roles(
                             change_owner_labels=change_type_labels_from_role(role),
                             context_file=ownership.context_file_ref,
                         ),
-                    ))
+                    )
+                    for role in owning_roles.values()
+                )
     return change_type_contexts
 
 

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/change_owners/tester.py

@@ -130,16 +130,14 @@ class AppInterfaceRepo:
             elif c.change_schema is None or c.change_schema == ctp.context_schema:
                 # the change happens in the self_service related files
                 for ssc in role.self_service or []:
-                    for df in ssc.datafiles or []:
-                        bundle_files.append(
-                            self.bundle_file_for_path(BundleFileType.DATAFILE, df.path)
-                        )
-                    for rf_path in ssc.resources or []:
-                        bundle_files.append(
-                            self.bundle_file_for_path(
-                                BundleFileType.RESOURCEFILE, rf_path
-                            )
-                        )
+                    bundle_files.extend(
+                        self.bundle_file_for_path(BundleFileType.DATAFILE, df.path)
+                        for df in ssc.datafiles or []
+                    )
+                    bundle_files.extend(
+                        self.bundle_file_for_path(BundleFileType.RESOURCEFILE, rf_path)
+                        for rf_path in ssc.resources or []
+                    )
         return bundle_files
 
     def bundle_files_with_schemas(self, schema: str) -> list[BundleFileChange]:

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/cli.py

@@ -256,7 +256,7 @@ def print_to_file(function):
 def config_name(function):
     function = click.option(
         "--config-name",
-        help="jenkins config name to print out." "must works with --print-only mode",
+        help="jenkins config name to print out.must works with --print-only mode",
         default=None,
     )(function)
 
@@ -308,7 +308,7 @@ def vault_output_path(function):
 def vault_throughput_path(function):
     function = click.option(
         "--vault-throughput-path",
-        help="path in Vault to find input resources " "and store output resources.",
+        help="path in Vault to find input resources and store output resources.",
         default="",
     )(function)
 
@@ -888,7 +888,7 @@ def openshift_users(ctx, thread_pool_size, internal, use_jump_host):
 
 
 @integration.command(
-    short_help="Use OpenShift ServiceAccount tokens " "across namespaces/clusters."
+    short_help="Use OpenShift ServiceAccount tokens across namespaces/clusters."
 )
 @threaded()
 @binary(["oc", "ssh"])
@@ -1601,8 +1601,7 @@ def openshift_tekton_resources(
 
 
 @integration.command(
-    short_help="Guesses and adds labels to merge requests "
-    "according to changed paths."
+    short_help="Guesses and adds labels to merge requests according to changed paths."
 )
 @click.argument("gitlab-project-id")
 @click.argument("gitlab-merge-request-id")
@@ -2801,7 +2800,7 @@ def ocm_addons_upgrade_scheduler_org(
 @click.option(
     "--ignore-sts-clusters",
     is_flag=True,
-    default=os.environ.get("IGNORE_STS_CLUSTERS", False),
+    default=bool(os.environ.get("IGNORE_STS_CLUSTERS")),
     help="Ignore STS clusters",
 )
 @click.pass_context
@@ -2938,7 +2937,7 @@ def ocm_addons(ctx, thread_pool_size):
 
 
 @integration.command(
-    short_help="Grants AWS infrastructure access " "to members in AWS groups via OCM."
+    short_help="Grants AWS infrastructure access to members in AWS groups via OCM."
 )
 @click.pass_context
 def ocm_aws_infrastructure_access(ctx):
@@ -3129,7 +3128,7 @@ def email_sender(ctx):
 
 
 @integration.command(
-    short_help="Send emails to users based on " "requests submitted to app-interface."
+    short_help="Send emails to users based on requests submitted to app-interface."
 )
 @click.pass_context
 def requests_sender(ctx):
@@ -3156,8 +3155,7 @@ def sql_query(ctx, enable_deletion):
 
 
 @integration.command(
-    short_help="Manages labels on gitlab merge requests "
-    "based on OWNERS files schema."
+    short_help="Manages labels on gitlab merge requests based on OWNERS files schema."
 )
 @threaded()
 @click.pass_context
@@ -3525,7 +3523,7 @@ def integrations_manager(
 @click.option(
     "--mr-management",
     is_flag=True,
-    default=os.environ.get("MR_MANAGEMENT", False),
+    default=bool(os.environ.get("MR_MANAGEMENT")),
     help="Manage MR labels and comments (default to false)",
 )
 @click.pass_context

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/closedbox_endpoint_monitoring_base.py

@@ -84,7 +84,7 @@ def parse_prober_url(url: str) -> dict[str, str]:
     parsed_url = urlparse(url)
     if parsed_url.scheme not in {"http", "https"}:
         raise ValueError(
-            "the prober URL needs to be an http:// or https:// one " f"but is {url}"
+            f"the prober URL needs to be an http:// or https:// one but is {url}"
         )
     data = {"url": parsed_url.netloc, "scheme": parsed_url.scheme}
     if parsed_url.path:

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/cna/integration.py

@@ -49,8 +49,8 @@ class CNAIntegration:
     ):
         self._cna_clients = cna_clients
         self._namespaces = namespaces
-        self._desired_states = desired_states if desired_states else defaultdict(State)
-        self._current_states = current_states if current_states else defaultdict(State)
+        self._desired_states = desired_states or defaultdict(State)
+        self._current_states = current_states or defaultdict(State)
 
     def assemble_desired_states(self):
         self._desired_states = defaultdict(State)

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/dashdotdb_base.py

@@ -57,7 +57,7 @@ class DashdotdbBase:
             return None
 
         params = {"scope": self.scope}
-        endpoint = f"{self.dashdotdb_url}/api/v1/" f"token"
+        endpoint = f"{self.dashdotdb_url}/api/v1/token"
         response = requests.get(
             url=endpoint,
             params=params,
@@ -81,7 +81,7 @@ class DashdotdbBase:
             return None
 
         params = {"scope": self.scope}
-        endpoint = f"{self.dashdotdb_url}/api/v1/" f"token/{self.dashdotdb_token}"
+        endpoint = f"{self.dashdotdb_url}/api/v1/token/{self.dashdotdb_token}"
         response = requests.delete(
             url=endpoint,
             params=params,

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/dashdotdb_cso.py

@@ -56,7 +56,7 @@ class DashdotdbCSO(DashdotdbBase):
             return response
 
         for item in imagemanifestvuln["items"]:
-            endpoint = f"{self.dashdotdb_url}/api/v1/" f"imagemanifestvuln/{cluster}"
+            endpoint = f"{self.dashdotdb_url}/api/v1/imagemanifestvuln/{cluster}"
             response = self._do_post(endpoint, item)
             try:
                 response.raise_for_status()

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/dashdotdb_dora.py

@@ -110,7 +110,7 @@ class DeploymentDB:
         trigger_reason_pattern: str,
         app_env_since_list: Iterable[tuple[AppEnv, datetime]],
     ) -> list[tuple[AppEnv, Deployment]]:
-        deployments = []
+        deployments: list[tuple[AppEnv, Deployment]] = []
         with self.conn.cursor() as cur:
             subq = [
                 sql.SQL(
@@ -137,11 +137,13 @@ class DeploymentDB:
 
             cur.execute(query)
 
-            for record in cur:
-                deployments.append((
+            deployments.extend(
+                (
                     AppEnv(record[0], record[1]),
                     Deployment(record[2], record[3]),
-                ))
+                )
+                for record in cur
+            )
 
         if not deployments:
             LOG.info("No deployments found")

{qontract_reconcile-0.10.2.dev13 → qontract_reconcile-0.10.2.dev15}/reconcile/dashdotdb_slo.py

@@ -78,7 +78,7 @@ class DashdotdbSLO(DashdotdbBase):
 
         for item in service_slos:
             slo_name = item.name
-            endpoint = f"{self.dashdotdb_url}/api/v1/" f"serviceslometrics/{slo_name}"
+            endpoint = f"{self.dashdotdb_url}/api/v1/serviceslometrics/{slo_name}"
             payload = item.dashdot_payload()
             if self.dry_run:
                 continue