qontract-reconcile 0.10.2.dev55__tar.gz → 0.10.2.dev57__tar.gz

{qontract_reconcile-0.10.2.dev55 → qontract_reconcile-0.10.2.dev57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev55
+Version: 0.10.2.dev57
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev55 → qontract_reconcile-0.10.2.dev57}/pyproject.toml

@@ -72,7 +72,7 @@ debugger = ["debugpy ~=1.6.0"]
 dev = [
     "anymarkup ~=0.8.1",
     # keep in sync project.dependencies.boto3
-    "boto3-stubs[account,dynamodb,ec2,ecr,elb,iam,logs,organizations,rds,route53,s3,service-quotas,sqs,sts,support]==1.34.94",
+    "boto3-stubs[account,ec2,s3,rds,iam,route53,organizations,service-quotas,sts,support,dynamodb]==1.34.94",
     "kubernetes-stubs==22.6.0",
     "MarkupSafe ~=2.1.1",
     "moto ~=2.2",
@@ -327,6 +327,19 @@ module = [
     "reconcile.terraform_aws_route53",
     "reconcile.terraform_cloudflare_resources",
     "reconcile.terraform_users",
+    "reconcile.utils.aggregated_list",
+    "reconcile.utils.aws_api",
+    "reconcile.utils.aws_helper",
+    "reconcile.utils.binary",
+    "reconcile.utils.config",
+    "reconcile.utils.data_structures",
+    "reconcile.utils.defer",
+    "reconcile.utils.elasticsearch_exceptions",
+    "reconcile.utils.environ",
+    "reconcile.utils.exceptions",
+    "reconcile.utils.git_secrets",
+    "reconcile.utils.git",
+    "reconcile.utils.gitlab_api",
     "reconcile.utils.gpg",
     "reconcile.utils.gql",
     "reconcile.utils.helpers",
@@ -405,6 +418,7 @@ module = [
     "reconcile.test.statuspage.test_statuspage_state",
     "reconcile.test.statuspage.test_statuspage_status",
     "reconcile.test.test_acs_rbac",
+    "reconcile.test.test_aggregated_list",
     "reconcile.test.test_aws_ami_share",
     "reconcile.test.test_aws_iam_keys",
     "reconcile.test.test_aws_iam_password_reset",
@@ -467,11 +481,15 @@ module = [
     "reconcile.test.test_vpc_peerings_validator",
     "reconcile.test.test_wrong_region",
     "reconcile.test.utils.internal_groups.test_internal_groups_client",
+    "reconcile.test.utils.test_aws_api",
+    "reconcile.test.utils.test_aws_helper",
     "reconcile.test.utils.test_aws_rds",
+    "reconcile.test.utils.test_data_structures",
     "reconcile.test.utils.test_expiration",
     "reconcile.test.utils.test_external_resource_spec",
     "reconcile.test.utils.test_external_resources",
     "reconcile.test.utils.test_github_api",
+    "reconcile.test.utils.test_gitlab_api",
     "reconcile.test.utils.test_gpg",
     "reconcile.test.utils.test_gql",
     "reconcile.test.utils.test_grouping",
@@ -524,8 +542,6 @@ module = [
     "jira.*",
     "jsonpath_ng.*",
     "kubernetes.*",
-    "matplotlib",
-    "matplotlib.*",
     "moto.*",
     "networkx.*",
     "openshift.*",

{qontract_reconcile-0.10.2.dev55 → qontract_reconcile-0.10.2.dev57}/reconcile/aws_cloudwatch_log_retention/integration.py

@@ -3,7 +3,10 @@ import re
 import typing
 from collections import defaultdict
 from collections.abc import Iterable
-from datetime import UTC, datetime, timedelta
+from datetime import (
+    datetime,
+    timedelta,
+)
 from enum import Enum
 from typing import TYPE_CHECKING
 
@@ -15,9 +18,9 @@ from reconcile.queries import get_aws_accounts
 from reconcile.utils.aws_api import AWSApi
 
 if TYPE_CHECKING:
-    from mypy_boto3_logs.type_defs import LogGroupTypeDef
+    from mypy_boto3_logs import CloudWatchLogsClient  # type: ignore
 else:
-    LogGroupTypeDef = object
+    CloudWatchLogsClient = object
 
 QONTRACT_INTEGRATION = "aws_cloudwatch_log_retention"
 MANAGED_BY_INTEGRATION_KEY = "managed_by_integration"
@@ -66,17 +69,21 @@ def create_awsapi_client(accounts: list, thread_pool_size: int) -> AWSApi:
     return AWSApi(thread_pool_size, accounts, settings=settings, init_users=False)
 
 
-def is_empty(log_group: LogGroupTypeDef) -> bool:
+def is_empty(
+    log_group: dict,
+) -> bool:
     return log_group["storedBytes"] == 0
 
 
 def is_longer_than_retention(
-    log_group: LogGroupTypeDef,
+    log_group: dict,
     desired_retention_days: int,
 ) -> bool:
-    return datetime.fromtimestamp(log_group["creationTime"] / 1000, tz=UTC) + timedelta(
-        days=desired_retention_days
-    ) < datetime.now(tz=UTC)
+    return (
+        datetime.fromtimestamp(log_group["creationTime"] / 1000)
+        + timedelta(days=desired_retention_days)
+        < datetime.utcnow()
+    )
 
 
 class TagStatus(Enum):
@@ -86,7 +93,7 @@ class TagStatus(Enum):
 
 
 def get_tag_status(
-    log_group: LogGroupTypeDef,
+    log_group: dict,
     account_name: str,
     region: str,
     aws_api: AWSApi,
@@ -106,7 +113,7 @@ def get_tag_status(
 
 def _reconcile_log_group(
     dry_run: bool,
-    aws_log_group: LogGroupTypeDef,
+    aws_log_group: dict,
     desired_cleanup_options: Iterable[AWSCloudwatchCleanupOption],
     account_name: str,
     region: str,

{qontract_reconcile-0.10.2.dev55 → qontract_reconcile-0.10.2.dev57}/reconcile/dashdotdb_dora.py

@@ -397,11 +397,10 @@ class DashdotdbDORA(DashdotdbBase):
         self, saastarget: SaasTarget, sha: str
     ) -> tuple[str | None, str | None]:
         try:
-            saas_file_yaml = self.gl_app_interface_get_file(saastarget.path, ref=sha)
-            if not saas_file_yaml:
-                LOG.info(f"failed to fetch saas file {saastarget.path} at {sha}")
-                return (None, None)
-            saas_file = yaml.safe_load(saas_file_yaml.decode())
+            saas_file_yaml = self.gl_app_interface_get_file(
+                saastarget.path, ref=sha
+            ).decode()
+            saas_file = yaml.safe_load(saas_file_yaml)
         except Exception as e:
             LOG.info(f"failed to decode saas file {saastarget.path} with error: {e}")
             return (None, None)

{qontract_reconcile-0.10.2.dev55 → qontract_reconcile-0.10.2.dev57}/reconcile/gitlab_housekeeping.py

@@ -11,11 +11,10 @@ from datetime import (
     timedelta,
 )
 from operator import itemgetter
-from typing import Any, cast
+from typing import Any
 
 import gitlab
 from gitlab.v4.objects import (
-    ProjectCommit,
     ProjectIssue,
     ProjectMergeRequest,
 )
@@ -336,15 +335,12 @@ def is_good_to_merge(labels):
 
 def is_rebased(mr, gl: GitLabApi) -> bool:
     target_branch = mr.target_branch
-    head = cast(
-        list[ProjectCommit],
-        gl.project.commits.list(
-            ref_name=target_branch,
-            per_page=1,
-            page=1,
-        ),
+    head = gl.project.commits.list(
+        ref_name=target_branch,
+        per_page=1,
+        page=1,
     )[0].id
-    result = cast(dict, gl.project.repository_compare(mr.sha, head))
+    result = gl.project.repository_compare(mr.sha, head)
     return len(result["commits"]) == 0
 
 

{qontract_reconcile-0.10.2.dev55 → qontract_reconcile-0.10.2.dev57}/reconcile/terraform_tgw_attachments.py

@@ -195,9 +195,9 @@ def _build_desired_state_tgw_connection(
         cluster_region,
         cluster_cidr_block,
         tags=peer_connection.tags or {},
-        route_tables=bool(peer_connection.manage_routes),
-        security_groups=bool(peer_connection.manage_security_groups),
-        route53_associations=bool(peer_connection.manage_route53_associations),
+        route_tables=peer_connection.manage_routes,
+        security_groups=peer_connection.manage_security_groups,
+        route53_associations=peer_connection.manage_route53_associations,
     )
     for tgw in account_tgws:
         connection_name = (
@@ -268,13 +268,13 @@ def _build_accepter(
     awsapi: AWSApi,
     private_hcp: bool = False,
 ) -> Accepter:
-    allow_hcp_private_api_access = bool(
+    allow_hcp_private_api_access = (
         private_hcp and peer_connection.allow_private_hcp_api_access
     )
     (vpc_id, route_table_ids, subnets_id_az, api_security_group_id) = (
         awsapi.get_cluster_vpc_details(
             account.dict(by_alias=True),
-            route_tables=bool(peer_connection.manage_routes),
+            route_tables=peer_connection.manage_routes,
             subnets=True,
             hcp_vpc_endpoint_sg=allow_hcp_private_api_access,
         )

{qontract_reconcile-0.10.2.dev55 → qontract_reconcile-0.10.2.dev57}/reconcile/terraform_vpc_peerings.py

@@ -198,7 +198,7 @@ def build_desired_state_single_cluster(
                 f"{peer_connection_name}"
             )
 
-        accepter_manage_routes = peer_info.get("manageRoutes", False)
+        accepter_manage_routes = peer_info.get("manageRoutes")
 
         infra_account_name, req_aws, acc_aws = aws_assume_roles_for_cluster_vpc_peering(
             peer_connection, cluster_info, peer_info, peer_cluster, ocm

{qontract_reconcile-0.10.2.dev55 → qontract_reconcile-0.10.2.dev57}/reconcile/utils/aggregated_list.py

@@ -1,27 +1,17 @@
 import json
 import logging
-from collections.abc import Callable, KeysView
-from typing import Any, TypedDict
-
-Action = Callable[[Any, list[Any]], bool]
-Cond = Callable[[Any], bool]
 
 
 class RunnerException(Exception):
     pass
 
 
-class AggregatedItem(TypedDict):
-    params: Any
-    items: list[Any]
-
-
 class AggregatedList:
-    def __init__(self) -> None:
-        self._dict: dict[int, AggregatedItem] = {}
+    def __init__(self):
+        self._dict = {}
 
-    def add(self, params: Any, new_items: Any | list[Any]) -> None:
-        params_hash: int = self.hash_params(params)
+    def add(self, params, new_items):
+        params_hash = self.hash_params(params)
 
         if self._dict.get(params_hash) is None:
             self._dict[params_hash] = {"params": params, "items": []}
@@ -33,20 +23,20 @@ class AggregatedList:
             if item not in self._dict[params_hash]["items"]:
                 self._dict[params_hash]["items"].append(item)
 
-    def get(self, params: Any) -> AggregatedItem:
+    def get(self, params):
         return self._dict[self.hash_params(params)]
 
-    def get_all_params_hash(self) -> KeysView[int]:
+    def get_all_params_hash(self):
         return self._dict.keys()
 
-    def get_by_params_hash(self, params_hash: int) -> AggregatedItem:
+    def get_by_params_hash(self, params_hash):
         return self._dict[params_hash]
 
-    def diff(self, right_state: "AggregatedList") -> dict[str, list[AggregatedItem]]:
+    def diff(self, right_state):
         left_params = self.get_all_params_hash()
         right_params = right_state.get_all_params_hash()
 
-        diff: dict[str, list[AggregatedItem]] = {
+        diff = {
             "insert": [
                 right_state.get_by_params_hash(p)
                 for p in right_params
@@ -62,11 +52,11 @@ class AggregatedList:
         union = [p for p in left_params if p in right_params]
 
         for p in union:
-            left: AggregatedItem = self.get_by_params_hash(p)
-            right: AggregatedItem = right_state.get_by_params_hash(p)
+            left = self.get_by_params_hash(p)
+            right = right_state.get_by_params_hash(p)
 
-            l_items: list[Any] = left["items"]
-            r_items: list[Any] = right["items"]
+            l_items = left["items"]
+            r_items = right["items"]
 
             update_insert = [i for i in r_items if i not in l_items]
             update_delete = [i for i in l_items if i not in r_items]
@@ -85,28 +75,28 @@ class AggregatedList:
 
         return diff
 
-    def dump(self) -> list[AggregatedItem]:
+    def dump(self):
         return list(self._dict.values())
 
-    def toJSON(self) -> str:
+    def toJSON(self):
         return json.dumps(self.dump(), indent=4)
 
     @staticmethod
-    def hash_params(params: Any) -> int:
+    def hash_params(params):
         return hash(json.dumps(params, sort_keys=True))
 
 
 class AggregatedDiffRunner:
-    def __init__(self, diff: dict[str, list[AggregatedItem]]) -> None:
+    def __init__(self, diff):
         self.diff = diff
-        self.actions: list[tuple[str, Action, Cond | None]] = []
+        self.actions = []
 
-    def register(self, on: str, action: Action, cond: Cond | None = None) -> None:
+    def register(self, on, action, cond=None):
         if on not in self.diff:
             raise Exception(f"Unknown diff key for 'on': {on}")
         self.actions.append((on, action, cond))
 
-    def run(self) -> bool:
+    def run(self):
         status = True
 
         for on, action, cond in self.actions: