qontract-reconcile 0.10.2.dev57__tar.gz → 0.10.2.dev58__tar.gz

{qontract_reconcile-0.10.2.dev57 → qontract_reconcile-0.10.2.dev58}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev57
+Version: 0.10.2.dev58
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev57 → qontract_reconcile-0.10.2.dev58}/pyproject.toml

@@ -72,7 +72,7 @@ debugger = ["debugpy ~=1.6.0"]
 dev = [
     "anymarkup ~=0.8.1",
     # keep in sync project.dependencies.boto3
-    "boto3-stubs[account,ec2,s3,rds,iam,route53,organizations,service-quotas,sts,support,dynamodb]==1.34.94",
+    "boto3-stubs[account,dynamodb,ec2,ecr,elb,iam,logs,organizations,rds,route53,s3,service-quotas,sqs,sts,support]==1.34.94",
     "kubernetes-stubs==22.6.0",
     "MarkupSafe ~=2.1.1",
     "moto ~=2.2",
@@ -327,19 +327,6 @@ module = [
     "reconcile.terraform_aws_route53",
     "reconcile.terraform_cloudflare_resources",
     "reconcile.terraform_users",
-    "reconcile.utils.aggregated_list",
-    "reconcile.utils.aws_api",
-    "reconcile.utils.aws_helper",
-    "reconcile.utils.binary",
-    "reconcile.utils.config",
-    "reconcile.utils.data_structures",
-    "reconcile.utils.defer",
-    "reconcile.utils.elasticsearch_exceptions",
-    "reconcile.utils.environ",
-    "reconcile.utils.exceptions",
-    "reconcile.utils.git_secrets",
-    "reconcile.utils.git",
-    "reconcile.utils.gitlab_api",
     "reconcile.utils.gpg",
     "reconcile.utils.gql",
     "reconcile.utils.helpers",
@@ -379,6 +366,10 @@ module = [
     "reconcile.utils.vault",
     "reconcile.utils.vaultsecretref",
     "reconcile.vpc_peerings_validator",
+    "tools.app_interface_reporter",
+    "tools.cli_commands.gpg_encrypt",
+    "tools.cli_commands.test.test_gpg_encrypt",
+    "tools.qontract_cli",
     "tools.sre_checkpoints.util",
     # untyped tests
     "reconcile.test.cna.test_integration",
@@ -418,7 +409,6 @@ module = [
     "reconcile.test.statuspage.test_statuspage_state",
     "reconcile.test.statuspage.test_statuspage_status",
     "reconcile.test.test_acs_rbac",
-    "reconcile.test.test_aggregated_list",
     "reconcile.test.test_aws_ami_share",
     "reconcile.test.test_aws_iam_keys",
     "reconcile.test.test_aws_iam_password_reset",
@@ -481,15 +471,11 @@ module = [
     "reconcile.test.test_vpc_peerings_validator",
     "reconcile.test.test_wrong_region",
     "reconcile.test.utils.internal_groups.test_internal_groups_client",
-    "reconcile.test.utils.test_aws_api",
-    "reconcile.test.utils.test_aws_helper",
     "reconcile.test.utils.test_aws_rds",
-    "reconcile.test.utils.test_data_structures",
     "reconcile.test.utils.test_expiration",
     "reconcile.test.utils.test_external_resource_spec",
     "reconcile.test.utils.test_external_resources",
     "reconcile.test.utils.test_github_api",
-    "reconcile.test.utils.test_gitlab_api",
     "reconcile.test.utils.test_gpg",
     "reconcile.test.utils.test_gql",
     "reconcile.test.utils.test_grouping",
@@ -518,6 +504,9 @@ module = [
     "reconcile.test.utils.test_terrascript_aws_client",
     "reconcile.test.utils.test_terrascript_cloudflare_client",
     "reconcile.test.utils.test_terrascript_cloudflare_resources",
+    "tools.test.test_qontract_cli",
+    "tools.test.test_sd_app_sre_alert_report",
+    "tools.test.test_sre_checkpoints",
 ]
 check_untyped_defs = false
 disallow_untyped_defs = false
@@ -542,6 +531,8 @@ module = [
     "jira.*",
     "jsonpath_ng.*",
     "kubernetes.*",
+    "matplotlib",
+    "matplotlib.*",
     "moto.*",
     "networkx.*",
     "openshift.*",

{qontract_reconcile-0.10.2.dev57 → qontract_reconcile-0.10.2.dev58}/reconcile/aws_cloudwatch_log_retention/integration.py

@@ -3,10 +3,7 @@ import re
 import typing
 from collections import defaultdict
 from collections.abc import Iterable
-from datetime import (
-    datetime,
-    timedelta,
-)
+from datetime import UTC, datetime, timedelta
 from enum import Enum
 from typing import TYPE_CHECKING
 
@@ -18,9 +15,9 @@ from reconcile.queries import get_aws_accounts
 from reconcile.utils.aws_api import AWSApi
 
 if TYPE_CHECKING:
-    from mypy_boto3_logs import CloudWatchLogsClient  # type: ignore
+    from mypy_boto3_logs.type_defs import LogGroupTypeDef
 else:
-    CloudWatchLogsClient = object
+    LogGroupTypeDef = object
 
 QONTRACT_INTEGRATION = "aws_cloudwatch_log_retention"
 MANAGED_BY_INTEGRATION_KEY = "managed_by_integration"
@@ -69,21 +66,17 @@ def create_awsapi_client(accounts: list, thread_pool_size: int) -> AWSApi:
     return AWSApi(thread_pool_size, accounts, settings=settings, init_users=False)
 
 
-def is_empty(
-    log_group: dict,
-) -> bool:
+def is_empty(log_group: LogGroupTypeDef) -> bool:
     return log_group["storedBytes"] == 0
 
 
 def is_longer_than_retention(
-    log_group: dict,
+    log_group: LogGroupTypeDef,
     desired_retention_days: int,
 ) -> bool:
-    return (
-        datetime.fromtimestamp(log_group["creationTime"] / 1000)
-        + timedelta(days=desired_retention_days)
-        < datetime.utcnow()
-    )
+    return datetime.fromtimestamp(log_group["creationTime"] / 1000, tz=UTC) + timedelta(
+        days=desired_retention_days
+    ) < datetime.now(tz=UTC)
 
 
 class TagStatus(Enum):
@@ -93,7 +86,7 @@ class TagStatus(Enum):
 
 
 def get_tag_status(
-    log_group: dict,
+    log_group: LogGroupTypeDef,
     account_name: str,
     region: str,
     aws_api: AWSApi,
@@ -113,7 +106,7 @@ def get_tag_status(
 
 def _reconcile_log_group(
     dry_run: bool,
-    aws_log_group: dict,
+    aws_log_group: LogGroupTypeDef,
     desired_cleanup_options: Iterable[AWSCloudwatchCleanupOption],
     account_name: str,
     region: str,

{qontract_reconcile-0.10.2.dev57 → qontract_reconcile-0.10.2.dev58}/reconcile/dashdotdb_dora.py

@@ -397,10 +397,11 @@ class DashdotdbDORA(DashdotdbBase):
         self, saastarget: SaasTarget, sha: str
     ) -> tuple[str | None, str | None]:
         try:
-            saas_file_yaml = self.gl_app_interface_get_file(
-                saastarget.path, ref=sha
-            ).decode()
-            saas_file = yaml.safe_load(saas_file_yaml)
+            saas_file_yaml = self.gl_app_interface_get_file(saastarget.path, ref=sha)
+            if not saas_file_yaml:
+                LOG.info(f"failed to fetch saas file {saastarget.path} at {sha}")
+                return (None, None)
+            saas_file = yaml.safe_load(saas_file_yaml.decode())
         except Exception as e:
             LOG.info(f"failed to decode saas file {saastarget.path} with error: {e}")
             return (None, None)

{qontract_reconcile-0.10.2.dev57 → qontract_reconcile-0.10.2.dev58}/reconcile/gitlab_housekeeping.py

@@ -11,10 +11,11 @@ from datetime import (
     timedelta,
 )
 from operator import itemgetter
-from typing import Any
+from typing import Any, cast
 
 import gitlab
 from gitlab.v4.objects import (
+    ProjectCommit,
     ProjectIssue,
     ProjectMergeRequest,
 )
@@ -335,12 +336,15 @@ def is_good_to_merge(labels):
 
 def is_rebased(mr, gl: GitLabApi) -> bool:
     target_branch = mr.target_branch
-    head = gl.project.commits.list(
-        ref_name=target_branch,
-        per_page=1,
-        page=1,
+    head = cast(
+        list[ProjectCommit],
+        gl.project.commits.list(
+            ref_name=target_branch,
+            per_page=1,
+            page=1,
+        ),
     )[0].id
-    result = gl.project.repository_compare(mr.sha, head)
+    result = cast(dict, gl.project.repository_compare(mr.sha, head))
     return len(result["commits"]) == 0
 
 

{qontract_reconcile-0.10.2.dev57 → qontract_reconcile-0.10.2.dev58}/reconcile/terraform_tgw_attachments.py

@@ -195,9 +195,9 @@ def _build_desired_state_tgw_connection(
         cluster_region,
         cluster_cidr_block,
         tags=peer_connection.tags or {},
-        route_tables=peer_connection.manage_routes,
-        security_groups=peer_connection.manage_security_groups,
-        route53_associations=peer_connection.manage_route53_associations,
+        route_tables=bool(peer_connection.manage_routes),
+        security_groups=bool(peer_connection.manage_security_groups),
+        route53_associations=bool(peer_connection.manage_route53_associations),
     )
     for tgw in account_tgws:
         connection_name = (
@@ -268,13 +268,13 @@ def _build_accepter(
     awsapi: AWSApi,
     private_hcp: bool = False,
 ) -> Accepter:
-    allow_hcp_private_api_access = (
+    allow_hcp_private_api_access = bool(
         private_hcp and peer_connection.allow_private_hcp_api_access
     )
     (vpc_id, route_table_ids, subnets_id_az, api_security_group_id) = (
         awsapi.get_cluster_vpc_details(
             account.dict(by_alias=True),
-            route_tables=peer_connection.manage_routes,
+            route_tables=bool(peer_connection.manage_routes),
             subnets=True,
             hcp_vpc_endpoint_sg=allow_hcp_private_api_access,
         )

{qontract_reconcile-0.10.2.dev57 → qontract_reconcile-0.10.2.dev58}/reconcile/terraform_vpc_peerings.py

@@ -198,7 +198,7 @@ def build_desired_state_single_cluster(
                 f"{peer_connection_name}"
             )
 
-        accepter_manage_routes = peer_info.get("manageRoutes")
+        accepter_manage_routes = peer_info.get("manageRoutes", False)
 
         infra_account_name, req_aws, acc_aws = aws_assume_roles_for_cluster_vpc_peering(
             peer_connection, cluster_info, peer_info, peer_cluster, ocm

{qontract_reconcile-0.10.2.dev57 → qontract_reconcile-0.10.2.dev58}/reconcile/utils/aggregated_list.py

@@ -1,17 +1,27 @@
 import json
 import logging
+from collections.abc import Callable, KeysView
+from typing import Any, TypedDict
+
+Action = Callable[[Any, list[Any]], bool]
+Cond = Callable[[Any], bool]
 
 
 class RunnerException(Exception):
     pass
 
 
+class AggregatedItem(TypedDict):
+    params: Any
+    items: list[Any]
+
+
 class AggregatedList:
-    def __init__(self):
-        self._dict = {}
+    def __init__(self) -> None:
+        self._dict: dict[int, AggregatedItem] = {}
 
-    def add(self, params, new_items):
-        params_hash = self.hash_params(params)
+    def add(self, params: Any, new_items: Any | list[Any]) -> None:
+        params_hash: int = self.hash_params(params)
 
         if self._dict.get(params_hash) is None:
             self._dict[params_hash] = {"params": params, "items": []}
@@ -23,20 +33,20 @@ class AggregatedList:
             if item not in self._dict[params_hash]["items"]:
                 self._dict[params_hash]["items"].append(item)
 
-    def get(self, params):
+    def get(self, params: Any) -> AggregatedItem:
         return self._dict[self.hash_params(params)]
 
-    def get_all_params_hash(self):
+    def get_all_params_hash(self) -> KeysView[int]:
         return self._dict.keys()
 
-    def get_by_params_hash(self, params_hash):
+    def get_by_params_hash(self, params_hash: int) -> AggregatedItem:
         return self._dict[params_hash]
 
-    def diff(self, right_state):
+    def diff(self, right_state: "AggregatedList") -> dict[str, list[AggregatedItem]]:
         left_params = self.get_all_params_hash()
         right_params = right_state.get_all_params_hash()
 
-        diff = {
+        diff: dict[str, list[AggregatedItem]] = {
             "insert": [
                 right_state.get_by_params_hash(p)
                 for p in right_params
@@ -52,11 +62,11 @@ class AggregatedList:
         union = [p for p in left_params if p in right_params]
 
         for p in union:
-            left = self.get_by_params_hash(p)
-            right = right_state.get_by_params_hash(p)
+            left: AggregatedItem = self.get_by_params_hash(p)
+            right: AggregatedItem = right_state.get_by_params_hash(p)
 
-            l_items = left["items"]
-            r_items = right["items"]
+            l_items: list[Any] = left["items"]
+            r_items: list[Any] = right["items"]
 
             update_insert = [i for i in r_items if i not in l_items]
             update_delete = [i for i in l_items if i not in r_items]
@@ -75,28 +85,28 @@ class AggregatedList:
 
         return diff
 
-    def dump(self):
+    def dump(self) -> list[AggregatedItem]:
         return list(self._dict.values())
 
-    def toJSON(self):
+    def toJSON(self) -> str:
         return json.dumps(self.dump(), indent=4)
 
     @staticmethod
-    def hash_params(params):
+    def hash_params(params: Any) -> int:
         return hash(json.dumps(params, sort_keys=True))
 
 
 class AggregatedDiffRunner:
-    def __init__(self, diff):
+    def __init__(self, diff: dict[str, list[AggregatedItem]]) -> None:
         self.diff = diff
-        self.actions = []
+        self.actions: list[tuple[str, Action, Cond | None]] = []
 
-    def register(self, on, action, cond=None):
+    def register(self, on: str, action: Action, cond: Cond | None = None) -> None:
         if on not in self.diff:
             raise Exception(f"Unknown diff key for 'on': {on}")
         self.actions.append((on, action, cond))
 
-    def run(self):
+    def run(self) -> bool:
         status = True
 
         for on, action, cond in self.actions: