qontract-reconcile 0.10.2.dev317__tar.gz → 0.10.2.dev319__tar.gz

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev317
+Version: 0.10.2.dev319
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/pyproject.toml

@@ -262,21 +262,12 @@ disallow_incomplete_defs = true
 [[tool.mypy.overrides]]
 # Untyped modules
 module = [
-    "reconcile.utils.gpg",
-    "reconcile.utils.imap_client",
-    "reconcile.utils.jjb_client",
-    "reconcile.utils.ldap_client",
-    "reconcile.utils.lean_terraform_client",
     "reconcile.utils.oc",
-    "reconcile.utils.ocm_base_client",
-    "reconcile.utils.ocm.ocm",
     "reconcile.utils.openshift_resource",
-    "reconcile.utils.raw_github_api",
     "reconcile.utils.vault",
     # untyped tests
     "reconcile.test.oc.test_oc_connection_parameters",
     "reconcile.test.oc.test_oc_map",
-    "reconcile.test.ocm_labels.test_ocm_labels_integration",
     "reconcile.test.test_acs_rbac",
     "reconcile.test.test_checkpoint",
     "reconcile.test.test_closedbox_endpoint_monitoring",
@@ -300,15 +291,11 @@ module = [
     "reconcile.test.utils.internal_groups.test_internal_groups_client",
     "reconcile.test.utils.test_aws_rds",
     "reconcile.test.utils.test_external_resource_spec",
-    "reconcile.test.utils.test_gpg",
     "reconcile.test.utils.test_helm",
-    "reconcile.test.utils.test_imap_client",
     "reconcile.test.utils.test_jira_client",
-    "reconcile.test.utils.test_jjb_client",
     "reconcile.test.utils.test_jsonpath",
     "reconcile.test.utils.test_models",
     "reconcile.test.utils.test_oc",
-    "reconcile.test.utils.test_ocm",
     "reconcile.test.utils.test_parse_dhms_duration",
     "reconcile.test.utils.test_promotion_state",
     "reconcile.test.utils.test_semver_helper",

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/reconcile/jenkins_job_builder.py

@@ -81,7 +81,7 @@ def run(
     if not print_only and config_name is not None:
         raise Exception("--config-name must works with --print-only mode")
     secret_reader = SecretReader(queries.get_secret_reader_settings())
-    jjb: JJB = init_jjb(secret_reader, instance_name, config_name, print_only)
+    jjb = init_jjb(secret_reader, instance_name, config_name, print_only)
     if defer:
         defer(jjb.cleanup)
 

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/reconcile/jenkins_worker_fleets.py

@@ -1,14 +1,12 @@
+from __future__ import annotations
+
 import logging
-from typing import (
-    Any,
-    cast,
-)
+from enum import Enum
+from typing import Any, cast
+
+from pydantic import BaseModel, Field
 
 from reconcile import queries
-from reconcile.jenkins.types import (
-    JenkinsWorkerFleet,
-    SSHConnector,
-)
 from reconcile.utils.external_resources import get_external_resource_specs
 from reconcile.utils.jenkins_api import JenkinsApi
 from reconcile.utils.secret_reader import SecretReader
@@ -17,6 +15,75 @@ from reconcile.utils.terrascript_aws_client import TerrascriptClient as Terrascr
 QONTRACT_INTEGRATION = "jenkins-worker-fleets"
 
 
+class SSHHostKeyVerificationStrategy(Enum):
+    MANUALLY_TRUSTED_KEY_VERIFICATION_STRATEGY = (
+        "manuallyTrustedKeyVerificationStrategy"
+    )
+    MANUALLY_PROVIDED_KEY_VERIFICATION_STRATEGY = (
+        "manuallyProvidedKeyVerificationStrategy"
+    )
+    NON_VERIFYING_KEY_VERIFICATION_STRATEGY = "nonVerifyingKeyVerificationStrategy"
+    KNOWN_HOSTS_FILE_KEY_VERIFICATION_STRATEGY = "knownHostsFileKeyVerificationStrategy"
+
+
+class SSHConnector(BaseModel):
+    credentials_id: str = Field(..., alias="credentialsId")
+    launch_timeout_seconds: int | None = Field(None, alias="launchTimeoutSeconds")
+    max_num_retries: int | None = Field(None, alias="maxNumRetries")
+    retry_wait_time: int | None = Field(None, alias="retryWaitTime")
+    port: int | None = 22
+    jvm_options: str | None = Field(None, alias="jvmOptions")
+    ssh_host_key_verification_strategy: SSHHostKeyVerificationStrategy = Field(
+        SSHHostKeyVerificationStrategy.NON_VERIFYING_KEY_VERIFICATION_STRATEGY,
+        alias="sshHostKeyVerificationStrategy",
+    )
+
+    class Config:
+        use_enum_values = True
+
+
+class ComputerConnector(BaseModel):
+    # alias name is defined by jcasc schema
+    ssh_connector: SSHConnector = Field(..., alias="sSHConnector")
+
+
+class JenkinsWorkerFleet(BaseModel):
+    # following options comes form https://github.com/jenkinsci/ec2-fleet-plugin/blob/master/docs/
+    name: str
+    fleet: str
+    region: str
+    min_size: int = Field(..., alias="minSize")
+    max_size: int = Field(..., alias="maxSize")
+    computer_connector: ComputerConnector = Field(..., alias="computerConnector")
+    fs_root: str = Field(..., alias="fsRoot")
+    label_string: str = Field(..., alias="labelString")
+    num_executors: int = Field(2, alias="numExecutors")
+    idle_minutes: int = Field(30, alias="idleMinutes")
+    min_spare_size: int = Field(0, alias="minSpareSize")
+    max_total_uses: int = Field(-1, alias="maxTotalUses")
+    no_delay_provision: bool = Field(False, alias="noDelayProvision")
+    add_node_only_if_running: bool = Field(True, alias="addNodeOnlyIfRunning")
+    always_reconnect: bool = Field(True, alias="alwaysReconnect")
+    private_ip_used: bool = Field(True, alias="privateIpUsed")
+    restrict_usage: bool = Field(True, alias="restrictUsage")
+
+    def __lt__(self, other: JenkinsWorkerFleet) -> bool:
+        return self.fleet < other.fleet
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, JenkinsWorkerFleet):
+            raise NotImplementedError(
+                "Cannot compare to non JenkinsWorkerFleet objects."
+            )
+        return self.fleet == other.fleet and self.region == other.region
+
+    def __hash__(self) -> int:
+        return hash(self.fleet + self.region)
+
+    def differ(self, other: JenkinsWorkerFleet) -> bool:
+        return self.dict() != other.dict()
+
+
 def get_current_state(jenkins: JenkinsApi) -> list[JenkinsWorkerFleet]:
     current_state = []
 

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/reconcile/oum/providers.py

@@ -33,7 +33,7 @@ class LdapGroupMemberProvider(GroupMemberProvider):
         if len(group_ids) == 0:
             return {}
         with self.ldap_client as lc:
-            groups_members_by_dn = lc.get_group_members(group_dn_mapping.keys())
+            groups_members_by_dn = lc.get_group_members(set(group_dn_mapping.keys()))
         return {
             group_dn_mapping[dn]: members
             for dn, members in groups_members_by_dn.items()

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/reconcile/terraform_vpc_peerings.py

@@ -73,6 +73,7 @@ def _build_infrastructure_assume_role(
     ocm: OCM | None,
     provided_assume_role: str | None,
 ) -> dict[str, Any] | None:
+    assume_role: str | None = None
     if provided_assume_role:
         assume_role = provided_assume_role
     elif cluster["spec"].get("account"):

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/reconcile/utils/gpg.py

@@ -8,7 +8,7 @@ from subprocess import (
 )
 
 
-def gpg_encrypt(content, public_gpg_key):
+def gpg_encrypt(content: str, public_gpg_key: str) -> str:
     public_gpg_key_dec = base64.b64decode(public_gpg_key)
 
     with tempfile.TemporaryDirectory() as gnupg_home_dir:
@@ -22,6 +22,8 @@ def gpg_encrypt(content, public_gpg_key):
         )
         out = proc.stdout.decode("utf-8")
         match = re.search(r"<\S+>", out)
+        if not match:
+            raise ValueError("No recipient found in GPG import output")
         recipient = match.group(0)[1:-1]
         # encrypt content
         proc = run(
@@ -41,5 +43,5 @@ def gpg_encrypt(content, public_gpg_key):
             stderr=STDOUT,
             check=True,
         )
-        out = proc.stdout
-    return out.decode("utf-8")
+        encrypted_out = proc.stdout
+    return encrypted_out.decode("utf-8")

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/reconcile/utils/imap_client.py

@@ -21,7 +21,7 @@ class ImapClient:
         self._server.login(self.user, self.password)
         return self
 
-    def __exit__(self, *args, **kwargs) -> None:
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
         if self._server:
             self._server.logout()
 

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/reconcile/utils/jjb_client.py

@@ -8,6 +8,7 @@ import shutil
 import subprocess
 import tempfile
 import xml.etree.ElementTree as ET
+from collections.abc import Iterable, Mapping
 from os import path
 from subprocess import (
     PIPE,
@@ -25,6 +26,8 @@ from sretoolbox.utils import retry
 
 from reconcile.utils import throughput
 from reconcile.utils.helpers import toggle_logger
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.state import State
 from reconcile.utils.vcs import GITHUB_BASE_URL
 
 JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
@@ -33,14 +36,22 @@ JJB_INI = "[jenkins]\nurl = https://JENKINS_URL"
 class JJB:
     """Wrapper around Jenkins Jobs"""
 
-    def __init__(self, configs, ssl_verify=True, secret_reader=None, print_only=False):
+    def __init__(
+        self,
+        configs: list[dict[str, Any]],
+        ssl_verify: bool = True,
+        secret_reader: SecretReaderBase | None = None,
+        print_only: bool = False,
+    ) -> None:
         self.print_only = print_only
         self.secret_reader = secret_reader
+        if not self.print_only and self.secret_reader is None:
+            raise ValueError("secret_reader must be provided if print_only is False")
         self.collect_configs(configs)
         self.modify_logger()
         self.python_https_verify = str(int(ssl_verify))
 
-    def collect_configs(self, configs):
+    def collect_configs(self, configs: list[dict[str, Any]]) -> None:
         instances = {
             c["instance"]["name"]: {
                 "serverUrl": c["instance"]["serverUrl"],
@@ -57,7 +68,7 @@ class JJB:
             server_url = data["serverUrl"]
             wd = tempfile.mkdtemp()
             ini = JJB_INI
-            if not self.print_only:
+            if not self.print_only and self.secret_reader:
                 ini = self.secret_reader.read(token)
                 ini = ini.replace('"', "")
                 ini = ini.replace("false", "False")
@@ -92,7 +103,7 @@ class JJB:
         self.instance_urls = instance_urls
         self.working_dirs = working_dirs
 
-    def overwrite_configs(self, configs):
+    def overwrite_configs(self, configs: Mapping[str, str] | State) -> None:
         """This function will override the existing
         config files in the working directories with
         the supplied configs"""
@@ -101,12 +112,12 @@ class JJB:
             with open(config_path, "w", encoding="locale") as f:
                 f.write(configs[name])
 
-    def sort(self, configs):
+    def sort(self, configs: list[dict[str, Any]]) -> None:
         configs.sort(key=self.sort_by_name)
-        configs.sort(key=self.sort_by_type)
+        configs.sort(key=lambda x: self.sort_by_type(x) or 0)
 
     @staticmethod
-    def sort_by_type(config):
+    def sort_by_type(config: Mapping[str, Any]) -> int:
         if config["type"] == "defaults":
             return 0
         if config["type"] == "global-defaults":
@@ -123,12 +134,13 @@ class JJB:
             return 40
         if config["type"] == "jobs":
             return 50
+        return 100
 
     @staticmethod
-    def sort_by_name(config):
+    def sort_by_name(config: Mapping[str, Any]) -> str:
         return config["name"]
 
-    def get_configs(self):
+    def get_configs(self) -> dict[str, str]:
         """This function gets the configs from the
         working directories"""
         configs = {}
@@ -139,7 +151,7 @@ class JJB:
 
         return configs
 
-    def generate(self, io_dir, fetch_state):
+    def generate(self, io_dir: str, fetch_state: str) -> None:
         """
         Generates job definitions from JJB configs
 
@@ -163,7 +175,7 @@ class JJB:
             self.execute(args)
             throughput.change_files_ownership(io_dir)
 
-    def print_diffs(self, io_dir, instance_name=None):
+    def print_diffs(self, io_dir: str, instance_name: str | None = None) -> None:
         """Print the diffs between the current and
         the desired job definitions"""
         current_path = path.join(io_dir, "jjb", "current")
@@ -179,7 +191,7 @@ class JJB:
         self.print_diff(delete, current_path, "delete")
         self.print_diff(common, desired_path, "update")
 
-    def print_diff(self, files, replace_path, action):
+    def print_diff(self, files: Iterable[str], replace_path: str, action: str) -> None:
         for f in files:
             if action == "update":
                 ft = self.toggle_cd(f)
@@ -210,11 +222,16 @@ class JJB:
                     ]
                     logging.debug("DIFF:\n" + "".join(diff))
 
-    def compare_files(self, from_files, subtract_files, in_op=False):
+    def compare_files(
+        self,
+        from_files: Iterable[str],
+        subtract_files: Iterable[str],
+        in_op: bool = False,
+    ) -> list[str]:
         return [f for f in from_files if (self.toggle_cd(f) in subtract_files) is in_op]
 
     @staticmethod
-    def get_files(search_path, instance_name=None):
+    def get_files(search_path: str, instance_name: str | None = None) -> list[str]:
         if instance_name is not None:
             search_path = path.join(search_path, instance_name)
         return [
@@ -222,7 +239,7 @@ class JJB:
         ]
 
     @staticmethod
-    def toggle_cd(file_name):
+    def toggle_cd(file_name: str) -> str:
         if "desired" in file_name:
             return file_name.replace("desired", "current")
         return file_name.replace("current", "desired")
@@ -248,28 +265,28 @@ class JJB:
                 raise
 
     @staticmethod
-    def get_jjb(args):
+    def get_jjb(args: Iterable[str]) -> Any:
         from jenkins_jobs.cli.entry import JenkinsJobs  # noqa: PLC0415
 
         return JenkinsJobs(args)
 
-    def execute(self, args):
+    def execute(self, args: Iterable[str]) -> None:
         jjb = self.get_jjb(args)
         with toggle_logger():
             jjb.execute()
 
-    def modify_logger(self):
+    def modify_logger(self) -> None:
         yaml.warnings({"YAMLLoadWarning": False})
         formatter = logging.Formatter("%(levelname)s: %(message)s")
         logger = logging.getLogger()
         logger.handlers[0].setFormatter(formatter)
 
-    def cleanup(self):
+    def cleanup(self) -> None:
         for wd in self.working_dirs.values():
             shutil.rmtree(wd)
 
     @retry(exceptions=(JenkinsJobsException))
-    def get_jobs(self, wd, name):
+    def get_jobs(self, wd: str, name: str) -> list[dict[str, Any]]:
         ini_path = f"{wd}/{name}.ini"
         config_path = f"{wd}/config.yaml"
 
@@ -283,8 +300,8 @@ class JJB:
 
         return jobs
 
-    def get_job_webhooks_data(self):
-        job_webhooks_data = {}
+    def get_job_webhooks_data(self) -> dict[str, list[dict[str, Any]]]:
+        job_webhooks_data: dict[str, list[dict[str, Any]]] = {}
         for name, wd in self.working_dirs.items():
             jobs = self.get_jobs(wd, name)
 
@@ -313,7 +330,7 @@ class JJB:
 
         return job_webhooks_data
 
-    def get_repos(self):
+    def get_repos(self) -> set[str]:
         repos = set()
         for name, wd in self.working_dirs.items():
             jobs = self.get_jobs(wd, name)
@@ -325,7 +342,7 @@ class JJB:
                     logging.debug(f"missing github url: {job_name}")
         return repos
 
-    def get_admins(self):
+    def get_admins(self) -> set[str]:
         admins = set()
         for name, wd in self.working_dirs.items():
             jobs = self.get_jobs(wd, name)
@@ -340,15 +357,17 @@ class JJB:
         return admins
 
     @staticmethod
-    def get_repo_url(job):
+    def get_repo_url(job: Mapping[str, Any]) -> str:
         repo_url_raw = job["properties"][0]["github"]["url"]
         return repo_url_raw.strip("/").replace(".git", "")
 
     @staticmethod
-    def get_ref(job: dict) -> str:
+    def get_ref(job: Mapping[str, Any]) -> str:
         return job["scm"][0]["git"]["branches"][0]
 
-    def get_all_jobs(self, job_types=None, instance_name=None) -> dict[str, list[dict]]:
+    def get_all_jobs(
+        self, job_types: Iterable[str] | None = None, instance_name: str | None = None
+    ) -> dict[str, list[dict[str, Any]]]:
         if job_types is None:
             job_types = []
         all_jobs: dict[str, list[dict]] = {}
@@ -366,8 +385,8 @@ class JJB:
 
         return all_jobs
 
-    def print_jobs(self, job_name=None):
-        all_jobs = {}
+    def print_jobs(self, job_name: str | None = None) -> None:
+        all_jobs: dict[str, list[dict[str, Any]]] = {}
         found = False
         for name, wd in self.working_dirs.items():
             logging.debug(f"getting jobs from {name}")
@@ -394,7 +413,7 @@ class JJB:
         raise ValueError(f"job with {job_type=} and {repo_url=} not found")
 
     @staticmethod
-    def get_trigger_phrases_regex(job: dict) -> str | None:
+    def get_trigger_phrases_regex(job: Mapping[str, Any]) -> str | None:
         for trigger in job["triggers"]:
             if "gitlab" in trigger:
                 return trigger["gitlab"].get("note-regex")
@@ -403,7 +422,7 @@ class JJB:
         return None
 
     @staticmethod
-    def get_gitlab_webhook_trigger(job: dict) -> list[str]:
+    def get_gitlab_webhook_trigger(job: Mapping[str, Any]) -> list[str]:
         gitlab_triggers = job["triggers"][0]["gitlab"]
         # pr-check job should be triggered by merge request events
         # and certain comments: [test]|/retest|/lgtm|/lgtm cancel|/hold|/hold cancel

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/reconcile/utils/ldap_client.py

@@ -1,5 +1,6 @@
 from collections import defaultdict
 from collections.abc import Iterable
+from typing import Any, Self
 
 from ldap3 import (
     ALL,
@@ -17,15 +18,15 @@ class LdapClient:
     appropriately.
     """
 
-    def __init__(self, base_dn: str, connection: Connection):
+    def __init__(self, base_dn: str, connection: Connection) -> None:
         self.base_dn = base_dn
         self.connection = connection
 
-    def __enter__(self):
+    def __enter__(self) -> Self:
         self.connection.bind()
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
         self.connection.unbind()
 
     def get_users(self, uids: Iterable[str]) -> set[str]:

{qontract_reconcile-0.10.2.dev317 → qontract_reconcile-0.10.2.dev319}/reconcile/utils/lean_terraform_client.py

@@ -6,7 +6,9 @@ from collections.abc import Mapping
 from typing import Any
 
 
-def state_rm_access_key(working_dirs, account, user):
+def state_rm_access_key(
+    working_dirs: Mapping[str, str], account: str, user: str
+) -> bool:
     wd = working_dirs[account]
     init_result = subprocess.run(["terraform", "init"], check=False, cwd=wd)
     if init_result.returncode != 0: