qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/jenkins_job_builder.py

@@ -1,13 +1,9 @@
 import logging
 import sys
 from collections.abc import Callable
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 from reconcile import queries
-from reconcile.utils import gql
 from reconcile.utils.defer import defer
 from reconcile.utils.jjb_client import JJB
 from reconcile.utils.secret_reader import (
@@ -16,48 +12,14 @@ from reconcile.utils.secret_reader import (
 )
 from reconcile.utils.state import init_state
 
-QUERY = """
-{
-  jenkins_configs: jenkins_configs_v1 {
-    name
-    instance {
-      name
-      serverUrl
-      token {
-        path
-        field
-        version
-        format
-      }
-      deleteMethod
-    }
-    type
-    config
-    config_path {
-      content
-    }
-  }
-}
-"""
-
 QONTRACT_INTEGRATION = "jenkins-job-builder"
 GENERATE_TYPE = ["jobs", "views"]
 
 
-def get_openshift_saas_deploy_job_name(saas_file_name, env_name, settings):
-    job_template_name = settings["saasDeployJobTemplate"]
-    return f"{job_template_name}-{saas_file_name}-{env_name}"
-
-
-def get_jenkins_configs():
-    gqlapi = gql.get_api()
-    return gqlapi.query(QUERY)["jenkins_configs"]
-
-
 def collect_configs(
-    instance_name: Optional[str], config_name: Optional[str]
+    instance_name: str | None, config_name: str | None
 ) -> list[dict[str, Any]]:
-    configs = get_jenkins_configs()
+    configs = queries.get_jenkins_configs()
     if instance_name is not None:
         configs = [n for n in configs if n["instance"]["name"] == instance_name]
         if not configs:
@@ -76,8 +38,8 @@ def collect_configs(
 
 def init_jjb(
     secret_reader: SecretReaderBase,
-    instance_name: Optional[str] = None,
-    config_name: Optional[str] = None,
+    instance_name: str | None = None,
+    config_name: str | None = None,
     print_only: bool = False,
 ) -> JJB:
     configs = collect_configs(instance_name, config_name)
@@ -101,7 +63,7 @@ def validate_repos_and_admins(jjb: JJB):
     )
     unknown_admins = [a for a in jjb_admins if a not in github_usernames]
     for a in unknown_admins:
-        logging.warning("admin is missing from users: {}".format(a))
+        logging.warning(f"admin is missing from users: {a}")
     if missing_repos:
         sys.exit(1)
 
@@ -111,10 +73,10 @@ def run(
     dry_run: bool,
     io_dir: str = "throughput/",
     print_only: bool = False,
-    config_name: Optional[str] = None,
-    job_name: Optional[str] = None,
-    instance_name: Optional[str] = None,
-    defer: Optional[Callable] = None,
+    config_name: str | None = None,
+    job_name: str | None = None,
+    instance_name: str | None = None,
+    defer: Callable | None = None,
 ) -> None:
     if not print_only and config_name is not None:
         raise Exception("--config-name must works with --print-only mode")

reconcile/jenkins_job_builds_cleaner.py

@@ -19,15 +19,13 @@ def delete_builds(jenkins, builds_todel, dry_run=True):
         job_name = build["job_name"]
         build_id = build["build_id"]
         progress_str = f"{idx}/{delete_builds_count}"
-        logging.debug(
-            [
-                progress_str,
-                job_name,
-                build["rule_name"],
-                build["rule_keep_hours"],
-                build_id,
-            ]
-        )
+        logging.debug([
+            progress_str,
+            job_name,
+            build["rule_name"],
+            build["rule_keep_hours"],
+            build_id,
+        ])
         if not dry_run:
             try:
                 jenkins.delete_build(build["job_name"], build["build_id"])
@@ -36,6 +34,20 @@ def delete_builds(jenkins, builds_todel, dry_run=True):
                 logging.exception(msg)
 
 
+def get_last_build_ids(builds):
+    builds_to_keep = []
+    sorted_builds = sorted(builds, key=lambda b: b["timestamp"], reverse=True)
+    if sorted_builds:
+        last_build = sorted_builds[0]
+        builds_to_keep.append(last_build["id"])
+
+    for build in sorted_builds:
+        if build["result"] == "SUCCESS":
+            builds_to_keep.append(build["id"])
+            break
+    return builds_to_keep
+
+
 def find_builds(jenkins, job_names, rules):
     # Current time in ms
     time_ms = time.time() * 1000
@@ -45,16 +57,21 @@ def find_builds(jenkins, job_names, rules):
         for rule in rules:
             if rule["name_re"].search(job_name):
                 builds = jenkins.get_builds(job_name)
+                # We need to keep very last and last successful builds (https://issues.redhat.com/browse/APPSRE-8701)
+                builds_to_keep = get_last_build_ids(builds)
                 for build in builds:
-                    if time_ms - rule["keep_ms"] > build["timestamp"]:
-                        builds_found.append(
-                            {
-                                "job_name": job_name,
-                                "rule_name": rule["name"],
-                                "rule_keep_hours": rule["keep_hours"],
-                                "build_id": build["id"],
-                            }
+                    if build["id"] in builds_to_keep:
+                        logging.debug(
+                            f"{jenkins.url}: {job_name} build: {build['id']} will be kept"
                         )
+                        continue
+                    if time_ms - rule["keep_ms"] > build["timestamp"]:
+                        builds_found.append({
+                            "job_name": job_name,
+                            "rule_name": rule["name"],
+                            "rule_keep_hours": rule["keep_hours"],
+                            "build_id": build["id"],
+                        })
                 # Only act on the first rule matched
                 break
     return builds_found
@@ -73,14 +90,12 @@ def run(dry_run):
         # Process cleanup rules, pre-compile as regexes
         cleanup_rules = []
         for rule in instance_cleanup_rules:
-            cleanup_rules.append(
-                {
-                    "name": rule["name"],
-                    "name_re": re.compile(rule["name"]),
-                    "keep_hours": rule["keep_hours"],
-                    "keep_ms": hours_to_ms(rule["keep_hours"]),
-                }
-            )
+            cleanup_rules.append({
+                "name": rule["name"],
+                "name_re": re.compile(rule["name"]),
+                "keep_hours": rule["keep_hours"],
+                "keep_ms": hours_to_ms(rule["keep_hours"]),
+            })
 
         token = instance["token"]
         instance_name = instance["name"]

reconcile/jenkins_job_cleaner.py

@@ -20,9 +20,7 @@ def get_managed_job_names(job_names, managed_projects):
 
 def get_desired_job_names(instance_name: str, secret_reader: SecretReader):
     jjb = init_jjb(secret_reader)
-    desired_jobs = jjb.get_all_jobs(instance_name=instance_name, include_test=True)[
-        instance_name
-    ]
+    desired_jobs = jjb.get_all_jobs(instance_name=instance_name)[instance_name]
     return [j["name"] for j in desired_jobs]
 
 

reconcile/jenkins_roles.py

@@ -86,9 +86,11 @@ def get_current_state(jenkins_map):
                 continue
 
             for user in users:
-                current_state.append(
-                    {"instance": instance, "role": role_name, "user": user}
-                )
+                current_state.append({
+                    "instance": instance,
+                    "role": role_name,
+                    "user": user,
+                })
 
     return current_state
 
@@ -104,24 +106,20 @@ def get_desired_state():
                 continue
 
             for u in r["users"]:
-                desired_state.append(
-                    {
-                        "instance": p["instance"]["name"],
-                        "role": p["role"],
-                        "user": u["org_username"],
-                    }
-                )
+                desired_state.append({
+                    "instance": p["instance"]["name"],
+                    "role": p["role"],
+                    "user": u["org_username"],
+                })
             for u in r["bots"]:
                 if u["org_username"] is None:
                     continue
 
-                desired_state.append(
-                    {
-                        "instance": p["instance"]["name"],
-                        "role": p["role"],
-                        "user": u["org_username"],
-                    }
-                )
+                desired_state.append({
+                    "instance": p["instance"]["name"],
+                    "role": p["role"],
+                    "user": u["org_username"],
+                })
 
     return desired_state
 
@@ -151,14 +149,12 @@ def subtract_states(from_state, subtract_state, action):
             found = True
             break
         if not found:
-            result.append(
-                {
-                    "action": action,
-                    "instance": f_user["instance"],
-                    "role": f_user["role"],
-                    "user": f_user["user"],
-                }
-            )
+            result.append({
+                "action": action,
+                "instance": f_user["instance"],
+                "role": f_user["role"],
+                "user": f_user["user"],
+            })
 
     return result
 
@@ -174,7 +170,7 @@ def act(diff, jenkins_map):
     elif action == "unassign_role_from_user":
         jenkins_map[instance].unassign_role_from_user(role, user)
     else:
-        raise Exception("invalid action: {}".format(action))
+        raise Exception(f"invalid action: {action}")
 
 
 def run(dry_run):

reconcile/jenkins_webhooks.py

@@ -3,10 +3,7 @@ import logging
 from typing import Any
 
 from reconcile import queries
-from reconcile.jenkins_job_builder import (
-    get_jenkins_configs,
-    init_jjb,
-)
+from reconcile.jenkins_job_builder import init_jjb
 from reconcile.utils.defer import defer
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.jjb_client import JJB
@@ -27,7 +24,13 @@ def get_hooks_to_add(desired_state, gl):
             current_hooks = gl.get_project_hooks(project_url)
             for h in current_hooks:
                 job_url = h.url
-                trigger = "mr" if h.merge_requests_events else "push"
+                trigger = []
+                if h.merge_requests_events:
+                    trigger.append("mr")
+                if h.push_events:
+                    trigger.append("push")
+                if h.note_events:
+                    trigger.append("note")
                 item = {
                     "job_url": job_url.strip("/"),
                     "trigger": trigger,
@@ -62,5 +65,5 @@ def run(dry_run, defer=None):
 
 def early_exit_desired_state(*args, **kwargs) -> dict[str, Any]:
     return {
-        "jenkins_configs": get_jenkins_configs(),
+        "jenkins_configs": queries.get_jenkins_configs(),
     }

reconcile/jenkins_worker_fleets.py

@@ -5,7 +5,10 @@ from typing import (
 )
 
 from reconcile import queries
-from reconcile.jenkins.types import JenkinsWorkerFleet
+from reconcile.jenkins.types import (
+    JenkinsWorkerFleet,
+    SSHConnector,
+)
 from reconcile.utils.external_resources import get_external_resource_specs
 from reconcile.utils.jenkins_api import JenkinsApi
 from reconcile.utils.secret_reader import SecretReader
@@ -41,6 +44,10 @@ def get_desired_state(
         namespace = f["namespace"]
         account = f["account"]
         identifier = f["identifier"]
+
+        s = {k: v for k, v in f["sshConnector"].items() if v is not None}
+        ssh_connector = SSHConnector(**s)
+
         specs = get_external_resource_specs(namespace)
         found = False
         for spec in specs:
@@ -57,10 +64,8 @@ def get_desired_state(
                 f["region"] = region
                 f["minSize"] = values.get("min_size")
                 f["maxSize"] = values.get("max_size")
-                f["computerConnector"] = {
-                    "sSHConnector": {"credentialsId": f["credentialsId"]}
-                }
-                f = dict((k, v) for k, v in f.items() if v is not None)
+                f["computerConnector"] = {"sSHConnector": ssh_connector}
+                f = {k: v for k, v in f.items() if v is not None}
                 desired_state.append(JenkinsWorkerFleet(**f))
                 break
         if not found:
@@ -102,7 +107,7 @@ def act(
         if not dry_run:
             d_clouds = []
             for d in desired_state:
-                d_clouds.append({"eC2Fleet": d.dict(by_alias=True)})
+                d_clouds.append({"eC2Fleet": d.dict(by_alias=True, exclude_none=True)})
             config = {"jenkins": {"clouds": d_clouds}}
             jenkins.apply_jcasc_config(config)