qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/gql_definitions/ocp_release_mirror/ocp_release_mirror.py

@@ -1,287 +0,0 @@
-"""
-Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
-"""
-from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
-from datetime import datetime  # noqa: F401 # pylint: disable=W0611
-from enum import Enum  # noqa: F401 # pylint: disable=W0611
-from typing import (  # noqa: F401 # pylint: disable=W0611
-    Any,
-    Optional,
-    Union,
-)
-
-from pydantic import (  # noqa: F401 # pylint: disable=W0611
-    BaseModel,
-    Extra,
-    Field,
-    Json,
-)
-
-from reconcile.gql_definitions.fragments.jumphost_common_fields import (
-    CommonJumphostFields,
-)
-from reconcile.gql_definitions.fragments.ocm_environment import OCMEnvironment
-from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
-
-
-DEFINITION = """
-fragment CommonJumphostFields on ClusterJumpHost_v1 {
-  hostname
-  knownHosts
-  user
-  port
-  remotePort
-  identity {
-    ... VaultSecret
-  }
-}
-
-fragment OCMEnvironment on OpenShiftClusterManagerEnvironment_v1 {
-    name
-    url
-    accessTokenClientId
-    accessTokenUrl
-    accessTokenClientSecret {
-        ... VaultSecret
-    }
-}
-
-fragment VaultSecret on VaultSecret_v1 {
-    path
-    field
-    version
-    format
-}
-
-query OCPReleaseMirror {
-  ocp_release_mirror: ocp_release_mirror_v1 {
-    hiveCluster {
-      name
-      serverUrl
-      insecureSkipTLSVerify
-      jumpHost {
-        ... CommonJumphostFields
-      }
-      managedGroups
-      ocm {
-        name
-        environment {
-          ... OCMEnvironment
-        }
-        orgId
-        accessTokenClientId
-        accessTokenUrl
-        accessTokenClientSecret {
-          ... VaultSecret
-        }
-      }
-      automationToken {
-        ... VaultSecret
-      }
-      clusterAdmin
-      clusterAdminAutomationToken {
-        ... VaultSecret
-      }
-      internal
-      disable {
-        integrations
-        e2eTests
-      }
-      auth {
-        service
-        ... on ClusterAuthGithubOrg_v1 {
-          org
-        }
-        ... on ClusterAuthGithubOrgTeam_v1 {
-          org
-          team
-        }
-        # ... on ClusterAuthOIDC_v1 {
-        # }
-      }
-    }
-    ecrResourcesNamespace {
-      name
-      managedExternalResources
-      externalResources {
-        provider
-        provisioner {
-          name
-        }
-        ... on NamespaceTerraformProviderResourceAWS_v1 {
-          resources {
-            provider
-            ... on NamespaceTerraformResourceECR_v1
-            {
-              region
-              identifier
-              output_resource_name
-            }
-          }
-        }
-      }
-      cluster
-      {
-        name
-        serverUrl
-        automationToken
-        {
-          ... VaultSecret
-        }
-        internal
-      }
-    }
-    quayTargetOrgs {
-      name
-      instance {
-        name
-      }
-    }
-    ocpReleaseEcrIdentifier
-    ocpArtDevEcrIdentifier
-    mirrorChannels
-  }
-}
-"""
-
-
-class ConfiguredBaseModel(BaseModel):
-    class Config:
-        smart_union = True
-        extra = Extra.forbid
-
-
-class OpenShiftClusterManagerV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    environment: OCMEnvironment = Field(..., alias="environment")
-    org_id: str = Field(..., alias="orgId")
-    access_token_client_id: Optional[str] = Field(..., alias="accessTokenClientId")
-    access_token_url: Optional[str] = Field(..., alias="accessTokenUrl")
-    access_token_client_secret: Optional[VaultSecret] = Field(
-        ..., alias="accessTokenClientSecret"
-    )
-
-
-class DisableClusterAutomationsV1(ConfiguredBaseModel):
-    integrations: Optional[list[str]] = Field(..., alias="integrations")
-    e2e_tests: Optional[list[str]] = Field(..., alias="e2eTests")
-
-
-class ClusterAuthV1(ConfiguredBaseModel):
-    service: str = Field(..., alias="service")
-
-
-class ClusterAuthGithubOrgV1(ClusterAuthV1):
-    org: str = Field(..., alias="org")
-
-
-class ClusterAuthGithubOrgTeamV1(ClusterAuthV1):
-    org: str = Field(..., alias="org")
-    team: str = Field(..., alias="team")
-
-
-class ClusterV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    server_url: str = Field(..., alias="serverUrl")
-    insecure_skip_tls_verify: Optional[bool] = Field(..., alias="insecureSkipTLSVerify")
-    jump_host: Optional[CommonJumphostFields] = Field(..., alias="jumpHost")
-    managed_groups: Optional[list[str]] = Field(..., alias="managedGroups")
-    ocm: Optional[OpenShiftClusterManagerV1] = Field(..., alias="ocm")
-    automation_token: Optional[VaultSecret] = Field(..., alias="automationToken")
-    cluster_admin: Optional[bool] = Field(..., alias="clusterAdmin")
-    cluster_admin_automation_token: Optional[VaultSecret] = Field(
-        ..., alias="clusterAdminAutomationToken"
-    )
-    internal: Optional[bool] = Field(..., alias="internal")
-    disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable")
-    auth: list[
-        Union[ClusterAuthGithubOrgTeamV1, ClusterAuthGithubOrgV1, ClusterAuthV1]
-    ] = Field(..., alias="auth")
-
-
-class ExternalResourcesProvisionerV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-
-
-class NamespaceExternalResourceV1(ConfiguredBaseModel):
-    provider: str = Field(..., alias="provider")
-    provisioner: ExternalResourcesProvisionerV1 = Field(..., alias="provisioner")
-
-
-class NamespaceTerraformResourceAWSV1(ConfiguredBaseModel):
-    provider: str = Field(..., alias="provider")
-
-
-class NamespaceTerraformResourceECRV1(NamespaceTerraformResourceAWSV1):
-    region: Optional[str] = Field(..., alias="region")
-    identifier: str = Field(..., alias="identifier")
-    output_resource_name: Optional[str] = Field(..., alias="output_resource_name")
-
-
-class NamespaceTerraformProviderResourceAWSV1(NamespaceExternalResourceV1):
-    resources: list[
-        Union[NamespaceTerraformResourceECRV1, NamespaceTerraformResourceAWSV1]
-    ] = Field(..., alias="resources")
-
-
-class NamespaceV1_ClusterV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    server_url: str = Field(..., alias="serverUrl")
-    automation_token: Optional[VaultSecret] = Field(..., alias="automationToken")
-    internal: Optional[bool] = Field(..., alias="internal")
-
-
-class NamespaceV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    managed_external_resources: Optional[bool] = Field(
-        ..., alias="managedExternalResources"
-    )
-    external_resources: Optional[
-        list[
-            Union[NamespaceTerraformProviderResourceAWSV1, NamespaceExternalResourceV1]
-        ]
-    ] = Field(..., alias="externalResources")
-    cluster: NamespaceV1_ClusterV1 = Field(..., alias="cluster")
-
-
-class QuayInstanceV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-
-
-class QuayOrgV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    instance: QuayInstanceV1 = Field(..., alias="instance")
-
-
-class OcpReleaseMirrorV1(ConfiguredBaseModel):
-    hive_cluster: ClusterV1 = Field(..., alias="hiveCluster")
-    ecr_resources_namespace: NamespaceV1 = Field(..., alias="ecrResourcesNamespace")
-    quay_target_orgs: list[QuayOrgV1] = Field(..., alias="quayTargetOrgs")
-    ocp_release_ecr_identifier: str = Field(..., alias="ocpReleaseEcrIdentifier")
-    ocp_art_dev_ecr_identifier: str = Field(..., alias="ocpArtDevEcrIdentifier")
-    mirror_channels: list[str] = Field(..., alias="mirrorChannels")
-
-
-class OCPReleaseMirrorQueryData(ConfiguredBaseModel):
-    ocp_release_mirror: Optional[list[OcpReleaseMirrorV1]] = Field(
-        ..., alias="ocp_release_mirror"
-    )
-
-
-def query(query_func: Callable, **kwargs: Any) -> OCPReleaseMirrorQueryData:
-    """
-    This is a convenience function which queries and parses the data into
-    concrete types. It should be compatible with most GQL clients.
-    You do not have to use it to consume the generated data classes.
-    Alternatively, you can also mime and alternate the behavior
-    of this function in the caller.
-
-    Parameters:
-        query_func (Callable): Function which queries your GQL Server
-        kwargs: optional arguments that will be passed to the query function
-
-    Returns:
-        OCPReleaseMirrorQueryData: queried data parsed into generated classes
-    """
-    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
-    return OCPReleaseMirrorQueryData(**raw_data)

reconcile/integrations_validator.py

@@ -1,18 +0,0 @@
-import logging
-import sys
-
-from reconcile import queries
-
-QONTRACT_INTEGRATION = "integrations-validator"
-
-
-def run(dry_run, integration_commands):
-    desired_integrations = [i["name"] for i in queries.get_integrations()]
-
-    missing = set(integration_commands) - set(desired_integrations)
-
-    for integration in missing:
-        logging.error(["missing_integration", integration])
-
-    if missing:
-        sys.exit(1)

reconcile/jenkins_plugins.py

@@ -1,129 +0,0 @@
-import logging
-from collections.abc import Mapping
-from typing import Any
-
-from reconcile import queries
-from reconcile.utils import gql
-from reconcile.utils.jenkins_api import JenkinsApi
-from reconcile.utils.secret_reader import SecretReader
-
-INSTANCES_QUERY = """
-{
-  instances: jenkins_instances_v1 {
-    name
-    token {
-      path
-      field
-      version
-      format
-    }
-    plugins
-  }
-}
-"""
-
-QONTRACT_INTEGRATION = "jenkins-plugins"
-
-
-def get_jenkins_map(
-    plugins_only=False, desired_instances=None
-) -> dict[str, JenkinsApi]:
-    gqlapi = gql.get_api()
-    jenkins_instances = gqlapi.query(INSTANCES_QUERY)["instances"]
-    secret_reader = SecretReader(queries.get_secret_reader_settings())
-
-    jenkins_map = {}
-    for instance in jenkins_instances:
-        instance_name = instance["name"]
-        if desired_instances and instance_name not in desired_instances:
-            continue
-        if instance_name in jenkins_map:
-            continue
-        if plugins_only and not instance["plugins"]:
-            continue
-
-        token = instance["token"]
-        jenkins = JenkinsApi.init_jenkins_from_secret(secret_reader, token)
-        jenkins_map[instance_name] = jenkins
-
-    return jenkins_map
-
-
-def get_current_state(jenkins_map: Mapping[str, JenkinsApi]) -> list[dict[str, Any]]:
-    current_state = []
-
-    for instance, jenkins in jenkins_map.items():
-        plugins = jenkins.list_plugins()
-        for plugin in plugins:
-            current_state.append({"instance": instance, "plugin": plugin["shortName"]})
-
-    return current_state
-
-
-def get_desired_state():
-    gqlapi = gql.get_api()
-    jenkins_instances = gqlapi.query(INSTANCES_QUERY)["instances"]
-
-    desired_state = []
-    for instance in jenkins_instances:
-        for plugin in instance["plugins"] or []:
-            desired_state.append({"instance": instance["name"], "plugin": plugin})
-
-    return desired_state
-
-
-def calculate_diff(current_state, desired_state):
-    diff = []
-    plugins_to_install = subtract_states(desired_state, current_state, "install_plugin")
-    diff.extend(plugins_to_install)
-
-    return diff
-
-
-def subtract_states(from_state, subtract_state, action):
-    result = []
-
-    for f_plugin in from_state:
-        found = False
-        for s_plugin in subtract_state:
-            if f_plugin != s_plugin:
-                continue
-            found = True
-            break
-        if not found:
-            result.append(
-                {
-                    "action": action,
-                    "instance": f_plugin["instance"],
-                    "plugin": f_plugin["plugin"],
-                }
-            )
-
-    return result
-
-
-def act(diff, jenkins_map):
-    instance = diff["instance"]
-    plugin = diff["plugin"]
-    action = diff["action"]
-
-    if action == "install_plugin":
-        jenkins_map[instance].install_plugin(plugin)
-    else:
-        raise Exception("invalid action: {}".format(action))
-
-
-def run(dry_run):
-    jenkins_map = get_jenkins_map(plugins_only=True)
-    current_state = get_current_state(jenkins_map)
-    desired_state = get_desired_state()
-    diffs = calculate_diff(current_state, desired_state)
-
-    for diff in diffs:
-        logging.info(list(diff.values()))
-
-        if not dry_run:
-            act(diff, jenkins_map)
-
-    for instance in jenkins_map.values():
-        instance.safe_restart()

reconcile/kafka_clusters.py

@@ -1,208 +0,0 @@
-import base64
-import copy
-import logging
-import sys
-
-import reconcile.openshift_base as ob
-from reconcile import queries
-from reconcile.status import ExitCodes
-from reconcile.utils.defer import defer
-from reconcile.utils.ocm import (
-    STATUS_FAILED,
-    STATUS_READY,
-    OCMMap,
-)
-from reconcile.utils.openshift_resource import OpenshiftResource as OR
-from reconcile.utils.semver_helper import make_semver
-from reconcile.utils.vault import VaultClient
-
-QONTRACT_INTEGRATION = "kafka-clusters"
-QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
-
-
-def construct_oc_resource(data):
-    body = {
-        "apiVersion": "v1",
-        "kind": "Secret",
-        "type": "Opaque",
-        "metadata": {"name": "kafka", "annotations": {"qontract.recycle": "true"}},
-        "data": {
-            k: base64.b64encode(v.encode()).decode("utf-8") for k, v in data.items()
-        },
-    }
-    return OR(body, QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION)
-
-
-def fetch_desired_state(clusters):
-    desired_state = []
-    for cluster_info in clusters:
-        item = {
-            "name": cluster_info["name"],
-            "cloud_provider": cluster_info["spec"]["provider"],
-            "region": cluster_info["spec"]["region"],
-            "multi_az": cluster_info["spec"]["multi_az"],
-        }
-        desired_state.append(item)
-    return desired_state
-
-
-def get_kafa_service_account(
-    kafka_service_accounts, kafka_cluster_name, vault_throughput_path, dry_run, ocm_map
-):
-    """
-    get a service account for the cluster
-    we match cluster to service account by name
-    """
-    service_accounts = [
-        sa for sa in kafka_service_accounts if sa["name"] == kafka_cluster_name
-    ]
-    if service_accounts:
-        result_sa = copy.deepcopy(service_accounts[0])
-        # since this is an existing service account
-        # we do not get it's client_secret. read it from vault
-        cs_key = "client_secret"
-        result_sa[cs_key] = read_input_from_vault(
-            vault_throughput_path, kafka_cluster_name, cs_key
-        )
-        # the name was only needed for matching
-        result_sa.pop("name", None)
-    else:
-        result_sa = {}
-        logging.info(["create_service_account", kafka_cluster_name])
-        if not dry_run:
-            ocm = ocm_map.get(kafka_cluster_name)
-            sa_fields = ["client_id", "client_secret"]
-            result_sa = ocm.create_kafka_service_account(
-                kafka_cluster_name, fields=sa_fields
-            )
-
-    return result_sa
-
-
-def read_input_from_vault(vault_path, name, field):
-    integration_name = QONTRACT_INTEGRATION
-    vault_client = VaultClient()
-    secret_path = f"{vault_path}/{integration_name}/{name}"
-    secret = {"path": secret_path, "field": field}
-    return vault_client.read(secret)
-
-
-def write_output_to_vault(vault_path, name, data):
-    integration_name = QONTRACT_INTEGRATION
-    vault_client = VaultClient()
-    secret_path = f"{vault_path}/{integration_name}/{name}"
-    secret = {"path": secret_path, "data": data}
-    vault_client.write(secret)
-
-
-@defer
-def run(
-    dry_run,
-    thread_pool_size=10,
-    internal=None,
-    use_jump_host=True,
-    vault_throughput_path=None,
-    defer=None,
-):
-    if not vault_throughput_path:
-        logging.error("must supply vault throughput path")
-        sys.exit(ExitCodes.ERROR)
-
-    kafka_clusters = queries.get_kafka_clusters()
-    if not kafka_clusters:
-        logging.debug("No Kafka clusters found in app-interface")
-        sys.exit(ExitCodes.SUCCESS)
-
-    settings = queries.get_app_interface_settings()
-    ocm_map = OCMMap(
-        clusters=kafka_clusters, integration=QONTRACT_INTEGRATION, settings=settings
-    )
-    namespaces = []
-    for kafka_cluster in kafka_clusters:
-        namespaces.extend(kafka_cluster["namespaces"])
-    ri, oc_map = ob.fetch_current_state(
-        namespaces=namespaces,
-        thread_pool_size=thread_pool_size,
-        integration=QONTRACT_INTEGRATION,
-        integration_version=QONTRACT_INTEGRATION_VERSION,
-        override_managed_types=["Secret"],
-        internal=internal,
-        use_jump_host=use_jump_host,
-    )
-    defer(oc_map.cleanup)
-
-    current_state = ocm_map.kafka_cluster_specs()
-    desired_state = fetch_desired_state(kafka_clusters)
-    kafka_service_accounts = ocm_map.kafka_service_account_specs()
-
-    for kafka_cluster in kafka_clusters:
-        kafka_cluster_name = kafka_cluster["name"]
-        desired_cluster = [c for c in desired_state if kafka_cluster_name == c["name"]][
-            0
-        ]
-        current_cluster = [c for c in current_state if kafka_cluster_name == c["name"]]
-        # check if cluster exists. if not - create it
-        if not current_cluster:
-            logging.info(["create_cluster", kafka_cluster_name])
-            if not dry_run:
-                ocm = ocm_map.get(kafka_cluster_name)
-                ocm.create_kafka_cluster(desired_cluster)
-            continue
-        # there should only be one cluster
-        current_cluster = current_cluster[0]
-        # check if desired cluster matches current cluster. if not - error
-        if not all(k in current_cluster.keys() for k in desired_cluster.keys()):
-            logging.error(
-                "[%s] desired spec %s is different " + "from current spec %s",
-                kafka_cluster_name,
-                desired_cluster,
-                current_cluster,
-            )
-            ri.register_error()
-            continue
-        # check if cluster is ready. if not - wait
-        status = current_cluster["status"]
-        if status != STATUS_READY:
-            # check if cluster is failed
-            if status == STATUS_FAILED:
-                failed_reason = current_cluster["failed_reason"]
-                logging.error(
-                    f"[{kafka_cluster_name}] cluster status is {status}. "
-                    f"reason: {failed_reason}"
-                )
-                ri.register_error()
-            else:
-                logging.warning(f"[{kafka_cluster_name}] cluster status is {status}")
-            continue
-        # we have a ready cluster!
-        # get a service account for the cluster
-        kafka_service_account = get_kafa_service_account(
-            kafka_service_accounts,
-            kafka_cluster_name,
-            vault_throughput_path,
-            dry_run,
-            ocm_map,
-        )
-        # let's create a Secret in all referencing namespaces
-        kafka_namespaces = kafka_cluster["namespaces"]
-        secret_fields = ["bootstrap_server_host"]
-        data = {k: v for k, v in current_cluster.items() if k in secret_fields}
-        data.update(kafka_service_account)
-        resource = construct_oc_resource(data)
-        for namespace_info in kafka_namespaces:
-            ri.add_desired(
-                namespace_info["cluster"]["name"],
-                namespace_info["name"],
-                resource.kind,
-                resource.name,
-                resource,
-            )
-        if not dry_run:
-            write_output_to_vault(
-                vault_throughput_path, kafka_cluster_name, resource.body["data"]
-            )
-
-    ob.realize_data(dry_run, oc_map, ri, thread_pool_size)
-
-    if ri.has_error_registered():
-        sys.exit(ExitCodes.ERROR)

reconcile/ocm_cluster_admin.py

@@ -1,42 +0,0 @@
-import logging
-import sys
-from typing import (
-    Any,
-    Mapping,
-)
-
-from reconcile import queries
-from reconcile.status import ExitCodes
-from reconcile.utils.disabled_integrations import integration_is_enabled
-from reconcile.utils.ocm import OCMMap
-
-QONTRACT_INTEGRATION = "ocm-cluster-admin"
-
-
-def _cluster_is_compatible(cluster: Mapping[str, Any]) -> bool:
-    return bool(cluster.get("ocm") is not None and cluster.get("clusterAdmin"))
-
-
-def run(dry_run: bool) -> None:
-    settings = queries.get_app_interface_settings()
-    clusters = queries.get_clusters()
-    clusters = [
-        c
-        for c in clusters
-        if integration_is_enabled(QONTRACT_INTEGRATION, c) and _cluster_is_compatible(c)
-    ]
-
-    if not clusters:
-        logging.debug("No cluster admin definitions found in app-interface")
-        sys.exit(ExitCodes.SUCCESS)
-
-    ocm_map = OCMMap(
-        clusters=clusters, integration=QONTRACT_INTEGRATION, settings=settings
-    )
-    for cluster in clusters:
-        cluster_name = cluster["name"]
-        ocm = ocm_map.get(cluster_name)
-        if not ocm.is_cluster_admin_enabled(cluster_name):
-            logging.info(["enable_cluster_admin", cluster_name])
-            if not dry_run:
-                ocm.enable_cluster_admin(cluster_name)