qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/ecr_mirror.py

@@ -1,152 +0,0 @@
-import base64
-import logging
-
-from sretoolbox.container import (
-    Image,
-    Skopeo,
-)
-from sretoolbox.container.skopeo import SkopeoCmdError
-from sretoolbox.utils import threaded
-
-from reconcile import queries
-from reconcile.utils.aws_api import AWSApi
-from reconcile.utils.external_resource_spec import ExternalResourceSpec
-from reconcile.utils.external_resources import (
-    get_external_resource_specs,
-    managed_external_resources,
-)
-from reconcile.utils.secret_reader import SecretReader
-
-QONTRACT_INTEGRATION = "ecr-mirror"
-LOG = logging.getLogger(__name__)
-
-
-class EcrMirror:
-    def __init__(self, instance: ExternalResourceSpec, dry_run: bool):
-        self.dry_run = dry_run
-        self.instance = instance
-        self.settings = queries.get_app_interface_settings()
-        self.secret_reader = SecretReader(settings=self.settings)
-        self.skopeo_cli = Skopeo(dry_run)
-        self.error = False
-
-        identifier = instance.identifier
-        account = instance.provisioner_name
-        region = instance.resource.get("region")
-
-        self.aws_cli = AWSApi(
-            thread_pool_size=1,
-            accounts=[self._get_aws_account_info(account)],
-            settings=self.settings,
-            init_ecr_auth_tokens=True,
-        )
-
-        self.aws_cli.map_ecr_resources()
-
-        self.ecr_uri = self._get_image_uri(
-            account=account,
-            repository=identifier,
-        )
-        if self.ecr_uri is None:
-            self.error = True
-            LOG.error(f"Could not find the ECR repository {identifier}")
-
-        self.ecr_username, self.ecr_password = self._get_ecr_creds(
-            account=account,
-            region=region,
-        )
-        self.ecr_auth = f"{self.ecr_username}:{self.ecr_password}"
-
-        self.image_username = None
-        self.image_password = None
-        self.image_auth = None
-        pull_secret = self.instance.resource["mirror"]["pullCredentials"]
-        if pull_secret is not None:
-            raw_data = self.secret_reader.read_all(pull_secret)
-            self.image_username = raw_data["user"]
-            self.image_password = raw_data["token"]
-            self.image_auth = f"{self.image_username}:{self.image_password}"
-
-    def _cleanup(self):
-        self.aws_cli.cleanup()
-
-    def run(self):
-        if self.error:
-            self._cleanup()
-            return
-
-        ecr_mirror = Image(
-            self.ecr_uri, username=self.ecr_username, password=self.ecr_password
-        )
-
-        image = Image(
-            self.instance.resource["mirror"]["url"],
-            username=self.image_username,
-            password=self.image_password,
-        )
-
-        LOG.debug("[checking %s -> %s]", image, ecr_mirror)
-        for tag in image:
-            if tag not in ecr_mirror:
-                try:
-                    self.skopeo_cli.copy(
-                        src_image=image[tag],
-                        src_creds=self.image_auth,
-                        dst_image=ecr_mirror[tag],
-                        dest_creds=self.ecr_auth,
-                    )
-                except SkopeoCmdError as details:
-                    LOG.error("[%s]", details)
-        self._cleanup()
-
-    def _get_ecr_creds(self, account, region):
-        if region is None:
-            region = self.aws_cli.accounts[account]["resourcesDefaultRegion"]
-        auth_token = f"{account}/{region}"
-        data = self.aws_cli.auth_tokens[auth_token]
-        auth_data = data["authorizationData"][0]
-        token = auth_data["authorizationToken"]
-        password = base64.b64decode(token).decode("utf-8").split(":")[1]
-        return "AWS", password
-
-    def _get_image_uri(self, account, repository):
-        for repo in self.aws_cli.resources[account]["ecr"]:
-            if repo["repositoryName"] == repository:
-                return repo["repositoryUri"]
-
-    @staticmethod
-    def _get_aws_account_info(account):
-        for account_info in queries.get_aws_accounts():
-            if "name" not in account_info:
-                continue
-            if account_info["name"] != account:
-                continue
-            return account_info
-
-
-def worker(ecr_mirror_instance):
-    return ecr_mirror_instance.run()
-
-
-def run(dry_run, thread_pool_size=10):
-    namespaces = queries.get_namespaces()
-
-    tfrs_to_mirror = []
-    for namespace in namespaces:
-
-        if not managed_external_resources(namespace):
-            continue
-
-        for spec in get_external_resource_specs(namespace):
-            if spec.provider != "ecr":
-                continue
-
-            if spec.resource.get("mirror") is None:
-                continue
-
-            tfrs_to_mirror.append(spec)
-
-    work_list = threaded.run(
-        EcrMirror, tfrs_to_mirror, thread_pool_size=thread_pool_size, dry_run=dry_run
-    )
-    threaded.run(worker, work_list, thread_pool_size=thread_pool_size)

reconcile/github_scanner.py

@@ -1,74 +0,0 @@
-import logging
-
-from github.GithubException import UnknownObjectException
-from sretoolbox.utils import (
-    retry,
-    threaded,
-)
-
-import reconcile.aws_support_cases_sos as aws_sos
-from reconcile import queries
-from reconcile.github_users import init_github
-from reconcile.utils import git_secrets
-from reconcile.utils.aws_api import AWSApi
-
-QONTRACT_INTEGRATION = "github-scanner"
-
-
-def strip_repo_url(repo_url):
-    return repo_url.rstrip("/").replace(".git", "")
-
-
-@retry(max_attempts=6)
-def get_all_repos_to_scan(repos):
-    logging.info("getting full list of repos")
-    all_repos = []
-    all_repos.extend([strip_repo_url(r) for r in repos])
-    g = init_github()
-    for r in repos:
-        logging.debug("getting forks: {}".format(r))
-        repo_name = r.replace("https://github.com/", "")
-        try:
-            repo = g.get_repo(repo_name)
-            forks = repo.get_forks()
-            all_repos.extend([strip_repo_url(f.clone_url) for f in forks])
-        except UnknownObjectException:
-            logging.error("not found {}".format(r))
-
-    return all_repos
-
-
-def run(dry_run, gitlab_project_id=None, thread_pool_size=10):
-    accounts = queries.get_aws_accounts()
-    settings = queries.get_app_interface_settings()
-    with AWSApi(thread_pool_size, accounts, settings=settings) as aws:
-        existing_keys = aws.get_users_keys()
-    existing_keys_list = [
-        key
-        for user_key in existing_keys.values()
-        for keys in user_key.values()
-        for key in keys
-    ]
-    logging.info("found {} existing keys".format(len(existing_keys_list)))
-
-    app_int_github_repos = queries.get_repos(server="https://github.com")
-    all_repos = get_all_repos_to_scan(app_int_github_repos)
-    logging.info("about to scan {} repos".format(len(all_repos)))
-
-    results = threaded.run(
-        git_secrets.scan_history,
-        all_repos,
-        thread_pool_size,
-        existing_keys=existing_keys_list,
-    )
-    all_leaked_keys = [key for keys in results for key in keys]
-
-    deleted_keys = aws_sos.get_deleted_keys(accounts)
-    keys_to_delete = [
-        {"account": account, "key": key}
-        for key in all_leaked_keys
-        for account, user_keys in existing_keys.items()
-        if key in [uk for uks in user_keys.values() for uk in uks]
-        and key not in deleted_keys[account]
-    ]
-    aws_sos.act(dry_run, gitlab_project_id, accounts, keys_to_delete)

reconcile/gitlab_integrations.py

@@ -1,63 +0,0 @@
-import logging
-
-from sretoolbox.utils import retry
-
-from reconcile import queries
-from reconcile.utils.defer import defer
-from reconcile.utils.gitlab_api import GitLabApi
-from reconcile.utils.secret_reader import SecretReader
-
-QONTRACT_INTEGRATION = "gitlab-integrations"
-
-
-@retry()
-def get_repo_services(gl, repo_url):
-    project = gl.get_project(repo_url=repo_url)
-    return project.services
-
-
-@defer
-def run(dry_run, defer=None):
-    instance = queries.get_gitlab_instance()
-    settings = queries.get_app_interface_settings()
-    gl = GitLabApi(instance, settings=settings)
-    if defer:
-        defer(gl.cleanup)
-    secret_reader = SecretReader(settings=settings)
-
-    # Jira
-    repos = queries.get_repos_gitlab_jira(server=gl.server)
-    for repo in repos:
-        skip = False
-        repo_url = repo["url"]
-        services = get_repo_services(gl, repo_url)
-        current_jira = services.get("jira")
-
-        desired_jira = repo["jira"]
-        desired_jira_url = desired_jira["serverUrl"]
-        desired_jira_crdentials = secret_reader.read_all(desired_jira["token"])
-
-        if current_jira.active:
-            properties = current_jira.properties
-            desired_jira_username = desired_jira_crdentials["username"]
-            if (
-                properties["url"] == desired_jira_url
-                and properties["username"] == desired_jira_username
-            ):
-                skip = True
-
-        if skip:
-            continue
-
-        logging.info(["update_jira", repo_url, desired_jira_url])
-        if not dry_run:
-            new_data = {
-                "active": True,
-                "url": desired_jira_url,
-                "username": desired_jira_crdentials["username"],
-                "password": desired_jira_crdentials["password"],
-                "commit_events": True,
-                "merge_requests_events": True,
-                "comment_on_event_enabled": False,
-            }
-            services.update("jira", new_data=new_data)

reconcile/gql_definitions/ocm_oidc_idp/clusters.py

@@ -1,195 +0,0 @@
-"""
-Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
-"""
-from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
-from datetime import datetime  # noqa: F401 # pylint: disable=W0611
-from enum import Enum  # noqa: F401 # pylint: disable=W0611
-from typing import (  # noqa: F401 # pylint: disable=W0611
-    Any,
-    Optional,
-    Union,
-)
-
-from pydantic import (  # noqa: F401 # pylint: disable=W0611
-    BaseModel,
-    Extra,
-    Field,
-    Json,
-)
-
-from reconcile.gql_definitions.fragments.ocm_environment import OCMEnvironment
-from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
-
-
-DEFINITION = """
-fragment OCMEnvironment on OpenShiftClusterManagerEnvironment_v1 {
-    name
-    url
-    accessTokenClientId
-    accessTokenUrl
-    accessTokenClientSecret {
-        ... VaultSecret
-    }
-}
-
-fragment VaultSecret on VaultSecret_v1 {
-    path
-    field
-    version
-    format
-}
-
-query OidcClusters($name: String) {
-  clusters: clusters_v1(name: $name) {
-    name
-    ocm {
-      name
-      environment {
-        ... OCMEnvironment
-      }
-      orgId
-      accessTokenClientId
-      accessTokenUrl
-      accessTokenClientSecret {
-        ...VaultSecret
-      }
-      blockedVersions
-      sectors {
-        name
-        dependencies {
-          name
-          ocm {
-            name
-          }
-        }
-      }
-    }
-    upgradePolicy {
-      workloads
-      schedule
-      conditions {
-        sector
-      }
-    }
-    disable {
-      integrations
-    }
-    auth {
-      service
-      ... on ClusterAuthOIDC_v1 {
-        name
-        issuer
-        claims {
-          email
-          name
-          username
-          groups
-        }
-      }
-    }
-  }
-}
-"""
-
-
-class ConfiguredBaseModel(BaseModel):
-    class Config:
-        smart_union = True
-        extra = Extra.forbid
-
-
-class OpenShiftClusterManagerSectorDependenciesV1_OpenShiftClusterManagerV1(
-    ConfiguredBaseModel
-):
-    name: str = Field(..., alias="name")
-
-
-class OpenShiftClusterManagerSectorDependenciesV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    ocm: Optional[
-        OpenShiftClusterManagerSectorDependenciesV1_OpenShiftClusterManagerV1
-    ] = Field(..., alias="ocm")
-
-
-class OpenShiftClusterManagerSectorV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    dependencies: Optional[list[OpenShiftClusterManagerSectorDependenciesV1]] = Field(
-        ..., alias="dependencies"
-    )
-
-
-class OpenShiftClusterManagerV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    environment: OCMEnvironment = Field(..., alias="environment")
-    org_id: str = Field(..., alias="orgId")
-    access_token_client_id: Optional[str] = Field(..., alias="accessTokenClientId")
-    access_token_url: Optional[str] = Field(..., alias="accessTokenUrl")
-    access_token_client_secret: Optional[VaultSecret] = Field(
-        ..., alias="accessTokenClientSecret"
-    )
-    blocked_versions: Optional[list[str]] = Field(..., alias="blockedVersions")
-    sectors: Optional[list[OpenShiftClusterManagerSectorV1]] = Field(
-        ..., alias="sectors"
-    )
-
-
-class ClusterUpgradePolicyConditionsV1(ConfiguredBaseModel):
-    sector: Optional[str] = Field(..., alias="sector")
-
-
-class ClusterUpgradePolicyV1(ConfiguredBaseModel):
-    workloads: list[str] = Field(..., alias="workloads")
-    schedule: str = Field(..., alias="schedule")
-    conditions: ClusterUpgradePolicyConditionsV1 = Field(..., alias="conditions")
-
-
-class DisableClusterAutomationsV1(ConfiguredBaseModel):
-    integrations: Optional[list[str]] = Field(..., alias="integrations")
-
-
-class ClusterAuthV1(ConfiguredBaseModel):
-    service: str = Field(..., alias="service")
-
-
-class ClusterAuthOIDCClaimsV1(ConfiguredBaseModel):
-    email: Optional[list[str]] = Field(..., alias="email")
-    name: Optional[list[str]] = Field(..., alias="name")
-    username: Optional[list[str]] = Field(..., alias="username")
-    groups: Optional[list[str]] = Field(..., alias="groups")
-
-
-class ClusterAuthOIDCV1(ClusterAuthV1):
-    name: str = Field(..., alias="name")
-    issuer: Optional[str] = Field(..., alias="issuer")
-    claims: Optional[ClusterAuthOIDCClaimsV1] = Field(..., alias="claims")
-
-
-class ClusterV1(ConfiguredBaseModel):
-    name: str = Field(..., alias="name")
-    ocm: Optional[OpenShiftClusterManagerV1] = Field(..., alias="ocm")
-    upgrade_policy: Optional[ClusterUpgradePolicyV1] = Field(..., alias="upgradePolicy")
-    disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable")
-    auth: list[Union[ClusterAuthOIDCV1, ClusterAuthV1]] = Field(..., alias="auth")
-
-
-class OidcClustersQueryData(ConfiguredBaseModel):
-    clusters: Optional[list[ClusterV1]] = Field(..., alias="clusters")
-
-
-def query(query_func: Callable, **kwargs: Any) -> OidcClustersQueryData:
-    """
-    This is a convenience function which queries and parses the data into
-    concrete types. It should be compatible with most GQL clients.
-    You do not have to use it to consume the generated data classes.
-    Alternatively, you can also mime and alternate the behavior
-    of this function in the caller.
-
-    Parameters:
-        query_func (Callable): Function which queries your GQL Server
-        kwargs: optional arguments that will be passed to the query function
-
-    Returns:
-        OidcClustersQueryData: queried data parsed into generated classes
-    """
-    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
-    return OidcClustersQueryData(**raw_data)