qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/aws_version_sync/merge_request_manager/merge_request_manager.py

@@ -0,0 +1,160 @@
+import logging
+
+from gitlab.exceptions import GitlabGetError
+from pydantic import BaseModel
+
+from reconcile.aws_version_sync.merge_request_manager.merge_request import (
+    AVS_LABEL,
+    AVSInfo,
+    Parser,
+    Renderer,
+)
+from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.merge_request_manager.merge_request_manager import (
+    MergeRequestManagerBase,
+)
+from reconcile.utils.mr.base import MergeRequestBase
+from reconcile.utils.mr.labels import AUTO_MERGE
+from reconcile.utils.vcs import VCS
+
+
+class AVSMR(MergeRequestBase):
+    name = "AVS"
+
+    def __init__(
+        self, title: str, description: str, path: str, content: str, labels: list[str]
+    ):
+        super().__init__()
+        self._title = title
+        self._description = description
+        self._path = path
+        self._content = content
+        self.labels = labels
+
+    @property
+    def title(self) -> str:
+        return self._title
+
+    @property
+    def description(self) -> str:
+        return self._description
+
+    def process(self, gitlab_cli: GitLabApi) -> None:
+        gitlab_cli.update_file(
+            branch_name=self.branch,
+            file_path=f"data{self._path}",
+            commit_message="aws version sync",
+            content=self._content,
+        )
+
+
+class MrData(BaseModel):
+    namespace_file: str
+    provider: str
+    provisioner_ref: str
+    provisioner_uid: str
+    resource_provider: str
+    resource_identifier: str
+    resource_engine: str
+    resource_engine_version: str
+
+
+class MergeRequestManager(MergeRequestManagerBase[AVSInfo]):
+    """
+    Manager for AVS merge requests. This class
+    is responsible for housekeeping (closing old/bad MRs) and
+    opening new MRs for external resources that have new versions.
+
+    For each external resource, there are exist just one MR to update
+    the version number in the App-Interface. Old or obsolete MRs are
+    closed automatically.
+    """
+
+    def __init__(
+        self, vcs: VCS, renderer: Renderer, parser: Parser, auto_merge_enabled: bool
+    ):
+        super().__init__(vcs, parser, AVS_LABEL)
+        self._renderer = renderer
+        self._auto_merge_enabled = auto_merge_enabled
+
+    def create_merge_request(
+        self,
+        data: MrData,
+    ) -> None:
+        """Open new MR (if not already present) for an external resource and close any outdated before."""
+        if not self._housekeeping_ran:
+            self.housekeeping()
+
+        namespace_file = data.namespace_file
+        provider = data.provider
+        provisioner_ref = data.provisioner_ref
+        provisioner_uid = data.provisioner_uid
+        resource_provider = data.resource_provider
+        resource_identifier = data.resource_identifier
+        resource_engine = data.resource_engine
+        resource_engine_version = data.resource_engine_version
+
+        if mr := self._merge_request_already_exists({
+            "provider": provider,
+            "account_id": provisioner_uid,
+            "resource_provider": resource_provider,
+            "resource_identifier": resource_identifier,
+            "resource_engine": resource_engine,
+        }):
+            if mr.mr_info.resource_engine_version == resource_engine_version:
+                # an MR for this external resource already exists
+                return None
+            logging.info(
+                "Found an outdated MR for '%s' - closing it.", resource_identifier
+            )
+            self._vcs.close_app_interface_mr(
+                mr.raw, "Closing this MR because it's outdated."
+            )
+            # don't open a new MR right now, because the deletion of the old MRs could be
+            # disabled. In this case, we would end up with multiple open MRs for the
+            # same external resource.
+            return None
+
+        try:
+            content = self._vcs.get_file_content_from_app_interface_master(
+                file_path=namespace_file
+            )
+        except GitlabGetError as e:
+            if e.response_code == 404:
+                logging.error(
+                    "The file %s does not exist anylonger. Most likely qontract-server data not in synch. This should resolve soon on its own.",
+                    namespace_file,
+                )
+                return None
+            raise e
+        content = self._renderer.render_merge_request_content(
+            current_content=content,
+            provider=provider,
+            provisioner_ref=provisioner_ref,
+            resource_provider=resource_provider,
+            resource_identifier=resource_identifier,
+            resource_engine_version=resource_engine_version,
+        )
+
+        description = self._renderer.render_description(
+            provider=provider,
+            account_id=provisioner_uid,
+            resource_provider=resource_provider,
+            resource_identifier=resource_identifier,
+            resource_engine=resource_engine,
+            resource_engine_version=resource_engine_version,
+        )
+        title = self._renderer.render_title(resource_identifier=resource_identifier)
+        logging.info("Open MR for %s (%s)", resource_identifier, resource_engine)
+        mr_labels = [AVS_LABEL]
+        if self._auto_merge_enabled:
+            mr_labels.append(AUTO_MERGE)
+        self._vcs.open_app_interface_merge_request(
+            mr=AVSMR(
+                path=namespace_file,
+                title=title,
+                description=description,
+                content=content,
+                labels=mr_labels,
+            )
+        )

reconcile/aws_version_sync/utils.py

@@ -0,0 +1,64 @@
+from collections.abc import (
+    Callable,
+    Iterable,
+    Mapping,
+)
+from typing import (
+    Any,
+    TypeVar,
+)
+from urllib.parse import urljoin
+
+import anymarkup
+import requests
+
+from reconcile.utils.exceptions import FetchResourceError
+from reconcile.utils.gql import GqlGetResourceError
+
+
+def prom_get(
+    url: str,
+    params: Mapping[Any, Any] | None = None,
+    token: str | None = None,
+    ssl_verify: bool = True,
+    uri: str = "api/v1/query",
+    timeout: int = 60,
+) -> list[dict[str, str]]:
+    url = urljoin(url, uri)
+    headers = {"accept": "application/json"}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+
+    response = requests.get(
+        url, params=params, headers=headers, verify=ssl_verify, timeout=timeout
+    )
+    response.raise_for_status()
+    return [r["metric"] for r in response.json()["data"]["result"]]
+
+
+Key = TypeVar("Key")
+T = TypeVar("T")
+
+
+def uniquify(key: Callable[[T], Key], items: Iterable[T]) -> list[T]:
+    return list({key(item): item for item in items}.values())
+
+
+def get_values(gql_get_resource_func: Callable, path: str) -> dict[str, Any]:
+    try:
+        raw_values = gql_get_resource_func(path)
+    except GqlGetResourceError as e:
+        raise FetchResourceError(str(e)) from e
+    try:
+        values = anymarkup.parse(raw_values["content"], force_types=None)
+        values.pop("$schema", None)
+    except anymarkup.AnyMarkupError as e:
+        e_msg = "Could not parse data. Skipping resource: {}"
+        raise FetchResourceError(e_msg.format(path)) from e
+    return values
+
+
+def override_values(values: Mapping, overrides: Mapping | None) -> dict:
+    if overrides is None:
+        return {**values}
+    return {**values, **overrides}

reconcile/blackbox_exporter_endpoint_monitoring.py

@@ -1,9 +1,6 @@
 import logging
 import sys
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 from reconcile import queries
 from reconcile.closedbox_endpoint_monitoring_base import (
@@ -69,7 +66,7 @@ def get_blackbox_providers() -> list[EndpointMonitoringProvider]:
 
 def build_probe(
     provider: EndpointMonitoringProvider, endpoints: list[Endpoint]
-) -> Optional[OpenshiftResource]:
+) -> OpenshiftResource | None:
     blackbox_exporter = provider.blackboxExporter
     if blackbox_exporter:
         prober_url = parse_prober_url(blackbox_exporter.exporterUrl)

reconcile/change_owners/README.md

@@ -0,0 +1,34 @@
+# Integration change-owners
+
+`change-owners` uses change-type declarations from `app-interface` to grant self-service change permissions to `app-interface` tenants.
+
+
+## Module structure
+
+* `reconcile.change_owners.diff` contains all functionality for low-level diffing on file content. It can be used to extract differences from files as `Diff` dataclasses.
+* `reconcile.change_owners.change_types` uses the `diff` module to extract relevant changes and tries to cover them with `change-types` from `app-interface`. The results are `BundleFileChange` dataclasses that contain all the found changes along with the `change-types` covering them. A change is considered `covered` when at least one `change-type` allows the changes in a context that provides approvers.
+* `reconcile.change_owners.self_service_roles` is a change coverage implementation that uses `RoleV1.self_service` configurations from `app-interface` to bring changes, change-types and approvers together in a `ChangeTypeContext`.
+* `reconcile.change_owners.decision` handles MR decision command parsing from eligible change approvers.
+* `reconcile.change_owners.change_owners` is the entry point for the integrations and acts mostly as a coordinator between the other modules.
+
+## Functionality
+
+`change_owners` uses the `qontract-server` diff endpoint to get a highlevel overview what changed in an MR. It leverages `change_types` to find fine grained differences in datafiles and resourcefiles and build `BundleFileChange` objects that hold the state of diffs and diff coverage.
+
+`change_owners` checks `BundleFileChange` objects for `change-types` that are `restrictive`. If the MR was created by a user, that has this `change-type` not assigned, the integration will fail. A user with this role assigned could issue an `/good-to-test` command to override this restriction.
+
+`change_owners` reachs out to pluggable functionality to find out which `change-types` can be applied to which changes with a set of approvers. Currently, the only module to provide such `ChangeTypeContext` is `self_service_roles` which looks for explicitly bound `change-types` and files in the context of a `Role` with users and bots will can act as approvers.
+
+The functionality provided by `self_service_roles` is very explicit because it sets up the self-service relationship by listing all involved components. Other mechanisms to provide `ChangeTypeContexts` based on other explicit or implicit ownership information can be added easily and plugged into the `change_owners.cover_changes()` function.
+
+The result of this coverage process is a list of `BundleFileChange` objects, each of them having a list of `DiffCoverage` objects, listing all differences and how (and if) they are covered by `ChangeTypeContexts`. That way every change has a list of `Approvers` provided by the context of the `ChangeType`.
+
+Next, `change_owners` reaches out to the `app-interface MR` it processes, to find the decisions of the approvers. Decisions are given as `/` command comments on the MR. The `decision` module parses those comments and applies them to the `BundleFileChanges` and their `DiffCoverage` objects. The result is a list of `ChangeDecisions` where every change is `approved` and/or on `hold`.
+
+The `change_owners` module inspects those `ChangeDecisions` to find out if all changes have been approved. If that is the case, it adds the `bot/approved` label to the MR so that `gitlab-housekeeper` can act on the MR and will try to merge it.
+
+If at least one `ChangeDecision` is on `hold`, the `bot/hold` label is applied, which prevents `gitlab-housekeeper` from merging.
+
+If at least one `ChangeDecision` is not `approved`, the the MR can not proceed without further approvals or without AppSRE intervention.
+
+In the case of changes not having a `ChangeTypeContext` attached allowing approvers to decide, the MR is considered `non-self-serviceable` and can't proceed without AppSRE intervention.

reconcile/change_owners/approver.py

@@ -1,6 +1,5 @@
 from dataclasses import dataclass
 from typing import (
-    Optional,
     Protocol,
 )
 
@@ -17,26 +16,25 @@ class Approver:
     """
 
     org_username: str
-    tag_on_merge_requests: Optional[bool] = False
+    tag_on_merge_requests: bool | None = False
 
 
 class ApproverResolver(Protocol):
-    def lookup_approver_by_path(self, path: str) -> Optional[Approver]:
-        ...
+    def lookup_approver_by_path(self, path: str) -> Approver | None: ...
 
 
 class GqlApproverResolver:
     def __init__(self, gqlapis: list[GqlApi]):
         self.gqlapis = gqlapis
 
-    def lookup_approver_by_path(self, path: str) -> Optional[Approver]:
+    def lookup_approver_by_path(self, path: str) -> Approver | None:
         for gqlapi in self.gqlapis:
             approver = self._lookup_approver_by_path(gqlapi, path)
             if approver:
                 return approver
         return None
 
-    def _lookup_approver_by_path(self, gqlapi: GqlApi, path: str) -> Optional[Approver]:
+    def _lookup_approver_by_path(self, gqlapi: GqlApi, path: str) -> Approver | None:
         approvers = gqlapi.query(
             """
             query Approvers($path: String) {
@@ -62,17 +60,17 @@ class GqlApproverResolver:
 
 
 class ApproverReachability(Protocol):
-    def render_for_mr_report(self) -> str:
-        ...
+    def render_for_mr_report(self) -> str: ...
 
 
 @dataclass
 class SlackGroupApproverReachability:
     slack_group: str
     workspace: str
+    channel: str
 
     def render_for_mr_report(self) -> str:
-        return f"Slack group {self.slack_group}/{self.workspace}"
+        return f"Slack group @{self.slack_group} in channel #{self.channel} (workspace {self.workspace})"
 
 
 @dataclass

reconcile/change_owners/bundle.py

@@ -1,31 +1,157 @@
 from abc import abstractmethod
 from dataclasses import dataclass
-from enum import Enum
+from enum import StrEnum
 from typing import (
     Any,
-    Optional,
     Protocol,
-    Tuple,
+)
+
+from pydantic import (
+    BaseModel,
+    Field,
 )
 
 from reconcile.utils.gql import get_diff
 
+DATAFILE_PATH_FIELD_NAME = "path"
+DATAFILE_SHA256SUM_FIELD_NAME = "$file_sha256sum"
+DATAFILE_SCHEMA_FIELD_NAME = "$schema"
 
-class BundleFileType(Enum):
+
+class BundleFileType(StrEnum):
     DATAFILE = "datafile"
     RESOURCEFILE = "resourcefile"
 
 
 @dataclass(frozen=True)
 class FileRef:
+    """
+    A reference to a file in a bundle.
+    """
+
     file_type: BundleFileType
     path: str
-    schema: Optional[str]
+    schema: str | None
+    json_path: str | None = None
 
     def __str__(self) -> str:
         return f"{self.file_type.value}:{self.path}"
 
 
+DATAFILE_CONTENT_CLEANUP_FIELDS = [
+    DATAFILE_SHA256SUM_FIELD_NAME,
+    DATAFILE_PATH_FIELD_NAME,
+    DATAFILE_SCHEMA_FIELD_NAME,
+]
+"""
+Datafile metadata fields that should be removed from the datafile content
+during BundleFileChange initialization.
+"""
+
+
+#
+# The following dataclasses represent the data returned by the
+# qontract-server /diff endpoint.
+#
+
+
+class QontractServerDatafileDiff(BaseModel):
+    """
+    Represents a datafile diff of an individual datafile returned by the qontract-server /diff endpoint.
+    """
+
+    datafilepath: str
+    datafileschema: str
+    old: dict[str, Any] | None
+    new: dict[str, Any] | None
+
+    @property
+    def old_datafilepath(self) -> str | None:
+        return self.old.get(DATAFILE_PATH_FIELD_NAME) if self.old else None
+
+    @property
+    def new_datafilepath(self) -> str | None:
+        return self.new.get(DATAFILE_PATH_FIELD_NAME) if self.new else None
+
+    @property
+    def old_data_sha(self) -> str | None:
+        return self.old.get(DATAFILE_SHA256SUM_FIELD_NAME) if self.old else None
+
+    @property
+    def new_data_sha(self) -> str | None:
+        return self.new.get(DATAFILE_SHA256SUM_FIELD_NAME) if self.new else None
+
+    @property
+    def cleaned_old_data(self) -> dict[str, Any] | None:
+        return _clean_datafile_content(self.old)
+
+    @property
+    def cleaned_new_data(self) -> dict[str, Any] | None:
+        return _clean_datafile_content(self.new)
+
+
+def _clean_datafile_content(data: dict[str, Any] | None) -> dict[str, Any] | None:
+    """
+    Sadly, datafiles mix and match data and metadata in the same file. This
+    function removes some metadata that is otherwise annoying to deal with.
+    """
+    if data is None:
+        return None
+    return {k: v for k, v in data.items() if k not in DATAFILE_CONTENT_CLEANUP_FIELDS}
+
+
+class QontractServerResourcefileBackref(BaseModel):
+    """
+    Represents a backref from a resourcefile to a datafile
+    """
+
+    path: str
+    datafileschema: str = Field(..., alias="datafileSchema")
+    jsonpath: str = Field(..., alias="jsonpath")
+
+
+class QontractServerResourcefileDiffState(BaseModel):
+    """
+    Represents the old or new state of a resourcefile returned by the qontract-server /diff endpoint.
+    """
+
+    path: str
+    content: str
+    resourcefileschema: str | None = Field(..., alias="$schema")
+    sha256sum: str
+    backrefs: list[QontractServerResourcefileBackref] | None
+
+
+class QontractServerResourcefileDiff(BaseModel):
+    """
+    Represents a resourcefile diff of an individual resourcefile returned by the qontract-server /diff endpoint.
+    """
+
+    resourcepath: str
+    old: QontractServerResourcefileDiffState | None = None
+    new: QontractServerResourcefileDiffState | None = None
+
+    @property
+    def resourcefileschema(self) -> str | None:
+        old_schema = self.old.resourcefileschema if self.old else None
+        new_schema = self.new.resourcefileschema if self.new else None
+        return new_schema or old_schema
+
+
+class QontractServerDiff(BaseModel):
+    """
+    Top level datastructure for datafile and resourcefile diffs returned by the qontract-server /diff endpoint.
+    """
+
+    datafiles: dict[str, QontractServerDatafileDiff]
+    resources: dict[str, QontractServerResourcefileDiff]
+
+
+#
+# File diff resolver help finding differences between two versions of a file.
+#
+
+
 class FileDiffResolver(Protocol):
     """
     A protocol to lookup the diff of a file given its FileRef.
@@ -34,8 +160,7 @@ class FileDiffResolver(Protocol):
     @abstractmethod
     def lookup_file_diff(
         self, file_ref: FileRef
-    ) -> Tuple[Optional[dict[str, Any]], Optional[dict[str, Any]]]:
-        ...
+    ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: ...
 
 
 @dataclass
@@ -49,7 +174,7 @@ class QontractServerFileDiffResolver:
 
     def lookup_file_diff(
         self, file_ref: FileRef
-    ) -> Tuple[Optional[dict[str, Any]], Optional[dict[str, Any]]]:
+    ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
         data = get_diff(
             old_sha=self.comparison_sha,
             file_type=file_ref.file_type.value,
@@ -66,7 +191,7 @@ class NoOpFileDiffResolver:
 
     def lookup_file_diff(
         self, file_ref: FileRef
-    ) -> Tuple[Optional[dict[str, Any]], Optional[dict[str, Any]]]:
+    ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
         raise Exception(
             "NoOpFileDiffResolver is not supposed to be used in "
             "runtime contexts where lookups are needed"