qontract-reconcile 0.10.2.dev361__py3-none-any.whl → 0.10.2.dev474__py3-none-any.whl

reconcile/utils/expiration.py

@@ -6,6 +6,8 @@ from typing import (
     cast,
 )
 
+from reconcile.utils.datetime_util import ensure_utc, utc_now
+
 DATE_FORMAT = "%Y-%m-%d"
 
 
@@ -17,12 +19,14 @@ DictsOrRoles = TypeVar("DictsOrRoles", bound=Iterable[FilterableRole] | Iterable
 
 
 def date_expired(date: str) -> bool:
-    exp_date = datetime.datetime.strptime(date, DATE_FORMAT).date()
-    current_date = datetime.datetime.utcnow().date()
+    exp_date = ensure_utc(datetime.datetime.strptime(date, DATE_FORMAT)).date()  # noqa: DTZ007
+    current_date = utc_now().date()
     return current_date >= exp_date
 
 
-def filter(roles: DictsOrRoles | None) -> DictsOrRoles:
+def filter[DictsOrRoles: Iterable[FilterableRole] | Iterable[dict]](
+    roles: DictsOrRoles | None,
+) -> DictsOrRoles:
     """Filters roles and returns the ones which are not yet expired."""
     filtered = []
     for r in roles or []:

reconcile/utils/external_resource_spec.py

@@ -157,6 +157,8 @@ class ExternalResourceSpec:
             tags["cost-center"] = cost_center
         if service_phase := self.namespace["environment"].get("servicePhase"):
             tags["service-phase"] = service_phase
+        if cost_center := self.namespace["environment"].get("costCenter"):
+            tags["cost-center"] = cost_center
 
         resource_tags_str = self.resource.get("tags")
         if resource_tags_str:

reconcile/utils/filtering.py

@@ -6,7 +6,7 @@ KeyType = TypeVar("KeyType")
 ValueType = TypeVar("ValueType")
 
 
-def remove_none_values_from_dict(
+def remove_none_values_from_dict[KeyType, ValueType](
     a_dict: dict[KeyType, ValueType | None],
 ) -> dict[KeyType, ValueType]:
     """

reconcile/utils/gitlab_api.py

@@ -263,13 +263,13 @@ class GitLabApi:
         # we can determine if a pending MR exists based on the title
         return any(mr.title == title for mr in mrs)
 
-    @retry()
+    @retry(no_retry_exceptions=(RuntimeError,))
     def get_project_maintainers(
         self, repo_url: str | None = None, query: dict | None = None
-    ) -> list[str] | None:
+    ) -> list[str]:
         project = self.project if repo_url is None else self.get_project(repo_url)
         if project is None:
-            return None
+            raise RuntimeError("project not found")
         members = project.members_all.list(iterator=True, query_parameters=query or {})
         return [m.username for m in members if m.access_level >= 40]
 
@@ -444,6 +444,8 @@ class GitLabApi:
     def get_merge_request_comments(
         merge_request: ProjectMergeRequest,
         include_description: bool = False,
+        include_approvals: bool = False,
+        approval_body: str = "",
     ) -> list[Comment]:
         comments = []
         if include_description:
@@ -455,6 +457,16 @@ class GitLabApi:
                     created_at=merge_request.created_at,
                 )
             )
+        if include_approvals:
+            comments.extend(
+                Comment(
+                    id=approval["user"]["id"],
+                    username=approval["user"]["username"],
+                    body=approval_body,
+                    created_at=approval["approved_at"],
+                )
+                for approval in merge_request.approvals.get().approved_by
+            )
         comments.extend(
             Comment(
                 id=note.id,
@@ -826,14 +838,16 @@ class GitLabApi:
         )
 
     def get_commit_sha(self, ref: str, repo_url: str) -> str:
-        project = self.get_project(repo_url)
+        if not (project := self.get_project(repo_url)):
+            raise ValueError(f"Project not found for repo_url: {repo_url}")
         commits = project.commits.list(ref_name=ref, per_page=1, page=1)
         return commits[0].id
 
     def repository_compare(
         self, repo_url: str, ref_from: str, ref_to: str
     ) -> list[dict[str, Any]]:
-        project = self.get_project(repo_url)
+        if not (project := self.get_project(repo_url)):
+            raise ValueError(f"Project not found for repo_url: {repo_url}")
         response: Any = project.repository_compare(ref_from, ref_to)
         return response.get("commits", [])
 

reconcile/utils/glitchtip/client.py

@@ -164,7 +164,9 @@ class GlitchtipClient(ApiBase):
         return ProjectAlert(
             **self._post(
                 f"/api/0/projects/{organization_slug}/{project_slug}/alerts/",
-                data=alert.dict(by_alias=True, exclude_unset=True, exclude_none=True),
+                data=alert.model_dump(
+                    mode="json", by_alias=True, exclude_unset=True, exclude_none=True
+                ),
             )
         )
 
@@ -183,7 +185,9 @@ class GlitchtipClient(ApiBase):
         return ProjectAlert(
             **self._put(
                 f"/api/0/projects/{organization_slug}/{project_slug}/alerts/{alert.pk}/",
-                data=alert.dict(by_alias=True, exclude_unset=True, exclude_none=True),
+                data=alert.model_dump(
+                    mode="json", by_alias=True, exclude_unset=True, exclude_none=True
+                ),
             )
         )
 

reconcile/utils/glitchtip/models.py

@@ -3,13 +3,13 @@ from __future__ import annotations
 import re
 from datetime import datetime
 from enum import Enum
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Self
 
 from pydantic import (
     BaseModel,
     Field,
-    root_validator,
-    validator,
+    field_validator,
+    model_validator,
 )
 
 if TYPE_CHECKING:
@@ -49,7 +49,8 @@ class Team(BaseModel):
     slug: str = ""
     users: list[User] = []
 
-    @root_validator(pre=True)
+    @model_validator(mode="before")
+    @classmethod
     def name_xor_slug_must_be_set(
         cls, values: MutableMapping[str, Any]
     ) -> MutableMapping[str, Any]:
@@ -58,11 +59,11 @@ class Team(BaseModel):
         ), "name xor slug must be set!"
         return values
 
-    @root_validator
-    def slugify(cls, values: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
-        values["slug"] = values.get("slug") or slugify(values.get("name", ""))
-        values["name"] = slugify(values.get("name", "")) or values.get("slug")
-        return values
+    @model_validator(mode="after")
+    def slugify(self) -> Self:
+        self.slug = self.slug or slugify(self.name)
+        self.name = slugify(self.name) or self.slug
+        return self
 
     def __lt__(self, other: Team) -> bool:
         return self.slug < other.slug
@@ -86,16 +87,15 @@ class RecipientType(Enum):
     WEBHOOK = "webhook"
 
 
-class ProjectAlertRecipient(BaseModel):
+class ProjectAlertRecipient(
+    BaseModel, validate_by_name=True, validate_by_alias=True, use_enum_values=True
+):
     pk: int | None = Field(None, alias="id")
     recipient_type: RecipientType = Field(..., alias="recipientType")
     url: str = ""
 
-    class Config:
-        allow_population_by_field_name = True
-        use_enum_values = True
-
-    @validator("recipient_type")
+    @field_validator("recipient_type")
+    @classmethod
     def recipient_type_enforce_enum_type(cls, v: str | RecipientType) -> RecipientType:
         if isinstance(v, RecipientType):
             return v
@@ -113,17 +113,15 @@ class ProjectAlertRecipient(BaseModel):
         return hash((self.recipient_type, self.url))
 
 
-class ProjectAlert(BaseModel):
+class ProjectAlert(BaseModel, validate_by_name=True, validate_by_alias=True):
     pk: int | None = Field(None, alias="id")
     name: str
     timespan_minutes: int = Field(..., alias="timespanMinutes")
     quantity: int
     recipients: list[ProjectAlertRecipient] = Field([], alias="alertRecipients")
 
-    class Config:
-        allow_population_by_field_name = True
-
-    @root_validator
+    @model_validator(mode="before")
+    @classmethod
     def empty_name(cls, values: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
         # name is an empty string if the alert was created manually because it can't be set via UI
         # use the pk instead.
@@ -141,20 +139,18 @@ class ProjectAlert(BaseModel):
         )
 
 
-class Project(BaseModel):
+class Project(BaseModel, validate_by_name=True, validate_by_alias=True):
     pk: int | None = Field(None, alias="id")
     name: str
     slug: str = ""
-    platform: str | None
+    platform: str | None = None
     teams: list[Team] = []
     alerts: list[ProjectAlert] = []
     event_throttle_rate: int = Field(0, alias="eventThrottleRate")
     organization: Organization | None = None
 
-    class Config:
-        allow_population_by_field_name = True
-
-    @root_validator
+    @model_validator(mode="before")
+    @classmethod
     def slugify(cls, values: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
         values["slug"] = values.get("slug") or slugify(values["name"])
         return values
@@ -195,7 +191,8 @@ class Organization(BaseModel):
     teams: list[Team] = []
     users: list[User] = []
 
-    @root_validator
+    @model_validator(mode="before")
+    @classmethod
     def slugify(cls, values: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
         values["slug"] = values.get("slug") or slugify(values["name"])
         return values
@@ -212,4 +209,4 @@ class Organization(BaseModel):
         return hash(self.name)
 
 
-Project.update_forward_refs()
+Project.model_rebuild()

reconcile/utils/gql.py

@@ -109,7 +109,7 @@ class GqlApi:
         if int_name:
             integrations = self.query(INTEGRATIONS_QUERY, skip_validation=True)
 
-            for integration in integrations["integrations"]:
+            for integration in integrations["integrations"] if integrations else []:
                 if integration["name"] == int_name:
                     self._valid_schemas = integration["schemas"]
                     break
@@ -142,7 +142,7 @@ class GqlApi:
         query: str,
         variables: dict[str, Any] | None = None,
         skip_validation: bool = False,
-    ) -> dict[str, Any] | None:
+    ) -> dict[str, Any]:
         try:
             result = self.client.execute(
                 gql(query), variables, get_execution_result=True
@@ -172,11 +172,8 @@ class GqlApi:
             if forbidden_schemas:
                 raise GqlApiErrorForbiddenSchemaError(forbidden_schemas)
 
-        # This is to appease mypy. This exception won't be thrown as this condition
-        # is already handled above with AssertionError
-        if result["data"] is None:
-            raise GqlApiError("`data` not received in GraphQL payload")
-
+        # make mypy happy
+        assert "data" in result and result["data"] is not None
         return result["data"]
 
     def get_template(self, path: str) -> dict[str, str]:

reconcile/utils/helpers.py

@@ -47,7 +47,7 @@ def flatten(
 Item = TypeVar("Item")
 
 
-def find_duplicates(items: Iterable[Item]) -> list[Item]:
+def find_duplicates[Item](items: Iterable[Item]) -> list[Item]:
     return [item for item, count in Counter(items).items() if count > 1]
 
 

reconcile/utils/instrumented_wrappers.py

@@ -40,7 +40,7 @@ class InstrumentedImage(Image):
 
 
 class InstrumentedSkopeo(Skopeo):
-    def copy(self, *args: Any, **kwargs: Any) -> bytes | str:
+    def copy(self, *args: Any, **kwargs: Any) -> None:
         metrics.copy_count.labels(
             integration=INTEGRATION_NAME, shard=SHARDS, shard_id=SHARD_ID
         ).inc()

reconcile/utils/internal_groups/client.py

@@ -140,7 +140,7 @@ class InternalGroupsClient:
         with self._api as api:
             return Group(
                 **api.create_group(
-                    data=group.dict(by_alias=True),
+                    data=group.model_dump(by_alias=True),
                 )
             )
 
@@ -155,6 +155,6 @@ class InternalGroupsClient:
             return Group(
                 **api.update_group(
                     name=group.name,
-                    data=group.dict(by_alias=True),
+                    data=group.model_dump(by_alias=True),
                 )
             )

reconcile/utils/internal_groups/models.py

@@ -27,7 +27,7 @@ class Entity(BaseModel):
         return hash(self.id)
 
 
-class Group(BaseModel):
+class Group(BaseModel, validate_by_name=True, validate_by_alias=True):
     name: str
     description: str
     member_approval_type: str = Field("self-service", alias="memberApprovalType")
@@ -35,16 +35,18 @@ class Group(BaseModel):
     owners: list[Entity]
     display_name: str = Field(..., alias="displayName")
     notes: str | None = None
-    rover_group_member_query: str | None = Field(None, alias="roverGroupMemberQuery")
+    rover_group_member_query: str | None = Field(
+        None, alias="roverGroupMemberQuery", exclude=True
+    )
     rover_group_inclusions: list[Entity] | None = Field(
-        None, alias="roverGroupInclusions"
+        None, alias="roverGroupInclusions", exclude=True
     )
     rover_group_exclusions: list[Entity] | None = Field(
-        None, alias="roverGroupExclusions"
+        None, alias="roverGroupExclusions", exclude=True
     )
     members: list[Entity] = []
-    member_of: list[str] | None = Field(None, alias="memberOf")
-    namespace: str | None = None
+    member_of: list[str] | None = Field(None, alias="memberOf", exclude=True)
+    namespace: str | None = Field(None, exclude=True)
 
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, Group):
@@ -58,14 +60,3 @@ class Group(BaseModel):
             and self.notes == other.notes
             and set(self.members) == set(other.members)
         )
-
-    class Config:
-        allow_population_by_field_name = True
-        # exclude read-only fields in the json/dict dumps
-        fields = {
-            "rover_group_member_query": {"exclude": True},
-            "rover_group_inclusions": {"exclude": True},
-            "rover_group_exclusions": {"exclude": True},
-            "member_of": {"exclude": True},
-            "namespace": {"exclude": True},
-        }

reconcile/utils/jinja2/utils.py

@@ -1,13 +1,10 @@
 import datetime
 import os
-import subprocess
-import tempfile
 from collections.abc import Mapping
 from functools import cache
 from typing import Any, Self
 
 import jinja2
-import yaml
 from github import Github
 from jinja2.sandbox import SandboxedEnvironment
 from pydantic import BaseModel
@@ -18,6 +15,7 @@ from reconcile.checkpoint import url_makes_sense
 from reconcile.github_org import get_default_config
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.github_api import GithubRepositoryApi
 from reconcile.utils.helpers import flatten
 from reconcile.utils.jinja2.extensions import B64EncodeExtension, RaiseErrorExtension
@@ -38,7 +36,7 @@ from reconcile.utils.secret_reader import (
     SecretReader,
     SecretReaderBase,
 )
-from reconcile.utils.sloth import process_sloth_output
+from reconcile.utils.sloth import generate_sloth_rules
 from reconcile.utils.vault import SecretFieldNotFoundError
 
 
@@ -47,14 +45,11 @@ class Jinja2TemplateError(Exception):
         super().__init__("error processing jinja2 template: " + str(msg))
 
 
-class TemplateRenderOptions(BaseModel):
+class TemplateRenderOptions(BaseModel, frozen=True):
     trim_blocks: bool
     lstrip_blocks: bool
     keep_trailing_newline: bool
 
-    class Config:
-        frozen = True
-
     @classmethod
     def create(
         cls,
@@ -77,7 +72,7 @@ def compile_jinja2_template(
 ) -> Any:
     if not template_render_options:
         template_render_options = TemplateRenderOptions.create()
-    env: dict[str, Any] = template_render_options.dict()
+    env: dict[str, Any] = template_render_options.model_dump()
     if extra_curly:
         env.update({
             "block_start_string": "{{%",
@@ -192,89 +187,6 @@ def list_s3_objects(
         )
 
 
-def sloth_alerts(
-    service: str,
-    slo_name: str,
-    objective: float,
-    error_query: str,
-    total_query: str,
-    version: str = "prometheus/v1",
-) -> str:
-    """Generate Prometheus rules using sloth: https://sloth.dev
-
-    Args:
-        service: Service name identifier
-        slo_name: Name of the SLO
-        objective: Target percentage (e.g. 99.9)
-        error_query: Prometheus query for error events
-        total_query: Prometheus query for total events
-        version: Spec version (default: "prometheus/v1")
-
-    Returns:
-        Generated Prometheus rules as YAML string
-    """
-    # Build the SLO definition
-    slo = {
-        "name": slo_name,
-        "objective": objective,
-        "description": f"{slo_name} SLO for {service}",
-        "sli": {
-            "events": {
-                "error_query": error_query.replace("{{window}}", "{{.window}}"),
-                "total_query": total_query.replace("{{window}}", "{{.window}}"),
-            }
-        },
-        "alerting": {
-            "name": f"{service.title()}{slo_name.title()}",
-            "annotations": {
-                "summary": f"High error rate on '{service}' {slo_name}",
-                "message": f"High error rate on '{service}' {slo_name}",
-            },
-            "page_alert": {
-                "labels": {
-                    "severity": "critical",
-                    "service": service,
-                    "slo": slo_name,
-                }
-            },
-            "ticket_alert": {
-                "labels": {
-                    "severity": "medium",
-                    "service": service,
-                    "slo": slo_name,
-                }
-            },
-        },
-    }
-
-    spec = {
-        "version": version,
-        "service": service,
-        "slos": [slo],
-    }
-
-    with (
-        tempfile.NamedTemporaryFile(
-            encoding="utf-8", mode="w", suffix=".yml"
-        ) as input_file,
-        tempfile.NamedTemporaryFile(
-            encoding="utf-8", mode="w", suffix=".yml"
-        ) as output_file,
-    ):
-        yaml.dump(spec, input_file, allow_unicode=True)
-        cmd = ["sloth", "generate", "-i", input_file.name, "-o", output_file.name]
-        try:
-            subprocess.run(cmd, capture_output=True, check=True, text=True)
-        except subprocess.CalledProcessError as e:
-            error_msg = f"{e}"
-            if e.stdout:
-                error_msg += f"\nstdout: {e.stdout}"
-            if e.stderr:
-                error_msg += f"\nstderr: {e.stderr}"
-            raise SlothGenerateError(error_msg) from e
-        return process_sloth_output(output_file.name)
-
-
 @retry()
 def lookup_secret(
     path: str,
@@ -345,10 +257,8 @@ def process_jinja2_template(
         "s3": lookup_s3_object,
         "s3_ls": list_s3_objects,
         "flatten_dict": flatten,
-        "yesterday": lambda: (datetime.datetime.now() - datetime.timedelta(1)).strftime(
-            "%Y-%m-%d"
-        ),
-        "sloth_alerts": sloth_alerts,
+        "yesterday": lambda: (utc_now() - datetime.timedelta(1)).strftime("%Y-%m-%d"),
+        "sloth_alerts": generate_sloth_rules,
     })
     if "_template_mocks" in vars:
         for k, v in vars["_template_mocks"].items():
@@ -381,11 +291,6 @@ def process_extracurlyjinja2_template(
     )
 
 
-class SlothGenerateError(Exception):
-    def __init__(self, msg: Any):
-        super().__init__("sloth generate failed: " + str(msg))
-
-
 class FetchSecretError(Exception):
     def __init__(self, msg: Any):
         super().__init__("error fetching secret: " + str(msg))