qontract-reconcile 0.10.2.dev310__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl

reconcile/utils/models.py

@@ -1,7 +1,5 @@
 from collections import UserList
 from collections.abc import (
-    Callable,
-    Generator,
     MutableMapping,
 )
 from typing import Any
@@ -9,75 +7,311 @@ from typing import Any
 from croniter import croniter
 from pydantic import (
     BaseModel,
+    GetCoreSchemaHandler,
     ValidationError,
 )
-from pydantic import errors as pydantic_errors
-from pydantic.fields import ModelField
+from pydantic_core import core_schema
 
 DEFAULT_STRING = "I was too lazy to define a string here"
 DEFAULT_INT = 42
+DEFAULT_BOOL = False
+DEFAULT_JSON_STR = "{}"
+DEFAULT_JSON: dict = {}
+
+DataType = MutableMapping[str, Any] | str | int | bool | list | None
+
+
+def _process_field_with_alias(
+    field_info: dict[str, Any],
+    data: DataType,
+    alias: str | None,
+    use_defaults: bool,
+    definitions: list[dict[str, Any]] | None,
+) -> DataType:
+    """Process a field by recursively calling data_default_none and handling alias.
+
+    This helper function encapsulates the common pattern used by 'default',
+    'nullable', and 'model' field types where we need to:
+    1. Recursively process the field's schema
+    2. Handle data extraction via alias if data is a dict
+    3. Update the data dict with the processed value if alias exists
+
+    Args:
+        field_info: Schema information for the field to process
+        data: Input data (can be dict, primitive, list, or None)
+        alias: Field alias for dict key access, or None for unnamed fields
+        use_defaults: Whether to use default values for missing required fields
+        definitions: Schema definitions for definition-ref fields
+
+    Returns:
+        Processed data value, or updated data dict if alias was provided
+    """
+    # Recursively process the field's schema
+    value = data_default_none(
+        field_info["schema"],
+        data.get(alias) if isinstance(data, MutableMapping) and alias else data,
+        use_defaults=use_defaults,
+        definitions=definitions,
+    )
+
+    # For unnamed fields (in recursive calls), return value directly
+    if not alias:
+        return value
+
+    # For named fields, update the data dict
+    if not isinstance(data, MutableMapping):
+        raise TypeError(
+            f"Expected MutableMapping for field '{alias}', got {type(data).__name__}"
+        )
+    data[alias] = value
+    return data
 
 
 def data_default_none(
-    klass: type[BaseModel], data: MutableMapping[str, Any], use_defaults: bool = True
-) -> MutableMapping[str, Any]:
-    """Set default values to None for required but optional fields."""
-    for field in klass.__fields__.values():
-        if not field.required:
-            continue
-
-        if field.alias not in data:
-            # Settings defaults
-            if field.allow_none:
-                data[field.alias] = None
-            elif not use_defaults:
-                raise ValueError(f"Field {field.alias} is required but not set.")
-            elif isinstance(field.type_, type) and issubclass(field.type_, str):
-                data[field.alias] = DEFAULT_STRING
-            elif isinstance(field.type_, type) and issubclass(field.type_, bool):
-                data[field.alias] = False
-            elif isinstance(field.type_, type) and issubclass(field.type_, int):
-                data[field.alias] = DEFAULT_INT
-        elif isinstance(field.type_, type) and issubclass(field.type_, BaseModel):
-            if isinstance(data[field.alias], dict):
-                data[field.alias] = data_default_none(field.type_, data[field.alias])
-            if isinstance(data[field.alias], list):
-                data[field.alias] = [
-                    data_default_none(field.type_, item)
-                    for item in data[field.alias]
-                    if isinstance(item, dict)
+    klass: type[BaseModel] | dict,
+    data: DataType,
+    use_defaults: bool = True,
+    definitions: list[dict[str, Any]] | None = None,
+) -> DataType:
+    """Set default values for required fields in Pydantic model data.
+
+    This function recursively processes Pydantic model schemas and fills in
+    missing fields with appropriate default values based on their type. It's
+    primarily used to handle GraphQL query results where fields may be missing.
+
+    Args:
+        klass: Pydantic BaseModel class or schema dict to process
+        data: Input data to fill with defaults (can be dict, primitive, list, or None)
+        use_defaults: If True, use type-based defaults for missing required fields.
+                     If False, raise ValueError for missing required fields.
+        definitions: Schema definitions for definition-ref fields (e.g., VaultSecret).
+                    Automatically extracted from models with "definitions" schema type.
+
+    Returns:
+        Data with defaults filled in. Return type matches input data structure.
+
+    Field handling behavior:
+        - Optional fields (T | None): Default to None if missing
+        - Required primitives: Use DEFAULT_STRING, DEFAULT_INT, or DEFAULT_BOOL
+        - Fields with explicit defaults: Use the provided default value
+        - Lists: Default to empty list [] if missing, recursively process items
+        - Nested models: Recursively process with data_default_none
+        - Union types: Try each variant until one validates ("first match wins")
+
+    Example:
+        >>> class User(BaseModel):
+        ...     name: str
+        ...     age: int | None
+        >>> data_default_none(User, {})
+        {'name': 'I was too lazy to define a string here', 'age': None}
+    """
+    # =========================================================================
+    # Extract schema from BaseModel class or use provided schema dict
+    # =========================================================================
+    if isinstance(klass, dict):
+        # Recursive calls pass schema dict directly
+        schema = klass
+    else:
+        # Extract schema from Pydantic BaseModel class
+        match klass.__pydantic_core_schema__["type"]:
+            case "definitions":
+                # Models using definition-ref (e.g., VaultSecret fragments)
+                schema = klass.__pydantic_core_schema__["schema"]["schema"]
+                definitions = klass.__pydantic_core_schema__["definitions"]
+            case _:
+                # Standard models
+                schema = klass.__pydantic_core_schema__["schema"]
+
+    # =========================================================================
+    # Get fields to process - either model fields or single schema
+    # =========================================================================
+    if schema["type"] == "model-fields":
+        # Multiple fields in a model
+        fields = schema["fields"].items()
+    else:
+        # Single field schema (used in recursive calls for field processing)
+        fields = [(None, schema)]
+
+    # =========================================================================
+    # Process each field according to its schema type
+    # =========================================================================
+    for name, field_info in fields:
+        # Use alias if defined (e.g., "serverUrl" instead of "server_url")
+        alias = field_info.get("validation_alias", name)
+
+        match field_info["type"]:
+            # =================================================================
+            # PRIMITIVE TYPES: str, int, bool
+            # =================================================================
+            case "str" | "int" | "bool" | "json":
+                # If data is provided, return it as-is (don't override user data)
+                if data is not None:
+                    return data
+
+                # If no defaults allowed, raise error for missing required field
+                if not use_defaults:
+                    raise ValueError(f"Field {alias or ''} is required but not set.")
+
+                # Return type-specific default value
+                match field_info["type"]:
+                    case "str":
+                        return DEFAULT_STRING
+                    case "int":
+                        return DEFAULT_INT
+                    case "bool":
+                        return DEFAULT_BOOL
+                    case "json":
+                        return DEFAULT_JSON_STR
+
+            # =================================================================
+            # FIELDS WITH EXPLICIT DEFAULT VALUES
+            # =================================================================
+            case "default":
+                # If no data provided, use the field's default value
+                if data is None:
+                    return field_info["default"]
+
+                # Data exists - recursively process it in case it's a complex type
+                return _process_field_with_alias(
+                    field_info, data, alias, use_defaults, definitions
+                )
+
+            # =================================================================
+            # NULLABLE/OPTIONAL FIELDS: T | None
+            # =================================================================
+            case "nullable":
+                # If no data provided, optional fields default to None
+                if data is None:
+                    return None
+
+                # Data exists - recursively process the non-None value
+                return _process_field_with_alias(
+                    field_info, data, alias, use_defaults, definitions
+                )
+
+            # =================================================================
+            # MODEL FIELDS: Nested Pydantic models within parent model
+            # =================================================================
+            case "model-field":
+                # Ensure data is a dict (create empty dict if needed)
+                # This allows processing models even when parent data is missing
+                if not isinstance(data, MutableMapping):
+                    data = {}
+
+                # Recursively process the nested model
+                data[alias] = data_default_none(
+                    field_info["schema"],
+                    data.get(alias),
+                    use_defaults=use_defaults,
+                    definitions=definitions,
+                )
+
+            # =================================================================
+            # STANDALONE MODELS: BaseModel used as field type
+            # =================================================================
+            case "model":
+                # Recursively process nested BaseModel
+                return _process_field_with_alias(
+                    field_info, data, alias, use_defaults, definitions
+                )
+
+            # =================================================================
+            # DEFINITION REFERENCES: e.g., VaultSecret fragments
+            # =================================================================
+            case "definition-ref":
+                # Definition-ref fields require a MutableMapping data structure
+                assert isinstance(data, MutableMapping)
+
+                # Definitions are required for resolving references
+                if not definitions:
+                    raise RuntimeError(
+                        "definitions parameter is required for definition-ref fields"
+                    )
+
+                # Find the referenced schema definition by ref ID
+                ref_schema = next(
+                    definition
+                    for definition in definitions
+                    if definition["ref"] == field_info["schema_ref"]
+                )
+
+                # Recursively process using the referenced schema
+                value = data_default_none(
+                    ref_schema["schema"],
+                    data.get(alias) if alias else data,
+                    use_defaults=use_defaults,
+                    definitions=definitions,
+                )
+
+                # For unnamed fields, return value directly
+                if not alias:
+                    return value
+
+                data[alias] = value
+
+            # =================================================================
+            # UNION TYPES: T1 | T2 | ...
+            # =================================================================
+            case "union":
+                # Strategy: Try each union variant until one validates successfully
+                # This is a "first match wins" approach
+                for sub_field in field_info["choices"]:
+                    try:
+                        # Try to process data with this union variant
+                        # use_defaults=False ensures we only match existing data
+                        sub_data = data_default_none(
+                            sub_field,
+                            # don't alter data structure here
+                            dict(data) if isinstance(data, MutableMapping) else data,
+                            use_defaults=use_defaults,
+                            definitions=definitions,
+                        )
+
+                        # Validate the result matches the expected type
+                        match sub_field["type"]:
+                            case "str":
+                                if not isinstance(sub_data, str):
+                                    raise ValueError()
+                            case "int":
+                                if not isinstance(sub_data, int):
+                                    raise ValueError()
+                            case "bool":
+                                if not isinstance(sub_data, bool):
+                                    raise ValueError()
+                            case "model":
+                                if not isinstance(sub_data, dict):
+                                    raise ValueError()
+                                # Validate the dict can be instantiated as the model
+                                sub_field["cls"].model_validate(sub_data)
+                                if isinstance(data, MutableMapping):
+                                    data.update(sub_data)
+
+                        # If we get here, this union variant matched successfully
+                        return data
+                    except (ValidationError, ValueError):
+                        # This variant didn't match, try the next one
+                        continue
+
+            # =================================================================
+            # LIST TYPES: list[T]
+            # =================================================================
+            case "list":
+                # If data is not a list, default to empty list
+                if not isinstance(data, list):
+                    return []
+
+                # Recursively process each list item
+                return [
+                    data_default_none(
+                        field_info["items_schema"],
+                        item.get(alias)
+                        if isinstance(item, MutableMapping) and alias
+                        else item,
+                        use_defaults=use_defaults,
+                        definitions=definitions,
+                    )
+                    for item in data
                 ]
-        elif field.sub_fields:
-            if all(
-                isinstance(sub_field.type_, type)
-                and issubclass(sub_field.type_, BaseModel)
-                for sub_field in field.sub_fields
-            ):
-                # Union[ClassA, ClassB] field
-                for sub_field in field.sub_fields:
-                    if isinstance(data[field.alias], dict):
-                        try:
-                            d = dict(data[field.alias])
-                            d.update(data_default_none(sub_field.type_, d))
-                            # Lets confirm we found a matching union class
-                            sub_field.type_(**d)
-                            data[field.alias] = d
-                            break
-                        except ValidationError:
-                            continue
-            elif isinstance(data[field.alias], list) and len(field.sub_fields) == 1:
-                # list[Union[ClassA, ClassB]] field
-                for sub_data in data[field.alias]:
-                    for sub_field in field.sub_fields[0].sub_fields or []:
-                        try:
-                            d = dict(sub_data)
-                            d.update(data_default_none(sub_field.type_, d))
-                            # Lets confirm we found a matching union class
-                            sub_field.type_(**d)
-                            sub_data.update(d)
-                            break
-                        except ValidationError:
-                            continue
 
     return data
 
@@ -89,28 +323,16 @@ class CSV(UserList[str]):
     """
 
     @classmethod
-    def __get_validators__(cls) -> Generator[Callable, None, None]:  # noqa: PLW3201
-        yield cls.validate
-        yield cls.length_validator
-
-    @classmethod
-    def validate(cls, value: str) -> list[str]:
-        items = [] if not value else value.split(",")
-        return items
+    def __get_pydantic_core_schema__(  # noqa: PLW3201
+        cls, source: type[Any], handler: GetCoreSchemaHandler
+    ) -> core_schema.CoreSchema:
+        return core_schema.with_info_before_validator_function(
+            cls._validate, core_schema.list_schema()
+        )
 
     @classmethod
-    def length_validator(
-        cls, v: "list[str]", values: dict, field: ModelField
-    ) -> "list[str]":
-        min_items = field.field_info.extra.get("csv_min_items")
-        max_items = field.field_info.extra.get("csv_max_items")
-
-        v_len = len(v)
-        if min_items is not None and v_len < min_items:
-            raise pydantic_errors.ListMinLengthError(limit_value=min_items)
-        if max_items is not None and v_len > max_items:
-            raise pydantic_errors.ListMaxLengthError(limit_value=max_items)
-        return v
+    def _validate(cls, __input_value: str, _: Any) -> list[str]:
+        return [] if not __input_value else __input_value.split(",")
 
 
 def cron_validator(value: str) -> str:

reconcile/utils/mr/__init__.py

@@ -1,3 +1,5 @@
+from typing import Any
+
 from reconcile.utils.mr.app_interface_reporter import CreateAppInterfaceReporter
 from reconcile.utils.mr.aws_access import CreateDeleteAwsAccessKey
 from reconcile.utils.mr.base import (
@@ -44,7 +46,7 @@ class UnknownMergeRequestTypeError(Exception):
     """
 
 
-def init_from_sqs_message(message) -> MergeRequestBase:
+def init_from_sqs_message(message: dict[str, Any]) -> MergeRequestBase:
     # First, let's find the classes that are inheriting from
     # MergeRequestBase and create a map where the class.name is
     # the key and the class itself is the value.

reconcile/utils/mr/app_interface_reporter.py

@@ -1,8 +1,9 @@
-from datetime import datetime
+from collections.abc import Iterable
 from pathlib import Path
 
 from ruamel.yaml.scalarstring import PreservedScalarString
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import (
     MergeRequestBase,
@@ -14,7 +15,9 @@ from reconcile.utils.mr.labels import AUTO_MERGE
 class CreateAppInterfaceReporter(MergeRequestBase):
     name = "create_app_interface_reporter_mr"
 
-    def __init__(self, reports, email_body, reports_path):
+    def __init__(
+        self, reports: Iterable[dict[str, str]], email_body: str, reports_path: str
+    ) -> None:
         self.reports = reports
         self.email_body = email_body
         self.reports_path = reports_path
@@ -23,7 +26,7 @@ class CreateAppInterfaceReporter(MergeRequestBase):
 
         self.labels = [AUTO_MERGE]
 
-        now = datetime.now()
+        now = utc_now()
         self.isodate = now.isoformat()
         self.ts = now.strftime("%Y%m%d%H%M%S")
 

reconcile/utils/mr/aws_access.py

@@ -18,7 +18,7 @@ BODY_TEMPLATE = PROJ_ROOT / "templates" / "aws_access_key_email.j2"
 class CreateDeleteAwsAccessKey(MergeRequestBase):
     name = "create_delete_aws_access_key_mr"
 
-    def __init__(self, account, path, key):
+    def __init__(self, account: str, path: str, key: str):
         self.account = account
         self.path = path.lstrip("/")
         self.key = key

reconcile/utils/mr/base.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from abc import (
     ABC,
@@ -11,10 +10,10 @@ from uuid import uuid4
 from gitlab.exceptions import GitlabError
 from jinja2 import Template
 
-from reconcile import typed_queries
 from reconcile.gql_definitions.fragments.user import User
 from reconcile.utils.constants import PROJ_ROOT
 from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.json import json_dumps
 from reconcile.utils.mr.labels import DO_NOT_MERGE_HOLD
 from reconcile.utils.sqs_gateway import SQSGateway
 
@@ -45,14 +44,14 @@ class MergeRequestBase(ABC):
 
     name = "merge-request-base"
 
-    def __init__(self):
+    def __init__(self) -> None:
         # Let's first get all the attributes from the instance
         # and use for the SQS Msg payload. With that, the msg
         # to the SQS is enough to create a new, similar, instance
         # of the child class.
         self.sqs_msg_data = {**self.__dict__}
 
-        self.labels = [DO_NOT_MERGE_HOLD]
+        self.labels: Iterable[str] = [DO_NOT_MERGE_HOLD]
 
         random_id = str(uuid4())[:6]
         self.branch = f"{self.name}-{random_id}"
@@ -110,19 +109,14 @@ class MergeRequestBase(ABC):
         }
 
     def infer_author(
-        self, author_email: str | None, all_users: Iterable[User] | None = None
+        self, github_user_id: str | None, all_users: Iterable[User] | None = None
     ) -> str | None:
-        if not author_email:
+        if not github_user_id:
             return None
         if not all_users:
             return None
 
-        username = author_email.split("@")[0]
-        users = None
-        if author_email.endswith(typed_queries.smtp.settings().mail_address):
-            users = [u for u in all_users if username == u.org_username]
-        elif author_email.endswith("users.noreply.github.com"):
-            users = [u for u in all_users if username == u.github_username]
+        users = [u for u in all_users if u.github_username == github_user_id]
 
         if users:
             return users[0].org_username
@@ -199,7 +193,7 @@ class MergeRequestBase(ABC):
             # logging this exception
             raise MergeRequestProcessingError(
                 f"error processing {self.name} changes "
-                f"{json.dumps(self.sqs_msg_data)} "
+                f"{json_dumps(self.sqs_msg_data)} "
                 f"into temporary branch {self.branch}. "
                 f"Reason: {err}"
             ) from err

reconcile/utils/mr/clusters_updates.py

@@ -1,4 +1,6 @@
+from collections.abc import Mapping
 from io import StringIO
+from typing import Any
 
 from ruamel.yaml import YAML
 
@@ -15,7 +17,7 @@ yaml.width = 4096
 class CreateClustersUpdates(MergeRequestBase):
     name = "create_clusters_updates_mr"
 
-    def __init__(self, clusters_updates):
+    def __init__(self, clusters_updates: Mapping[str, dict[str, Any]]):
         self.clusters_updates = clusters_updates
 
         super().__init__()
@@ -30,7 +32,7 @@ class CreateClustersUpdates(MergeRequestBase):
     def description(self) -> str:
         return DecisionCommand.APPROVED.value
 
-    def process(self, gitlab_cli: GitLabApi):
+    def process(self, gitlab_cli: GitLabApi) -> None:
         changes = False
         for cluster_name, cluster_updates in self.clusters_updates.items():
             if not cluster_updates:

reconcile/utils/mr/notificator.py

@@ -1,9 +1,9 @@
 import logging
-from datetime import datetime
 from pathlib import Path
 
 from pydantic import BaseModel
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import (
     MergeRequestBase,
@@ -35,7 +35,7 @@ class CreateAppInterfaceNotificator(MergeRequestBase):
         email_base_path: Path = Path("data") / "app-interface" / "emails",
         dry_run: bool = False,
     ):
-        self._notification_as_dict = notification.dict()
+        self._notification_as_dict = notification.model_dump()
         super().__init__()
         self._notification = notification
         self._email_base_path = email_base_path
@@ -58,7 +58,7 @@ class CreateAppInterfaceNotificator(MergeRequestBase):
         )
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        now = datetime.now()
+        now = utc_now()
         ts = now.strftime("%Y%m%d%H%M%S")
         short_date = now.strftime("%Y-%m-%d")
 

reconcile/utils/mr/ocm_upgrade_scheduler_org_updates.py

@@ -1,3 +1,6 @@
+from collections.abc import Mapping
+from typing import Any
+
 from ruamel import yaml
 
 from reconcile.utils.gitlab_api import GitLabApi
@@ -8,7 +11,7 @@ from reconcile.utils.mr.labels import AUTO_MERGE
 class CreateOCMUpgradeSchedulerOrgUpdates(MergeRequestBase):
     name = "create_ocm_upgrade_scheduler_org_updates_mr"
 
-    def __init__(self, updates_info):
+    def __init__(self, updates_info: Mapping[str, Any]):
         self.updates_info = updates_info
 
         super().__init__()