qontract-reconcile 0.10.2.dev361__py3-none-any.whl → 0.10.2.dev474__py3-none-any.whl

reconcile/utils/models.py

@@ -1,7 +1,5 @@
 from collections import UserList
 from collections.abc import (
-    Callable,
-    Generator,
     MutableMapping,
 )
 from typing import Any
@@ -9,75 +7,311 @@ from typing import Any
 from croniter import croniter
 from pydantic import (
     BaseModel,
+    GetCoreSchemaHandler,
     ValidationError,
 )
-from pydantic import errors as pydantic_errors
-from pydantic.fields import ModelField
+from pydantic_core import core_schema
 
 DEFAULT_STRING = "I was too lazy to define a string here"
 DEFAULT_INT = 42
+DEFAULT_BOOL = False
+DEFAULT_JSON_STR = "{}"
+DEFAULT_JSON: dict = {}
+
+DataType = MutableMapping[str, Any] | str | int | bool | list | None
+
+
+def _process_field_with_alias(
+    field_info: dict[str, Any],
+    data: DataType,
+    alias: str | None,
+    use_defaults: bool,
+    definitions: list[dict[str, Any]] | None,
+) -> DataType:
+    """Process a field by recursively calling data_default_none and handling alias.
+
+    This helper function encapsulates the common pattern used by 'default',
+    'nullable', and 'model' field types where we need to:
+    1. Recursively process the field's schema
+    2. Handle data extraction via alias if data is a dict
+    3. Update the data dict with the processed value if alias exists
+
+    Args:
+        field_info: Schema information for the field to process
+        data: Input data (can be dict, primitive, list, or None)
+        alias: Field alias for dict key access, or None for unnamed fields
+        use_defaults: Whether to use default values for missing required fields
+        definitions: Schema definitions for definition-ref fields
+
+    Returns:
+        Processed data value, or updated data dict if alias was provided
+    """
+    # Recursively process the field's schema
+    value = data_default_none(
+        field_info["schema"],
+        data.get(alias) if isinstance(data, MutableMapping) and alias else data,
+        use_defaults=use_defaults,
+        definitions=definitions,
+    )
+
+    # For unnamed fields (in recursive calls), return value directly
+    if not alias:
+        return value
+
+    # For named fields, update the data dict
+    if not isinstance(data, MutableMapping):
+        raise TypeError(
+            f"Expected MutableMapping for field '{alias}', got {type(data).__name__}"
+        )
+    data[alias] = value
+    return data
 
 
 def data_default_none(
-    klass: type[BaseModel], data: MutableMapping[str, Any], use_defaults: bool = True
-) -> MutableMapping[str, Any]:
-    """Set default values to None for required but optional fields."""
-    for field in klass.__fields__.values():
-        if not field.required:
-            continue
-
-        if field.alias not in data:
-            # Settings defaults
-            if field.allow_none:
-                data[field.alias] = None
-            elif not use_defaults:
-                raise ValueError(f"Field {field.alias} is required but not set.")
-            elif isinstance(field.type_, type) and issubclass(field.type_, str):
-                data[field.alias] = DEFAULT_STRING
-            elif isinstance(field.type_, type) and issubclass(field.type_, bool):
-                data[field.alias] = False
-            elif isinstance(field.type_, type) and issubclass(field.type_, int):
-                data[field.alias] = DEFAULT_INT
-        elif isinstance(field.type_, type) and issubclass(field.type_, BaseModel):
-            if isinstance(data[field.alias], dict):
-                data[field.alias] = data_default_none(field.type_, data[field.alias])
-            if isinstance(data[field.alias], list):
-                data[field.alias] = [
-                    data_default_none(field.type_, item)
-                    for item in data[field.alias]
-                    if isinstance(item, dict)
+    klass: type[BaseModel] | dict,
+    data: DataType,
+    use_defaults: bool = True,
+    definitions: list[dict[str, Any]] | None = None,
+) -> DataType:
+    """Set default values for required fields in Pydantic model data.
+
+    This function recursively processes Pydantic model schemas and fills in
+    missing fields with appropriate default values based on their type. It's
+    primarily used to handle GraphQL query results where fields may be missing.
+
+    Args:
+        klass: Pydantic BaseModel class or schema dict to process
+        data: Input data to fill with defaults (can be dict, primitive, list, or None)
+        use_defaults: If True, use type-based defaults for missing required fields.
+                     If False, raise ValueError for missing required fields.
+        definitions: Schema definitions for definition-ref fields (e.g., VaultSecret).
+                    Automatically extracted from models with "definitions" schema type.
+
+    Returns:
+        Data with defaults filled in. Return type matches input data structure.
+
+    Field handling behavior:
+        - Optional fields (T | None): Default to None if missing
+        - Required primitives: Use DEFAULT_STRING, DEFAULT_INT, or DEFAULT_BOOL
+        - Fields with explicit defaults: Use the provided default value
+        - Lists: Default to empty list [] if missing, recursively process items
+        - Nested models: Recursively process with data_default_none
+        - Union types: Try each variant until one validates ("first match wins")
+
+    Example:
+        >>> class User(BaseModel):
+        ...     name: str
+        ...     age: int | None
+        >>> data_default_none(User, {})
+        {'name': 'I was too lazy to define a string here', 'age': None}
+    """
+    # =========================================================================
+    # Extract schema from BaseModel class or use provided schema dict
+    # =========================================================================
+    if isinstance(klass, dict):
+        # Recursive calls pass schema dict directly
+        schema = klass
+    else:
+        # Extract schema from Pydantic BaseModel class
+        match klass.__pydantic_core_schema__["type"]:
+            case "definitions":
+                # Models using definition-ref (e.g., VaultSecret fragments)
+                schema = klass.__pydantic_core_schema__["schema"]["schema"]
+                definitions = klass.__pydantic_core_schema__["definitions"]
+            case _:
+                # Standard models
+                schema = klass.__pydantic_core_schema__["schema"]
+
+    # =========================================================================
+    # Get fields to process - either model fields or single schema
+    # =========================================================================
+    if schema["type"] == "model-fields":
+        # Multiple fields in a model
+        fields = schema["fields"].items()
+    else:
+        # Single field schema (used in recursive calls for field processing)
+        fields = [(None, schema)]
+
+    # =========================================================================
+    # Process each field according to its schema type
+    # =========================================================================
+    for name, field_info in fields:
+        # Use alias if defined (e.g., "serverUrl" instead of "server_url")
+        alias = field_info.get("validation_alias", name)
+
+        match field_info["type"]:
+            # =================================================================
+            # PRIMITIVE TYPES: str, int, bool
+            # =================================================================
+            case "str" | "int" | "bool" | "json":
+                # If data is provided, return it as-is (don't override user data)
+                if data is not None:
+                    return data
+
+                # If no defaults allowed, raise error for missing required field
+                if not use_defaults:
+                    raise ValueError(f"Field {alias or ''} is required but not set.")
+
+                # Return type-specific default value
+                match field_info["type"]:
+                    case "str":
+                        return DEFAULT_STRING
+                    case "int":
+                        return DEFAULT_INT
+                    case "bool":
+                        return DEFAULT_BOOL
+                    case "json":
+                        return DEFAULT_JSON_STR
+
+            # =================================================================
+            # FIELDS WITH EXPLICIT DEFAULT VALUES
+            # =================================================================
+            case "default":
+                # If no data provided, use the field's default value
+                if data is None:
+                    return field_info["default"]
+
+                # Data exists - recursively process it in case it's a complex type
+                return _process_field_with_alias(
+                    field_info, data, alias, use_defaults, definitions
+                )
+
+            # =================================================================
+            # NULLABLE/OPTIONAL FIELDS: T | None
+            # =================================================================
+            case "nullable":
+                # If no data provided, optional fields default to None
+                if data is None:
+                    return None
+
+                # Data exists - recursively process the non-None value
+                return _process_field_with_alias(
+                    field_info, data, alias, use_defaults, definitions
+                )
+
+            # =================================================================
+            # MODEL FIELDS: Nested Pydantic models within parent model
+            # =================================================================
+            case "model-field":
+                # Ensure data is a dict (create empty dict if needed)
+                # This allows processing models even when parent data is missing
+                if not isinstance(data, MutableMapping):
+                    data = {}
+
+                # Recursively process the nested model
+                data[alias] = data_default_none(
+                    field_info["schema"],
+                    data.get(alias),
+                    use_defaults=use_defaults,
+                    definitions=definitions,
+                )
+
+            # =================================================================
+            # STANDALONE MODELS: BaseModel used as field type
+            # =================================================================
+            case "model":
+                # Recursively process nested BaseModel
+                return _process_field_with_alias(
+                    field_info, data, alias, use_defaults, definitions
+                )
+
+            # =================================================================
+            # DEFINITION REFERENCES: e.g., VaultSecret fragments
+            # =================================================================
+            case "definition-ref":
+                # Definition-ref fields require a MutableMapping data structure
+                assert isinstance(data, MutableMapping)
+
+                # Definitions are required for resolving references
+                if not definitions:
+                    raise RuntimeError(
+                        "definitions parameter is required for definition-ref fields"
+                    )
+
+                # Find the referenced schema definition by ref ID
+                ref_schema = next(
+                    definition
+                    for definition in definitions
+                    if definition["ref"] == field_info["schema_ref"]
+                )
+
+                # Recursively process using the referenced schema
+                value = data_default_none(
+                    ref_schema["schema"],
+                    data.get(alias) if alias else data,
+                    use_defaults=use_defaults,
+                    definitions=definitions,
+                )
+
+                # For unnamed fields, return value directly
+                if not alias:
+                    return value
+
+                data[alias] = value
+
+            # =================================================================
+            # UNION TYPES: T1 | T2 | ...
+            # =================================================================
+            case "union":
+                # Strategy: Try each union variant until one validates successfully
+                # This is a "first match wins" approach
+                for sub_field in field_info["choices"]:
+                    try:
+                        # Try to process data with this union variant
+                        # use_defaults=False ensures we only match existing data
+                        sub_data = data_default_none(
+                            sub_field,
+                            # don't alter data structure here
+                            dict(data) if isinstance(data, MutableMapping) else data,
+                            use_defaults=use_defaults,
+                            definitions=definitions,
+                        )
+
+                        # Validate the result matches the expected type
+                        match sub_field["type"]:
+                            case "str":
+                                if not isinstance(sub_data, str):
+                                    raise ValueError()
+                            case "int":
+                                if not isinstance(sub_data, int):
+                                    raise ValueError()
+                            case "bool":
+                                if not isinstance(sub_data, bool):
+                                    raise ValueError()
+                            case "model":
+                                if not isinstance(sub_data, dict):
+                                    raise ValueError()
+                                # Validate the dict can be instantiated as the model
+                                sub_field["cls"].model_validate(sub_data)
+                                if isinstance(data, MutableMapping):
+                                    data.update(sub_data)
+
+                        # If we get here, this union variant matched successfully
+                        return data
+                    except (ValidationError, ValueError):
+                        # This variant didn't match, try the next one
+                        continue
+
+            # =================================================================
+            # LIST TYPES: list[T]
+            # =================================================================
+            case "list":
+                # If data is not a list, default to empty list
+                if not isinstance(data, list):
+                    return []
+
+                # Recursively process each list item
+                return [
+                    data_default_none(
+                        field_info["items_schema"],
+                        item.get(alias)
+                        if isinstance(item, MutableMapping) and alias
+                        else item,
+                        use_defaults=use_defaults,
+                        definitions=definitions,
+                    )
+                    for item in data
                 ]
-        elif field.sub_fields:
-            if all(
-                isinstance(sub_field.type_, type)
-                and issubclass(sub_field.type_, BaseModel)
-                for sub_field in field.sub_fields
-            ):
-                # Union[ClassA, ClassB] field
-                for sub_field in field.sub_fields:
-                    if isinstance(data[field.alias], dict):
-                        try:
-                            d = dict(data[field.alias])
-                            d.update(data_default_none(sub_field.type_, d))
-                            # Lets confirm we found a matching union class
-                            sub_field.type_(**d)
-                            data[field.alias] = d
-                            break
-                        except ValidationError:
-                            continue
-            elif isinstance(data[field.alias], list) and len(field.sub_fields) == 1:
-                # list[Union[ClassA, ClassB]] field
-                for sub_data in data[field.alias]:
-                    for sub_field in field.sub_fields[0].sub_fields or []:
-                        try:
-                            d = dict(sub_data)
-                            d.update(data_default_none(sub_field.type_, d))
-                            # Lets confirm we found a matching union class
-                            sub_field.type_(**d)
-                            sub_data.update(d)
-                            break
-                        except ValidationError:
-                            continue
 
     return data
 
@@ -89,28 +323,16 @@ class CSV(UserList[str]):
     """
 
     @classmethod
-    def __get_validators__(cls) -> Generator[Callable, None, None]:  # noqa: PLW3201
-        yield cls.validate
-        yield cls.length_validator
-
-    @classmethod
-    def validate(cls, value: str) -> list[str]:
-        items = [] if not value else value.split(",")
-        return items
+    def __get_pydantic_core_schema__(  # noqa: PLW3201
+        cls, source: type[Any], handler: GetCoreSchemaHandler
+    ) -> core_schema.CoreSchema:
+        return core_schema.with_info_before_validator_function(
+            cls._validate, core_schema.list_schema()
+        )
 
     @classmethod
-    def length_validator(
-        cls, v: "list[str]", values: dict, field: ModelField
-    ) -> "list[str]":
-        min_items = field.field_info.extra.get("csv_min_items")
-        max_items = field.field_info.extra.get("csv_max_items")
-
-        v_len = len(v)
-        if min_items is not None and v_len < min_items:
-            raise pydantic_errors.ListMinLengthError(limit_value=min_items)
-        if max_items is not None and v_len > max_items:
-            raise pydantic_errors.ListMaxLengthError(limit_value=max_items)
-        return v
+    def _validate(cls, __input_value: str, _: Any) -> list[str]:
+        return [] if not __input_value else __input_value.split(",")
 
 
 def cron_validator(value: str) -> str:

reconcile/utils/mr/app_interface_reporter.py

@@ -1,9 +1,9 @@
 from collections.abc import Iterable
-from datetime import datetime
 from pathlib import Path
 
 from ruamel.yaml.scalarstring import PreservedScalarString
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import (
     MergeRequestBase,
@@ -26,7 +26,7 @@ class CreateAppInterfaceReporter(MergeRequestBase):
 
         self.labels = [AUTO_MERGE]
 
-        now = datetime.now()
+        now = utc_now()
         self.isodate = now.isoformat()
         self.ts = now.strftime("%Y%m%d%H%M%S")
 

reconcile/utils/mr/notificator.py

@@ -1,9 +1,9 @@
 import logging
-from datetime import datetime
 from pathlib import Path
 
 from pydantic import BaseModel
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import (
     MergeRequestBase,
@@ -35,7 +35,7 @@ class CreateAppInterfaceNotificator(MergeRequestBase):
         email_base_path: Path = Path("data") / "app-interface" / "emails",
         dry_run: bool = False,
     ):
-        self._notification_as_dict = notification.dict()
+        self._notification_as_dict = notification.model_dump()
         super().__init__()
         self._notification = notification
         self._email_base_path = email_base_path
@@ -58,7 +58,7 @@ class CreateAppInterfaceNotificator(MergeRequestBase):
         )
 
     def process(self, gitlab_cli: GitLabApi) -> None:
-        now = datetime.now()
+        now = utc_now()
         ts = now.strftime("%Y%m%d%H%M%S")
         short_date = now.strftime("%Y-%m-%d")
 

reconcile/utils/mr/update_access_report_base.py

@@ -1,14 +1,13 @@
 import logging
 from abc import abstractmethod
 from collections.abc import Sequence
-from datetime import UTC, date
-from datetime import datetime as dt
 from pathlib import Path
 from typing import TypeVar
 
 from jinja2 import Template
 from pydantic import BaseModel
 
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.gitlab_api import GitLabApi
 from reconcile.utils.mr.base import MergeRequestBase
 from reconcile.utils.mr.labels import AUTO_MERGE
@@ -27,7 +26,7 @@ class UpdateAccessReportBase(MergeRequestBase):
         self.labels = [AUTO_MERGE]
         self._users = users
         self._workbook_file_name = str(workbook_path)
-        self._isodate = dt.now(tz=UTC).isoformat()
+        self._isodate = utc_now().isoformat()
         self._dry_run = dry_run
 
     @property
@@ -65,7 +64,7 @@ class UpdateAccessReportBase(MergeRequestBase):
 
     def _render_tracking_table_row(self, old_number_of_users: int) -> str:
         # | Date Reviewed | Number of Current Users | +/- Red Hat Users |
-        return f"| {date.today()} | {len(self._users)} | {len(self._users) - old_number_of_users} |\n"
+        return f"| {utc_now().date()} | {len(self._users)} | {len(self._users) - old_number_of_users} |\n"
 
     def _update_workbook(self, workbook_md: str) -> str:
         new_workbook_md = ""

reconcile/utils/mr/user_maintenance.py

@@ -2,7 +2,7 @@ from collections.abc import Iterable
 from enum import Enum
 from pathlib import Path
 
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, field_validator
 from ruamel import yaml
 
 from reconcile.utils.gitlab_api import GitLabApi
@@ -23,7 +23,8 @@ class PathSpec(BaseModel):
     type: PathTypes
     path: str
 
-    @validator("path")
+    @field_validator("path")
+    @classmethod
     def prepend_data_to_path(cls, v: str) -> str:
         return "data" + v