qontract-reconcile 0.10.2.dev310__py3-none-any.whl → 0.10.2.dev439__py3-none-any.whl

reconcile/database_access_manager.py

@@ -9,10 +9,9 @@ from string import (
 from typing import (
     Any,
     TypedDict,
-    cast,
 )
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 from sretoolbox.container.image import Image
 
 from reconcile import openshift_base, queries
@@ -44,7 +43,6 @@ from reconcile.utils.state import (
 )
 from reconcile.utils.vault import (
     VaultClient,
-    _VaultClient,
 )
 
 QONTRACT_INTEGRATION = "database-access-manager"
@@ -63,17 +61,19 @@ def get_database_access_namespaces(
     return query(query_func).namespaces_v1 or []
 
 
-class DatabaseConnectionParameters(BaseModel):
-    host: str
-    port: str
-    user: str
-    password: str
-    database: str
+class DatabaseConnectionParameters(
+    BaseModel, validate_by_name=True, validate_by_alias=True
+):
+    host: str = Field(..., alias="db.host")
+    port: str = Field(..., alias="db.port")
+    user: str = Field(..., alias="db.user")
+    password: str = Field(..., alias="db.password")
+    database: str = Field(..., alias="db.name")
 
 
 class PSQLScriptGenerator(BaseModel):
     db_access: DatabaseAccessV1
-    current_db_access: DatabaseAccessV1 | None
+    current_db_access: DatabaseAccessV1 | None = None
     connection_parameter: DatabaseConnectionParameters
     admin_connection_parameter: DatabaseConnectionParameters
     engine: str
@@ -227,7 +227,7 @@ def get_db_engine(resource: NamespaceTerraformResourceRDSV1) -> str:
 
 class JobData(BaseModel):
     engine: str
-    name_suffix: str
+    name: str
     image: str
     service_account_name: str
     rds_admin_secret_name: str
@@ -236,7 +236,7 @@ class JobData(BaseModel):
 
 
 def get_job_spec(job_data: JobData) -> OpenshiftResource:
-    job_name = f"dbam-{job_data.name_suffix}"
+    job_name = job_data.name
 
     image_tag = Image(job_data.image).tag
     image_pull_policy = "Always" if image_tag == "latest" else "IfNotPresent"
@@ -393,14 +393,6 @@ def get_service_account_spec(name: str) -> OpenshiftResource:
     )
 
 
-class DBAMResource(BaseModel):
-    resource: OpenshiftResource
-    clean_up: bool
-
-    class Config:
-        arbitrary_types_allowed = True
-
-
 class JobStatusCondition(BaseModel):
     type: str
 
@@ -426,18 +418,12 @@ def _populate_resources(
     user_connection: DatabaseConnectionParameters,
     admin_connection: DatabaseConnectionParameters,
     current_db_access: DatabaseAccessV1 | None = None,
-) -> list[DBAMResource]:
+) -> list[OpenshiftResource]:
     if user_connection.database == admin_connection.database:
         raise ValueError(f"Can not use default database {admin_connection.database}")
 
-    managed_resources: list[DBAMResource] = []
     # create service account
-    managed_resources.append(
-        DBAMResource(
-            resource=get_service_account_spec(resource_prefix),
-            clean_up=True,
-        )
-    )
+    service_account_resource = get_service_account_spec(resource_prefix)
 
     # create script secret
     generator = PSQLScriptGenerator(
@@ -448,74 +434,107 @@ def _populate_resources(
         engine=engine,
     )
     script_secret_name = f"{resource_prefix}-script"
-    managed_resources.extend([
-        DBAMResource(
-            resource=generate_script_secret_spec(
-                script_secret_name,
-                generator.generate_script(),
-            ),
-            clean_up=True,
-        ),
-        # create user secret
-        DBAMResource(
-            resource=generate_user_secret_spec(resource_prefix, user_connection),
-            clean_up=False,
-        ),
-    ])
+    secret_script_resource = generate_script_secret_spec(
+        script_secret_name,
+        generator.generate_script(),
+    )
+
+    # create user secret
+    user_secret_resource = generate_user_secret_spec(
+        resource_prefix,
+        user_connection,
+    )
+
     # create pull secret
-    labels = pull_secret["labels"] or {}
-    pull_secret_resources = orb.fetch_provider_vault_secret(
+    pull_secret_resource = orb.fetch_provider_vault_secret(
         path=pull_secret["path"],
         version=pull_secret["version"],
         name=f"{resource_prefix}-pull-secret",
-        labels=labels,
+        labels=pull_secret["labels"] or {},
         annotations=pull_secret["annotations"] or {},
         type=pull_secret["type"],
         integration=QONTRACT_INTEGRATION,
         integration_version=QONTRACT_INTEGRATION_VERSION,
         settings=settings,
     )
-    managed_resources.extend([
-        DBAMResource(resource=pull_secret_resources, clean_up=True),
-        # create job
-        DBAMResource(
-            resource=get_job_spec(
-                JobData(
-                    engine=engine,
-                    name_suffix=db_access.name,
-                    image=job_image,
-                    service_account_name=resource_prefix,
-                    rds_admin_secret_name=admin_secret_name,
-                    script_secret_name=script_secret_name,
-                    pull_secret=f"{resource_prefix}-pull-secret",
-                )
-            ),
-            clean_up=True,
-        ),
-    ])
 
-    return managed_resources
+    job_resource = get_job_spec(
+        JobData(
+            engine=engine,
+            name=resource_prefix,
+            image=job_image,
+            service_account_name=resource_prefix,
+            rds_admin_secret_name=admin_secret_name,
+            script_secret_name=script_secret_name,
+            pull_secret=f"{resource_prefix}-pull-secret",
+        )
+    )
+
+    return [
+        service_account_resource,
+        secret_script_resource,
+        user_secret_resource,
+        pull_secret_resource,
+        job_resource,
+    ]
 
 
 def _generate_password() -> str:
     return "".join(choices(ascii_letters + digits, k=32))
 
 
-class _DBDonnections(TypedDict):
+class DBConnections(TypedDict):
     user: DatabaseConnectionParameters
     admin: DatabaseConnectionParameters
 
 
+def _decode_secret_value(value: str) -> str:
+    return base64.b64decode(value).decode("utf-8")
+
+
+def _build_user_connection(
+    db_access: DatabaseAccessV1,
+    admin_secret: dict[str, Any],
+    user_secret: dict[str, Any],
+) -> DatabaseConnectionParameters:
+    """
+    Build user connection parameters.
+
+    If user secret exists, use values from it as that's the one used to provision new database user.
+    Otherwise, generate a new password, this info will be saved as cluster secret.
+    After job completes, the secret will be saved to vault then deleted from the cluster.
+
+    Args:
+        db_access (DatabaseAccessV1): Database access definition from app-interface.
+        admin_secret (dict[str, Any]): Admin secret fetched from the cluster.
+        user_secret (dict[str, Any]): User secret fetched from the cluster, may be empty if not exists.
+    Returns:
+        DatabaseConnectionParameters: Connection parameters for the database user.
+    """
+    if user_secret:
+        return DatabaseConnectionParameters(
+            host=_decode_secret_value(user_secret["data"]["db.host"]),
+            port=_decode_secret_value(user_secret["data"]["db.port"]),
+            user=_decode_secret_value(user_secret["data"]["db.user"]),
+            password=_decode_secret_value(user_secret["data"]["db.password"]),
+            database=_decode_secret_value(user_secret["data"]["db.name"]),
+        )
+    return DatabaseConnectionParameters(
+        host=_decode_secret_value(admin_secret["data"]["db.host"]),
+        port=_decode_secret_value(admin_secret["data"]["db.port"]),
+        user=db_access.username,
+        password=_generate_password(),
+        database=db_access.database,
+    )
+
+
 def _create_database_connection_parameter(
     db_access: DatabaseAccessV1,
     namespace_name: str,
     oc: OCClient,
     admin_secret_name: str,
     user_secret_name: str,
-) -> _DBDonnections:
-    def _decode_secret_value(value: str) -> str:
-        return base64.b64decode(value).decode("utf-8")
-
+) -> DBConnections:
     user_secret = oc.get(
         namespace_name,
         "Secret",
@@ -528,26 +547,11 @@ def _create_database_connection_parameter(
         admin_secret_name,
         allow_not_found=False,
     )
-
-    if user_secret:
-        password = _decode_secret_value(user_secret["data"]["db.password"])
-        host = _decode_secret_value(user_secret["data"]["db.host"])
-        user = _decode_secret_value(user_secret["data"]["db.user"])
-        port = _decode_secret_value(user_secret["data"]["db.port"])
-        database = _decode_secret_value(user_secret["data"]["db.name"])
-    else:
-        host = _decode_secret_value(admin_secret["data"]["db.host"])
-        port = _decode_secret_value(admin_secret["data"]["db.port"])
-        user = db_access.username
-        password = _generate_password()
-        database = db_access.database
-    return _DBDonnections(
-        user=DatabaseConnectionParameters(
-            host=host,
-            port=port,
-            user=user,
-            password=password,
-            database=database,
+    return DBConnections(
+        user=_build_user_connection(
+            db_access=db_access,
+            admin_secret=admin_secret,
+            user_secret=user_secret,
         ),
         admin=DatabaseConnectionParameters(
             host=_decode_secret_value(admin_secret["data"]["db.host"]),
@@ -559,10 +563,10 @@ def _create_database_connection_parameter(
     )
 
 
-def _db_access_acccess_is_valid(db_acces: DatabaseAccessV1) -> bool:
+def _db_access_access_is_valid(db_access: DatabaseAccessV1) -> bool:
     found_schema: set[str] = set()
 
-    for schema in db_acces.access or []:
+    for schema in db_access.access or []:
         if schema.target.dbschema in found_schema:
             return False
         found_schema.add(schema.target.dbschema)
@@ -577,21 +581,22 @@ class JobFailedError(Exception):
 def _process_db_access(
     dry_run: bool,
     state: State,
+    state_key: str,
     db_access: DatabaseAccessV1,
     namespace: NamespaceV1,
     admin_secret_name: str,
     engine: str,
     settings: dict[Any, Any],
     vault_output_path: str,
-    vault_client: _VaultClient,
+    vault_client: VaultClient,
 ) -> None:
-    if not _db_access_acccess_is_valid(db_access):
+    if not _db_access_access_is_valid(db_access):
         raise ValueError("Duplicate schema in access list.")
 
     current_db_access: DatabaseAccessV1 | None = None
-    if state.exists(db_access.name):
-        current_state = state.get(db_access.name)
-        if current_state == db_access.dict(by_alias=True):
+    if state.exists(state_key):
+        current_state = state.get(state_key)
+        if current_state == db_access.model_dump(by_alias=True):
             return
         current_db_access = DatabaseAccessV1(**current_state)
         if current_db_access.database != db_access.database:
@@ -602,9 +607,9 @@ def _process_db_access(
     cluster_name = namespace.cluster.name
     namespace_name = namespace.name
 
-    resource_prefix = f"dbam-{db_access.name}"
+    resource_prefix = f"dbam-{state_key.replace('/', '-')}"
     with OC_Map(
-        clusters=[namespace.cluster.dict(by_alias=True)],
+        clusters=[namespace.cluster.model_dump(by_alias=True)],
         integration=QONTRACT_INTEGRATION,
         settings=settings,
     ) as oc_map:
@@ -615,7 +620,7 @@ def _process_db_access(
             namespace_name,
             oc,
             admin_secret_name,
-            resource_prefix,
+            user_secret_name=resource_prefix,
         )
 
         sql_query_settings = settings.get("sqlQuery")
@@ -644,7 +649,7 @@ def _process_db_access(
         job = oc.get(
             namespace_name,
             "Job",
-            f"dbam-{db_access.name}",
+            resource_prefix,
             allow_not_found=True,
         )
         if not job:
@@ -654,8 +659,8 @@ def _process_db_access(
                     oc_map=oc_map,
                     cluster=cluster_name,
                     namespace=namespace_name,
-                    resource_type=r.resource.kind,
-                    resource=r.resource,
+                    resource_type=r.kind,
+                    resource=r,
                     wait_for_namespace=False,
                 )
             return
@@ -667,34 +672,31 @@ def _process_db_access(
         )
         if job_status.is_complete():
             if job_status.has_errors():
-                raise JobFailedError(
-                    f"Job dbam-{db_access.name} failed, please check logs"
-                )
+                raise JobFailedError(f"Job {resource_prefix} failed, please check logs")
             if not dry_run and not db_access.delete:
                 secret = {
-                    "path": f"{vault_output_path}/{QONTRACT_INTEGRATION}/{cluster_name}/{namespace_name}/{db_access.name}",
-                    "data": connections["user"].dict(by_alias=True),
+                    "path": f"{vault_output_path}/{QONTRACT_INTEGRATION}/{state_key}",
+                    "data": connections["user"].model_dump(by_alias=True),
                 }
                 vault_client.write(secret, decode_base64=False)
             logging.debug("job completed, cleaning up")
             for r in managed_resources:
-                if r.clean_up:
-                    openshift_base.delete(
-                        dry_run=dry_run,
-                        oc_map=oc_map,
-                        cluster=cluster_name,
-                        namespace=namespace_name,
-                        resource_type=r.resource.kind,
-                        name=r.resource.name,
-                        enable_deletion=True,
-                    )
+                openshift_base.delete(
+                    dry_run=dry_run,
+                    oc_map=oc_map,
+                    cluster=cluster_name,
+                    namespace=namespace_name,
+                    resource_type=r.kind,
+                    name=r.name,
+                    enable_deletion=True,
+                )
             state.add(
-                db_access.name,
-                value=db_access.dict(by_alias=True),
+                state_key,
+                value=db_access.model_dump(by_alias=True),
                 force=True,
             )
         else:
-            logging.info(f"Job dbam-{db_access.name} appears to be still running")
+            logging.info(f"Job {resource_prefix} appears to be still running")
 
 
 class DBAMIntegrationParams(PydanticRunParams):
@@ -708,7 +710,7 @@ class DatabaseAccessManagerIntegration(QontractReconcileIntegration):
 
     def run(self, dry_run: bool) -> None:
         settings = queries.get_app_interface_settings()
-        vault_client = cast("_VaultClient", VaultClient())
+        vault_client = VaultClient.get_instance()
 
         state = init_state(
             integration=QONTRACT_INTEGRATION, secret_reader=self.secret_reader
@@ -736,9 +738,11 @@ class DatabaseAccessManagerIntegration(QontractReconcileIntegration):
 
                     for db_access in resource.database_access or []:
                         try:
+                            state_key = f"{external_resource.provisioner.name}/{resource.identifier}/{db_access.name}"
                             _process_db_access(
                                 dry_run,
                                 state,
+                                state_key,
                                 db_access,
                                 namespace,
                                 admin_secret_name,

reconcile/deadmanssnitch.py

@@ -1,7 +1,4 @@
 import logging
-from typing import (
-    cast,
-)
 
 from pydantic import BaseModel
 
@@ -22,7 +19,6 @@ from reconcile.utils.runtime.integration import (
 from reconcile.utils.semver_helper import make_semver
 from reconcile.utils.vault import (
     VaultClient,
-    _VaultClient,
 )
 
 QONTRACT_INTEGRATION = "deadmanssnitch"
@@ -51,7 +47,7 @@ class DeadMansSnitchIntegration(QontractReconcileIntegration[NoParams]):
         super().__init__(NoParams())
         self.qontract_integration_version = make_semver(0, 1, 0)
         self.settings = get_deadmanssnitch_settings()
-        self.vault_client = cast("_VaultClient", VaultClient())
+        self.vault_client = VaultClient.get_instance()
 
     @staticmethod
     def get_snitch_name(cluster: ClusterV1) -> str:

reconcile/dynatrace_token_provider/integration.py

@@ -82,7 +82,7 @@ class DynatraceTokenProviderIntegration(QontractReconcileIntegration[NoParams]):
         return {
             "version": QONTRACT_INTEGRATION_VERSION,
             "specs": {
-                spec.name: spec.dict()
+                spec.name: spec.model_dump()
                 for spec in get_dynatrace_token_provider_token_specs()
             },
         }

reconcile/endpoints_discovery/integration.py

@@ -150,7 +150,10 @@ class EndpointsDiscoveryIntegration(
             return []
 
         routes = defaultdict(list)
-        for item in oc.get_items(kind="Route", namespace=namespace.name):
+        for item in oc.get_items(
+            kind="Route.route.openshift.io",
+            namespace=namespace.name,
+        ):
             tls = bool(item["spec"].get("tls"))
             host = item["spec"]["host"]
             # group all routes with the same hostname/tls

reconcile/endpoints_discovery/merge_request.py

@@ -89,7 +89,7 @@ class Renderer:
 
     def render_description(self, hash: str) -> str:
         """Render the description for a merge request."""
-        return DESC.safe_substitute(EPDInfo(hash=hash).dict())
+        return DESC.safe_substitute(EPDInfo(hash=hash).model_dump())
 
     def render_title(self) -> str:
         """Render the title for a merge request."""

reconcile/endpoints_discovery/merge_request_manager.py

@@ -1,8 +1,7 @@
 import hashlib
-import json
 import logging
 from collections.abc import Sequence
-from typing import Any, TypeAlias
+from typing import Any
 
 from gitlab.exceptions import GitlabGetError
 from pydantic import BaseModel
@@ -16,6 +15,7 @@ from reconcile.endpoints_discovery.merge_request import (
     Renderer,
 )
 from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.json import json_dumps
 from reconcile.utils.merge_request_manager.merge_request_manager import (
     MergeRequestManagerBase,
 )
@@ -64,22 +64,20 @@ class Endpoint(BaseModel):
 
     @property
     def hash(self) -> str:
-        return hashlib.sha256(
-            json.dumps(self.dict(), sort_keys=True).encode()
-        ).hexdigest()
+        return hashlib.sha256(json_dumps(self.model_dump()).encode()).hexdigest()
 
 
-EndpointsToAdd: TypeAlias = list[Endpoint]
-EndpointsToChange: TypeAlias = list[Endpoint]
-EndpointsToDelete: TypeAlias = list[Endpoint]
+EndpointsToAdd = list[Endpoint]
+EndpointsToChange = list[Endpoint]
+EndpointsToDelete = list[Endpoint]
 
 
 class App(BaseModel):
     name: str
     path: str
-    endpoints_to_add: EndpointsToAdd = EndpointsToAdd()
-    endpoints_to_change: EndpointsToChange = EndpointsToChange()
-    endpoints_to_delete: EndpointsToDelete = EndpointsToDelete()
+    endpoints_to_add: EndpointsToAdd = []
+    endpoints_to_change: EndpointsToChange = []
+    endpoints_to_delete: EndpointsToDelete = []
 
     @property
     def hash(self) -> str:

reconcile/external_resources/factories.py

@@ -2,7 +2,7 @@ from abc import (
     ABC,
     abstractmethod,
 )
-from typing import Generic, TypeVar
+from typing import TypeVar
 
 from reconcile.external_resources.aws import (
     AWSDefaultResourceFactory,
@@ -32,16 +32,8 @@ from reconcile.utils.secret_reader import SecretReaderBase
 
 T = TypeVar("T")
 
-AWS_DEFAULT_TAGS = [
-    {
-        "tags": {
-            "app": "app-sre-infra",
-        }
-    }
-]
 
-
-class ObjectFactory(Generic[T]):
+class ObjectFactory[T]:
     def __init__(
         self, factories: dict[str, T], default_factory: T | None = None
     ) -> None:
@@ -123,12 +115,14 @@ class AWSExternalResourceFactory(ExternalResourceFactory):
         secret_reader: SecretReaderBase,
         provision_factories: ObjectFactory[ModuleProvisionDataFactory],
         resource_factories: ObjectFactory[AWSResourceFactory],
+        default_tags: dict[str, str],
     ):
         self.provision_factories = provision_factories
         self.resource_factories = resource_factories
         self.module_inventory = module_inventory
         self.er_inventory = er_inventory
         self.secret_reader = secret_reader
+        self.default_tags = default_tags
 
     def create_external_resource(
         self,
@@ -137,8 +131,7 @@ class AWSExternalResourceFactory(ExternalResourceFactory):
     ) -> ExternalResource:
         f = self.resource_factories.get_factory(spec.provider)
         data = f.resolve(spec, module_conf)
-        data["tags"] = spec.tags(integration=QONTRACT_INTEGRATION)
-        data["default_tags"] = AWS_DEFAULT_TAGS
+        data["tags"] = self.default_tags | spec.tags(integration=QONTRACT_INTEGRATION)
 
         region = data.get("region")
         if region:

reconcile/external_resources/integration.py

@@ -45,7 +45,7 @@ from reconcile.utils.secret_reader import SecretReaderBase, create_secret_reader
 def fetch_current_state(
     ri: ResourceInventory, oc: OCCli, cluster: str, namespace: str
 ) -> None:
-    for item in oc.get_items("Job", namespace=namespace):
+    for item in oc.get_items("Job.batch", namespace=namespace):
         r = OpenshiftResource(item, QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION)
         ri.add_current(cluster, namespace, "Job", r.name, r)
 

reconcile/external_resources/manager.py

@@ -1,7 +1,7 @@
 import logging
 from collections import Counter
 from collections.abc import Iterable
-from datetime import UTC, datetime
+from typing import cast
 
 from sretoolbox.utils import threaded
 
@@ -41,6 +41,7 @@ from reconcile.external_resources.state import (
 from reconcile.gql_definitions.external_resources.external_resources_settings import (
     ExternalResourcesSettingsV1,
 )
+from reconcile.utils.datetime_util import utc_now
 from reconcile.utils.external_resource_spec import (
     ExternalResourceSpec,
 )
@@ -67,6 +68,7 @@ def setup_factories(
                 resource_factories=setup_aws_resource_factories(
                     er_inventory, secret_reader
                 ),
+                default_tags=cast("dict[str, str]", settings.default_tags),
             )
         }
     )
@@ -141,7 +143,7 @@ class ExternalResourcesManager:
     def _resource_drift_detection_ttl_expired(
         self, reconciliation: Reconciliation, state: ExternalResourceState
     ) -> bool:
-        return (datetime.now(state.ts.tzinfo) - state.ts).total_seconds() > (
+        return (utc_now() - state.ts).total_seconds() > (
             reconciliation.module_configuration.reconcile_drift_interval_minutes * 60
         )
 
@@ -242,7 +244,7 @@ class ExternalResourcesManager:
             reconciliation = Reconciliation(
                 key=key,
                 resource_hash=resource.hash(),
-                input=resource.json(),
+                input=resource.export(),
                 action=Action.APPLY,
                 module_configuration=module_conf,
                 linked_resources=self._find_linked_resources(spec),
@@ -355,7 +357,7 @@ class ExternalResourcesManager:
     def _set_resource_reconciliation_in_progress(
         self, r: Reconciliation, state: ExternalResourceState
     ) -> None:
-        state.ts = datetime.now(UTC)
+        state.ts = utc_now()
         if r.action == Action.APPLY:
             state.resource_status = ResourceStatus.IN_PROGRESS
         elif r.action == Action.DESTROY:
@@ -448,7 +450,8 @@ class ExternalResourcesManager:
             r
             for r in desired_r.union(deleted_r)
             if self._reconciliation_needs_dry_run_run(
-                r, self.state_mgr.get_external_resource_state(key=r.key)
+                r,
+                self.state_mgr.get_external_resource_state(key=r.key),
             )
         }
 

reconcile/external_resources/meta.py

@@ -10,7 +10,6 @@ SECRET_ANN_PROVISIONER = SECRET_ANN_PREFIX + "/provisioner_name"
 SECRET_ANN_PROVIDER = SECRET_ANN_PREFIX + "/provider"
 SECRET_ANN_IDENTIFIER = SECRET_ANN_PREFIX + "/identifier"
 SECRET_UPDATED_AT = SECRET_ANN_PREFIX + "/updated_at"
-SECRET_UPDATED_AT_TIMEFORMAT = "%Y-%m-%dT%H:%M:%SZ"
 
 FLAG_RESOURCE_MANAGED_BY_ERV2 = "managed_by_erv2"
 FLAG_DELETE_RESOURCE = "delete"

reconcile/external_resources/metrics.py

@@ -13,7 +13,7 @@ from reconcile.utils.metrics import (
 
 
 class ExternalResourcesBaseMetric(BaseModel):
-    integration = normalize_integration_name(QONTRACT_INTEGRATION)
+    integration: str = normalize_integration_name(QONTRACT_INTEGRATION)
     app: str
     environment: str
     provision_provider: str