qontract-reconcile 0.10.1rc762__py3-none-any.whl → 0.10.1rc764__py3-none-any.whl

reconcile/external_resources/state.py

@@ -0,0 +1,246 @@
+import logging
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any
+
+import boto3
+from pydantic import BaseModel
+
+from reconcile.external_resources.model import (
+    ExternalResourceKey,
+    ExternalResourceModuleConfiguration,
+    Reconciliation,
+)
+
+DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+
+class StateNotFoundError(Exception):
+    pass
+
+
+class ReconcileStatus(str, Enum):
+    SUCCESS: str = "SUCCESS"
+    ERROR: str = "ERROR"
+    IN_PROGRESS: str = "IN_PROGRESS"
+    NOT_EXISTS: str = "NOT_EXISTS"
+
+
+class ResourceStatus(str, Enum):
+    CREATED: str = "CREATED"
+    DELETED: str = "DELETED"
+    ABANDONED: str = "ABANDONED"
+    NOT_EXISTS: str = "NOT_EXISTS"
+    IN_PROGRESS: str = "IN_PROGRESS"
+    DELETE_IN_PROGRESS: str = "DELETE_IN_PROGRESS"
+    ERROR: str = "ERROR"
+
+
+class ExternalResourceState(BaseModel):
+    key: ExternalResourceKey
+    ts: datetime
+    resource_status: ResourceStatus
+    reconciliation: Reconciliation
+    reconciliation_errors: int = 0
+
+
+class DynamoDBStateAdapter:
+    # Table PK
+    ER_KEY_HASH = "external_resource_key_hash"
+
+    RESOURCE_STATUS = "resource_status"
+    TIMESTAMP = "time_stamp"
+    RECONCILIATION_ERRORS = "reconciliation_errors"
+
+    ER_KEY = "external_resource_key"
+    ER_KEY_PROVISION_PROVIDER = "provision_provider"
+    ER_KEY_PROVISIONER_NAME = "provisioner_name"
+    ER_KEY_PROVIDER = "provider"
+    ER_KEY_IDENTIFIER = "identifier"
+
+    RECONC = "reconciliation"
+    RECONC_RESOURCE_HASH = "resource_hash"
+    RECONC_INPUT = "input"
+    RECONC_ACTION = "action"
+
+    MODCONF = "module_configuration"
+    MODCONF_IMAGE = "image"
+    MODCONF_VERSION = "version"
+    MODCONF_DRIFT_MINS = "drift_detection_minutes"
+    MODCONF_TIMEOUT_MINS = "timeout_minutes"
+
+    def _get_value(self, item: Mapping[str, Any], key: str, _type: str = "S") -> Any:
+        return item[key][_type]
+
+    def deserialize(
+        self, item: Mapping[str, Any], partial_data: bool = False
+    ) -> ExternalResourceState:
+        _key = self._get_value(item, self.ER_KEY, _type="M")
+        key = ExternalResourceKey(
+            provision_provider=self._get_value(_key, self.ER_KEY_PROVISION_PROVIDER),
+            provisioner_name=self._get_value(_key, self.ER_KEY_PROVISIONER_NAME),
+            provider=self._get_value(_key, self.ER_KEY_PROVIDER),
+            identifier=self._get_value(_key, self.ER_KEY_IDENTIFIER),
+        )
+        _reconciliation = self._get_value(item, self.RECONC, _type="M")
+
+        if partial_data:
+            r = Reconciliation(
+                key=key,
+                resource_hash=self._get_value(
+                    _reconciliation, self.RECONC_RESOURCE_HASH
+                ),
+            )
+        else:
+            _modconf = self._get_value(_reconciliation, self.MODCONF, _type="M")
+            r = Reconciliation(
+                key=key,
+                resource_hash=self._get_value(
+                    _reconciliation, self.RECONC_RESOURCE_HASH
+                ),
+                input=self._get_value(_reconciliation, self.RECONC_INPUT),
+                action=self._get_value(_reconciliation, self.RECONC_ACTION),
+                module_configuration=ExternalResourceModuleConfiguration(
+                    image=self._get_value(_modconf, self.MODCONF_IMAGE),
+                    version=self._get_value(_modconf, self.MODCONF_VERSION),
+                    reconcile_drift_interval_minutes=self._get_value(
+                        _modconf, self.MODCONF_DRIFT_MINS, _type="N"
+                    ),
+                    reconcile_timeout_minutes=self._get_value(
+                        _modconf, self.MODCONF_TIMEOUT_MINS, _type="N"
+                    ),
+                ),
+            )
+
+        return ExternalResourceState(
+            key=key,
+            ts=self._get_value(item, self.TIMESTAMP),
+            resource_status=self._get_value(item, self.RESOURCE_STATUS),
+            reconciliation=r,
+            reconciliation_errors=int(
+                self._get_value(item, self.RECONCILIATION_ERRORS, _type="N")
+            ),
+        )
+
+    def serialize(self, state: ExternalResourceState) -> dict[str, Any]:
+        return {
+            self.ER_KEY_HASH: {"S": state.key.hash()},
+            self.TIMESTAMP: {"S": state.ts.isoformat()},
+            self.RESOURCE_STATUS: {"S": state.resource_status.value},
+            self.RECONCILIATION_ERRORS: {"N": str(state.reconciliation_errors)},
+            self.ER_KEY: {
+                "M": {
+                    self.ER_KEY_PROVISION_PROVIDER: {"S": state.key.provision_provider},
+                    self.ER_KEY_PROVISIONER_NAME: {"S": state.key.provisioner_name},
+                    self.ER_KEY_PROVIDER: {"S": state.key.provider},
+                    self.ER_KEY_IDENTIFIER: {"S": state.key.identifier},
+                }
+            },
+            self.RECONC: {
+                "M": {
+                    self.RECONC_RESOURCE_HASH: {
+                        "S": state.reconciliation.resource_hash
+                    },
+                    self.RECONC_ACTION: {"S": state.reconciliation.action.value},
+                    self.RECONC_INPUT: {"S": state.reconciliation.input},
+                    self.MODCONF: {
+                        "M": {
+                            self.MODCONF_IMAGE: {
+                                "S": state.reconciliation.module_configuration.image
+                            },
+                            self.MODCONF_VERSION: {
+                                "S": state.reconciliation.module_configuration.version
+                            },
+                            self.MODCONF_DRIFT_MINS: {
+                                "N": str(
+                                    state.reconciliation.module_configuration.reconcile_drift_interval_minutes
+                                )
+                            },
+                            self.MODCONF_TIMEOUT_MINS: {
+                                "N": str(
+                                    state.reconciliation.module_configuration.reconcile_timeout_minutes
+                                )
+                            },
+                        }
+                    },
+                }
+            },
+        }
+
+
+class ExternalResourcesStateDynamoDB:
+    PARTIALS_PROJECTED_VALUES = ",".join([
+        DynamoDBStateAdapter.ER_KEY,
+        DynamoDBStateAdapter.TIMESTAMP,
+        DynamoDBStateAdapter.RESOURCE_STATUS,
+        DynamoDBStateAdapter.RECONCILIATION_ERRORS,
+        f"{DynamoDBStateAdapter.RECONC}.{DynamoDBStateAdapter.RECONC_RESOURCE_HASH}",
+    ])
+
+    def __init__(self, table_name: str, region_name: str) -> None:
+        self.adapter = DynamoDBStateAdapter()
+        self.client = boto3.client("dynamodb", region_name=region_name)
+        self._table = table_name
+        self.partial_resources = self._get_partial_resources()
+
+    def get_external_resource_state(
+        self, key: ExternalResourceKey
+    ) -> ExternalResourceState:
+        data = self.client.get_item(
+            TableName=self._table,
+            ConsistentRead=True,
+            Key={self.adapter.ER_KEY_HASH: {"S": key.hash()}},
+        )
+        if "Item" in data:
+            return self.adapter.deserialize(data["Item"])
+        else:
+            return ExternalResourceState(
+                key=key,
+                ts=datetime.now(timezone.utc),
+                resource_status=ResourceStatus.NOT_EXISTS,
+                reconciliation=Reconciliation(key=key),
+                reconciliation_errors=0,
+            )
+
+    def set_external_resource_state(
+        self,
+        state: ExternalResourceState,
+    ) -> None:
+        self.client.put_item(TableName=self._table, Item=self.adapter.serialize(state))
+
+    def del_external_resource_state(self, key: ExternalResourceKey) -> None:
+        self.client.delete_item(
+            TableName=self._table,
+            Key={self.adapter.ER_KEY_HASH: {"S": key.hash()}},
+        )
+
+    def _get_partial_resources(
+        self,
+    ) -> dict[ExternalResourceKey, ExternalResourceState]:
+        """A Partial Resoure is the minimum resource data reguired
+        to check if a resource has been removed from the configuration.
+        Getting less data from DynamoDb saves money and the logic does not need it.
+        """
+        logging.info("Getting Managed resources from DynamoDb")
+        partials = {}
+        for item in self.client.scan(
+            TableName=self._table, ProjectionExpression=self.PARTIALS_PROJECTED_VALUES
+        ).get("Items", []):
+            s = self.adapter.deserialize(item, partial_data=True)
+            partials[s.key] = s
+        return partials
+
+    def get_all_resource_keys(self) -> set[ExternalResourceKey]:
+        return {k for k in self.partial_resources.keys()}
+
+    def update_resource_status(
+        self, key: ExternalResourceKey, status: ResourceStatus
+    ) -> None:
+        self.client.update_item(
+            TableName=self._table,
+            Key={self.adapter.ER_KEY_HASH: {"S": key.hash()}},
+            UpdateExpression="set resource_status=:new_value",
+            ExpressionAttributeValues={":new_value": {"S": status.value}},
+            ReturnValues="UPDATED_NEW",
+        )

reconcile/gql_definitions/aws_account_manager/aws_accounts.py

@@ -80,6 +80,8 @@ query AWSAccountManagerAccounts {
       quotaLimits {
         path
       }
+      resourcesDefaultRegion
+      supportedDeploymentRegions
     }
     organization_accounts {
       ... AWSAccountManaged
@@ -129,6 +131,8 @@ class AWSAccountRequestV1(ConfiguredBaseModel):
     account_owner: OwnerV1 = Field(..., alias="accountOwner")
     organization: AWSOrganizationV1 = Field(..., alias="organization")
     quota_limits: Optional[list[AWSQuotaLimitsV1]] = Field(..., alias="quotaLimits")
+    resources_default_region: str = Field(..., alias="resourcesDefaultRegion")
+    supported_deployment_regions: Optional[list[str]] = Field(..., alias="supportedDeploymentRegions")
 
 
 class AWSAccountV1(AWSAccountManaged):