qontract-reconcile 0.10.2.dev477__tar.gz → 0.10.2.dev478__tar.gz

{qontract_reconcile-0.10.2.dev477 → qontract_reconcile-0.10.2.dev478}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev477
+Version: 0.10.2.dev478
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

qontract_reconcile-0.10.2.dev478/reconcile/external_resources/cloudflare.py

@@ -0,0 +1,80 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from reconcile.external_resources.model import (
+    ExternalResource,
+    ExternalResourceKey,
+    ExternalResourceModuleConfiguration,
+    ExternalResourcesInventory,
+)
+from reconcile.utils.exceptions import SecretIncompleteError
+from reconcile.utils.external_resource_spec import (
+    ExternalResourceSpec,
+)
+from reconcile.utils.external_resources import ResourceValueResolver
+from reconcile.utils.secret_reader import SecretReaderBase
+
+
+class CloudflareResourceFactory(ABC):
+    def __init__(
+        self, er_inventory: ExternalResourcesInventory, secret_reader: SecretReaderBase
+    ):
+        self.er_inventory = er_inventory
+        self.secret_reader = secret_reader
+
+    @abstractmethod
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]: ...
+
+    @abstractmethod
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None: ...
+
+    def find_linked_resources(
+        self, spec: ExternalResourceSpec
+    ) -> set[ExternalResourceKey]:
+        """Method to find dependant resources. Resources in this list
+        will be reconciled every time the parent resource finishes its reconciliation."""
+        return set()
+
+
+def get_account_id(
+    secret_reader: SecretReaderBase, api_credentials: dict[str, Any]
+) -> str:
+    creds = secret_reader.read_all(api_credentials)
+    account_id = creds.get("account_id")
+    if not account_id:
+        raise SecretIncompleteError(
+            f"secret {api_credentials['path']} incomplete: account_id missing"
+        )
+    return account_id
+
+
+class CloudflareDefaultResourceFactory(CloudflareResourceFactory):
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]:
+        account_id = get_account_id(
+            self.secret_reader,
+            spec.provisioner["api_credentials"],
+        )
+        resolved_values = ResourceValueResolver(
+            spec=spec, identifier_as_value=True
+        ).resolve()
+        return resolved_values | {
+            "account_id": account_id,
+        }
+
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None: ...

{qontract_reconcile-0.10.2.dev477 → qontract_reconcile-0.10.2.dev478}/reconcile/external_resources/factories.py

@@ -11,6 +11,10 @@ from reconcile.external_resources.aws import (
     AWSRdsFactory,
     AWSResourceFactory,
 )
+from reconcile.external_resources.cloudflare import (
+    CloudflareDefaultResourceFactory,
+    CloudflareResourceFactory,
+)
 from reconcile.external_resources.meta import QONTRACT_INTEGRATION
 from reconcile.external_resources.model import (
     ExternalResource,
@@ -171,3 +175,67 @@ class AWSExternalResourceFactory(ExternalResourceFactory):
     ) -> set[ExternalResourceKey]:
         f = self.resource_factories.get_factory(spec.provider)
         return f.find_linked_resources(spec)
+
+
+def setup_cloudflare_resource_factories(
+    er_inventory: ExternalResourcesInventory, secret_reader: SecretReaderBase
+) -> ObjectFactory[CloudflareResourceFactory]:
+    return ObjectFactory[CloudflareResourceFactory](
+        factories={},
+        default_factory=CloudflareDefaultResourceFactory(er_inventory, secret_reader),
+    )
+
+
+class CloudflareExternalResourceFactory(ExternalResourceFactory):
+    def __init__(
+        self,
+        module_inventory: ModuleInventory,
+        er_inventory: ExternalResourcesInventory,
+        secret_reader: SecretReaderBase,
+        provision_factories: ObjectFactory[ModuleProvisionDataFactory],
+        resource_factories: ObjectFactory[CloudflareResourceFactory],
+    ):
+        self.provision_factories = provision_factories
+        self.resource_factories = resource_factories
+        self.module_inventory = module_inventory
+        self.er_inventory = er_inventory
+        self.secret_reader = secret_reader
+
+    def create_external_resource(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> ExternalResource:
+        f = self.resource_factories.get_factory(spec.provider)
+        data = f.resolve(spec, module_conf)
+
+        module_type = self.module_inventory.get_from_spec(spec).module_type
+        provision_factory = self.provision_factories.get_factory(module_type)
+        module_provision_data = provision_factory.create_provision_data(spec)
+
+        provision = ExternalResourceProvision(
+            provision_provider=spec.provision_provider,
+            provisioner=spec.provisioner_name,
+            provider=spec.provider,
+            identifier=spec.identifier,
+            target_cluster=spec.cluster_name,
+            target_namespace=spec.namespace_name,
+            target_secret_name=spec.output_resource_name,
+            module_provision_data=module_provision_data,
+        )
+
+        return ExternalResource(data=data, provision=provision)
+
+    def validate_external_resource(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
+        f = self.resource_factories.get_factory(resource.provision.provider)
+        f.validate(resource, module_conf)
+
+    def find_linked_resources(
+        self, spec: ExternalResourceSpec
+    ) -> set[ExternalResourceKey]:
+        f = self.resource_factories.get_factory(spec.provider)
+        return f.find_linked_resources(spec)

{qontract_reconcile-0.10.2.dev477 → qontract_reconcile-0.10.2.dev478}/reconcile/external_resources/manager.py

@@ -7,11 +7,13 @@ from sretoolbox.utils import threaded
 
 from reconcile.external_resources.factories import (
     AWSExternalResourceFactory,
+    CloudflareExternalResourceFactory,
     ExternalResourceFactory,
     ModuleProvisionDataFactory,
     ObjectFactory,
     TerraformModuleProvisionDataFactory,
     setup_aws_resource_factories,
+    setup_cloudflare_resource_factories,
 )
 from reconcile.external_resources.metrics import publish_metrics
 from reconcile.external_resources.model import (
@@ -55,6 +57,9 @@ def setup_factories(
     secret_reader: SecretReaderBase,
 ) -> ObjectFactory[ExternalResourceFactory]:
     tf_factory = TerraformModuleProvisionDataFactory(settings=settings)
+    provision_factories = ObjectFactory[ModuleProvisionDataFactory](
+        factories={"terraform": tf_factory}
+    )
 
     return ObjectFactory[ExternalResourceFactory](
         factories={
@@ -62,14 +67,21 @@ def setup_factories(
                 module_inventory=module_inventory,
                 er_inventory=er_inventory,
                 secret_reader=secret_reader,
-                provision_factories=ObjectFactory[ModuleProvisionDataFactory](
-                    factories={"terraform": tf_factory, "cdktf": tf_factory}
-                ),
+                provision_factories=provision_factories,
                 resource_factories=setup_aws_resource_factories(
                     er_inventory, secret_reader
                 ),
                 default_tags=cast("dict[str, str]", settings.default_tags),
-            )
+            ),
+            "cloudflare": CloudflareExternalResourceFactory(
+                module_inventory=module_inventory,
+                er_inventory=er_inventory,
+                secret_reader=secret_reader,
+                provision_factories=provision_factories,
+                resource_factories=setup_cloudflare_resource_factories(
+                    er_inventory, secret_reader
+                ),
+            ),
         }
     )
 

{qontract_reconcile-0.10.2.dev477 → qontract_reconcile-0.10.2.dev478}/reconcile/external_resources/model.py

@@ -15,6 +15,8 @@ from reconcile.gql_definitions.external_resources.external_resources_modules imp
 )
 from reconcile.gql_definitions.external_resources.external_resources_namespaces import (
     NamespaceTerraformProviderResourceAWSV1,
+    NamespaceTerraformProviderResourceCloudflareV1,
+    NamespaceTerraformResourceCloudflareZoneV1,
     NamespaceTerraformResourceCloudWatchV1,
     NamespaceTerraformResourceElastiCacheV1,
     NamespaceTerraformResourceKMSV1,
@@ -90,7 +92,10 @@ class ExternalResourceKey(BaseModel, frozen=True):
         return f"{self.provision_provider}/{self.provisioner_name}/{self.provider}/{self.identifier}"
 
 
-SUPPORTED_RESOURCE_PROVIDERS = NamespaceTerraformProviderResourceAWSV1
+SUPPORTED_RESOURCE_PROVIDERS = (
+    NamespaceTerraformProviderResourceAWSV1
+    | NamespaceTerraformProviderResourceCloudflareV1
+)
 SUPPORTED_RESOURCE_TYPES = (
     NamespaceTerraformResourceRDSV1
     | NamespaceTerraformResourceMskV1
@@ -98,6 +103,7 @@ SUPPORTED_RESOURCE_TYPES = (
     | NamespaceTerraformResourceKMSV1
     | NamespaceTerraformResourceCloudWatchV1
     | NamespaceTerraformResourceRDSProxyV1
+    | NamespaceTerraformResourceCloudflareZoneV1
 )
 
 
@@ -138,7 +144,8 @@ class ExternalResourcesInventory(MutableMapping):
                     FLAG_RESOURCE_MANAGED_BY_ERV2,
                     FLAG_DELETE_RESOURCE,
                     MODULE_OVERRIDES,
-                }
+                },
+                by_alias=True,
             ),
             namespace=namespace.model_dump(by_alias=True),
         )
@@ -406,7 +413,7 @@ class ModuleProvisionData(BaseModel):
 
 
 class TerraformModuleProvisionData(ModuleProvisionData):
-    """Specific Provision Options for modules based on Terraform or CDKTF"""
+    """Specific Provision Options for modules based on Terraform"""
 
     tf_state_bucket: str
     tf_state_region: str

{qontract_reconcile-0.10.2.dev477 → qontract_reconcile-0.10.2.dev478}/reconcile/external_resources/reconciler.py

@@ -6,6 +6,7 @@ from typing import Any
 from kubernetes.client import (
     V1Container,
     V1EmptyDirVolumeSource,
+    V1EnvFromSource,
     V1EnvVar,
     V1EnvVarSource,
     V1JobSpec,
@@ -15,6 +16,7 @@ from kubernetes.client import (
     V1PodSpec,
     V1PodTemplateSpec,
     V1ResourceRequirements,
+    V1SecretEnvSource,
     V1SecretVolumeSource,
     V1Volume,
     V1VolumeMount,
@@ -116,6 +118,14 @@ class ReconciliationK8sJob(K8sJob, BaseModel, frozen=True):
                     value=self.reconciliation.action.value,
                 ),
             ],
+            env_from=[
+                V1EnvFromSource(
+                    secret_ref=V1SecretEnvSource(
+                        name=f"dotenv-{self.reconciliation.key.provision_provider}-{self.reconciliation.key.provisioner_name}",
+                        optional=True,
+                    )
+                ),
+            ],
             volume_mounts=[
                 V1VolumeMount(
                     name="credentials",
@@ -192,7 +202,7 @@ class ReconciliationK8sJob(K8sJob, BaseModel, frozen=True):
                         V1Volume(
                             name="credentials",
                             secret=V1SecretVolumeSource(
-                                secret_name=f"credentials-{self.reconciliation.key.provisioner_name}",
+                                secret_name=f"credentials-{self.reconciliation.key.provision_provider}-{self.reconciliation.key.provisioner_name}",
                             ),
                         ),
                         V1Volume(

{qontract_reconcile-0.10.2.dev477 → qontract_reconcile-0.10.2.dev478}/reconcile/gql_definitions/external_resources/external_resources_namespaces.py

@@ -570,6 +570,26 @@ query ExternalResourcesNamespaces {
           }
         }
       }
+      ... on NamespaceTerraformProviderResourceCloudflare_v1 {
+        provisioner {
+          name
+          apiCredentials {
+          ...VaultSecret
+          }
+        }
+        resources {
+          ... on NamespaceTerraformResourceCloudflareZone_v1 {
+            provider
+            identifier
+            defaults
+            delete
+            managed_by_erv2
+            module_overrides {
+              ...ExternalResourcesModuleOverrides
+            }
+          }
+        }
+      }
     }
     environment {
       name
@@ -1173,6 +1193,29 @@ class NamespaceTerraformProviderResourceAWSV1(NamespaceExternalResourceV1):
     resources: list[Union[NamespaceTerraformResourceRDSV1, NamespaceTerraformResourceRosaAuthenticatorV1, NamespaceTerraformResourceALBV1, NamespaceTerraformResourceS3V1, NamespaceTerraformResourceElastiCacheV1, NamespaceTerraformResourceCloudWatchV1, NamespaceTerraformResourceASGV1, NamespaceTerraformResourceRDSProxyV1, NamespaceTerraformResourceRoleV1, NamespaceTerraformResourceKMSV1, NamespaceTerraformResourceMskV1, NamespaceTerraformResourceSNSTopicV1, NamespaceTerraformResourceServiceAccountV1, NamespaceTerraformResourceS3SQSV1, NamespaceTerraformResourceKinesisV1, NamespaceTerraformResourceRosaAuthenticatorVPCEV1, NamespaceTerraformResourceS3CloudFrontV1, NamespaceTerraformResourceElasticSearchV1, NamespaceTerraformResourceACMV1, NamespaceTerraformResourceRoute53ZoneV1, NamespaceTerraformResourceSQSV1, NamespaceTerraformResourceDynamoDBV1, NamespaceTerraformResourceECRV1, NamespaceTerraformResourceS3CloudFrontPublicKeyV1, NamespaceTerraformResourceSecretsManagerV1, NamespaceTerraformResourceSecretsManagerServiceAccountV1, NamespaceTerraformResourceAWSV1]] = Field(..., alias="resources")
 
 
+class CloudflareAccountV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    api_credentials: VaultSecret = Field(..., alias="apiCredentials")
+
+
+class NamespaceTerraformResourceCloudflareV1(ConfiguredBaseModel):
+    ...
+
+
+class NamespaceTerraformResourceCloudflareZoneV1(NamespaceTerraformResourceCloudflareV1):
+    provider: str = Field(..., alias="provider")
+    identifier: str = Field(..., alias="identifier")
+    defaults: str = Field(..., alias="defaults")
+    delete: Optional[bool] = Field(..., alias="delete")
+    managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
+    module_overrides: Optional[ExternalResourcesModuleOverrides] = Field(..., alias="module_overrides")
+
+
+class NamespaceTerraformProviderResourceCloudflareV1(NamespaceExternalResourceV1):
+    provisioner: CloudflareAccountV1 = Field(..., alias="provisioner")
+    resources: list[Union[NamespaceTerraformResourceCloudflareZoneV1, NamespaceTerraformResourceCloudflareV1]] = Field(..., alias="resources")
+
+
 class EnvironmentV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
     labels: str = Field(..., alias="labels")
@@ -1213,7 +1256,7 @@ class NamespaceV1(ConfiguredBaseModel):
     delete: Optional[bool] = Field(..., alias="delete")
     cluster_admin: Optional[bool] = Field(..., alias="clusterAdmin")
     managed_external_resources: Optional[bool] = Field(..., alias="managedExternalResources")
-    external_resources: Optional[list[Union[NamespaceTerraformProviderResourceAWSV1, NamespaceExternalResourceV1]]] = Field(..., alias="externalResources")
+    external_resources: Optional[list[Union[NamespaceTerraformProviderResourceAWSV1, NamespaceTerraformProviderResourceCloudflareV1, NamespaceExternalResourceV1]]] = Field(..., alias="externalResources")
     environment: EnvironmentV1 = Field(..., alias="environment")
     app: AppV1 = Field(..., alias="app")
     cluster: NamespaceV1_ClusterV1 = Field(..., alias="cluster")

{qontract_reconcile-0.10.2.dev477 → qontract_reconcile-0.10.2.dev478}/reconcile/gql_definitions/introspection.json

@@ -42711,6 +42711,22 @@
                             "isDeprecated": false,
                             "deprecationReason": null
                         },
+                        {
+                            "name": "defaults",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "SCALAR",
+                                    "name": "String",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
                         {
                             "name": "zone",
                             "description": null,
@@ -42727,6 +42743,42 @@
                             "isDeprecated": false,
                             "deprecationReason": null
                         },
+                        {
+                            "name": "delete",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "Boolean",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "managed_by_erv2",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "Boolean",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "module_overrides",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "OBJECT",
+                                "name": "ExternalResourcesModuleOverrides_v1",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
                         {
                             "name": "plan",
                             "description": null,
@@ -42800,7 +42852,7 @@
                             "deprecationReason": null
                         },
                         {
-                            "name": "records",
+                            "name": "dns_records",
                             "description": null,
                             "args": [],
                             "type": {
@@ -42891,6 +42943,113 @@
                     "enumValues": null,
                     "possibleTypes": null
                 },
+                {
+                    "kind": "OBJECT",
+                    "name": "ExternalResourcesModuleOverrides_v1",
+                    "description": null,
+                    "fields": [
+                        {
+                            "name": "module_type",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "String",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "channel",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "String",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "image",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "String",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "version",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "String",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "reconcile_timeout_minutes",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "Int",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "outputs_secret_image",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "String",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "outputs_secret_version",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "String",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
+                        {
+                            "name": "resources",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "OBJECT",
+                                "name": "DeployResources_v1",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        }
+                    ],
+                    "inputFields": null,
+                    "interfaces": [],
+                    "enumValues": null,
+                    "possibleTypes": null
+                },
                 {
                     "kind": "OBJECT",
                     "name": "CloudflareZoneArgo_v1",
@@ -45058,113 +45217,6 @@
                     "enumValues": null,
                     "possibleTypes": null
                 },
-                {
-                    "kind": "OBJECT",
-                    "name": "ExternalResourcesModuleOverrides_v1",
-                    "description": null,
-                    "fields": [
-                        {
-                            "name": "module_type",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "SCALAR",
-                                "name": "String",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "channel",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "SCALAR",
-                                "name": "String",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "image",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "SCALAR",
-                                "name": "String",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "version",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "SCALAR",
-                                "name": "String",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "reconcile_timeout_minutes",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "SCALAR",
-                                "name": "Int",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "outputs_secret_image",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "SCALAR",
-                                "name": "String",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "outputs_secret_version",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "SCALAR",
-                                "name": "String",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
-                        {
-                            "name": "resources",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "OBJECT",
-                                "name": "DeployResources_v1",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        }
-                    ],
-                    "inputFields": null,
-                    "interfaces": [],
-                    "enumValues": null,
-                    "possibleTypes": null
-                },
                 {
                     "kind": "OBJECT",
                     "name": "NamespaceTerraformResourceRDS_v1",