dbt-platform-helper 12.3.0__py3-none-any.whl → 12.4.1__py3-none-any.whl

dbt_platform_helper/utils/validation.py

@@ -1,14 +1,9 @@
-import ipaddress
 import os
 import re
 from pathlib import Path
 
 import click
 import yaml
-from schema import Optional
-from schema import Or
-from schema import Regex
-from schema import Schema
 from schema import SchemaError
 from yaml.parser import ParserError
 from yamllint import config
@@ -18,66 +13,14 @@ from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
 from dbt_platform_helper.constants import ENVIRONMENTS_KEY
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
+from dbt_platform_helper.providers.platform_config_schema import EXTENSION_SCHEMAS
+from dbt_platform_helper.providers.platform_config_schema import PLATFORM_CONFIG_SCHEMA
 from dbt_platform_helper.utils.aws import get_supported_opensearch_versions
 from dbt_platform_helper.utils.aws import get_supported_redis_versions
 from dbt_platform_helper.utils.files import apply_environment_defaults
 from dbt_platform_helper.utils.messages import abort_with_error
 
 
-def validate_string(regex_pattern: str):
-    def validator(string):
-        if not re.match(regex_pattern, string):
-            raise SchemaError(
-                f"String '{string}' does not match the required pattern '{regex_pattern}'. For more details on valid string patterns see: https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/"
-            )
-        return string
-
-    return validator
-
-
-S3_BUCKET_NAME_ERROR_TEMPLATE = "Bucket name '{}' is invalid:\n{}"
-AVAILABILITY_UNCERTAIN_TEMPLATE = (
-    "Warning: Could not determine the availability of bucket name '{}'."
-)
-BUCKET_NAME_IN_USE_TEMPLATE = "Warning: Bucket name '{}' is already in use. Check your AWS accounts to see if this is a problem."
-
-
-def validate_s3_bucket_name(name: str):
-    errors = []
-    if not (2 < len(name) < 64):
-        errors.append("Length must be between 3 and 63 characters inclusive.")
-
-    if not re.match(r"^[a-z0-9].*[a-z0-9]$", name):
-        errors.append("Names must start and end with 0-9 or a-z.")
-
-    if not re.match(r"^[a-z0-9.-]*$", name):
-        errors.append("Names can only contain the characters 0-9, a-z, '.' and '-'.")
-
-    if ".." in name:
-        errors.append("Names cannot contain two adjacent periods.")
-
-    try:
-        ipaddress.ip_address(name)
-        errors.append("Names cannot be IP addresses.")
-    except ValueError:
-        pass
-
-    for prefix in ("xn--", "sthree-"):
-        if name.startswith(prefix):
-            errors.append(f"Names cannot be prefixed '{prefix}'.")
-
-    for suffix in ("-s3alias", "--ol-s3"):
-        if name.endswith(suffix):
-            errors.append(f"Names cannot be suffixed '{suffix}'.")
-
-    if errors:
-        raise SchemaError(
-            S3_BUCKET_NAME_ERROR_TEMPLATE.format(name, "\n".join(f"  {e}" for e in errors))
-        )
-
-    return True
-
-
 def validate_addons(addons: dict):
     """
     Validate the addons file and return a dictionary of addon: error message.
@@ -90,7 +33,7 @@ def validate_addons(addons: dict):
             if not addon_type:
                 errors[addon_name] = f"Missing addon type in addon '{addon_name}'"
                 continue
-            schema = SCHEMA_MAP.get(addon_type, None)
+            schema = EXTENSION_SCHEMAS.get(addon_type, None)
             if not schema:
                 errors[addon_name] = (
                     f"Unsupported addon type '{addon_type}' in addon '{addon_name}'"
@@ -116,15 +59,6 @@ def validate_addons(addons: dict):
     return errors
 
 
-def int_between(lower, upper):
-    def is_between(value):
-        if isinstance(value, int) and lower <= value <= upper:
-            return True
-        raise SchemaError(f"should be an integer between {lower} and {upper}")
-
-    return is_between
-
-
 def float_between_with_halfstep(lower, upper):
     def is_between(value):
         is_number = isinstance(value, int) or isinstance(value, float)
@@ -137,420 +71,6 @@ def float_between_with_halfstep(lower, upper):
     return is_between
 
 
-ENV_NAME = Regex(
-    r"^([a-z][a-zA-Z0-9]*|\*)$",
-    error="Environment name {} is invalid: names must only contain lowercase alphanumeric characters, or be the '*' default environment",
-    # For values the "error" parameter works and outputs the custom text. For keys the custom text doesn't get reported in the exception for some reason.
-)
-
-range_validator = validate_string(r"^\d+-\d+$")
-seconds_validator = validate_string(r"^\d+s$")
-branch_wildcard_validator = validate_string(r"^((?!\*).)*(\*)?$")
-
-NUMBER = Or(int, float)
-DELETION_POLICY = Or("Delete", "Retain")
-DB_DELETION_POLICY = Or("Delete", "Retain", "Snapshot")
-DELETION_PROTECTION = bool
-
-REDIS_PLANS = Or(
-    "micro",
-    "micro-ha",
-    "tiny",
-    "tiny-ha",
-    "small",
-    "small-ha",
-    "medium",
-    "medium-ha",
-    "large",
-    "large-ha",
-    "x-large",
-    "x-large-ha",
-)
-
-REDIS_ENGINE_VERSIONS = str
-
-REDIS_DEFINITION = {
-    "type": "redis",
-    Optional("environments"): {
-        ENV_NAME: {
-            Optional("plan"): REDIS_PLANS,
-            Optional("engine"): REDIS_ENGINE_VERSIONS,
-            Optional("replicas"): int_between(0, 5),
-            Optional("deletion_policy"): DELETION_POLICY,
-            Optional("apply_immediately"): bool,
-            Optional("automatic_failover_enabled"): bool,
-            Optional("instance"): str,
-            Optional("multi_az_enabled"): bool,
-        }
-    },
-}
-
-POSTGRES_PLANS = Or(
-    "tiny",
-    "small",
-    "small-ha",
-    "small-high-io",
-    "medium",
-    "medium-ha",
-    "medium-high-io",
-    "large",
-    "large-ha",
-    "large-high-io",
-    "x-large",
-    "x-large-ha",
-    "x-large-high-io",
-)
-POSTGRES_STORAGE_TYPES = Or("gp2", "gp3", "io1", "io2")
-
-RETENTION_POLICY = Or(
-    None,
-    {
-        "mode": Or("GOVERNANCE", "COMPLIANCE"),
-        Or("days", "years", only_one=True): int,
-    },
-)
-
-DATABASE_COPY = {
-    "from": ENV_NAME,
-    "to": ENV_NAME,
-    Optional("from_account"): str,
-    Optional("to_account"): str,
-}
-
-POSTGRES_DEFINITION = {
-    "type": "postgres",
-    "version": NUMBER,
-    Optional("deletion_policy"): DB_DELETION_POLICY,
-    Optional("environments"): {
-        ENV_NAME: {
-            Optional("plan"): POSTGRES_PLANS,
-            Optional("volume_size"): int_between(20, 10000),
-            Optional("iops"): int_between(1000, 9950),
-            Optional("snapshot_id"): str,
-            Optional("deletion_policy"): DB_DELETION_POLICY,
-            Optional("deletion_protection"): DELETION_PROTECTION,
-            Optional("multi_az"): bool,
-            Optional("storage_type"): POSTGRES_STORAGE_TYPES,
-            Optional("backup_retention_days"): int_between(1, 35),
-        }
-    },
-    Optional("database_copy"): [DATABASE_COPY],
-    Optional("objects"): [
-        {
-            "key": str,
-            Optional("body"): str,
-        }
-    ],
-}
-
-LIFECYCLE_RULE = {
-    Optional("filter_prefix"): str,
-    "expiration_days": int,
-    "enabled": bool,
-}
-
-
-def kms_key_arn_regex(key):
-    return Regex(
-        r"^arn:aws:kms:.*:\d{12}:(key|alias).*",
-        error=f"{key} must contain a valid ARN for a KMS key",
-    )
-
-
-def s3_bucket_arn_regex(key):
-    return Regex(
-        r"^arn:aws:s3::.*",
-        error=f"{key} must contain a valid ARN for an S3 bucket",
-    )
-
-
-def iam_role_arn_regex(key):
-    return Regex(
-        r"^arn:aws:iam::\d{12}:role/.*",
-        error=f"{key} must contain a valid ARN for an IAM role",
-    )
-
-
-def dbt_email_address_regex(key):
-    return Regex(
-        r"^[\w.-]+@(businessandtrade.gov.uk|digital.trade.gov.uk)$",
-        error=f"{key} must contain a valid DBT email address",
-    )
-
-
-EXTERNAL_ROLE_ACCESS = {
-    "role_arn": iam_role_arn_regex("role_arn"),
-    "read": bool,
-    "write": bool,
-    "cyber_sign_off_by": dbt_email_address_regex("cyber_sign_off_by"),
-}
-
-EXTERNAL_ROLE_ACCESS_NAME = Regex(
-    r"^([a-z][a-zA-Z0-9_-]*)$",
-    error="External role access block name {} is invalid: names must only contain lowercase alphanumeric characters separated by hypen or underscore",
-)
-
-DATA_IMPORT = {
-    Optional("source_kms_key_arn"): kms_key_arn_regex("source_kms_key_arn"),
-    "source_bucket_arn": s3_bucket_arn_regex("source_bucket_arn"),
-    "worker_role_arn": iam_role_arn_regex("worker_role_arn"),
-}
-
-DATA_MIGRATION = {
-    "import": DATA_IMPORT,
-}
-
-S3_BASE = {
-    Optional("readonly"): bool,
-    Optional("serve_static_content"): bool,
-    Optional("services"): Or("__all__", [str]),
-    Optional("environments"): {
-        ENV_NAME: {
-            "bucket_name": validate_s3_bucket_name,
-            Optional("deletion_policy"): DELETION_POLICY,
-            Optional("retention_policy"): RETENTION_POLICY,
-            Optional("versioning"): bool,
-            Optional("lifecycle_rules"): [LIFECYCLE_RULE],
-            Optional("data_migration"): DATA_MIGRATION,
-            Optional("external_role_access"): {EXTERNAL_ROLE_ACCESS_NAME: EXTERNAL_ROLE_ACCESS},
-        },
-    },
-}
-
-S3_POLICY_DEFINITION = dict(S3_BASE)
-S3_POLICY_DEFINITION.update({"type": "s3-policy"})
-
-S3_DEFINITION = dict(S3_BASE)
-S3_DEFINITION.update(
-    {
-        "type": "s3",
-        Optional("objects"): [{"key": str, Optional("body"): str, Optional("content_type"): str}],
-    }
-)
-
-MONITORING_DEFINITION = {
-    "type": "monitoring",
-    Optional("environments"): {
-        ENV_NAME: {
-            Optional("enable_ops_center"): bool,
-        }
-    },
-}
-
-OPENSEARCH_PLANS = Or(
-    "tiny", "small", "small-ha", "medium", "medium-ha", "large", "large-ha", "x-large", "x-large-ha"
-)
-OPENSEARCH_ENGINE_VERSIONS = str
-OPENSEARCH_MIN_VOLUME_SIZE = 10
-OPENSEARCH_MAX_VOLUME_SIZE = {
-    "tiny": 100,
-    "small": 200,
-    "small-ha": 200,
-    "medium": 512,
-    "medium-ha": 512,
-    "large": 1000,
-    "large-ha": 1000,
-    "x-large": 1500,
-    "x-large-ha": 1500,
-}
-
-OPENSEARCH_DEFINITION = {
-    "type": "opensearch",
-    Optional("environments"): {
-        ENV_NAME: {
-            Optional("engine"): OPENSEARCH_ENGINE_VERSIONS,
-            Optional("deletion_policy"): DELETION_POLICY,
-            Optional("plan"): OPENSEARCH_PLANS,
-            Optional("volume_size"): int,
-            Optional("ebs_throughput"): int,
-            Optional("ebs_volume_type"): str,
-            Optional("instance"): str,
-            Optional("instances"): int,
-            Optional("master"): bool,
-            Optional("es_app_log_retention_in_days"): int,
-            Optional("index_slow_log_retention_in_days"): int,
-            Optional("audit_log_retention_in_days"): int,
-            Optional("search_slow_log_retention_in_days"): int,
-            Optional("password_special_characters"): str,
-            Optional("urlencode_password"): bool,
-        }
-    },
-}
-
-CACHE_POLICY_DEFINITION = {
-    "min_ttl": int,
-    "max_ttl": int,
-    "default_ttl": int,
-    "cookies_config": Or("none", "whitelist", "allExcept", "all"),
-    "header": Or("none", "whitelist"),
-    "query_string_behavior": Or("none", "whitelist", "allExcept", "all"),
-    Optional("cookie_list"): list,
-    Optional("headers_list"): list,
-    Optional("cache_policy_query_strings"): list,
-}
-
-PATHS_DEFINITION = {
-    Optional("default"): {
-        "cache": str,
-        "request": str,
-    },
-    Optional("additional"): list[
-        {
-            "path": str,
-            "cache": str,
-            "request": str,
-        }
-    ],
-}
-
-ALB_DEFINITION = {
-    "type": "alb",
-    Optional("environments"): {
-        ENV_NAME: Or(
-            {
-                Optional("additional_address_list"): list,
-                Optional("allowed_methods"): list,
-                Optional("cached_methods"): list,
-                Optional("cdn_compress"): bool,
-                Optional("cdn_domains_list"): dict,
-                Optional("cdn_geo_locations"): list,
-                Optional("cdn_geo_restriction_type"): str,
-                Optional("cdn_logging_bucket"): str,
-                Optional("cdn_logging_bucket_prefix"): str,
-                Optional("cdn_timeout_seconds"): int,
-                Optional("default_waf"): str,
-                Optional("domain_prefix"): str,
-                Optional("enable_logging"): bool,
-                Optional("env_root"): str,
-                Optional("forwarded_values_forward"): str,
-                Optional("forwarded_values_headers"): list,
-                Optional("forwarded_values_query_string"): bool,
-                Optional("origin_protocol_policy"): str,
-                Optional("origin_ssl_protocols"): list,
-                Optional("slack_alert_channel_alb_secret_rotation"): str,
-                Optional("viewer_certificate_minimum_protocol_version"): str,
-                Optional("viewer_certificate_ssl_support_method"): str,
-                Optional("viewer_protocol_policy"): str,
-                Optional("cache_policy"): dict({str: CACHE_POLICY_DEFINITION}),
-                Optional("origin_request_policy"): dict({str: {}}),
-                Optional("paths"): dict({str: PATHS_DEFINITION}),
-            },
-            None,
-        )
-    },
-}
-
-PROMETHEUS_POLICY_DEFINITION = {
-    "type": "prometheus-policy",
-    Optional("services"): Or("__all__", [str]),
-    Optional("environments"): {
-        ENV_NAME: {
-            "role_arn": str,
-        }
-    },
-}
-
-_DEFAULT_VERSIONS_DEFINITION = {
-    Optional("terraform-platform-modules"): str,
-    Optional("platform-helper"): str,
-}
-_ENVIRONMENTS_VERSIONS_OVERRIDES = {
-    Optional("terraform-platform-modules"): str,
-}
-_PIPELINE_VERSIONS_OVERRIDES = {
-    Optional("platform-helper"): str,
-}
-
-_ENVIRONMENTS_PARAMS = {
-    Optional("accounts"): {
-        "deploy": {
-            "name": str,
-            "id": str,
-        },
-        "dns": {
-            "name": str,
-            "id": str,
-        },
-    },
-    Optional("requires_approval"): bool,
-    Optional("versions"): _ENVIRONMENTS_VERSIONS_OVERRIDES,
-    Optional("vpc"): str,
-}
-
-ENVIRONMENTS_DEFINITION = {str: Or(None, _ENVIRONMENTS_PARAMS)}
-
-CODEBASE_PIPELINES_DEFINITION = [
-    {
-        "name": str,
-        "repository": str,
-        Optional("additional_ecr_repository"): str,
-        Optional("deploy_repository_branch"): str,
-        "services": list[str],
-        "pipelines": [
-            Or(
-                {
-                    "name": str,
-                    "branch": branch_wildcard_validator,
-                    "environments": [
-                        {
-                            "name": str,
-                            Optional("requires_approval"): bool,
-                        }
-                    ],
-                },
-                {
-                    "name": str,
-                    "tag": bool,
-                    "environments": [
-                        {
-                            "name": str,
-                            Optional("requires_approval"): bool,
-                        }
-                    ],
-                },
-            ),
-        ],
-    },
-]
-
-ENVIRONMENT_PIPELINES_DEFINITION = {
-    str: {
-        Optional("account"): str,
-        Optional("branch", default="main"): str,
-        Optional("pipeline_to_trigger"): str,
-        Optional("versions"): _PIPELINE_VERSIONS_OVERRIDES,
-        "slack_channel": str,
-        "trigger_on_push": bool,
-        "environments": {str: Or(None, _ENVIRONMENTS_PARAMS)},
-    }
-}
-
-PLATFORM_CONFIG_SCHEMA = Schema(
-    {
-        # The following line is for the AWS Copilot version, will be removed under DBTP-1002
-        "application": str,
-        Optional("legacy_project", default=False): bool,
-        Optional("default_versions"): _DEFAULT_VERSIONS_DEFINITION,
-        Optional("accounts"): list[str],
-        Optional("environments"): ENVIRONMENTS_DEFINITION,
-        Optional("codebase_pipelines"): CODEBASE_PIPELINES_DEFINITION,
-        Optional("extensions"): {
-            str: Or(
-                REDIS_DEFINITION,
-                POSTGRES_DEFINITION,
-                S3_DEFINITION,
-                S3_POLICY_DEFINITION,
-                MONITORING_DEFINITION,
-                OPENSEARCH_DEFINITION,
-                ALB_DEFINITION,
-                PROMETHEUS_POLICY_DEFINITION,
-            )
-        },
-        Optional("environment_pipelines"): ENVIRONMENT_PIPELINES_DEFINITION,
-    }
-)
-
-
 def validate_platform_config(config):
     PLATFORM_CONFIG_SCHEMA.validate(config)
     enriched_config = apply_environment_defaults(config)
@@ -832,71 +352,3 @@ def config_file_check(path=PLATFORM_CONFIG_FILE):
     if errors:
         click.secho("\n".join(errors), bg="red", fg="white")
         exit(1)
-
-
-S3_SCHEMA = Schema(S3_DEFINITION)
-S3_POLICY_SCHEMA = Schema(S3_POLICY_DEFINITION)
-POSTGRES_SCHEMA = Schema(POSTGRES_DEFINITION)
-REDIS_SCHEMA = Schema(REDIS_DEFINITION)
-
-
-class ConditionalSchema(Schema):
-    def validate(self, data, _is_conditional_schema=True):
-        data = super(ConditionalSchema, self).validate(data, _is_conditional_schema=False)
-        if _is_conditional_schema:
-            default_plan = None
-            default_volume_size = None
-
-            default_environment_config = data["environments"].get(
-                "*", data["environments"].get("default", None)
-            )
-            if default_environment_config:
-                default_plan = default_environment_config.get("plan", None)
-                default_volume_size = default_environment_config.get("volume_size", None)
-
-            for env in data["environments"]:
-                volume_size = data["environments"][env].get("volume_size", default_volume_size)
-                plan = data["environments"][env].get("plan", default_plan)
-
-                if volume_size:
-                    if not plan:
-                        raise SchemaError(f"Missing key: 'plan'")
-
-                    if volume_size < OPENSEARCH_MIN_VOLUME_SIZE:
-                        raise SchemaError(
-                            f"Key 'environments' error: Key '{env}' error: Key 'volume_size' error: should be an integer greater than {OPENSEARCH_MIN_VOLUME_SIZE}"
-                        )
-
-                    for key in OPENSEARCH_MAX_VOLUME_SIZE:
-                        if plan == key and not volume_size <= OPENSEARCH_MAX_VOLUME_SIZE[key]:
-                            raise SchemaError(
-                                f"Key 'environments' error: Key '{env}' error: Key 'volume_size' error: should be an integer between {OPENSEARCH_MIN_VOLUME_SIZE} and {OPENSEARCH_MAX_VOLUME_SIZE[key]} for plan {plan}"
-                            )
-
-        return data
-
-
-OPENSEARCH_SCHEMA = ConditionalSchema(OPENSEARCH_DEFINITION)
-MONITORING_SCHEMA = Schema(MONITORING_DEFINITION)
-ALB_SCHEMA = Schema(ALB_DEFINITION)
-PROMETHEUS_POLICY_SCHEMA = Schema(PROMETHEUS_POLICY_DEFINITION)
-
-
-def no_param_schema(schema_type):
-    return Schema({"type": schema_type, Optional("services"): Or("__all__", [str])})
-
-
-SCHEMA_MAP = {
-    "s3": S3_SCHEMA,
-    "s3-policy": S3_POLICY_SCHEMA,
-    "postgres": POSTGRES_SCHEMA,
-    "redis": REDIS_SCHEMA,
-    "opensearch": OPENSEARCH_SCHEMA,
-    "monitoring": MONITORING_SCHEMA,
-    "appconfig-ipfilter": no_param_schema("appconfig-ipfilter"),
-    "subscription-filter": no_param_schema("subscription-filter"),
-    "vpc": no_param_schema("vpc"),
-    "xray": no_param_schema("xray"),
-    "alb": ALB_SCHEMA,
-    "prometheus-policy": PROMETHEUS_POLICY_SCHEMA,
-}

dbt_platform_helper/utils/versioning.py

@@ -13,9 +13,9 @@ import requests
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.constants import PLATFORM_HELPER_VERSION_FILE
-from dbt_platform_helper.exceptions import IncompatibleMajorVersion
-from dbt_platform_helper.exceptions import IncompatibleMinorVersion
-from dbt_platform_helper.exceptions import ValidationException
+from dbt_platform_helper.providers.validation import IncompatibleMajorVersionException
+from dbt_platform_helper.providers.validation import IncompatibleMinorVersionException
+from dbt_platform_helper.providers.validation import ValidationException
 from dbt_platform_helper.utils.platform_config import load_unvalidated_config_file
 
 VersionTuple = Optional[Tuple[int, int, int]]
@@ -198,13 +198,13 @@ def validate_version_compatibility(
     if (app_major == 0 and check_major == 0) and (
         app_minor != check_minor or app_patch != check_patch
     ):
-        raise IncompatibleMajorVersion(app_version_as_string, check_version_as_string)
+        raise IncompatibleMajorVersionException(app_version_as_string, check_version_as_string)
 
     if app_major != check_major:
-        raise IncompatibleMajorVersion(app_version_as_string, check_version_as_string)
+        raise IncompatibleMajorVersionException(app_version_as_string, check_version_as_string)
 
     if app_minor != check_minor:
-        raise IncompatibleMinorVersion(app_version_as_string, check_version_as_string)
+        raise IncompatibleMinorVersionException(app_version_as_string, check_version_as_string)
 
 
 def check_version_on_file_compatibility(
@@ -248,9 +248,9 @@ def check_platform_helper_version_needs_update():
     )
     try:
         validate_version_compatibility(local_version, latest_release)
-    except IncompatibleMajorVersion:
+    except IncompatibleMajorVersionException:
         click.secho(message, fg="red")
-    except IncompatibleMinorVersion:
+    except IncompatibleMinorVersionException:
         click.secho(message, fg="yellow")
 
 

{dbt_platform_helper-12.3.0.dist-info → dbt_platform_helper-12.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbt-platform-helper
-Version: 12.3.0
+Version: 12.4.1
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team