dbt-platform-helper 15.2.0__tar.gz → 15.2.1__tar.gz

{dbt_platform_helper-15.2.0 → dbt_platform_helper-15.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbt-platform-helper
-Version: 15.2.0
+Version: 15.2.1
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-15.2.0 → dbt_platform_helper-15.2.1}/dbt_platform_helper/domain/copilot.py

@@ -10,6 +10,7 @@ import botocore.errorfactory
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
 from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
+from dbt_platform_helper.domain.plans import PlanLoader
 from dbt_platform_helper.providers.config import ConfigProvider
 from dbt_platform_helper.providers.files import FileProvider
 from dbt_platform_helper.providers.io import ClickIOProvider
@@ -45,6 +46,7 @@ class Copilot:
         kms_provider: KMSProvider,
         session,
         io: ClickIOProvider = ClickIOProvider(),
+        plan_manager: PlanLoader = PlanLoader(),
         yaml_file_provider: YamlFileProvider = YamlFileProvider,
     ):
         self.config_provider = config_provider
@@ -53,6 +55,7 @@ class Copilot:
         self.copilot_templating = copilot_templating
         self.kms_provider = kms_provider
         self.io = io
+        self.plan_manager = plan_manager
         self.yaml_file_provider = yaml_file_provider
         self.session = session
 
@@ -169,7 +172,7 @@ class Copilot:
         def _normalise_keys(source: dict):
             return {k.replace("-", "_"): v for k, v in source.items()}
 
-        addon_plans = self.yaml_file_provider.load(self.PACKAGE_DIR / "addon-plans.yml")
+        addon_plans = self.plan_manager.load()
 
         # load and validate config
         config = self.yaml_file_provider.load(config_file)

dbt_platform_helper-15.2.1/dbt_platform_helper/domain/plans.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+
+from dbt_platform_helper.providers.yaml_file import YamlFileProvider
+
+
+class PlanLoader:
+
+    PROJECT_DIR = Path(__file__).resolve().parent.parent.parent
+
+    def __init__(
+        self,
+        extensions: dict = None,
+        terraform_dir: str = "terraform",
+        loader: YamlFileProvider = YamlFileProvider,
+    ):
+        self.path = terraform_dir
+        self.loader = loader
+        self._cache = {}
+        self.extensions = extensions or {
+            "redis": "elasticache-redis",
+            "opensearch": "opensearch",
+            "postgres": "postgres",
+        }
+
+    def load(self):
+        result = {}
+        for key, value in self.extensions.items():
+            result[key] = self._load_plan(key, f"{self.PROJECT_DIR}/{self.path}/{value}/plans.yml")
+        return result
+
+    def _load_plan(self, name, path):
+        if name in self._cache:
+            return self._cache[name]
+        else:
+            plan = self.loader.load(path)
+            self._cache[name] = plan
+            return plan
+
+    def get_plan_names(self, extension):
+        plans = self.load()
+        return list(plans[extension].keys())

{dbt_platform_helper-15.2.0 → dbt_platform_helper-15.2.1}/dbt_platform_helper/entities/platform_config_schema.py

@@ -9,6 +9,10 @@ from schema import Schema
 from schema import SchemaError
 
 from dbt_platform_helper.constants import PLATFORM_CONFIG_SCHEMA_VERSION
+from dbt_platform_helper.domain.plans import PlanLoader
+
+plan_manager = PlanLoader()
+plan_manager.load()
 
 
 class PlatformConfigSchema:
@@ -248,17 +252,7 @@ class PlatformConfigSchema:
     @staticmethod
     def __opensearch_schema() -> dict:
         # TODO: DBTP-1943: Move to OpenSearch provider?
-        _valid_opensearch_plans = Or(
-            "tiny",
-            "small",
-            "small-ha",
-            "medium",
-            "medium-ha",
-            "large",
-            "large-ha",
-            "x-large",
-            "x-large-ha",
-        )
+        _valid_opensearch_plans = Or(*plan_manager.get_plan_names("opensearch"))
 
         return {
             "type": "opensearch",
@@ -285,28 +279,7 @@ class PlatformConfigSchema:
 
     @staticmethod
     def __postgres_schema() -> dict:
-        # TODO: DBTP-1943: Move to Postgres provider?
-        _valid_postgres_plans = Or(
-            "tiny",
-            "small",
-            "small-ha",
-            "small-high-io",
-            "medium",
-            "medium-ha",
-            "medium-high-io",
-            "large",
-            "large-ha",
-            "large-high-io",
-            "x-large",
-            "x-large-ha",
-            "x-large-high-io",
-            "2x-large",
-            "2x-large-ha",
-            "2x-large-high-io",
-            "4x-large",
-            "4x-large-ha",
-            "4x-large-high-io",
-        )
+        _valid_postgres_plans = Or(*plan_manager.get_plan_names("postgres"))
 
         # TODO: DBTP-1943: Move to Postgres provider?
         _valid_postgres_storage_types = Or("gp2", "gp3", "io1", "io2")
@@ -361,21 +334,7 @@ class PlatformConfigSchema:
 
     @staticmethod
     def __redis_schema() -> dict:
-        # TODO: DBTP-1943: move to Redis provider?
-        _valid_redis_plans = Or(
-            "micro",
-            "micro-ha",
-            "tiny",
-            "tiny-ha",
-            "small",
-            "small-ha",
-            "medium",
-            "medium-ha",
-            "large",
-            "large-ha",
-            "x-large",
-            "x-large-ha",
-        )
+        _valid_redis_plans = Or(*plan_manager.get_plan_names("redis"))
 
         return {
             "type": "redis",

{dbt_platform_helper-15.2.0 → dbt_platform_helper-15.2.1}/pyproject.toml

@@ -3,14 +3,17 @@ line-length = 100
 
 [tool.poetry]
 name = "dbt-platform-helper"
-version = "15.2.0"
+version = "15.2.1"
 description = "Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot."
 authors = ["Department for Business and Trade Platform Team <sre-team@digital.trade.gov.uk>"]
 license = "MIT"
 readme = "dbt_platform_helper/README.md"
 packages = [
     { include = "dbt_platform_helper" },
-    { include = "platform_helper.py" }
+    { include = "platform_helper.py" },
+    { include = "terraform/elasticache-redis/plans.yml" },
+    { include = "terraform/opensearch/plans.yml" },
+    { include = "terraform/postgres/plans.yml" },
 ]
 
 [tool.poetry.scripts]

dbt_platform_helper-15.2.1/terraform/elasticache-redis/plans.yml

@@ -0,0 +1,85 @@
+# t-shirt sizes for ElastiCache Redis
+
+# 0.5GB RAM, single node, no failover.
+micro:
+  replicas: 0
+  instance: cache.t4g.micro
+  automatic_failover_enabled: false
+  multi_az_enabled: false
+
+# 0.5GB RAM, highly-available.
+micro-ha:
+  replicas: 2
+  instance: cache.t4g.micro
+  automatic_failover_enabled: true
+  multi_az_enabled: true
+
+# 1.37GB RAM, single node, no failover.
+tiny:
+  replicas: 0
+  instance: cache.t4g.small
+  automatic_failover_enabled: false
+  multi_az_enabled: false
+
+# 1.37GB RAM, highly-available
+tiny-ha:
+  replicas: 2
+  instance: cache.t4g.small
+  automatic_failover_enabled: true
+  multi_az_enabled: true
+
+# 3.09GB RAM, single node, no failover.
+small:
+  replicas: 0
+  instance: cache.t4g.medium
+  automatic_failover_enabled: false
+  multi_az_enabled: false
+
+# 3.09GB RAM, highly-available
+small-ha:
+  replicas: 2
+  instance: cache.t4g.medium
+  automatic_failover_enabled: true
+  multi_az_enabled: true
+
+# 6.38GB RAM, single node, no failover.
+medium:
+  replicas: 0
+  instance: cache.m6g.large
+  automatic_failover_enabled: false
+  multi_az_enabled: false
+
+# 6.38GB RAM, highly-available
+medium-ha:
+  replicas: 2
+  instance: 'cache.m6g.large'
+  automatic_failover_enabled: true
+  multi_az_enabled: true
+
+# 12.93GB RAM, single node, no failover.
+large:
+  replicas: 0
+  instance: cache.m6g.xlarge
+  automatic_failover_enabled: false
+  multi_az_enabled: false
+
+# 12.93GB RAM, highly-available
+large-ha:
+  replicas: 2
+  instance: cache.m6g.xlarge
+  automatic_failover_enabled: true
+  multi_az_enabled: true
+
+# 26.04GB RAM, single node, no failover.
+x-large:
+  replicas: 0
+  instance: cache.m6g.2xlarge
+  automatic_failover_enabled: false
+  multi_az_enabled: false
+
+# 26.04GB RAM, highly-available
+x-large-ha:
+  replicas: 2
+  instance: cache.m6g.2xlarge
+  automatic_failover_enabled: true
+  multi_az_enabled: true

dbt_platform_helper-15.2.1/terraform/opensearch/plans.yml

@@ -0,0 +1,71 @@
+# t-shirt sizes for OpenSearch
+
+# 2 vCPU, 2GB RAM, volume size range 10-100GB
+tiny:
+  volume_size: 80
+  instances: 1
+  instance: t3.small.search
+  enable_ha: false
+
+# 2 vCPU, 2GB RAM, volume size range 10-100GB
+tiny-ha:
+  volume_size: 80
+  instances: 2
+  instance: t3.small.search
+  enable_ha: true
+
+# 2 vCPU, 4GB RAM, volume size range 10-200GB
+small:
+  volume_size: 200
+  instances: 1
+  instance: t3.medium.search
+  enable_ha: false
+
+# 2 nodes with 2 vCPU, 4GB RAM, volume size range 10-200GB
+small-ha:
+  volume_size: 200
+  instances: 2
+  instance: t3.medium.search
+  enable_ha: true
+
+# 2 vCPU, 8GB RAM, volume size range 10-512GB
+medium:
+  volume_size: 512
+  instances: 1
+  instance: m6g.large.search
+  enable_ha: false
+
+# 2 nodes with 2 vCPU, 8GB RAM, volume size range 10-512GB
+medium-ha:
+  volume_size: 512
+  instances: 2
+  instance: m6g.large.search
+  enable_ha: true
+
+# 4 vCPU, 16GB RAM, volume size range 10-1000GB
+large:
+  volume_size: 1000
+  instances: 1
+  instance: m6g.xlarge.search
+  enable_ha: false
+
+# 2 nodes with 4 vCPU, 16GB RAM, volume size range 10-1000GB
+large-ha:
+  volume_size: 1000
+  instances: 2
+  instance: m6g.xlarge.search
+  enable_ha: true
+
+# 8 vCPU, 32GB RAM, volume size range 10-1500GB
+x-large:
+  volume_size: 1500
+  instances: 1
+  instance: m6g.2xlarge.search
+  enable_ha: false
+
+# 2 nodes with 8 vCPU, 32GB RAM, volume size range 10-1500GB
+x-large-ha:
+  volume_size: 1500
+  instances: 2
+  instance: m6g.2xlarge.search
+  enable_ha: true

dbt_platform_helper-15.2.1/terraform/postgres/plans.yml

@@ -0,0 +1,128 @@
+# t-shirt sizes for Postgres
+# Aligned with the instance types from https://aws.amazon.com/rds/instance-types/
+
+# 2v CPU, 1GB RAM, 100GB Storage
+tiny:
+  volume_size: 100
+  multi_az: false
+  instance: db.t3.micro
+
+# 2v CPU, 2GB RAM, 100GB Storage
+small:
+  volume_size: 100
+  multi_az: false
+  instance: db.t3.small
+
+# 2v CPU, 2GB RAM, 100GB Storage, multi AZ
+small-ha:
+  volume_size: 100
+  multi_az: true
+  instance: db.t3.small
+
+# 2v CPU, 2GB RAM, 100GB Storage, multi AZ, high IOPS
+small-high-io:
+  volume_size: 100
+  multi_az: true
+  instance: db.t3.small
+  storage_type: io2
+  iops: 40000
+
+# # 2v CPU, 4GB RAM, 100GB Storage
+medium:
+  volume_size: 100
+  multi_az: false
+  instance: db.t3.medium
+
+# # 2v CPU, 4GB RAM, 100GB Storage, multi AZ
+medium-ha:
+  volume_size: 100
+  multi_az: true
+  instance: db.t3.medium
+
+# 2v CPU, 4GB RAM, 100GB Storage, multi AZ, high IOPS
+medium-high-io:
+  volume_size: 100
+  multi_az: true
+  instance: db.t3.medium
+  storage_type: io2
+  iops: 40000
+
+# 2v CPU, 8GB RAM, 100GB Storage
+large:
+  volume_size: 100
+  multi_az: false
+  instance: db.m5.large
+
+# 2v CPU, 8GB RAM, 100GB Storage, multi AZ
+large-ha:
+  volume_size: 100
+  multi_az: true
+  instance: db.m5.large
+
+# 2v CPU, 8GB RAM, 100GB Storage, multi AZ, high IOPS
+large-high-io:
+  volume_size: 100
+  multi_az: true
+  instance: db.m5.large
+  storage_type: io2
+  iops: 40000
+
+# 4v CPU, 16GB RAM, 100GB Storage
+x-large:
+  volume_size: 100
+  multi_az: false
+  instance: db.m5.xlarge
+
+# 4v CPU, 16GB RAM, 100GB Storage, multi AZ
+x-large-ha:
+  volume_size: 100
+  multi_az: true
+  instance: db.m5.xlarge
+
+# 4v CPU, 16GB RAM, 100GB Storage, multi AZ, high IOPS
+x-large-high-io:
+  volume_size: 100
+  multi_az: true
+  instance: db.m5.xlarge
+  storage_type: io2
+  iops: 40000
+
+# 8v CPU, 32GB RAM, 100GB Storage
+2x-large:
+  volume_size: 100
+  multi_az: false
+  instance: db.m5.2xlarge
+
+# 8v CPU, 32GB RAM, 100GB Storage, multi AZ
+2x-large-ha:
+  volume_size: 100
+  multi_az: true
+  instance: db.m5.2xlarge
+
+# 8v CPU, 32GB RAM, 100GB Storage, multi AZ, high IOPS
+2x-large-high-io:
+  volume_size: 100
+  multi_az: true
+  instance: db.m5.2xlarge
+  storage_type: io2
+  iops: 40000
+
+# 16v CPU, 64GB RAM, 100GB Storage
+4x-large:
+  volume_size: 100
+  multi_az: false
+  instance: db.m5.4xlarge
+
+# 16v CPU, 64GB RAM, 100GB Storage, multi AZ
+4x-large-ha:
+  volume_size: 100
+  multi_az: true
+  instance: db.m5.4xlarge
+
+# 16v CPU, 64GB RAM, 100GB Storage, multi AZ, high IOPS
+4x-large-high-io:
+  volume_size: 100
+  multi_az: true
+  instance: db.m5.4xlarge
+  storage_type: io2
+  iops: 40000

dbt_platform_helper-15.2.0/dbt_platform_helper/addon-plans.yml

@@ -1,224 +0,0 @@
-# t-shirt sizes for addons that support configurable storage/cpu etc. such as RDS, Opensearch and Elasticache Redis
-
-redis:
-  # 0.5GB RAM, single node, no failover.
-  micro:
-    replicas: 0
-    instance: cache.t4g.micro
-
-  # 0.5GB RAM, highly-available.
-  micro-ha:
-    replicas: 2
-    instance: cache.t4g.micro
-
-  # 1.37GB RAM, single node, no failover.
-  tiny:
-    replicas: 0
-    instance: cache.t4g.small
-
-  # 1.37GB RAM, highly-available
-  tiny-ha:
-    replicas: 2
-    instance: cache.t4g.small
-
-  # 3.09GB RAM, single node, no failover.
-  small:
-    replicas: 0
-    instance: cache.t4g.medium
-
-  # 3.09GB RAM, highly-available
-  small-ha:
-    replicas: 2
-    instance: cache.t4g.medium
-
-  # 6.38GB RAM, single node, no failover.
-  medium:
-    replicas: 0
-    instance: cache.m6g.large
-
-  # 6.38GB RAM, highly-available
-  medium-ha:
-    replicas: 2
-    instance: 'cache.m6g.large'
-
-  # 12.93GB RAM, single node, no failover.
-  large:
-    replicas: 0
-    instance: cache.m6g.xlarge
-
-  # 12.93GB RAM, highly-available
-  large-ha:
-    replicas: 2
-    instance: cache.m6g.xlarge
-
-  # 26.04GB RAM, single node, no failover.
-  x-large:
-    replicas: 0
-    instance: cache.m6g.2xlarge
-
-  # 26.04GB RAM, highly-available
-  x-large-ha:
-    replicas: 2
-    instance: cache.m6g.2xlarge
-
-opensearch:
-  # 2 vCPU, 2GB RAM, volume size range 10-100GB
-  tiny:
-    volume_size: 80
-    instances: 1
-    master: false
-    instance: t3.small.search
-
-  # 2 vCPU, 4GB RAM, volume size range 10-200GB
-  small:
-    volume_size: 200
-    instances: 1
-    master: false
-    instance: t3.medium.search
-
-  # 2 nodes with 2 vCPU, 4GB RAM, volume size range 10-200GB
-  small-ha:
-    volume_size: 200
-    instances: 2
-    master: false
-    instance: t3.medium.search
-
-  # 2 vCPU, 8GB RAM, volume size range 10-512GB
-  medium:
-    volume_size: 512
-    instances: 1
-    master: false
-    instance: m6g.large.search
-
-  # 2 nodes with 2 vCPU, 8GB RAM, volume size range 10-512GB
-  medium-ha:
-    volume_size: 512
-    instances: 2
-    master: false
-    instance: m6g.large.search
-
-  # 4 vCPU, 16GB RAM, volume size range 10-1000GB
-  large:
-    volume_size: 1000
-    instances: 1
-    master: false
-    instance: m6g.xlarge.search
-
-  # 2 nodes with 4 vCPU, 16GB RAM, volume size range 10-1000GB
-  large-ha:
-    volume_size: 1000
-    instances: 2
-    master: false
-    instance: m6g.xlarge.search
-
-  # 8 vCPU, 32GB RAM, volume size range 10-1500GB
-  x-large:
-    volume_size: 1500
-    instances: 1
-    master: false
-    instance: m6g.2xlarge.search
-
-  # 2 nodes with 8 vCPU, 32GB RAM, volume size range 10-1500GB
-  x-large-ha:
-    volume_size: 1500
-    instances: 2
-    master: false
-    instance: m6g.2xlarge.search
-
-# RDS/Postgres Instances
-postgres:
-  # 2v CPU, 1GB RAM, 20GB Storage
-  tiny:
-    volume_size: 20
-    multi_az: false
-    instance: db.t3.micro
-
-  # 2v CPU, 2GB RAM, 100GB Storage
-  small:
-    volume_size: 100
-    multi_az: false
-    instance: db.t3.small
-
-  # 2v CPU, 2GB RAM, 100GB Storage, multi AZ
-  small-ha:
-    volume_size: 100
-    multi_az: true
-    instance: db.t3.small
-
-  # 2v CPU, 2GB RAM, 100GB Storage, multi AZ
-  small-high-io:
-    volume_size: 100
-    multi_az: true
-    instance: db.t3.small
-    storage_type: io1
-    iops: 40000
-
-  # 2v CPU, 8GB RAM, 100GB Storage
-  medium:
-    volume_size: 100
-    multi_az: false
-    instance: db.m5.large
-
-  # 2v CPU, 8GB RAM, 100GB Storage, multi AZ
-  medium-ha:
-    volume_size: 100
-    multi_az: true
-    instance: db.m5.large
-
-  # 2v CPU, 8GB RAM, 100GB Storage, multi AZ
-  medium-high-io:
-    volume_size: 100
-    multi_az: true
-    instance: db.m5.large
-    storage_type: io1
-    iops: 40000
-
-  # 8v CPU, 32GB RAM, 564GB Storage
-  large:
-    volume_size: 564
-    multi_az: false
-    instance: db.m5.2xlarge
-
-  # 8v CPU, 32GB RAM, 564GB Storage, multi AZ
-  large-ha:
-    volume_size: 564
-    multi_az: true
-    instance: db.m5.2xlarge
-
-  # 8v CPU, 32GB RAM, 564GB Storage, multi AZ
-  large-high-io:
-    volume_size: 564
-    multi_az: true
-    instance: db.m5.2xlarge
-    storage_type: io1
-    iops: 40000
-
-  # 16v CPU, 64GB RAM, 2000GB Storage
-  x-large:
-    volume_size: 2000
-    multi_az: false
-    instance: db.m5.4xlarge
-
-  # 16v CPU, 64GB RAM, 2000GB Storage, multi AZ
-  x-large-ha:
-    volume_size: 2000
-    multi_az: true
-    instance: db.m5.4xlarge
-
-  # 16v CPU, 64GB RAM, 2000GB Storage, multi AZ
-  x-large-high-io:
-    volume_size: 2000
-    multi_az: true
-    instance: db.m5.4xlarge
-    storage_type: io1
-    iops: 40000
-
-s3: {}
-
-s3-policy: {}
-
-monitoring: {}
-
-alb: {}
-
-prometheus-policy: {}