service-capacity-modeling 0.3.73__py3-none-any.whl → 0.3.79__py3-none-any.whl

service_capacity_modeling/capacity_planner.py

@@ -190,7 +190,7 @@ def model_desires_percentiles(
 def _set_instance_objects(
     desires: CapacityDesires,
     hardware: Hardware,
-):
+) -> None:
     if desires.current_clusters:
         for zonal_cluster_capacity in desires.current_clusters.zonal:
             if zonal_cluster_capacity.cluster_instance_name in hardware.instances:
@@ -290,7 +290,9 @@ def _regret(
     return plans_by_regret
 
 
-def _add_requirement(requirement, accum):
+def _add_requirement(
+    requirement: CapacityRequirement, accum: Dict[str, Dict[str, List[Interval]]]
+) -> None:
     if requirement.requirement_type not in accum:
         accum[requirement.requirement_type] = {}
 
@@ -305,7 +307,11 @@ def _add_requirement(requirement, accum):
                 requirements[field].append(d)
 
 
-def _merge_models(plans_by_model, zonal_requirements, regional_requirements):
+def _merge_models(
+    plans_by_model: List[List[CapacityPlan]],
+    zonal_requirements: Dict[str, Dict[str, List[Interval]]],
+    regional_requirements: Dict[str, Dict[str, List[Interval]]],
+) -> List[CapacityPlan]:
     capacity_plans = []
     for composed in zip(*filter(lambda x: x, plans_by_model)):
         merged_plans = [functools.reduce(merge_plan, composed)]
@@ -331,10 +337,10 @@ def _in_allowed(inp: str, allowed: Sequence[str]) -> bool:
 class CapacityPlanner:
     def __init__(
         self,
-        default_num_simulations=128,
-        default_num_results=2,
-        default_lifecycles=(Lifecycle.stable, Lifecycle.beta),
-    ):
+        default_num_simulations: int = 128,
+        default_num_results: int = 2,
+        default_lifecycles: Tuple[Lifecycle, ...] = (Lifecycle.stable, Lifecycle.beta),
+    ) -> None:
         self._shapes: HardwareShapes = shapes
         self._models: Dict[str, CapacityModel] = {}
 
@@ -343,11 +349,11 @@ class CapacityPlanner:
         self._default_regret_params = CapacityRegretParameters()
         self._default_lifecycles = default_lifecycles
 
-    def register_group(self, group: Callable[[], Dict[str, CapacityModel]]):
+    def register_group(self, group: Callable[[], Dict[str, CapacityModel]]) -> None:
         for name, model in group().items():
             self.register_model(name, model)
 
-    def register_model(self, name: str, capacity_model: CapacityModel):
+    def register_model(self, name: str, capacity_model: CapacityModel) -> None:
         self._models[name] = capacity_model
 
     @property
@@ -429,16 +435,16 @@ class CapacityPlanner:
 
     def _group_plans_by_percentile(  # pylint: disable=too-many-positional-arguments
         self,
-        drives,
-        extra_model_arguments,
-        instance_families,
-        lifecycles,
-        num_regions,
-        num_results,
-        region,
-        model_percentile_desires,
-        sorted_percentiles,
-    ):
+        drives: Optional[Sequence[str]],
+        extra_model_arguments: Dict[str, Any],
+        instance_families: Optional[Sequence[str]],
+        lifecycles: Sequence[Lifecycle],
+        num_regions: int,
+        num_results: Optional[int],
+        region: str,
+        model_percentile_desires: Any,
+        sorted_percentiles: List[int],
+    ) -> Dict[int, Sequence[CapacityPlan]]:
         percentile_plans = {}
         for index, percentile in enumerate(sorted_percentiles):
             percentile_plan = []
@@ -470,15 +476,15 @@ class CapacityPlanner:
 
     def _mean_plan(  # pylint: disable=too-many-positional-arguments
         self,
-        drives,
-        extra_model_arguments,
-        instance_families,
-        lifecycles,
-        num_regions,
-        num_results,
-        region,
-        model_mean_desires,
-    ):
+        drives: Optional[Sequence[str]],
+        extra_model_arguments: Dict[str, Any],
+        instance_families: Optional[Sequence[str]],
+        lifecycles: Sequence[Lifecycle],
+        num_regions: int,
+        num_results: Optional[int],
+        region: str,
+        model_mean_desires: Dict[str, CapacityDesires],
+    ) -> Sequence[CapacityPlan]:
         mean_plans = []
         for mean_sub_model, mean_sub_desire in model_mean_desires.items():
             mean_sub_plan = self._plan_certain(
@@ -585,7 +591,7 @@ class CapacityPlanner:
         ]
 
     # Calculates the minimum cpu, memory, and network requirements based on desires.
-    def _per_instance_requirements(self, desires) -> Tuple[int, float]:
+    def _per_instance_requirements(self, desires: CapacityDesires) -> Tuple[int, float]:
         # Applications often set fixed reservations of heap or OS memory
         per_instance_mem = (
             desires.data_shape.reserved_instance_app_mem_gib
@@ -621,14 +627,14 @@ class CapacityPlanner:
 
     def generate_scenarios(  # pylint: disable=too-many-positional-arguments
         self,
-        model,
-        region,
-        desires,
-        num_regions,
-        lifecycles,
-        instance_families,
-        drives,
-    ):
+        model: CapacityModel,
+        region: str,
+        desires: CapacityDesires,
+        num_regions: int,
+        lifecycles: Optional[Sequence[Lifecycle]],
+        instance_families: Optional[Sequence[str]],
+        drives: Optional[Sequence[str]],
+    ) -> Generator[Tuple[Instance, Drive, RegionContext], None, None]:
         lifecycles = lifecycles or self._default_lifecycles
         instance_families = instance_families or []
         drives = drives or []
@@ -718,8 +724,8 @@ class CapacityPlanner:
         lifecycles = lifecycles or self._default_lifecycles
 
         # requirement types -> values
-        zonal_requirements: Dict[str, Dict] = {}
-        regional_requirements: Dict[str, Dict] = {}
+        zonal_requirements: Dict[str, Dict[str, List[Interval]]] = {}
+        regional_requirements: Dict[str, Dict[str, List[Interval]]] = {}
 
         regret_clusters_by_model: Dict[
             str, Sequence[Tuple[CapacityPlan, CapacityDesires, float]]
@@ -836,7 +842,7 @@ class CapacityPlanner:
         model_name: str,
         desires: CapacityDesires,
         extra_model_arguments: Dict[str, Any],
-    ):
+    ) -> Generator[Tuple[str, CapacityDesires], None, None]:
         queue: List[Tuple[CapacityDesires, str]] = [(desires, model_name)]
         models_used = []
 

service_capacity_modeling/hardware/__init__.py

@@ -5,6 +5,8 @@ import logging
 import os
 from functools import reduce
 from pathlib import Path
+from typing import Any
+from typing import cast
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -20,11 +22,11 @@ from service_capacity_modeling.interface import Service
 logger = logging.getLogger(__name__)
 
 
-def load_pricing(pricing: Dict) -> Pricing:
+def load_pricing(pricing: Dict[str, Any]) -> Pricing:
     return Pricing(regions=pricing)
 
 
-def load_hardware(hardware: Dict) -> Hardware:
+def load_hardware(hardware: Dict[str, Any]) -> Hardware:
     return Hardware(**hardware)
 
 
@@ -103,7 +105,9 @@ def merge_hardware(existing: Hardware, override: Hardware) -> Hardware:
             merged[key] = override_obj.get(key)
         elif isinstance(existing_field, Dict):
             override_field = override_obj.get(key)
-            merged_field = merged.setdefault(key, {})
+            if override_field is None:
+                override_field = {}
+            merged_field = cast(Dict[str, Any], merged.setdefault(key, {}))
 
             existing_keys = existing_field.keys()
             override_keys = override_obj.get(key, {}).keys()
@@ -119,7 +123,7 @@ def merge_hardware(existing: Hardware, override: Hardware) -> Hardware:
     return Hardware(**merged)
 
 
-def merge_pricing(existing: Dict, override: Dict) -> Dict:
+def merge_pricing(existing: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
     merged = existing.copy()
     for region, override_pricing in override.items():
         if region not in merged:
@@ -165,7 +169,7 @@ def load_hardware_from_disk(
     if shape_paths is None:
         shape_paths = []
 
-    combined_pricing: Dict = {}
+    combined_pricing: Dict[str, Any] = {}
 
     logger.debug("Loading pricing from: %s", price_paths)
     for price_path in price_paths:
@@ -185,7 +189,7 @@ def load_hardware_from_disk(
     return price_hardware(hardware=hardware, pricing=pricing)
 
 
-def load_hardware_from_s3(bucket, path) -> GlobalHardware:
+def load_hardware_from_s3(bucket: str, path: str) -> GlobalHardware:
     try:
         # boto is a heavy dependency so we only want to take it if
         # someone will be using it ...
@@ -205,7 +209,7 @@ def load_hardware_from_s3(bucket, path) -> GlobalHardware:
 
 
 class HardwareShapes:
-    def __init__(self):
+    def __init__(self) -> None:
         self._hardware: Optional[GlobalHardware] = None
 
     def load(self, new_hardware: GlobalHardware) -> None:

service_capacity_modeling/hardware/profiles/shapes/aws/auto_i3en.json

@@ -0,0 +1,172 @@
+{
+  "instances": {
+    "i3en.large": {
+      "name": "i3en.large",
+      "cpu": 2,
+      "cpu_cores": 1,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 15.26,
+      "net_mbps": 2100.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 1164,
+        "read_io_per_s": 42500,
+        "write_io_per_s": 32500,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.xlarge": {
+      "name": "i3en.xlarge",
+      "cpu": 4,
+      "cpu_cores": 2,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 30.52,
+      "net_mbps": 4200.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 2328,
+        "read_io_per_s": 85000,
+        "write_io_per_s": 65000,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.2xlarge": {
+      "name": "i3en.2xlarge",
+      "cpu": 8,
+      "cpu_cores": 4,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 61.04,
+      "net_mbps": 8400.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 4657,
+        "read_io_per_s": 170000,
+        "write_io_per_s": 130000,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.3xlarge": {
+      "name": "i3en.3xlarge",
+      "cpu": 12,
+      "cpu_cores": 6,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 91.55,
+      "net_mbps": 12500.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 6985,
+        "read_io_per_s": 255000,
+        "write_io_per_s": 195000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.6xlarge": {
+      "name": "i3en.6xlarge",
+      "cpu": 24,
+      "cpu_cores": 12,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 183.11,
+      "net_mbps": 25000.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 13970,
+        "read_io_per_s": 510000,
+        "write_io_per_s": 390000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.12xlarge": {
+      "name": "i3en.12xlarge",
+      "cpu": 48,
+      "cpu_cores": 24,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 366.21,
+      "net_mbps": 50000.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 27940,
+        "read_io_per_s": 1020000,
+        "write_io_per_s": 780000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.24xlarge": {
+      "name": "i3en.24xlarge",
+      "cpu": 96,
+      "cpu_cores": 48,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 732.42,
+      "net_mbps": 100000.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 55879,
+        "read_io_per_s": 2040000,
+        "write_io_per_s": 1560000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    }
+  }
+}

service_capacity_modeling/hardware/profiles/shapes/aws/auto_i4i.json

@@ -0,0 +1,220 @@
+{
+  "instances": {
+    "i4i.large": {
+      "name": "i4i.large",
+      "cpu": 2,
+      "cpu_cores": 1,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 15.26,
+      "net_mbps": 781.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 436,
+        "read_io_per_s": 50000,
+        "write_io_per_s": 27500,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.xlarge": {
+      "name": "i4i.xlarge",
+      "cpu": 4,
+      "cpu_cores": 2,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 30.52,
+      "net_mbps": 1875.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 873,
+        "read_io_per_s": 100000,
+        "write_io_per_s": 55000,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.2xlarge": {
+      "name": "i4i.2xlarge",
+      "cpu": 8,
+      "cpu_cores": 4,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 61.04,
+      "net_mbps": 4687.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 1746,
+        "read_io_per_s": 200000,
+        "write_io_per_s": 110000,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.4xlarge": {
+      "name": "i4i.4xlarge",
+      "cpu": 16,
+      "cpu_cores": 8,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 122.07,
+      "net_mbps": 9375.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 3492,
+        "read_io_per_s": 400000,
+        "write_io_per_s": 220000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.8xlarge": {
+      "name": "i4i.8xlarge",
+      "cpu": 32,
+      "cpu_cores": 16,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 244.14,
+      "net_mbps": 18750.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 6985,
+        "read_io_per_s": 800000,
+        "write_io_per_s": 440000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.12xlarge": {
+      "name": "i4i.12xlarge",
+      "cpu": 48,
+      "cpu_cores": 24,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 366.21,
+      "net_mbps": 28125.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 10477,
+        "read_io_per_s": 1200000,
+        "write_io_per_s": 660000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.16xlarge": {
+      "name": "i4i.16xlarge",
+      "cpu": 64,
+      "cpu_cores": 32,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 488.28,
+      "net_mbps": 37500.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 13970,
+        "read_io_per_s": 1600000,
+        "write_io_per_s": 880000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.24xlarge": {
+      "name": "i4i.24xlarge",
+      "cpu": 96,
+      "cpu_cores": 48,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 732.42,
+      "net_mbps": 56250.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 20955,
+        "read_io_per_s": 2400000,
+        "write_io_per_s": 1320000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.32xlarge": {
+      "name": "i4i.32xlarge",
+      "cpu": 128,
+      "cpu_cores": 64,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 976.56,
+      "net_mbps": 75000.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 27940,
+        "read_io_per_s": 3200000,
+        "write_io_per_s": 1760000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    }
+  }
+}