service-capacity-modeling 0.3.47__tar.gz → 0.3.49__tar.gz

{service_capacity_modeling-0.3.47 → service_capacity_modeling-0.3.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.47
+Version: 0.3.49
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com

{service_capacity_modeling-0.3.47 → service_capacity_modeling-0.3.49}/service_capacity_modeling/models/common.py

@@ -374,6 +374,9 @@ def compute_stateful_zone(  # pylint: disable=too-many-positional-arguments
     min_count: int = 0,
     adjusted_disk_io_needed: float = 0.0,
     read_write_ratio: float = 0.0,
+    # Max attached EBS volume size per node. Higher value here could allow
+    # for a lower instance count (allows more vertical scaling vs forcing horizontal)
+    max_attached_disk_gib: Optional[float] = None,
 ) -> ZoneClusterCapacity:
     # Datastores often require disk headroom for e.g. compaction and such
     if instance.drive is not None:
@@ -461,6 +464,9 @@ def compute_stateful_zone(  # pylint: disable=too-many-positional-arguments
         # 1/3 the maximum volume size in one node (preferring more nodes
         # with smaller volumes)
         max_size = drive.max_size_gib / 3
+        if max_attached_disk_gib is not None:
+            max_size = max_attached_disk_gib
+
         if ebs_gib > max_size > 0:
             ratio = ebs_gib / max_size
             count = max(cluster_size(math.ceil(count * ratio)), min_count)

{service_capacity_modeling-0.3.47 → service_capacity_modeling-0.3.49}/service_capacity_modeling/models/org/netflix/kafka.py

@@ -18,6 +18,7 @@ from service_capacity_modeling.interface import certain_float
 from service_capacity_modeling.interface import certain_int
 from service_capacity_modeling.interface import Clusters
 from service_capacity_modeling.interface import Consistency
+from service_capacity_modeling.interface import CurrentZoneClusterCapacity
 from service_capacity_modeling.interface import DataShape
 from service_capacity_modeling.interface import Drive
 from service_capacity_modeling.interface import GIB_IN_BYTES
@@ -47,13 +48,30 @@ def target_cpu_utilization(tier: int) -> float:
     """
     Returns the target average cluster CPU utilization for a given tier
     """
-    if tier == 0:
-        return 0.25
-    if tier == 1:
-        return 0.30
-    if tier == 2:
-        return 0.35
-    return 0.40
+    if tier in (0, 1):
+        return 0.40
+    return 0.50
+
+
+def _get_current_zonal_cluster(
+    desires: CapacityDesires,
+) -> Optional[CurrentZoneClusterCapacity]:
+    return (
+        None
+        if desires.current_clusters is None
+        else (
+            desires.current_clusters.zonal[0]
+            if len(desires.current_clusters.zonal)
+            else None
+        )
+    )
+
+
+def _is_same_instance_family(cluster, target_family):
+    """Check if cluster has a different instance family than the target."""
+    if not cluster or not cluster.cluster_instance:
+        return False
+    return cluster.cluster_instance.family == target_family
 
 
 def _estimate_kafka_requirement(  # pylint: disable=too-many-positional-arguments
@@ -90,15 +108,7 @@ def _estimate_kafka_requirement(  # pylint: disable=too-many-positional-argument
         write_mib_per_second
     )
     # use the current cluster capacity if available
-    current_zonal_cluster = (
-        None
-        if desires.current_clusters is None
-        else (
-            desires.current_clusters.zonal[0]
-            if len(desires.current_clusters.zonal)
-            else None
-        )
-    )
+    current_zonal_cluster = _get_current_zonal_cluster(desires)
 
     if (
         current_zonal_cluster
@@ -230,6 +240,7 @@ def _estimate_kafka_cluster_zonal(  # pylint: disable=too-many-positional-argume
     max_local_disk_gib: int = 1024 * 5,
     min_instance_cpu: int = 2,
     min_instance_memory_gib: int = 12,
+    require_same_instance_family: bool = True,
 ) -> Optional[CapacityPlan]:
 
     # Kafka doesn't like to deploy on single CPU instances or with < 12 GiB of ram
@@ -248,6 +259,15 @@ def _estimate_kafka_cluster_zonal(  # pylint: disable=too-many-positional-argume
     if instance.drive is None and drive.name != "gp3":
         return None
 
+    # If there is a current cluster, check if we are restricted to same instance family
+    current_zonal_cluster = _get_current_zonal_cluster(desires)
+    if (
+        current_zonal_cluster
+        and require_same_instance_family
+        and not _is_same_instance_family(current_zonal_cluster, instance.family)
+    ):
+        return None
+
     requirement, regrets = _estimate_kafka_requirement(
         instance=instance,
         desires=desires,
@@ -317,6 +337,8 @@ def _estimate_kafka_cluster_zonal(  # pylint: disable=too-many-positional-argume
         # Sidecars and Variable OS Memory
         # Kafka currently uses 8GiB fixed, might want to change to min(30, x // 2)
         reserve_memory=lambda instance_mem_gib: base_mem + 8,
+        # allow up to 8TiB of attached EBS
+        max_attached_disk_gib=8 * 1024,
     )
 
     # Communicate to the actual provision that if we want reduced RF
@@ -461,6 +483,10 @@ class NflxKafkaCapacityModel(CapacityModel):
         required_zone_size: Optional[int] = extra_model_arguments.get(
             "required_zone_size", None
         )
+        # By default, for existing clusters, restrict to only using same instance family
+        require_same_instance_family: bool = extra_model_arguments.get(
+            "require_same_instance_family", True
+        )
 
         return _estimate_kafka_cluster_zonal(
             instance=instance,
@@ -476,6 +502,7 @@ class NflxKafkaCapacityModel(CapacityModel):
             min_instance_cpu=min_instance_cpu,
             min_instance_memory_gib=min_instance_memory_gib,
             hot_retention_seconds=hot_retention_seconds,
+            require_same_instance_family=require_same_instance_family,
         )
 
     @staticmethod

{service_capacity_modeling-0.3.47 → service_capacity_modeling-0.3.49}/service_capacity_modeling.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.47
+Version: 0.3.49
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com