service-capacity-modeling 0.3.53__tar.gz → 0.3.55__tar.gz

{service_capacity_modeling-0.3.53 → service_capacity_modeling-0.3.55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.53
+Version: 0.3.55
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com

{service_capacity_modeling-0.3.53 → service_capacity_modeling-0.3.55}/service_capacity_modeling/capacity_planner.py

@@ -511,6 +511,7 @@ class CapacityPlanner:
         num_results: Optional[int] = None,
         num_regions: int = 3,
         extra_model_arguments: Optional[Dict[str, Any]] = None,
+        max_results_per_family: int = 1,
     ) -> Sequence[CapacityPlan]:
         if model_name not in self._models:
             raise ValueError(
@@ -538,6 +539,7 @@ class CapacityPlanner:
                 lifecycles=lifecycles,
                 instance_families=instance_families,
                 drives=drives,
+                max_results_per_family=max_results_per_family,
             )
             if sub_plan:
                 results.append(sub_plan)
@@ -555,6 +557,7 @@ class CapacityPlanner:
         instance_families: Optional[Sequence[str]] = None,
         drives: Optional[Sequence[str]] = None,
         extra_model_arguments: Optional[Dict[str, Any]] = None,
+        max_results_per_family: int = 1,
     ) -> Sequence[CapacityPlan]:
         extra_model_arguments = extra_model_arguments or {}
         model = self._models[model_name]
@@ -577,7 +580,9 @@ class CapacityPlanner:
         plans.sort(key=lambda p: (p.rank, p.candidate_clusters.total_annual_cost))
 
         num_results = num_results or self._default_num_results
-        return reduce_by_family(plans)[:num_results]
+        return reduce_by_family(plans, max_results_per_family=max_results_per_family)[
+            :num_results
+        ]
 
     # Calculates the minimum cpu, memory, and network requirements based on desires.
     def _per_instance_requirements(self, desires) -> Tuple[int, float]:
@@ -696,6 +701,7 @@ class CapacityPlanner:
         regret_params: Optional[CapacityRegretParameters] = None,
         extra_model_arguments: Optional[Dict[str, Any]] = None,
         explain: bool = False,
+        max_results_per_family: int = 1,
     ) -> UncertainCapacityPlan:
         extra_model_arguments = extra_model_arguments or {}
 
@@ -741,6 +747,7 @@ class CapacityPlanner:
                             lifecycles=lifecycles,
                             instance_families=instance_families,
                             drives=drives,
+                            max_results_per_family=max_results_per_family,
                         ),
                     )
                 )
@@ -763,7 +770,8 @@ class CapacityPlanner:
                 ],
                 zonal_requirements,
                 regional_requirements,
-            )
+            ),
+            max_results_per_family=max_results_per_family,
         )[:num_results]
 
         low_p, high_p = sorted(percentiles)[0], sorted(percentiles)[-1]

{service_capacity_modeling-0.3.53 → service_capacity_modeling-0.3.55}/service_capacity_modeling/models/org/netflix/cassandra.py

@@ -144,6 +144,7 @@ def _estimate_cassandra_requirement(  # pylint: disable=too-many-positional-argu
     disk_buffer = buffer_for_components(
         buffers=desires.buffers, components=[BufferComponent.disk]
     )
+    memory_preserve = False
     reference_shape = desires.reference_shape
     current_capacity = (
         None
@@ -169,6 +170,9 @@ def _estimate_cassandra_requirement(  # pylint: disable=too-many-positional-argu
             * current_capacity.cluster_instance_count.mid
             * (disk_scale or 1)
         )
+        _, memory_preserve = derived_buffer_for_component(
+            desires.buffers.derived, ["storage", "memory"]
+        )
     else:
         # If the cluster is not yet provisioned
         capacity_requirement = _zonal_requirement_for_new_cluster(
@@ -202,13 +206,19 @@ def _estimate_cassandra_requirement(  # pylint: disable=too-many-positional-argu
     )
     rps_working_set = min(1.0, disk_rps / max_rps_to_disk)
 
-    # If disk RPS will be smaller than our target because there are no
-    # reads, we don't need to hold as much data in memory.
-    # For c*, we can skip memory buffer and can just keep using the heap and write buffer calc
-    # Eventually we'll want to phrase those heap, read cache, and write cache as buffers
-    needed_memory = min(working_set, rps_working_set) * disk_used_gib * zones_per_region
-    # Now convert to per zone
-    needed_memory = max(1, int(needed_memory // zones_per_region))
+    if memory_preserve:
+        needed_memory = capacity_requirement.mem_gib.mid
+    else:
+        # If disk RPS will be smaller than our target because there are no
+        # reads, we don't need to hold as much data in memory.
+        # For c*, we can skip memory buffer and can just keep using the heap and write buffer calc
+        # Eventually we'll want to phrase those heap, read cache, and write cache as buffers
+        needed_memory = (
+            min(working_set, rps_working_set) * disk_used_gib * zones_per_region
+        )
+        # Now convert to per zone
+        needed_memory = max(1, int(needed_memory // zones_per_region))
+
     logger.debug(
         "Need (cpu, mem, disk, working) = (%s, %s, %s, %f)",
         needed_cores,
@@ -381,6 +391,7 @@ def _estimate_cassandra_cluster_zonal(  # pylint: disable=too-many-positional-ar
         # C* clusters provision in powers of 2 because doubling
         cluster_size=next_power_of_2,
         min_count=max(min_count, required_cluster_size or 0),
+        # TODO: Take reserve memory calculation into account during buffer calculation
         # C* heap usage takes away from OS page cache memory
         reserve_memory=lambda x: base_mem + heap_fn(x),
         # C* heap buffers the writes at roughly a rate of

{service_capacity_modeling-0.3.53 → service_capacity_modeling-0.3.55}/service_capacity_modeling/models/utils.py

@@ -1,19 +1,26 @@
 import math
+from typing import Dict
 from typing import Iterable
 from typing import List
-from typing import Set
 from typing import Tuple
 
 from service_capacity_modeling.models import CapacityPlan
 
 
-def reduce_by_family(plans: Iterable[CapacityPlan]) -> List[CapacityPlan]:
+def reduce_by_family(
+    plans: Iterable[CapacityPlan], max_results_per_family: int = 1
+) -> List[CapacityPlan]:
     """Groups a potential set of clusters by hardware family sorted by cost.
 
     Useful for showing different family options.
+
+    Args:
+        plans: Iterable of CapacityPlan objects to filter
+        max_results_per_family: Maximum number of results to return per
+            family combination
     """
-    zonal_families: Set[Tuple[Tuple[str, str], ...]] = set()
-    regional_families: Set[Tuple[Tuple[str, str], ...]] = set()
+    zonal_families: Dict[Tuple[Tuple[str, str], ...], int] = {}
+    regional_families: Dict[Tuple[Tuple[str, str], ...], int] = {}
 
     result: List[CapacityPlan] = []
     for plan in plans:
@@ -31,11 +38,20 @@ def reduce_by_family(plans: Iterable[CapacityPlan]) -> List[CapacityPlan]:
                 sorted({(c.cluster_type, c.instance.family) for c in topo.zonal})
             )
 
-        if not (zonal_type in zonal_families and regional_type in regional_families):
+        # Count how many of each family combination we've seen
+        zonal_count = zonal_families.get(zonal_type, 0)
+        regional_count = regional_families.get(regional_type, 0)
+
+        # Add the plan if we haven't reached the maximum for either family type
+        if (
+            zonal_count < max_results_per_family
+            or regional_count < max_results_per_family
+        ):
             result.append(plan)
 
-        regional_families.add(regional_type)
-        zonal_families.add(zonal_type)
+            # Update counters
+            zonal_families[zonal_type] = zonal_count + 1
+            regional_families[regional_type] = regional_count + 1
 
     return result
 

{service_capacity_modeling-0.3.53 → service_capacity_modeling-0.3.55}/service_capacity_modeling.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.53
+Version: 0.3.55
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com

{service_capacity_modeling-0.3.53 → service_capacity_modeling-0.3.55}/service_capacity_modeling.egg-info/SOURCES.txt

@@ -90,4 +90,5 @@ tests/test_io2.py
 tests/test_model_dump.py
 tests/test_reproducible.py
 tests/test_simulation.py
+tests/test_utils.py
 tests/test_working_set.py

service_capacity_modeling-0.3.55/tests/test_utils.py

@@ -0,0 +1,175 @@
+from decimal import Decimal
+from typing import Dict
+from typing import List
+
+from service_capacity_modeling.interface import CapacityPlan
+from service_capacity_modeling.interface import Clusters
+from service_capacity_modeling.interface import Instance
+from service_capacity_modeling.interface import Requirements
+from service_capacity_modeling.interface import ZoneClusterCapacity
+from service_capacity_modeling.models.utils import reduce_by_family
+
+
+# Create mock hardware instances with different families for all tests
+def get_test_instances():
+    shape_family_a1 = Instance(
+        name="family_a.a1",
+        family_separator=".",
+        cpu=2,
+        cpu_ghz=2.4,
+        ram_gib=8,
+        net_mbps=1000,
+    )
+    shape_family_a2 = Instance(
+        name="family_a.a2",
+        family_separator=".",
+        cpu=4,
+        cpu_ghz=2.4,
+        ram_gib=16,
+        net_mbps=2000,
+    )
+    shape_family_a3 = Instance(
+        name="family_a.a3",
+        family_separator=".",
+        cpu=8,
+        cpu_ghz=2.4,
+        ram_gib=32,
+        net_mbps=4000,
+    )
+
+    shape_family_b1 = Instance(
+        name="family_b.b1",
+        family_separator=".",
+        cpu=2,
+        cpu_ghz=2.4,
+        ram_gib=8,
+        net_mbps=1000,
+    )
+    shape_family_b2 = Instance(
+        name="family_b.b2",
+        family_separator=".",
+        cpu=4,
+        cpu_ghz=2.4,
+        ram_gib=16,
+        net_mbps=2000,
+    )
+
+    return (
+        shape_family_a1,
+        shape_family_a2,
+        shape_family_a3,
+        shape_family_b1,
+        shape_family_b2,
+    )
+
+
+def create_test_capacity_plans() -> List[CapacityPlan]:
+    """Create test capacity plans with different hardware families for testing."""
+    shapes = get_test_instances()
+    (
+        shape_family_a1,
+        shape_family_a2,
+        shape_family_a3,
+        shape_family_b1,
+        shape_family_b2,
+    ) = shapes
+
+    plans = []
+
+    # Family A plans
+    for i, shape in enumerate([shape_family_a1, shape_family_a2, shape_family_a3]):
+        annual_cost = (i + 1) * 100.0  # Different costs
+
+        cluster = ZoneClusterCapacity(
+            cluster_type="test_cluster",
+            count=i + 1,
+            instance=shape,
+            annual_cost=annual_cost,
+        )
+
+        annual_costs_a: Dict[str, Decimal] = {"test_cluster": Decimal(str(annual_cost))}
+
+        plans.append(
+            CapacityPlan(
+                requirements=Requirements(),
+                candidate_clusters=Clusters(
+                    annual_costs=annual_costs_a, zonal=[cluster], regional=[]
+                ),
+                cost=annual_cost,
+                efficiency=1.0,
+            )
+        )
+
+    # Family B plans
+    for i, shape in enumerate([shape_family_b1, shape_family_b2]):
+        annual_cost = (i + 1) * 200.0  # Different costs
+
+        cluster = ZoneClusterCapacity(
+            cluster_type="test_cluster",
+            count=i + 1,
+            instance=shape,
+            annual_cost=annual_cost,
+        )
+
+        annual_costs_b: Dict[str, Decimal] = {"test_cluster": Decimal(str(annual_cost))}
+
+        plans.append(
+            CapacityPlan(
+                requirements=Requirements(),
+                candidate_clusters=Clusters(
+                    annual_costs=annual_costs_b, zonal=[cluster], regional=[]
+                ),
+                cost=annual_cost,
+                efficiency=1.0,
+            )
+        )
+
+    return plans
+
+
+def test_reduce_by_family_default():
+    """Test that reduce_by_family with default parameter returns one plan per family."""
+    plans = create_test_capacity_plans()
+    result = reduce_by_family(plans)
+
+    # Should return only 2 plans - one from family_a and one from family_b
+    assert len(result) == 2
+
+    # Verify we have one from each family
+    families = set()
+    for plan in result:
+        for cluster in plan.candidate_clusters.zonal:
+            families.add(cluster.instance.family)
+
+    assert families == {"family_a", "family_b"}
+
+
+def test_reduce_by_family_multiple():
+    """Test that reduce_by_family with max_results_per_family > 1
+    returns multiple plans per family."""
+    plans = create_test_capacity_plans()
+    result = reduce_by_family(plans, max_results_per_family=2)
+
+    # Should return 4 plans - two from family_a and two from family_b
+    assert len(result) == 4
+
+    # Count plans per family
+    family_counts = {"family_a": 0, "family_b": 0}
+    for plan in result:
+        for cluster in plan.candidate_clusters.zonal:
+            family_counts[cluster.instance.family] += 1
+
+    # Verify we have exactly 2 from each family
+    assert family_counts["family_a"] == 2
+    assert family_counts["family_b"] == 2
+
+
+def test_reduce_by_family_unlimited():
+    """Test that reduce_by_family with max_results_per_family > available plans
+    returns all plans."""
+    plans = create_test_capacity_plans()
+    # Set max_results_per_family higher than the number of plans we have
+    result = reduce_by_family(plans, max_results_per_family=10)
+
+    # Should return all 5 plans since we have 3 from family_a and 2 from family_b
+    assert len(result) == 5