service-capacity-modeling 0.3.79__py3-none-any.whl → 0.3.81__py3-none-any.whl

service_capacity_modeling/capacity_planner.py

@@ -85,7 +85,7 @@ def model_desires(
     data_shape = desires.data_shape.model_copy(deep=True)
 
     query_pattern_simulation = {}
-    for field in sorted(query_pattern.model_fields):
+    for field in sorted(QueryPattern.model_fields):
         d = getattr(query_pattern, field)
         if isinstance(d, Interval):
             query_pattern_simulation[field] = simulate_interval(d, field)(num_sims)
@@ -93,7 +93,7 @@ def model_desires(
             query_pattern_simulation[field] = [d] * num_sims
 
     data_shape_simulation = {}
-    for field in sorted(data_shape.model_fields):
+    for field in sorted(DataShape.model_fields):
         d = getattr(data_shape, field)
         if isinstance(d, Interval):
             data_shape_simulation[field] = simulate_interval(d, field)(num_sims)
@@ -104,14 +104,11 @@ def model_desires(
         query_pattern = QueryPattern(
             **{
                 f: query_pattern_simulation[f][sim]
-                for f in sorted(query_pattern.model_fields)
+                for f in sorted(QueryPattern.model_fields)
             }
         )
         data_shape = DataShape(
-            **{
-                f: data_shape_simulation[f][sim]
-                for f in sorted(data_shape.model_fields)
-            }
+            **{f: data_shape_simulation[f][sim] for f in sorted(DataShape.model_fields)}
         )
 
         d = desires.model_copy(update={"query_pattern": None, "data_shape": None})
@@ -129,7 +126,7 @@ def model_desires_percentiles(
 
     query_pattern_simulation = {}
     query_pattern_means = {}
-    for field in sorted(query_pattern.model_fields):
+    for field in sorted(QueryPattern.model_fields):
         d = getattr(query_pattern, field)
         if isinstance(d, Interval):
             query_pattern_simulation[field] = interval_percentile(d, percentiles)
@@ -143,7 +140,7 @@ def model_desires_percentiles(
 
     data_shape_simulation = {}
     data_shape_means = {}
-    for field in sorted(data_shape.model_fields):
+    for field in sorted(DataShape.model_fields):
         d = getattr(data_shape, field)
         if isinstance(d, Interval):
             data_shape_simulation[field] = interval_percentile(d, percentiles)
@@ -161,13 +158,13 @@ def model_desires_percentiles(
             query_pattern = QueryPattern(
                 **{
                     f: query_pattern_simulation[f][i]
-                    for f in sorted(query_pattern.model_fields)
+                    for f in sorted(QueryPattern.model_fields)
                 }
             )
         except Exception as exp:
             raise exp
         data_shape = DataShape(
-            **{f: data_shape_simulation[f][i] for f in sorted(data_shape.model_fields)}
+            **{f: data_shape_simulation[f][i] for f in sorted(DataShape.model_fields)}
         )
         d = desires.model_copy(deep=True)
         d.query_pattern = query_pattern
@@ -175,10 +172,10 @@ def model_desires_percentiles(
         results.append(d)
 
     mean_qp = QueryPattern(
-        **{f: query_pattern_means[f] for f in sorted(query_pattern.model_fields)}
+        **{f: query_pattern_means[f] for f in sorted(QueryPattern.model_fields)}
     )
     mean_ds = DataShape(
-        **{f: data_shape_means[f] for f in sorted(data_shape.model_fields)}
+        **{f: data_shape_means[f] for f in sorted(DataShape.model_fields)}
     )
     d = desires.model_copy(deep=True)
     d.query_pattern = mean_qp
@@ -298,7 +295,7 @@ def _add_requirement(
 
     requirements = accum[requirement.requirement_type]
 
-    for field in sorted(requirement.model_fields):
+    for field in sorted(CapacityRequirement.model_fields):
         d = getattr(requirement, field)
         if isinstance(d, Interval):
             if field not in requirements:

service_capacity_modeling/models/org/netflix/__init__.py

@@ -3,6 +3,7 @@ from typing import Dict
 
 from .aurora import nflx_aurora_capacity_model
 from .cassandra import nflx_cassandra_capacity_model
+from .control import nflx_control_capacity_model
 from .counter import nflx_counter_capacity_model
 from .crdb import nflx_cockroachdb_capacity_model
 from .ddb import nflx_ddb_capacity_model
@@ -38,6 +39,7 @@ def models() -> Dict[str, Any]:
         "org.netflix.elasticsearch.master": nflx_elasticsearch_master_capacity_model,
         "org.netflix.elasticsearch.search": nflx_elasticsearch_search_capacity_model,
         "org.netflix.entity": nflx_entity_capacity_model,
+        "org.netflix.control": nflx_control_capacity_model,
         "org.netflix.cockroachdb": nflx_cockroachdb_capacity_model,
         "org.netflix.aurora": nflx_aurora_capacity_model,
         "org.netflix.postgres": nflx_postgres_capacity_model,

service_capacity_modeling/models/org/netflix/control.py

@@ -0,0 +1,146 @@
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import Optional
+from typing import Tuple
+
+from .stateless_java import nflx_java_app_capacity_model
+from service_capacity_modeling.interface import AccessConsistency
+from service_capacity_modeling.interface import AccessPattern
+from service_capacity_modeling.interface import CapacityDesires
+from service_capacity_modeling.interface import CapacityPlan
+from service_capacity_modeling.interface import Consistency
+from service_capacity_modeling.interface import DataShape
+from service_capacity_modeling.interface import Drive
+from service_capacity_modeling.interface import FixedInterval
+from service_capacity_modeling.interface import GlobalConsistency
+from service_capacity_modeling.interface import Instance
+from service_capacity_modeling.interface import Interval
+from service_capacity_modeling.interface import QueryPattern
+from service_capacity_modeling.interface import RegionContext
+from service_capacity_modeling.models import CapacityModel
+from service_capacity_modeling.interface import certain_int
+
+
+class NflxControlCapacityModel(CapacityModel):
+    @staticmethod
+    def capacity_plan(
+        instance: Instance,
+        drive: Drive,
+        context: RegionContext,
+        desires: CapacityDesires,
+        extra_model_arguments: Dict[str, Any],
+    ) -> Optional[CapacityPlan]:
+        # Control wants 20GiB root volumes
+        extra_model_arguments.setdefault("root_disk_gib", 20)
+
+        # Ensure Java app has enough memory to cache the whole dataset
+        modified_desires = desires.model_copy(deep=True)
+        if modified_desires.data_shape.estimated_state_size_gib:
+            # double buffer the cache
+            additional_mem = 2 * desires.data_shape.estimated_state_size_gib.mid
+            modified_desires.data_shape.reserved_instance_app_mem_gib += additional_mem
+
+        control_app = nflx_java_app_capacity_model.capacity_plan(
+            instance=instance,
+            drive=drive,
+            context=context,
+            desires=modified_desires,
+            extra_model_arguments=extra_model_arguments,
+        )
+        if control_app is None:
+            return None
+
+        for cluster in control_app.candidate_clusters.regional:
+            cluster.cluster_type = "dgwcontrol"
+        return control_app
+
+    @staticmethod
+    def description() -> str:
+        return "Netflix Control Model"
+
+    @staticmethod
+    def extra_model_arguments_schema() -> Dict[str, Any]:
+        return nflx_java_app_capacity_model.extra_model_arguments_schema()
+
+    @staticmethod
+    def compose_with(
+        user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
+    ) -> Tuple[Tuple[str, Callable[[CapacityDesires], CapacityDesires]], ...]:
+        def _modify_aurora_desires(
+            user_desires: CapacityDesires,
+        ) -> CapacityDesires:
+            relaxed = user_desires.model_copy(deep=True)
+
+            # Aurora doesn't support tier 0, so downgrade to tier 1
+            if relaxed.service_tier == 0:
+                relaxed.service_tier = 1
+
+            # Control caches reads in memory, only writes go to Aurora
+            # Set read QPS to minimal since Aurora only handles writes
+            if relaxed.query_pattern.estimated_read_per_second:
+                relaxed.query_pattern.estimated_read_per_second = certain_int(1)
+
+            return relaxed
+
+        return (("org.netflix.aurora", _modify_aurora_desires),)
+
+    @staticmethod
+    def default_desires(
+        user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
+    ) -> CapacityDesires:
+        return CapacityDesires(
+            query_pattern=QueryPattern(
+                access_pattern=AccessPattern.latency,
+                access_consistency=GlobalConsistency(
+                    same_region=Consistency(
+                        target_consistency=AccessConsistency.read_your_writes,
+                    ),
+                    cross_region=Consistency(
+                        target_consistency=AccessConsistency.eventual,
+                    ),
+                ),
+                estimated_mean_read_size_bytes=Interval(
+                    low=128, mid=1024, high=65536, confidence=0.95
+                ),
+                estimated_mean_write_size_bytes=Interval(
+                    low=128, mid=1024, high=65536, confidence=0.95
+                ),
+                estimated_mean_read_latency_ms=Interval(
+                    low=0.2, mid=1, high=2, confidence=0.98
+                ),
+                estimated_mean_write_latency_ms=Interval(
+                    low=0.2, mid=1, high=2, confidence=0.98
+                ),
+                # "Single digit milliseconds SLO"
+                read_latency_slo_ms=FixedInterval(
+                    minimum_value=0.2,
+                    maximum_value=10,
+                    low=1,
+                    mid=3,
+                    high=6,
+                    confidence=0.98,
+                ),
+                write_latency_slo_ms=FixedInterval(
+                    minimum_value=0.2,
+                    maximum_value=10,
+                    low=0.4,
+                    mid=2,
+                    high=5,
+                    confidence=0.98,
+                ),
+            ),
+            # Most Control clusters are small
+            data_shape=DataShape(
+                estimated_state_size_gib=Interval(
+                    low=0.1, mid=1, high=10, confidence=0.98
+                ),
+                estimated_state_item_count=Interval(
+                    low=100000, mid=1000000, high=10000000, confidence=0.98
+                ),
+                reserved_instance_app_mem_gib=8,
+            ),
+        )
+
+
+nflx_control_capacity_model = NflxControlCapacityModel()

service_capacity_modeling/stats.py

@@ -45,7 +45,11 @@ def _gamma_fn_from_params(
 
     def f(k: float) -> float:
         zero = high / low
-        return float(gammaf(k, high_p * k / mid) / gammaf(k, low_p * k / mid) - zero)
+        high_x = high_p * k / mid
+        low_x = low_p * k / mid
+        result = gammaf(k, high_x) / gammaf(k, low_x) - zero
+        # Use .item() to convert NumPy scalar to Python float
+        return result.item() if hasattr(result, "item") else float(result)
 
     return f
 

{service_capacity_modeling-0.3.79.dist-info → service_capacity_modeling-0.3.81.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.79
+Version: 0.3.81
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com

{service_capacity_modeling-0.3.79.dist-info → service_capacity_modeling-0.3.81.dist-info}/RECORD

@@ -1,7 +1,7 @@
 service_capacity_modeling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-service_capacity_modeling/capacity_planner.py,sha256=nmjAWpUtL5EKbEtQzSGWrM8gPSAVc7EX5M4ApiLbQnA,32828
+service_capacity_modeling/capacity_planner.py,sha256=JRagEFlg3u_zB1N5GzGsKAN55JZLad6p4IF_PmL8kcg,32780
 service_capacity_modeling/interface.py,sha256=p-pOUQUdA6VA-moTjFSBLQAZCRvcjFaAfb2jOmxMWsY,39074
-service_capacity_modeling/stats.py,sha256=ievz2f1H-a6u8wHwXBQ47e2EhqH9BDrdL6VZAXQDK2w,5964
+service_capacity_modeling/stats.py,sha256=LCNUcQPfwF5hhIZwsfAsDe4ZbnuhDnl3vQHKfpK61Xc,6142
 service_capacity_modeling/hardware/__init__.py,sha256=P5ostvoSOMUqPODtepeFYb4qfTVH0E73mMFraP49rYU,9196
 service_capacity_modeling/hardware/profiles/__init__.py,sha256=7-y3JbCBkgzaAjFla2RIymREcImdZ51HTl3yn3vzoGw,1602
 service_capacity_modeling/hardware/profiles/profiles.txt,sha256=tOfSR3B0E0uAOaXd5SLI3ioq83UYZ3yhK7UHhsK4awQ,49
@@ -50,9 +50,10 @@ service_capacity_modeling/models/common.py,sha256=Bs-G1eHrb9qmS9qEWzx0rzN9QjQ419
 service_capacity_modeling/models/headroom_strategy.py,sha256=rGo_d7nxkQDjx0_hIAXKKZAWnQDBtqZhc0eTMouVh8s,682
 service_capacity_modeling/models/utils.py,sha256=WosEEg4o1_WSbTb5mL-M1v8JuWJgvS2oWvnDS3qNz3k,2662
 service_capacity_modeling/models/org/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-service_capacity_modeling/models/org/netflix/__init__.py,sha256=W6rKkhSdPhjD-awm7mYakAhw7VKLNJYkqv-U3LfkMew,2444
+service_capacity_modeling/models/org/netflix/__init__.py,sha256=2Ld2NPxiO3vbYtOMqHtrV4f1nEH390Hoxqo3I5NbBDI,2553
 service_capacity_modeling/models/org/netflix/aurora.py,sha256=Js33ZjxCtt34HiDPsWRT9mjKCAsnnCo9du15QArVFMo,13073
 service_capacity_modeling/models/org/netflix/cassandra.py,sha256=Fp37bHtWRJctJYYJhW78YbLeFXoX26QxREP_BMEItNE,39003
+service_capacity_modeling/models/org/netflix/control.py,sha256=wkJnqG7Nn7kQNcKEMzl9LFlA9xTAji2wYxL011VAKPI,5762
 service_capacity_modeling/models/org/netflix/counter.py,sha256=XopKlNMdvO5EbxAggn-KW_q7L3aKtXLXbry4ocl6i5Q,10494
 service_capacity_modeling/models/org/netflix/crdb.py,sha256=iW7tyG8jpXhHIdXrw3DPYSHRAknPN42MlCRLJO4o9C8,20826
 service_capacity_modeling/models/org/netflix/ddb.py,sha256=9qRiuTqWev9zbYFFzewyowU7M41uALsuLklYx20yAXw,26502
@@ -75,9 +76,9 @@ service_capacity_modeling/tools/auto_shape.py,sha256=Jx9H2ay9-H_kUDjtB141owQNxGF
 service_capacity_modeling/tools/fetch_pricing.py,sha256=Qp-XMymkY1dvtyS51RufmEpfgOHv-IQ-XyzS8wp2-qM,4021
 service_capacity_modeling/tools/generate_missing.py,sha256=F7YqvMJAV4nZc20GNrlIsnQSF8_77sLgwYZqc5k4LDg,3099
 service_capacity_modeling/tools/instance_families.py,sha256=e5RuYkCLUITvsAazDH12B6KjX_PaBsv6Ne3mj0HK_sQ,9223
-service_capacity_modeling-0.3.79.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-service_capacity_modeling-0.3.79.dist-info/METADATA,sha256=nM1qflhu7AXKNcD-AqIXyg_EZQ_ITg77Rj8b8ur_D8A,10361
-service_capacity_modeling-0.3.79.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-service_capacity_modeling-0.3.79.dist-info/entry_points.txt,sha256=ZsjzpG5SomWpT1zCE19n1uSXKH2gTI_yc33sdl0vmJg,146
-service_capacity_modeling-0.3.79.dist-info/top_level.txt,sha256=H8XjTCLgR3enHq5t3bIbxt9SeUkUT8HT_SDv2dgIT_A,26
-service_capacity_modeling-0.3.79.dist-info/RECORD,,
+service_capacity_modeling-0.3.81.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+service_capacity_modeling-0.3.81.dist-info/METADATA,sha256=g1x4TGRElgvLbJiQZ_D_H5wN-Mhiynf4nPhOEVbkIEg,10361
+service_capacity_modeling-0.3.81.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+service_capacity_modeling-0.3.81.dist-info/entry_points.txt,sha256=ZsjzpG5SomWpT1zCE19n1uSXKH2gTI_yc33sdl0vmJg,146
+service_capacity_modeling-0.3.81.dist-info/top_level.txt,sha256=H8XjTCLgR3enHq5t3bIbxt9SeUkUT8HT_SDv2dgIT_A,26
+service_capacity_modeling-0.3.81.dist-info/RECORD,,