service-capacity-modeling 0.3.78__tar.gz → 0.3.79__tar.gz

{service_capacity_modeling-0.3.78 → service_capacity_modeling-0.3.79}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.78
+Version: 0.3.79
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com

{service_capacity_modeling-0.3.78 → service_capacity_modeling-0.3.79}/service_capacity_modeling/models/org/netflix/counter.py

@@ -46,7 +46,8 @@ class NflxCounterArguments(NflxJavaAppArguments):
     )
     counter_cardinality: NflxCounterCardinality = Field(
         alias="counter.cardinality",
-        description="Low means < 1,000, medium (1,000—100,000), high means > 100,000.",
+        description="Low means < 10,000, medium (10,000—1,000,000), high means "
+        "> 1,000,000.",
     )
     counter_mode: NflxCounterMode = Field(
         alias="counter.mode",
@@ -92,36 +93,57 @@ class NflxCounterCapacityModel(CapacityModel):
     def compose_with(
         user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
     ) -> Tuple[Tuple[str, Callable[[CapacityDesires], CapacityDesires]], ...]:
-        stores = [("org.netflix.evcache", lambda x: x)]
-        if extra_model_arguments["counter.mode"] != NflxCounterMode.best_effort.value:
+        stores = []
 
+        if extra_model_arguments["counter.mode"] == NflxCounterMode.best_effort.value:
+            stores.append(("org.netflix.evcache", lambda x: x))
+        else:
+            # Shared evcache cluster is used for eventual and exact counters
             def _modify_cassandra_desires(
                 user_desires: CapacityDesires,
             ) -> CapacityDesires:
                 modified = user_desires.model_copy(deep=True)
+                counter_cardinality = extra_model_arguments["counter.cardinality"]
+
+                counter_deltas_per_second = (
+                    user_desires.query_pattern.estimated_write_per_second
+                )
 
-                # counts per second
-                cps = user_desires.query_pattern.estimated_write_per_second
+                # low cardinality : rollups happen once every 60 seconds
+                # medium cardinality : rollups happen once every 30 seconds
+                # high cardinality : rollups happen once every 10 seconds
+                # TODO: Account for read amplification from time slice configs
+                #       for better model accuracy
+                if counter_cardinality == NflxCounterCardinality.low.value:
+                    rollups_per_second = counter_deltas_per_second.scale(0.0167)
+                elif counter_cardinality == NflxCounterCardinality.medium.value:
+                    rollups_per_second = counter_deltas_per_second.scale(0.0333)
+                else:
+                    rollups_per_second = counter_deltas_per_second.scale(0.1)
 
-                # rollups happen once every 10 seconds after a write
-                rps = cps.scale(0.1)
-                modified.query_pattern.estimated_read_per_second = rps
+                modified.query_pattern.estimated_read_per_second = rollups_per_second
 
                 # storage size fix
-                event_size = 128  # bytes
-                count_size = 64  # bytes
+                delta_event_size = 256  # bytes
+                rolled_up_count_size = 128  # bytes
                 GiB = 1024 * 1024 * 1024
-                retention = timedelta(days=1).total_seconds()
+
+                # Events can be discarded as soon as rollup is complete
+                # We default to a 1 day slice with 2 day retention
+                retention = timedelta(days=2).total_seconds()
+
                 cardinality = {
-                    "low": 1_000,
-                    "medium": 10_000,
-                    "high": 100_000,
+                    "low": 10_000,
+                    "medium": 100_000,
+                    "high": 1_000_000,
                 }[extra_model_arguments["counter.cardinality"]]
 
-                event_store = cps.scale(count_size * retention / GiB)
-                count_store = event_size * cardinality / GiB
-                store = event_store.offset(count_store)
-                modified.data_shape.estimated_state_size_gib = store
+                event_storage_size = counter_deltas_per_second.scale(
+                    delta_event_size * retention / GiB
+                )
+                rollup_storage_size = rolled_up_count_size * cardinality / GiB
+                total_store_size = event_storage_size.offset(rollup_storage_size)
+                modified.data_shape.estimated_state_size_gib = total_store_size
 
                 return modified
 

{service_capacity_modeling-0.3.78 → service_capacity_modeling-0.3.79}/service_capacity_modeling.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.78
+Version: 0.3.79
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com