service-capacity-modeling 0.3.78__py3-none-any.whl → 0.3.80__py3-none-any.whl

service_capacity_modeling/capacity_planner.py

@@ -85,7 +85,7 @@ def model_desires(
     data_shape = desires.data_shape.model_copy(deep=True)
 
     query_pattern_simulation = {}
-    for field in sorted(query_pattern.model_fields):
+    for field in sorted(QueryPattern.model_fields):
         d = getattr(query_pattern, field)
         if isinstance(d, Interval):
             query_pattern_simulation[field] = simulate_interval(d, field)(num_sims)
@@ -93,7 +93,7 @@ def model_desires(
             query_pattern_simulation[field] = [d] * num_sims
 
     data_shape_simulation = {}
-    for field in sorted(data_shape.model_fields):
+    for field in sorted(DataShape.model_fields):
         d = getattr(data_shape, field)
         if isinstance(d, Interval):
             data_shape_simulation[field] = simulate_interval(d, field)(num_sims)
@@ -104,14 +104,11 @@ def model_desires(
         query_pattern = QueryPattern(
             **{
                 f: query_pattern_simulation[f][sim]
-                for f in sorted(query_pattern.model_fields)
+                for f in sorted(QueryPattern.model_fields)
             }
         )
         data_shape = DataShape(
-            **{
-                f: data_shape_simulation[f][sim]
-                for f in sorted(data_shape.model_fields)
-            }
+            **{f: data_shape_simulation[f][sim] for f in sorted(DataShape.model_fields)}
         )
 
         d = desires.model_copy(update={"query_pattern": None, "data_shape": None})
@@ -129,7 +126,7 @@ def model_desires_percentiles(
 
     query_pattern_simulation = {}
     query_pattern_means = {}
-    for field in sorted(query_pattern.model_fields):
+    for field in sorted(QueryPattern.model_fields):
         d = getattr(query_pattern, field)
         if isinstance(d, Interval):
             query_pattern_simulation[field] = interval_percentile(d, percentiles)
@@ -143,7 +140,7 @@ def model_desires_percentiles(
 
     data_shape_simulation = {}
     data_shape_means = {}
-    for field in sorted(data_shape.model_fields):
+    for field in sorted(DataShape.model_fields):
         d = getattr(data_shape, field)
         if isinstance(d, Interval):
             data_shape_simulation[field] = interval_percentile(d, percentiles)
@@ -161,13 +158,13 @@ def model_desires_percentiles(
             query_pattern = QueryPattern(
                 **{
                     f: query_pattern_simulation[f][i]
-                    for f in sorted(query_pattern.model_fields)
+                    for f in sorted(QueryPattern.model_fields)
                 }
             )
         except Exception as exp:
             raise exp
         data_shape = DataShape(
-            **{f: data_shape_simulation[f][i] for f in sorted(data_shape.model_fields)}
+            **{f: data_shape_simulation[f][i] for f in sorted(DataShape.model_fields)}
         )
         d = desires.model_copy(deep=True)
         d.query_pattern = query_pattern
@@ -175,10 +172,10 @@ def model_desires_percentiles(
         results.append(d)
 
     mean_qp = QueryPattern(
-        **{f: query_pattern_means[f] for f in sorted(query_pattern.model_fields)}
+        **{f: query_pattern_means[f] for f in sorted(QueryPattern.model_fields)}
     )
     mean_ds = DataShape(
-        **{f: data_shape_means[f] for f in sorted(data_shape.model_fields)}
+        **{f: data_shape_means[f] for f in sorted(DataShape.model_fields)}
     )
     d = desires.model_copy(deep=True)
     d.query_pattern = mean_qp
@@ -298,7 +295,7 @@ def _add_requirement(
 
     requirements = accum[requirement.requirement_type]
 
-    for field in sorted(requirement.model_fields):
+    for field in sorted(CapacityRequirement.model_fields):
         d = getattr(requirement, field)
         if isinstance(d, Interval):
             if field not in requirements:

service_capacity_modeling/models/org/netflix/counter.py

@@ -46,7 +46,8 @@ class NflxCounterArguments(NflxJavaAppArguments):
     )
     counter_cardinality: NflxCounterCardinality = Field(
         alias="counter.cardinality",
-        description="Low means < 1,000, medium (1,000—100,000), high means > 100,000.",
+        description="Low means < 10,000, medium (10,000—1,000,000), high means "
+        "> 1,000,000.",
     )
     counter_mode: NflxCounterMode = Field(
         alias="counter.mode",
@@ -92,36 +93,57 @@ class NflxCounterCapacityModel(CapacityModel):
     def compose_with(
         user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
     ) -> Tuple[Tuple[str, Callable[[CapacityDesires], CapacityDesires]], ...]:
-        stores = [("org.netflix.evcache", lambda x: x)]
-        if extra_model_arguments["counter.mode"] != NflxCounterMode.best_effort.value:
+        stores = []
 
+        if extra_model_arguments["counter.mode"] == NflxCounterMode.best_effort.value:
+            stores.append(("org.netflix.evcache", lambda x: x))
+        else:
+            # Shared evcache cluster is used for eventual and exact counters
             def _modify_cassandra_desires(
                 user_desires: CapacityDesires,
             ) -> CapacityDesires:
                 modified = user_desires.model_copy(deep=True)
+                counter_cardinality = extra_model_arguments["counter.cardinality"]
+
+                counter_deltas_per_second = (
+                    user_desires.query_pattern.estimated_write_per_second
+                )
 
-                # counts per second
-                cps = user_desires.query_pattern.estimated_write_per_second
+                # low cardinality : rollups happen once every 60 seconds
+                # medium cardinality : rollups happen once every 30 seconds
+                # high cardinality : rollups happen once every 10 seconds
+                # TODO: Account for read amplification from time slice configs
+                #       for better model accuracy
+                if counter_cardinality == NflxCounterCardinality.low.value:
+                    rollups_per_second = counter_deltas_per_second.scale(0.0167)
+                elif counter_cardinality == NflxCounterCardinality.medium.value:
+                    rollups_per_second = counter_deltas_per_second.scale(0.0333)
+                else:
+                    rollups_per_second = counter_deltas_per_second.scale(0.1)
 
-                # rollups happen once every 10 seconds after a write
-                rps = cps.scale(0.1)
-                modified.query_pattern.estimated_read_per_second = rps
+                modified.query_pattern.estimated_read_per_second = rollups_per_second
 
                 # storage size fix
-                event_size = 128  # bytes
-                count_size = 64  # bytes
+                delta_event_size = 256  # bytes
+                rolled_up_count_size = 128  # bytes
                 GiB = 1024 * 1024 * 1024
-                retention = timedelta(days=1).total_seconds()
+
+                # Events can be discarded as soon as rollup is complete
+                # We default to a 1 day slice with 2 day retention
+                retention = timedelta(days=2).total_seconds()
+
                 cardinality = {
-                    "low": 1_000,
-                    "medium": 10_000,
-                    "high": 100_000,
+                    "low": 10_000,
+                    "medium": 100_000,
+                    "high": 1_000_000,
                 }[extra_model_arguments["counter.cardinality"]]
 
-                event_store = cps.scale(count_size * retention / GiB)
-                count_store = event_size * cardinality / GiB
-                store = event_store.offset(count_store)
-                modified.data_shape.estimated_state_size_gib = store
+                event_storage_size = counter_deltas_per_second.scale(
+                    delta_event_size * retention / GiB
+                )
+                rollup_storage_size = rolled_up_count_size * cardinality / GiB
+                total_store_size = event_storage_size.offset(rollup_storage_size)
+                modified.data_shape.estimated_state_size_gib = total_store_size
 
                 return modified
 

service_capacity_modeling/stats.py

@@ -45,7 +45,11 @@ def _gamma_fn_from_params(
 
     def f(k: float) -> float:
         zero = high / low
-        return float(gammaf(k, high_p * k / mid) / gammaf(k, low_p * k / mid) - zero)
+        high_x = high_p * k / mid
+        low_x = low_p * k / mid
+        result = gammaf(k, high_x) / gammaf(k, low_x) - zero
+        # Use .item() to convert NumPy scalar to Python float
+        return result.item() if hasattr(result, "item") else float(result)
 
     return f
 

{service_capacity_modeling-0.3.78.dist-info → service_capacity_modeling-0.3.80.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.78
+Version: 0.3.80
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com

{service_capacity_modeling-0.3.78.dist-info → service_capacity_modeling-0.3.80.dist-info}/RECORD

@@ -1,7 +1,7 @@
 service_capacity_modeling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-service_capacity_modeling/capacity_planner.py,sha256=nmjAWpUtL5EKbEtQzSGWrM8gPSAVc7EX5M4ApiLbQnA,32828
+service_capacity_modeling/capacity_planner.py,sha256=JRagEFlg3u_zB1N5GzGsKAN55JZLad6p4IF_PmL8kcg,32780
 service_capacity_modeling/interface.py,sha256=p-pOUQUdA6VA-moTjFSBLQAZCRvcjFaAfb2jOmxMWsY,39074
-service_capacity_modeling/stats.py,sha256=ievz2f1H-a6u8wHwXBQ47e2EhqH9BDrdL6VZAXQDK2w,5964
+service_capacity_modeling/stats.py,sha256=LCNUcQPfwF5hhIZwsfAsDe4ZbnuhDnl3vQHKfpK61Xc,6142
 service_capacity_modeling/hardware/__init__.py,sha256=P5ostvoSOMUqPODtepeFYb4qfTVH0E73mMFraP49rYU,9196
 service_capacity_modeling/hardware/profiles/__init__.py,sha256=7-y3JbCBkgzaAjFla2RIymREcImdZ51HTl3yn3vzoGw,1602
 service_capacity_modeling/hardware/profiles/profiles.txt,sha256=tOfSR3B0E0uAOaXd5SLI3ioq83UYZ3yhK7UHhsK4awQ,49
@@ -53,7 +53,7 @@ service_capacity_modeling/models/org/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeu
 service_capacity_modeling/models/org/netflix/__init__.py,sha256=W6rKkhSdPhjD-awm7mYakAhw7VKLNJYkqv-U3LfkMew,2444
 service_capacity_modeling/models/org/netflix/aurora.py,sha256=Js33ZjxCtt34HiDPsWRT9mjKCAsnnCo9du15QArVFMo,13073
 service_capacity_modeling/models/org/netflix/cassandra.py,sha256=Fp37bHtWRJctJYYJhW78YbLeFXoX26QxREP_BMEItNE,39003
-service_capacity_modeling/models/org/netflix/counter.py,sha256=T-lBgxUMxZUojDyMJBR3HQI1u6fJujuPiQ6rGTZaMl4,9278
+service_capacity_modeling/models/org/netflix/counter.py,sha256=XopKlNMdvO5EbxAggn-KW_q7L3aKtXLXbry4ocl6i5Q,10494
 service_capacity_modeling/models/org/netflix/crdb.py,sha256=iW7tyG8jpXhHIdXrw3DPYSHRAknPN42MlCRLJO4o9C8,20826
 service_capacity_modeling/models/org/netflix/ddb.py,sha256=9qRiuTqWev9zbYFFzewyowU7M41uALsuLklYx20yAXw,26502
 service_capacity_modeling/models/org/netflix/elasticsearch.py,sha256=zPrC6b2LNrAh3IWE3HCMUEYASacjYbHChbO4WZSMma4,25234
@@ -75,9 +75,9 @@ service_capacity_modeling/tools/auto_shape.py,sha256=Jx9H2ay9-H_kUDjtB141owQNxGF
 service_capacity_modeling/tools/fetch_pricing.py,sha256=Qp-XMymkY1dvtyS51RufmEpfgOHv-IQ-XyzS8wp2-qM,4021
 service_capacity_modeling/tools/generate_missing.py,sha256=F7YqvMJAV4nZc20GNrlIsnQSF8_77sLgwYZqc5k4LDg,3099
 service_capacity_modeling/tools/instance_families.py,sha256=e5RuYkCLUITvsAazDH12B6KjX_PaBsv6Ne3mj0HK_sQ,9223
-service_capacity_modeling-0.3.78.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-service_capacity_modeling-0.3.78.dist-info/METADATA,sha256=p1ASGJf0gody4zS66LKxxJQH8wyXFcxWTSfpfVfDhME,10361
-service_capacity_modeling-0.3.78.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-service_capacity_modeling-0.3.78.dist-info/entry_points.txt,sha256=ZsjzpG5SomWpT1zCE19n1uSXKH2gTI_yc33sdl0vmJg,146
-service_capacity_modeling-0.3.78.dist-info/top_level.txt,sha256=H8XjTCLgR3enHq5t3bIbxt9SeUkUT8HT_SDv2dgIT_A,26
-service_capacity_modeling-0.3.78.dist-info/RECORD,,
+service_capacity_modeling-0.3.80.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+service_capacity_modeling-0.3.80.dist-info/METADATA,sha256=XBKHEm8Iy72isKJusZ0FifYh8KrWacjh45kd-oXp7m0,10361
+service_capacity_modeling-0.3.80.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+service_capacity_modeling-0.3.80.dist-info/entry_points.txt,sha256=ZsjzpG5SomWpT1zCE19n1uSXKH2gTI_yc33sdl0vmJg,146
+service_capacity_modeling-0.3.80.dist-info/top_level.txt,sha256=H8XjTCLgR3enHq5t3bIbxt9SeUkUT8HT_SDv2dgIT_A,26
+service_capacity_modeling-0.3.80.dist-info/RECORD,,