service-capacity-modeling 0.3.77__py3-none-any.whl → 0.3.79__py3-none-any.whl

service_capacity_modeling/models/org/netflix/cassandra.py

@@ -377,7 +377,7 @@ def _estimate_cassandra_cluster_zonal(  # pylint: disable=too-many-positional-ar
     desires: CapacityDesires,
     zones_per_region: int = 3,
     copies_per_region: int = 3,
-    require_local_disks: bool = True,
+    require_local_disks: bool = False,
     require_attached_disks: bool = False,
     required_cluster_size: Optional[int] = None,
     max_rps_to_disk: int = 500,
@@ -644,7 +644,7 @@ class NflxCassandraArguments(BaseModel):
         " this will be deduced from durability and consistency desires",
     )
     require_local_disks: bool = Field(
-        default=True,
+        default=False,
         description="If local (ephemeral) drives are required",
     )
     require_attached_disks: bool = Field(
@@ -732,7 +732,7 @@ class NflxCassandraCapacityModel(CapacityModel):
             desires, extra_model_arguments.get("copies_per_region", None)
         )
         require_local_disks: bool = extra_model_arguments.get(
-            "require_local_disks", True
+            "require_local_disks", False
         )
         require_attached_disks: bool = extra_model_arguments.get(
             "require_attached_disks", False

service_capacity_modeling/models/org/netflix/counter.py

@@ -46,7 +46,8 @@ class NflxCounterArguments(NflxJavaAppArguments):
     )
     counter_cardinality: NflxCounterCardinality = Field(
         alias="counter.cardinality",
-        description="Low means < 1,000, medium (1,000—100,000), high means > 100,000.",
+        description="Low means < 10,000, medium (10,000—1,000,000), high means "
+        "> 1,000,000.",
     )
     counter_mode: NflxCounterMode = Field(
         alias="counter.mode",
@@ -92,36 +93,57 @@ class NflxCounterCapacityModel(CapacityModel):
     def compose_with(
         user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
     ) -> Tuple[Tuple[str, Callable[[CapacityDesires], CapacityDesires]], ...]:
-        stores = [("org.netflix.evcache", lambda x: x)]
-        if extra_model_arguments["counter.mode"] != NflxCounterMode.best_effort.value:
+        stores = []
 
+        if extra_model_arguments["counter.mode"] == NflxCounterMode.best_effort.value:
+            stores.append(("org.netflix.evcache", lambda x: x))
+        else:
+            # Shared evcache cluster is used for eventual and exact counters
             def _modify_cassandra_desires(
                 user_desires: CapacityDesires,
             ) -> CapacityDesires:
                 modified = user_desires.model_copy(deep=True)
+                counter_cardinality = extra_model_arguments["counter.cardinality"]
+
+                counter_deltas_per_second = (
+                    user_desires.query_pattern.estimated_write_per_second
+                )
 
-                # counts per second
-                cps = user_desires.query_pattern.estimated_write_per_second
+                # low cardinality : rollups happen once every 60 seconds
+                # medium cardinality : rollups happen once every 30 seconds
+                # high cardinality : rollups happen once every 10 seconds
+                # TODO: Account for read amplification from time slice configs
+                #       for better model accuracy
+                if counter_cardinality == NflxCounterCardinality.low.value:
+                    rollups_per_second = counter_deltas_per_second.scale(0.0167)
+                elif counter_cardinality == NflxCounterCardinality.medium.value:
+                    rollups_per_second = counter_deltas_per_second.scale(0.0333)
+                else:
+                    rollups_per_second = counter_deltas_per_second.scale(0.1)
 
-                # rollups happen once every 10 seconds after a write
-                rps = cps.scale(0.1)
-                modified.query_pattern.estimated_read_per_second = rps
+                modified.query_pattern.estimated_read_per_second = rollups_per_second
 
                 # storage size fix
-                event_size = 128  # bytes
-                count_size = 64  # bytes
+                delta_event_size = 256  # bytes
+                rolled_up_count_size = 128  # bytes
                 GiB = 1024 * 1024 * 1024
-                retention = timedelta(days=1).total_seconds()
+
+                # Events can be discarded as soon as rollup is complete
+                # We default to a 1 day slice with 2 day retention
+                retention = timedelta(days=2).total_seconds()
+
                 cardinality = {
-                    "low": 1_000,
-                    "medium": 10_000,
-                    "high": 100_000,
+                    "low": 10_000,
+                    "medium": 100_000,
+                    "high": 1_000_000,
                 }[extra_model_arguments["counter.cardinality"]]
 
-                event_store = cps.scale(count_size * retention / GiB)
-                count_store = event_size * cardinality / GiB
-                store = event_store.offset(count_store)
-                modified.data_shape.estimated_state_size_gib = store
+                event_storage_size = counter_deltas_per_second.scale(
+                    delta_event_size * retention / GiB
+                )
+                rollup_storage_size = rolled_up_count_size * cardinality / GiB
+                total_store_size = event_storage_size.offset(rollup_storage_size)
+                modified.data_shape.estimated_state_size_gib = total_store_size
 
                 return modified
 

{service_capacity_modeling-0.3.77.dist-info → service_capacity_modeling-0.3.79.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.77
+Version: 0.3.79
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com

{service_capacity_modeling-0.3.77.dist-info → service_capacity_modeling-0.3.79.dist-info}/RECORD

@@ -52,8 +52,8 @@ service_capacity_modeling/models/utils.py,sha256=WosEEg4o1_WSbTb5mL-M1v8JuWJgvS2
 service_capacity_modeling/models/org/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 service_capacity_modeling/models/org/netflix/__init__.py,sha256=W6rKkhSdPhjD-awm7mYakAhw7VKLNJYkqv-U3LfkMew,2444
 service_capacity_modeling/models/org/netflix/aurora.py,sha256=Js33ZjxCtt34HiDPsWRT9mjKCAsnnCo9du15QArVFMo,13073
-service_capacity_modeling/models/org/netflix/cassandra.py,sha256=biLt-A0FSW5ZXiM3T8Y8TaMG5xJOA-fzf1-OCqxVXx8,39000
-service_capacity_modeling/models/org/netflix/counter.py,sha256=T-lBgxUMxZUojDyMJBR3HQI1u6fJujuPiQ6rGTZaMl4,9278
+service_capacity_modeling/models/org/netflix/cassandra.py,sha256=Fp37bHtWRJctJYYJhW78YbLeFXoX26QxREP_BMEItNE,39003
+service_capacity_modeling/models/org/netflix/counter.py,sha256=XopKlNMdvO5EbxAggn-KW_q7L3aKtXLXbry4ocl6i5Q,10494
 service_capacity_modeling/models/org/netflix/crdb.py,sha256=iW7tyG8jpXhHIdXrw3DPYSHRAknPN42MlCRLJO4o9C8,20826
 service_capacity_modeling/models/org/netflix/ddb.py,sha256=9qRiuTqWev9zbYFFzewyowU7M41uALsuLklYx20yAXw,26502
 service_capacity_modeling/models/org/netflix/elasticsearch.py,sha256=zPrC6b2LNrAh3IWE3HCMUEYASacjYbHChbO4WZSMma4,25234
@@ -75,9 +75,9 @@ service_capacity_modeling/tools/auto_shape.py,sha256=Jx9H2ay9-H_kUDjtB141owQNxGF
 service_capacity_modeling/tools/fetch_pricing.py,sha256=Qp-XMymkY1dvtyS51RufmEpfgOHv-IQ-XyzS8wp2-qM,4021
 service_capacity_modeling/tools/generate_missing.py,sha256=F7YqvMJAV4nZc20GNrlIsnQSF8_77sLgwYZqc5k4LDg,3099
 service_capacity_modeling/tools/instance_families.py,sha256=e5RuYkCLUITvsAazDH12B6KjX_PaBsv6Ne3mj0HK_sQ,9223
-service_capacity_modeling-0.3.77.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
-service_capacity_modeling-0.3.77.dist-info/METADATA,sha256=hD72H-7tgG6_2MkUOy0mn9-RRXfuXomq9OyIPfNJkiw,10361
-service_capacity_modeling-0.3.77.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-service_capacity_modeling-0.3.77.dist-info/entry_points.txt,sha256=ZsjzpG5SomWpT1zCE19n1uSXKH2gTI_yc33sdl0vmJg,146
-service_capacity_modeling-0.3.77.dist-info/top_level.txt,sha256=H8XjTCLgR3enHq5t3bIbxt9SeUkUT8HT_SDv2dgIT_A,26
-service_capacity_modeling-0.3.77.dist-info/RECORD,,
+service_capacity_modeling-0.3.79.dist-info/licenses/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
+service_capacity_modeling-0.3.79.dist-info/METADATA,sha256=nM1qflhu7AXKNcD-AqIXyg_EZQ_ITg77Rj8b8ur_D8A,10361
+service_capacity_modeling-0.3.79.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+service_capacity_modeling-0.3.79.dist-info/entry_points.txt,sha256=ZsjzpG5SomWpT1zCE19n1uSXKH2gTI_yc33sdl0vmJg,146
+service_capacity_modeling-0.3.79.dist-info/top_level.txt,sha256=H8XjTCLgR3enHq5t3bIbxt9SeUkUT8HT_SDv2dgIT_A,26
+service_capacity_modeling-0.3.79.dist-info/RECORD,,