service-capacity-modeling 0.3.73__py3-none-any.whl → 0.3.79__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of service-capacity-modeling might be problematic. Click here for more details.

Files changed (40) hide show
  1. service_capacity_modeling/capacity_planner.py +46 -40
  2. service_capacity_modeling/hardware/__init__.py +11 -7
  3. service_capacity_modeling/hardware/profiles/shapes/aws/auto_i3en.json +172 -0
  4. service_capacity_modeling/hardware/profiles/shapes/aws/auto_i4i.json +220 -0
  5. service_capacity_modeling/hardware/profiles/shapes/aws/manual_instances.json +0 -184
  6. service_capacity_modeling/interface.py +48 -22
  7. service_capacity_modeling/models/__init__.py +21 -2
  8. service_capacity_modeling/models/common.py +268 -190
  9. service_capacity_modeling/models/headroom_strategy.py +2 -1
  10. service_capacity_modeling/models/org/netflix/__init__.py +4 -1
  11. service_capacity_modeling/models/org/netflix/aurora.py +12 -7
  12. service_capacity_modeling/models/org/netflix/cassandra.py +39 -24
  13. service_capacity_modeling/models/org/netflix/counter.py +44 -20
  14. service_capacity_modeling/models/org/netflix/crdb.py +7 -4
  15. service_capacity_modeling/models/org/netflix/ddb.py +9 -5
  16. service_capacity_modeling/models/org/netflix/elasticsearch.py +8 -6
  17. service_capacity_modeling/models/org/netflix/entity.py +5 -3
  18. service_capacity_modeling/models/org/netflix/evcache.py +21 -25
  19. service_capacity_modeling/models/org/netflix/graphkv.py +5 -3
  20. service_capacity_modeling/models/org/netflix/iso_date_math.py +12 -9
  21. service_capacity_modeling/models/org/netflix/kafka.py +13 -7
  22. service_capacity_modeling/models/org/netflix/key_value.py +4 -2
  23. service_capacity_modeling/models/org/netflix/postgres.py +4 -2
  24. service_capacity_modeling/models/org/netflix/rds.py +10 -5
  25. service_capacity_modeling/models/org/netflix/stateless_java.py +4 -2
  26. service_capacity_modeling/models/org/netflix/time_series.py +4 -2
  27. service_capacity_modeling/models/org/netflix/time_series_config.py +3 -3
  28. service_capacity_modeling/models/org/netflix/wal.py +4 -2
  29. service_capacity_modeling/models/org/netflix/zookeeper.py +5 -3
  30. service_capacity_modeling/stats.py +14 -11
  31. service_capacity_modeling/tools/auto_shape.py +10 -6
  32. service_capacity_modeling/tools/fetch_pricing.py +13 -6
  33. service_capacity_modeling/tools/generate_missing.py +4 -3
  34. service_capacity_modeling/tools/instance_families.py +18 -7
  35. {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/METADATA +9 -5
  36. {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/RECORD +40 -38
  37. {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/WHEEL +0 -0
  38. {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/entry_points.txt +0 -0
  39. {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/licenses/LICENSE +0 -0
  40. {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/top_level.txt +0 -0
@@ -1,6 +1,7 @@
1
1
  import logging
2
2
  import math
3
3
  from typing import Any
4
+ from typing import Callable
4
5
  from typing import Dict
5
6
  from typing import Optional
6
7
  from typing import Tuple
@@ -86,7 +87,9 @@ def _estimate_aurora_requirement(
86
87
 
87
88
  # MySQL default block size is 16KiB, PostGreSQL is 8KiB Number of reads for B-Tree
88
89
  # are given by log of total pages to the base of B-Tree fan out factor
89
- def _rds_required_disk_ios(disk_size_gib: int, db_type: str, btree_fan_out: int = 100):
90
+ def _rds_required_disk_ios(
91
+ disk_size_gib: int, db_type: str, btree_fan_out: int = 100
92
+ ) -> float:
90
93
  disk_size_kb = disk_size_gib * 1024 * 1024
91
94
  if db_type == "postgres":
92
95
  default_block_size = 8 # KiB
@@ -100,11 +103,11 @@ def _rds_required_disk_ios(disk_size_gib: int, db_type: str, btree_fan_out: int
100
103
  # This is a start, we should iterate based on the actual work load
101
104
  def _estimate_io_cost(
102
105
  db_type: str,
103
- desires,
106
+ desires: CapacityDesires,
104
107
  read_io_price: float,
105
108
  write_io_price: float,
106
109
  cache_hit_rate: float = 0.8,
107
- ):
110
+ ) -> float:
108
111
  if db_type == "postgres":
109
112
  read_byte_per_io = 8192
110
113
  else:
@@ -134,8 +137,8 @@ def _compute_aurora_region( # pylint: disable=too-many-positional-arguments
134
137
  needed_disk_gib: int,
135
138
  needed_memory_gib: int,
136
139
  needed_network_mbps: float,
137
- required_disk_ios,
138
- required_disk_space,
140
+ required_disk_ios: Callable[[int], float],
141
+ required_disk_space: Callable[[int], float],
139
142
  db_type: str,
140
143
  desires: CapacityDesires,
141
144
  ) -> Optional[RegionClusterCapacity]:
@@ -295,7 +298,7 @@ class NflxAuroraCapacityModel(CapacityModel):
295
298
  )
296
299
 
297
300
  @staticmethod
298
- def description():
301
+ def description() -> str:
299
302
  return "Netflix Aurora Cluster Model"
300
303
 
301
304
  @staticmethod
@@ -307,7 +310,9 @@ class NflxAuroraCapacityModel(CapacityModel):
307
310
  return Platform.aurora_mysql, Platform.aurora_mysql
308
311
 
309
312
  @staticmethod
310
- def default_desires(user_desires, extra_model_arguments):
313
+ def default_desires(
314
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
315
+ ) -> CapacityDesires:
311
316
  return CapacityDesires(
312
317
  query_pattern=QueryPattern(
313
318
  access_pattern=AccessPattern.latency,
@@ -35,7 +35,7 @@ from service_capacity_modeling.interface import ServiceCapacity
35
35
  from service_capacity_modeling.models import CapacityModel
36
36
  from service_capacity_modeling.models.common import buffer_for_components
37
37
  from service_capacity_modeling.models.common import compute_stateful_zone
38
- from service_capacity_modeling.models.common import derived_buffer_for_component
38
+ from service_capacity_modeling.models.common import DerivedBuffers
39
39
  from service_capacity_modeling.models.common import get_effective_disk_per_node_gib
40
40
  from service_capacity_modeling.models.common import network_services
41
41
  from service_capacity_modeling.models.common import normalize_cores
@@ -79,7 +79,7 @@ def _write_buffer_gib_zone(
79
79
  return float(write_buffer_gib) / zones_per_region
80
80
 
81
81
 
82
- def _get_cores_from_desires(desires, instance):
82
+ def _get_cores_from_desires(desires: CapacityDesires, instance: Instance) -> int:
83
83
  cpu_buffer = buffer_for_components(
84
84
  buffers=desires.buffers, components=[BACKGROUND_BUFFER]
85
85
  )
@@ -97,7 +97,7 @@ def _get_cores_from_desires(desires, instance):
97
97
  return needed_cores
98
98
 
99
99
 
100
- def _get_disk_from_desires(desires, copies_per_region):
100
+ def _get_disk_from_desires(desires: CapacityDesires, copies_per_region: int) -> int:
101
101
  disk_buffer = buffer_for_components(
102
102
  buffers=desires.buffers, components=[BufferComponent.disk]
103
103
  )
@@ -116,7 +116,7 @@ def _get_min_count(
116
116
  needed_disk_gib: float,
117
117
  disk_per_node_gib: float,
118
118
  cluster_size_lambda: Callable[[int], int],
119
- ):
119
+ ) -> int:
120
120
  """
121
121
  Compute the minimum number of nodes required for a zone.
122
122
 
@@ -158,7 +158,10 @@ def _get_min_count(
158
158
 
159
159
 
160
160
  def _zonal_requirement_for_new_cluster(
161
- desires, instance, copies_per_region, zones_per_region
161
+ desires: CapacityDesires,
162
+ instance: Instance,
163
+ copies_per_region: int,
164
+ zones_per_region: int,
162
165
  ) -> CapacityRequirement:
163
166
  needed_cores = _get_cores_from_desires(desires, instance)
164
167
  needed_disk = _get_disk_from_desires(desires, copies_per_region)
@@ -181,7 +184,9 @@ def _zonal_requirement_for_new_cluster(
181
184
  )
182
185
 
183
186
 
184
- def _estimate_cassandra_requirement( # pylint: disable=too-many-positional-arguments
187
+ # pylint: disable=too-many-locals
188
+ # pylint: disable=too-many-positional-arguments
189
+ def _estimate_cassandra_requirement(
185
190
  instance: Instance,
186
191
  desires: CapacityDesires,
187
192
  working_set: float,
@@ -205,19 +210,22 @@ def _estimate_cassandra_requirement( # pylint: disable=too-many-positional-argu
205
210
  # If the cluster is already provisioned
206
211
  if current_capacity and desires.current_clusters is not None:
207
212
  capacity_requirement = zonal_requirements_from_current(
208
- desires.current_clusters, desires.buffers, instance, reference_shape
213
+ desires.current_clusters,
214
+ desires.buffers,
215
+ instance,
216
+ reference_shape,
209
217
  )
210
- disk_scale, _ = derived_buffer_for_component(
211
- desires.buffers.derived, ["storage", "disk"]
218
+ disk_derived_buffer = DerivedBuffers.for_components(
219
+ desires.buffers.derived, [BufferComponent.disk]
212
220
  )
213
221
  disk_used_gib = (
214
222
  current_capacity.disk_utilization_gib.mid
215
223
  * current_capacity.cluster_instance_count.mid
216
- * (disk_scale or 1)
217
- )
218
- _, memory_preserve = derived_buffer_for_component(
219
- desires.buffers.derived, ["storage", "memory"]
224
+ * disk_derived_buffer.scale
220
225
  )
226
+ memory_preserve = DerivedBuffers.for_components(
227
+ desires.buffers.derived, [BufferComponent.memory]
228
+ ).preserve
221
229
  else:
222
230
  # If the cluster is not yet provisioned
223
231
  capacity_requirement = _zonal_requirement_for_new_cluster(
@@ -320,14 +328,14 @@ def _estimate_cassandra_requirement( # pylint: disable=too-many-positional-argu
320
328
  )
321
329
 
322
330
 
323
- def _get_current_cluster_size(desires) -> int:
331
+ def _get_current_cluster_size(desires: CapacityDesires) -> int:
324
332
  current_capacity = _get_current_capacity(desires)
325
333
  if current_capacity is None:
326
334
  return 0
327
335
  return math.ceil(current_capacity.cluster_instance_count.mid)
328
336
 
329
337
 
330
- def _get_current_capacity(desires) -> Optional[CurrentClusterCapacity]:
338
+ def _get_current_capacity(desires: CapacityDesires) -> Optional[CurrentClusterCapacity]:
331
339
  current_capacity = (
332
340
  None
333
341
  if desires.current_clusters is None
@@ -340,7 +348,7 @@ def _get_current_capacity(desires) -> Optional[CurrentClusterCapacity]:
340
348
  return current_capacity
341
349
 
342
350
 
343
- def _upsert_params(cluster, params):
351
+ def _upsert_params(cluster: Any, params: Dict[str, Any]) -> None:
344
352
  if cluster.cluster_params:
345
353
  cluster.cluster_params.update(params)
346
354
  else:
@@ -369,7 +377,7 @@ def _estimate_cassandra_cluster_zonal( # pylint: disable=too-many-positional-ar
369
377
  desires: CapacityDesires,
370
378
  zones_per_region: int = 3,
371
379
  copies_per_region: int = 3,
372
- require_local_disks: bool = True,
380
+ require_local_disks: bool = False,
373
381
  require_attached_disks: bool = False,
374
382
  required_cluster_size: Optional[int] = None,
375
383
  max_rps_to_disk: int = 500,
@@ -570,7 +578,7 @@ def _estimate_cassandra_cluster_zonal( # pylint: disable=too-many-positional-ar
570
578
 
571
579
 
572
580
  # C* LCS has 160 MiB sstables by default and 10 sstables per level
573
- def _cass_io_per_read(node_size_gib, sstable_size_mb=160):
581
+ def _cass_io_per_read(node_size_gib: float, sstable_size_mb: int = 160) -> int:
574
582
  gb = node_size_gib * 1024
575
583
  sstables = max(1, gb // sstable_size_mb)
576
584
  # 10 sstables per level, plus 1 for L0 (avg)
@@ -580,7 +588,7 @@ def _cass_io_per_read(node_size_gib, sstable_size_mb=160):
580
588
  return 2 * levels
581
589
 
582
590
 
583
- def _get_base_memory(desires: CapacityDesires):
591
+ def _get_base_memory(desires: CapacityDesires) -> float:
584
592
  return (
585
593
  desires.data_shape.reserved_instance_app_mem_gib
586
594
  + desires.data_shape.reserved_instance_system_mem_gib
@@ -636,7 +644,7 @@ class NflxCassandraArguments(BaseModel):
636
644
  " this will be deduced from durability and consistency desires",
637
645
  )
638
646
  require_local_disks: bool = Field(
639
- default=True,
647
+ default=False,
640
648
  description="If local (ephemeral) drives are required",
641
649
  )
642
650
  require_attached_disks: bool = Field(
@@ -674,8 +682,13 @@ class NflxCassandraArguments(BaseModel):
674
682
 
675
683
 
676
684
  class NflxCassandraCapacityModel(CapacityModel):
685
+ def __init__(self) -> None:
686
+ pass
687
+
677
688
  @staticmethod
678
- def get_required_cluster_size(tier, extra_model_arguments):
689
+ def get_required_cluster_size(
690
+ tier: int, extra_model_arguments: Dict[str, Any]
691
+ ) -> Optional[int]:
679
692
  required_cluster_size: Optional[int] = (
680
693
  math.ceil(extra_model_arguments["required_cluster_size"])
681
694
  if "required_cluster_size" in extra_model_arguments
@@ -719,7 +732,7 @@ class NflxCassandraCapacityModel(CapacityModel):
719
732
  desires, extra_model_arguments.get("copies_per_region", None)
720
733
  )
721
734
  require_local_disks: bool = extra_model_arguments.get(
722
- "require_local_disks", True
735
+ "require_local_disks", False
723
736
  )
724
737
  require_attached_disks: bool = extra_model_arguments.get(
725
738
  "require_attached_disks", False
@@ -770,7 +783,7 @@ class NflxCassandraCapacityModel(CapacityModel):
770
783
  )
771
784
 
772
785
  @staticmethod
773
- def description():
786
+ def description() -> str:
774
787
  return "Netflix Streaming Cassandra Model"
775
788
 
776
789
  @staticmethod
@@ -798,7 +811,9 @@ class NflxCassandraCapacityModel(CapacityModel):
798
811
  )
799
812
 
800
813
  @staticmethod
801
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
814
+ def default_desires(
815
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
816
+ ) -> CapacityDesires:
802
817
  acceptable_consistency = {
803
818
  None,
804
819
  AccessConsistency.best_effort,
@@ -46,7 +46,8 @@ class NflxCounterArguments(NflxJavaAppArguments):
46
46
  )
47
47
  counter_cardinality: NflxCounterCardinality = Field(
48
48
  alias="counter.cardinality",
49
- description="Low means < 1,000, medium (1,000—100,000), high means > 100,000.",
49
+ description="Low means < 10,000, medium (10,000—1,000,000), high means "
50
+ "> 1,000,000.",
50
51
  )
51
52
  counter_mode: NflxCounterMode = Field(
52
53
  alias="counter.mode",
@@ -81,7 +82,7 @@ class NflxCounterCapacityModel(CapacityModel):
81
82
  return counter_app
82
83
 
83
84
  @staticmethod
84
- def description():
85
+ def description() -> str:
85
86
  return "Netflix Streaming Counter Model"
86
87
 
87
88
  @staticmethod
@@ -92,36 +93,57 @@ class NflxCounterCapacityModel(CapacityModel):
92
93
  def compose_with(
93
94
  user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
94
95
  ) -> Tuple[Tuple[str, Callable[[CapacityDesires], CapacityDesires]], ...]:
95
- stores = [("org.netflix.evcache", lambda x: x)]
96
- if extra_model_arguments["counter.mode"] != NflxCounterMode.best_effort.value:
96
+ stores = []
97
97
 
98
+ if extra_model_arguments["counter.mode"] == NflxCounterMode.best_effort.value:
99
+ stores.append(("org.netflix.evcache", lambda x: x))
100
+ else:
101
+ # Shared evcache cluster is used for eventual and exact counters
98
102
  def _modify_cassandra_desires(
99
103
  user_desires: CapacityDesires,
100
104
  ) -> CapacityDesires:
101
105
  modified = user_desires.model_copy(deep=True)
106
+ counter_cardinality = extra_model_arguments["counter.cardinality"]
107
+
108
+ counter_deltas_per_second = (
109
+ user_desires.query_pattern.estimated_write_per_second
110
+ )
102
111
 
103
- # counts per second
104
- cps = user_desires.query_pattern.estimated_write_per_second
112
+ # low cardinality : rollups happen once every 60 seconds
113
+ # medium cardinality : rollups happen once every 30 seconds
114
+ # high cardinality : rollups happen once every 10 seconds
115
+ # TODO: Account for read amplification from time slice configs
116
+ # for better model accuracy
117
+ if counter_cardinality == NflxCounterCardinality.low.value:
118
+ rollups_per_second = counter_deltas_per_second.scale(0.0167)
119
+ elif counter_cardinality == NflxCounterCardinality.medium.value:
120
+ rollups_per_second = counter_deltas_per_second.scale(0.0333)
121
+ else:
122
+ rollups_per_second = counter_deltas_per_second.scale(0.1)
105
123
 
106
- # rollups happen once every 10 seconds after a write
107
- rps = cps.scale(0.1)
108
- modified.query_pattern.estimated_read_per_second = rps
124
+ modified.query_pattern.estimated_read_per_second = rollups_per_second
109
125
 
110
126
  # storage size fix
111
- event_size = 128 # bytes
112
- count_size = 64 # bytes
127
+ delta_event_size = 256 # bytes
128
+ rolled_up_count_size = 128 # bytes
113
129
  GiB = 1024 * 1024 * 1024
114
- retention = timedelta(days=1).total_seconds()
130
+
131
+ # Events can be discarded as soon as rollup is complete
132
+ # We default to a 1 day slice with 2 day retention
133
+ retention = timedelta(days=2).total_seconds()
134
+
115
135
  cardinality = {
116
- "low": 1_000,
117
- "medium": 10_000,
118
- "high": 100_000,
136
+ "low": 10_000,
137
+ "medium": 100_000,
138
+ "high": 1_000_000,
119
139
  }[extra_model_arguments["counter.cardinality"]]
120
140
 
121
- event_store = cps.scale(count_size * retention / GiB)
122
- count_store = event_size * cardinality / GiB
123
- store = event_store.offset(count_store)
124
- modified.data_shape.estimated_state_size_gib = store
141
+ event_storage_size = counter_deltas_per_second.scale(
142
+ delta_event_size * retention / GiB
143
+ )
144
+ rollup_storage_size = rolled_up_count_size * cardinality / GiB
145
+ total_store_size = event_storage_size.offset(rollup_storage_size)
146
+ modified.data_shape.estimated_state_size_gib = total_store_size
125
147
 
126
148
  return modified
127
149
 
@@ -129,7 +151,9 @@ class NflxCounterCapacityModel(CapacityModel):
129
151
  return tuple(stores)
130
152
 
131
153
  @staticmethod
132
- def default_desires(user_desires, extra_model_arguments):
154
+ def default_desires(
155
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
156
+ ) -> CapacityDesires:
133
157
  if user_desires.query_pattern.access_pattern == AccessPattern.latency:
134
158
  return CapacityDesires(
135
159
  query_pattern=QueryPattern(
@@ -29,6 +29,7 @@ from service_capacity_modeling.interface import Interval
29
29
  from service_capacity_modeling.interface import QueryPattern
30
30
  from service_capacity_modeling.interface import RegionContext
31
31
  from service_capacity_modeling.interface import Requirements
32
+ from service_capacity_modeling.interface import ZoneClusterCapacity
32
33
  from service_capacity_modeling.models import CapacityModel
33
34
  from service_capacity_modeling.models.common import buffer_for_components
34
35
  from service_capacity_modeling.models.common import compute_stateful_zone
@@ -44,7 +45,7 @@ logger = logging.getLogger(__name__)
44
45
 
45
46
  # Pebble does Leveled compaction with tieres of size??
46
47
  # (FIXME) What does pebble actually do
47
- def _crdb_io_per_read(node_size_gib, sstable_size_mb=1000):
48
+ def _crdb_io_per_read(node_size_gib: float, sstable_size_mb: int = 1000) -> int:
48
49
  gb = node_size_gib * 1024
49
50
  sstables = max(1, gb // sstable_size_mb)
50
51
  # 10 sstables per level, plus 1 for L0 (avg)
@@ -128,7 +129,7 @@ def _estimate_cockroachdb_requirement( # noqa=E501 pylint: disable=too-many-pos
128
129
  )
129
130
 
130
131
 
131
- def _upsert_params(cluster, params):
132
+ def _upsert_params(cluster: ZoneClusterCapacity, params: Dict[str, Any]) -> None:
132
133
  if cluster.cluster_params:
133
134
  cluster.cluster_params.update(params)
134
135
  else:
@@ -329,7 +330,7 @@ class NflxCockroachDBCapacityModel(CapacityModel):
329
330
  )
330
331
 
331
332
  @staticmethod
332
- def description():
333
+ def description() -> str:
333
334
  return "Netflix Streaming CockroachDB Model"
334
335
 
335
336
  @staticmethod
@@ -337,7 +338,9 @@ class NflxCockroachDBCapacityModel(CapacityModel):
337
338
  return NflxCockroachDBArguments.model_json_schema()
338
339
 
339
340
  @staticmethod
340
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
341
+ def default_desires(
342
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
343
+ ) -> CapacityDesires:
341
344
  acceptable_consistency = {
342
345
  None,
343
346
  AccessConsistency.linearizable,
@@ -139,11 +139,12 @@ def _get_read_consistency_percentages(
139
139
  )
140
140
  if total_percent == 0:
141
141
  access_consistency = desires.query_pattern.access_consistency.same_region
142
- if access_consistency == AccessConsistency.serializable:
142
+ target_consistency = access_consistency.target_consistency
143
+ if target_consistency == AccessConsistency.serializable:
143
144
  transactional_read_percent = 1.0
144
145
  eventual_read_percent = 0.0
145
146
  strong_read_percent = 0.0
146
- elif access_consistency in (
147
+ elif target_consistency in (
147
148
  AccessConsistency.read_your_writes,
148
149
  AccessConsistency.linearizable,
149
150
  ):
@@ -184,7 +185,8 @@ def _get_write_consistency_percentages(
184
185
  total_percent = transactional_write_percent + non_transactional_write_percent
185
186
  if total_percent == 0:
186
187
  access_consistency = desires.query_pattern.access_consistency.same_region
187
- if access_consistency == AccessConsistency.serializable:
188
+ target_consistency = access_consistency.target_consistency
189
+ if target_consistency == AccessConsistency.serializable:
188
190
  transactional_write_percent = 1.0
189
191
  non_transactional_write_percent = 0.0
190
192
  else:
@@ -547,7 +549,7 @@ class NflxDynamoDBCapacityModel(CapacityModel):
547
549
  )
548
550
 
549
551
  @staticmethod
550
- def description():
552
+ def description() -> str:
551
553
  return "Netflix Streaming DynamoDB Model"
552
554
 
553
555
  @staticmethod
@@ -559,7 +561,9 @@ class NflxDynamoDBCapacityModel(CapacityModel):
559
561
  return False
560
562
 
561
563
  @staticmethod
562
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
564
+ def default_desires(
565
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
566
+ ) -> CapacityDesires:
563
567
  acceptable_consistency = {
564
568
  "same_region": {
565
569
  None,
@@ -58,7 +58,7 @@ def _target_rf(desires: CapacityDesires, user_copies: Optional[int]) -> int:
58
58
  # segments of 512 megs per
59
59
  # https://lucene.apache.org/core/8_1_0/core/org/apache/lucene/index/TieredMergePolicy.html#setSegmentsPerTier(double)
60
60
  # (FIXME) Verify what elastic merge actually does
61
- def _es_io_per_read(node_size_gib, segment_size_mb=512):
61
+ def _es_io_per_read(node_size_gib: float, segment_size_mb: int = 512) -> int:
62
62
  size_mib = node_size_gib * 1024
63
63
  segments = max(1, size_mib // segment_size_mb)
64
64
  # 10 segments per tier, plus 1 for L0 (avg)
@@ -75,7 +75,7 @@ def _estimate_elasticsearch_requirement( # noqa: E501 pylint: disable=too-many-
75
75
  max_rps_to_disk: int,
76
76
  zones_in_region: int = 3,
77
77
  copies_per_region: int = 3,
78
- jvm_memory_overhead=1.2,
78
+ jvm_memory_overhead: float = 1.2,
79
79
  ) -> CapacityRequirement:
80
80
  """Estimate the capacity required for one zone given a regional desire
81
81
 
@@ -153,7 +153,7 @@ def _estimate_elasticsearch_requirement( # noqa: E501 pylint: disable=too-many-
153
153
  )
154
154
 
155
155
 
156
- def _upsert_params(cluster, params):
156
+ def _upsert_params(cluster: ZoneClusterCapacity, params: Dict[str, Any]) -> None:
157
157
  if cluster.cluster_params:
158
158
  cluster.cluster_params.update(params)
159
159
  else:
@@ -194,7 +194,7 @@ class NflxElasticsearchDataCapacityModel(CapacityModel):
194
194
 
195
195
  @staticmethod
196
196
  def default_desires(
197
- user_desires, extra_model_arguments: Dict[str, Any]
197
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
198
198
  ) -> CapacityDesires:
199
199
  desires = CapacityModel.default_desires(user_desires, extra_model_arguments)
200
200
  desires.buffers = NflxElasticsearchDataCapacityModel.default_buffers()
@@ -452,7 +452,7 @@ class NflxElasticsearchCapacityModel(CapacityModel):
452
452
  return None
453
453
 
454
454
  @staticmethod
455
- def description():
455
+ def description() -> str:
456
456
  return "Netflix Streaming Elasticsearch Model"
457
457
 
458
458
  @staticmethod
@@ -482,7 +482,9 @@ class NflxElasticsearchCapacityModel(CapacityModel):
482
482
  return NflxElasticsearchArguments.model_json_schema()
483
483
 
484
484
  @staticmethod
485
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
485
+ def default_desires(
486
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
487
+ ) -> CapacityDesires:
486
488
  acceptable_consistency = {
487
489
  AccessConsistency.best_effort,
488
490
  AccessConsistency.eventual,
@@ -48,7 +48,7 @@ class NflxEntityCapacityModel(CapacityModel):
48
48
  return entity_app
49
49
 
50
50
  @staticmethod
51
- def description():
51
+ def description() -> str:
52
52
  return "Netflix Streaming Entity Model"
53
53
 
54
54
  @staticmethod
@@ -76,7 +76,7 @@ class NflxEntityCapacityModel(CapacityModel):
76
76
  / 1024**3
77
77
  )
78
78
  else:
79
- item_size_gib = 10 / 1024**2
79
+ item_size_gib = 10 / 1024**2 # type: ignore[unreachable]
80
80
  item_count = user_desires.data_shape.estimated_state_size_gib.scale(
81
81
  1 / item_size_gib
82
82
  )
@@ -102,7 +102,9 @@ class NflxEntityCapacityModel(CapacityModel):
102
102
  )
103
103
 
104
104
  @staticmethod
105
- def default_desires(user_desires, extra_model_arguments):
105
+ def default_desires(
106
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
107
+ ) -> CapacityDesires:
106
108
  if user_desires.query_pattern.access_pattern == AccessPattern.latency:
107
109
  return CapacityDesires(
108
110
  query_pattern=QueryPattern(
@@ -33,13 +33,10 @@ from service_capacity_modeling.interface import Requirements
33
33
  from service_capacity_modeling.models import CapacityModel
34
34
  from service_capacity_modeling.models.common import buffer_for_components
35
35
  from service_capacity_modeling.models.common import compute_stateful_zone
36
- from service_capacity_modeling.models.common import get_cores_from_current_capacity
37
- from service_capacity_modeling.models.common import get_disk_from_current_capacity
38
36
  from service_capacity_modeling.models.common import get_effective_disk_per_node_gib
39
- from service_capacity_modeling.models.common import get_memory_from_current_capacity
40
- from service_capacity_modeling.models.common import get_network_from_current_capacity
41
37
  from service_capacity_modeling.models.common import network_services
42
38
  from service_capacity_modeling.models.common import normalize_cores
39
+ from service_capacity_modeling.models.common import RequirementFromCurrentCapacity
43
40
  from service_capacity_modeling.models.common import simple_network_mbps
44
41
  from service_capacity_modeling.models.common import sqrt_staffed_cores
45
42
  from service_capacity_modeling.models.common import working_set_from_drive_and_slo
@@ -72,7 +69,9 @@ def calculate_read_cpu_time_evcache_ms(read_size_bytes: float) -> float:
72
69
  return max(read_latency_ms, 0.005)
73
70
 
74
71
 
75
- def calculate_spread_cost(cluster_size: int, max_cost=100000, min_cost=0.0) -> float:
72
+ def calculate_spread_cost(
73
+ cluster_size: int, max_cost: float = 100000, min_cost: float = 0.0
74
+ ) -> float:
76
75
  if cluster_size > 10:
77
76
  return min_cost
78
77
  if cluster_size < 2:
@@ -85,14 +84,14 @@ def calculate_vitals_for_capacity_planner(
85
84
  instance: Instance,
86
85
  current_memory_gib: float,
87
86
  current_disk_gib: float,
88
- ):
87
+ ) -> Tuple[float, float, float, float]:
89
88
  # First calculate assuming new deployment
90
89
  needed_cores = normalize_cores(
91
90
  core_count=sqrt_staffed_cores(desires),
92
91
  target_shape=instance,
93
92
  reference_shape=desires.reference_shape,
94
93
  )
95
- needed_network_mbps = simple_network_mbps(desires)
94
+ needed_network_mbps = float(simple_network_mbps(desires))
96
95
  needed_memory_gib = current_memory_gib
97
96
  needed_disk_gib = current_disk_gib
98
97
 
@@ -104,25 +103,20 @@ def calculate_vitals_for_capacity_planner(
104
103
  )
105
104
  if not current_capacity:
106
105
  return needed_cores, needed_network_mbps, needed_memory_gib, needed_disk_gib
106
+ requirements = RequirementFromCurrentCapacity(
107
+ current_capacity=current_capacity,
108
+ buffers=desires.buffers,
109
+ )
107
110
  needed_cores = normalize_cores(
108
- core_count=get_cores_from_current_capacity(
109
- current_capacity, desires.buffers, instance
110
- ),
111
+ core_count=requirements.cpu(instance_candidate=instance),
111
112
  target_shape=instance,
112
113
  reference_shape=current_capacity.cluster_instance,
113
114
  )
114
- needed_network_mbps = get_network_from_current_capacity(
115
- current_capacity, desires.buffers
116
- )
117
- needed_memory_gib = get_memory_from_current_capacity(
118
- current_capacity, desires.buffers
115
+ needed_network_mbps = float(requirements.network_mbps)
116
+ needed_disk_gib = float(
117
+ requirements.disk_gib if current_capacity.cluster_drive is not None else 0.0
119
118
  )
120
- if current_capacity.cluster_drive is None:
121
- needed_disk_gib = 0.0
122
- else:
123
- needed_disk_gib = get_disk_from_current_capacity(
124
- current_capacity, desires.buffers
125
- )
119
+ needed_memory_gib = requirements.mem_gib
126
120
  return needed_cores, needed_network_mbps, needed_memory_gib, needed_disk_gib
127
121
 
128
122
 
@@ -204,7 +198,7 @@ def _estimate_evcache_requirement(
204
198
  )
205
199
 
206
200
 
207
- def _upsert_params(cluster, params):
201
+ def _upsert_params(cluster: Any, params: Dict[str, Any]) -> None:
208
202
  if cluster.cluster_params:
209
203
  cluster.cluster_params.update(params)
210
204
  else:
@@ -271,7 +265,7 @@ def _estimate_evcache_cluster_zonal( # noqa: C901,E501 pylint: disable=too-many
271
265
  # larger to account for additional overhead. Note that the
272
266
  # reserved_instance_system_mem_gib has a base of 1 GiB OSMEM so this
273
267
  # just represents the variable component
274
- def reserve_memory(instance_mem_gib):
268
+ def reserve_memory(instance_mem_gib: float) -> float:
275
269
  # (Joey) From the chart it appears to be about a 3% overhead for
276
270
  # OS memory.
277
271
  variable_os = int(instance_mem_gib * 0.03)
@@ -447,7 +441,7 @@ class NflxEVCacheCapacityModel(CapacityModel):
447
441
  )
448
442
 
449
443
  @staticmethod
450
- def description():
444
+ def description() -> str:
451
445
  return "Netflix Streaming EVCache (memcached) Model"
452
446
 
453
447
  @staticmethod
@@ -455,7 +449,9 @@ class NflxEVCacheCapacityModel(CapacityModel):
455
449
  return NflxEVCacheArguments.model_json_schema()
456
450
 
457
451
  @staticmethod
458
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
452
+ def default_desires(
453
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
454
+ ) -> CapacityDesires:
459
455
  acceptable_consistency = {
460
456
  AccessConsistency.best_effort,
461
457
  AccessConsistency.never,
@@ -45,7 +45,7 @@ class NflxGraphKVCapacityModel(CapacityModel):
45
45
  return graphkv_app
46
46
 
47
47
  @staticmethod
48
- def description():
48
+ def description() -> str:
49
49
  return "Netflix Streaming Graph Abstraction"
50
50
 
51
51
  @staticmethod
@@ -89,7 +89,7 @@ class NflxGraphKVCapacityModel(CapacityModel):
89
89
  / 1024**3
90
90
  )
91
91
  else:
92
- item_size_gib = 1 / 1024**2
92
+ item_size_gib = 1 / 1024**2 # type: ignore[unreachable]
93
93
  item_count = user_desires.data_shape.estimated_state_size_gib.scale(
94
94
  1 / item_size_gib
95
95
  )
@@ -102,7 +102,9 @@ class NflxGraphKVCapacityModel(CapacityModel):
102
102
  return (("org.netflix.key-value", _modify_kv_desires),)
103
103
 
104
104
  @staticmethod
105
- def default_desires(user_desires, extra_model_arguments):
105
+ def default_desires(
106
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
107
+ ) -> CapacityDesires:
106
108
  if user_desires.query_pattern.access_pattern == AccessPattern.latency:
107
109
  return CapacityDesires(
108
110
  query_pattern=QueryPattern(