service-capacity-modeling 0.3.76__py3-none-any.whl → 0.3.78__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of service-capacity-modeling might be problematic. Click here for more details.

Files changed (37) hide show
  1. service_capacity_modeling/capacity_planner.py +46 -40
  2. service_capacity_modeling/hardware/__init__.py +11 -7
  3. service_capacity_modeling/interface.py +20 -18
  4. service_capacity_modeling/models/__init__.py +21 -2
  5. service_capacity_modeling/models/common.py +20 -19
  6. service_capacity_modeling/models/headroom_strategy.py +2 -1
  7. service_capacity_modeling/models/org/netflix/__init__.py +4 -1
  8. service_capacity_modeling/models/org/netflix/aurora.py +12 -7
  9. service_capacity_modeling/models/org/netflix/cassandra.py +25 -15
  10. service_capacity_modeling/models/org/netflix/counter.py +4 -2
  11. service_capacity_modeling/models/org/netflix/crdb.py +7 -4
  12. service_capacity_modeling/models/org/netflix/ddb.py +9 -5
  13. service_capacity_modeling/models/org/netflix/elasticsearch.py +8 -6
  14. service_capacity_modeling/models/org/netflix/entity.py +5 -3
  15. service_capacity_modeling/models/org/netflix/evcache.py +13 -9
  16. service_capacity_modeling/models/org/netflix/graphkv.py +5 -3
  17. service_capacity_modeling/models/org/netflix/iso_date_math.py +12 -9
  18. service_capacity_modeling/models/org/netflix/kafka.py +13 -7
  19. service_capacity_modeling/models/org/netflix/key_value.py +4 -2
  20. service_capacity_modeling/models/org/netflix/postgres.py +4 -2
  21. service_capacity_modeling/models/org/netflix/rds.py +10 -5
  22. service_capacity_modeling/models/org/netflix/stateless_java.py +4 -2
  23. service_capacity_modeling/models/org/netflix/time_series.py +4 -2
  24. service_capacity_modeling/models/org/netflix/time_series_config.py +3 -3
  25. service_capacity_modeling/models/org/netflix/wal.py +4 -2
  26. service_capacity_modeling/models/org/netflix/zookeeper.py +5 -3
  27. service_capacity_modeling/stats.py +14 -11
  28. service_capacity_modeling/tools/auto_shape.py +10 -6
  29. service_capacity_modeling/tools/fetch_pricing.py +13 -6
  30. service_capacity_modeling/tools/generate_missing.py +4 -3
  31. service_capacity_modeling/tools/instance_families.py +4 -1
  32. {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.78.dist-info}/METADATA +1 -1
  33. {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.78.dist-info}/RECORD +37 -37
  34. {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.78.dist-info}/WHEEL +0 -0
  35. {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.78.dist-info}/entry_points.txt +0 -0
  36. {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.78.dist-info}/licenses/LICENSE +0 -0
  37. {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.78.dist-info}/top_level.txt +0 -0
@@ -79,7 +79,7 @@ def _write_buffer_gib_zone(
79
79
  return float(write_buffer_gib) / zones_per_region
80
80
 
81
81
 
82
- def _get_cores_from_desires(desires, instance):
82
+ def _get_cores_from_desires(desires: CapacityDesires, instance: Instance) -> int:
83
83
  cpu_buffer = buffer_for_components(
84
84
  buffers=desires.buffers, components=[BACKGROUND_BUFFER]
85
85
  )
@@ -97,7 +97,7 @@ def _get_cores_from_desires(desires, instance):
97
97
  return needed_cores
98
98
 
99
99
 
100
- def _get_disk_from_desires(desires, copies_per_region):
100
+ def _get_disk_from_desires(desires: CapacityDesires, copies_per_region: int) -> int:
101
101
  disk_buffer = buffer_for_components(
102
102
  buffers=desires.buffers, components=[BufferComponent.disk]
103
103
  )
@@ -116,7 +116,7 @@ def _get_min_count(
116
116
  needed_disk_gib: float,
117
117
  disk_per_node_gib: float,
118
118
  cluster_size_lambda: Callable[[int], int],
119
- ):
119
+ ) -> int:
120
120
  """
121
121
  Compute the minimum number of nodes required for a zone.
122
122
 
@@ -158,7 +158,10 @@ def _get_min_count(
158
158
 
159
159
 
160
160
  def _zonal_requirement_for_new_cluster(
161
- desires, instance, copies_per_region, zones_per_region
161
+ desires: CapacityDesires,
162
+ instance: Instance,
163
+ copies_per_region: int,
164
+ zones_per_region: int,
162
165
  ) -> CapacityRequirement:
163
166
  needed_cores = _get_cores_from_desires(desires, instance)
164
167
  needed_disk = _get_disk_from_desires(desires, copies_per_region)
@@ -325,14 +328,14 @@ def _estimate_cassandra_requirement(
325
328
  )
326
329
 
327
330
 
328
- def _get_current_cluster_size(desires) -> int:
331
+ def _get_current_cluster_size(desires: CapacityDesires) -> int:
329
332
  current_capacity = _get_current_capacity(desires)
330
333
  if current_capacity is None:
331
334
  return 0
332
335
  return math.ceil(current_capacity.cluster_instance_count.mid)
333
336
 
334
337
 
335
- def _get_current_capacity(desires) -> Optional[CurrentClusterCapacity]:
338
+ def _get_current_capacity(desires: CapacityDesires) -> Optional[CurrentClusterCapacity]:
336
339
  current_capacity = (
337
340
  None
338
341
  if desires.current_clusters is None
@@ -345,7 +348,7 @@ def _get_current_capacity(desires) -> Optional[CurrentClusterCapacity]:
345
348
  return current_capacity
346
349
 
347
350
 
348
- def _upsert_params(cluster, params):
351
+ def _upsert_params(cluster: Any, params: Dict[str, Any]) -> None:
349
352
  if cluster.cluster_params:
350
353
  cluster.cluster_params.update(params)
351
354
  else:
@@ -374,7 +377,7 @@ def _estimate_cassandra_cluster_zonal( # pylint: disable=too-many-positional-ar
374
377
  desires: CapacityDesires,
375
378
  zones_per_region: int = 3,
376
379
  copies_per_region: int = 3,
377
- require_local_disks: bool = True,
380
+ require_local_disks: bool = False,
378
381
  require_attached_disks: bool = False,
379
382
  required_cluster_size: Optional[int] = None,
380
383
  max_rps_to_disk: int = 500,
@@ -575,7 +578,7 @@ def _estimate_cassandra_cluster_zonal( # pylint: disable=too-many-positional-ar
575
578
 
576
579
 
577
580
  # C* LCS has 160 MiB sstables by default and 10 sstables per level
578
- def _cass_io_per_read(node_size_gib, sstable_size_mb=160):
581
+ def _cass_io_per_read(node_size_gib: float, sstable_size_mb: int = 160) -> int:
579
582
  gb = node_size_gib * 1024
580
583
  sstables = max(1, gb // sstable_size_mb)
581
584
  # 10 sstables per level, plus 1 for L0 (avg)
@@ -585,7 +588,7 @@ def _cass_io_per_read(node_size_gib, sstable_size_mb=160):
585
588
  return 2 * levels
586
589
 
587
590
 
588
- def _get_base_memory(desires: CapacityDesires):
591
+ def _get_base_memory(desires: CapacityDesires) -> float:
589
592
  return (
590
593
  desires.data_shape.reserved_instance_app_mem_gib
591
594
  + desires.data_shape.reserved_instance_system_mem_gib
@@ -641,7 +644,7 @@ class NflxCassandraArguments(BaseModel):
641
644
  " this will be deduced from durability and consistency desires",
642
645
  )
643
646
  require_local_disks: bool = Field(
644
- default=True,
647
+ default=False,
645
648
  description="If local (ephemeral) drives are required",
646
649
  )
647
650
  require_attached_disks: bool = Field(
@@ -679,8 +682,13 @@ class NflxCassandraArguments(BaseModel):
679
682
 
680
683
 
681
684
  class NflxCassandraCapacityModel(CapacityModel):
685
+ def __init__(self) -> None:
686
+ pass
687
+
682
688
  @staticmethod
683
- def get_required_cluster_size(tier, extra_model_arguments):
689
+ def get_required_cluster_size(
690
+ tier: int, extra_model_arguments: Dict[str, Any]
691
+ ) -> Optional[int]:
684
692
  required_cluster_size: Optional[int] = (
685
693
  math.ceil(extra_model_arguments["required_cluster_size"])
686
694
  if "required_cluster_size" in extra_model_arguments
@@ -724,7 +732,7 @@ class NflxCassandraCapacityModel(CapacityModel):
724
732
  desires, extra_model_arguments.get("copies_per_region", None)
725
733
  )
726
734
  require_local_disks: bool = extra_model_arguments.get(
727
- "require_local_disks", True
735
+ "require_local_disks", False
728
736
  )
729
737
  require_attached_disks: bool = extra_model_arguments.get(
730
738
  "require_attached_disks", False
@@ -775,7 +783,7 @@ class NflxCassandraCapacityModel(CapacityModel):
775
783
  )
776
784
 
777
785
  @staticmethod
778
- def description():
786
+ def description() -> str:
779
787
  return "Netflix Streaming Cassandra Model"
780
788
 
781
789
  @staticmethod
@@ -803,7 +811,9 @@ class NflxCassandraCapacityModel(CapacityModel):
803
811
  )
804
812
 
805
813
  @staticmethod
806
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
814
+ def default_desires(
815
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
816
+ ) -> CapacityDesires:
807
817
  acceptable_consistency = {
808
818
  None,
809
819
  AccessConsistency.best_effort,
@@ -81,7 +81,7 @@ class NflxCounterCapacityModel(CapacityModel):
81
81
  return counter_app
82
82
 
83
83
  @staticmethod
84
- def description():
84
+ def description() -> str:
85
85
  return "Netflix Streaming Counter Model"
86
86
 
87
87
  @staticmethod
@@ -129,7 +129,9 @@ class NflxCounterCapacityModel(CapacityModel):
129
129
  return tuple(stores)
130
130
 
131
131
  @staticmethod
132
- def default_desires(user_desires, extra_model_arguments):
132
+ def default_desires(
133
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
134
+ ) -> CapacityDesires:
133
135
  if user_desires.query_pattern.access_pattern == AccessPattern.latency:
134
136
  return CapacityDesires(
135
137
  query_pattern=QueryPattern(
@@ -29,6 +29,7 @@ from service_capacity_modeling.interface import Interval
29
29
  from service_capacity_modeling.interface import QueryPattern
30
30
  from service_capacity_modeling.interface import RegionContext
31
31
  from service_capacity_modeling.interface import Requirements
32
+ from service_capacity_modeling.interface import ZoneClusterCapacity
32
33
  from service_capacity_modeling.models import CapacityModel
33
34
  from service_capacity_modeling.models.common import buffer_for_components
34
35
  from service_capacity_modeling.models.common import compute_stateful_zone
@@ -44,7 +45,7 @@ logger = logging.getLogger(__name__)
44
45
 
45
46
  # Pebble does Leveled compaction with tieres of size??
46
47
  # (FIXME) What does pebble actually do
47
- def _crdb_io_per_read(node_size_gib, sstable_size_mb=1000):
48
+ def _crdb_io_per_read(node_size_gib: float, sstable_size_mb: int = 1000) -> int:
48
49
  gb = node_size_gib * 1024
49
50
  sstables = max(1, gb // sstable_size_mb)
50
51
  # 10 sstables per level, plus 1 for L0 (avg)
@@ -128,7 +129,7 @@ def _estimate_cockroachdb_requirement( # noqa=E501 pylint: disable=too-many-pos
128
129
  )
129
130
 
130
131
 
131
- def _upsert_params(cluster, params):
132
+ def _upsert_params(cluster: ZoneClusterCapacity, params: Dict[str, Any]) -> None:
132
133
  if cluster.cluster_params:
133
134
  cluster.cluster_params.update(params)
134
135
  else:
@@ -329,7 +330,7 @@ class NflxCockroachDBCapacityModel(CapacityModel):
329
330
  )
330
331
 
331
332
  @staticmethod
332
- def description():
333
+ def description() -> str:
333
334
  return "Netflix Streaming CockroachDB Model"
334
335
 
335
336
  @staticmethod
@@ -337,7 +338,9 @@ class NflxCockroachDBCapacityModel(CapacityModel):
337
338
  return NflxCockroachDBArguments.model_json_schema()
338
339
 
339
340
  @staticmethod
340
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
341
+ def default_desires(
342
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
343
+ ) -> CapacityDesires:
341
344
  acceptable_consistency = {
342
345
  None,
343
346
  AccessConsistency.linearizable,
@@ -139,11 +139,12 @@ def _get_read_consistency_percentages(
139
139
  )
140
140
  if total_percent == 0:
141
141
  access_consistency = desires.query_pattern.access_consistency.same_region
142
- if access_consistency == AccessConsistency.serializable:
142
+ target_consistency = access_consistency.target_consistency
143
+ if target_consistency == AccessConsistency.serializable:
143
144
  transactional_read_percent = 1.0
144
145
  eventual_read_percent = 0.0
145
146
  strong_read_percent = 0.0
146
- elif access_consistency in (
147
+ elif target_consistency in (
147
148
  AccessConsistency.read_your_writes,
148
149
  AccessConsistency.linearizable,
149
150
  ):
@@ -184,7 +185,8 @@ def _get_write_consistency_percentages(
184
185
  total_percent = transactional_write_percent + non_transactional_write_percent
185
186
  if total_percent == 0:
186
187
  access_consistency = desires.query_pattern.access_consistency.same_region
187
- if access_consistency == AccessConsistency.serializable:
188
+ target_consistency = access_consistency.target_consistency
189
+ if target_consistency == AccessConsistency.serializable:
188
190
  transactional_write_percent = 1.0
189
191
  non_transactional_write_percent = 0.0
190
192
  else:
@@ -547,7 +549,7 @@ class NflxDynamoDBCapacityModel(CapacityModel):
547
549
  )
548
550
 
549
551
  @staticmethod
550
- def description():
552
+ def description() -> str:
551
553
  return "Netflix Streaming DynamoDB Model"
552
554
 
553
555
  @staticmethod
@@ -559,7 +561,9 @@ class NflxDynamoDBCapacityModel(CapacityModel):
559
561
  return False
560
562
 
561
563
  @staticmethod
562
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
564
+ def default_desires(
565
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
566
+ ) -> CapacityDesires:
563
567
  acceptable_consistency = {
564
568
  "same_region": {
565
569
  None,
@@ -58,7 +58,7 @@ def _target_rf(desires: CapacityDesires, user_copies: Optional[int]) -> int:
58
58
  # segments of 512 megs per
59
59
  # https://lucene.apache.org/core/8_1_0/core/org/apache/lucene/index/TieredMergePolicy.html#setSegmentsPerTier(double)
60
60
  # (FIXME) Verify what elastic merge actually does
61
- def _es_io_per_read(node_size_gib, segment_size_mb=512):
61
+ def _es_io_per_read(node_size_gib: float, segment_size_mb: int = 512) -> int:
62
62
  size_mib = node_size_gib * 1024
63
63
  segments = max(1, size_mib // segment_size_mb)
64
64
  # 10 segments per tier, plus 1 for L0 (avg)
@@ -75,7 +75,7 @@ def _estimate_elasticsearch_requirement( # noqa: E501 pylint: disable=too-many-
75
75
  max_rps_to_disk: int,
76
76
  zones_in_region: int = 3,
77
77
  copies_per_region: int = 3,
78
- jvm_memory_overhead=1.2,
78
+ jvm_memory_overhead: float = 1.2,
79
79
  ) -> CapacityRequirement:
80
80
  """Estimate the capacity required for one zone given a regional desire
81
81
 
@@ -153,7 +153,7 @@ def _estimate_elasticsearch_requirement( # noqa: E501 pylint: disable=too-many-
153
153
  )
154
154
 
155
155
 
156
- def _upsert_params(cluster, params):
156
+ def _upsert_params(cluster: ZoneClusterCapacity, params: Dict[str, Any]) -> None:
157
157
  if cluster.cluster_params:
158
158
  cluster.cluster_params.update(params)
159
159
  else:
@@ -194,7 +194,7 @@ class NflxElasticsearchDataCapacityModel(CapacityModel):
194
194
 
195
195
  @staticmethod
196
196
  def default_desires(
197
- user_desires, extra_model_arguments: Dict[str, Any]
197
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
198
198
  ) -> CapacityDesires:
199
199
  desires = CapacityModel.default_desires(user_desires, extra_model_arguments)
200
200
  desires.buffers = NflxElasticsearchDataCapacityModel.default_buffers()
@@ -452,7 +452,7 @@ class NflxElasticsearchCapacityModel(CapacityModel):
452
452
  return None
453
453
 
454
454
  @staticmethod
455
- def description():
455
+ def description() -> str:
456
456
  return "Netflix Streaming Elasticsearch Model"
457
457
 
458
458
  @staticmethod
@@ -482,7 +482,9 @@ class NflxElasticsearchCapacityModel(CapacityModel):
482
482
  return NflxElasticsearchArguments.model_json_schema()
483
483
 
484
484
  @staticmethod
485
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
485
+ def default_desires(
486
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
487
+ ) -> CapacityDesires:
486
488
  acceptable_consistency = {
487
489
  AccessConsistency.best_effort,
488
490
  AccessConsistency.eventual,
@@ -48,7 +48,7 @@ class NflxEntityCapacityModel(CapacityModel):
48
48
  return entity_app
49
49
 
50
50
  @staticmethod
51
- def description():
51
+ def description() -> str:
52
52
  return "Netflix Streaming Entity Model"
53
53
 
54
54
  @staticmethod
@@ -76,7 +76,7 @@ class NflxEntityCapacityModel(CapacityModel):
76
76
  / 1024**3
77
77
  )
78
78
  else:
79
- item_size_gib = 10 / 1024**2
79
+ item_size_gib = 10 / 1024**2 # type: ignore[unreachable]
80
80
  item_count = user_desires.data_shape.estimated_state_size_gib.scale(
81
81
  1 / item_size_gib
82
82
  )
@@ -102,7 +102,9 @@ class NflxEntityCapacityModel(CapacityModel):
102
102
  )
103
103
 
104
104
  @staticmethod
105
- def default_desires(user_desires, extra_model_arguments):
105
+ def default_desires(
106
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
107
+ ) -> CapacityDesires:
106
108
  if user_desires.query_pattern.access_pattern == AccessPattern.latency:
107
109
  return CapacityDesires(
108
110
  query_pattern=QueryPattern(
@@ -69,7 +69,9 @@ def calculate_read_cpu_time_evcache_ms(read_size_bytes: float) -> float:
69
69
  return max(read_latency_ms, 0.005)
70
70
 
71
71
 
72
- def calculate_spread_cost(cluster_size: int, max_cost=100000, min_cost=0.0) -> float:
72
+ def calculate_spread_cost(
73
+ cluster_size: int, max_cost: float = 100000, min_cost: float = 0.0
74
+ ) -> float:
73
75
  if cluster_size > 10:
74
76
  return min_cost
75
77
  if cluster_size < 2:
@@ -82,14 +84,14 @@ def calculate_vitals_for_capacity_planner(
82
84
  instance: Instance,
83
85
  current_memory_gib: float,
84
86
  current_disk_gib: float,
85
- ):
87
+ ) -> Tuple[float, float, float, float]:
86
88
  # First calculate assuming new deployment
87
89
  needed_cores = normalize_cores(
88
90
  core_count=sqrt_staffed_cores(desires),
89
91
  target_shape=instance,
90
92
  reference_shape=desires.reference_shape,
91
93
  )
92
- needed_network_mbps = simple_network_mbps(desires)
94
+ needed_network_mbps = float(simple_network_mbps(desires))
93
95
  needed_memory_gib = current_memory_gib
94
96
  needed_disk_gib = current_disk_gib
95
97
 
@@ -110,8 +112,8 @@ def calculate_vitals_for_capacity_planner(
110
112
  target_shape=instance,
111
113
  reference_shape=current_capacity.cluster_instance,
112
114
  )
113
- needed_network_mbps = requirements.network_mbps
114
- needed_disk_gib = (
115
+ needed_network_mbps = float(requirements.network_mbps)
116
+ needed_disk_gib = float(
115
117
  requirements.disk_gib if current_capacity.cluster_drive is not None else 0.0
116
118
  )
117
119
  needed_memory_gib = requirements.mem_gib
@@ -196,7 +198,7 @@ def _estimate_evcache_requirement(
196
198
  )
197
199
 
198
200
 
199
- def _upsert_params(cluster, params):
201
+ def _upsert_params(cluster: Any, params: Dict[str, Any]) -> None:
200
202
  if cluster.cluster_params:
201
203
  cluster.cluster_params.update(params)
202
204
  else:
@@ -263,7 +265,7 @@ def _estimate_evcache_cluster_zonal( # noqa: C901,E501 pylint: disable=too-many
263
265
  # larger to account for additional overhead. Note that the
264
266
  # reserved_instance_system_mem_gib has a base of 1 GiB OSMEM so this
265
267
  # just represents the variable component
266
- def reserve_memory(instance_mem_gib):
268
+ def reserve_memory(instance_mem_gib: float) -> float:
267
269
  # (Joey) From the chart it appears to be about a 3% overhead for
268
270
  # OS memory.
269
271
  variable_os = int(instance_mem_gib * 0.03)
@@ -439,7 +441,7 @@ class NflxEVCacheCapacityModel(CapacityModel):
439
441
  )
440
442
 
441
443
  @staticmethod
442
- def description():
444
+ def description() -> str:
443
445
  return "Netflix Streaming EVCache (memcached) Model"
444
446
 
445
447
  @staticmethod
@@ -447,7 +449,9 @@ class NflxEVCacheCapacityModel(CapacityModel):
447
449
  return NflxEVCacheArguments.model_json_schema()
448
450
 
449
451
  @staticmethod
450
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
452
+ def default_desires(
453
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
454
+ ) -> CapacityDesires:
451
455
  acceptable_consistency = {
452
456
  AccessConsistency.best_effort,
453
457
  AccessConsistency.never,
@@ -45,7 +45,7 @@ class NflxGraphKVCapacityModel(CapacityModel):
45
45
  return graphkv_app
46
46
 
47
47
  @staticmethod
48
- def description():
48
+ def description() -> str:
49
49
  return "Netflix Streaming Graph Abstraction"
50
50
 
51
51
  @staticmethod
@@ -89,7 +89,7 @@ class NflxGraphKVCapacityModel(CapacityModel):
89
89
  / 1024**3
90
90
  )
91
91
  else:
92
- item_size_gib = 1 / 1024**2
92
+ item_size_gib = 1 / 1024**2 # type: ignore[unreachable]
93
93
  item_count = user_desires.data_shape.estimated_state_size_gib.scale(
94
94
  1 / item_size_gib
95
95
  )
@@ -102,7 +102,9 @@ class NflxGraphKVCapacityModel(CapacityModel):
102
102
  return (("org.netflix.key-value", _modify_kv_desires),)
103
103
 
104
104
  @staticmethod
105
- def default_desires(user_desires, extra_model_arguments):
105
+ def default_desires(
106
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
107
+ ) -> CapacityDesires:
106
108
  if user_desires.query_pattern.access_pattern == AccessPattern.latency:
107
109
  return CapacityDesires(
108
110
  query_pattern=QueryPattern(
@@ -1,8 +1,9 @@
1
1
  import math
2
2
  from datetime import timedelta
3
3
  from decimal import Decimal
4
+ from typing import Union
4
5
 
5
- import isodate
6
+ import isodate # type: ignore
6
7
  from isodate import parse_duration
7
8
 
8
9
  DURATION_1M = 2592000
@@ -11,24 +12,26 @@ D_YEAR = Decimal(DURATION_1Y)
11
12
  D_MONTH = Decimal(DURATION_1M)
12
13
 
13
14
 
14
- def _iso_to_proto_duration(iso_duration: str):
15
+ def _iso_to_proto_duration(iso_duration: str) -> str:
15
16
  parsed = parse_duration(iso_duration)
16
17
  return f"{int(parsed.total_seconds())}s"
17
18
 
18
19
 
19
- def iso_to_seconds(iso_duration: str, unlimited=0) -> int:
20
+ def iso_to_seconds(iso_duration: str, unlimited: int = 0) -> int:
20
21
  if iso_duration == "unlimited":
21
22
  return unlimited
22
- parsed = isodate.parse_duration(iso_duration)
23
+ parsed: Union[timedelta, isodate.Duration] = isodate.parse_duration(iso_duration)
23
24
  # isodate package parse_duration returns either timedelta OR Duration
24
25
  if isinstance(parsed, isodate.Duration):
25
- return math.ceil(
26
- parsed.years * D_YEAR
27
- + parsed.months * D_MONTH
28
- + Decimal(parsed.tdelta.total_seconds())
26
+ return int(
27
+ math.ceil(
28
+ parsed.years * D_YEAR
29
+ + parsed.months * D_MONTH
30
+ + Decimal(parsed.tdelta.total_seconds())
31
+ )
29
32
  )
30
33
  return int(parsed.total_seconds())
31
34
 
32
35
 
33
- def _iso_to_timedelta(iso_duration: str, unlimited=0) -> timedelta:
36
+ def _iso_to_timedelta(iso_duration: str, unlimited: int = 0) -> timedelta:
34
37
  return timedelta(seconds=iso_to_seconds(iso_duration, unlimited))
@@ -64,7 +64,9 @@ def _get_current_zonal_cluster(
64
64
  )
65
65
 
66
66
 
67
- def _is_same_instance_family(cluster, target_family):
67
+ def _is_same_instance_family(
68
+ cluster: Optional[CurrentZoneClusterCapacity], target_family: str
69
+ ) -> bool:
68
70
  """Check if cluster has a different instance family than the target."""
69
71
  if not cluster or not cluster.cluster_instance:
70
72
  return False
@@ -213,14 +215,16 @@ def _estimate_kafka_requirement( # pylint: disable=too-many-positional-argument
213
215
  )
214
216
 
215
217
 
216
- def _upsert_params(cluster, params):
218
+ def _upsert_params(cluster: Any, params: Dict[str, Any]) -> None:
217
219
  if cluster.cluster_params:
218
220
  cluster.cluster_params.update(params)
219
221
  else:
220
222
  cluster.cluster_params = params
221
223
 
222
224
 
223
- def _kafka_read_io(rps, io_size_kib, size_gib, recovery_seconds: int) -> float:
225
+ def _kafka_read_io(
226
+ rps: float, io_size_kib: float, size_gib: float, recovery_seconds: int
227
+ ) -> float:
224
228
  # Get enough disk read IO capacity for some reads
225
229
  # In practice we have cache reducing this by 99% or more
226
230
  read_ios = rps * 0.05
@@ -229,7 +233,7 @@ def _kafka_read_io(rps, io_size_kib, size_gib, recovery_seconds: int) -> float:
229
233
  size_kib = size_gib * 0.5 * (1024 * 1024)
230
234
  recovery_ios = max(1, size_kib / io_size_kib) / recovery_seconds
231
235
  # Leave 50% headroom for read IOs since generally we will hit cache
232
- return (read_ios + int(round(recovery_ios))) * 1.5
236
+ return float((read_ios + int(round(recovery_ios))) * 1.5)
233
237
 
234
238
 
235
239
  # pylint: disable=too-many-locals
@@ -239,7 +243,7 @@ def _estimate_kafka_cluster_zonal( # noqa: C901
239
243
  instance: Instance,
240
244
  drive: Drive,
241
245
  desires: CapacityDesires,
242
- hot_retention_seconds,
246
+ hot_retention_seconds: float,
243
247
  zones_per_region: int = 3,
244
248
  copies_per_region: int = 2,
245
249
  require_local_disks: bool = False,
@@ -532,7 +536,7 @@ class NflxKafkaCapacityModel(CapacityModel):
532
536
  )
533
537
 
534
538
  @staticmethod
535
- def description():
539
+ def description() -> str:
536
540
  return "Netflix Streaming Kafka Model"
537
541
 
538
542
  @staticmethod
@@ -545,7 +549,9 @@ class NflxKafkaCapacityModel(CapacityModel):
545
549
  return ("gp3",)
546
550
 
547
551
  @staticmethod
548
- def default_desires(user_desires, extra_model_arguments: Dict[str, Any]):
552
+ def default_desires(
553
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
554
+ ) -> CapacityDesires:
549
555
  # Default to 10MiB/s and a single reader
550
556
  concurrent_readers = max(
551
557
  1, int(user_desires.query_pattern.estimated_read_per_second.mid)
@@ -48,7 +48,7 @@ class NflxKeyValueCapacityModel(CapacityModel):
48
48
  return kv_app
49
49
 
50
50
  @staticmethod
51
- def description():
51
+ def description() -> str:
52
52
  return "Netflix Streaming Key-Value Model"
53
53
 
54
54
  @staticmethod
@@ -119,7 +119,9 @@ class NflxKeyValueCapacityModel(CapacityModel):
119
119
  return (("org.netflix.cassandra", lambda x: x),)
120
120
 
121
121
  @staticmethod
122
- def default_desires(user_desires, extra_model_arguments):
122
+ def default_desires(
123
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
124
+ ) -> CapacityDesires:
123
125
  if user_desires.query_pattern.access_pattern == AccessPattern.latency:
124
126
  return CapacityDesires(
125
127
  query_pattern=QueryPattern(
@@ -45,7 +45,7 @@ class NflxPostgresCapacityModel(CapacityModel):
45
45
  return plan
46
46
 
47
47
  @staticmethod
48
- def description():
48
+ def description() -> str:
49
49
  return "Netflix Postgres Model"
50
50
 
51
51
  @staticmethod
@@ -53,7 +53,9 @@ class NflxPostgresCapacityModel(CapacityModel):
53
53
  return Platform.aurora_postgres, Platform.amd64
54
54
 
55
55
  @staticmethod
56
- def default_desires(user_desires, extra_model_arguments):
56
+ def default_desires(
57
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
58
+ ) -> CapacityDesires:
57
59
  return CapacityDesires(
58
60
  query_pattern=QueryPattern(
59
61
  access_pattern=AccessPattern.latency,
@@ -1,6 +1,7 @@
1
1
  import logging
2
2
  import math
3
3
  from typing import Any
4
+ from typing import Callable
4
5
  from typing import Dict
5
6
  from typing import Optional
6
7
 
@@ -83,7 +84,9 @@ def _estimate_rds_requirement(
83
84
 
84
85
  # MySQL default block size is 16KiB, PostGreSQL is 8KiB Number of reads for B-Tree
85
86
  # are given by log of total pages to the base of B-Tree fan out factor
86
- def _rds_required_disk_ios(disk_size_gib: int, db_type: str, btree_fan_out: int = 100):
87
+ def _rds_required_disk_ios(
88
+ disk_size_gib: int, db_type: str, btree_fan_out: int = 100
89
+ ) -> float:
87
90
  disk_size_kb = disk_size_gib * 1024 * 1024
88
91
  if db_type == "postgres":
89
92
  default_block_size = 8 # KiB
@@ -101,8 +104,8 @@ def _compute_rds_region( # pylint: disable=too-many-positional-arguments
101
104
  needed_disk_gib: int,
102
105
  needed_memory_gib: int,
103
106
  needed_network_mbps: float,
104
- required_disk_ios,
105
- required_disk_space,
107
+ required_disk_ios: Callable[[int], float],
108
+ required_disk_space: Callable[[int], int],
106
109
  reference_shape: Instance,
107
110
  ) -> Optional[RegionClusterCapacity]:
108
111
  """Computes a regional cluster of a RDS service
@@ -239,7 +242,7 @@ class NflxRDSCapacityModel(CapacityModel):
239
242
  )
240
243
 
241
244
  @staticmethod
242
- def description():
245
+ def description() -> str:
243
246
  return "Netflix RDS Cluster Model"
244
247
 
245
248
  @staticmethod
@@ -247,7 +250,9 @@ class NflxRDSCapacityModel(CapacityModel):
247
250
  return NflxRDSArguments.model_json_schema()
248
251
 
249
252
  @staticmethod
250
- def default_desires(user_desires, extra_model_arguments):
253
+ def default_desires(
254
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
255
+ ) -> CapacityDesires:
251
256
  return CapacityDesires(
252
257
  query_pattern=QueryPattern(
253
258
  access_pattern=AccessPattern.latency,
@@ -176,7 +176,7 @@ class NflxJavaAppCapacityModel(CapacityModel):
176
176
  )
177
177
 
178
178
  @staticmethod
179
- def description():
179
+ def description() -> str:
180
180
  return "Netflix Streaming Java App Model"
181
181
 
182
182
  @staticmethod
@@ -196,7 +196,9 @@ class NflxJavaAppCapacityModel(CapacityModel):
196
196
  return regret
197
197
 
198
198
  @staticmethod
199
- def default_desires(user_desires, extra_model_arguments):
199
+ def default_desires(
200
+ user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
201
+ ) -> CapacityDesires:
200
202
  if user_desires.query_pattern.access_pattern == AccessPattern.latency:
201
203
  return CapacityDesires(
202
204
  query_pattern=QueryPattern(