service-capacity-modeling 0.3.76__py3-none-any.whl → 0.3.77__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of service-capacity-modeling might be problematic. Click here for more details.
- service_capacity_modeling/capacity_planner.py +46 -40
- service_capacity_modeling/hardware/__init__.py +11 -7
- service_capacity_modeling/interface.py +20 -18
- service_capacity_modeling/models/__init__.py +21 -2
- service_capacity_modeling/models/common.py +20 -19
- service_capacity_modeling/models/headroom_strategy.py +2 -1
- service_capacity_modeling/models/org/netflix/__init__.py +4 -1
- service_capacity_modeling/models/org/netflix/aurora.py +12 -7
- service_capacity_modeling/models/org/netflix/cassandra.py +22 -12
- service_capacity_modeling/models/org/netflix/counter.py +4 -2
- service_capacity_modeling/models/org/netflix/crdb.py +7 -4
- service_capacity_modeling/models/org/netflix/ddb.py +9 -5
- service_capacity_modeling/models/org/netflix/elasticsearch.py +8 -6
- service_capacity_modeling/models/org/netflix/entity.py +5 -3
- service_capacity_modeling/models/org/netflix/evcache.py +13 -9
- service_capacity_modeling/models/org/netflix/graphkv.py +5 -3
- service_capacity_modeling/models/org/netflix/iso_date_math.py +12 -9
- service_capacity_modeling/models/org/netflix/kafka.py +13 -7
- service_capacity_modeling/models/org/netflix/key_value.py +4 -2
- service_capacity_modeling/models/org/netflix/postgres.py +4 -2
- service_capacity_modeling/models/org/netflix/rds.py +10 -5
- service_capacity_modeling/models/org/netflix/stateless_java.py +4 -2
- service_capacity_modeling/models/org/netflix/time_series.py +4 -2
- service_capacity_modeling/models/org/netflix/time_series_config.py +3 -3
- service_capacity_modeling/models/org/netflix/wal.py +4 -2
- service_capacity_modeling/models/org/netflix/zookeeper.py +5 -3
- service_capacity_modeling/stats.py +14 -11
- service_capacity_modeling/tools/auto_shape.py +10 -6
- service_capacity_modeling/tools/fetch_pricing.py +13 -6
- service_capacity_modeling/tools/generate_missing.py +4 -3
- service_capacity_modeling/tools/instance_families.py +4 -1
- {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.77.dist-info}/METADATA +1 -1
- {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.77.dist-info}/RECORD +37 -37
- {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.77.dist-info}/WHEEL +0 -0
- {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.77.dist-info}/entry_points.txt +0 -0
- {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.77.dist-info}/licenses/LICENSE +0 -0
- {service_capacity_modeling-0.3.76.dist-info → service_capacity_modeling-0.3.77.dist-info}/top_level.txt +0 -0
|
@@ -79,7 +79,7 @@ def _write_buffer_gib_zone(
|
|
|
79
79
|
return float(write_buffer_gib) / zones_per_region
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
def _get_cores_from_desires(desires, instance):
|
|
82
|
+
def _get_cores_from_desires(desires: CapacityDesires, instance: Instance) -> int:
|
|
83
83
|
cpu_buffer = buffer_for_components(
|
|
84
84
|
buffers=desires.buffers, components=[BACKGROUND_BUFFER]
|
|
85
85
|
)
|
|
@@ -97,7 +97,7 @@ def _get_cores_from_desires(desires, instance):
|
|
|
97
97
|
return needed_cores
|
|
98
98
|
|
|
99
99
|
|
|
100
|
-
def _get_disk_from_desires(desires, copies_per_region):
|
|
100
|
+
def _get_disk_from_desires(desires: CapacityDesires, copies_per_region: int) -> int:
|
|
101
101
|
disk_buffer = buffer_for_components(
|
|
102
102
|
buffers=desires.buffers, components=[BufferComponent.disk]
|
|
103
103
|
)
|
|
@@ -116,7 +116,7 @@ def _get_min_count(
|
|
|
116
116
|
needed_disk_gib: float,
|
|
117
117
|
disk_per_node_gib: float,
|
|
118
118
|
cluster_size_lambda: Callable[[int], int],
|
|
119
|
-
):
|
|
119
|
+
) -> int:
|
|
120
120
|
"""
|
|
121
121
|
Compute the minimum number of nodes required for a zone.
|
|
122
122
|
|
|
@@ -158,7 +158,10 @@ def _get_min_count(
|
|
|
158
158
|
|
|
159
159
|
|
|
160
160
|
def _zonal_requirement_for_new_cluster(
|
|
161
|
-
desires
|
|
161
|
+
desires: CapacityDesires,
|
|
162
|
+
instance: Instance,
|
|
163
|
+
copies_per_region: int,
|
|
164
|
+
zones_per_region: int,
|
|
162
165
|
) -> CapacityRequirement:
|
|
163
166
|
needed_cores = _get_cores_from_desires(desires, instance)
|
|
164
167
|
needed_disk = _get_disk_from_desires(desires, copies_per_region)
|
|
@@ -325,14 +328,14 @@ def _estimate_cassandra_requirement(
|
|
|
325
328
|
)
|
|
326
329
|
|
|
327
330
|
|
|
328
|
-
def _get_current_cluster_size(desires) -> int:
|
|
331
|
+
def _get_current_cluster_size(desires: CapacityDesires) -> int:
|
|
329
332
|
current_capacity = _get_current_capacity(desires)
|
|
330
333
|
if current_capacity is None:
|
|
331
334
|
return 0
|
|
332
335
|
return math.ceil(current_capacity.cluster_instance_count.mid)
|
|
333
336
|
|
|
334
337
|
|
|
335
|
-
def _get_current_capacity(desires) -> Optional[CurrentClusterCapacity]:
|
|
338
|
+
def _get_current_capacity(desires: CapacityDesires) -> Optional[CurrentClusterCapacity]:
|
|
336
339
|
current_capacity = (
|
|
337
340
|
None
|
|
338
341
|
if desires.current_clusters is None
|
|
@@ -345,7 +348,7 @@ def _get_current_capacity(desires) -> Optional[CurrentClusterCapacity]:
|
|
|
345
348
|
return current_capacity
|
|
346
349
|
|
|
347
350
|
|
|
348
|
-
def _upsert_params(cluster, params):
|
|
351
|
+
def _upsert_params(cluster: Any, params: Dict[str, Any]) -> None:
|
|
349
352
|
if cluster.cluster_params:
|
|
350
353
|
cluster.cluster_params.update(params)
|
|
351
354
|
else:
|
|
@@ -575,7 +578,7 @@ def _estimate_cassandra_cluster_zonal( # pylint: disable=too-many-positional-ar
|
|
|
575
578
|
|
|
576
579
|
|
|
577
580
|
# C* LCS has 160 MiB sstables by default and 10 sstables per level
|
|
578
|
-
def _cass_io_per_read(node_size_gib, sstable_size_mb=160):
|
|
581
|
+
def _cass_io_per_read(node_size_gib: float, sstable_size_mb: int = 160) -> int:
|
|
579
582
|
gb = node_size_gib * 1024
|
|
580
583
|
sstables = max(1, gb // sstable_size_mb)
|
|
581
584
|
# 10 sstables per level, plus 1 for L0 (avg)
|
|
@@ -585,7 +588,7 @@ def _cass_io_per_read(node_size_gib, sstable_size_mb=160):
|
|
|
585
588
|
return 2 * levels
|
|
586
589
|
|
|
587
590
|
|
|
588
|
-
def _get_base_memory(desires: CapacityDesires):
|
|
591
|
+
def _get_base_memory(desires: CapacityDesires) -> float:
|
|
589
592
|
return (
|
|
590
593
|
desires.data_shape.reserved_instance_app_mem_gib
|
|
591
594
|
+ desires.data_shape.reserved_instance_system_mem_gib
|
|
@@ -679,8 +682,13 @@ class NflxCassandraArguments(BaseModel):
|
|
|
679
682
|
|
|
680
683
|
|
|
681
684
|
class NflxCassandraCapacityModel(CapacityModel):
|
|
685
|
+
def __init__(self) -> None:
|
|
686
|
+
pass
|
|
687
|
+
|
|
682
688
|
@staticmethod
|
|
683
|
-
def get_required_cluster_size(
|
|
689
|
+
def get_required_cluster_size(
|
|
690
|
+
tier: int, extra_model_arguments: Dict[str, Any]
|
|
691
|
+
) -> Optional[int]:
|
|
684
692
|
required_cluster_size: Optional[int] = (
|
|
685
693
|
math.ceil(extra_model_arguments["required_cluster_size"])
|
|
686
694
|
if "required_cluster_size" in extra_model_arguments
|
|
@@ -775,7 +783,7 @@ class NflxCassandraCapacityModel(CapacityModel):
|
|
|
775
783
|
)
|
|
776
784
|
|
|
777
785
|
@staticmethod
|
|
778
|
-
def description():
|
|
786
|
+
def description() -> str:
|
|
779
787
|
return "Netflix Streaming Cassandra Model"
|
|
780
788
|
|
|
781
789
|
@staticmethod
|
|
@@ -803,7 +811,9 @@ class NflxCassandraCapacityModel(CapacityModel):
|
|
|
803
811
|
)
|
|
804
812
|
|
|
805
813
|
@staticmethod
|
|
806
|
-
def default_desires(
|
|
814
|
+
def default_desires(
|
|
815
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
816
|
+
) -> CapacityDesires:
|
|
807
817
|
acceptable_consistency = {
|
|
808
818
|
None,
|
|
809
819
|
AccessConsistency.best_effort,
|
|
@@ -81,7 +81,7 @@ class NflxCounterCapacityModel(CapacityModel):
|
|
|
81
81
|
return counter_app
|
|
82
82
|
|
|
83
83
|
@staticmethod
|
|
84
|
-
def description():
|
|
84
|
+
def description() -> str:
|
|
85
85
|
return "Netflix Streaming Counter Model"
|
|
86
86
|
|
|
87
87
|
@staticmethod
|
|
@@ -129,7 +129,9 @@ class NflxCounterCapacityModel(CapacityModel):
|
|
|
129
129
|
return tuple(stores)
|
|
130
130
|
|
|
131
131
|
@staticmethod
|
|
132
|
-
def default_desires(
|
|
132
|
+
def default_desires(
|
|
133
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
134
|
+
) -> CapacityDesires:
|
|
133
135
|
if user_desires.query_pattern.access_pattern == AccessPattern.latency:
|
|
134
136
|
return CapacityDesires(
|
|
135
137
|
query_pattern=QueryPattern(
|
|
@@ -29,6 +29,7 @@ from service_capacity_modeling.interface import Interval
|
|
|
29
29
|
from service_capacity_modeling.interface import QueryPattern
|
|
30
30
|
from service_capacity_modeling.interface import RegionContext
|
|
31
31
|
from service_capacity_modeling.interface import Requirements
|
|
32
|
+
from service_capacity_modeling.interface import ZoneClusterCapacity
|
|
32
33
|
from service_capacity_modeling.models import CapacityModel
|
|
33
34
|
from service_capacity_modeling.models.common import buffer_for_components
|
|
34
35
|
from service_capacity_modeling.models.common import compute_stateful_zone
|
|
@@ -44,7 +45,7 @@ logger = logging.getLogger(__name__)
|
|
|
44
45
|
|
|
45
46
|
# Pebble does Leveled compaction with tieres of size??
|
|
46
47
|
# (FIXME) What does pebble actually do
|
|
47
|
-
def _crdb_io_per_read(node_size_gib, sstable_size_mb=1000):
|
|
48
|
+
def _crdb_io_per_read(node_size_gib: float, sstable_size_mb: int = 1000) -> int:
|
|
48
49
|
gb = node_size_gib * 1024
|
|
49
50
|
sstables = max(1, gb // sstable_size_mb)
|
|
50
51
|
# 10 sstables per level, plus 1 for L0 (avg)
|
|
@@ -128,7 +129,7 @@ def _estimate_cockroachdb_requirement( # noqa=E501 pylint: disable=too-many-pos
|
|
|
128
129
|
)
|
|
129
130
|
|
|
130
131
|
|
|
131
|
-
def _upsert_params(cluster, params):
|
|
132
|
+
def _upsert_params(cluster: ZoneClusterCapacity, params: Dict[str, Any]) -> None:
|
|
132
133
|
if cluster.cluster_params:
|
|
133
134
|
cluster.cluster_params.update(params)
|
|
134
135
|
else:
|
|
@@ -329,7 +330,7 @@ class NflxCockroachDBCapacityModel(CapacityModel):
|
|
|
329
330
|
)
|
|
330
331
|
|
|
331
332
|
@staticmethod
|
|
332
|
-
def description():
|
|
333
|
+
def description() -> str:
|
|
333
334
|
return "Netflix Streaming CockroachDB Model"
|
|
334
335
|
|
|
335
336
|
@staticmethod
|
|
@@ -337,7 +338,9 @@ class NflxCockroachDBCapacityModel(CapacityModel):
|
|
|
337
338
|
return NflxCockroachDBArguments.model_json_schema()
|
|
338
339
|
|
|
339
340
|
@staticmethod
|
|
340
|
-
def default_desires(
|
|
341
|
+
def default_desires(
|
|
342
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
343
|
+
) -> CapacityDesires:
|
|
341
344
|
acceptable_consistency = {
|
|
342
345
|
None,
|
|
343
346
|
AccessConsistency.linearizable,
|
|
@@ -139,11 +139,12 @@ def _get_read_consistency_percentages(
|
|
|
139
139
|
)
|
|
140
140
|
if total_percent == 0:
|
|
141
141
|
access_consistency = desires.query_pattern.access_consistency.same_region
|
|
142
|
-
|
|
142
|
+
target_consistency = access_consistency.target_consistency
|
|
143
|
+
if target_consistency == AccessConsistency.serializable:
|
|
143
144
|
transactional_read_percent = 1.0
|
|
144
145
|
eventual_read_percent = 0.0
|
|
145
146
|
strong_read_percent = 0.0
|
|
146
|
-
elif
|
|
147
|
+
elif target_consistency in (
|
|
147
148
|
AccessConsistency.read_your_writes,
|
|
148
149
|
AccessConsistency.linearizable,
|
|
149
150
|
):
|
|
@@ -184,7 +185,8 @@ def _get_write_consistency_percentages(
|
|
|
184
185
|
total_percent = transactional_write_percent + non_transactional_write_percent
|
|
185
186
|
if total_percent == 0:
|
|
186
187
|
access_consistency = desires.query_pattern.access_consistency.same_region
|
|
187
|
-
|
|
188
|
+
target_consistency = access_consistency.target_consistency
|
|
189
|
+
if target_consistency == AccessConsistency.serializable:
|
|
188
190
|
transactional_write_percent = 1.0
|
|
189
191
|
non_transactional_write_percent = 0.0
|
|
190
192
|
else:
|
|
@@ -547,7 +549,7 @@ class NflxDynamoDBCapacityModel(CapacityModel):
|
|
|
547
549
|
)
|
|
548
550
|
|
|
549
551
|
@staticmethod
|
|
550
|
-
def description():
|
|
552
|
+
def description() -> str:
|
|
551
553
|
return "Netflix Streaming DynamoDB Model"
|
|
552
554
|
|
|
553
555
|
@staticmethod
|
|
@@ -559,7 +561,9 @@ class NflxDynamoDBCapacityModel(CapacityModel):
|
|
|
559
561
|
return False
|
|
560
562
|
|
|
561
563
|
@staticmethod
|
|
562
|
-
def default_desires(
|
|
564
|
+
def default_desires(
|
|
565
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
566
|
+
) -> CapacityDesires:
|
|
563
567
|
acceptable_consistency = {
|
|
564
568
|
"same_region": {
|
|
565
569
|
None,
|
|
@@ -58,7 +58,7 @@ def _target_rf(desires: CapacityDesires, user_copies: Optional[int]) -> int:
|
|
|
58
58
|
# segments of 512 megs per
|
|
59
59
|
# https://lucene.apache.org/core/8_1_0/core/org/apache/lucene/index/TieredMergePolicy.html#setSegmentsPerTier(double)
|
|
60
60
|
# (FIXME) Verify what elastic merge actually does
|
|
61
|
-
def _es_io_per_read(node_size_gib, segment_size_mb=512):
|
|
61
|
+
def _es_io_per_read(node_size_gib: float, segment_size_mb: int = 512) -> int:
|
|
62
62
|
size_mib = node_size_gib * 1024
|
|
63
63
|
segments = max(1, size_mib // segment_size_mb)
|
|
64
64
|
# 10 segments per tier, plus 1 for L0 (avg)
|
|
@@ -75,7 +75,7 @@ def _estimate_elasticsearch_requirement( # noqa: E501 pylint: disable=too-many-
|
|
|
75
75
|
max_rps_to_disk: int,
|
|
76
76
|
zones_in_region: int = 3,
|
|
77
77
|
copies_per_region: int = 3,
|
|
78
|
-
jvm_memory_overhead=1.2,
|
|
78
|
+
jvm_memory_overhead: float = 1.2,
|
|
79
79
|
) -> CapacityRequirement:
|
|
80
80
|
"""Estimate the capacity required for one zone given a regional desire
|
|
81
81
|
|
|
@@ -153,7 +153,7 @@ def _estimate_elasticsearch_requirement( # noqa: E501 pylint: disable=too-many-
|
|
|
153
153
|
)
|
|
154
154
|
|
|
155
155
|
|
|
156
|
-
def _upsert_params(cluster, params):
|
|
156
|
+
def _upsert_params(cluster: ZoneClusterCapacity, params: Dict[str, Any]) -> None:
|
|
157
157
|
if cluster.cluster_params:
|
|
158
158
|
cluster.cluster_params.update(params)
|
|
159
159
|
else:
|
|
@@ -194,7 +194,7 @@ class NflxElasticsearchDataCapacityModel(CapacityModel):
|
|
|
194
194
|
|
|
195
195
|
@staticmethod
|
|
196
196
|
def default_desires(
|
|
197
|
-
user_desires, extra_model_arguments: Dict[str, Any]
|
|
197
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
198
198
|
) -> CapacityDesires:
|
|
199
199
|
desires = CapacityModel.default_desires(user_desires, extra_model_arguments)
|
|
200
200
|
desires.buffers = NflxElasticsearchDataCapacityModel.default_buffers()
|
|
@@ -452,7 +452,7 @@ class NflxElasticsearchCapacityModel(CapacityModel):
|
|
|
452
452
|
return None
|
|
453
453
|
|
|
454
454
|
@staticmethod
|
|
455
|
-
def description():
|
|
455
|
+
def description() -> str:
|
|
456
456
|
return "Netflix Streaming Elasticsearch Model"
|
|
457
457
|
|
|
458
458
|
@staticmethod
|
|
@@ -482,7 +482,9 @@ class NflxElasticsearchCapacityModel(CapacityModel):
|
|
|
482
482
|
return NflxElasticsearchArguments.model_json_schema()
|
|
483
483
|
|
|
484
484
|
@staticmethod
|
|
485
|
-
def default_desires(
|
|
485
|
+
def default_desires(
|
|
486
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
487
|
+
) -> CapacityDesires:
|
|
486
488
|
acceptable_consistency = {
|
|
487
489
|
AccessConsistency.best_effort,
|
|
488
490
|
AccessConsistency.eventual,
|
|
@@ -48,7 +48,7 @@ class NflxEntityCapacityModel(CapacityModel):
|
|
|
48
48
|
return entity_app
|
|
49
49
|
|
|
50
50
|
@staticmethod
|
|
51
|
-
def description():
|
|
51
|
+
def description() -> str:
|
|
52
52
|
return "Netflix Streaming Entity Model"
|
|
53
53
|
|
|
54
54
|
@staticmethod
|
|
@@ -76,7 +76,7 @@ class NflxEntityCapacityModel(CapacityModel):
|
|
|
76
76
|
/ 1024**3
|
|
77
77
|
)
|
|
78
78
|
else:
|
|
79
|
-
item_size_gib = 10 / 1024**2
|
|
79
|
+
item_size_gib = 10 / 1024**2 # type: ignore[unreachable]
|
|
80
80
|
item_count = user_desires.data_shape.estimated_state_size_gib.scale(
|
|
81
81
|
1 / item_size_gib
|
|
82
82
|
)
|
|
@@ -102,7 +102,9 @@ class NflxEntityCapacityModel(CapacityModel):
|
|
|
102
102
|
)
|
|
103
103
|
|
|
104
104
|
@staticmethod
|
|
105
|
-
def default_desires(
|
|
105
|
+
def default_desires(
|
|
106
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
107
|
+
) -> CapacityDesires:
|
|
106
108
|
if user_desires.query_pattern.access_pattern == AccessPattern.latency:
|
|
107
109
|
return CapacityDesires(
|
|
108
110
|
query_pattern=QueryPattern(
|
|
@@ -69,7 +69,9 @@ def calculate_read_cpu_time_evcache_ms(read_size_bytes: float) -> float:
|
|
|
69
69
|
return max(read_latency_ms, 0.005)
|
|
70
70
|
|
|
71
71
|
|
|
72
|
-
def calculate_spread_cost(
|
|
72
|
+
def calculate_spread_cost(
|
|
73
|
+
cluster_size: int, max_cost: float = 100000, min_cost: float = 0.0
|
|
74
|
+
) -> float:
|
|
73
75
|
if cluster_size > 10:
|
|
74
76
|
return min_cost
|
|
75
77
|
if cluster_size < 2:
|
|
@@ -82,14 +84,14 @@ def calculate_vitals_for_capacity_planner(
|
|
|
82
84
|
instance: Instance,
|
|
83
85
|
current_memory_gib: float,
|
|
84
86
|
current_disk_gib: float,
|
|
85
|
-
):
|
|
87
|
+
) -> Tuple[float, float, float, float]:
|
|
86
88
|
# First calculate assuming new deployment
|
|
87
89
|
needed_cores = normalize_cores(
|
|
88
90
|
core_count=sqrt_staffed_cores(desires),
|
|
89
91
|
target_shape=instance,
|
|
90
92
|
reference_shape=desires.reference_shape,
|
|
91
93
|
)
|
|
92
|
-
needed_network_mbps = simple_network_mbps(desires)
|
|
94
|
+
needed_network_mbps = float(simple_network_mbps(desires))
|
|
93
95
|
needed_memory_gib = current_memory_gib
|
|
94
96
|
needed_disk_gib = current_disk_gib
|
|
95
97
|
|
|
@@ -110,8 +112,8 @@ def calculate_vitals_for_capacity_planner(
|
|
|
110
112
|
target_shape=instance,
|
|
111
113
|
reference_shape=current_capacity.cluster_instance,
|
|
112
114
|
)
|
|
113
|
-
needed_network_mbps = requirements.network_mbps
|
|
114
|
-
needed_disk_gib = (
|
|
115
|
+
needed_network_mbps = float(requirements.network_mbps)
|
|
116
|
+
needed_disk_gib = float(
|
|
115
117
|
requirements.disk_gib if current_capacity.cluster_drive is not None else 0.0
|
|
116
118
|
)
|
|
117
119
|
needed_memory_gib = requirements.mem_gib
|
|
@@ -196,7 +198,7 @@ def _estimate_evcache_requirement(
|
|
|
196
198
|
)
|
|
197
199
|
|
|
198
200
|
|
|
199
|
-
def _upsert_params(cluster, params):
|
|
201
|
+
def _upsert_params(cluster: Any, params: Dict[str, Any]) -> None:
|
|
200
202
|
if cluster.cluster_params:
|
|
201
203
|
cluster.cluster_params.update(params)
|
|
202
204
|
else:
|
|
@@ -263,7 +265,7 @@ def _estimate_evcache_cluster_zonal( # noqa: C901,E501 pylint: disable=too-many
|
|
|
263
265
|
# larger to account for additional overhead. Note that the
|
|
264
266
|
# reserved_instance_system_mem_gib has a base of 1 GiB OSMEM so this
|
|
265
267
|
# just represents the variable component
|
|
266
|
-
def reserve_memory(instance_mem_gib):
|
|
268
|
+
def reserve_memory(instance_mem_gib: float) -> float:
|
|
267
269
|
# (Joey) From the chart it appears to be about a 3% overhead for
|
|
268
270
|
# OS memory.
|
|
269
271
|
variable_os = int(instance_mem_gib * 0.03)
|
|
@@ -439,7 +441,7 @@ class NflxEVCacheCapacityModel(CapacityModel):
|
|
|
439
441
|
)
|
|
440
442
|
|
|
441
443
|
@staticmethod
|
|
442
|
-
def description():
|
|
444
|
+
def description() -> str:
|
|
443
445
|
return "Netflix Streaming EVCache (memcached) Model"
|
|
444
446
|
|
|
445
447
|
@staticmethod
|
|
@@ -447,7 +449,9 @@ class NflxEVCacheCapacityModel(CapacityModel):
|
|
|
447
449
|
return NflxEVCacheArguments.model_json_schema()
|
|
448
450
|
|
|
449
451
|
@staticmethod
|
|
450
|
-
def default_desires(
|
|
452
|
+
def default_desires(
|
|
453
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
454
|
+
) -> CapacityDesires:
|
|
451
455
|
acceptable_consistency = {
|
|
452
456
|
AccessConsistency.best_effort,
|
|
453
457
|
AccessConsistency.never,
|
|
@@ -45,7 +45,7 @@ class NflxGraphKVCapacityModel(CapacityModel):
|
|
|
45
45
|
return graphkv_app
|
|
46
46
|
|
|
47
47
|
@staticmethod
|
|
48
|
-
def description():
|
|
48
|
+
def description() -> str:
|
|
49
49
|
return "Netflix Streaming Graph Abstraction"
|
|
50
50
|
|
|
51
51
|
@staticmethod
|
|
@@ -89,7 +89,7 @@ class NflxGraphKVCapacityModel(CapacityModel):
|
|
|
89
89
|
/ 1024**3
|
|
90
90
|
)
|
|
91
91
|
else:
|
|
92
|
-
item_size_gib = 1 / 1024**2
|
|
92
|
+
item_size_gib = 1 / 1024**2 # type: ignore[unreachable]
|
|
93
93
|
item_count = user_desires.data_shape.estimated_state_size_gib.scale(
|
|
94
94
|
1 / item_size_gib
|
|
95
95
|
)
|
|
@@ -102,7 +102,9 @@ class NflxGraphKVCapacityModel(CapacityModel):
|
|
|
102
102
|
return (("org.netflix.key-value", _modify_kv_desires),)
|
|
103
103
|
|
|
104
104
|
@staticmethod
|
|
105
|
-
def default_desires(
|
|
105
|
+
def default_desires(
|
|
106
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
107
|
+
) -> CapacityDesires:
|
|
106
108
|
if user_desires.query_pattern.access_pattern == AccessPattern.latency:
|
|
107
109
|
return CapacityDesires(
|
|
108
110
|
query_pattern=QueryPattern(
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import math
|
|
2
2
|
from datetime import timedelta
|
|
3
3
|
from decimal import Decimal
|
|
4
|
+
from typing import Union
|
|
4
5
|
|
|
5
|
-
import isodate
|
|
6
|
+
import isodate # type: ignore
|
|
6
7
|
from isodate import parse_duration
|
|
7
8
|
|
|
8
9
|
DURATION_1M = 2592000
|
|
@@ -11,24 +12,26 @@ D_YEAR = Decimal(DURATION_1Y)
|
|
|
11
12
|
D_MONTH = Decimal(DURATION_1M)
|
|
12
13
|
|
|
13
14
|
|
|
14
|
-
def _iso_to_proto_duration(iso_duration: str):
|
|
15
|
+
def _iso_to_proto_duration(iso_duration: str) -> str:
|
|
15
16
|
parsed = parse_duration(iso_duration)
|
|
16
17
|
return f"{int(parsed.total_seconds())}s"
|
|
17
18
|
|
|
18
19
|
|
|
19
|
-
def iso_to_seconds(iso_duration: str, unlimited=0) -> int:
|
|
20
|
+
def iso_to_seconds(iso_duration: str, unlimited: int = 0) -> int:
|
|
20
21
|
if iso_duration == "unlimited":
|
|
21
22
|
return unlimited
|
|
22
|
-
parsed = isodate.parse_duration(iso_duration)
|
|
23
|
+
parsed: Union[timedelta, isodate.Duration] = isodate.parse_duration(iso_duration)
|
|
23
24
|
# isodate package parse_duration returns either timedelta OR Duration
|
|
24
25
|
if isinstance(parsed, isodate.Duration):
|
|
25
|
-
return
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
26
|
+
return int(
|
|
27
|
+
math.ceil(
|
|
28
|
+
parsed.years * D_YEAR
|
|
29
|
+
+ parsed.months * D_MONTH
|
|
30
|
+
+ Decimal(parsed.tdelta.total_seconds())
|
|
31
|
+
)
|
|
29
32
|
)
|
|
30
33
|
return int(parsed.total_seconds())
|
|
31
34
|
|
|
32
35
|
|
|
33
|
-
def _iso_to_timedelta(iso_duration: str, unlimited=0) -> timedelta:
|
|
36
|
+
def _iso_to_timedelta(iso_duration: str, unlimited: int = 0) -> timedelta:
|
|
34
37
|
return timedelta(seconds=iso_to_seconds(iso_duration, unlimited))
|
|
@@ -64,7 +64,9 @@ def _get_current_zonal_cluster(
|
|
|
64
64
|
)
|
|
65
65
|
|
|
66
66
|
|
|
67
|
-
def _is_same_instance_family(
|
|
67
|
+
def _is_same_instance_family(
|
|
68
|
+
cluster: Optional[CurrentZoneClusterCapacity], target_family: str
|
|
69
|
+
) -> bool:
|
|
68
70
|
"""Check if cluster has a different instance family than the target."""
|
|
69
71
|
if not cluster or not cluster.cluster_instance:
|
|
70
72
|
return False
|
|
@@ -213,14 +215,16 @@ def _estimate_kafka_requirement( # pylint: disable=too-many-positional-argument
|
|
|
213
215
|
)
|
|
214
216
|
|
|
215
217
|
|
|
216
|
-
def _upsert_params(cluster, params):
|
|
218
|
+
def _upsert_params(cluster: Any, params: Dict[str, Any]) -> None:
|
|
217
219
|
if cluster.cluster_params:
|
|
218
220
|
cluster.cluster_params.update(params)
|
|
219
221
|
else:
|
|
220
222
|
cluster.cluster_params = params
|
|
221
223
|
|
|
222
224
|
|
|
223
|
-
def _kafka_read_io(
|
|
225
|
+
def _kafka_read_io(
|
|
226
|
+
rps: float, io_size_kib: float, size_gib: float, recovery_seconds: int
|
|
227
|
+
) -> float:
|
|
224
228
|
# Get enough disk read IO capacity for some reads
|
|
225
229
|
# In practice we have cache reducing this by 99% or more
|
|
226
230
|
read_ios = rps * 0.05
|
|
@@ -229,7 +233,7 @@ def _kafka_read_io(rps, io_size_kib, size_gib, recovery_seconds: int) -> float:
|
|
|
229
233
|
size_kib = size_gib * 0.5 * (1024 * 1024)
|
|
230
234
|
recovery_ios = max(1, size_kib / io_size_kib) / recovery_seconds
|
|
231
235
|
# Leave 50% headroom for read IOs since generally we will hit cache
|
|
232
|
-
return (read_ios + int(round(recovery_ios))) * 1.5
|
|
236
|
+
return float((read_ios + int(round(recovery_ios))) * 1.5)
|
|
233
237
|
|
|
234
238
|
|
|
235
239
|
# pylint: disable=too-many-locals
|
|
@@ -239,7 +243,7 @@ def _estimate_kafka_cluster_zonal( # noqa: C901
|
|
|
239
243
|
instance: Instance,
|
|
240
244
|
drive: Drive,
|
|
241
245
|
desires: CapacityDesires,
|
|
242
|
-
hot_retention_seconds,
|
|
246
|
+
hot_retention_seconds: float,
|
|
243
247
|
zones_per_region: int = 3,
|
|
244
248
|
copies_per_region: int = 2,
|
|
245
249
|
require_local_disks: bool = False,
|
|
@@ -532,7 +536,7 @@ class NflxKafkaCapacityModel(CapacityModel):
|
|
|
532
536
|
)
|
|
533
537
|
|
|
534
538
|
@staticmethod
|
|
535
|
-
def description():
|
|
539
|
+
def description() -> str:
|
|
536
540
|
return "Netflix Streaming Kafka Model"
|
|
537
541
|
|
|
538
542
|
@staticmethod
|
|
@@ -545,7 +549,9 @@ class NflxKafkaCapacityModel(CapacityModel):
|
|
|
545
549
|
return ("gp3",)
|
|
546
550
|
|
|
547
551
|
@staticmethod
|
|
548
|
-
def default_desires(
|
|
552
|
+
def default_desires(
|
|
553
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
554
|
+
) -> CapacityDesires:
|
|
549
555
|
# Default to 10MiB/s and a single reader
|
|
550
556
|
concurrent_readers = max(
|
|
551
557
|
1, int(user_desires.query_pattern.estimated_read_per_second.mid)
|
|
@@ -48,7 +48,7 @@ class NflxKeyValueCapacityModel(CapacityModel):
|
|
|
48
48
|
return kv_app
|
|
49
49
|
|
|
50
50
|
@staticmethod
|
|
51
|
-
def description():
|
|
51
|
+
def description() -> str:
|
|
52
52
|
return "Netflix Streaming Key-Value Model"
|
|
53
53
|
|
|
54
54
|
@staticmethod
|
|
@@ -119,7 +119,9 @@ class NflxKeyValueCapacityModel(CapacityModel):
|
|
|
119
119
|
return (("org.netflix.cassandra", lambda x: x),)
|
|
120
120
|
|
|
121
121
|
@staticmethod
|
|
122
|
-
def default_desires(
|
|
122
|
+
def default_desires(
|
|
123
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
124
|
+
) -> CapacityDesires:
|
|
123
125
|
if user_desires.query_pattern.access_pattern == AccessPattern.latency:
|
|
124
126
|
return CapacityDesires(
|
|
125
127
|
query_pattern=QueryPattern(
|
|
@@ -45,7 +45,7 @@ class NflxPostgresCapacityModel(CapacityModel):
|
|
|
45
45
|
return plan
|
|
46
46
|
|
|
47
47
|
@staticmethod
|
|
48
|
-
def description():
|
|
48
|
+
def description() -> str:
|
|
49
49
|
return "Netflix Postgres Model"
|
|
50
50
|
|
|
51
51
|
@staticmethod
|
|
@@ -53,7 +53,9 @@ class NflxPostgresCapacityModel(CapacityModel):
|
|
|
53
53
|
return Platform.aurora_postgres, Platform.amd64
|
|
54
54
|
|
|
55
55
|
@staticmethod
|
|
56
|
-
def default_desires(
|
|
56
|
+
def default_desires(
|
|
57
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
58
|
+
) -> CapacityDesires:
|
|
57
59
|
return CapacityDesires(
|
|
58
60
|
query_pattern=QueryPattern(
|
|
59
61
|
access_pattern=AccessPattern.latency,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import math
|
|
3
3
|
from typing import Any
|
|
4
|
+
from typing import Callable
|
|
4
5
|
from typing import Dict
|
|
5
6
|
from typing import Optional
|
|
6
7
|
|
|
@@ -83,7 +84,9 @@ def _estimate_rds_requirement(
|
|
|
83
84
|
|
|
84
85
|
# MySQL default block size is 16KiB, PostGreSQL is 8KiB Number of reads for B-Tree
|
|
85
86
|
# are given by log of total pages to the base of B-Tree fan out factor
|
|
86
|
-
def _rds_required_disk_ios(
|
|
87
|
+
def _rds_required_disk_ios(
|
|
88
|
+
disk_size_gib: int, db_type: str, btree_fan_out: int = 100
|
|
89
|
+
) -> float:
|
|
87
90
|
disk_size_kb = disk_size_gib * 1024 * 1024
|
|
88
91
|
if db_type == "postgres":
|
|
89
92
|
default_block_size = 8 # KiB
|
|
@@ -101,8 +104,8 @@ def _compute_rds_region( # pylint: disable=too-many-positional-arguments
|
|
|
101
104
|
needed_disk_gib: int,
|
|
102
105
|
needed_memory_gib: int,
|
|
103
106
|
needed_network_mbps: float,
|
|
104
|
-
required_disk_ios,
|
|
105
|
-
required_disk_space,
|
|
107
|
+
required_disk_ios: Callable[[int], float],
|
|
108
|
+
required_disk_space: Callable[[int], int],
|
|
106
109
|
reference_shape: Instance,
|
|
107
110
|
) -> Optional[RegionClusterCapacity]:
|
|
108
111
|
"""Computes a regional cluster of a RDS service
|
|
@@ -239,7 +242,7 @@ class NflxRDSCapacityModel(CapacityModel):
|
|
|
239
242
|
)
|
|
240
243
|
|
|
241
244
|
@staticmethod
|
|
242
|
-
def description():
|
|
245
|
+
def description() -> str:
|
|
243
246
|
return "Netflix RDS Cluster Model"
|
|
244
247
|
|
|
245
248
|
@staticmethod
|
|
@@ -247,7 +250,9 @@ class NflxRDSCapacityModel(CapacityModel):
|
|
|
247
250
|
return NflxRDSArguments.model_json_schema()
|
|
248
251
|
|
|
249
252
|
@staticmethod
|
|
250
|
-
def default_desires(
|
|
253
|
+
def default_desires(
|
|
254
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
255
|
+
) -> CapacityDesires:
|
|
251
256
|
return CapacityDesires(
|
|
252
257
|
query_pattern=QueryPattern(
|
|
253
258
|
access_pattern=AccessPattern.latency,
|
|
@@ -176,7 +176,7 @@ class NflxJavaAppCapacityModel(CapacityModel):
|
|
|
176
176
|
)
|
|
177
177
|
|
|
178
178
|
@staticmethod
|
|
179
|
-
def description():
|
|
179
|
+
def description() -> str:
|
|
180
180
|
return "Netflix Streaming Java App Model"
|
|
181
181
|
|
|
182
182
|
@staticmethod
|
|
@@ -196,7 +196,9 @@ class NflxJavaAppCapacityModel(CapacityModel):
|
|
|
196
196
|
return regret
|
|
197
197
|
|
|
198
198
|
@staticmethod
|
|
199
|
-
def default_desires(
|
|
199
|
+
def default_desires(
|
|
200
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
201
|
+
) -> CapacityDesires:
|
|
200
202
|
if user_desires.query_pattern.access_pattern == AccessPattern.latency:
|
|
201
203
|
return CapacityDesires(
|
|
202
204
|
query_pattern=QueryPattern(
|
|
@@ -49,7 +49,7 @@ class NflxTimeSeriesCapacityModel(CapacityModel):
|
|
|
49
49
|
return ts_app
|
|
50
50
|
|
|
51
51
|
@staticmethod
|
|
52
|
-
def description():
|
|
52
|
+
def description() -> str:
|
|
53
53
|
return "Netflix Streaming TimeSeries Model"
|
|
54
54
|
|
|
55
55
|
@staticmethod
|
|
@@ -91,7 +91,9 @@ class NflxTimeSeriesCapacityModel(CapacityModel):
|
|
|
91
91
|
return (("org.netflix.cassandra", _modify_cassandra_desires),)
|
|
92
92
|
|
|
93
93
|
@staticmethod
|
|
94
|
-
def default_desires(
|
|
94
|
+
def default_desires(
|
|
95
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
96
|
+
) -> CapacityDesires:
|
|
95
97
|
if user_desires.query_pattern.access_pattern == AccessPattern.latency:
|
|
96
98
|
return CapacityDesires(
|
|
97
99
|
query_pattern=QueryPattern(
|