service-capacity-modeling 0.3.73__py3-none-any.whl → 0.3.79__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of service-capacity-modeling might be problematic. Click here for more details.
- service_capacity_modeling/capacity_planner.py +46 -40
- service_capacity_modeling/hardware/__init__.py +11 -7
- service_capacity_modeling/hardware/profiles/shapes/aws/auto_i3en.json +172 -0
- service_capacity_modeling/hardware/profiles/shapes/aws/auto_i4i.json +220 -0
- service_capacity_modeling/hardware/profiles/shapes/aws/manual_instances.json +0 -184
- service_capacity_modeling/interface.py +48 -22
- service_capacity_modeling/models/__init__.py +21 -2
- service_capacity_modeling/models/common.py +268 -190
- service_capacity_modeling/models/headroom_strategy.py +2 -1
- service_capacity_modeling/models/org/netflix/__init__.py +4 -1
- service_capacity_modeling/models/org/netflix/aurora.py +12 -7
- service_capacity_modeling/models/org/netflix/cassandra.py +39 -24
- service_capacity_modeling/models/org/netflix/counter.py +44 -20
- service_capacity_modeling/models/org/netflix/crdb.py +7 -4
- service_capacity_modeling/models/org/netflix/ddb.py +9 -5
- service_capacity_modeling/models/org/netflix/elasticsearch.py +8 -6
- service_capacity_modeling/models/org/netflix/entity.py +5 -3
- service_capacity_modeling/models/org/netflix/evcache.py +21 -25
- service_capacity_modeling/models/org/netflix/graphkv.py +5 -3
- service_capacity_modeling/models/org/netflix/iso_date_math.py +12 -9
- service_capacity_modeling/models/org/netflix/kafka.py +13 -7
- service_capacity_modeling/models/org/netflix/key_value.py +4 -2
- service_capacity_modeling/models/org/netflix/postgres.py +4 -2
- service_capacity_modeling/models/org/netflix/rds.py +10 -5
- service_capacity_modeling/models/org/netflix/stateless_java.py +4 -2
- service_capacity_modeling/models/org/netflix/time_series.py +4 -2
- service_capacity_modeling/models/org/netflix/time_series_config.py +3 -3
- service_capacity_modeling/models/org/netflix/wal.py +4 -2
- service_capacity_modeling/models/org/netflix/zookeeper.py +5 -3
- service_capacity_modeling/stats.py +14 -11
- service_capacity_modeling/tools/auto_shape.py +10 -6
- service_capacity_modeling/tools/fetch_pricing.py +13 -6
- service_capacity_modeling/tools/generate_missing.py +4 -3
- service_capacity_modeling/tools/instance_families.py +18 -7
- {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/METADATA +9 -5
- {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/RECORD +40 -38
- {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/WHEEL +0 -0
- {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/entry_points.txt +0 -0
- {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/licenses/LICENSE +0 -0
- {service_capacity_modeling-0.3.73.dist-info → service_capacity_modeling-0.3.79.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import math
|
|
3
3
|
from typing import Any
|
|
4
|
+
from typing import Callable
|
|
4
5
|
from typing import Dict
|
|
5
6
|
from typing import Optional
|
|
6
7
|
from typing import Tuple
|
|
@@ -86,7 +87,9 @@ def _estimate_aurora_requirement(
|
|
|
86
87
|
|
|
87
88
|
# MySQL default block size is 16KiB, PostGreSQL is 8KiB Number of reads for B-Tree
|
|
88
89
|
# are given by log of total pages to the base of B-Tree fan out factor
|
|
89
|
-
def _rds_required_disk_ios(
|
|
90
|
+
def _rds_required_disk_ios(
|
|
91
|
+
disk_size_gib: int, db_type: str, btree_fan_out: int = 100
|
|
92
|
+
) -> float:
|
|
90
93
|
disk_size_kb = disk_size_gib * 1024 * 1024
|
|
91
94
|
if db_type == "postgres":
|
|
92
95
|
default_block_size = 8 # KiB
|
|
@@ -100,11 +103,11 @@ def _rds_required_disk_ios(disk_size_gib: int, db_type: str, btree_fan_out: int
|
|
|
100
103
|
# This is a start, we should iterate based on the actual work load
|
|
101
104
|
def _estimate_io_cost(
|
|
102
105
|
db_type: str,
|
|
103
|
-
desires,
|
|
106
|
+
desires: CapacityDesires,
|
|
104
107
|
read_io_price: float,
|
|
105
108
|
write_io_price: float,
|
|
106
109
|
cache_hit_rate: float = 0.8,
|
|
107
|
-
):
|
|
110
|
+
) -> float:
|
|
108
111
|
if db_type == "postgres":
|
|
109
112
|
read_byte_per_io = 8192
|
|
110
113
|
else:
|
|
@@ -134,8 +137,8 @@ def _compute_aurora_region( # pylint: disable=too-many-positional-arguments
|
|
|
134
137
|
needed_disk_gib: int,
|
|
135
138
|
needed_memory_gib: int,
|
|
136
139
|
needed_network_mbps: float,
|
|
137
|
-
required_disk_ios,
|
|
138
|
-
required_disk_space,
|
|
140
|
+
required_disk_ios: Callable[[int], float],
|
|
141
|
+
required_disk_space: Callable[[int], float],
|
|
139
142
|
db_type: str,
|
|
140
143
|
desires: CapacityDesires,
|
|
141
144
|
) -> Optional[RegionClusterCapacity]:
|
|
@@ -295,7 +298,7 @@ class NflxAuroraCapacityModel(CapacityModel):
|
|
|
295
298
|
)
|
|
296
299
|
|
|
297
300
|
@staticmethod
|
|
298
|
-
def description():
|
|
301
|
+
def description() -> str:
|
|
299
302
|
return "Netflix Aurora Cluster Model"
|
|
300
303
|
|
|
301
304
|
@staticmethod
|
|
@@ -307,7 +310,9 @@ class NflxAuroraCapacityModel(CapacityModel):
|
|
|
307
310
|
return Platform.aurora_mysql, Platform.aurora_mysql
|
|
308
311
|
|
|
309
312
|
@staticmethod
|
|
310
|
-
def default_desires(
|
|
313
|
+
def default_desires(
|
|
314
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
315
|
+
) -> CapacityDesires:
|
|
311
316
|
return CapacityDesires(
|
|
312
317
|
query_pattern=QueryPattern(
|
|
313
318
|
access_pattern=AccessPattern.latency,
|
|
@@ -35,7 +35,7 @@ from service_capacity_modeling.interface import ServiceCapacity
|
|
|
35
35
|
from service_capacity_modeling.models import CapacityModel
|
|
36
36
|
from service_capacity_modeling.models.common import buffer_for_components
|
|
37
37
|
from service_capacity_modeling.models.common import compute_stateful_zone
|
|
38
|
-
from service_capacity_modeling.models.common import
|
|
38
|
+
from service_capacity_modeling.models.common import DerivedBuffers
|
|
39
39
|
from service_capacity_modeling.models.common import get_effective_disk_per_node_gib
|
|
40
40
|
from service_capacity_modeling.models.common import network_services
|
|
41
41
|
from service_capacity_modeling.models.common import normalize_cores
|
|
@@ -79,7 +79,7 @@ def _write_buffer_gib_zone(
|
|
|
79
79
|
return float(write_buffer_gib) / zones_per_region
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
def _get_cores_from_desires(desires, instance):
|
|
82
|
+
def _get_cores_from_desires(desires: CapacityDesires, instance: Instance) -> int:
|
|
83
83
|
cpu_buffer = buffer_for_components(
|
|
84
84
|
buffers=desires.buffers, components=[BACKGROUND_BUFFER]
|
|
85
85
|
)
|
|
@@ -97,7 +97,7 @@ def _get_cores_from_desires(desires, instance):
|
|
|
97
97
|
return needed_cores
|
|
98
98
|
|
|
99
99
|
|
|
100
|
-
def _get_disk_from_desires(desires, copies_per_region):
|
|
100
|
+
def _get_disk_from_desires(desires: CapacityDesires, copies_per_region: int) -> int:
|
|
101
101
|
disk_buffer = buffer_for_components(
|
|
102
102
|
buffers=desires.buffers, components=[BufferComponent.disk]
|
|
103
103
|
)
|
|
@@ -116,7 +116,7 @@ def _get_min_count(
|
|
|
116
116
|
needed_disk_gib: float,
|
|
117
117
|
disk_per_node_gib: float,
|
|
118
118
|
cluster_size_lambda: Callable[[int], int],
|
|
119
|
-
):
|
|
119
|
+
) -> int:
|
|
120
120
|
"""
|
|
121
121
|
Compute the minimum number of nodes required for a zone.
|
|
122
122
|
|
|
@@ -158,7 +158,10 @@ def _get_min_count(
|
|
|
158
158
|
|
|
159
159
|
|
|
160
160
|
def _zonal_requirement_for_new_cluster(
|
|
161
|
-
desires
|
|
161
|
+
desires: CapacityDesires,
|
|
162
|
+
instance: Instance,
|
|
163
|
+
copies_per_region: int,
|
|
164
|
+
zones_per_region: int,
|
|
162
165
|
) -> CapacityRequirement:
|
|
163
166
|
needed_cores = _get_cores_from_desires(desires, instance)
|
|
164
167
|
needed_disk = _get_disk_from_desires(desires, copies_per_region)
|
|
@@ -181,7 +184,9 @@ def _zonal_requirement_for_new_cluster(
|
|
|
181
184
|
)
|
|
182
185
|
|
|
183
186
|
|
|
184
|
-
|
|
187
|
+
# pylint: disable=too-many-locals
|
|
188
|
+
# pylint: disable=too-many-positional-arguments
|
|
189
|
+
def _estimate_cassandra_requirement(
|
|
185
190
|
instance: Instance,
|
|
186
191
|
desires: CapacityDesires,
|
|
187
192
|
working_set: float,
|
|
@@ -205,19 +210,22 @@ def _estimate_cassandra_requirement( # pylint: disable=too-many-positional-argu
|
|
|
205
210
|
# If the cluster is already provisioned
|
|
206
211
|
if current_capacity and desires.current_clusters is not None:
|
|
207
212
|
capacity_requirement = zonal_requirements_from_current(
|
|
208
|
-
desires.current_clusters,
|
|
213
|
+
desires.current_clusters,
|
|
214
|
+
desires.buffers,
|
|
215
|
+
instance,
|
|
216
|
+
reference_shape,
|
|
209
217
|
)
|
|
210
|
-
|
|
211
|
-
desires.buffers.derived, [
|
|
218
|
+
disk_derived_buffer = DerivedBuffers.for_components(
|
|
219
|
+
desires.buffers.derived, [BufferComponent.disk]
|
|
212
220
|
)
|
|
213
221
|
disk_used_gib = (
|
|
214
222
|
current_capacity.disk_utilization_gib.mid
|
|
215
223
|
* current_capacity.cluster_instance_count.mid
|
|
216
|
-
*
|
|
217
|
-
)
|
|
218
|
-
_, memory_preserve = derived_buffer_for_component(
|
|
219
|
-
desires.buffers.derived, ["storage", "memory"]
|
|
224
|
+
* disk_derived_buffer.scale
|
|
220
225
|
)
|
|
226
|
+
memory_preserve = DerivedBuffers.for_components(
|
|
227
|
+
desires.buffers.derived, [BufferComponent.memory]
|
|
228
|
+
).preserve
|
|
221
229
|
else:
|
|
222
230
|
# If the cluster is not yet provisioned
|
|
223
231
|
capacity_requirement = _zonal_requirement_for_new_cluster(
|
|
@@ -320,14 +328,14 @@ def _estimate_cassandra_requirement( # pylint: disable=too-many-positional-argu
|
|
|
320
328
|
)
|
|
321
329
|
|
|
322
330
|
|
|
323
|
-
def _get_current_cluster_size(desires) -> int:
|
|
331
|
+
def _get_current_cluster_size(desires: CapacityDesires) -> int:
|
|
324
332
|
current_capacity = _get_current_capacity(desires)
|
|
325
333
|
if current_capacity is None:
|
|
326
334
|
return 0
|
|
327
335
|
return math.ceil(current_capacity.cluster_instance_count.mid)
|
|
328
336
|
|
|
329
337
|
|
|
330
|
-
def _get_current_capacity(desires) -> Optional[CurrentClusterCapacity]:
|
|
338
|
+
def _get_current_capacity(desires: CapacityDesires) -> Optional[CurrentClusterCapacity]:
|
|
331
339
|
current_capacity = (
|
|
332
340
|
None
|
|
333
341
|
if desires.current_clusters is None
|
|
@@ -340,7 +348,7 @@ def _get_current_capacity(desires) -> Optional[CurrentClusterCapacity]:
|
|
|
340
348
|
return current_capacity
|
|
341
349
|
|
|
342
350
|
|
|
343
|
-
def _upsert_params(cluster, params):
|
|
351
|
+
def _upsert_params(cluster: Any, params: Dict[str, Any]) -> None:
|
|
344
352
|
if cluster.cluster_params:
|
|
345
353
|
cluster.cluster_params.update(params)
|
|
346
354
|
else:
|
|
@@ -369,7 +377,7 @@ def _estimate_cassandra_cluster_zonal( # pylint: disable=too-many-positional-ar
|
|
|
369
377
|
desires: CapacityDesires,
|
|
370
378
|
zones_per_region: int = 3,
|
|
371
379
|
copies_per_region: int = 3,
|
|
372
|
-
require_local_disks: bool =
|
|
380
|
+
require_local_disks: bool = False,
|
|
373
381
|
require_attached_disks: bool = False,
|
|
374
382
|
required_cluster_size: Optional[int] = None,
|
|
375
383
|
max_rps_to_disk: int = 500,
|
|
@@ -570,7 +578,7 @@ def _estimate_cassandra_cluster_zonal( # pylint: disable=too-many-positional-ar
|
|
|
570
578
|
|
|
571
579
|
|
|
572
580
|
# C* LCS has 160 MiB sstables by default and 10 sstables per level
|
|
573
|
-
def _cass_io_per_read(node_size_gib, sstable_size_mb=160):
|
|
581
|
+
def _cass_io_per_read(node_size_gib: float, sstable_size_mb: int = 160) -> int:
|
|
574
582
|
gb = node_size_gib * 1024
|
|
575
583
|
sstables = max(1, gb // sstable_size_mb)
|
|
576
584
|
# 10 sstables per level, plus 1 for L0 (avg)
|
|
@@ -580,7 +588,7 @@ def _cass_io_per_read(node_size_gib, sstable_size_mb=160):
|
|
|
580
588
|
return 2 * levels
|
|
581
589
|
|
|
582
590
|
|
|
583
|
-
def _get_base_memory(desires: CapacityDesires):
|
|
591
|
+
def _get_base_memory(desires: CapacityDesires) -> float:
|
|
584
592
|
return (
|
|
585
593
|
desires.data_shape.reserved_instance_app_mem_gib
|
|
586
594
|
+ desires.data_shape.reserved_instance_system_mem_gib
|
|
@@ -636,7 +644,7 @@ class NflxCassandraArguments(BaseModel):
|
|
|
636
644
|
" this will be deduced from durability and consistency desires",
|
|
637
645
|
)
|
|
638
646
|
require_local_disks: bool = Field(
|
|
639
|
-
default=
|
|
647
|
+
default=False,
|
|
640
648
|
description="If local (ephemeral) drives are required",
|
|
641
649
|
)
|
|
642
650
|
require_attached_disks: bool = Field(
|
|
@@ -674,8 +682,13 @@ class NflxCassandraArguments(BaseModel):
|
|
|
674
682
|
|
|
675
683
|
|
|
676
684
|
class NflxCassandraCapacityModel(CapacityModel):
|
|
685
|
+
def __init__(self) -> None:
|
|
686
|
+
pass
|
|
687
|
+
|
|
677
688
|
@staticmethod
|
|
678
|
-
def get_required_cluster_size(
|
|
689
|
+
def get_required_cluster_size(
|
|
690
|
+
tier: int, extra_model_arguments: Dict[str, Any]
|
|
691
|
+
) -> Optional[int]:
|
|
679
692
|
required_cluster_size: Optional[int] = (
|
|
680
693
|
math.ceil(extra_model_arguments["required_cluster_size"])
|
|
681
694
|
if "required_cluster_size" in extra_model_arguments
|
|
@@ -719,7 +732,7 @@ class NflxCassandraCapacityModel(CapacityModel):
|
|
|
719
732
|
desires, extra_model_arguments.get("copies_per_region", None)
|
|
720
733
|
)
|
|
721
734
|
require_local_disks: bool = extra_model_arguments.get(
|
|
722
|
-
"require_local_disks",
|
|
735
|
+
"require_local_disks", False
|
|
723
736
|
)
|
|
724
737
|
require_attached_disks: bool = extra_model_arguments.get(
|
|
725
738
|
"require_attached_disks", False
|
|
@@ -770,7 +783,7 @@ class NflxCassandraCapacityModel(CapacityModel):
|
|
|
770
783
|
)
|
|
771
784
|
|
|
772
785
|
@staticmethod
|
|
773
|
-
def description():
|
|
786
|
+
def description() -> str:
|
|
774
787
|
return "Netflix Streaming Cassandra Model"
|
|
775
788
|
|
|
776
789
|
@staticmethod
|
|
@@ -798,7 +811,9 @@ class NflxCassandraCapacityModel(CapacityModel):
|
|
|
798
811
|
)
|
|
799
812
|
|
|
800
813
|
@staticmethod
|
|
801
|
-
def default_desires(
|
|
814
|
+
def default_desires(
|
|
815
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
816
|
+
) -> CapacityDesires:
|
|
802
817
|
acceptable_consistency = {
|
|
803
818
|
None,
|
|
804
819
|
AccessConsistency.best_effort,
|
|
@@ -46,7 +46,8 @@ class NflxCounterArguments(NflxJavaAppArguments):
|
|
|
46
46
|
)
|
|
47
47
|
counter_cardinality: NflxCounterCardinality = Field(
|
|
48
48
|
alias="counter.cardinality",
|
|
49
|
-
description="Low means <
|
|
49
|
+
description="Low means < 10,000, medium (10,000—1,000,000), high means "
|
|
50
|
+
"> 1,000,000.",
|
|
50
51
|
)
|
|
51
52
|
counter_mode: NflxCounterMode = Field(
|
|
52
53
|
alias="counter.mode",
|
|
@@ -81,7 +82,7 @@ class NflxCounterCapacityModel(CapacityModel):
|
|
|
81
82
|
return counter_app
|
|
82
83
|
|
|
83
84
|
@staticmethod
|
|
84
|
-
def description():
|
|
85
|
+
def description() -> str:
|
|
85
86
|
return "Netflix Streaming Counter Model"
|
|
86
87
|
|
|
87
88
|
@staticmethod
|
|
@@ -92,36 +93,57 @@ class NflxCounterCapacityModel(CapacityModel):
|
|
|
92
93
|
def compose_with(
|
|
93
94
|
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
94
95
|
) -> Tuple[Tuple[str, Callable[[CapacityDesires], CapacityDesires]], ...]:
|
|
95
|
-
stores = [
|
|
96
|
-
if extra_model_arguments["counter.mode"] != NflxCounterMode.best_effort.value:
|
|
96
|
+
stores = []
|
|
97
97
|
|
|
98
|
+
if extra_model_arguments["counter.mode"] == NflxCounterMode.best_effort.value:
|
|
99
|
+
stores.append(("org.netflix.evcache", lambda x: x))
|
|
100
|
+
else:
|
|
101
|
+
# Shared evcache cluster is used for eventual and exact counters
|
|
98
102
|
def _modify_cassandra_desires(
|
|
99
103
|
user_desires: CapacityDesires,
|
|
100
104
|
) -> CapacityDesires:
|
|
101
105
|
modified = user_desires.model_copy(deep=True)
|
|
106
|
+
counter_cardinality = extra_model_arguments["counter.cardinality"]
|
|
107
|
+
|
|
108
|
+
counter_deltas_per_second = (
|
|
109
|
+
user_desires.query_pattern.estimated_write_per_second
|
|
110
|
+
)
|
|
102
111
|
|
|
103
|
-
#
|
|
104
|
-
|
|
112
|
+
# low cardinality : rollups happen once every 60 seconds
|
|
113
|
+
# medium cardinality : rollups happen once every 30 seconds
|
|
114
|
+
# high cardinality : rollups happen once every 10 seconds
|
|
115
|
+
# TODO: Account for read amplification from time slice configs
|
|
116
|
+
# for better model accuracy
|
|
117
|
+
if counter_cardinality == NflxCounterCardinality.low.value:
|
|
118
|
+
rollups_per_second = counter_deltas_per_second.scale(0.0167)
|
|
119
|
+
elif counter_cardinality == NflxCounterCardinality.medium.value:
|
|
120
|
+
rollups_per_second = counter_deltas_per_second.scale(0.0333)
|
|
121
|
+
else:
|
|
122
|
+
rollups_per_second = counter_deltas_per_second.scale(0.1)
|
|
105
123
|
|
|
106
|
-
|
|
107
|
-
rps = cps.scale(0.1)
|
|
108
|
-
modified.query_pattern.estimated_read_per_second = rps
|
|
124
|
+
modified.query_pattern.estimated_read_per_second = rollups_per_second
|
|
109
125
|
|
|
110
126
|
# storage size fix
|
|
111
|
-
|
|
112
|
-
|
|
127
|
+
delta_event_size = 256 # bytes
|
|
128
|
+
rolled_up_count_size = 128 # bytes
|
|
113
129
|
GiB = 1024 * 1024 * 1024
|
|
114
|
-
|
|
130
|
+
|
|
131
|
+
# Events can be discarded as soon as rollup is complete
|
|
132
|
+
# We default to a 1 day slice with 2 day retention
|
|
133
|
+
retention = timedelta(days=2).total_seconds()
|
|
134
|
+
|
|
115
135
|
cardinality = {
|
|
116
|
-
"low":
|
|
117
|
-
"medium":
|
|
118
|
-
"high":
|
|
136
|
+
"low": 10_000,
|
|
137
|
+
"medium": 100_000,
|
|
138
|
+
"high": 1_000_000,
|
|
119
139
|
}[extra_model_arguments["counter.cardinality"]]
|
|
120
140
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
141
|
+
event_storage_size = counter_deltas_per_second.scale(
|
|
142
|
+
delta_event_size * retention / GiB
|
|
143
|
+
)
|
|
144
|
+
rollup_storage_size = rolled_up_count_size * cardinality / GiB
|
|
145
|
+
total_store_size = event_storage_size.offset(rollup_storage_size)
|
|
146
|
+
modified.data_shape.estimated_state_size_gib = total_store_size
|
|
125
147
|
|
|
126
148
|
return modified
|
|
127
149
|
|
|
@@ -129,7 +151,9 @@ class NflxCounterCapacityModel(CapacityModel):
|
|
|
129
151
|
return tuple(stores)
|
|
130
152
|
|
|
131
153
|
@staticmethod
|
|
132
|
-
def default_desires(
|
|
154
|
+
def default_desires(
|
|
155
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
156
|
+
) -> CapacityDesires:
|
|
133
157
|
if user_desires.query_pattern.access_pattern == AccessPattern.latency:
|
|
134
158
|
return CapacityDesires(
|
|
135
159
|
query_pattern=QueryPattern(
|
|
@@ -29,6 +29,7 @@ from service_capacity_modeling.interface import Interval
|
|
|
29
29
|
from service_capacity_modeling.interface import QueryPattern
|
|
30
30
|
from service_capacity_modeling.interface import RegionContext
|
|
31
31
|
from service_capacity_modeling.interface import Requirements
|
|
32
|
+
from service_capacity_modeling.interface import ZoneClusterCapacity
|
|
32
33
|
from service_capacity_modeling.models import CapacityModel
|
|
33
34
|
from service_capacity_modeling.models.common import buffer_for_components
|
|
34
35
|
from service_capacity_modeling.models.common import compute_stateful_zone
|
|
@@ -44,7 +45,7 @@ logger = logging.getLogger(__name__)
|
|
|
44
45
|
|
|
45
46
|
# Pebble does Leveled compaction with tieres of size??
|
|
46
47
|
# (FIXME) What does pebble actually do
|
|
47
|
-
def _crdb_io_per_read(node_size_gib, sstable_size_mb=1000):
|
|
48
|
+
def _crdb_io_per_read(node_size_gib: float, sstable_size_mb: int = 1000) -> int:
|
|
48
49
|
gb = node_size_gib * 1024
|
|
49
50
|
sstables = max(1, gb // sstable_size_mb)
|
|
50
51
|
# 10 sstables per level, plus 1 for L0 (avg)
|
|
@@ -128,7 +129,7 @@ def _estimate_cockroachdb_requirement( # noqa=E501 pylint: disable=too-many-pos
|
|
|
128
129
|
)
|
|
129
130
|
|
|
130
131
|
|
|
131
|
-
def _upsert_params(cluster, params):
|
|
132
|
+
def _upsert_params(cluster: ZoneClusterCapacity, params: Dict[str, Any]) -> None:
|
|
132
133
|
if cluster.cluster_params:
|
|
133
134
|
cluster.cluster_params.update(params)
|
|
134
135
|
else:
|
|
@@ -329,7 +330,7 @@ class NflxCockroachDBCapacityModel(CapacityModel):
|
|
|
329
330
|
)
|
|
330
331
|
|
|
331
332
|
@staticmethod
|
|
332
|
-
def description():
|
|
333
|
+
def description() -> str:
|
|
333
334
|
return "Netflix Streaming CockroachDB Model"
|
|
334
335
|
|
|
335
336
|
@staticmethod
|
|
@@ -337,7 +338,9 @@ class NflxCockroachDBCapacityModel(CapacityModel):
|
|
|
337
338
|
return NflxCockroachDBArguments.model_json_schema()
|
|
338
339
|
|
|
339
340
|
@staticmethod
|
|
340
|
-
def default_desires(
|
|
341
|
+
def default_desires(
|
|
342
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
343
|
+
) -> CapacityDesires:
|
|
341
344
|
acceptable_consistency = {
|
|
342
345
|
None,
|
|
343
346
|
AccessConsistency.linearizable,
|
|
@@ -139,11 +139,12 @@ def _get_read_consistency_percentages(
|
|
|
139
139
|
)
|
|
140
140
|
if total_percent == 0:
|
|
141
141
|
access_consistency = desires.query_pattern.access_consistency.same_region
|
|
142
|
-
|
|
142
|
+
target_consistency = access_consistency.target_consistency
|
|
143
|
+
if target_consistency == AccessConsistency.serializable:
|
|
143
144
|
transactional_read_percent = 1.0
|
|
144
145
|
eventual_read_percent = 0.0
|
|
145
146
|
strong_read_percent = 0.0
|
|
146
|
-
elif
|
|
147
|
+
elif target_consistency in (
|
|
147
148
|
AccessConsistency.read_your_writes,
|
|
148
149
|
AccessConsistency.linearizable,
|
|
149
150
|
):
|
|
@@ -184,7 +185,8 @@ def _get_write_consistency_percentages(
|
|
|
184
185
|
total_percent = transactional_write_percent + non_transactional_write_percent
|
|
185
186
|
if total_percent == 0:
|
|
186
187
|
access_consistency = desires.query_pattern.access_consistency.same_region
|
|
187
|
-
|
|
188
|
+
target_consistency = access_consistency.target_consistency
|
|
189
|
+
if target_consistency == AccessConsistency.serializable:
|
|
188
190
|
transactional_write_percent = 1.0
|
|
189
191
|
non_transactional_write_percent = 0.0
|
|
190
192
|
else:
|
|
@@ -547,7 +549,7 @@ class NflxDynamoDBCapacityModel(CapacityModel):
|
|
|
547
549
|
)
|
|
548
550
|
|
|
549
551
|
@staticmethod
|
|
550
|
-
def description():
|
|
552
|
+
def description() -> str:
|
|
551
553
|
return "Netflix Streaming DynamoDB Model"
|
|
552
554
|
|
|
553
555
|
@staticmethod
|
|
@@ -559,7 +561,9 @@ class NflxDynamoDBCapacityModel(CapacityModel):
|
|
|
559
561
|
return False
|
|
560
562
|
|
|
561
563
|
@staticmethod
|
|
562
|
-
def default_desires(
|
|
564
|
+
def default_desires(
|
|
565
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
566
|
+
) -> CapacityDesires:
|
|
563
567
|
acceptable_consistency = {
|
|
564
568
|
"same_region": {
|
|
565
569
|
None,
|
|
@@ -58,7 +58,7 @@ def _target_rf(desires: CapacityDesires, user_copies: Optional[int]) -> int:
|
|
|
58
58
|
# segments of 512 megs per
|
|
59
59
|
# https://lucene.apache.org/core/8_1_0/core/org/apache/lucene/index/TieredMergePolicy.html#setSegmentsPerTier(double)
|
|
60
60
|
# (FIXME) Verify what elastic merge actually does
|
|
61
|
-
def _es_io_per_read(node_size_gib, segment_size_mb=512):
|
|
61
|
+
def _es_io_per_read(node_size_gib: float, segment_size_mb: int = 512) -> int:
|
|
62
62
|
size_mib = node_size_gib * 1024
|
|
63
63
|
segments = max(1, size_mib // segment_size_mb)
|
|
64
64
|
# 10 segments per tier, plus 1 for L0 (avg)
|
|
@@ -75,7 +75,7 @@ def _estimate_elasticsearch_requirement( # noqa: E501 pylint: disable=too-many-
|
|
|
75
75
|
max_rps_to_disk: int,
|
|
76
76
|
zones_in_region: int = 3,
|
|
77
77
|
copies_per_region: int = 3,
|
|
78
|
-
jvm_memory_overhead=1.2,
|
|
78
|
+
jvm_memory_overhead: float = 1.2,
|
|
79
79
|
) -> CapacityRequirement:
|
|
80
80
|
"""Estimate the capacity required for one zone given a regional desire
|
|
81
81
|
|
|
@@ -153,7 +153,7 @@ def _estimate_elasticsearch_requirement( # noqa: E501 pylint: disable=too-many-
|
|
|
153
153
|
)
|
|
154
154
|
|
|
155
155
|
|
|
156
|
-
def _upsert_params(cluster, params):
|
|
156
|
+
def _upsert_params(cluster: ZoneClusterCapacity, params: Dict[str, Any]) -> None:
|
|
157
157
|
if cluster.cluster_params:
|
|
158
158
|
cluster.cluster_params.update(params)
|
|
159
159
|
else:
|
|
@@ -194,7 +194,7 @@ class NflxElasticsearchDataCapacityModel(CapacityModel):
|
|
|
194
194
|
|
|
195
195
|
@staticmethod
|
|
196
196
|
def default_desires(
|
|
197
|
-
user_desires, extra_model_arguments: Dict[str, Any]
|
|
197
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
198
198
|
) -> CapacityDesires:
|
|
199
199
|
desires = CapacityModel.default_desires(user_desires, extra_model_arguments)
|
|
200
200
|
desires.buffers = NflxElasticsearchDataCapacityModel.default_buffers()
|
|
@@ -452,7 +452,7 @@ class NflxElasticsearchCapacityModel(CapacityModel):
|
|
|
452
452
|
return None
|
|
453
453
|
|
|
454
454
|
@staticmethod
|
|
455
|
-
def description():
|
|
455
|
+
def description() -> str:
|
|
456
456
|
return "Netflix Streaming Elasticsearch Model"
|
|
457
457
|
|
|
458
458
|
@staticmethod
|
|
@@ -482,7 +482,9 @@ class NflxElasticsearchCapacityModel(CapacityModel):
|
|
|
482
482
|
return NflxElasticsearchArguments.model_json_schema()
|
|
483
483
|
|
|
484
484
|
@staticmethod
|
|
485
|
-
def default_desires(
|
|
485
|
+
def default_desires(
|
|
486
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
487
|
+
) -> CapacityDesires:
|
|
486
488
|
acceptable_consistency = {
|
|
487
489
|
AccessConsistency.best_effort,
|
|
488
490
|
AccessConsistency.eventual,
|
|
@@ -48,7 +48,7 @@ class NflxEntityCapacityModel(CapacityModel):
|
|
|
48
48
|
return entity_app
|
|
49
49
|
|
|
50
50
|
@staticmethod
|
|
51
|
-
def description():
|
|
51
|
+
def description() -> str:
|
|
52
52
|
return "Netflix Streaming Entity Model"
|
|
53
53
|
|
|
54
54
|
@staticmethod
|
|
@@ -76,7 +76,7 @@ class NflxEntityCapacityModel(CapacityModel):
|
|
|
76
76
|
/ 1024**3
|
|
77
77
|
)
|
|
78
78
|
else:
|
|
79
|
-
item_size_gib = 10 / 1024**2
|
|
79
|
+
item_size_gib = 10 / 1024**2 # type: ignore[unreachable]
|
|
80
80
|
item_count = user_desires.data_shape.estimated_state_size_gib.scale(
|
|
81
81
|
1 / item_size_gib
|
|
82
82
|
)
|
|
@@ -102,7 +102,9 @@ class NflxEntityCapacityModel(CapacityModel):
|
|
|
102
102
|
)
|
|
103
103
|
|
|
104
104
|
@staticmethod
|
|
105
|
-
def default_desires(
|
|
105
|
+
def default_desires(
|
|
106
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
107
|
+
) -> CapacityDesires:
|
|
106
108
|
if user_desires.query_pattern.access_pattern == AccessPattern.latency:
|
|
107
109
|
return CapacityDesires(
|
|
108
110
|
query_pattern=QueryPattern(
|
|
@@ -33,13 +33,10 @@ from service_capacity_modeling.interface import Requirements
|
|
|
33
33
|
from service_capacity_modeling.models import CapacityModel
|
|
34
34
|
from service_capacity_modeling.models.common import buffer_for_components
|
|
35
35
|
from service_capacity_modeling.models.common import compute_stateful_zone
|
|
36
|
-
from service_capacity_modeling.models.common import get_cores_from_current_capacity
|
|
37
|
-
from service_capacity_modeling.models.common import get_disk_from_current_capacity
|
|
38
36
|
from service_capacity_modeling.models.common import get_effective_disk_per_node_gib
|
|
39
|
-
from service_capacity_modeling.models.common import get_memory_from_current_capacity
|
|
40
|
-
from service_capacity_modeling.models.common import get_network_from_current_capacity
|
|
41
37
|
from service_capacity_modeling.models.common import network_services
|
|
42
38
|
from service_capacity_modeling.models.common import normalize_cores
|
|
39
|
+
from service_capacity_modeling.models.common import RequirementFromCurrentCapacity
|
|
43
40
|
from service_capacity_modeling.models.common import simple_network_mbps
|
|
44
41
|
from service_capacity_modeling.models.common import sqrt_staffed_cores
|
|
45
42
|
from service_capacity_modeling.models.common import working_set_from_drive_and_slo
|
|
@@ -72,7 +69,9 @@ def calculate_read_cpu_time_evcache_ms(read_size_bytes: float) -> float:
|
|
|
72
69
|
return max(read_latency_ms, 0.005)
|
|
73
70
|
|
|
74
71
|
|
|
75
|
-
def calculate_spread_cost(
|
|
72
|
+
def calculate_spread_cost(
|
|
73
|
+
cluster_size: int, max_cost: float = 100000, min_cost: float = 0.0
|
|
74
|
+
) -> float:
|
|
76
75
|
if cluster_size > 10:
|
|
77
76
|
return min_cost
|
|
78
77
|
if cluster_size < 2:
|
|
@@ -85,14 +84,14 @@ def calculate_vitals_for_capacity_planner(
|
|
|
85
84
|
instance: Instance,
|
|
86
85
|
current_memory_gib: float,
|
|
87
86
|
current_disk_gib: float,
|
|
88
|
-
):
|
|
87
|
+
) -> Tuple[float, float, float, float]:
|
|
89
88
|
# First calculate assuming new deployment
|
|
90
89
|
needed_cores = normalize_cores(
|
|
91
90
|
core_count=sqrt_staffed_cores(desires),
|
|
92
91
|
target_shape=instance,
|
|
93
92
|
reference_shape=desires.reference_shape,
|
|
94
93
|
)
|
|
95
|
-
needed_network_mbps = simple_network_mbps(desires)
|
|
94
|
+
needed_network_mbps = float(simple_network_mbps(desires))
|
|
96
95
|
needed_memory_gib = current_memory_gib
|
|
97
96
|
needed_disk_gib = current_disk_gib
|
|
98
97
|
|
|
@@ -104,25 +103,20 @@ def calculate_vitals_for_capacity_planner(
|
|
|
104
103
|
)
|
|
105
104
|
if not current_capacity:
|
|
106
105
|
return needed_cores, needed_network_mbps, needed_memory_gib, needed_disk_gib
|
|
106
|
+
requirements = RequirementFromCurrentCapacity(
|
|
107
|
+
current_capacity=current_capacity,
|
|
108
|
+
buffers=desires.buffers,
|
|
109
|
+
)
|
|
107
110
|
needed_cores = normalize_cores(
|
|
108
|
-
core_count=
|
|
109
|
-
current_capacity, desires.buffers, instance
|
|
110
|
-
),
|
|
111
|
+
core_count=requirements.cpu(instance_candidate=instance),
|
|
111
112
|
target_shape=instance,
|
|
112
113
|
reference_shape=current_capacity.cluster_instance,
|
|
113
114
|
)
|
|
114
|
-
needed_network_mbps =
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
needed_memory_gib = get_memory_from_current_capacity(
|
|
118
|
-
current_capacity, desires.buffers
|
|
115
|
+
needed_network_mbps = float(requirements.network_mbps)
|
|
116
|
+
needed_disk_gib = float(
|
|
117
|
+
requirements.disk_gib if current_capacity.cluster_drive is not None else 0.0
|
|
119
118
|
)
|
|
120
|
-
|
|
121
|
-
needed_disk_gib = 0.0
|
|
122
|
-
else:
|
|
123
|
-
needed_disk_gib = get_disk_from_current_capacity(
|
|
124
|
-
current_capacity, desires.buffers
|
|
125
|
-
)
|
|
119
|
+
needed_memory_gib = requirements.mem_gib
|
|
126
120
|
return needed_cores, needed_network_mbps, needed_memory_gib, needed_disk_gib
|
|
127
121
|
|
|
128
122
|
|
|
@@ -204,7 +198,7 @@ def _estimate_evcache_requirement(
|
|
|
204
198
|
)
|
|
205
199
|
|
|
206
200
|
|
|
207
|
-
def _upsert_params(cluster, params):
|
|
201
|
+
def _upsert_params(cluster: Any, params: Dict[str, Any]) -> None:
|
|
208
202
|
if cluster.cluster_params:
|
|
209
203
|
cluster.cluster_params.update(params)
|
|
210
204
|
else:
|
|
@@ -271,7 +265,7 @@ def _estimate_evcache_cluster_zonal( # noqa: C901,E501 pylint: disable=too-many
|
|
|
271
265
|
# larger to account for additional overhead. Note that the
|
|
272
266
|
# reserved_instance_system_mem_gib has a base of 1 GiB OSMEM so this
|
|
273
267
|
# just represents the variable component
|
|
274
|
-
def reserve_memory(instance_mem_gib):
|
|
268
|
+
def reserve_memory(instance_mem_gib: float) -> float:
|
|
275
269
|
# (Joey) From the chart it appears to be about a 3% overhead for
|
|
276
270
|
# OS memory.
|
|
277
271
|
variable_os = int(instance_mem_gib * 0.03)
|
|
@@ -447,7 +441,7 @@ class NflxEVCacheCapacityModel(CapacityModel):
|
|
|
447
441
|
)
|
|
448
442
|
|
|
449
443
|
@staticmethod
|
|
450
|
-
def description():
|
|
444
|
+
def description() -> str:
|
|
451
445
|
return "Netflix Streaming EVCache (memcached) Model"
|
|
452
446
|
|
|
453
447
|
@staticmethod
|
|
@@ -455,7 +449,9 @@ class NflxEVCacheCapacityModel(CapacityModel):
|
|
|
455
449
|
return NflxEVCacheArguments.model_json_schema()
|
|
456
450
|
|
|
457
451
|
@staticmethod
|
|
458
|
-
def default_desires(
|
|
452
|
+
def default_desires(
|
|
453
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
454
|
+
) -> CapacityDesires:
|
|
459
455
|
acceptable_consistency = {
|
|
460
456
|
AccessConsistency.best_effort,
|
|
461
457
|
AccessConsistency.never,
|
|
@@ -45,7 +45,7 @@ class NflxGraphKVCapacityModel(CapacityModel):
|
|
|
45
45
|
return graphkv_app
|
|
46
46
|
|
|
47
47
|
@staticmethod
|
|
48
|
-
def description():
|
|
48
|
+
def description() -> str:
|
|
49
49
|
return "Netflix Streaming Graph Abstraction"
|
|
50
50
|
|
|
51
51
|
@staticmethod
|
|
@@ -89,7 +89,7 @@ class NflxGraphKVCapacityModel(CapacityModel):
|
|
|
89
89
|
/ 1024**3
|
|
90
90
|
)
|
|
91
91
|
else:
|
|
92
|
-
item_size_gib = 1 / 1024**2
|
|
92
|
+
item_size_gib = 1 / 1024**2 # type: ignore[unreachable]
|
|
93
93
|
item_count = user_desires.data_shape.estimated_state_size_gib.scale(
|
|
94
94
|
1 / item_size_gib
|
|
95
95
|
)
|
|
@@ -102,7 +102,9 @@ class NflxGraphKVCapacityModel(CapacityModel):
|
|
|
102
102
|
return (("org.netflix.key-value", _modify_kv_desires),)
|
|
103
103
|
|
|
104
104
|
@staticmethod
|
|
105
|
-
def default_desires(
|
|
105
|
+
def default_desires(
|
|
106
|
+
user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
|
|
107
|
+
) -> CapacityDesires:
|
|
106
108
|
if user_desires.query_pattern.access_pattern == AccessPattern.latency:
|
|
107
109
|
return CapacityDesires(
|
|
108
110
|
query_pattern=QueryPattern(
|