service-capacity-modeling 0.3.74__tar.gz → 0.3.79__tar.gz

{service_capacity_modeling-0.3.74 → service_capacity_modeling-0.3.79}/PKG-INFO

@@ -1,19 +1,22 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.74
+Version: 0.3.79
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com
 License: Apache 2.0
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9,<3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: pydantic>2.0
 Requires-Dist: scipy
 Requires-Dist: numpy
-Requires-Dist: importlib_resources; python_version < "3.7"
 Requires-Dist: isodate
 Provides-Extra: aws
 Requires-Dist: boto3; extra == "aws"
@@ -26,6 +29,7 @@ Dynamic: license
 Dynamic: license-file
 Dynamic: provides-extra
 Dynamic: requires-dist
+Dynamic: requires-python
 Dynamic: summary
 
 # Service Capacity Modeling
@@ -43,10 +47,10 @@ remember this repository is public when making changes to it.
 Run the tests:
 ```bash
 # Test the capacity planner on included netflix models
-$ tox -e py38
+$ tox -e py39
 
 # Run a single test with a debugger attached if the test fails
-$ .tox/py38/bin/pytest -n0 -k test_java_heap_heavy --pdb --pdbcls=IPython.terminal.debugger:Pdb
+$ .tox/py39/bin/pytest -n0 -k test_java_heap_heavy --pdb --pdbcls=IPython.terminal.debugger:Pdb
 
 # Verify all type contracts
 $ tox -e mypy
@@ -247,7 +251,7 @@ To contribute to this project:
 2. Write a unit test using `pytest` in the `tests` folder.
 3. Ensure your tests pass via `tox` or debug them with:
 ```
-tox -e py38 -- -k test_<your_functionality> --pdb --pdbcls=IPython.terminal.debugger:Pdb
+tox -e py39 -- -k test_<your_functionality> --pdb --pdbcls=IPython.terminal.debugger:Pdb
 ```
 
 ### Pre-commit / Linting

{service_capacity_modeling-0.3.74 → service_capacity_modeling-0.3.79}/README.md

@@ -13,10 +13,10 @@ remember this repository is public when making changes to it.
 Run the tests:
 ```bash
 # Test the capacity planner on included netflix models
-$ tox -e py38
+$ tox -e py39
 
 # Run a single test with a debugger attached if the test fails
-$ .tox/py38/bin/pytest -n0 -k test_java_heap_heavy --pdb --pdbcls=IPython.terminal.debugger:Pdb
+$ .tox/py39/bin/pytest -n0 -k test_java_heap_heavy --pdb --pdbcls=IPython.terminal.debugger:Pdb
 
 # Verify all type contracts
 $ tox -e mypy
@@ -217,7 +217,7 @@ To contribute to this project:
 2. Write a unit test using `pytest` in the `tests` folder.
 3. Ensure your tests pass via `tox` or debug them with:
 ```
-tox -e py38 -- -k test_<your_functionality> --pdb --pdbcls=IPython.terminal.debugger:Pdb
+tox -e py39 -- -k test_<your_functionality> --pdb --pdbcls=IPython.terminal.debugger:Pdb
 ```
 
 ### Pre-commit / Linting

{service_capacity_modeling-0.3.74 → service_capacity_modeling-0.3.79}/service_capacity_modeling/capacity_planner.py

@@ -190,7 +190,7 @@ def model_desires_percentiles(
 def _set_instance_objects(
     desires: CapacityDesires,
     hardware: Hardware,
-):
+) -> None:
     if desires.current_clusters:
         for zonal_cluster_capacity in desires.current_clusters.zonal:
             if zonal_cluster_capacity.cluster_instance_name in hardware.instances:
@@ -290,7 +290,9 @@ def _regret(
     return plans_by_regret
 
 
-def _add_requirement(requirement, accum):
+def _add_requirement(
+    requirement: CapacityRequirement, accum: Dict[str, Dict[str, List[Interval]]]
+) -> None:
     if requirement.requirement_type not in accum:
         accum[requirement.requirement_type] = {}
 
@@ -305,7 +307,11 @@ def _add_requirement(requirement, accum):
                 requirements[field].append(d)
 
 
-def _merge_models(plans_by_model, zonal_requirements, regional_requirements):
+def _merge_models(
+    plans_by_model: List[List[CapacityPlan]],
+    zonal_requirements: Dict[str, Dict[str, List[Interval]]],
+    regional_requirements: Dict[str, Dict[str, List[Interval]]],
+) -> List[CapacityPlan]:
     capacity_plans = []
     for composed in zip(*filter(lambda x: x, plans_by_model)):
         merged_plans = [functools.reduce(merge_plan, composed)]
@@ -331,10 +337,10 @@ def _in_allowed(inp: str, allowed: Sequence[str]) -> bool:
 class CapacityPlanner:
     def __init__(
         self,
-        default_num_simulations=128,
-        default_num_results=2,
-        default_lifecycles=(Lifecycle.stable, Lifecycle.beta),
-    ):
+        default_num_simulations: int = 128,
+        default_num_results: int = 2,
+        default_lifecycles: Tuple[Lifecycle, ...] = (Lifecycle.stable, Lifecycle.beta),
+    ) -> None:
         self._shapes: HardwareShapes = shapes
         self._models: Dict[str, CapacityModel] = {}
 
@@ -343,11 +349,11 @@ class CapacityPlanner:
         self._default_regret_params = CapacityRegretParameters()
         self._default_lifecycles = default_lifecycles
 
-    def register_group(self, group: Callable[[], Dict[str, CapacityModel]]):
+    def register_group(self, group: Callable[[], Dict[str, CapacityModel]]) -> None:
         for name, model in group().items():
             self.register_model(name, model)
 
-    def register_model(self, name: str, capacity_model: CapacityModel):
+    def register_model(self, name: str, capacity_model: CapacityModel) -> None:
         self._models[name] = capacity_model
 
     @property
@@ -429,16 +435,16 @@ class CapacityPlanner:
 
     def _group_plans_by_percentile(  # pylint: disable=too-many-positional-arguments
         self,
-        drives,
-        extra_model_arguments,
-        instance_families,
-        lifecycles,
-        num_regions,
-        num_results,
-        region,
-        model_percentile_desires,
-        sorted_percentiles,
-    ):
+        drives: Optional[Sequence[str]],
+        extra_model_arguments: Dict[str, Any],
+        instance_families: Optional[Sequence[str]],
+        lifecycles: Sequence[Lifecycle],
+        num_regions: int,
+        num_results: Optional[int],
+        region: str,
+        model_percentile_desires: Any,
+        sorted_percentiles: List[int],
+    ) -> Dict[int, Sequence[CapacityPlan]]:
         percentile_plans = {}
         for index, percentile in enumerate(sorted_percentiles):
             percentile_plan = []
@@ -470,15 +476,15 @@ class CapacityPlanner:
 
     def _mean_plan(  # pylint: disable=too-many-positional-arguments
         self,
-        drives,
-        extra_model_arguments,
-        instance_families,
-        lifecycles,
-        num_regions,
-        num_results,
-        region,
-        model_mean_desires,
-    ):
+        drives: Optional[Sequence[str]],
+        extra_model_arguments: Dict[str, Any],
+        instance_families: Optional[Sequence[str]],
+        lifecycles: Sequence[Lifecycle],
+        num_regions: int,
+        num_results: Optional[int],
+        region: str,
+        model_mean_desires: Dict[str, CapacityDesires],
+    ) -> Sequence[CapacityPlan]:
         mean_plans = []
         for mean_sub_model, mean_sub_desire in model_mean_desires.items():
             mean_sub_plan = self._plan_certain(
@@ -585,7 +591,7 @@ class CapacityPlanner:
         ]
 
     # Calculates the minimum cpu, memory, and network requirements based on desires.
-    def _per_instance_requirements(self, desires) -> Tuple[int, float]:
+    def _per_instance_requirements(self, desires: CapacityDesires) -> Tuple[int, float]:
         # Applications often set fixed reservations of heap or OS memory
         per_instance_mem = (
             desires.data_shape.reserved_instance_app_mem_gib
@@ -621,14 +627,14 @@ class CapacityPlanner:
 
     def generate_scenarios(  # pylint: disable=too-many-positional-arguments
         self,
-        model,
-        region,
-        desires,
-        num_regions,
-        lifecycles,
-        instance_families,
-        drives,
-    ):
+        model: CapacityModel,
+        region: str,
+        desires: CapacityDesires,
+        num_regions: int,
+        lifecycles: Optional[Sequence[Lifecycle]],
+        instance_families: Optional[Sequence[str]],
+        drives: Optional[Sequence[str]],
+    ) -> Generator[Tuple[Instance, Drive, RegionContext], None, None]:
         lifecycles = lifecycles or self._default_lifecycles
         instance_families = instance_families or []
         drives = drives or []
@@ -718,8 +724,8 @@ class CapacityPlanner:
         lifecycles = lifecycles or self._default_lifecycles
 
         # requirement types -> values
-        zonal_requirements: Dict[str, Dict] = {}
-        regional_requirements: Dict[str, Dict] = {}
+        zonal_requirements: Dict[str, Dict[str, List[Interval]]] = {}
+        regional_requirements: Dict[str, Dict[str, List[Interval]]] = {}
 
         regret_clusters_by_model: Dict[
             str, Sequence[Tuple[CapacityPlan, CapacityDesires, float]]
@@ -836,7 +842,7 @@ class CapacityPlanner:
         model_name: str,
         desires: CapacityDesires,
         extra_model_arguments: Dict[str, Any],
-    ):
+    ) -> Generator[Tuple[str, CapacityDesires], None, None]:
         queue: List[Tuple[CapacityDesires, str]] = [(desires, model_name)]
         models_used = []
 

{service_capacity_modeling-0.3.74 → service_capacity_modeling-0.3.79}/service_capacity_modeling/hardware/__init__.py

@@ -5,6 +5,8 @@ import logging
 import os
 from functools import reduce
 from pathlib import Path
+from typing import Any
+from typing import cast
 from typing import Dict
 from typing import List
 from typing import Optional
@@ -20,11 +22,11 @@ from service_capacity_modeling.interface import Service
 logger = logging.getLogger(__name__)
 
 
-def load_pricing(pricing: Dict) -> Pricing:
+def load_pricing(pricing: Dict[str, Any]) -> Pricing:
     return Pricing(regions=pricing)
 
 
-def load_hardware(hardware: Dict) -> Hardware:
+def load_hardware(hardware: Dict[str, Any]) -> Hardware:
     return Hardware(**hardware)
 
 
@@ -103,7 +105,9 @@ def merge_hardware(existing: Hardware, override: Hardware) -> Hardware:
             merged[key] = override_obj.get(key)
         elif isinstance(existing_field, Dict):
             override_field = override_obj.get(key)
-            merged_field = merged.setdefault(key, {})
+            if override_field is None:
+                override_field = {}
+            merged_field = cast(Dict[str, Any], merged.setdefault(key, {}))
 
             existing_keys = existing_field.keys()
             override_keys = override_obj.get(key, {}).keys()
@@ -119,7 +123,7 @@ def merge_hardware(existing: Hardware, override: Hardware) -> Hardware:
     return Hardware(**merged)
 
 
-def merge_pricing(existing: Dict, override: Dict) -> Dict:
+def merge_pricing(existing: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
     merged = existing.copy()
     for region, override_pricing in override.items():
         if region not in merged:
@@ -165,7 +169,7 @@ def load_hardware_from_disk(
     if shape_paths is None:
         shape_paths = []
 
-    combined_pricing: Dict = {}
+    combined_pricing: Dict[str, Any] = {}
 
     logger.debug("Loading pricing from: %s", price_paths)
     for price_path in price_paths:
@@ -185,7 +189,7 @@ def load_hardware_from_disk(
     return price_hardware(hardware=hardware, pricing=pricing)
 
 
-def load_hardware_from_s3(bucket, path) -> GlobalHardware:
+def load_hardware_from_s3(bucket: str, path: str) -> GlobalHardware:
     try:
         # boto is a heavy dependency so we only want to take it if
         # someone will be using it ...
@@ -205,7 +209,7 @@ def load_hardware_from_s3(bucket, path) -> GlobalHardware:
 
 
 class HardwareShapes:
-    def __init__(self):
+    def __init__(self) -> None:
         self._hardware: Optional[GlobalHardware] = None
 
     def load(self, new_hardware: GlobalHardware) -> None:

{service_capacity_modeling-0.3.74 → service_capacity_modeling-0.3.79}/service_capacity_modeling/interface.py

@@ -27,12 +27,12 @@ MEGABIT_IN_BYTES = (1000 * 1000) / 8
 
 
 class ExcludeUnsetModel(BaseModel):
-    def model_dump(self, *args, **kwargs):
+    def model_dump(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:
         if "exclude_unset" not in kwargs:
             kwargs["exclude_unset"] = True
         return super().model_dump(*args, **kwargs)
 
-    def model_dump_json(self, *args, **kwargs):
+    def model_dump_json(self, *args: Any, **kwargs: Any) -> str:
         if "exclude_unset" not in kwargs:
             kwargs["exclude_unset"] = True
         return super().model_dump_json(*args, **kwargs)
@@ -44,10 +44,10 @@ class ExcludeUnsetModel(BaseModel):
 
 
 class IntervalModel(str, Enum):
-    def __str__(self):
+    def __str__(self) -> str:
         return str(self.value)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return f"D({self.value})"
 
     gamma = "gamma"
@@ -71,11 +71,11 @@ class Interval(ExcludeUnsetModel):
     model_config = ConfigDict(frozen=True, protected_namespaces=())
 
     @property
-    def can_simulate(self):
+    def can_simulate(self) -> bool:
         return self.confidence <= 0.99 and self.allow_simulate
 
     @property
-    def minimum(self):
+    def minimum(self) -> float:
         if self.minimum_value is None:
             if self.confidence == 1.0:
                 return self.low * 0.999
@@ -84,17 +84,19 @@ class Interval(ExcludeUnsetModel):
         return self.minimum_value
 
     @property
-    def maximum(self):
+    def maximum(self) -> float:
         if self.maximum_value is None:
             if self.confidence == 1.0:
                 return self.high * 1.001
             return self.high * 2
         return self.maximum_value
 
-    def __hash__(self):
+    def __hash__(self) -> int:
         return hash((type(self),) + tuple(self.__dict__.values()))
 
-    def __eq__(self, other):
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, Interval):
+            return False
         return self.__hash__() == other.__hash__()
 
     def scale(self, factor: float) -> Interval:
@@ -264,14 +266,14 @@ class Drive(ExcludeUnsetModel):
         return max(self.block_size_kib, self.group_size_kib)
 
     @property
-    def max_size_gib(self):
+    def max_size_gib(self) -> float:
         if self.max_scale_size_gib != 0:
             return self.max_scale_size_gib
         else:
             return self.size_gib
 
     @property
-    def max_io_per_s(self):
+    def max_io_per_s(self) -> int:
         if self.max_scale_io_per_s != 0:
             return self.max_scale_io_per_s
         else:
@@ -279,7 +281,7 @@ class Drive(ExcludeUnsetModel):
 
     @computed_field(return_type=float)  # type: ignore
     @property
-    def annual_cost(self):
+    def annual_cost(self) -> float:
         size = self.size_gib or 0
         r_ios = self.read_io_per_s or 0
         w_ios = self.write_io_per_s or 0
@@ -382,15 +384,15 @@ class Instance(ExcludeUnsetModel):
     family_separator: str = "."
 
     @property
-    def family(self):
+    def family(self) -> str:
         return self.name.rsplit(self.family_separator, 1)[0]
 
     @property
-    def size(self):
+    def size(self) -> str:
         return self.name.rsplit(self.family_separator, 1)[1]
 
     @property
-    def cores(self):
+    def cores(self) -> int:
         if self.cpu_cores is not None:
             return self.cpu_cores
         return self.cpu // 2
@@ -456,7 +458,7 @@ class Service(ExcludeUnsetModel):
         low=1, mid=10, high=50, confidence=0.9
     )
 
-    def annual_cost_gib(self, data_gib: float = 0):
+    def annual_cost_gib(self, data_gib: float = 0) -> float:
         if isinstance(self.annual_cost_per_gib, float):
             return self.annual_cost_per_gib * data_gib
         else:
@@ -779,23 +781,48 @@ class BufferComponent(str, Enum):
     compute = "compute"
     # [Data Shape]    a.k.a. "Dataset" related buffers, e.g. Disk and Memory
     storage = "storage"
-
     # Resource specific component
     cpu = "cpu"
     network = "network"
     disk = "disk"
     memory = "memory"
 
+    @staticmethod
+    def is_generic(component: str) -> bool:
+        return component in {BufferComponent.compute, BufferComponent.storage}
+
+    @staticmethod
+    def is_specific(component: str) -> bool:
+        return not BufferComponent.is_generic(component)
+
 
 class BufferIntent(str, Enum):
     # Most buffers show "desired" buffer, this is the default
     desired = "desired"
     # ratio on top of existing buffers to ensure exists. Generally combined
     # with a different desired buffer to ensure we don't just scale needlessly
+    # This means we can scale up or down as as long as we meet the desired buffer.
     scale = "scale"
-    # Ignore model preferences, just preserve existing buffers
+
+    # DEPRECATED: Use scale_up/scale_down instead
+    # Ignores model preferences, just preserve existing buffers
+    # We rarely actually want to do this since it can cause severe over provisioning
     preserve = "preserve"
 
+    # Scale up if necessary to meet the desired buffer.
+    # If the existing resource is over-provisioned, do not reduce the requirement.
+    # If under-provisioned, the requirement can be increased to meet the desired buffer.
+    # Example: need 20 cores but have 10 → scale up to 20 cores.
+    # Example 2: need 20 cores but have 40 → do not scale down and require at
+    # least 40 cores
+    scale_up = "scale_up"
+    # Scale down if necessary to meet the desired buffer.
+    # If the existing resource is under-provisioned, do not increase the requirement.
+    # If over-provisioned, the requirement can be decreased to meet the desired buffer.
+    # Example: need 20 cores but have 10 → maintain buffer and do not scale up.
+    # Example 2: need 20 cores but have 40 → scale down to 20 cores.
+    scale_down = "scale_down"
+
 
 class Buffer(ExcludeUnsetModel):
     # The value of the buffer expressed as a ratio over "normal" load e.g. 1.5x
@@ -819,7 +846,6 @@ class Buffers(ExcludeUnsetModel):
             "compute": Buffer(ratio: 1.5),
         }
     )
-
     And then models layer in their buffers, for example if a workload
     requires 10 CPU cores, but the operator of that workload  likes to build in
     2x buffer for background work (20 cores provisioned), they would express that
@@ -955,7 +981,7 @@ class CapacityRequirement(ExcludeUnsetModel):
     network_mbps: Interval = certain_int(0)
     disk_gib: Interval = certain_int(0)
 
-    context: Dict = {}
+    context: Dict[str, Any] = {}
 
 
 class ClusterCapacity(ExcludeUnsetModel):
@@ -968,7 +994,7 @@ class ClusterCapacity(ExcludeUnsetModel):
     # When provisioning services we might need to signal they
     # should have certain configuration, for example flags that
     # affect durability shut off
-    cluster_params: Dict = {}
+    cluster_params: Dict[str, Any] = {}
 
 
 class ServiceCapacity(ExcludeUnsetModel):
@@ -979,7 +1005,7 @@ class ServiceCapacity(ExcludeUnsetModel):
     regret_cost: bool = False
     # Often while provisioning cloud services we need to represent
     # parameters to the cloud APIs, use this to inject those from models
-    service_params: Dict = {}
+    service_params: Dict[str, Any] = {}
 
 
 # For services that are provisioned by zone (e.g. Cassandra, EVCache)

{service_capacity_modeling-0.3.74 → service_capacity_modeling-0.3.79}/service_capacity_modeling/models/__init__.py

@@ -20,6 +20,25 @@ from service_capacity_modeling.interface import Platform
 from service_capacity_modeling.interface import QueryPattern
 from service_capacity_modeling.interface import RegionContext
 
+__all__ = [
+    "AccessConsistency",
+    "AccessPattern",
+    "CapacityDesires",
+    "CapacityPlan",
+    "CapacityRegretParameters",
+    "certain_float",
+    "Consistency",
+    "DataShape",
+    "Drive",
+    "FixedInterval",
+    "GlobalConsistency",
+    "Instance",
+    "Platform",
+    "QueryPattern",
+    "RegionContext",
+    "CapacityModel",
+]
+
 __common_regrets__ = frozenset(("spend", "disk", "mem"))
 
 
@@ -85,7 +104,7 @@ class CapacityModel:
 
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         pass
 
     @staticmethod
@@ -270,7 +289,7 @@ class CapacityModel:
     @staticmethod
     def default_desires(
         user_desires: CapacityDesires, extra_model_arguments: Dict[str, Any]
-    ):
+    ) -> CapacityDesires:
         """Optional defaults to apply given a user desires
 
         Often users do not know what the on-cpu time of their queries