service-capacity-modeling 0.3.72__tar.gz → 0.3.74__tar.gz

{service_capacity_modeling-0.3.72 → service_capacity_modeling-0.3.74}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.72
+Version: 0.3.74
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com

service_capacity_modeling-0.3.74/service_capacity_modeling/hardware/profiles/shapes/aws/auto_i3en.json

@@ -0,0 +1,172 @@
+{
+  "instances": {
+    "i3en.large": {
+      "name": "i3en.large",
+      "cpu": 2,
+      "cpu_cores": 1,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 15.26,
+      "net_mbps": 2100.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 1164,
+        "read_io_per_s": 42500,
+        "write_io_per_s": 32500,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.xlarge": {
+      "name": "i3en.xlarge",
+      "cpu": 4,
+      "cpu_cores": 2,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 30.52,
+      "net_mbps": 4200.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 2328,
+        "read_io_per_s": 85000,
+        "write_io_per_s": 65000,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.2xlarge": {
+      "name": "i3en.2xlarge",
+      "cpu": 8,
+      "cpu_cores": 4,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 61.04,
+      "net_mbps": 8400.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 4657,
+        "read_io_per_s": 170000,
+        "write_io_per_s": 130000,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.3xlarge": {
+      "name": "i3en.3xlarge",
+      "cpu": 12,
+      "cpu_cores": 6,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 91.55,
+      "net_mbps": 12500.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 6985,
+        "read_io_per_s": 255000,
+        "write_io_per_s": 195000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.6xlarge": {
+      "name": "i3en.6xlarge",
+      "cpu": 24,
+      "cpu_cores": 12,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 183.11,
+      "net_mbps": 25000.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 13970,
+        "read_io_per_s": 510000,
+        "write_io_per_s": 390000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.12xlarge": {
+      "name": "i3en.12xlarge",
+      "cpu": 48,
+      "cpu_cores": 24,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 366.21,
+      "net_mbps": 50000.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 27940,
+        "read_io_per_s": 1020000,
+        "write_io_per_s": 780000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i3en.24xlarge": {
+      "name": "i3en.24xlarge",
+      "cpu": 96,
+      "cpu_cores": 48,
+      "cpu_ghz": 3.1,
+      "cpu_ipc_scale": 1.0,
+      "ram_gib": 732.42,
+      "net_mbps": 100000.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 55879,
+        "read_io_per_s": 2040000,
+        "write_io_per_s": 1560000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.08,
+          "mid": 0.125,
+          "high": 0.2,
+          "confidence": 0.9,
+          "minimum_value": 0.07,
+          "maximum_value": 2.0
+        }
+      }
+    }
+  }
+}

service_capacity_modeling-0.3.74/service_capacity_modeling/hardware/profiles/shapes/aws/auto_i4i.json

@@ -0,0 +1,220 @@
+{
+  "instances": {
+    "i4i.large": {
+      "name": "i4i.large",
+      "cpu": 2,
+      "cpu_cores": 1,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 15.26,
+      "net_mbps": 781.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 436,
+        "read_io_per_s": 50000,
+        "write_io_per_s": 27500,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.xlarge": {
+      "name": "i4i.xlarge",
+      "cpu": 4,
+      "cpu_cores": 2,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 30.52,
+      "net_mbps": 1875.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 873,
+        "read_io_per_s": 100000,
+        "write_io_per_s": 55000,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.2xlarge": {
+      "name": "i4i.2xlarge",
+      "cpu": 8,
+      "cpu_cores": 4,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 61.04,
+      "net_mbps": 4687.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 1746,
+        "read_io_per_s": 200000,
+        "write_io_per_s": 110000,
+        "single_tenant": false,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.4xlarge": {
+      "name": "i4i.4xlarge",
+      "cpu": 16,
+      "cpu_cores": 8,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 122.07,
+      "net_mbps": 9375.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 3492,
+        "read_io_per_s": 400000,
+        "write_io_per_s": 220000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.8xlarge": {
+      "name": "i4i.8xlarge",
+      "cpu": 32,
+      "cpu_cores": 16,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 244.14,
+      "net_mbps": 18750.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 6985,
+        "read_io_per_s": 800000,
+        "write_io_per_s": 440000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.12xlarge": {
+      "name": "i4i.12xlarge",
+      "cpu": 48,
+      "cpu_cores": 24,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 366.21,
+      "net_mbps": 28125.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 10477,
+        "read_io_per_s": 1200000,
+        "write_io_per_s": 660000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.16xlarge": {
+      "name": "i4i.16xlarge",
+      "cpu": 64,
+      "cpu_cores": 32,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 488.28,
+      "net_mbps": 37500.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 13970,
+        "read_io_per_s": 1600000,
+        "write_io_per_s": 880000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.24xlarge": {
+      "name": "i4i.24xlarge",
+      "cpu": 96,
+      "cpu_cores": 48,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 732.42,
+      "net_mbps": 56250.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 20955,
+        "read_io_per_s": 2400000,
+        "write_io_per_s": 1320000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    },
+    "i4i.32xlarge": {
+      "name": "i4i.32xlarge",
+      "cpu": 128,
+      "cpu_cores": 64,
+      "cpu_ghz": 3.5,
+      "cpu_ipc_scale": 1.15,
+      "ram_gib": 976.56,
+      "net_mbps": 75000.0,
+      "drive": {
+        "name": "ephem",
+        "size_gib": 27940,
+        "read_io_per_s": 3200000,
+        "write_io_per_s": 1760000,
+        "single_tenant": true,
+        "read_io_latency_ms": {
+          "low": 0.1,
+          "mid": 0.125,
+          "high": 0.17,
+          "confidence": 0.9,
+          "minimum_value": 0.05,
+          "maximum_value": 2.0
+        }
+      }
+    }
+  }
+}

{service_capacity_modeling-0.3.72 → service_capacity_modeling-0.3.74}/service_capacity_modeling/hardware/profiles/shapes/aws/manual_instances.json

@@ -55,190 +55,6 @@
         "block_size_kib": 4, "single_tenant": true
       }
     },
-    "i4i.large": {
-      "name": "i4i.large",
-      "cpu": 2,
-      "cpu_ghz": 3.5,
-      "cpu_ipc_scale": 1.15,
-      "ram_gib": 15.48,
-      "net_mbps": 781,
-      "drive": {
-        "name": "ephem", "size_gib": 436,
-        "read_io_latency_ms": {
-            "minimum_value":0.05,
-            "low":0.10, "mid":0.125, "high":0.17,
-            "maximum_value":2.05, "confidence":0.9
-        },
-        "read_io_per_s": 50000, "write_io_per_s": 27500,
-        "block_size_kib": 4, "single_tenant": false
-      }
-    },
-    "i4i.xlarge": {
-      "name": "i4i.xlarge",
-      "cpu": 4,
-      "cpu_ghz": 3.5,
-      "cpu_ipc_scale": 1.15,
-      "ram_gib": 30.955,
-      "net_mbps": 1875,
-      "drive": {
-        "name": "ephem", "size_gib": 873,
-        "read_io_latency_ms": {
-            "minimum_value": 0.05,
-            "low": 0.10, "mid": 0.125, "high": 0.17,
-            "maximum_value": 2.05, "confidence": 0.9
-        },
-        "read_io_per_s": 100000, "write_io_per_s": 55000,
-        "block_size_kib": 4, "single_tenant": false
-      }
-    },
-    "i4i.2xlarge": {
-      "name": "i4i.2xlarge",
-      "cpu": 8,
-      "cpu_ghz": 3.5,
-      "cpu_ipc_scale": 1.15,
-      "ram_gib": 61.91,
-      "net_mbps": 4687.5 ,
-      "drive": {
-        "name": "ephem", "size_gib": 1746,
-        "read_io_latency_ms": {
-            "minimum_value": 0.05,
-            "low": 0.10, "mid": 0.125, "high": 0.17,
-            "maximum_value": 2.05, "confidence": 0.9
-        },
-        "read_io_per_s": 200000, "write_io_per_s": 110000,
-        "block_size_kib": 4, "single_tenant": true
-      }
-    },
-    "i4i.4xlarge": {
-      "name": "i4i.4xlarge",
-      "cpu": 16,
-      "cpu_ghz": 3.5,
-      "cpu_ipc_scale": 1.15,
-      "ram_gib": 123.82,
-      "net_mbps": 9375,
-      "drive": {
-        "name": "ephem", "size_gib": 3492,
-        "read_io_latency_ms": {
-            "minimum_value":0.05,
-            "low":0.10, "mid":0.125, "high":0.17,
-            "maximum_value":2.05, "confidence":0.9
-        },
-        "read_io_per_s": 400000, "write_io_per_s": 220000,
-        "block_size_kib": 4, "single_tenant": true
-      }
-    },
-    "i4i.8xlarge": {
-      "name": "i4i.8xlarge",
-      "cpu": 32,
-      "cpu_ghz": 3.5,
-      "cpu_ipc_scale": 1.15,
-      "ram_gib": 247.76,
-      "net_mbps": 18750,
-      "drive": {
-        "name": "ephem", "size_gib": 6984,
-        "read_io_latency_ms": {
-            "minimum_value":0.05,
-            "low":0.10, "mid":0.125, "high":0.17,
-            "maximum_value":2.05, "confidence":0.9
-        },
-        "read_io_per_s": 800000, "write_io_per_s": 440000,
-        "block_size_kib": 4, "single_tenant": true
-      }
-    },
-    "i4i.16xlarge": {
-      "name": "i4i.16xlarge",
-      "cpu": 64,
-      "cpu_ghz": 3.5,
-      "cpu_ipc_scale": 1.15,
-      "ram_gib": 495.82,
-      "net_mbps": 35000,
-      "drive": {
-        "name": "ephem", "size_gib": 13968,
-        "read_io_latency_ms": {
-            "minimum_value":0.05,
-            "low":0.10, "mid":0.125, "high":0.17,
-            "maximum_value":2.05, "confidence":0.9
-        },
-        "read_io_per_s": 16000000, "write_io_per_s": 880000,
-        "block_size_kib": 4, "single_tenant": true
-      }
-    },
-
-    "i3en.large": {
-      "name": "i3en.large",
-      "cpu": 2,
-      "cpu_ghz": 3.1,
-      "ram_gib": 15.8,
-      "net_mbps": 4000,
-      "drive": {
-        "name": "ephem", "size_gib": 1150,
-        "read_io_latency_ms": {
-          "minimum_value":0.07,
-          "low":0.08, "mid":0.12, "high":0.20,
-          "maximum_value":2, "confidence":0.9
-        },
-        "read_io_per_s": 42500, "write_io_per_s": 32500,
-        "block_size_kib": 4, "single_tenant": false
-      }
-    },
-    "i3en.xlarge": {
-      "name": "i3en.xlarge",
-      "cpu": 4,
-      "cpu_ghz": 3.1,
-      "ram_gib": 31.7,
-      "net_mbps": 4000,
-      "drive": {
-        "name": "ephem", "size_gib": 2300,
-        "read_io_latency_ms": {
-            "minimum_value":0.07,
-            "low":0.08, "mid":0.12, "high":0.20,
-            "maximum_value":2, "confidence":0.9
-        },
-        "read_io_per_s": 85000, "write_io_per_s": 65000,
-        "block_size_kib": 4, "single_tenant": false
-      }
-    },
-    "i3en.2xlarge": {
-      "name": "i3en.2xlarge",
-      "cpu": 8,
-      "cpu_ghz": 3.1,
-      "ram_gib": 63.62,
-      "net_mbps": 8000,
-      "drive": {
-        "name": "ephem", "size_gib": 4600,
-        "read_io_latency_ms": {
-            "minimum_value": 0.07,
-            "low":0.08, "mid":0.12, "high": 0.20,
-            "maximum_value":2, "confidence":0.9
-        },
-        "read_io_per_s": 170000, "write_io_per_s": 130000,
-        "block_size_kib": 4, "single_tenant": false
-      }
-    },
-    "i3en.3xlarge": {
-      "name": "i3en.3xlarge",
-      "cpu": 12,
-      "cpu_ghz": 3.1,
-      "ram_gib": 95.54,
-      "net_mbps": 12000,
-      "drive": {"name": "ephem", "size_gib": 6819,
-        "read_io_latency_ms": {"minimum_value":0.07, "low":0.08, "mid":0.12, "high":0.16, "maximum_value":2, "confidence":0.9},
-        "read_io_per_s": 250000, "write_io_per_s": 200000,
-        "block_size_kib": 4, "single_tenant": true
-      }
-    },
-    "i3en.6xlarge": {
-      "name": "i3en.6xlarge",
-      "cpu": 24,
-      "cpu_ghz": 3.1,
-      "ram_gib": 186.62,
-      "net_mbps": 24000,
-      "drive": {"name": "ephem", "size_gib": 14000,
-        "read_io_latency_ms": {"minimum_value":0.07, "low":0.08, "mid":0.12, "high":0.16, "maximum_value":2, "confidence":0.9},
-        "read_io_per_s": 500000, "write_io_per_s": 400000,
-        "block_size_kib": 4, "single_tenant": true
-      }
-    },
     "m5d.large": {
       "name": "m5d.large",
       "cpu": 2,

{service_capacity_modeling-0.3.72 → service_capacity_modeling-0.3.74}/service_capacity_modeling/models/org/netflix/elasticsearch.py

@@ -82,6 +82,11 @@ def _estimate_elasticsearch_requirement(  # noqa: E501 pylint: disable=too-many-
     The input desires should be the **regional** desire, and this function will
     return the zonal capacity requirement
     """
+    # NOTE: ElasticSearch is only using buffers for disk but CPU and disk
+    # have hard coded values. In a follow-up change these values (1.5 and 2)
+    # should be converted to buffers as well to be idiomatic with the other
+    # stateful models (e.g. cassandra / EVCache, kafka)
+
     # Keep half of the cores free for background work (merging mostly)
     needed_cores = normalize_cores(
         core_count=math.ceil(sqrt_staffed_cores(desires) * 1.5),
@@ -191,9 +196,9 @@ class NflxElasticsearchDataCapacityModel(CapacityModel):
     def default_desires(
         user_desires, extra_model_arguments: Dict[str, Any]
     ) -> CapacityDesires:
-        return CapacityDesires(
-            buffers=NflxElasticsearchDataCapacityModel.default_buffers()
-        )
+        desires = CapacityModel.default_desires(user_desires, extra_model_arguments)
+        desires.buffers = NflxElasticsearchDataCapacityModel.default_buffers()
+        return desires
 
     @staticmethod
     def capacity_plan(

{service_capacity_modeling-0.3.72 → service_capacity_modeling-0.3.74}/service_capacity_modeling/tools/instance_families.py

@@ -95,12 +95,6 @@ INSTANCE_TYPES = {
     #     'cpu_ipc_scale': None
     # },
     # "i3": {'xl_iops': None, 'io_latency_curve': 'ssd', 'cpu_ipc_scale': None},
-    # "i3en": {'xl_iops': None, 'io_latency_curve': 'ssd', 'cpu_ipc_scale': None},
-    # "i4i": {
-    #     'xl_iops': None,
-    #     'io_latency_curve': '5th-gen-ssd',
-    #     'cpu_ipc_scale': None
-    # },
     "i7i": {
         "xl_iops": "150,000/82,500",
         "io_latency_curve": "7th-gen-ephemeral",
@@ -108,6 +102,20 @@ INSTANCE_TYPES = {
         "cpu_turbo_single_ghz": 4.0,
         "cpu_turbo_all_ghz": 3.2,
     },
+    "i3en": {
+        "xl_iops": None,
+        "io_latency_curve": "ssd",
+        "cpu_ipc_scale": SKYLAKE_IPC,
+        "cpu_turbo_single_ghz": 3.1,
+        "cpu_turbo_all_ghz": 3.1,
+    },
+    "i4i": {
+        "xl_iops": None,
+        "io_latency_curve": "ssd",
+        "cpu_ipc_scale": ICELAKE_IPC,
+        "cpu_turbo_single_ghz": 3.5,
+        "cpu_turbo_all_ghz": 3.5,
+    },
     "m4": {
         "xl_iops": None,
         "io_latency_curve": None,

{service_capacity_modeling-0.3.72 → service_capacity_modeling-0.3.74}/service_capacity_modeling.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: service-capacity-modeling
-Version: 0.3.72
+Version: 0.3.74
 Summary: Contains utilities for modeling capacity for pluggable workloads
 Author: Joseph Lynch
 Author-email: josephl@netflix.com

{service_capacity_modeling-0.3.72 → service_capacity_modeling-0.3.74}/service_capacity_modeling.egg-info/SOURCES.txt

@@ -26,6 +26,8 @@ service_capacity_modeling/hardware/profiles/shapes/aws/auto_c6id.json
 service_capacity_modeling/hardware/profiles/shapes/aws/auto_c7a.json
 service_capacity_modeling/hardware/profiles/shapes/aws/auto_c7i.json
 service_capacity_modeling/hardware/profiles/shapes/aws/auto_c8i.json
+service_capacity_modeling/hardware/profiles/shapes/aws/auto_i3en.json
+service_capacity_modeling/hardware/profiles/shapes/aws/auto_i4i.json
 service_capacity_modeling/hardware/profiles/shapes/aws/auto_i7i.json
 service_capacity_modeling/hardware/profiles/shapes/aws/auto_m4.json
 service_capacity_modeling/hardware/profiles/shapes/aws/auto_m5.json