redis-benchmarks-specification 0.2.18__py3-none-any.whl → 0.2.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of redis-benchmarks-specification might be problematic. Click here for more details.
- redis_benchmarks_specification/__common__/timeseries.py +21 -7
- redis_benchmarks_specification/__self_contained_coordinator__/args.py +6 -0
- redis_benchmarks_specification/__self_contained_coordinator__/docker.py +5 -3
- redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py +4 -2
- redis_benchmarks_specification/__self_contained_coordinator__/runners.py +7 -3
- redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +204 -104
- redis_benchmarks_specification/setups/topologies/topologies.yml +9 -9
- {redis_benchmarks_specification-0.2.18.dist-info → redis_benchmarks_specification-0.2.20.dist-info}/METADATA +1 -1
- {redis_benchmarks_specification-0.2.18.dist-info → redis_benchmarks_specification-0.2.20.dist-info}/RECORD +12 -12
- {redis_benchmarks_specification-0.2.18.dist-info → redis_benchmarks_specification-0.2.20.dist-info}/LICENSE +0 -0
- {redis_benchmarks_specification-0.2.18.dist-info → redis_benchmarks_specification-0.2.20.dist-info}/WHEEL +0 -0
- {redis_benchmarks_specification-0.2.18.dist-info → redis_benchmarks_specification-0.2.20.dist-info}/entry_points.txt +0 -0
|
@@ -139,9 +139,13 @@ def extract_results_table(
|
|
|
139
139
|
|
|
140
140
|
# Debug logging for JSON path resolution
|
|
141
141
|
if "ALL STATS" in metric_jsonpath or "ALL_STATS" in metric_jsonpath:
|
|
142
|
-
logging.info(
|
|
142
|
+
logging.info(
|
|
143
|
+
f"DEBUG: Found {len(find_res)} results for JSONPath '{metric_jsonpath}'"
|
|
144
|
+
)
|
|
143
145
|
for i, result in enumerate(find_res):
|
|
144
|
-
logging.info(
|
|
146
|
+
logging.info(
|
|
147
|
+
f" Result {i}: path='{result.path}', value={result.value}, context='{result.context.path}'"
|
|
148
|
+
)
|
|
145
149
|
|
|
146
150
|
for metric in find_res:
|
|
147
151
|
metric_name = str(metric.path)
|
|
@@ -209,11 +213,17 @@ def extract_results_table(
|
|
|
209
213
|
)
|
|
210
214
|
# Debug logging for missing metrics - show available keys
|
|
211
215
|
if "ALL STATS" in str(jsonpath) or "ALL_STATS" in str(jsonpath):
|
|
212
|
-
logging.info(
|
|
216
|
+
logging.info(
|
|
217
|
+
f"DEBUG: Available top-level keys in results_dict: {list(results_dict.keys())}"
|
|
218
|
+
)
|
|
213
219
|
if "ALL STATS" in results_dict:
|
|
214
|
-
logging.info(
|
|
220
|
+
logging.info(
|
|
221
|
+
f"DEBUG: Keys in 'ALL STATS': {list(results_dict['ALL STATS'].keys())}"
|
|
222
|
+
)
|
|
215
223
|
if "ALL_STATS" in results_dict:
|
|
216
|
-
logging.info(
|
|
224
|
+
logging.info(
|
|
225
|
+
f"DEBUG: Keys in 'ALL_STATS': {list(results_dict['ALL_STATS'].keys())}"
|
|
226
|
+
)
|
|
217
227
|
return results_matrix
|
|
218
228
|
|
|
219
229
|
|
|
@@ -422,9 +432,13 @@ def common_timeseries_extraction(
|
|
|
422
432
|
use_metric_context_path = cleaned_metric[5]
|
|
423
433
|
|
|
424
434
|
# Debug logging for metric extraction
|
|
425
|
-
logging.info(
|
|
435
|
+
logging.info(
|
|
436
|
+
f"Extracted metric - JSONPath: '{metric_jsonpath}', Name: '{metric_name}', Value: {metric_value}"
|
|
437
|
+
)
|
|
426
438
|
if "ALL_STATS.Totals.Ops/sec" in metric_name or "ALL STATS" in metric_jsonpath:
|
|
427
|
-
logging.warning(
|
|
439
|
+
logging.warning(
|
|
440
|
+
f"DEBUG ALL_STATS metric - JSONPath: '{metric_jsonpath}', Name: '{metric_name}', Value: {metric_value}, Context: '{metric_context_path}')"
|
|
441
|
+
)
|
|
428
442
|
|
|
429
443
|
target_table_keyname, target_table_dict = from_metric_kv_to_timeserie(
|
|
430
444
|
break_by_key,
|
|
@@ -207,4 +207,10 @@ def create_self_contained_coordinator_args(project_name):
|
|
|
207
207
|
action="store_true",
|
|
208
208
|
help="Skip automatically clearing pending messages and resetting consumer group position on startup. By default, pending messages are cleared and consumer group is reset to latest position to skip old work and recover from crashes.",
|
|
209
209
|
)
|
|
210
|
+
parser.add_argument(
|
|
211
|
+
"--enable-cpu-distribution",
|
|
212
|
+
default=False,
|
|
213
|
+
action="store_true",
|
|
214
|
+
help="Enable CPU distribution optimization using Docker cpuset_cpus and nano_cpus parameters. This forces better CPU core distribution for Redis and client containers. Disabled by default.",
|
|
215
|
+
)
|
|
210
216
|
return parser
|
|
@@ -86,7 +86,7 @@ def spin_docker_standalone_redis(
|
|
|
86
86
|
ceil_db_cpu_limit, current_cpu_pos
|
|
87
87
|
)
|
|
88
88
|
# Calculate nano_cpus for better CPU distribution
|
|
89
|
-
redis_cpu_count = len(db_cpuset_cpus.split(
|
|
89
|
+
redis_cpu_count = len(db_cpuset_cpus.split(","))
|
|
90
90
|
redis_nano_cpus = int(redis_cpu_count * 1e9) # 1 CPU = 1e9 nano_cpus
|
|
91
91
|
|
|
92
92
|
logging.info(
|
|
@@ -94,7 +94,9 @@ def spin_docker_standalone_redis(
|
|
|
94
94
|
run_image, db_cpuset_cpus, command_str
|
|
95
95
|
)
|
|
96
96
|
)
|
|
97
|
-
logging.info(
|
|
97
|
+
logging.info(
|
|
98
|
+
f"Redis container will use {redis_cpu_count} CPUs (nano_cpus={redis_nano_cpus}) on cores {db_cpuset_cpus}"
|
|
99
|
+
)
|
|
98
100
|
|
|
99
101
|
container = docker_client.containers.run(
|
|
100
102
|
image=run_image,
|
|
@@ -109,7 +111,7 @@ def spin_docker_standalone_redis(
|
|
|
109
111
|
detach=True,
|
|
110
112
|
cpuset_cpus=db_cpuset_cpus,
|
|
111
113
|
nano_cpus=redis_nano_cpus, # Force CPU distribution
|
|
112
|
-
|
|
114
|
+
# pid_mode="host",
|
|
113
115
|
)
|
|
114
116
|
redis_containers.append(container)
|
|
115
117
|
return current_cpu_pos
|
|
@@ -68,10 +68,12 @@ def data_prepopulation_step(
|
|
|
68
68
|
preload_start_time = datetime.datetime.now()
|
|
69
69
|
|
|
70
70
|
# Calculate nano_cpus for better CPU distribution
|
|
71
|
-
preload_cpu_count = len(client_cpuset_cpus.split(
|
|
71
|
+
preload_cpu_count = len(client_cpuset_cpus.split(","))
|
|
72
72
|
preload_nano_cpus = int(preload_cpu_count * 1e9) # 1 CPU = 1e9 nano_cpus
|
|
73
73
|
|
|
74
|
-
logging.info(
|
|
74
|
+
logging.info(
|
|
75
|
+
f"Preload container will use {preload_cpu_count} CPUs (nano_cpus={preload_nano_cpus}) on cores {client_cpuset_cpus}"
|
|
76
|
+
)
|
|
75
77
|
|
|
76
78
|
client_container_stdout = docker_client.containers.run(
|
|
77
79
|
image=preload_image,
|
|
@@ -541,10 +541,14 @@ def process_self_contained_coordinator_stream(
|
|
|
541
541
|
benchmark_start_time = datetime.datetime.now()
|
|
542
542
|
|
|
543
543
|
# Calculate nano_cpus for better CPU distribution
|
|
544
|
-
client_cpu_count = len(client_cpuset_cpus.split(
|
|
545
|
-
client_nano_cpus = int(
|
|
544
|
+
client_cpu_count = len(client_cpuset_cpus.split(","))
|
|
545
|
+
client_nano_cpus = int(
|
|
546
|
+
client_cpu_count * 1e9
|
|
547
|
+
) # 1 CPU = 1e9 nano_cpus
|
|
546
548
|
|
|
547
|
-
logging.info(
|
|
549
|
+
logging.info(
|
|
550
|
+
f"Client container will use {client_cpu_count} CPUs (nano_cpus={client_nano_cpus}) on cores {client_cpuset_cpus}"
|
|
551
|
+
)
|
|
548
552
|
|
|
549
553
|
client_container_stdout = docker_client.containers.run(
|
|
550
554
|
image=client_container_image,
|
|
@@ -772,6 +772,7 @@ def main():
|
|
|
772
772
|
priority_upper_limit,
|
|
773
773
|
default_baseline_branch,
|
|
774
774
|
default_metrics_str,
|
|
775
|
+
args=args,
|
|
775
776
|
)
|
|
776
777
|
|
|
777
778
|
|
|
@@ -808,6 +809,7 @@ def self_contained_coordinator_blocking_read(
|
|
|
808
809
|
default_metrics_str="ALL_STATS.Totals.Ops/sec",
|
|
809
810
|
docker_keep_env=False,
|
|
810
811
|
restore_build_artifacts_default=True,
|
|
812
|
+
args=None,
|
|
811
813
|
):
|
|
812
814
|
num_process_streams = 0
|
|
813
815
|
num_process_test_suites = 0
|
|
@@ -1405,6 +1407,10 @@ def process_self_contained_coordinator_stream(
|
|
|
1405
1407
|
redis_containers,
|
|
1406
1408
|
run_image,
|
|
1407
1409
|
temporary_dir,
|
|
1410
|
+
auto_remove=False,
|
|
1411
|
+
enable_cpu_distribution=(
|
|
1412
|
+
args.enable_cpu_distribution if args else False
|
|
1413
|
+
),
|
|
1408
1414
|
)
|
|
1409
1415
|
|
|
1410
1416
|
r = redis.StrictRedis(
|
|
@@ -1600,63 +1606,119 @@ def process_self_contained_coordinator_stream(
|
|
|
1600
1606
|
f"Using default container timeout: {container_timeout}s"
|
|
1601
1607
|
)
|
|
1602
1608
|
|
|
1603
|
-
#
|
|
1604
|
-
|
|
1605
|
-
|
|
1609
|
+
# Prepare client container arguments
|
|
1610
|
+
client_container_args = {
|
|
1611
|
+
"image": client_container_image,
|
|
1612
|
+
"volumes": {
|
|
1613
|
+
temporary_dir_client: {
|
|
1614
|
+
"bind": client_mnt_point,
|
|
1615
|
+
"mode": "rw",
|
|
1616
|
+
},
|
|
1617
|
+
},
|
|
1618
|
+
"auto_remove": False, # Don't auto-remove so we can get logs if timeout
|
|
1619
|
+
"privileged": True,
|
|
1620
|
+
"working_dir": benchmark_tool_workdir,
|
|
1621
|
+
"command": benchmark_command_str,
|
|
1622
|
+
"network_mode": "host",
|
|
1623
|
+
"detach": True, # Detach to enable timeout
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
# Add CPU distribution settings if enabled
|
|
1627
|
+
enable_cpu_distribution = (
|
|
1628
|
+
args.enable_cpu_distribution if args else False
|
|
1629
|
+
)
|
|
1630
|
+
if enable_cpu_distribution:
|
|
1631
|
+
client_cpu_count = len(
|
|
1632
|
+
client_cpuset_cpus.split(",")
|
|
1633
|
+
)
|
|
1634
|
+
client_nano_cpus = int(
|
|
1635
|
+
client_cpu_count * 1e9
|
|
1636
|
+
) # 1 CPU = 1e9 nano_cpus
|
|
1637
|
+
|
|
1638
|
+
client_container_args["cpuset_cpus"] = (
|
|
1639
|
+
client_cpuset_cpus
|
|
1640
|
+
)
|
|
1641
|
+
client_container_args["nano_cpus"] = (
|
|
1642
|
+
client_nano_cpus
|
|
1643
|
+
)
|
|
1606
1644
|
|
|
1607
|
-
|
|
1645
|
+
logging.info(
|
|
1646
|
+
f"Client container will use {client_cpu_count} CPUs (nano_cpus={client_nano_cpus}) on cores {client_cpuset_cpus} [CPU distribution enabled]"
|
|
1647
|
+
)
|
|
1648
|
+
else:
|
|
1649
|
+
logging.info(
|
|
1650
|
+
f"Client container will use default CPU allocation [CPU distribution disabled]"
|
|
1651
|
+
)
|
|
1608
1652
|
|
|
1609
1653
|
try:
|
|
1610
1654
|
# Start container with detach=True to enable timeout handling
|
|
1611
1655
|
container = docker_client.containers.run(
|
|
1612
|
-
|
|
1613
|
-
volumes={
|
|
1614
|
-
temporary_dir_client: {
|
|
1615
|
-
"bind": client_mnt_point,
|
|
1616
|
-
"mode": "rw",
|
|
1617
|
-
},
|
|
1618
|
-
},
|
|
1619
|
-
auto_remove=False, # Don't auto-remove so we can get logs if timeout
|
|
1620
|
-
privileged=True,
|
|
1621
|
-
working_dir=benchmark_tool_workdir,
|
|
1622
|
-
command=benchmark_command_str,
|
|
1623
|
-
network_mode="host",
|
|
1624
|
-
detach=True, # Detach to enable timeout
|
|
1625
|
-
cpuset_cpus=client_cpuset_cpus,
|
|
1626
|
-
nano_cpus=client_nano_cpus, # Force CPU distribution
|
|
1656
|
+
**client_container_args
|
|
1627
1657
|
)
|
|
1628
1658
|
|
|
1629
1659
|
logging.info(
|
|
1630
1660
|
f"Started container {container.name} ({container.id[:12]}) with {container_timeout}s timeout"
|
|
1631
1661
|
)
|
|
1632
1662
|
|
|
1633
|
-
# Apply CPU affinity using taskset
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1663
|
+
# Apply CPU affinity using taskset if CPU distribution is enabled
|
|
1664
|
+
if enable_cpu_distribution:
|
|
1665
|
+
try:
|
|
1666
|
+
container_info = (
|
|
1667
|
+
docker_client.api.inspect_container(
|
|
1668
|
+
container.id
|
|
1669
|
+
)
|
|
1670
|
+
)
|
|
1671
|
+
container_pid = container_info["State"][
|
|
1672
|
+
"Pid"
|
|
1673
|
+
]
|
|
1637
1674
|
|
|
1638
|
-
|
|
1675
|
+
logging.info(
|
|
1676
|
+
f"Setting CPU affinity for client container PID {container_pid} to cores {client_cpuset_cpus}"
|
|
1677
|
+
)
|
|
1639
1678
|
|
|
1640
|
-
|
|
1641
|
-
|
|
1679
|
+
# Set CPU affinity for the main process and all its threads
|
|
1680
|
+
subprocess.run(
|
|
1681
|
+
f"taskset -cp {client_cpuset_cpus} {container_pid}",
|
|
1682
|
+
shell=True,
|
|
1683
|
+
check=True,
|
|
1684
|
+
)
|
|
1642
1685
|
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1686
|
+
# Wait a moment for client to start its threads, then set affinity for all child processes
|
|
1687
|
+
time.sleep(1)
|
|
1688
|
+
result = subprocess.run(
|
|
1689
|
+
f"pgrep -P {container_pid}",
|
|
1690
|
+
shell=True,
|
|
1691
|
+
capture_output=True,
|
|
1692
|
+
text=True,
|
|
1693
|
+
)
|
|
1694
|
+
if result.returncode == 0:
|
|
1695
|
+
child_pids = (
|
|
1696
|
+
result.stdout.strip().split("\n")
|
|
1697
|
+
)
|
|
1698
|
+
for child_pid in child_pids:
|
|
1699
|
+
if child_pid.strip():
|
|
1700
|
+
try:
|
|
1701
|
+
subprocess.run(
|
|
1702
|
+
f"taskset -cp {client_cpuset_cpus} {child_pid.strip()}",
|
|
1703
|
+
shell=True,
|
|
1704
|
+
check=True,
|
|
1705
|
+
)
|
|
1706
|
+
logging.info(
|
|
1707
|
+
f"Set CPU affinity for client child process {child_pid.strip()}"
|
|
1708
|
+
)
|
|
1709
|
+
except (
|
|
1710
|
+
subprocess.CalledProcessError
|
|
1711
|
+
):
|
|
1712
|
+
pass # Child process may have exited
|
|
1655
1713
|
|
|
1656
|
-
|
|
1714
|
+
logging.info(
|
|
1715
|
+
f"✅ Applied CPU affinity to client container and all child processes"
|
|
1716
|
+
)
|
|
1657
1717
|
|
|
1658
|
-
|
|
1659
|
-
|
|
1718
|
+
except Exception as e:
|
|
1719
|
+
logging.warning(
|
|
1720
|
+
f"Failed to set CPU affinity for client container: {e}"
|
|
1721
|
+
)
|
|
1660
1722
|
|
|
1661
1723
|
# Wait for container with timeout
|
|
1662
1724
|
try:
|
|
@@ -2266,6 +2328,7 @@ def start_redis_container(
|
|
|
2266
2328
|
run_image,
|
|
2267
2329
|
temporary_dir,
|
|
2268
2330
|
auto_remove=False,
|
|
2331
|
+
enable_cpu_distribution=False,
|
|
2269
2332
|
):
|
|
2270
2333
|
logging.info(
|
|
2271
2334
|
"Running redis-server on docker image {} (cpuset={}) with the following args: {}".format(
|
|
@@ -2284,55 +2347,80 @@ def start_redis_container(
|
|
|
2284
2347
|
logging.info(f"setting volume as follow: {volumes}. working_dir={mnt_point}")
|
|
2285
2348
|
working_dir = mnt_point
|
|
2286
2349
|
|
|
2287
|
-
#
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2350
|
+
# Prepare container arguments
|
|
2351
|
+
container_args = {
|
|
2352
|
+
"image": run_image,
|
|
2353
|
+
"volumes": volumes,
|
|
2354
|
+
"auto_remove": auto_remove,
|
|
2355
|
+
"privileged": True,
|
|
2356
|
+
"working_dir": mnt_point,
|
|
2357
|
+
"command": command_str,
|
|
2358
|
+
"network_mode": "host",
|
|
2359
|
+
"detach": True,
|
|
2360
|
+
"publish_all_ports": True,
|
|
2361
|
+
}
|
|
2362
|
+
|
|
2363
|
+
# Add CPU distribution settings if enabled
|
|
2364
|
+
if enable_cpu_distribution:
|
|
2365
|
+
redis_cpu_count = len(db_cpuset_cpus.split(","))
|
|
2366
|
+
redis_nano_cpus = int(redis_cpu_count * 1e9) # 1 CPU = 1e9 nano_cpus
|
|
2367
|
+
|
|
2368
|
+
container_args["cpuset_cpus"] = db_cpuset_cpus
|
|
2369
|
+
container_args["nano_cpus"] = redis_nano_cpus
|
|
2370
|
+
|
|
2371
|
+
logging.info(
|
|
2372
|
+
f"Redis container will use {redis_cpu_count} CPUs (nano_cpus={redis_nano_cpus}) on cores {db_cpuset_cpus} [CPU distribution enabled]"
|
|
2373
|
+
)
|
|
2374
|
+
else:
|
|
2375
|
+
logging.info(
|
|
2376
|
+
f"Redis container will use default CPU allocation [CPU distribution disabled]"
|
|
2377
|
+
)
|
|
2378
|
+
|
|
2379
|
+
redis_container = docker_client.containers.run(**container_args)
|
|
2307
2380
|
time.sleep(5)
|
|
2308
2381
|
|
|
2309
|
-
# Apply CPU affinity using taskset
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2382
|
+
# Apply CPU affinity using taskset if CPU distribution is enabled
|
|
2383
|
+
if enable_cpu_distribution:
|
|
2384
|
+
try:
|
|
2385
|
+
container_info = docker_client.api.inspect_container(redis_container.id)
|
|
2386
|
+
container_pid = container_info["State"]["Pid"]
|
|
2313
2387
|
|
|
2314
|
-
|
|
2388
|
+
logging.info(
|
|
2389
|
+
f"Setting CPU affinity for Redis container PID {container_pid} to cores {db_cpuset_cpus}"
|
|
2390
|
+
)
|
|
2315
2391
|
|
|
2316
|
-
|
|
2317
|
-
|
|
2392
|
+
# Set CPU affinity for the main Redis process and all its threads
|
|
2393
|
+
subprocess.run(
|
|
2394
|
+
f"taskset -cp {db_cpuset_cpus} {container_pid}", shell=True, check=True
|
|
2395
|
+
)
|
|
2318
2396
|
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2397
|
+
# Wait a moment for Redis to start its IO threads, then set affinity for all Redis processes
|
|
2398
|
+
time.sleep(2)
|
|
2399
|
+
result = subprocess.run(
|
|
2400
|
+
f"pgrep -P {container_pid}", shell=True, capture_output=True, text=True
|
|
2401
|
+
)
|
|
2402
|
+
if result.returncode == 0:
|
|
2403
|
+
child_pids = result.stdout.strip().split("\n")
|
|
2404
|
+
for child_pid in child_pids:
|
|
2405
|
+
if child_pid.strip():
|
|
2406
|
+
try:
|
|
2407
|
+
subprocess.run(
|
|
2408
|
+
f"taskset -cp {db_cpuset_cpus} {child_pid.strip()}",
|
|
2409
|
+
shell=True,
|
|
2410
|
+
check=True,
|
|
2411
|
+
)
|
|
2412
|
+
logging.info(
|
|
2413
|
+
f"Set CPU affinity for Redis child process {child_pid.strip()}"
|
|
2414
|
+
)
|
|
2415
|
+
except subprocess.CalledProcessError:
|
|
2416
|
+
pass # Child process may have exited
|
|
2331
2417
|
|
|
2332
|
-
|
|
2418
|
+
logging.info(
|
|
2419
|
+
f"✅ Applied CPU affinity to Redis container and all child processes"
|
|
2420
|
+
)
|
|
2333
2421
|
|
|
2334
|
-
|
|
2335
|
-
|
|
2422
|
+
except Exception as e:
|
|
2423
|
+
logging.warning(f"Failed to set CPU affinity for Redis container: {e}")
|
|
2336
2424
|
|
|
2337
2425
|
redis_containers.append(redis_container)
|
|
2338
2426
|
return redis_container
|
|
@@ -2522,31 +2610,43 @@ def data_prepopulation_step(
|
|
|
2522
2610
|
preload_timeout = 1800 # 30 minutes default for data loading
|
|
2523
2611
|
logging.info(f"Starting preload container with {preload_timeout}s timeout")
|
|
2524
2612
|
|
|
2525
|
-
#
|
|
2526
|
-
|
|
2527
|
-
|
|
2613
|
+
# Prepare preload container arguments
|
|
2614
|
+
preload_container_args = {
|
|
2615
|
+
"image": preload_image,
|
|
2616
|
+
"volumes": {
|
|
2617
|
+
temporary_dir: {
|
|
2618
|
+
"bind": client_mnt_point,
|
|
2619
|
+
"mode": "rw",
|
|
2620
|
+
},
|
|
2621
|
+
},
|
|
2622
|
+
"auto_remove": False, # Don't auto-remove so we can get logs if timeout
|
|
2623
|
+
"privileged": True,
|
|
2624
|
+
"working_dir": benchmark_tool_workdir,
|
|
2625
|
+
"command": preload_command_str,
|
|
2626
|
+
"network_mode": "host",
|
|
2627
|
+
"detach": True, # Detach to enable timeout
|
|
2628
|
+
}
|
|
2629
|
+
|
|
2630
|
+
# Add CPU distribution settings if enabled
|
|
2631
|
+
enable_cpu_distribution = args.enable_cpu_distribution if args else False
|
|
2632
|
+
if enable_cpu_distribution:
|
|
2633
|
+
preload_cpu_count = len(client_cpuset_cpus.split(","))
|
|
2634
|
+
preload_nano_cpus = int(preload_cpu_count * 1e9) # 1 CPU = 1e9 nano_cpus
|
|
2528
2635
|
|
|
2529
|
-
|
|
2636
|
+
preload_container_args["cpuset_cpus"] = client_cpuset_cpus
|
|
2637
|
+
preload_container_args["nano_cpus"] = preload_nano_cpus
|
|
2638
|
+
|
|
2639
|
+
logging.info(
|
|
2640
|
+
f"Preload container will use {preload_cpu_count} CPUs (nano_cpus={preload_nano_cpus}) on cores {client_cpuset_cpus} [CPU distribution enabled]"
|
|
2641
|
+
)
|
|
2642
|
+
else:
|
|
2643
|
+
logging.info(
|
|
2644
|
+
f"Preload container will use default CPU allocation [CPU distribution disabled]"
|
|
2645
|
+
)
|
|
2530
2646
|
|
|
2531
2647
|
try:
|
|
2532
2648
|
# Start container with detach=True to enable timeout handling
|
|
2533
|
-
container = docker_client.containers.run(
|
|
2534
|
-
image=preload_image,
|
|
2535
|
-
volumes={
|
|
2536
|
-
temporary_dir: {
|
|
2537
|
-
"bind": client_mnt_point,
|
|
2538
|
-
"mode": "rw",
|
|
2539
|
-
},
|
|
2540
|
-
},
|
|
2541
|
-
auto_remove=False, # Don't auto-remove so we can get logs if timeout
|
|
2542
|
-
privileged=True,
|
|
2543
|
-
working_dir=benchmark_tool_workdir,
|
|
2544
|
-
command=preload_command_str,
|
|
2545
|
-
network_mode="host",
|
|
2546
|
-
detach=True, # Detach to enable timeout
|
|
2547
|
-
cpuset_cpus=client_cpuset_cpus,
|
|
2548
|
-
nano_cpus=preload_nano_cpus, # Force CPU distribution
|
|
2549
|
-
)
|
|
2649
|
+
container = docker_client.containers.run(**preload_container_args)
|
|
2550
2650
|
|
|
2551
2651
|
logging.info(
|
|
2552
2652
|
f"Started preload container {container.name} ({container.id[:12]}) with {preload_timeout}s timeout"
|
|
@@ -26,10 +26,10 @@ spec:
|
|
|
26
26
|
redis_topology:
|
|
27
27
|
primaries: 1
|
|
28
28
|
replicas: 0
|
|
29
|
-
redis_arguments: --io-threads 2 --io-threads-do-reads yes
|
|
29
|
+
redis_arguments: --io-threads 2 --io-threads-do-reads yes
|
|
30
30
|
resources:
|
|
31
31
|
requests:
|
|
32
|
-
cpus: "
|
|
32
|
+
cpus: "2"
|
|
33
33
|
memory: "10g"
|
|
34
34
|
|
|
35
35
|
- name: oss-standalone-04-io-threads
|
|
@@ -37,10 +37,10 @@ spec:
|
|
|
37
37
|
redis_topology:
|
|
38
38
|
primaries: 1
|
|
39
39
|
replicas: 0
|
|
40
|
-
redis_arguments: --io-threads 4 --io-threads-do-reads yes
|
|
40
|
+
redis_arguments: --io-threads 4 --io-threads-do-reads yes
|
|
41
41
|
resources:
|
|
42
42
|
requests:
|
|
43
|
-
cpus: "
|
|
43
|
+
cpus: "4"
|
|
44
44
|
memory: "10g"
|
|
45
45
|
|
|
46
46
|
- name: oss-standalone-08-io-threads
|
|
@@ -48,10 +48,10 @@ spec:
|
|
|
48
48
|
redis_topology:
|
|
49
49
|
primaries: 1
|
|
50
50
|
replicas: 0
|
|
51
|
-
redis_arguments: --io-threads 8 --io-threads-do-reads yes
|
|
51
|
+
redis_arguments: --io-threads 8 --io-threads-do-reads yes
|
|
52
52
|
resources:
|
|
53
53
|
requests:
|
|
54
|
-
cpus: "
|
|
54
|
+
cpus: "8"
|
|
55
55
|
memory: "10g"
|
|
56
56
|
|
|
57
57
|
- name: oss-standalone-16-io-threads
|
|
@@ -62,7 +62,7 @@ spec:
|
|
|
62
62
|
redis_arguments: --io-threads 16 --io-threads-do-reads yes
|
|
63
63
|
resources:
|
|
64
64
|
requests:
|
|
65
|
-
cpus: "
|
|
65
|
+
cpus: "16"
|
|
66
66
|
memory: "10g"
|
|
67
67
|
|
|
68
68
|
- name: oss-standalone-32-io-threads
|
|
@@ -73,7 +73,7 @@ spec:
|
|
|
73
73
|
redis_arguments: --io-threads 32 --io-threads-do-reads yes
|
|
74
74
|
resources:
|
|
75
75
|
requests:
|
|
76
|
-
cpus: "
|
|
76
|
+
cpus: "32"
|
|
77
77
|
memory: "10g"
|
|
78
78
|
|
|
79
79
|
- name: oss-standalone-64-io-threads
|
|
@@ -84,7 +84,7 @@ spec:
|
|
|
84
84
|
redis_arguments: --io-threads 64 --io-threads-do-reads yes
|
|
85
85
|
resources:
|
|
86
86
|
requests:
|
|
87
|
-
cpus: "
|
|
87
|
+
cpus: "64"
|
|
88
88
|
memory: "10g"
|
|
89
89
|
- name: oss-standalone-1replica
|
|
90
90
|
type: oss-standalone
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: redis-benchmarks-specification
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.20
|
|
4
4
|
Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
|
|
5
5
|
Author: filipecosta90
|
|
6
6
|
Author-email: filipecosta.90@gmail.com
|
|
@@ -18,7 +18,7 @@ redis_benchmarks_specification/__common__/package.py,sha256=4uVt1BAZ999LV2rZkq--
|
|
|
18
18
|
redis_benchmarks_specification/__common__/runner.py,sha256=TKMUFJ3nLSfmSU7P_ok9oM5-pI4L4tFxsWLUWaUHhbI,16733
|
|
19
19
|
redis_benchmarks_specification/__common__/spec.py,sha256=D_SN48wg6NMthW_-OS1H5bydSDiuZpfd4WPPj7Vfwmc,5760
|
|
20
20
|
redis_benchmarks_specification/__common__/suppress_warnings.py,sha256=xpOjJ_piGYWlGq9ITr-ZwSCl2GpreA9juZIBao4fDRs,691
|
|
21
|
-
redis_benchmarks_specification/__common__/timeseries.py,sha256=
|
|
21
|
+
redis_benchmarks_specification/__common__/timeseries.py,sha256=HxCLQmS1JCaxympxMxMk92HHnmfKehIigZ6GZZWHoLw,56045
|
|
22
22
|
redis_benchmarks_specification/__compare__/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
|
|
23
23
|
redis_benchmarks_specification/__compare__/args.py,sha256=CNtA7pI9CJDTBJPGL2pNVfis7VDdxLautwRyka7oUCI,8911
|
|
24
24
|
redis_benchmarks_specification/__compare__/compare.py,sha256=_AbuV3FZxtUZIdq4qq24LNzPNIdtQQaqrk8bUjn9blk,84327
|
|
@@ -28,16 +28,16 @@ redis_benchmarks_specification/__runner__/args.py,sha256=K3VGmBC0-9lSv9H6VDp0N-6
|
|
|
28
28
|
redis_benchmarks_specification/__runner__/remote_profiling.py,sha256=R7obNQju8mmY9oKkcndjI4aAuxi84OCLhDSqqaYu1SU,18610
|
|
29
29
|
redis_benchmarks_specification/__runner__/runner.py,sha256=lWIpjThVeYW1hxR2dl2OSIeAwUXrWVHKHfQqsD9mrF8,155505
|
|
30
30
|
redis_benchmarks_specification/__self_contained_coordinator__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
|
|
31
|
-
redis_benchmarks_specification/__self_contained_coordinator__/args.py,sha256=
|
|
31
|
+
redis_benchmarks_specification/__self_contained_coordinator__/args.py,sha256=hZo-zfVzj_LUamMgFva2QZQv5PwhUf1dHCMxdu9voMc,7648
|
|
32
32
|
redis_benchmarks_specification/__self_contained_coordinator__/artifacts.py,sha256=OVHqJzDgeSSRfUSiKp1ZTAVv14PvSbk-5yJsAAoUfpw,936
|
|
33
33
|
redis_benchmarks_specification/__self_contained_coordinator__/build_info.py,sha256=vlg8H8Rxu2falW8xp1GvL1SV1fyBguSbz6Apxc7A2yM,2282
|
|
34
34
|
redis_benchmarks_specification/__self_contained_coordinator__/clients.py,sha256=EL1V4-i-tTav1mcF_CUosqPF3Q1qi9BZL0zFajEk70c,1878
|
|
35
35
|
redis_benchmarks_specification/__self_contained_coordinator__/cpuset.py,sha256=sRvtoJIitppcOpm3R5LbVmSfPEAqPumOqVATnF5Wbek,594
|
|
36
|
-
redis_benchmarks_specification/__self_contained_coordinator__/docker.py,sha256=
|
|
36
|
+
redis_benchmarks_specification/__self_contained_coordinator__/docker.py,sha256=TuJrnijaK_5Sw9GVL6vi9H1m7f9p3g9c1f7kpU_RHuM,3542
|
|
37
37
|
redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py,sha256=sVLKNnWdAqYY9DjVdqRC5tDaIrVSaI3Ca7w8-DQ-LRM,776
|
|
38
|
-
redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py,sha256=
|
|
39
|
-
redis_benchmarks_specification/__self_contained_coordinator__/runners.py,sha256=
|
|
40
|
-
redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py,sha256=
|
|
38
|
+
redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py,sha256=30SU8dAKmRpznjAKbs1n0iHO-CjG5goCuR5TYx7XIMc,3390
|
|
39
|
+
redis_benchmarks_specification/__self_contained_coordinator__/runners.py,sha256=yHgRpNeuBcft6sEeiT12qQ012xXypXsVM8Ov91CE2k0,34234
|
|
40
|
+
redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py,sha256=Yn1oEaBlHZB-GbiNnQgo_x-FsK-Gw_Ni89zmWjdycQY,122909
|
|
41
41
|
redis_benchmarks_specification/__setups__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
42
42
|
redis_benchmarks_specification/__setups__/topologies.py,sha256=xQ1IJkcTji_ZjLiJd3vOxZpvbNtBLZw9cPkw5hGJKHU,481
|
|
43
43
|
redis_benchmarks_specification/__spec__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
|
|
@@ -51,7 +51,7 @@ redis_benchmarks_specification/commands/commands.py,sha256=hJbKkGzAFt_l40fJyQLfB
|
|
|
51
51
|
redis_benchmarks_specification/setups/builders/gcc:15.2.0-amd64-debian-bookworm-default.yml,sha256=UobsjPRRQALKNvAkOqvYJJs8HLrlG9AbfJwuIkLpwHU,528
|
|
52
52
|
redis_benchmarks_specification/setups/builders/gcc:15.2.0-arm64-debian-bookworm-default.yml,sha256=zexg-qwlrdjNEsJDigcwQgm-CluwtrWHPygvXzv0wwo,528
|
|
53
53
|
redis_benchmarks_specification/setups/platforms/aws-ec2-1node-c5.4xlarge.yml,sha256=l7HsjccpebwZXeutnt3SHSETw4iiRwQ9dCDXLOySSRQ,622
|
|
54
|
-
redis_benchmarks_specification/setups/topologies/topologies.yml,sha256=
|
|
54
|
+
redis_benchmarks_specification/setups/topologies/topologies.yml,sha256=37NQPN0H5WGk6oOnlxzkOVbj3-bzVedRHEEYAULFF9g,3264
|
|
55
55
|
redis_benchmarks_specification/test-suites/defaults.yml,sha256=EJHv9INdjoNVMOgHY8qo4IVCHfvXVz5sv7Vxtr3DAIE,1392
|
|
56
56
|
redis_benchmarks_specification/test-suites/generate.py,sha256=1QJBuWiouJ5OLil_r4OMG_UtZkmA8TLcyPlQAUuxCUw,4175
|
|
57
57
|
redis_benchmarks_specification/test-suites/memtier_benchmark-100Kkeys-hash-hgetall-50-fields-100B-values.yml,sha256=BYmI-u_QEoSVXczuWPIaoCU_IQdhPrNCdXj8SO4s60g,2317
|
|
@@ -287,8 +287,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-st
|
|
|
287
287
|
redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-storage-1k-sessions.yml,sha256=2egtIxPxCze2jlbAfgsk4v9JSQHNMoPLbDWFEW8olDg,7006
|
|
288
288
|
redis_benchmarks_specification/test-suites/template.txt,sha256=ezqGiRPOvuSDO0iG7GEf-AGXNfHbgXI89_G0RUEzL88,481
|
|
289
289
|
redis_benchmarks_specification/vector-search-test-suites/vector_db_benchmark_test.yml,sha256=PD7ow-k4Ll2BkhEC3aIqiaCZt8Hc4aJIp96Lw3J3mcI,791
|
|
290
|
-
redis_benchmarks_specification-0.2.
|
|
291
|
-
redis_benchmarks_specification-0.2.
|
|
292
|
-
redis_benchmarks_specification-0.2.
|
|
293
|
-
redis_benchmarks_specification-0.2.
|
|
294
|
-
redis_benchmarks_specification-0.2.
|
|
290
|
+
redis_benchmarks_specification-0.2.20.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
291
|
+
redis_benchmarks_specification-0.2.20.dist-info/METADATA,sha256=fFnbhd_k5arAd8Ek0E_BADyrkSXfnd6VoDV5I6A1VdU,22767
|
|
292
|
+
redis_benchmarks_specification-0.2.20.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
293
|
+
redis_benchmarks_specification-0.2.20.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
|
|
294
|
+
redis_benchmarks_specification-0.2.20.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|