simoptlib 1.2.0.post2__tar.gz → 1.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/PKG-INFO +12 -2
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/pyproject.toml +16 -11
- simoptlib-1.2.2/simopt/bootstrap.py +433 -0
- simoptlib-1.2.2/simopt/data_farming_base.py +428 -0
- simoptlib-1.2.2/simopt/experiment/__init__.py +7 -0
- simoptlib-1.2.2/simopt/experiment/multiple.py +815 -0
- simoptlib-1.2.2/simopt/experiment/post_normalize.py +294 -0
- simoptlib-1.2.2/simopt/experiment/single.py +1233 -0
- simoptlib-1.2.2/simopt/experiment_base.py +356 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/input_models.py +3 -3
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/model.py +0 -4
- simoptlib-1.2.2/simopt/models/_ext.py +56 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/amusementpark.py +6 -1
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/chessmm.py +2 -2
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/cntnv.py +2 -2
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/dualsourcing.py +2 -2
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/dynamnews.py +2 -2
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/facilitysizing.py +2 -2
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/hotel.py +1 -1
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/ironore.py +2 -2
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/network.py +2 -2
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/rmitd.py +1 -1
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/tableallocation.py +7 -4
- simoptlib-1.2.2/simopt/plot_type.py +43 -0
- simoptlib-1.2.2/simopt/plots/__init__.py +21 -0
- simoptlib-1.2.2/simopt/plots/area_scatterplot.py +292 -0
- simoptlib-1.2.2/simopt/plots/feasibility_progress.py +301 -0
- simoptlib-1.2.2/simopt/plots/progress_curve.py +267 -0
- simoptlib-1.2.2/simopt/plots/solvability_cdf.py +208 -0
- simoptlib-1.2.2/simopt/plots/solvability_profile.py +578 -0
- simoptlib-1.2.2/simopt/plots/terminal_feasibility.py +478 -0
- simoptlib-1.2.2/simopt/plots/terminal_progress.py +175 -0
- simoptlib-1.2.2/simopt/plots/terminal_scatterplot.py +131 -0
- simoptlib-1.2.2/simopt/plots/utils.py +507 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simoptlib.egg-info/PKG-INFO +12 -2
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simoptlib.egg-info/SOURCES.txt +17 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simoptlib.egg-info/requires.txt +13 -0
- simoptlib-1.2.0.post2/simopt/data_farming_base.py +0 -901
- simoptlib-1.2.0.post2/simopt/experiment_base.py +0 -5908
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/LICENSE +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/README.md +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/setup.cfg +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/GUI.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/__init__.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/__main__.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/base.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/curve.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/curve_utils.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/data_farming/data_farming_core.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/data_farming/nolhs.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/data_farming/nolhs_design_table.npz +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/directory.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/experimental.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/feasibility.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/gui/data_farming_window.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/gui/df_object.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/gui/main_menu.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/gui/new_experiment_window.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/gui/plot_window.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/gui/toplevel_custom.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/__init__.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/contam.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/example.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/fixedsan.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/mm1queue.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/paramesti.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/san.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/models/sscont.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/problem.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/problem_types.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solver.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solvers/__init__.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solvers/adam.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solvers/aloe.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solvers/astrodf.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solvers/fcsa.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solvers/neldmd.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solvers/randomsearch.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solvers/spsa.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solvers/strong.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/solvers/utils.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/utils.py +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simopt/utils.pyi +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simoptlib.egg-info/dependency_links.txt +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/simoptlib.egg-info/top_level.txt +0 -0
- {simoptlib-1.2.0.post2 → simoptlib-1.2.2}/test/test_experiments.py +0 -0
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: simoptlib
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.2
|
|
4
4
|
Summary: A testbed for simulation-optimization experiments.
|
|
5
|
-
Author-email: David Eckman <eckman@tamu.edu>, Shane Henderson <sgh9@cornell.edu>, Sara Shashaani <sshasha2@ncsu.edu>, William Grochocinski <
|
|
5
|
+
Author-email: David Eckman <eckman@tamu.edu>, Shane Henderson <sgh9@cornell.edu>, Sara Shashaani <sshasha2@ncsu.edu>, William Grochocinski <williamgrochocinski@gmail.com>
|
|
6
6
|
License: MIT License
|
|
7
7
|
|
|
8
8
|
Copyright (c) 2025 simopt-admin
|
|
@@ -44,10 +44,20 @@ Requires-Dist: pillow>=12.0.0
|
|
|
44
44
|
Requires-Dist: pydantic>=2.12.3
|
|
45
45
|
Requires-Dist: scipy>=1.16.3
|
|
46
46
|
Requires-Dist: seaborn>=0.13.2
|
|
47
|
+
Provides-Extra: dev
|
|
48
|
+
Requires-Dist: jupytext>=1.18.1; extra == "dev"
|
|
49
|
+
Requires-Dist: pytest-xdist>=3.8.0; extra == "dev"
|
|
50
|
+
Requires-Dist: pytest>=9.0.1; extra == "dev"
|
|
51
|
+
Requires-Dist: ruff>=0.14.2; extra == "dev"
|
|
52
|
+
Requires-Dist: ty>=0.0.1a26; extra == "dev"
|
|
47
53
|
Provides-Extra: docs
|
|
48
54
|
Requires-Dist: sphinx>=8.2.3; extra == "docs"
|
|
49
55
|
Requires-Dist: sphinx-autoapi>=3.6.1; extra == "docs"
|
|
50
56
|
Requires-Dist: sphinx-rtd-theme>=3.0.2; extra == "docs"
|
|
57
|
+
Provides-Extra: notebooks
|
|
58
|
+
Requires-Dist: ipykernel>=7.1.0; extra == "notebooks"
|
|
59
|
+
Provides-Extra: ext
|
|
60
|
+
Requires-Dist: simopt-extensions; extra == "ext"
|
|
51
61
|
Dynamic: license-file
|
|
52
62
|
|
|
53
63
|

|
|
@@ -11,14 +11,17 @@ exclude = ["build*"]
|
|
|
11
11
|
[tool.setuptools.package-data]
|
|
12
12
|
"simopt.data_farming" = ["*.npz"]
|
|
13
13
|
|
|
14
|
+
[tool.uv.sources]
|
|
15
|
+
simopt-extensions = { git = "https://github.com/cenwangumass/simopt-extensions", rev = "2bcf002" }
|
|
16
|
+
|
|
14
17
|
[project]
|
|
15
18
|
name = "simoptlib"
|
|
16
|
-
version = "1.2.
|
|
19
|
+
version = "1.2.2"
|
|
17
20
|
authors = [
|
|
18
21
|
{ name = "David Eckman", email = "eckman@tamu.edu" },
|
|
19
22
|
{ name = "Shane Henderson", email = "sgh9@cornell.edu" },
|
|
20
23
|
{ name = "Sara Shashaani", email = "sshasha2@ncsu.edu" },
|
|
21
|
-
{ name = "William Grochocinski", email = "
|
|
24
|
+
{ name = "William Grochocinski", email = "williamgrochocinski@gmail.com" },
|
|
22
25
|
]
|
|
23
26
|
description = "A testbed for simulation-optimization experiments."
|
|
24
27
|
readme = "README.md"
|
|
@@ -44,17 +47,19 @@ dependencies = [
|
|
|
44
47
|
]
|
|
45
48
|
|
|
46
49
|
[project.optional-dependencies]
|
|
47
|
-
|
|
48
|
-
"
|
|
49
|
-
"
|
|
50
|
-
"
|
|
50
|
+
dev = [
|
|
51
|
+
"jupytext>=1.18.1",
|
|
52
|
+
"pytest-xdist>=3.8.0",
|
|
53
|
+
"pytest>=9.0.1",
|
|
54
|
+
"ruff>=0.14.2",
|
|
55
|
+
"ty>=0.0.1a26",
|
|
56
|
+
]
|
|
57
|
+
docs = ["sphinx>=8.2.3", "sphinx-autoapi>=3.6.1", "sphinx-rtd-theme>=3.0.2"]
|
|
58
|
+
notebooks = ["ipykernel>=7.1.0"]
|
|
59
|
+
ext = [
|
|
60
|
+
"simopt-extensions",
|
|
51
61
|
]
|
|
52
62
|
|
|
53
63
|
[project.urls]
|
|
54
64
|
"Homepage" = "https://github.com/simopt-admin/simopt"
|
|
55
65
|
"Documentation" = "https://simopt.readthedocs.io/en/latest/"
|
|
56
|
-
|
|
57
|
-
[dependency-groups]
|
|
58
|
-
dev = [
|
|
59
|
-
"ruff>=0.14.2",
|
|
60
|
-
]
|
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
"""Bootstrapping procedures."""
|
|
2
|
+
|
|
3
|
+
from typing import Literal
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
import simopt.curve_utils as curve_utils
|
|
8
|
+
from mrg32k3a.mrg32k3a import MRG32k3a
|
|
9
|
+
from simopt.curve import Curve
|
|
10
|
+
from simopt.experiment import ProblemSolver
|
|
11
|
+
from simopt.plot_type import PlotType
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def bootstrap_sample_all(
|
|
15
|
+
experiments: list[list[ProblemSolver]],
|
|
16
|
+
bootstrap_rng: MRG32k3a,
|
|
17
|
+
normalize: bool = True,
|
|
18
|
+
feasibility_score_method: Literal["inf_norm", "norm"] = "inf_norm",
|
|
19
|
+
feasibility_norm_degree: int = 1,
|
|
20
|
+
feasibility_two_sided: bool = False,
|
|
21
|
+
) -> tuple[list[list[list[Curve]]], list[list[list[Curve]]]]:
|
|
22
|
+
"""Generates bootstrap samples of progress and feasibility curves.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
experiments (list[list[ProblemSolver]]): Grid of problem-solver pairs, where
|
|
26
|
+
each inner list corresponds to different problems for a given solver.
|
|
27
|
+
bootstrap_rng (MRG32k3a): Random number generator used for bootstrapping.
|
|
28
|
+
normalize (bool, optional): If True, normalize progress curves by optimality
|
|
29
|
+
gaps. Defaults to True.
|
|
30
|
+
feasibility_score_method (Literal["inf_norm", "norm"], optional): Feasibility
|
|
31
|
+
scoring rule.
|
|
32
|
+
feasibility_norm_degree (int, optional): Degree of the norm when
|
|
33
|
+
``feasibility_score_method == "norm"``.
|
|
34
|
+
feasibility_two_sided (bool, optional): Whether to give feasible solutions a
|
|
35
|
+
non-zero score based on the best violation.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
tuple[list[list[list[Curve]]], list[list[list[Curve]]]]:
|
|
39
|
+
Bootstrapped progress/objective curves and feasibility curves for all
|
|
40
|
+
solutions from all macroreplications, grouped by solver and problem.
|
|
41
|
+
"""
|
|
42
|
+
n_solvers = len(experiments)
|
|
43
|
+
n_problems = len(experiments[0])
|
|
44
|
+
bootstrap_curves = [[[] for _ in range(n_problems)] for _ in range(n_solvers)]
|
|
45
|
+
bootstrap_feasibility_curves = [
|
|
46
|
+
[[] for _ in range(n_problems)] for _ in range(n_solvers)
|
|
47
|
+
]
|
|
48
|
+
# Obtain a bootstrap sample from each experiment.
|
|
49
|
+
for solver_idx in range(n_solvers):
|
|
50
|
+
for problem_idx in range(n_problems):
|
|
51
|
+
experiment = experiments[solver_idx][problem_idx]
|
|
52
|
+
objective_curves, feasibility_curves = experiment.bootstrap_sample(
|
|
53
|
+
bootstrap_rng,
|
|
54
|
+
normalize,
|
|
55
|
+
feasibility_score_method,
|
|
56
|
+
feasibility_norm_degree,
|
|
57
|
+
feasibility_two_sided,
|
|
58
|
+
)
|
|
59
|
+
bootstrap_curves[solver_idx][problem_idx] = objective_curves
|
|
60
|
+
bootstrap_feasibility_curves[solver_idx][problem_idx] = feasibility_curves
|
|
61
|
+
# Reset substream for next solver-problem pair.
|
|
62
|
+
bootstrap_rng.reset_substream()
|
|
63
|
+
# Advance substream of random number generator to prepare for next bootstrap sample.
|
|
64
|
+
bootstrap_rng.advance_substream()
|
|
65
|
+
return bootstrap_curves, bootstrap_feasibility_curves
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def bootstrap_procedure(
|
|
69
|
+
experiments: list[list[ProblemSolver]],
|
|
70
|
+
n_bootstraps: int,
|
|
71
|
+
conf_level: float,
|
|
72
|
+
plot_type: PlotType,
|
|
73
|
+
beta: float | None = None,
|
|
74
|
+
solve_tol: float | None = None,
|
|
75
|
+
estimator: float | Curve | None = None,
|
|
76
|
+
normalize: bool = True,
|
|
77
|
+
feasibility_score_method: Literal["inf_norm", "norm"] = "inf_norm",
|
|
78
|
+
feasibility_norm_degree: int = 1,
|
|
79
|
+
feasibility_two_sided: bool = False,
|
|
80
|
+
) -> tuple[float, float] | tuple[Curve, Curve]:
|
|
81
|
+
"""Performs bootstrapping and computes confidence intervals for progress curves.
|
|
82
|
+
|
|
83
|
+
Args:
|
|
84
|
+
experiments (list[list[ProblemSolver]]): Grid of problem-solver pairs.
|
|
85
|
+
n_bootstraps (int): Number of bootstrap samples to generate.
|
|
86
|
+
conf_level (float): Confidence level for the interval (0 < conf_level < 1).
|
|
87
|
+
plot_type (PlotType): Type of plot/metric for which to compute the interval.
|
|
88
|
+
beta (float, optional): Quantile level (0 < beta < 1), used with some plot
|
|
89
|
+
types.
|
|
90
|
+
solve_tol (float, optional): Relative optimality gap that defines a "solved"
|
|
91
|
+
instance.
|
|
92
|
+
estimator (float or Curve, optional): Reference estimator for difference plot
|
|
93
|
+
types.
|
|
94
|
+
normalize (bool, optional): Whether to normalize progress curves. Defaults to
|
|
95
|
+
True.
|
|
96
|
+
feasibility_score_method (Literal["inf_norm", "norm"], optional): Feasibility
|
|
97
|
+
scoring rule used when `plot_type` corresponds to feasibility metrics.
|
|
98
|
+
feasibility_norm_degree (int, optional): Degree of the norm when
|
|
99
|
+
``feasibility_score_method == "norm"``.
|
|
100
|
+
feasibility_two_sided (bool, optional): Whether to assign a non-zero score to
|
|
101
|
+
feasible solutions based on the best violation.
|
|
102
|
+
|
|
103
|
+
Returns:
|
|
104
|
+
tuple[float, float] or tuple[Curve, Curve]: Lower and upper bounds of the CI.
|
|
105
|
+
"""
|
|
106
|
+
# Value checking
|
|
107
|
+
if n_bootstraps < 1:
|
|
108
|
+
error_msg = "Number of bootstraps must be a positive integer."
|
|
109
|
+
raise ValueError(error_msg)
|
|
110
|
+
if not 0 < conf_level < 1:
|
|
111
|
+
error_msg = "Confidence level must be in (0, 1)."
|
|
112
|
+
raise ValueError(error_msg)
|
|
113
|
+
if beta is not None and not 0 < beta < 1:
|
|
114
|
+
error_msg = "Beta quantile must be in (0, 1)."
|
|
115
|
+
raise ValueError(error_msg)
|
|
116
|
+
if solve_tol is not None and not 0 < solve_tol <= 1:
|
|
117
|
+
error_msg = "Solve tolerance must be in (0, 1]."
|
|
118
|
+
raise ValueError(error_msg)
|
|
119
|
+
acceptable_plot_types = [
|
|
120
|
+
PlotType.MEAN,
|
|
121
|
+
PlotType.QUANTILE,
|
|
122
|
+
PlotType.AREA_MEAN,
|
|
123
|
+
PlotType.AREA_STD_DEV,
|
|
124
|
+
PlotType.SOLVE_TIME_QUANTILE,
|
|
125
|
+
PlotType.SOLVE_TIME_CDF,
|
|
126
|
+
PlotType.CDF_SOLVABILITY,
|
|
127
|
+
PlotType.QUANTILE_SOLVABILITY,
|
|
128
|
+
PlotType.DIFFERENCE_OF_CDF_SOLVABILITY,
|
|
129
|
+
PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY,
|
|
130
|
+
PlotType.MEAN_FEASIBILITY_PROGRESS,
|
|
131
|
+
PlotType.QUANTILE_FEASIBILITY_PROGRESS,
|
|
132
|
+
]
|
|
133
|
+
if plot_type not in acceptable_plot_types:
|
|
134
|
+
error_msg = (
|
|
135
|
+
f"Plot type must be one of {acceptable_plot_types}.\nReceived: {plot_type}"
|
|
136
|
+
)
|
|
137
|
+
raise ValueError(error_msg)
|
|
138
|
+
|
|
139
|
+
# Create random number generator for bootstrap sampling.
|
|
140
|
+
# Stream 1 dedicated for bootstrapping.
|
|
141
|
+
bootstrap_rng = MRG32k3a(s_ss_sss_index=[1, 0, 0])
|
|
142
|
+
# Obtain n_bootstrap replications.
|
|
143
|
+
bootstrap_replications = []
|
|
144
|
+
for _ in range(n_bootstraps):
|
|
145
|
+
# Generate bootstrap sample of estimated objective/progress curves.
|
|
146
|
+
bootstrap_curves, bootstrap_feasibility_curves = bootstrap_sample_all(
|
|
147
|
+
experiments,
|
|
148
|
+
bootstrap_rng,
|
|
149
|
+
normalize,
|
|
150
|
+
feasibility_score_method,
|
|
151
|
+
feasibility_norm_degree,
|
|
152
|
+
feasibility_two_sided,
|
|
153
|
+
)
|
|
154
|
+
if plot_type in (
|
|
155
|
+
PlotType.MEAN_FEASIBILITY_PROGRESS,
|
|
156
|
+
PlotType.QUANTILE_FEASIBILITY_PROGRESS,
|
|
157
|
+
):
|
|
158
|
+
curves = bootstrap_feasibility_curves
|
|
159
|
+
else:
|
|
160
|
+
curves = bootstrap_curves
|
|
161
|
+
# Apply the functional of the bootstrap sample.
|
|
162
|
+
bootstrap_replications.append(
|
|
163
|
+
functional_of_curves(curves, plot_type, beta=beta, solve_tol=solve_tol)
|
|
164
|
+
)
|
|
165
|
+
# Distinguish cases where functional returns a scalar vs a curve.
|
|
166
|
+
if plot_type in [
|
|
167
|
+
PlotType.AREA_MEAN,
|
|
168
|
+
PlotType.AREA_STD_DEV,
|
|
169
|
+
PlotType.SOLVE_TIME_QUANTILE,
|
|
170
|
+
]:
|
|
171
|
+
if estimator is None:
|
|
172
|
+
error_msg = (
|
|
173
|
+
"Estimator must be provided for functional that returns a scalar."
|
|
174
|
+
)
|
|
175
|
+
raise ValueError(error_msg)
|
|
176
|
+
if isinstance(estimator, Curve):
|
|
177
|
+
error_msg = (
|
|
178
|
+
"Estimator must be a scalar for functional that returns a scalar."
|
|
179
|
+
)
|
|
180
|
+
raise ValueError(error_msg)
|
|
181
|
+
# Functional returns a scalar.
|
|
182
|
+
computed_bootstrap = compute_bootstrap_conf_int(
|
|
183
|
+
bootstrap_replications,
|
|
184
|
+
conf_level=conf_level,
|
|
185
|
+
bias_correction=True,
|
|
186
|
+
overall_estimator=estimator,
|
|
187
|
+
)
|
|
188
|
+
# Get the first and second float values from the computed bootstrap.
|
|
189
|
+
float_1 = computed_bootstrap[0]
|
|
190
|
+
float_2 = computed_bootstrap[1]
|
|
191
|
+
# Keep indexing into them until they are floats.
|
|
192
|
+
while not isinstance(float_1, (int, float)):
|
|
193
|
+
float_1 = float_1[0]
|
|
194
|
+
while not isinstance(float_2, (int, float)):
|
|
195
|
+
float_2 = float_2[0]
|
|
196
|
+
return float_1, float_2
|
|
197
|
+
# Functional returns a curve.
|
|
198
|
+
unique_budget_list = list(
|
|
199
|
+
np.unique(
|
|
200
|
+
[budget for curve in bootstrap_replications for budget in curve.x_vals]
|
|
201
|
+
)
|
|
202
|
+
)
|
|
203
|
+
bs_conf_int_lower_bound_list: list[np.ndarray] = []
|
|
204
|
+
bs_conf_int_upper_bound_list: list[np.ndarray] = []
|
|
205
|
+
for budget in unique_budget_list:
|
|
206
|
+
budget_float = float(budget)
|
|
207
|
+
bootstrap_subreplications = [
|
|
208
|
+
curve.lookup(budget_float) for curve in bootstrap_replications
|
|
209
|
+
]
|
|
210
|
+
if estimator is None:
|
|
211
|
+
error_msg = (
|
|
212
|
+
"Estimator must be provided for functional that returns a curve."
|
|
213
|
+
)
|
|
214
|
+
raise ValueError(error_msg)
|
|
215
|
+
if isinstance(estimator, (int, float)):
|
|
216
|
+
error_msg = (
|
|
217
|
+
"Estimator must be a Curve object for functional that returns a curve."
|
|
218
|
+
)
|
|
219
|
+
raise ValueError(error_msg)
|
|
220
|
+
sub_estimator = estimator.lookup(budget_float)
|
|
221
|
+
bs_conf_int_lower_bound, bs_conf_int_upper_bound = compute_bootstrap_conf_int(
|
|
222
|
+
bootstrap_subreplications,
|
|
223
|
+
conf_level=conf_level,
|
|
224
|
+
bias_correction=True,
|
|
225
|
+
overall_estimator=sub_estimator,
|
|
226
|
+
)
|
|
227
|
+
bs_conf_int_lower_bound_list.append(bs_conf_int_lower_bound)
|
|
228
|
+
bs_conf_int_upper_bound_list.append(bs_conf_int_upper_bound)
|
|
229
|
+
# Create the curves for the lower and upper bounds of the bootstrap
|
|
230
|
+
# confidence intervals.
|
|
231
|
+
unique_budget_list_floats = [float(val) for val in unique_budget_list]
|
|
232
|
+
lower_bound_list = [float(val) for val in bs_conf_int_lower_bound_list]
|
|
233
|
+
bs_conf_int_lower_bounds = Curve(
|
|
234
|
+
x_vals=unique_budget_list_floats, y_vals=lower_bound_list
|
|
235
|
+
)
|
|
236
|
+
upper_bound_list = [float(val) for val in bs_conf_int_upper_bound_list]
|
|
237
|
+
bs_conf_int_upper_bounds = Curve(
|
|
238
|
+
x_vals=unique_budget_list_floats, y_vals=upper_bound_list
|
|
239
|
+
)
|
|
240
|
+
return bs_conf_int_lower_bounds, bs_conf_int_upper_bounds
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def functional_of_curves(
|
|
244
|
+
bootstrap_curves: list[list[list[Curve]]],
|
|
245
|
+
plot_type: PlotType,
|
|
246
|
+
beta: float | None = 0.5,
|
|
247
|
+
solve_tol: float | None = 0.1,
|
|
248
|
+
) -> float | Curve:
|
|
249
|
+
"""Computes a functional of bootstrapped objective or progress curves.
|
|
250
|
+
|
|
251
|
+
Args:
|
|
252
|
+
bootstrap_curves (list[list[list[Curve]]]): Bootstrapped curves for all
|
|
253
|
+
solutions across all macroreplications.
|
|
254
|
+
plot_type (PlotType): Type of functional to compute:
|
|
255
|
+
- PlotType.MEAN
|
|
256
|
+
- PlotType.QUANTILE
|
|
257
|
+
- PlotType.AREA_MEAN
|
|
258
|
+
- PlotType.AREA_STD_DEV
|
|
259
|
+
- PlotType.SOLVE_TIME_QUANTILE
|
|
260
|
+
- PlotType.SOLVE_TIME_CDF
|
|
261
|
+
- PlotType.CDF_SOLVABILITY
|
|
262
|
+
- PlotType.QUANTILE_SOLVABILITY
|
|
263
|
+
- PlotType.DIFFERENCE_OF_CDF_SOLVABILITY
|
|
264
|
+
- PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY
|
|
265
|
+
beta (float, optional): Quantile level (0 < beta < 1). Defaults to 0.5.
|
|
266
|
+
solve_tol (float, optional): Optimality gap for defining a solved instance
|
|
267
|
+
(0 < solve_tol ≤ 1). Defaults to 0.1.
|
|
268
|
+
|
|
269
|
+
Returns:
|
|
270
|
+
Curve or float: The computed functional of the curves.
|
|
271
|
+
|
|
272
|
+
Raises:
|
|
273
|
+
ValueError: If input values are invalid or unsupported for the given plot_type.
|
|
274
|
+
"""
|
|
275
|
+
# Set default arguments
|
|
276
|
+
if beta is None:
|
|
277
|
+
beta = 0.5
|
|
278
|
+
if solve_tol is None:
|
|
279
|
+
solve_tol = 0.1
|
|
280
|
+
# Value checking
|
|
281
|
+
if not 0 < beta < 1:
|
|
282
|
+
error_msg = "Beta quantile must be in (0, 1)."
|
|
283
|
+
raise ValueError(error_msg)
|
|
284
|
+
if not 0 < solve_tol <= 1:
|
|
285
|
+
error_msg = "Solve tolerance must be in (0, 1]."
|
|
286
|
+
raise ValueError(error_msg)
|
|
287
|
+
|
|
288
|
+
single_curves = bootstrap_curves[0][0]
|
|
289
|
+
solver_1_curves = bootstrap_curves[0]
|
|
290
|
+
solver_2_curves = bootstrap_curves[1] if len(bootstrap_curves) > 1 else None
|
|
291
|
+
|
|
292
|
+
dispatch = {
|
|
293
|
+
PlotType.MEAN: lambda: curve_utils.mean_of_curves(single_curves),
|
|
294
|
+
PlotType.QUANTILE: lambda: curve_utils.quantile_of_curves(
|
|
295
|
+
single_curves, beta=beta
|
|
296
|
+
),
|
|
297
|
+
PlotType.AREA_MEAN: lambda: float(
|
|
298
|
+
np.mean([c.compute_area_under_curve() for c in single_curves])
|
|
299
|
+
),
|
|
300
|
+
PlotType.AREA_STD_DEV: lambda: float(
|
|
301
|
+
np.std([c.compute_area_under_curve() for c in single_curves], ddof=1)
|
|
302
|
+
),
|
|
303
|
+
PlotType.SOLVE_TIME_QUANTILE: lambda: float(
|
|
304
|
+
np.quantile(
|
|
305
|
+
[c.compute_crossing_time(threshold=solve_tol) for c in single_curves],
|
|
306
|
+
q=beta,
|
|
307
|
+
)
|
|
308
|
+
),
|
|
309
|
+
PlotType.SOLVE_TIME_CDF: lambda: curve_utils.cdf_of_curves_crossing_times(
|
|
310
|
+
single_curves, threshold=solve_tol
|
|
311
|
+
),
|
|
312
|
+
PlotType.CDF_SOLVABILITY: lambda: curve_utils.mean_of_curves(
|
|
313
|
+
[
|
|
314
|
+
curve_utils.cdf_of_curves_crossing_times(curves, threshold=solve_tol)
|
|
315
|
+
for curves in solver_1_curves
|
|
316
|
+
]
|
|
317
|
+
),
|
|
318
|
+
PlotType.QUANTILE_SOLVABILITY: lambda: curve_utils.mean_of_curves(
|
|
319
|
+
[
|
|
320
|
+
curve_utils.quantile_cross_jump(curves, threshold=solve_tol, beta=beta)
|
|
321
|
+
for curves in solver_1_curves
|
|
322
|
+
]
|
|
323
|
+
),
|
|
324
|
+
PlotType.DIFFERENCE_OF_CDF_SOLVABILITY: lambda: curve_utils.difference_of_curves( # noqa: E501
|
|
325
|
+
curve_utils.mean_of_curves(
|
|
326
|
+
[
|
|
327
|
+
curve_utils.cdf_of_curves_crossing_times(
|
|
328
|
+
curves, threshold=solve_tol
|
|
329
|
+
)
|
|
330
|
+
for curves in solver_1_curves
|
|
331
|
+
]
|
|
332
|
+
),
|
|
333
|
+
curve_utils.mean_of_curves(
|
|
334
|
+
[
|
|
335
|
+
curve_utils.cdf_of_curves_crossing_times(
|
|
336
|
+
curves, threshold=solve_tol
|
|
337
|
+
)
|
|
338
|
+
for curves in solver_2_curves # type: ignore
|
|
339
|
+
]
|
|
340
|
+
),
|
|
341
|
+
),
|
|
342
|
+
PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY: lambda: curve_utils.difference_of_curves( # noqa: E501
|
|
343
|
+
curve_utils.mean_of_curves(
|
|
344
|
+
[
|
|
345
|
+
curve_utils.quantile_cross_jump(
|
|
346
|
+
curves, threshold=solve_tol, beta=beta
|
|
347
|
+
)
|
|
348
|
+
for curves in solver_1_curves
|
|
349
|
+
]
|
|
350
|
+
),
|
|
351
|
+
curve_utils.mean_of_curves(
|
|
352
|
+
[
|
|
353
|
+
curve_utils.quantile_cross_jump(
|
|
354
|
+
curves, threshold=solve_tol, beta=beta
|
|
355
|
+
)
|
|
356
|
+
for curves in solver_2_curves # type: ignore
|
|
357
|
+
]
|
|
358
|
+
),
|
|
359
|
+
),
|
|
360
|
+
PlotType.MEAN_FEASIBILITY_PROGRESS: lambda: curve_utils.mean_of_curves(
|
|
361
|
+
single_curves
|
|
362
|
+
),
|
|
363
|
+
PlotType.QUANTILE_FEASIBILITY_PROGRESS: lambda: curve_utils.quantile_of_curves(
|
|
364
|
+
single_curves, beta=beta
|
|
365
|
+
),
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
try:
|
|
369
|
+
return dispatch[plot_type]()
|
|
370
|
+
except KeyError as e:
|
|
371
|
+
raise NotImplementedError(f"'{plot_type.value}' is not implemented.") from e
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
# TODO: double check observations type and return type
|
|
375
|
+
def compute_bootstrap_conf_int(
|
|
376
|
+
observations: list[float | int],
|
|
377
|
+
conf_level: float,
|
|
378
|
+
bias_correction: bool = True,
|
|
379
|
+
overall_estimator: float | None = None,
|
|
380
|
+
) -> tuple[np.ndarray, np.ndarray]:
|
|
381
|
+
"""Construct a bootstrap confidence interval for an estimator.
|
|
382
|
+
|
|
383
|
+
Args:
|
|
384
|
+
observations (list[float | int]): Estimators from all bootstrap instances.
|
|
385
|
+
conf_level (float): Confidence level for confidence intervals, i.e., 1 - gamma;
|
|
386
|
+
must be in (0, 1).
|
|
387
|
+
bias_correction (bool, optional): Whether to use bias-corrected bootstrap CIs
|
|
388
|
+
(via the percentile method). Defaults to True.
|
|
389
|
+
overall_estimator (float | None, optional): The estimator to compute the CI
|
|
390
|
+
around. Required if`bias_correction` is True.
|
|
391
|
+
|
|
392
|
+
Returns:
|
|
393
|
+
tuple[np.ndarray, np.ndarray]: A tuple containing the lower and upper bounds of
|
|
394
|
+
the bootstrap confidence interval.
|
|
395
|
+
|
|
396
|
+
Raises:
|
|
397
|
+
ValueError: If `conf_level` is not in (0, 1), or if `overall_estimator` is None
|
|
398
|
+
when `bias_correction` is True.
|
|
399
|
+
"""
|
|
400
|
+
# Value checking
|
|
401
|
+
if not 0 < conf_level < 1:
|
|
402
|
+
error_msg = "Confidence level must be in (0, 1)."
|
|
403
|
+
raise ValueError(error_msg)
|
|
404
|
+
if bias_correction and overall_estimator is None:
|
|
405
|
+
error_msg = "Overall estimator must be provided for bias correction."
|
|
406
|
+
raise ValueError(error_msg)
|
|
407
|
+
|
|
408
|
+
# Compute bootstrapping confidence interval via percentile method.
|
|
409
|
+
# See Efron (1981) "Nonparameteric Standard Errors and Confidence Intervals."
|
|
410
|
+
if bias_correction:
|
|
411
|
+
if overall_estimator is None:
|
|
412
|
+
error_msg = "Overall estimator must be provided for bias correction."
|
|
413
|
+
raise ValueError(error_msg)
|
|
414
|
+
# Lazy imports
|
|
415
|
+
from scipy.stats import norm
|
|
416
|
+
|
|
417
|
+
# For biased-corrected CIs, see equation (4.4) on page 146.
|
|
418
|
+
z0 = norm.ppf(np.mean([obs < overall_estimator for obs in observations]))
|
|
419
|
+
zconflvl = norm.ppf(conf_level)
|
|
420
|
+
q_lower = norm.cdf(2 * z0 - zconflvl)
|
|
421
|
+
q_upper = norm.cdf(2 * z0 + zconflvl)
|
|
422
|
+
else:
|
|
423
|
+
# For uncorrected CIs, see equation (4.3) on page 146.
|
|
424
|
+
q_lower = (1 - conf_level) / 2
|
|
425
|
+
q_upper = 1 - (1 - conf_level) / 2
|
|
426
|
+
bs_conf_int_lower_bound = np.quantile(observations, q=q_lower)
|
|
427
|
+
bs_conf_int_upper_bound = np.quantile(observations, q=q_upper)
|
|
428
|
+
# Sometimes quantile returns a scalar, so convert to array.
|
|
429
|
+
if not isinstance(bs_conf_int_lower_bound, np.ndarray):
|
|
430
|
+
bs_conf_int_lower_bound = np.array([bs_conf_int_lower_bound])
|
|
431
|
+
if not isinstance(bs_conf_int_upper_bound, np.ndarray):
|
|
432
|
+
bs_conf_int_upper_bound = np.array([bs_conf_int_upper_bound])
|
|
433
|
+
return bs_conf_int_lower_bound, bs_conf_int_upper_bound
|