simoptlib 1.2.0.post1__tar.gz → 1.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/PKG-INFO +11 -3
  2. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/README.md +1 -1
  3. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/pyproject.toml +10 -11
  4. simoptlib-1.2.1/simopt/bootstrap.py +433 -0
  5. simoptlib-1.2.1/simopt/data_farming_base.py +428 -0
  6. simoptlib-1.2.1/simopt/experiment/__init__.py +7 -0
  7. simoptlib-1.2.1/simopt/experiment/multiple.py +815 -0
  8. simoptlib-1.2.1/simopt/experiment/post_normalize.py +294 -0
  9. simoptlib-1.2.1/simopt/experiment/single.py +1233 -0
  10. simoptlib-1.2.1/simopt/experiment_base.py +356 -0
  11. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/model.py +0 -4
  12. simoptlib-1.2.1/simopt/plot_type.py +43 -0
  13. simoptlib-1.2.1/simopt/plots/__init__.py +21 -0
  14. simoptlib-1.2.1/simopt/plots/area_scatterplot.py +292 -0
  15. simoptlib-1.2.1/simopt/plots/feasibility_progress.py +301 -0
  16. simoptlib-1.2.1/simopt/plots/progress_curve.py +267 -0
  17. simoptlib-1.2.1/simopt/plots/solvability_cdf.py +208 -0
  18. simoptlib-1.2.1/simopt/plots/solvability_profile.py +578 -0
  19. simoptlib-1.2.1/simopt/plots/terminal_feasibility.py +478 -0
  20. simoptlib-1.2.1/simopt/plots/terminal_progress.py +175 -0
  21. simoptlib-1.2.1/simopt/plots/terminal_scatterplot.py +131 -0
  22. simoptlib-1.2.1/simopt/plots/utils.py +507 -0
  23. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simoptlib.egg-info/PKG-INFO +11 -3
  24. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simoptlib.egg-info/SOURCES.txt +16 -0
  25. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simoptlib.egg-info/requires.txt +10 -0
  26. simoptlib-1.2.0.post1/simopt/data_farming_base.py +0 -901
  27. simoptlib-1.2.0.post1/simopt/experiment_base.py +0 -5908
  28. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/LICENSE +0 -0
  29. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/setup.cfg +0 -0
  30. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/GUI.py +0 -0
  31. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/__init__.py +0 -0
  32. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/__main__.py +0 -0
  33. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/base.py +0 -0
  34. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/curve.py +0 -0
  35. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/curve_utils.py +0 -0
  36. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/data_farming/data_farming_core.py +0 -0
  37. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/data_farming/nolhs.py +0 -0
  38. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/data_farming/nolhs_design_table.npz +0 -0
  39. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/directory.py +0 -0
  40. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/experimental.py +0 -0
  41. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/feasibility.py +0 -0
  42. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/gui/data_farming_window.py +0 -0
  43. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/gui/df_object.py +0 -0
  44. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/gui/main_menu.py +0 -0
  45. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/gui/new_experiment_window.py +0 -0
  46. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/gui/plot_window.py +0 -0
  47. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/gui/toplevel_custom.py +0 -0
  48. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/input_models.py +0 -0
  49. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/__init__.py +0 -0
  50. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/amusementpark.py +0 -0
  51. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/chessmm.py +0 -0
  52. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/cntnv.py +0 -0
  53. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/contam.py +0 -0
  54. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/dualsourcing.py +0 -0
  55. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/dynamnews.py +0 -0
  56. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/example.py +0 -0
  57. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/facilitysizing.py +0 -0
  58. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/fixedsan.py +0 -0
  59. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/hotel.py +0 -0
  60. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/ironore.py +0 -0
  61. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/mm1queue.py +0 -0
  62. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/network.py +0 -0
  63. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/paramesti.py +0 -0
  64. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/rmitd.py +0 -0
  65. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/san.py +0 -0
  66. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/sscont.py +0 -0
  67. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/models/tableallocation.py +0 -0
  68. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/problem.py +0 -0
  69. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/problem_types.py +0 -0
  70. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solver.py +0 -0
  71. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solvers/__init__.py +0 -0
  72. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solvers/adam.py +0 -0
  73. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solvers/aloe.py +0 -0
  74. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solvers/astrodf.py +0 -0
  75. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solvers/fcsa.py +0 -0
  76. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solvers/neldmd.py +0 -0
  77. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solvers/randomsearch.py +0 -0
  78. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solvers/spsa.py +0 -0
  79. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solvers/strong.py +0 -0
  80. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/solvers/utils.py +0 -0
  81. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/utils.py +0 -0
  82. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simopt/utils.pyi +0 -0
  83. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simoptlib.egg-info/dependency_links.txt +0 -0
  84. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/simoptlib.egg-info/top_level.txt +0 -0
  85. {simoptlib-1.2.0.post1 → simoptlib-1.2.1}/test/test_experiments.py +0 -0
@@ -1,8 +1,8 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: simoptlib
3
- Version: 1.2.0.post1
3
+ Version: 1.2.1
4
4
  Summary: A testbed for simulation-optimization experiments.
5
- Author-email: David Eckman <eckman@tamu.edu>, Shane Henderson <sgh9@cornell.edu>, Sara Shashaani <sshasha2@ncsu.edu>, William Grochocinski <wagrocho@ncsu.edu>
5
+ Author-email: David Eckman <eckman@tamu.edu>, Shane Henderson <sgh9@cornell.edu>, Sara Shashaani <sshasha2@ncsu.edu>, William Grochocinski <williamgrochocinski@gmail.com>
6
6
  License: MIT License
7
7
 
8
8
  Copyright (c) 2025 simopt-admin
@@ -44,13 +44,21 @@ Requires-Dist: pillow>=12.0.0
44
44
  Requires-Dist: pydantic>=2.12.3
45
45
  Requires-Dist: scipy>=1.16.3
46
46
  Requires-Dist: seaborn>=0.13.2
47
+ Provides-Extra: dev
48
+ Requires-Dist: jupytext>=1.18.1; extra == "dev"
49
+ Requires-Dist: pytest-xdist>=3.8.0; extra == "dev"
50
+ Requires-Dist: pytest>=9.0.1; extra == "dev"
51
+ Requires-Dist: ruff>=0.14.2; extra == "dev"
52
+ Requires-Dist: ty>=0.0.1a26; extra == "dev"
47
53
  Provides-Extra: docs
48
54
  Requires-Dist: sphinx>=8.2.3; extra == "docs"
49
55
  Requires-Dist: sphinx-autoapi>=3.6.1; extra == "docs"
50
56
  Requires-Dist: sphinx-rtd-theme>=3.0.2; extra == "docs"
57
+ Provides-Extra: notebooks
58
+ Requires-Dist: ipykernel>=7.1.0; extra == "notebooks"
51
59
  Dynamic: license-file
52
60
 
53
- ![SimOpt Logo](.github/resources/logo_full_magnifying_glass.png)
61
+ ![SimOpt Logo](https://raw.githubusercontent.com/simopt-admin/simopt/master/.github/resources/logo_full_magnifying_glass.png)
54
62
 
55
63
  ## About the Project
56
64
  SimOpt is a testbed of simulation-optimization problems and solvers. Its purpose is to encourage the development and constructive comparison of simulation-optimization (SO) solvers (algorithms). We are particularly interested in the finite-time performance of solvers, rather than the asymptotic results that one often finds in related literature.
@@ -1,4 +1,4 @@
1
- ![SimOpt Logo](.github/resources/logo_full_magnifying_glass.png)
1
+ ![SimOpt Logo](https://raw.githubusercontent.com/simopt-admin/simopt/master/.github/resources/logo_full_magnifying_glass.png)
2
2
 
3
3
  ## About the Project
4
4
  SimOpt is a testbed of simulation-optimization problems and solvers. Its purpose is to encourage the development and constructive comparison of simulation-optimization (SO) solvers (algorithms). We are particularly interested in the finite-time performance of solvers, rather than the asymptotic results that one often finds in related literature.
@@ -13,12 +13,12 @@ exclude = ["build*"]
13
13
 
14
14
  [project]
15
15
  name = "simoptlib"
16
- version = "1.2.0.post1"
16
+ version = "1.2.1"
17
17
  authors = [
18
18
  { name = "David Eckman", email = "eckman@tamu.edu" },
19
19
  { name = "Shane Henderson", email = "sgh9@cornell.edu" },
20
20
  { name = "Sara Shashaani", email = "sshasha2@ncsu.edu" },
21
- { name = "William Grochocinski", email = "wagrocho@ncsu.edu" },
21
+ { name = "William Grochocinski", email = "williamgrochocinski@gmail.com" },
22
22
  ]
23
23
  description = "A testbed for simulation-optimization experiments."
24
24
  readme = "README.md"
@@ -44,17 +44,16 @@ dependencies = [
44
44
  ]
45
45
 
46
46
  [project.optional-dependencies]
47
- docs = [
48
- "sphinx>=8.2.3",
49
- "sphinx-autoapi>=3.6.1",
50
- "sphinx-rtd-theme>=3.0.2",
47
+ dev = [
48
+ "jupytext>=1.18.1",
49
+ "pytest-xdist>=3.8.0",
50
+ "pytest>=9.0.1",
51
+ "ruff>=0.14.2",
52
+ "ty>=0.0.1a26",
51
53
  ]
54
+ docs = ["sphinx>=8.2.3", "sphinx-autoapi>=3.6.1", "sphinx-rtd-theme>=3.0.2"]
55
+ notebooks = ["ipykernel>=7.1.0"]
52
56
 
53
57
  [project.urls]
54
58
  "Homepage" = "https://github.com/simopt-admin/simopt"
55
59
  "Documentation" = "https://simopt.readthedocs.io/en/latest/"
56
-
57
- [dependency-groups]
58
- dev = [
59
- "ruff>=0.14.2",
60
- ]
@@ -0,0 +1,433 @@
1
+ """Bootstrapping procedures."""
2
+
3
+ from typing import Literal
4
+
5
+ import numpy as np
6
+
7
+ import simopt.curve_utils as curve_utils
8
+ from mrg32k3a.mrg32k3a import MRG32k3a
9
+ from simopt.curve import Curve
10
+ from simopt.experiment import ProblemSolver
11
+ from simopt.plot_type import PlotType
12
+
13
+
14
+ def bootstrap_sample_all(
15
+ experiments: list[list[ProblemSolver]],
16
+ bootstrap_rng: MRG32k3a,
17
+ normalize: bool = True,
18
+ feasibility_score_method: Literal["inf_norm", "norm"] = "inf_norm",
19
+ feasibility_norm_degree: int = 1,
20
+ feasibility_two_sided: bool = False,
21
+ ) -> tuple[list[list[list[Curve]]], list[list[list[Curve]]]]:
22
+ """Generates bootstrap samples of progress and feasibility curves.
23
+
24
+ Args:
25
+ experiments (list[list[ProblemSolver]]): Grid of problem-solver pairs, where
26
+ each inner list corresponds to different problems for a given solver.
27
+ bootstrap_rng (MRG32k3a): Random number generator used for bootstrapping.
28
+ normalize (bool, optional): If True, normalize progress curves by optimality
29
+ gaps. Defaults to True.
30
+ feasibility_score_method (Literal["inf_norm", "norm"], optional): Feasibility
31
+ scoring rule.
32
+ feasibility_norm_degree (int, optional): Degree of the norm when
33
+ ``feasibility_score_method == "norm"``.
34
+ feasibility_two_sided (bool, optional): Whether to give feasible solutions a
35
+ non-zero score based on the best violation.
36
+
37
+ Returns:
38
+ tuple[list[list[list[Curve]]], list[list[list[Curve]]]]:
39
+ Bootstrapped progress/objective curves and feasibility curves for all
40
+ solutions from all macroreplications, grouped by solver and problem.
41
+ """
42
+ n_solvers = len(experiments)
43
+ n_problems = len(experiments[0])
44
+ bootstrap_curves = [[[] for _ in range(n_problems)] for _ in range(n_solvers)]
45
+ bootstrap_feasibility_curves = [
46
+ [[] for _ in range(n_problems)] for _ in range(n_solvers)
47
+ ]
48
+ # Obtain a bootstrap sample from each experiment.
49
+ for solver_idx in range(n_solvers):
50
+ for problem_idx in range(n_problems):
51
+ experiment = experiments[solver_idx][problem_idx]
52
+ objective_curves, feasibility_curves = experiment.bootstrap_sample(
53
+ bootstrap_rng,
54
+ normalize,
55
+ feasibility_score_method,
56
+ feasibility_norm_degree,
57
+ feasibility_two_sided,
58
+ )
59
+ bootstrap_curves[solver_idx][problem_idx] = objective_curves
60
+ bootstrap_feasibility_curves[solver_idx][problem_idx] = feasibility_curves
61
+ # Reset substream for next solver-problem pair.
62
+ bootstrap_rng.reset_substream()
63
+ # Advance substream of random number generator to prepare for next bootstrap sample.
64
+ bootstrap_rng.advance_substream()
65
+ return bootstrap_curves, bootstrap_feasibility_curves
66
+
67
+
68
+ def bootstrap_procedure(
69
+ experiments: list[list[ProblemSolver]],
70
+ n_bootstraps: int,
71
+ conf_level: float,
72
+ plot_type: PlotType,
73
+ beta: float | None = None,
74
+ solve_tol: float | None = None,
75
+ estimator: float | Curve | None = None,
76
+ normalize: bool = True,
77
+ feasibility_score_method: Literal["inf_norm", "norm"] = "inf_norm",
78
+ feasibility_norm_degree: int = 1,
79
+ feasibility_two_sided: bool = False,
80
+ ) -> tuple[float, float] | tuple[Curve, Curve]:
81
+ """Performs bootstrapping and computes confidence intervals for progress curves.
82
+
83
+ Args:
84
+ experiments (list[list[ProblemSolver]]): Grid of problem-solver pairs.
85
+ n_bootstraps (int): Number of bootstrap samples to generate.
86
+ conf_level (float): Confidence level for the interval (0 < conf_level < 1).
87
+ plot_type (PlotType): Type of plot/metric for which to compute the interval.
88
+ beta (float, optional): Quantile level (0 < beta < 1), used with some plot
89
+ types.
90
+ solve_tol (float, optional): Relative optimality gap that defines a "solved"
91
+ instance.
92
+ estimator (float or Curve, optional): Reference estimator for difference plot
93
+ types.
94
+ normalize (bool, optional): Whether to normalize progress curves. Defaults to
95
+ True.
96
+ feasibility_score_method (Literal["inf_norm", "norm"], optional): Feasibility
97
+ scoring rule used when `plot_type` corresponds to feasibility metrics.
98
+ feasibility_norm_degree (int, optional): Degree of the norm when
99
+ ``feasibility_score_method == "norm"``.
100
+ feasibility_two_sided (bool, optional): Whether to assign a non-zero score to
101
+ feasible solutions based on the best violation.
102
+
103
+ Returns:
104
+ tuple[float, float] or tuple[Curve, Curve]: Lower and upper bounds of the CI.
105
+ """
106
+ # Value checking
107
+ if n_bootstraps < 1:
108
+ error_msg = "Number of bootstraps must be a positive integer."
109
+ raise ValueError(error_msg)
110
+ if not 0 < conf_level < 1:
111
+ error_msg = "Confidence level must be in (0, 1)."
112
+ raise ValueError(error_msg)
113
+ if beta is not None and not 0 < beta < 1:
114
+ error_msg = "Beta quantile must be in (0, 1)."
115
+ raise ValueError(error_msg)
116
+ if solve_tol is not None and not 0 < solve_tol <= 1:
117
+ error_msg = "Solve tolerance must be in (0, 1]."
118
+ raise ValueError(error_msg)
119
+ acceptable_plot_types = [
120
+ PlotType.MEAN,
121
+ PlotType.QUANTILE,
122
+ PlotType.AREA_MEAN,
123
+ PlotType.AREA_STD_DEV,
124
+ PlotType.SOLVE_TIME_QUANTILE,
125
+ PlotType.SOLVE_TIME_CDF,
126
+ PlotType.CDF_SOLVABILITY,
127
+ PlotType.QUANTILE_SOLVABILITY,
128
+ PlotType.DIFFERENCE_OF_CDF_SOLVABILITY,
129
+ PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY,
130
+ PlotType.MEAN_FEASIBILITY_PROGRESS,
131
+ PlotType.QUANTILE_FEASIBILITY_PROGRESS,
132
+ ]
133
+ if plot_type not in acceptable_plot_types:
134
+ error_msg = (
135
+ f"Plot type must be one of {acceptable_plot_types}.\nReceived: {plot_type}"
136
+ )
137
+ raise ValueError(error_msg)
138
+
139
+ # Create random number generator for bootstrap sampling.
140
+ # Stream 1 dedicated for bootstrapping.
141
+ bootstrap_rng = MRG32k3a(s_ss_sss_index=[1, 0, 0])
142
+ # Obtain n_bootstrap replications.
143
+ bootstrap_replications = []
144
+ for _ in range(n_bootstraps):
145
+ # Generate bootstrap sample of estimated objective/progress curves.
146
+ bootstrap_curves, bootstrap_feasibility_curves = bootstrap_sample_all(
147
+ experiments,
148
+ bootstrap_rng,
149
+ normalize,
150
+ feasibility_score_method,
151
+ feasibility_norm_degree,
152
+ feasibility_two_sided,
153
+ )
154
+ if plot_type in (
155
+ PlotType.MEAN_FEASIBILITY_PROGRESS,
156
+ PlotType.QUANTILE_FEASIBILITY_PROGRESS,
157
+ ):
158
+ curves = bootstrap_feasibility_curves
159
+ else:
160
+ curves = bootstrap_curves
161
+ # Apply the functional of the bootstrap sample.
162
+ bootstrap_replications.append(
163
+ functional_of_curves(curves, plot_type, beta=beta, solve_tol=solve_tol)
164
+ )
165
+ # Distinguish cases where functional returns a scalar vs a curve.
166
+ if plot_type in [
167
+ PlotType.AREA_MEAN,
168
+ PlotType.AREA_STD_DEV,
169
+ PlotType.SOLVE_TIME_QUANTILE,
170
+ ]:
171
+ if estimator is None:
172
+ error_msg = (
173
+ "Estimator must be provided for functional that returns a scalar."
174
+ )
175
+ raise ValueError(error_msg)
176
+ if isinstance(estimator, Curve):
177
+ error_msg = (
178
+ "Estimator must be a scalar for functional that returns a scalar."
179
+ )
180
+ raise ValueError(error_msg)
181
+ # Functional returns a scalar.
182
+ computed_bootstrap = compute_bootstrap_conf_int(
183
+ bootstrap_replications,
184
+ conf_level=conf_level,
185
+ bias_correction=True,
186
+ overall_estimator=estimator,
187
+ )
188
+ # Get the first and second float values from the computed bootstrap.
189
+ float_1 = computed_bootstrap[0]
190
+ float_2 = computed_bootstrap[1]
191
+ # Keep indexing into them until they are floats.
192
+ while not isinstance(float_1, (int, float)):
193
+ float_1 = float_1[0]
194
+ while not isinstance(float_2, (int, float)):
195
+ float_2 = float_2[0]
196
+ return float_1, float_2
197
+ # Functional returns a curve.
198
+ unique_budget_list = list(
199
+ np.unique(
200
+ [budget for curve in bootstrap_replications for budget in curve.x_vals]
201
+ )
202
+ )
203
+ bs_conf_int_lower_bound_list: list[np.ndarray] = []
204
+ bs_conf_int_upper_bound_list: list[np.ndarray] = []
205
+ for budget in unique_budget_list:
206
+ budget_float = float(budget)
207
+ bootstrap_subreplications = [
208
+ curve.lookup(budget_float) for curve in bootstrap_replications
209
+ ]
210
+ if estimator is None:
211
+ error_msg = (
212
+ "Estimator must be provided for functional that returns a curve."
213
+ )
214
+ raise ValueError(error_msg)
215
+ if isinstance(estimator, (int, float)):
216
+ error_msg = (
217
+ "Estimator must be a Curve object for functional that returns a curve."
218
+ )
219
+ raise ValueError(error_msg)
220
+ sub_estimator = estimator.lookup(budget_float)
221
+ bs_conf_int_lower_bound, bs_conf_int_upper_bound = compute_bootstrap_conf_int(
222
+ bootstrap_subreplications,
223
+ conf_level=conf_level,
224
+ bias_correction=True,
225
+ overall_estimator=sub_estimator,
226
+ )
227
+ bs_conf_int_lower_bound_list.append(bs_conf_int_lower_bound)
228
+ bs_conf_int_upper_bound_list.append(bs_conf_int_upper_bound)
229
+ # Create the curves for the lower and upper bounds of the bootstrap
230
+ # confidence intervals.
231
+ unique_budget_list_floats = [float(val) for val in unique_budget_list]
232
+ lower_bound_list = [float(val) for val in bs_conf_int_lower_bound_list]
233
+ bs_conf_int_lower_bounds = Curve(
234
+ x_vals=unique_budget_list_floats, y_vals=lower_bound_list
235
+ )
236
+ upper_bound_list = [float(val) for val in bs_conf_int_upper_bound_list]
237
+ bs_conf_int_upper_bounds = Curve(
238
+ x_vals=unique_budget_list_floats, y_vals=upper_bound_list
239
+ )
240
+ return bs_conf_int_lower_bounds, bs_conf_int_upper_bounds
241
+
242
+
243
+ def functional_of_curves(
244
+ bootstrap_curves: list[list[list[Curve]]],
245
+ plot_type: PlotType,
246
+ beta: float | None = 0.5,
247
+ solve_tol: float | None = 0.1,
248
+ ) -> float | Curve:
249
+ """Computes a functional of bootstrapped objective or progress curves.
250
+
251
+ Args:
252
+ bootstrap_curves (list[list[list[Curve]]]): Bootstrapped curves for all
253
+ solutions across all macroreplications.
254
+ plot_type (PlotType): Type of functional to compute:
255
+ - PlotType.MEAN
256
+ - PlotType.QUANTILE
257
+ - PlotType.AREA_MEAN
258
+ - PlotType.AREA_STD_DEV
259
+ - PlotType.SOLVE_TIME_QUANTILE
260
+ - PlotType.SOLVE_TIME_CDF
261
+ - PlotType.CDF_SOLVABILITY
262
+ - PlotType.QUANTILE_SOLVABILITY
263
+ - PlotType.DIFFERENCE_OF_CDF_SOLVABILITY
264
+ - PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY
265
+ beta (float, optional): Quantile level (0 < beta < 1). Defaults to 0.5.
266
+ solve_tol (float, optional): Optimality gap for defining a solved instance
267
+ (0 < solve_tol ≤ 1). Defaults to 0.1.
268
+
269
+ Returns:
270
+ Curve or float: The computed functional of the curves.
271
+
272
+ Raises:
273
+ ValueError: If input values are invalid or unsupported for the given plot_type.
274
+ """
275
+ # Set default arguments
276
+ if beta is None:
277
+ beta = 0.5
278
+ if solve_tol is None:
279
+ solve_tol = 0.1
280
+ # Value checking
281
+ if not 0 < beta < 1:
282
+ error_msg = "Beta quantile must be in (0, 1)."
283
+ raise ValueError(error_msg)
284
+ if not 0 < solve_tol <= 1:
285
+ error_msg = "Solve tolerance must be in (0, 1]."
286
+ raise ValueError(error_msg)
287
+
288
+ single_curves = bootstrap_curves[0][0]
289
+ solver_1_curves = bootstrap_curves[0]
290
+ solver_2_curves = bootstrap_curves[1] if len(bootstrap_curves) > 1 else None
291
+
292
+ dispatch = {
293
+ PlotType.MEAN: lambda: curve_utils.mean_of_curves(single_curves),
294
+ PlotType.QUANTILE: lambda: curve_utils.quantile_of_curves(
295
+ single_curves, beta=beta
296
+ ),
297
+ PlotType.AREA_MEAN: lambda: float(
298
+ np.mean([c.compute_area_under_curve() for c in single_curves])
299
+ ),
300
+ PlotType.AREA_STD_DEV: lambda: float(
301
+ np.std([c.compute_area_under_curve() for c in single_curves], ddof=1)
302
+ ),
303
+ PlotType.SOLVE_TIME_QUANTILE: lambda: float(
304
+ np.quantile(
305
+ [c.compute_crossing_time(threshold=solve_tol) for c in single_curves],
306
+ q=beta,
307
+ )
308
+ ),
309
+ PlotType.SOLVE_TIME_CDF: lambda: curve_utils.cdf_of_curves_crossing_times(
310
+ single_curves, threshold=solve_tol
311
+ ),
312
+ PlotType.CDF_SOLVABILITY: lambda: curve_utils.mean_of_curves(
313
+ [
314
+ curve_utils.cdf_of_curves_crossing_times(curves, threshold=solve_tol)
315
+ for curves in solver_1_curves
316
+ ]
317
+ ),
318
+ PlotType.QUANTILE_SOLVABILITY: lambda: curve_utils.mean_of_curves(
319
+ [
320
+ curve_utils.quantile_cross_jump(curves, threshold=solve_tol, beta=beta)
321
+ for curves in solver_1_curves
322
+ ]
323
+ ),
324
+ PlotType.DIFFERENCE_OF_CDF_SOLVABILITY: lambda: curve_utils.difference_of_curves( # noqa: E501
325
+ curve_utils.mean_of_curves(
326
+ [
327
+ curve_utils.cdf_of_curves_crossing_times(
328
+ curves, threshold=solve_tol
329
+ )
330
+ for curves in solver_1_curves
331
+ ]
332
+ ),
333
+ curve_utils.mean_of_curves(
334
+ [
335
+ curve_utils.cdf_of_curves_crossing_times(
336
+ curves, threshold=solve_tol
337
+ )
338
+ for curves in solver_2_curves # type: ignore
339
+ ]
340
+ ),
341
+ ),
342
+ PlotType.DIFFERENCE_OF_QUANTILE_SOLVABILITY: lambda: curve_utils.difference_of_curves( # noqa: E501
343
+ curve_utils.mean_of_curves(
344
+ [
345
+ curve_utils.quantile_cross_jump(
346
+ curves, threshold=solve_tol, beta=beta
347
+ )
348
+ for curves in solver_1_curves
349
+ ]
350
+ ),
351
+ curve_utils.mean_of_curves(
352
+ [
353
+ curve_utils.quantile_cross_jump(
354
+ curves, threshold=solve_tol, beta=beta
355
+ )
356
+ for curves in solver_2_curves # type: ignore
357
+ ]
358
+ ),
359
+ ),
360
+ PlotType.MEAN_FEASIBILITY_PROGRESS: lambda: curve_utils.mean_of_curves(
361
+ single_curves
362
+ ),
363
+ PlotType.QUANTILE_FEASIBILITY_PROGRESS: lambda: curve_utils.quantile_of_curves(
364
+ single_curves, beta=beta
365
+ ),
366
+ }
367
+
368
+ try:
369
+ return dispatch[plot_type]()
370
+ except KeyError as e:
371
+ raise NotImplementedError(f"'{plot_type.value}' is not implemented.") from e
372
+
373
+
374
+ # TODO: double check observations type and return type
375
+ def compute_bootstrap_conf_int(
376
+ observations: list[float | int],
377
+ conf_level: float,
378
+ bias_correction: bool = True,
379
+ overall_estimator: float | None = None,
380
+ ) -> tuple[np.ndarray, np.ndarray]:
381
+ """Construct a bootstrap confidence interval for an estimator.
382
+
383
+ Args:
384
+ observations (list[float | int]): Estimators from all bootstrap instances.
385
+ conf_level (float): Confidence level for confidence intervals, i.e., 1 - gamma;
386
+ must be in (0, 1).
387
+ bias_correction (bool, optional): Whether to use bias-corrected bootstrap CIs
388
+ (via the percentile method). Defaults to True.
389
+ overall_estimator (float | None, optional): The estimator to compute the CI
390
+ around. Required if`bias_correction` is True.
391
+
392
+ Returns:
393
+ tuple[np.ndarray, np.ndarray]: A tuple containing the lower and upper bounds of
394
+ the bootstrap confidence interval.
395
+
396
+ Raises:
397
+ ValueError: If `conf_level` is not in (0, 1), or if `overall_estimator` is None
398
+ when `bias_correction` is True.
399
+ """
400
+ # Value checking
401
+ if not 0 < conf_level < 1:
402
+ error_msg = "Confidence level must be in (0, 1)."
403
+ raise ValueError(error_msg)
404
+ if bias_correction and overall_estimator is None:
405
+ error_msg = "Overall estimator must be provided for bias correction."
406
+ raise ValueError(error_msg)
407
+
408
+ # Compute bootstrapping confidence interval via percentile method.
409
+ # See Efron (1981) "Nonparameteric Standard Errors and Confidence Intervals."
410
+ if bias_correction:
411
+ if overall_estimator is None:
412
+ error_msg = "Overall estimator must be provided for bias correction."
413
+ raise ValueError(error_msg)
414
+ # Lazy imports
415
+ from scipy.stats import norm
416
+
417
+ # For biased-corrected CIs, see equation (4.4) on page 146.
418
+ z0 = norm.ppf(np.mean([obs < overall_estimator for obs in observations]))
419
+ zconflvl = norm.ppf(conf_level)
420
+ q_lower = norm.cdf(2 * z0 - zconflvl)
421
+ q_upper = norm.cdf(2 * z0 + zconflvl)
422
+ else:
423
+ # For uncorrected CIs, see equation (4.3) on page 146.
424
+ q_lower = (1 - conf_level) / 2
425
+ q_upper = 1 - (1 - conf_level) / 2
426
+ bs_conf_int_lower_bound = np.quantile(observations, q=q_lower)
427
+ bs_conf_int_upper_bound = np.quantile(observations, q=q_upper)
428
+ # Sometimes quantile returns a scalar, so convert to array.
429
+ if not isinstance(bs_conf_int_lower_bound, np.ndarray):
430
+ bs_conf_int_lower_bound = np.array([bs_conf_int_lower_bound])
431
+ if not isinstance(bs_conf_int_upper_bound, np.ndarray):
432
+ bs_conf_int_upper_bound = np.array([bs_conf_int_upper_bound])
433
+ return bs_conf_int_lower_bound, bs_conf_int_upper_bound