desdeo 1.2__py3-none-any.whl → 2.1.0__py3-none-any.whl

desdeo/problem/simulator_evaluator.py

@@ -0,0 +1,348 @@
+"""Evaluators are defined to evaluate simulator based and surrogate based objectives, constraints and extras."""
+
+import json
+import subprocess
+import sys
+from inspect import getfullargspec
+from pathlib import Path
+from urllib.parse import urlparse
+
+import joblib
+import numpy as np
+import polars as pl
+import requests
+
+from desdeo.problem import (
+    MathParser,
+    ObjectiveTypeEnum,
+    PolarsEvaluator,
+    PolarsEvaluatorModesEnum,
+    Problem,
+)
+from desdeo.problem.external import ProviderParams, get_resolver, supported_schemes
+
+# external resolver to resolve providers for problems defined externally of DESDEO
+_external_resolver = get_resolver()
+
+
+class EvaluatorError(Exception):
+    """Error raised when exceptions are encountered in an Evaluator."""
+
+
+class SimulatorEvaluator:
+    """A class for creating evaluators for simulator based and surrogate based objectives, constraints and extras."""
+
+    def __init__(  # noqa: PLR0912
+        self,
+        problem: Problem,
+        params: dict[str, dict] | ProviderParams | None = None,
+        surrogate_paths: dict[str, Path] | None = None,
+    ):
+        """Creating an evaluator for simulator based and surrogate based objectives, constraints and extras.
+
+        Args:
+            problem (Problem): The problem as a pydantic 'Problem' data class.
+            params (dict[str, dict], optional): Parameters for the different simulators used in the problem.
+                Given as dict with the simulators' symbols as keys and the corresponding simulator parameters
+                as a dict as values. Defaults to None.
+            surrogate_paths (dict[str, Path], optional): A dictionary where the keys are the names of the objectives,
+                constraints and extra functions and the values are the paths to the surrogate models saved on disk.
+                The names of the objectives, constraints and extra functions should match the names of the objectives,
+                constraints and extra functions in the problem JSON. Defaults to None.
+        """
+        self.problem = problem
+        # store the symbol and min or max multiplier as well (symbol, min/max multiplier [1 | -1])
+        self.objective_mix_max_mult = [
+            (objective.symbol, -1 if objective.maximize else 1) for objective in problem.objectives
+        ]
+        # Gather symbols objectives of different types into their own lists
+        self.analytical_symbols = [
+            obj.symbol
+            for obj in list(filter(lambda x: x.objective_type == ObjectiveTypeEnum.analytical, problem.objectives))
+        ]
+        self.data_based_symbols = [
+            obj.symbol for obj in problem.objectives if obj.objective_type == ObjectiveTypeEnum.data_based
+        ]
+        self.simulator_symbols = [
+            obj.symbol
+            for obj in list(filter(lambda x: x.objective_type == ObjectiveTypeEnum.simulator, problem.objectives))
+        ]
+        self.surrogate_symbols = [
+            obj.symbol
+            for obj in list(filter(lambda x: x.objective_type == ObjectiveTypeEnum.surrogate, problem.objectives))
+        ]
+        if problem.scalarization_funcs is not None:
+            parser = MathParser()
+            self.scalarization_funcs = [
+                (func.symbol, parser.parse(func.func))
+                for func in problem.scalarization_funcs
+                if func.symbol is not None
+            ]
+        else:
+            self.scalarization_funcs = []
+        # Gather any constraints' symbols
+        if problem.constraints is not None:
+            self.analytical_symbols = self.analytical_symbols + [
+                con.symbol for con in list(filter(lambda x: x.func is not None, problem.constraints))
+            ]
+            self.simulator_symbols = self.simulator_symbols + [
+                con.symbol for con in list(filter(lambda x: x.simulator_path is not None, problem.constraints))
+            ]
+            self.surrogate_symbols = self.surrogate_symbols + [
+                con.symbol for con in list(filter(lambda x: x.surrogates is not None, problem.constraints))
+            ]
+
+        # Gather any extra functions' symbols
+        if problem.extra_funcs is not None:
+            self.analytical_symbols = self.analytical_symbols + [
+                extra.symbol for extra in list(filter(lambda x: x.func is not None, problem.extra_funcs))
+            ]
+            self.simulator_symbols = self.simulator_symbols + [
+                extra.symbol for extra in list(filter(lambda x: x.simulator_path is not None, problem.extra_funcs))
+            ]
+            self.surrogate_symbols = self.surrogate_symbols + [
+                extra.symbol for extra in list(filter(lambda x: x.surrogates is not None, problem.extra_funcs))
+            ]
+
+        # Gather all the symbols of objectives, constraints and extra functions
+        self.problem_symbols = (
+            self.analytical_symbols + self.data_based_symbols + self.simulator_symbols + self.surrogate_symbols
+        )
+
+        # Gather the possible simulators
+        self.simulators = problem.simulators if problem.simulators is not None else []
+
+        # Gather the possibly given parameters
+        if params and not isinstance(params, dict):
+            params = params.model_dump()
+        self.params = {}
+        for sim in self.simulators:
+            sim_params = params.get(sim.name, {}) if params is not None else {}
+            if sim.parameter_options is not None:
+                for key in sim.parameter_options:
+                    sim_params[key] = sim.parameter_options[key]
+            self.params[sim.name] = sim_params
+
+        self.surrogates = {}
+        if surrogate_paths is not None:
+            self._load_surrogates(surrogate_paths)
+        else:
+            self._load_surrogates()
+
+        if len(self.surrogate_symbols) > 0:
+            missing_surrogates = []
+            for symbol in self.surrogate_symbols:
+                if symbol not in self.surrogates:
+                    missing_surrogates.append(symbol)
+
+            if len(missing_surrogates) > 0:
+                raise EvaluatorError(f"Some surrogates missing: {missing_surrogates}.")
+
+    def _evaluate_simulator(self, xs: dict[str, list[int | float]]) -> pl.DataFrame:
+        """Evaluate the problem for the given decision variables using the problem's simulators.
+
+        Args:
+            xs (dict[str, list[int | float]]): The decision variables for which the functions are to be evaluated.
+                Given as a dictionary with the decision variable symbols as keys and a list of decision variable values
+                as the values. The length of the lists is the number of samples and each list should have the same
+                length (same number of samples).
+
+        Returns:
+            pl.DataFrame: The objective, constraint and extra function values for the given decision variables as
+                a polars dataframe. The symbols of the objectives, constraints and extra functions are the column names
+                and the length of the columns is the number of samples. Will return those objective, constraint and
+                extra function values that are gained from simulators listed in the problem object.
+        """
+        res_df = pl.DataFrame()
+        for sim in self.simulators:
+            # gather the possible parameters for the simulator
+            params = self.params.get(sim.name, {})
+            if sim.file is not None:
+                # call the simulator with the decision variable values and parameters as dicts
+                res = subprocess.run(
+                    [sys.executable, sim.file, "-d", str(xs), "-p", str(params)], capture_output=True, text=True
+                )
+                if res.returncode == 0:
+                    # gather the simulation results (a dict) into the results dataframe
+                    res_df = res_df.hstack(pl.DataFrame(json.loads(res.stdout)))
+                else:
+                    raise EvaluatorError(res.stderr)
+            elif sim.url is not None:
+                # call the endpoint
+                try:
+                    if isinstance(xs, pl.DataFrame):
+                        # if xs is a polars dataframe, convert it to a dict
+                        xs = xs.to_dict(as_series=False)
+                    scheme = urlparse(sim.url.url).scheme
+                    if scheme in supported_schemes:
+                        # desdeo
+                        res = _external_resolver.evaluate(sim.url.url, params, xs)
+                        res_df = res_df.hstack(pl.DataFrame(res))
+                        # parse res
+                    else:
+                        # http, https, etc...
+                        res = requests.get(sim.url.url, auth=sim.url.auth, json={"d": xs, "p": params})
+                        res.raise_for_status()  # raise an error if the request failed
+                        res_df = res_df.hstack(pl.DataFrame(res.json()))
+                except requests.RequestException as e:
+                    raise EvaluatorError(
+                        f"Failed to call the simulator at {sim.url}. Is the simulator server running?"
+                    ) from e
+
+        # Evaluate the minimization form of the objective functions
+        min_obj_columns = pl.DataFrame()
+        for symbol, min_max_mult in self.objective_mix_max_mult:
+            if symbol in res_df.columns:
+                min_obj_columns = min_obj_columns.hstack(
+                    res_df.select((min_max_mult * pl.col(f"{symbol}")).alias(f"{symbol}_min"))
+                )
+
+        res_df = res_df.hstack(min_obj_columns)
+        # If there are scalarization functions, evaluate them as well
+        scalarization_columns = res_df.select(*[expr.alias(symbol) for symbol, expr in self.scalarization_funcs])
+        return res_df.hstack(scalarization_columns)
+
+    def _evaluate_surrogates(self, xs: dict[str, list[int | float]]) -> pl.DataFrame:
+        """Evaluate the problem for the given decision variables using the surrogate models.
+
+        Args:
+            xs (dict[str, list[int | float]]): The decision variables for which the functions are to be evaluated.
+                Given as a dictionary with the decision variable symbols as keys and a list of decision variable values
+                as the values. The length of the lists is the number of samples and each list should have the same
+                length (same number of samples).
+
+        Returns:
+            pl.DataFrame: The values of the evaluated objectives, constraints and extra functions as a polars
+                dataframe. The uncertainty prediction values are also returned. If a model does not provide
+                uncertainty predictions, then they are set as NaN.
+        """
+        res = pl.DataFrame()
+        var = np.array([value for _, value in xs.items()]).T  # has to be transpose (at least for sklearn models)
+        for symbol in self.surrogates:
+            # get a list of args accepted by the model's predict function
+            accepted_args = getfullargspec(self.surrogates[symbol].predict).args
+            # if "return_std" accepted, gather the uncertainty predictions as well
+            if "return_std" in accepted_args:
+                value, uncertainty = self.surrogates[symbol].predict(var, return_std=True)
+            # otherwise, set the uncertainties as NaN
+            else:
+                value = self.surrogates[symbol].predict(var)
+                uncertainty = np.full(np.shape(value), np.nan)
+            # add the objects, constraints and extra functions into the polars dataframe
+            # values go into columns with the symbol as the column names
+            res = res.with_columns(pl.Series(value).alias(symbol))
+            # uncertainties go into columns with {symbol}_uncert as the column names
+            res = res.with_columns(pl.Series(uncertainty).alias(f"{symbol}_uncert"))
+
+        # Evaluate the minimization form of the objective functions
+        min_obj_columns = pl.DataFrame()
+        for symbol, min_max_mult in self.objective_mix_max_mult:
+            if symbol in res.columns:
+                min_obj_columns = min_obj_columns.hstack(
+                    res.select((min_max_mult * pl.col(f"{symbol}")).alias(f"{symbol}_min"))
+                )
+        res_df = res.hstack(min_obj_columns)
+        # If there are scalarization functions, evaluate them as well
+        scalarization_columns = res_df.select(*[expr.alias(symbol) for symbol, expr in self.scalarization_funcs])
+        return res_df.hstack(scalarization_columns)
+
+    def _load_surrogates(self, surrogate_paths: dict[str, Path] | None = None):
+        """Load the surrogate models from disk and store them within the evaluator.
+
+        This is used during initialization of the evaluator or when the analyst wants to replace the current surrogate
+        models with other models. However if a new model is trained after initialization of the evaluator, the problem
+        JSON should be updated with the new model paths and the evaluator should be re-initialized. This can happen
+        with any solver that does model management.
+
+        Args:
+            surrogate_paths (dict[str, Path]): A dictionary where the keys are the names of the objectives, constraints
+                and extra functions and the values are the paths to the surrogate models saved on disk. The names of
+                the objectives should match the names of the objectives in the problem JSON. At the moment the supported
+                file format is .skops (through skops.io). TODO: if skops.io used, should be added to pyproject.toml.
+        """
+        if surrogate_paths is not None:
+            for symbol in surrogate_paths:
+                with Path.open(f"{surrogate_paths[symbol]}", "rb") as file:
+                    self.surrogates[symbol] = joblib.load(file)
+                    """unknown_types = sio.get_untrusted_types(file=file)
+                    if len(unknown_types) == 0:
+                        self.surrogates[symbol] = sio.load(file, unknown_types)
+                    else: # TODO: if there are unknown types they should be checked
+                        self.surrogates[symbol] = sio.load(file, unknown_types)
+                        #raise EvaluatorError(f"Untrusted types found in the model of {obj.symbol}: {unknown_types}")"""
+        else:
+            # check each surrogate based objective, constraint and extra function for surrogate path
+            for obj in self.problem.objectives:
+                if obj.surrogates is not None:
+                    with Path.open(f"{obj.surrogates[0]}", "rb") as file:
+                        self.surrogates[obj.symbol] = joblib.load(file)
+                        """unknown_types = sio.get_untrusted_types(file=file)
+                        if len(unknown_types) == 0:
+                            self.surrogates[obj.symbol] = sio.load(file, unknown_types)
+                        else: # TODO: if there are unknown types they should be checked
+                            self.surrogates[obj.symbol] = sio.load(file, unknown_types)
+                            #raise EvaluatorError(f"Untrusted types found in the model of {obj.symbol}: {unknown_types}")"""
+            for con in self.problem.constraints or []:  # if there are no constraints, an empty list is used
+                if con.surrogates is not None:
+                    with Path.open(f"{con.surrogates[0]}", "rb") as file:
+                        self.surrogates[con.symbol] = joblib.load(file)
+                        """unknown_types = sio.get_untrusted_types(file=file)
+                        if len(unknown_types) == 0:
+                            self.surrogates[con.symbol] = sio.load(file, unknown_types)
+                        else: # TODO: if there are unknown types they should be checked
+                            self.surrogates[con.symbol] = sio.load(file, unknown_types)
+                            #raise EvaluatorError(f"Untrusted types found in the model of {obj.symbol}: {unknown_types}")"""
+            for extra in self.problem.extra_funcs or []:  # if there are no extra functions, an empty list is used
+                if extra.surrogates is not None:
+                    with Path.open(f"{extra.surrogates[0]}", "rb") as file:
+                        self.surrogates[extra.symbol] = joblib.load(file)
+                        """unknown_types = sio.get_untrusted_types(file=file)
+                        if len(unknown_types) == 0:
+                            self.surrogates[extra.symbol] = sio.load(file, unknown_types)
+                        else: # TODO: if there are unknown types they should be checked
+                            self.surrogates[extra.symbol] = sio.load(file, unknown_types)
+                            #raise EvaluatorError(f"Untrusted types found in the model of {obj.symbol}: {unknown_types}")"""
+
+    def evaluate(self, xs: dict[str, list[int | float]], flat: bool = False) -> pl.DataFrame:
+        """Evaluate the functions for the given decision variables.
+
+        Evaluates analytical, simulation based and surrogate based functions. For now, the evaluator assumes that there
+        are no data based objectives.
+
+        Args:
+            xs (dict[str, list[int | float]]): The decision variables for which the functions are to be evaluated.
+                Given as a dictionary with the decision variable symbols as keys and a list of decision variable values
+                as the values. The length of the lists is the number of samples and each list should have the same
+                length (same number of samples).
+            flat (bool, optional): whether the valuation is done using flattened variables or not. Defaults to False.
+
+        Returns:
+            pl.DataFrame: polars dataframe with the evaluated function values.
+        """
+        # TODO (@gialmisi): Make work with polars dataframes as well in addition to dict.
+        # See, e.g., PolarsEvaluator._polars_evaluate. Then, remove the arg `flat`.
+        res = pl.DataFrame()
+
+        # Evaluate the analytical functions
+        if len(self.analytical_symbols + self.data_based_symbols) > 0:
+            polars_evaluator = PolarsEvaluator(self.problem, evaluator_mode=PolarsEvaluatorModesEnum.mixed)
+            analytical_values = (
+                polars_evaluator._polars_evaluate(xs) if not flat else polars_evaluator._polars_evaluate_flat(xs)
+            )
+            res = res.hstack(analytical_values)
+
+        # Evaluate the simulator based functions
+        if len(self.simulator_symbols) > 0:
+            simulator_values = self._evaluate_simulator(xs)
+            res = res.hstack(simulator_values)
+
+        # Evaluate the surrogate based functions
+        if len(self.surrogate_symbols) > 0:
+            surrogate_values = self._evaluate_surrogates(xs)
+            res = res.hstack(surrogate_values)
+
+        # Check that everything is evaluated
+        for symbol in self.problem_symbols:
+            if symbol not in res.columns:
+                raise EvaluatorError(f"{symbol} not evaluated.")
+        return res

desdeo/problem/sympy_evaluator.py

@@ -0,0 +1,244 @@
+"""Implements and evaluator based on sympy expressions."""
+
+from copy import deepcopy
+
+import sympy as sp
+
+from desdeo.problem.evaluator import variable_dimension_enumerate
+from desdeo.problem.json_parser import FormatEnum, MathParser
+from desdeo.problem.schema import Problem
+
+SUPPORTED_VAR_DIMENSIONS = ["scalar"]
+
+
+class SympyEvaluatorError(Exception):
+    """Raised when an exception with a Sympy evaluator is encountered."""
+
+
+class SympyEvaluator:
+    """Defines an evaluator that can be used to evaluate instances of Problem utilizing sympy."""
+
+    def __init__(self, problem: Problem):
+        """Initializes the evaluator.
+
+        Args:
+            problem (Problem): the problem to be evaluated.
+        """
+        if variable_dimension_enumerate(problem) not in SUPPORTED_VAR_DIMENSIONS:
+            msg = "SymPy evaluator does not yet support tensors."
+            raise SympyEvaluatorError(msg)
+
+        # Collect all the symbols and expressions in the problem
+        parser = MathParser(to_format=FormatEnum.sympy)
+
+        self.variable_symbols = [var.symbol for var in problem.variables]
+        self.constant_expressions = (
+            {const.symbol: parser.parse(const.value) for const in problem.constants}
+            if problem.constants is not None
+            else None
+        )
+
+        self.extra_expressions = (
+            {extra.symbol: parser.parse(extra.func) for extra in problem.extra_funcs}
+            if problem.extra_funcs is not None
+            else None
+        )
+
+        self.objective_expressions = {obj.symbol: parser.parse(obj.func) for obj in problem.objectives}
+
+        self.constraint_expressions = (
+            {con.symbol: parser.parse(con.func) for con in problem.constraints}
+            if problem.constraints is not None
+            else None
+        )
+
+        self.scalarization_expressions = (
+            {scal.symbol: parser.parse(scal.func) for scal in problem.scalarization_funcs}
+            if problem.scalarization_funcs is not None
+            else None
+        )
+
+        # replace symbols and create lambda functions ready to be called
+        # replace constants in extra functions, if they exist
+        if self.extra_expressions is not None:
+            _extra_expressions = (
+                {
+                    k: self.extra_expressions[k].subs(self.constant_expressions, evaluate=False)
+                    for k in self.extra_expressions
+                }
+                if self.constant_expressions is not None
+                else deepcopy(self.extra_expressions)
+            )
+        else:
+            _extra_expressions = None
+
+        # replace constants in objective functions, if constants have been defined
+        _objective_expressions = (
+            {
+                k: self.objective_expressions[k].subs(self.constant_expressions, evaluate=False)
+                for k in self.objective_expressions
+            }
+            if self.constant_expressions is not None
+            else deepcopy(self.objective_expressions)
+        )
+
+        # replace extra functions in objective functions, if extra functions have been defined
+        _objective_expressions = (
+            (
+                {
+                    k: _objective_expressions[k].subs(self.extra_expressions, evaluate=False)
+                    for k in _objective_expressions
+                }
+            )
+            if self.extra_expressions is not None
+            else _objective_expressions
+        )
+
+        # always minimized objective expressions
+        _objective_expressions_min = {
+            f"{obj.symbol}_min": -_objective_expressions[obj.symbol]
+            if obj.maximize
+            else _objective_expressions[obj.symbol]
+            for obj in problem.objectives
+        }
+
+        # replace stuff in the constraint expressions if any are defined
+        if self.constraint_expressions is not None:
+            # replace constants
+            _constraint_expressions = (
+                {
+                    k: self.constraint_expressions[k].subs(self.constant_expressions, evaluate=False)
+                    for k in self.constraint_expressions
+                }
+                if self.constant_expressions is not None
+                else deepcopy(self.constraint_expressions)
+            )
+
+            # replace extra functions
+            _constraint_expressions = (
+                {
+                    k: _constraint_expressions[k].subs(_extra_expressions, evaluate=False)
+                    for k in _constraint_expressions
+                }
+                if _extra_expressions is not None
+                else _constraint_expressions
+            )
+
+            # replace objective functions
+            _constraint_expressions = {
+                k: _constraint_expressions[k].subs(_objective_expressions, evaluate=False)
+                for k in _constraint_expressions
+            }
+            _constraint_expressions = {
+                k: _constraint_expressions[k].subs(_objective_expressions_min, evaluate=False)
+                for k in _constraint_expressions
+            }
+
+        else:
+            _constraint_expressions = None
+
+        # replace stuff in scalarization expressions if any are defined
+        if self.scalarization_expressions is not None:
+            # replace constants
+            _scalarization_expressions = (
+                {
+                    k: self.scalarization_expressions[k].subs(self.constant_expressions, evaluate=False)
+                    for k in self.scalarization_expressions
+                }
+                if self.constant_expressions is not None
+                else deepcopy(self.scalarization_expressions)
+            )
+
+            # replace extra functions
+            _scalarization_expressions = (
+                {
+                    k: _scalarization_expressions[k].subs(_extra_expressions, evaluate=False)
+                    for k in _scalarization_expressions
+                }
+                if _extra_expressions is not None
+                else _scalarization_expressions
+            )
+
+            # replace constraints
+            _scalarization_expressions = (
+                {
+                    k: _scalarization_expressions[k].subs(_constraint_expressions, evaluate=False)
+                    for k in _scalarization_expressions
+                }
+                if _constraint_expressions is not None
+                else _scalarization_expressions
+            )
+
+            # replace objectives
+            _scalarization_expressions = {
+                k: _scalarization_expressions[k].subs(_objective_expressions, evaluate=False)
+                for k in _scalarization_expressions
+            }
+
+            _scalarization_expressions = {
+                k: _scalarization_expressions[k].subs(_objective_expressions_min, evaluate=False)
+                for k in _scalarization_expressions
+            }
+
+        else:
+            _scalarization_expressions = None
+
+        # initialize callable lambdas
+        self.lambda_exprs = {
+            _k: _v
+            for _d in [
+                {k: sp.lambdify(self.variable_symbols, d[k]) for k in d}
+                for d in [
+                    _extra_expressions,
+                    _objective_expressions,
+                    _objective_expressions_min,
+                    _constraint_expressions,
+                    _scalarization_expressions,
+                ]
+                if d is not None
+            ]
+            for _k, _v in _d.items()
+        }
+
+        self.problem = problem
+        self.parser = parser
+
+    def evaluate(self, xs: dict[str, float | int | bool]) -> dict[str, float | int | bool]:
+        """Evaluate the the whole problem with a given decision variable dict.
+
+        Args:
+            xs (dict[str, float  |  int  |  bool]): a dict with keys representing decision variable
+                symbols and values with the decision variable value.
+
+        Returns:
+            dict[str, float | int | bool]: a dict with keys corresponding to each symbol
+                defined for the problem being evaluated and the corresponding expression's
+                value.
+        """
+        return {k: self.lambda_exprs[k](**xs) for k in self.lambda_exprs} | xs
+
+    def evaluate_target(self, xs: dict[str, float | int | bool], target: str) -> float:
+        """Evaluates only the specified target with given decision variables.
+
+        Args:
+            xs (dict[str, float  |  int  |  bool]): a dict with keys representing decision variable
+                symbols and values with the decision variable value.
+            target (str): the symbol of the function expressions to be evaluated.
+
+        Returns:
+            float: the value of the target once evaluated.
+        """
+        return self.lambda_exprs[target](**xs)
+
+    def evaluate_constraints(self, xs: dict[str, float | int | bool]) -> dict[str, float | int | bool]:
+        """Evaluates the constraints of the problem with given decision variables.
+
+        Args:
+            xs (dict[str, float  |  int  |  bool]): a dict with keys representing decision variable
+                symbols and values with the decision variable value.
+
+        Returns:
+            dict[str, float | int | bool]: a dict with keys being the constraints symbols
+                and values being the value of the corresponding constraint.
+        """
+        return {k: self.lambda_exprs[k](**xs) for k in [constr.symbol for constr in self.problem.constraints]}

desdeo/problem/testproblems/__init__.py

@@ -0,0 +1,88 @@
+"""Pre-defined multiobjective optimization problems.
+
+Pre-defined problems for, e.g.,
+testing and illustration purposed are defined here.
+"""
+
+__all__ = [  # noqa: RUF022
+    "binh_and_korn",
+    "dtlz2",
+    "forest_problem",
+    "forest_problem_discrete",
+    "mcwb_equilateral_tbeam_problem",
+    "mcwb_hollow_rectangular_problem",
+    "mcwb_ragsdell1976_problem",
+    "mcwb_solid_rectangular_problem",
+    "mcwb_square_channel_problem",
+    "mcwb_tapered_channel_problem",
+    "mixed_variable_dimensions_problem",
+    "momip_ti2",
+    "momip_ti7",
+    "multi_valued_constraint_problem",
+    "nimbus_test_problem",
+    "pareto_navigator_test_problem",
+    "re21",
+    "re22",
+    "re23",
+    "re24",
+    "river_pollution_problem",
+    "river_pollution_problem_discrete",
+    "river_pollution_scenario",
+    "rocket_injector_design",
+    "simple_data_problem",
+    "simple_integer_test_problem",
+    "simple_knapsack",
+    "simple_knapsack_vectors",
+    "simple_linear_test_problem",
+    "simple_scenario_test_problem",
+    "simple_test_problem",
+    "simulator_problem",
+    "spanish_sustainability_problem",
+    "spanish_sustainability_problem_discrete",
+    "zdt1",
+    "zdt2",
+    "zdt3",
+    "best_cake_problem",
+    "dmitry_forest_problem_disc",
+]
+
+
+from .binh_and_korn_problem import binh_and_korn
+from .cake_problem import best_cake_problem
+from .dmitry_forest_problem_discrete import dmitry_forest_problem_disc
+from .dtlz2_problem import dtlz2
+from .forest_problem import forest_problem, forest_problem_discrete
+from .knapsack_problem import simple_knapsack, simple_knapsack_vectors
+from .mcwb_problem import (
+    mcwb_equilateral_tbeam_problem,
+    mcwb_hollow_rectangular_problem,
+    mcwb_ragsdell1976_problem,
+    mcwb_solid_rectangular_problem,
+    mcwb_square_channel_problem,
+    mcwb_tapered_channel_problem,
+)
+from .mixed_variable_dimenrions_problem import mixed_variable_dimensions_problem
+from .momip_problem import momip_ti2, momip_ti7
+from .multi_valued_constraints import multi_valued_constraint_problem
+from .nimbus_problem import nimbus_test_problem
+from .pareto_navigator_problem import pareto_navigator_test_problem
+from .re_problem import re21, re22, re23, re24
+from .river_pollution_problems import (
+    river_pollution_problem,
+    river_pollution_problem_discrete,
+    river_pollution_scenario,
+)
+from .rocket_injector_design_problem import rocket_injector_design
+from .simple_problem import (
+    simple_data_problem,
+    simple_integer_test_problem,
+    simple_linear_test_problem,
+    simple_scenario_test_problem,
+    simple_test_problem,
+)
+from .simulator_problem import simulator_problem
+from .spanish_sustainability_problem import (
+    spanish_sustainability_problem,
+    spanish_sustainability_problem_discrete,
+)
+from .zdt_problem import zdt1, zdt2, zdt3