desdeo 1.2__py3-none-any.whl → 2.1.0__py3-none-any.whl

desdeo/problem/external/core.py

@@ -0,0 +1,356 @@
+"""Implements a interface to interface into external (test) problem suites."""
+
+import json
+from typing import Any, Literal, Protocol
+from urllib.parse import urlparse
+
+import requests
+from pydantic import BaseModel, Field
+
+from desdeo.problem import ConstraintTypeEnum, VariableType, VariableTypeEnum
+
+Operation = Literal["info", "evaluate"]
+Scheme = Literal["desdeo", "http"]
+
+
+class ExternalProblemInfo(BaseModel):
+    """A model to represent problem information of problems, which are not native to DESDEO."""
+
+    name: str = Field(description="Name of the problem")
+    description: str | None = Field(description="Description of the problem. Default to 'None'.", default=None)
+    variable_symbols: list[str] = Field(description="The symbols of the variables.")
+    variable_names: dict[str, str] | None = Field(
+        description=(
+            "The names of the variables. It is expected that the keys are the same as the provided symbols. "
+            "Defaults to 'None', in which case the symbols are used."
+        )
+    )
+    variable_type: dict[str, VariableTypeEnum] | None = Field(
+        description=(
+            "The type of each variable (real, integer, binary). It is "
+            "expected that the keys are the same as the provided symbols. If 'None', "
+            "the type 'real' is assumed for all variables. Defaults to 'None'."
+        )
+    )
+    variable_lower_bounds: dict[str, VariableType | None] | None = Field(
+        description=(
+            "The lower bound of each variable. It is "
+            "expected that the keys are the same as the provided symbols. If 'None', "
+            "the value is None, no bounds are assumed for all lower bounds. variables. Defaults to 'None'."
+        )
+    )
+    variable_upper_bounds: dict[str, VariableType | None] | None = Field(
+        description=(
+            "The upper bound of each variable. It is "
+            "expected that the keys are the same as the provided symbols. If 'None', "
+            "the value is None, no bounds are assumed for all upper bounds. variables. Defaults to 'None'."
+        )
+    )
+    objective_symbols: list[str] = Field(description="The names of the objective functions.")
+    objective_names: dict[str, str] | None = Field(
+        description=(
+            "The names of the objectives. It is expected that the keys are the same as the provided symbols. "
+            "Defaults to 'None', in which case the symbols are used."
+        )
+    )
+    objective_maximize: dict[str, bool] | None = Field(
+        description=(
+            "Whether objective are to be maximized. It is "
+            "expected that the keys are the same as the provided symbols. "
+            "Defaults to 'None', in which case minimization is assumed."
+        )
+    )
+    ideal_point: dict[str, float] | None = Field(
+        description=(
+            "The ideal point of the problem. The keys should match those of the objective functions. "
+            "Defaults to 'None'."
+        ),
+        default=None,
+    )
+    nadir_point: dict[str, float] | None = Field(
+        description=(
+            "The nadir point of the problem. The keys should match those of the objective functions. "
+            "Defaults to 'None'."
+        ),
+        default=None,
+    )
+    constraint_symbols: list[str] | None = Field(
+        description=(
+            "The symbols of constraints. If 'None', no constraints are defined for the problem. Defaults to None."
+        )
+    )
+    constraint_names: dict[str, str] | None = Field(
+        description=(
+            "The names of the constraints. It is expected that the keys are the same as the provided symbols. "
+            "Defaults to 'None', in which case the symbols are used."
+        )
+    )
+    constraint_types: dict[str, ConstraintTypeEnum] | None = Field(
+        description=(
+            "The types (LTE, EQ...) of the constraints. It is expected that the "
+            "keys are the same as the provided symbols. Defaults to 'None', in "
+            "which case no symbols are assumed to be defined."
+        )
+    )
+
+
+class ExternalProblemParams(BaseModel):
+    """A model to represent parameters that can be used to generate problems, which are not native to DESDEO."""
+
+
+class ProviderParams(BaseModel):
+    """A model to represent parameters that external libraries can use to build problem."""
+
+
+class Locator(BaseModel):
+    """A model to represent a locator, i.e., a URI."""
+
+    scheme: Scheme = Field(description="The source of the problem.")
+    provider: str | None = Field(description="The provider of the operation. Defaults to 'None'.", default=None)
+    op: Operation | None = Field(description="The associated operator. Defaults to 'None'.", default=None)
+    raw: str = Field(description="The raw uri.")
+
+    # for http
+    http_url: str | None = Field(description="Uri for web-access. Defaults to 'None'.", default=None)
+
+
+class LocatorParseError(ValueError):
+    """Raised when errors emerge parsing a locator."""
+
+
+def parse_locator(uri: str) -> Locator:
+    """Parses a string representing a uri into a Locator model.
+
+    Args:
+        uri (str): a string containing a uri. Should follow the format 'scheme://provider/operation'.
+
+    Raises:
+        LocatorParsesError: when parsing the uri goes wrong for one reason or another.
+
+    Returns:
+        Locator: a locator with the information parsed from the uri.
+    """
+    p = urlparse(uri)
+    expected_segments = 2
+
+    if p.scheme in ("http", "https"):
+        segments = [s for s in p.path.split("/") if s]
+        if len(segments) == expected_segments:
+            provider, op = segments
+        else:
+            raise LocatorParseError(f"Invalid path; expected 2 segments in {uri!r}")
+
+        return Locator(
+            scheme="http",
+            op=None,
+            provider=None,
+            raw=uri,
+            http_url=uri,
+        )
+
+    if p.scheme != "desdeo":
+        raise LocatorParseError(f"Unsupported scheme: {p.scheme!r} in {uri!r}")
+
+    segments = [s for s in p.path.split("/") if s]
+    if len(segments) == expected_segments:
+        provider, op = segments
+    else:
+        raise LocatorParseError(f"Invalid path; expected 2 segments in {uri!r}")
+
+    if op not in ("info", "evaluate"):
+        raise LocatorParseError(f"Unsupported operation {op!r} in {uri!r}")
+
+    if not provider or not (provider[0].isalpha() and all(c.isalnum() or c in "_-" for c in provider)):
+        raise LocatorParseError(f"Invalid provider identifier {provider!r} in {uri!r}")
+
+    return Locator(
+        scheme="desdeo",
+        op=op,
+        provider=provider,
+        raw=uri,
+        http_url=None,
+    )
+
+
+class Provider(Protocol):
+    """The minimal interface any implemented provider should fulfill."""
+
+    def info(self, params: ExternalProblemParams) -> ExternalProblemInfo:
+        """Return the information of an external problem the provider exposes.
+
+        Args:
+            params (ExternalProblemParams): the parameters to generate the external problem.
+
+        Returns:
+            ExternalProblemInfo: information on the external problem.
+        """
+
+    def evaluate(
+        self, xs: dict[str, VariableType | list[VariableType]], params: ProviderParams | dict[str, Any]
+    ) -> dict[str, float]:
+        """Given a set of variable values, evalate the external problem.
+
+        Args:
+            xs (dict[str, VariableType]): a set of variables to be evaluated.
+                Expected format is {variable symbol: [values]}.
+            params (ProviderParams | dict[str, Any]): the parameters that can be used to generate the external problem.
+                Can also be a dict.
+
+        Returns:
+            dict[str, float]: a dict with keys corresponding to evaluated fields of the problem, e.g.,
+                objective functions, constraints, etc., and values consisting of lists.
+
+        Note:
+            When multiple values are provided for the variables, it is assumed that
+            the external problem is evaluated pointwise and that the returned values
+            correspond to the input values in the same order.
+        """
+
+
+class UnknownProviderError(KeyError):
+    """Raised when a non-existing provider is encountered."""
+
+
+class ProviderRegistry:
+    """A registry of available providers."""
+
+    def __init__(self) -> None:
+        """Initializes the registry with an empty listing of providers."""
+        self._providers: dict[str, Provider] = {}
+
+    def register(self, name: str, provider: Provider) -> None:
+        """Register a new provider.
+
+        Args:
+            name (str): the name of the provider.
+            provider (Provider): the provider to be registered.
+        """
+        self._providers[name] = provider
+
+    def get(self, name: str) -> Provider:
+        """Get a provider from the registry based on its name."""
+        try:
+            return self._providers[name]
+        except KeyError as e:
+            raise UnknownProviderError(f"Unknown provider: {name!r}") from e
+
+    def has(self, name: str) -> bool:
+        """Checks if a provider already exists in the registry.
+
+        Args:
+            name (str): name of the provider.
+
+        Returns:
+            bool: whether it exists in the registry.
+        """
+        return name in self._providers
+
+    def items(self) -> list[Provider]:
+        """Return the current providers in the registry.
+
+        Returns:
+            list[Provider]: list with the providers in the registry.
+        """
+        return self._providers.items()
+
+
+class ProviderResolverError(RuntimeError):
+    """Raised when the resolver fails to resolve to an existing simulator."""
+
+
+def _stable_json(d: dict[str, Any]) -> str:
+    # stable key for caching etc.
+    return json.dumps(d, sort_keys=True, separators=(",", ":"))
+
+
+class ProviderResolver:
+    """Defines a resolver for registered providers."""
+
+    def __init__(self, registry: ProviderRegistry) -> None:
+        """Initialize the resolver with a registry of providers.
+
+        Args:
+            registry (ProviderRegistry): a registry of providers the resolver should be aware of.
+        """
+        self.registry = registry
+        self._info_cache: dict[tuple[str, str], dict[str, Any]] = {}
+
+    def clear_caches(self) -> None:
+        """Clears the info cache."""
+        self._info_cache.clear()
+
+    def info(self, locator_uri: str, params: dict[str, Any]) -> ExternalProblemInfo:
+        """Get info on the problem from the resolved provider.
+
+        Args:
+            locator_uri (str): the uri to locate the provider of the problem's info.
+            params (dict[str, Any]): parameters given to the provider.
+
+        Raises:
+            ProviderResolverError: when the provider cannot be resolved.
+
+        Returns:
+            dict[str, Any]:
+        """
+        loc = parse_locator(locator_uri)
+
+        if loc.scheme == "http":
+            raise ProviderResolverError("TODO: HTTP info not implemented.")
+
+        if loc.provider is None:
+            raise ProviderResolverError("Could not resolve the provider.")
+
+        cache_key = (loc.provider, _stable_json(params))
+
+        if cache_key in self._info_cache:
+            return self._info_cache[cache_key]
+
+        provider = self.registry.get(loc.provider)
+        out = provider.info(params)
+
+        self._info_cache[cache_key] = out
+
+        return out
+
+    def evaluate(
+        self,
+        locator_uri: str,
+        params: ProviderParams | dict[str, Any],
+        xs: dict[str, VariableType | list[VariableType]],
+    ) -> dict[str, float]:
+        """Evaluate the problem with a resolved provider.
+
+        Args:
+            locator_uri (str): the uri to locate the provider.
+            params (ProviderParams | dict[str, Any]): parameters given to the provider.
+            xs (dict[str, VariableType]): a set of variables to be evaluated.
+                Expected format is {variable symbol: [values]}.
+
+        Raises:
+            ProviderResolverError: could not evaluate the problem.
+
+        Returns:
+            dict[str, float]: a dict with keys corresponding to evaluated fields of the problem, e.g.,
+                objective functions, constraints, etc., and values consisting of lists.
+
+        Note:
+            When multiple values are provided for the variables, it is assumed that
+            the external problem is evaluated pointwise and that the returned values
+            correspond to the input values in the same order.
+        """
+        loc = parse_locator(locator_uri)
+        if loc.scheme == "http":
+            # remote
+            try:
+                r = requests.post(loc.http_url, json={"params": params, "X": xs}, timeout=30)
+                r.raise_for_status()
+                return r.json()
+            except requests.RequestException as e:
+                raise ProviderResolverError(f"HTTP evaluation failed for {loc.http_url!r}") from e
+
+        if loc.provider is None:
+            raise ProviderResolverError("Could not resolve the provider.")
+
+        provider = self.registry.get(loc.provider)
+
+        return provider.evaluate(xs, params)

desdeo/problem/external/pymoo_provider.py

@@ -0,0 +1,266 @@
+"""Implements a provider for the test problems found in Pymoo."""
+
+import json
+from functools import lru_cache
+from typing import Any
+
+import numpy as np
+from pydantic import Field
+from pymoo.problems import get_problem as pymoo_get_problem
+
+from desdeo.problem import (
+    Constraint,
+    ConstraintTypeEnum,
+    Objective,
+    Problem,
+    Simulator,
+    Url,
+    Variable,
+    VariableType,
+    VariableTypeEnum,
+)
+
+from .core import ExternalProblemInfo, ExternalProblemParams, Provider
+
+_pymoo_locator_evaluate = "desdeo://external/pymoo/evaluate"
+
+
+class PymooProblemParams(ExternalProblemParams):
+    """Parameters to generate Pymoo test problems.
+
+    See: https://pymoo.org/problems/test_problems.html
+    """
+
+    name: str = Field(description="The name of the model. See https://pymoo.org/problems/test_problems.html")
+    n_var: int | None = Field(description="The number of desirable variables (if relevant).", default=None)
+    n_obj: int | None = Field(description="The number of desirable objective functions (if relevant).", default=None)
+    extra: dict[str, Any] | None = Field(description="Any other extra parameters.", default=None)
+
+
+def _stable_json(d: dict[str, Any]) -> str:
+    return json.dumps(d, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+
+
+def _params_key(params: dict) -> str:
+    """Canonical cache key for a PymooParams instance.
+
+    Canonical cache key for a PymooParams instance.
+    Includes 'extra' (if present) and omits None fields.
+    """
+    return _stable_json(params)
+
+
+@lru_cache(maxsize=256)
+def _get_cached_pymoo_problem(params_key: str):
+    """Cache pymoo Problem instances by a stable JSON key.
+
+    Note: per-process cache (fine for typical single-worker runs).
+    """
+    params_dict = json.loads(params_key)
+
+    extra = params_dict.pop("extra", None)
+    if isinstance(extra, dict):
+        params_dict |= extra
+
+    return pymoo_get_problem(**{k: v for k, v in params_dict.items() if v is not None})
+
+
+@lru_cache(maxsize=256)
+def _get_cached_info(params_key: str) -> ExternalProblemInfo:
+    """Cache ExternalProblemInfo too, since it derives from the cached pymoo problem."""
+    pymoo_problem = _get_cached_pymoo_problem(params_key)
+
+    p_name = pymoo_problem.name()  # why?
+    p_n_var = pymoo_problem.n_var
+    p_n_obj = pymoo_problem.n_obj
+    p_n_ieq_constr = pymoo_problem.n_ieq_constr
+    p_n_eq_constr = pymoo_problem.n_eq_constr
+    p_xl = pymoo_problem.xl
+    p_xu = pymoo_problem.xu
+    p_vtype = pymoo_problem.vtype
+
+    try:
+        p_ideal = pymoo_problem.ideal_point()
+    except Exception:
+        p_ideal = None
+    try:
+        p_nadir = pymoo_problem.nadir_point()
+    except Exception:
+        p_nadir = None
+
+    var_symbols = [f"x_{i}" for i in range(1, p_n_var + 1)]
+    obj_symbols = [f"f_{i}" for i in range(1, p_n_obj + 1)]
+    constr_symbols = (
+        [f"c_{i}" for i in range(1, p_n_ieq_constr + p_n_eq_constr + 1)] if p_n_ieq_constr + p_n_eq_constr > 0 else None
+    )
+    constr_types = [ConstraintTypeEnum.LTE] * p_n_ieq_constr + [ConstraintTypeEnum.EQ] * p_n_eq_constr
+
+    var_type_mapping = {float: VariableTypeEnum.real, int: VariableTypeEnum.integer, bool: VariableTypeEnum.binary}
+
+    return ExternalProblemInfo(
+        name=p_name,
+        description=f"The {p_name} problem as defined in the Pymoo library.",
+        variable_symbols=var_symbols,
+        variable_names=dict(zip(var_symbols, var_symbols, strict=True)),
+        variable_type=dict.fromkeys(var_symbols, var_type_mapping[p_vtype]),  # assumed all same
+        variable_lower_bounds=dict(zip(var_symbols, p_xl, strict=True)),
+        variable_upper_bounds=dict(zip(var_symbols, p_xu, strict=True)),
+        objective_symbols=obj_symbols,
+        objective_names=dict(zip(obj_symbols, obj_symbols, strict=True)),
+        objective_maximize=dict.fromkeys(obj_symbols, False),  # assumed all min
+        ideal_point=dict(zip(obj_symbols, p_ideal, strict=True)) if p_ideal is not None else None,
+        nadir_point=dict(zip(obj_symbols, p_nadir, strict=True)) if p_nadir is not None else None,
+        constraint_symbols=constr_symbols,
+        constraint_names=dict(zip(constr_symbols, constr_symbols, strict=True)) if constr_symbols is not None else None,
+        constraint_types=dict(zip(constr_symbols, constr_types, strict=True)) if len(constr_types) > 0 else None,
+    )
+
+
+class PymooProvider(Provider):
+    """Provider to get info on and evaluate Pymoo test problems.
+
+    Note:
+        the methods `info` and `evaluate` use caching so that the Pymoo problem is generated only once
+            in case of multiple consecutive evaluations of the same problem.
+    """
+
+    def info(self, params: PymooProblemParams | dict[str, Any]) -> ExternalProblemInfo:
+        """Get info on a Pymoo test problem.
+
+        Args:
+            params (PymooProblemParams | dict[str, Any]): parameters to generate the problem.
+
+        Returns:
+            ExternalProblemInfo: info on the problem generated based on the provided parameters.
+        """
+        return _get_cached_info(_params_key(params if isinstance(params, dict) else params.model_dump()))
+
+    def evaluate(
+        self, xs: dict[str, VariableType | list[VariableType]], params: PymooProblemParams | dict[str, Any]
+    ) -> dict[str, float]:
+        """Evaluate a Pymoo test problem.
+
+        Args:
+            xs (dict[str, VariableType]): a set of variables to be evaluated.
+                Expected format is {variable symbol: [values]}.
+            params (ProviderParams | dict[str, Any]): parameters that generate the problem Pymoo
+                problem to be evaluated.
+
+        Returns:
+            dict[str, float]: a dict with keys corresponding to evaluated fields of the problem, e.g.,
+                objective functions, constraints, etc., and values consisting of lists.
+
+        Note:
+            When multiple values are provided for the variables, it is assumed that
+            the external problem is evaluated pointwise and that the returned values
+            correspond to the input values in the same order.
+        """
+        params_key = _params_key(params if isinstance(params, dict) else params.model_dump())
+        problem = _get_cached_pymoo_problem(params_key)
+        info = _get_cached_info(params_key)
+
+        out = {"F": None, "G": None, "H": None}
+
+        xs_arr = np.atleast_2d(np.column_stack([xs[k] for k in info.variable_symbols]))
+        problem._evaluate(np.asarray(xs_arr, dtype=float), out)
+
+        # parse output
+        obj_results = dict(zip(info.objective_symbols, np.atleast_2d(out["F"].T).tolist(), strict=True))
+        _constrs = (np.atleast_2d(out["G"].T).tolist() if out["G"] is not None else []) + (
+            np.atleast_2d(out["H"].T).tolist() if out["H"] is not None else []
+        )
+        constr_results = dict(zip(info.constraint_symbols, _constrs, strict=True)) if len(_constrs) > 0 else {}
+
+        return obj_results | constr_results
+
+
+def create_pymoo_problem(params: PymooProblemParams) -> Problem:
+    """Create a Pymoo test problem based on given parameters.
+
+    Args:
+        params (PymooProblemParams): the parameters to generate the Pymoo test problem.
+
+    Returns:
+        Problem: an instance of a Pymoo test problem generated based on the given parameters.
+
+    Note:
+        Any `Problem` generated with this function should be considered to be black-box, i.e.,
+        the exact mathematical forms are not available. However, if the user is knowledgeable
+        about the properties of the problem, they can still use, for example, some gradient-based
+        solvers available in DESDEO to try and solve the problem, if the problem is differentiable.
+
+        Ideal and nadir point values will be available, if they are available in Pymoo for a given problem.
+
+        For info on the possible problems to generate, see https://pymoo.org/problems/test_problems.html
+    """
+    provider = PymooProvider()
+    info = provider.info(params)
+
+    simulator_url = Url(url=_pymoo_locator_evaluate)
+
+    variables: list[Variable] = []
+    for sym in info.variable_symbols:
+        v_name = info.variable_names[sym] if info.variable_names is not None else sym
+        v_type = info.variable_type[sym] if info.variable_type is not None else VariableTypeEnum.real
+        lb = info.variable_lower_bounds[sym] if info.variable_lower_bounds is not None else None
+        ub = info.variable_upper_bounds[sym] if info.variable_upper_bounds is not None else None
+
+        variables.append(
+            Variable(
+                name=v_name,
+                symbol=sym,
+                lowerbound=lb,
+                upperbound=ub,
+                variable_type=v_type,
+            )
+        )
+
+    objectives: list[Objective] = []
+    for sym in info.objective_symbols:
+        o_name = info.objective_names[sym] if info.objective_names is not None else sym
+        maximize = info.objective_maximize[sym] if info.objective_maximize is not None else False
+
+        objectives.append(
+            Objective(
+                name=o_name,
+                symbol=sym,
+                simulator_path=simulator_url,
+                objective_type="simulator",
+                maximize=maximize,
+            )
+        )
+
+    # Constraints (optional)
+    constraints: list[Constraint] = []
+    if info.constraint_symbols is not None and len(info.constraint_symbols) > 0:
+        for sym in info.constraint_symbols:
+            c_name = info.constraint_names[sym] if info.constraint_names is not None else sym
+            c_type = info.constraint_types[sym] if info.constraint_types is not None else ConstraintTypeEnum.LTE
+
+            constraints.append(
+                Constraint(
+                    name=c_name,
+                    symbol=sym,
+                    simulator_path=simulator_url,
+                    cons_type=c_type,
+                )
+            )
+
+    simulator = Simulator(
+        name="pymoo_sim",
+        symbol="pymoo_sim",
+        url=simulator_url,
+        parameter_options=params.model_dump(exclude_none=True),
+    )
+
+    problem = Problem(
+        name=info.name,
+        description=info.description,
+        variables=variables,
+        objectives=objectives,
+        constraints=constraints if len(constraints) > 0 else None,
+        simulators=[simulator],
+    )
+
+    # add ideal and nadir (might be None)
+    return problem.update_ideal_and_nadir(info.ideal_point, info.nadir_point)

desdeo/problem/external/runtime.py

@@ -0,0 +1,44 @@
+"""Runtime which owns a general ProviderResolver singleton and exposes functions to manage it."""
+
+from .core import Provider, ProviderRegistry, ProviderResolver
+
+_registry = ProviderRegistry()
+_resolver = ProviderResolver(_registry)
+
+# if uri's of other type than 'desdeo://...' are to be supported, update this list
+supported_schemes = ["desdeo"]
+
+
+def get_registry() -> ProviderRegistry:
+    """Get the runtime registry."""
+    return _registry
+
+
+def get_resolver() -> ProviderResolver:
+    """Get the runtime provider resolver."""
+    return _resolver
+
+
+def register_provider(name: str, provider: Provider, *, overwrite: bool = False, clear_cache: bool = True) -> None:
+    """Register a provider to the current runtime resolver.
+
+    Args:
+        name (str): name of the provider.
+        provider (Provider): the instance of the provider.
+        overwrite (bool, optional): should an existing provider with the same
+            name be overwritten, if it already exits? Defaults to False.
+        clear_cache (bool, optional): should the resolver's cache be cleared? Defaults to True.
+
+    Raises:
+        KeyError: if overwrite is 'False' and a provider with the given `name`
+            already exists in the register of the resolver.
+    """
+    reg = get_registry()
+
+    if reg.has(name) and not overwrite:
+        raise KeyError(f"Provider {name!r} already registered")
+
+    reg.register(name, provider)
+
+    if clear_cache:
+        get_resolver().clear_caches()