fcmaes 1.3.17__py3-none-any.whl → 1.6.9__py3-none-any.whl

fcmaes/mode.py

@@ -2,14 +2,12 @@
 #
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory.
+from __future__ import annotations
 
 """ Numpy based implementation of multi objective
-    Differential Evolution using either DE/rand/1 or DE/best/1 strategy ('best' refers to the current pareto front'). 
-     
-    Can switch to NSGA-II like population update via parameter 'nsga_update'.
-    Then it works essentially like NSGA-II but instead of the tournament selection
-    the whole population is sorted and the best individuals survive. To do this
-    efficiently the crowd distance ordering is slightly inaccurate. 
+    Differential Evolution using either the DE/rand/1 strategy
+    or a NSGA-II like population update (parameter 'nsga_update=True)'.
+    Then it works similar to NSGA-II.
     
     Supports parallel fitness function evaluation. 
     
@@ -31,37 +29,56 @@
     parameter to parallelize objective function evaluation. The workers parameter is limited by the 
     population size.
     
+    The ints parameter is a boolean array indicating which parameters are discrete integer values. This 
+    parameter was introduced after observing non optimal DE-results for the ESP2 benchmark problem: 
+    https://github.com/AlgTUDelft/ExpensiveOptimBenchmark/blob/master/expensiveoptimbenchmark/problems/DockerCFDBenchmark.py
+    If defined it causes a "special treatment" for discrete variables: They are rounded to the next integer value and
+    there is an additional mutation to avoid getting stuck at local minima. This behavior is specified by the internal
+    function _modifier which can be overwritten by providing the optional modifier argument. If modifier is defined,
+    ints is ignored. 
+    
     See https://github.com/dietmarwo/fast-cma-es/blob/master/tutorials/MODE.adoc for a detailed description.
 """
 
 import numpy as np
-import os
-import time
+import os, sys, time
 import ctypes as ct
-from numpy.random import Generator, MT19937
-from fcmaes.evaluator import Evaluator
+from numpy.random import Generator, PCG64DXSM
+from scipy.optimize import Bounds
+
+from fcmaes.evaluator import Evaluator, parallel_mo
 from fcmaes import moretry
 import multiprocessing as mp
-from fcmaes.optimizer import logger
-from fcmaes import moretry
+from fcmaes.optimizer import dtime
+from fcmaes.retry import Shared2d
+from loguru import logger
+from typing import Optional, Callable, Tuple
+from numpy.typing import ArrayLike
 
 os.environ['MKL_DEBUG_CPU_TYPE'] = '5'
 
-def minimize(mofun, 
-             nobj,
-             ncon, 
-             bounds,
-             popsize = 64, 
-             max_evaluations = 100000, 
-             workers = None,
-             f = 0.5, 
-             cr = 0.9, 
-             nsga_update = False,
-             pareto_update = 0,
-             rg = Generator(MT19937()),
-             logger = None,
-             plot_name = None,
-             store = None):  
+def minimize(mofun: Callable[[ArrayLike], ArrayLike], 
+             nobj: int,
+             ncon: int,
+             bounds: Bounds,
+             guess: Optional[np.ndarray] = None,
+             popsize: Optional[int] = 64,
+             max_evaluations: Optional[int] = 100000,
+             workers: Optional[int] = 1,
+             f: Optional[float] = 0.5,
+             cr: Optional[float] = 0.9,
+             pro_c: Optional[float] = 0.5,
+             dis_c: Optional[float] = 15.0,
+             pro_m: Optional[float] = 0.9,
+             dis_m: Optional[float] = 20.0,
+             nsga_update: Optional[bool] = True,
+             pareto_update: Optional[int] = 0,
+             ints: Optional[ArrayLike] = None,
+             modifier: Callable = None,
+             min_mutate: Optional[float] = 0.1,
+             max_mutate: Optional[float] = 0.5,
+             rg: Optional[Generator] = Generator(PCG64DXSM()),
+             store: Optional[store] = None) -> Tuple[np.ndarray, np.ndarray]:  
       
     """Minimization of a multi objjective function of one or more variables using
     Differential Evolution.
@@ -70,10 +87,8 @@ def minimize(mofun,
     ----------
     mofun : callable
         The objective function to be minimized.
-            ``mofun(x, *args) -> list(float)``
-        where ``x`` is an 1-D array with shape (n,) and ``args``
-        is a tuple of the fixed parameters needed to completely
-        specify the function.
+            ``mofun(x) -> ndarray(float)``
+        where ``x`` is an 1-D array with shape (n,)
     nobj : int
         number of objectives
     ncon : int
@@ -84,33 +99,40 @@ def minimize(mofun,
             1. Instance of the `scipy.Bounds` class.
             2. Sequence of ``(min, max)`` pairs for each element in `x`. None
                is used to specify no bound.
+    guess : ndarray, shape (popsize,dim) or Tuple
+        Initial guess. 
     popsize : int, optional
         Population size.
     max_evaluations : int, optional
-        Forced termination after ``max_evaluations`` function evaluations.
+        Forced termination after ``max_evaluations`` function evaluations. 
     workers : int or None, optional
-        If not workers is None, function evaluation is performed in parallel for the whole population. 
+        workers > 1, function evaluation is performed in parallel for the whole population. 
         Useful for costly objective functions     
     f = float, optional
         The mutation constant. In the literature this is also known as differential weight, 
         being denoted by F. Should be in the range [0, 2].
     cr = float, optional
         The recombination constant. Should be in the range [0, 1]. 
-        In the literature this is also known as the crossover probability.     
+        In the literature this is also known as the crossover probability.   
+    pro_c, dis_c, pro_m, dis_m = float, optional
+        NSGA population update parameters, usually leave at default.   
     nsga_update = boolean, optional
-        Use of NSGA-II or DE population update. Default is False    
+        Use of NSGA-II/SBX or DE population update. Default is True    
     pareto_update = float, optional
-        Only applied if nsga_update = False. Use the pareto front for population update 
-        with probability pareto_update, else use the whole population. Default 0 - use always 
-        the whole population.      
+        Only applied if nsga_update = False. Favor better solutions for sample generation. Default 0 - 
+        use all population members with the same probability.   
+    ints = list or array, optional
+        indicating which parameters are discrete integer values. If defined these parameters will be
+        rounded to the next integer and some additional mutation of discrete parameters are performed.
+    min_mutate = float, optional
+        Determines the minimal mutation rate for discrete integer parameters.
+    max_mutate = float, optional
+        Determines the maximal mutation rate for discrete integer parameters. 
+    modifier = callable, optional
+        used to overwrite the default behaviour induced by ints. If defined, the ints parameter is
+        ignored. Modifies all generated x vectors.
     rg = numpy.random.Generator, optional
         Random generator for creating random guesses.
-    logger : logger, optional
-        logger for log output for tell_one, If None, logging
-        is switched off. Default is a logger which logs both to stdout and
-        appends to a file ``optimizer.log``.
-    plot_name : plot_name, optional
-        if defined the pareto front is plotted during the optimization to monitor progress
     store : result store, optional
         if defined the optimization results are added to the result store. For multi threaded execution.
         use workers=1 if you call minimize from multiple threads
@@ -119,17 +141,19 @@ def minimize(mofun,
     -------
     x, y: list of argument vectors and corresponding value vectors of the optimization results. """
 
-    mode = MODE(nobj, ncon, bounds, popsize, workers if not workers is None else 0, 
-            f, cr, nsga_update, pareto_update, rg, logger, plot_name)
-    try:
-        if workers and workers > 1:
-            x, y, evals, iterations, stop = mode.do_optimize_delayed_update(mofun, max_evaluations, workers)
-        else:      
-            x, y, evals, iterations, stop = mode.do_optimize(mofun, max_evaluations)
+    try:   
+        mode = MODE(nobj, ncon, bounds, popsize,
+            f, cr, pro_c, dis_c, pro_m, dis_m, nsga_update, pareto_update, rg, ints, min_mutate, max_mutate, modifier)
+        mode.set_guess(guess, mofun, rg)
+        if workers <= 1:
+            x, y, = mode.minimize_ser(mofun, max_evaluations)
+        else:
+            x, y = mode.minimize_par(mofun, max_evaluations, workers)
         if not store is None:
             store.add_results(x, y)
         return x, y
     except Exception as ex:
+        print(str(ex))  
         return None, None  
 
     
@@ -144,7 +168,8 @@ class store():
     nobj : int
         number of objectives
     capacity : int, optional
-        capacity of the result store.
+        capacity of the store collecting all solutions. If full, its content is replaced by its
+        pareto front. 
     """
     
     def __init__(self, dim, nobj, capacity = mp.cpu_count()*512):    
@@ -152,58 +177,83 @@ class store():
         self.nobj = nobj
         self.capacity = capacity
         self.add_mutex = mp.Lock()    
-        self.xs = mp.RawArray(ct.c_double, self.capacity * self.dim)
-        self.ys = mp.RawArray(ct.c_double, self.capacity * self.nobj)  
+        self.xs = Shared2d(np.empty((self.capacity, self.dim), dtype = np.float64))
+        self.ys = Shared2d(np.empty((self.capacity, self.nobj), dtype = np.float64)) 
+        self.create_views()
         self.num_stored = mp.RawValue(ct.c_int, 0) 
         self.num_added = mp.RawValue(ct.c_int, 0) 
 
+    def create_views(self): # needs to be called in the target process
+        self.xs_view = self.xs.view()
+        self.ys_view = self.ys.view()
+
+    def get_xs(self) -> np.ndarray:
+        return self.xs.view()
+
+    def get_ys(self) -> np.ndarray:
+        return self.ys.view()
+
+    def add_result(self, x, y):
+        with self.add_mutex:
+            self.num_added.value += 1
+            i = self.num_stored.value
+            if i < self.capacity:                
+                self.xs_view[i] = x
+                self.ys_view[i] = y
+            self.num_stored.value = i + 1
+
     def add_results(self, xs, ys):
         with self.add_mutex:
             self.num_added.value += 1
             i = self.num_stored.value
             for j in range(len(xs)):
-                if i < self.capacity:                      
-                    self.set_x(i, xs[j]) 
-                    self.set_y(i, ys[j])
+                if i < self.capacity:                
+                    self.xs_view[i] = xs[j] 
+                    self.ys_view[i] = ys[j][:self.nobj]
                     i += 1
-                else: # store is full, replace with pareto front
-                    xf, yf = self.get_front()
-                    i = 0
-                    for k in range(len(xf)):                   
-                        self.set_x(i, xf[k]) 
-                        self.set_y(i, yf[k])
-                        i += 1
-                    if i == self.capacity:
-                        break
+                else:
+                    self.get_front(update=True)
+                    i = self.num_stored.value
+                    if i > 0.9*self.capacity: # give up
+                        return
             self.num_stored.value = i
                       
-    def get_front(self):
-        return moretry.pareto(self.get_xs(), self.get_ys())
-       
-    def get_xs(self):
-        return np.array([self.get_x(i) for i in range(self.num_stored.value)])
-
-    def get_ys(self):
-        return np.array([self.get_y(i) for i in range(self.num_stored.value)])
-    
-    def get_x(self, i):
-        return self.xs[i*self.dim:(i+1)*self.dim]
- 
-    def set_x(self, i, x):
-        self.xs[i*self.dim:(i+1)*self.dim] = x[:]          
-     
-    def get_y(self, i):
-        return self.ys[i*self.nobj:(i+1)*self.nobj]
- 
-    def set_y(self, i, y):
-        self.ys[i*self.nobj:(i+1)*self.nobj] = y[:]          
+    def get_front(self, update=False):
+        stored = self.num_stored.value
+        xs = self.xs_view[:stored]
+        ys = self.ys_view[:stored]
+        xf, yf = moretry.pareto(xs, ys)
+        if update:
+            n = len(yf)
+            self.xs_view[:n] = xf 
+            self.ys_view[:n] = yf                                
+            self.num_stored.value = n
+        return xf, yf
 
+    def get_content(self):
+        stored = self.num_stored.value
+        return self.xs_view[:stored], self.ys_view[:stored]
 
 class MODE(object):
     
-    def __init__(self, nobj, ncon, bounds, popsize = 64, workers = 0,
-                 F = 0.5, Cr = 0.9, nsga_update = False, pareto_update = False, 
-                 rg = Generator(MT19937()), logger = None, plot_name = None):
+    def __init__(self, 
+                nobj: int,
+                ncon: int, 
+                bounds: Bounds,
+                popsize: Optional[int] = 64, 
+                F: Optional[float] = 0.5, 
+                Cr: Optional[float] = 0.9, 
+                pro_c: Optional[float] = 0.5,
+                dis_c: Optional[float] = 15.0,
+                pro_m: Optional[float] = 0.9,
+                dis_m: Optional[float] = 20.0,
+                nsga_update: Optional[bool] = True,
+                pareto_update: Optional[int] = 0,
+                rg: Optional[Generator] = Generator(PCG64DXSM()),
+                ints: Optional[ArrayLike] = None,
+                min_mutate: Optional[float] = 0.1,
+                max_mutate: Optional[float] = 0.5,   
+                modifier: Callable = None):
         self.nobj = nobj
         self.ncon = ncon
         self.dim, self.lower, self.upper = _check_bounds(bounds, None)
@@ -212,10 +262,13 @@ class MODE(object):
         if popsize % 2 == 1 and nsga_update: # nsga update requires even popsize
             popsize += 1
         self.popsize = popsize
-        self.workers = workers 
         self.rg = rg
         self.F0 = F
         self.Cr0 = Cr
+        self.pro_c = pro_c
+        self.dis_c = dis_c
+        self.pro_m = pro_m
+        self.dis_m = dis_m    
         self.nsga_update = nsga_update
         self.pareto_update = pareto_update
         self.stop = 0
@@ -223,154 +276,103 @@ class MODE(object):
         self.evals = 0
         self.mutex = mp.Lock()
         self.p = 0
-        self.best_p = None
+        # nsga update doesn't support mixed integer
+        self.ints = None if (ints is None or nsga_update) else np.array(ints)
+        self.min_mutate = min_mutate
+        self.max_mutate = max_mutate        
+        # use default variable modifier for int variables if modifier is None
+        if modifier is None and not ints is None:
+            self.lower = self.lower.astype(float)
+            self.upper = self.upper.astype(float)
+            self.modifier = self._modifier
+        else:
+            self.modifier = modifier
         self._init()
-        if not logger is None:
-            self.logger = logger
-            self.n_evals = mp.RawValue(ct.c_long, 0)
-            self.time_0 = time.perf_counter()
-            if not plot_name is None:
-                self.plot_name = plot_name + '_mode_' + str(popsize) + '_' + \
-                    ('nsga_update' if nsga_update else ('de_update_' + str(pareto_update)))
-        
-    def ask(self):
-        """ask for one new argument vector.
-        
-        Returns
-        -------
-        p : int population index 
-        x : dim sized argument ."""
-        
-        p = self.p
-        x = self._next_x(p)
-        self.p = (self.p + 1) % self.popsize
-        return p, x
-
-    def tell(self, p, y, x):      
-        """tell function value for a argument list retrieved by ask_one().
-    
-        Parameters
-        ----------
-        p : int population index 
-        y : function value
-        x : dim sized argument list
- 
-        Returns
-        -------
-        stop : int termination criteria, if != 0 loop should stop."""
-        
-        with self.mutex:  
-            for dp in range(len(self.done)):
-                if not self.done[dp]:
-                    break
-            self.nx[dp] = x
-            self.ny[dp] = y      
-            self.done[dp] = True
-            if sum(self.done) >= self.popsize:
-                done_p = np.arange(len(self.ny))
-                done_p = done_p[self.done]
-                p = self.popsize
-                for dp in done_p:
-                    self.x[p] = self.nx[dp]
-                    self.y[p] = self.ny[dp]
-                    self.done[dp] = False
-                    if p >= len(self.y):
-                        break
-                    p += 1
-                self.pop_update()
-        
-        if hasattr(self, 'logger'):
-            self.n_evals.value += 1
-            if self.n_evals.value % 100000 == 99999:           
-                t = time.perf_counter() - self.time_0
-                c = self.n_evals.value
-                message = '"c/t={0:.2f} c={1:d} t={2:.2f} y={3!s} x={4!s}'.format(
-                    c/t, c, t, str(list(y)), str(list(x)))
-                self.logger.info(message)   
-                if hasattr(self, "plot_name") and not self.plot_name is None:           
-                    name = self.plot_name + ' '  + str(self.n_evals.value)
-                    np.savez_compressed(name, xs=self.x, ys=self.y) 
-                    moretry.plot(name, self.ncon, self.x, self.y)
-
-        return self.stop 
-
+               
+    def set_guess(self, guess, mofun, rg = None):
+        if not guess is None:
+            if isinstance(guess, np.ndarray):
+                ys = np.array([mofun(x) for x in guess])
+            else:
+                guess, ys = guess
+            if rg is None:
+                rg = Generator(PCG64DXSM())
+            choice = rg.choice(len(ys), self.popsize, 
+                                    replace = (len(ys) < self.popsize))
+            self.tell(ys[choice], guess[choice])
+                   
+    def ask(self) -> np.ndarray:
+        for p in range(self.popsize):
+            self.x[p + self.popsize] = self._next_x(p)
+        return self.x[self.popsize:]
+                
+    def tell(self, ys: np.ndarray, xs: Optional[np.ndarray] = None):
+        if not xs is None:
+            for p in range(self.popsize):
+                self.x[p + self.popsize] = xs[p]
+        for p in range(self.popsize):
+            self.y[p + self.popsize] = ys[p]
+        self.pop_update()
+                         
     def _init(self):
         self.x = np.empty((2*self.popsize, self.dim))
         self.y = np.empty((2*self.popsize, self.nobj + self.ncon))
         for i in range(self.popsize):
             self.x[i] = self._sample()
             self.y[i] = np.array([1E99]*(self.nobj + self.ncon))
-        
-        next_size = 2*(max(self.workers, self.popsize))
-        self.done = np.zeros(next_size, dtype=bool)
-        self.nx = np.empty((next_size, self.dim))
-        self.ny = np.empty((next_size, self.nobj + self.ncon))
         self.vx = self.x.copy()
         self.vp = 0
-                                    
-    def do_optimize(self, fun, max_evals):
-        self.fun = fun
-        self.max_evals = max_evals    
-        self.iterations = 0
-        self.evals = 0
-        while self.evals < self.max_evals:
-            for p in range(self.popsize):
-                x = self._next_x(p)
-                self.y[self.popsize + p] = self.fun(x)
-                self.x[self.popsize + p] = x
-                self.evals += 1
-            self.pop_update()
-        x, y = filter(self.x, self.y)
-        return x, y, self.evals, self.iterations, self.stop
+        self.ycon = None
+        self.eps = 0
 
-    def do_optimize_delayed_update(self, fun, max_evals, workers=mp.cpu_count()):
-        self.fun = fun
-        self.max_evals = max_evals    
-        evaluator = Evaluator(self.fun)
-        evaluator.start(workers)
-        evals_x = {}
-        self.iterations = 0
-        self.evals = 0
-        self.p = 0
-        for _ in range(workers): # fill queue with initial population
-            p, x = self.ask()
-            evaluator.pipe[0].send((self.evals, x))
-            evals_x[self.evals] = p, x # store x
-            self.evals += 1
-            
-        while True: # read from pipe, tell de and create new x
-            evals, y = evaluator.pipe[0].recv()            
-            p, x = evals_x[evals] # retrieve evaluated x
-            del evals_x[evals]
-            self.tell(p, y, x) # tell evaluated x
-            if self.stop != 0 or self.evals >= self.max_evals:
-                break # shutdown worker if stop criteria met
-            
-            p, x = self.ask() # create new x          
-            evaluator.pipe[0].send((self.evals, x))       
-            evals_x[self.evals] = p, x  # store x
-            self.evals += 1
-            
-        evaluator.stop()
-        x, y = filter(self.x, self.y)
-        return x, y, self.evals, self.iterations, self.stop
+    def minimize_ser(self, 
+                     fun: Callable[[ArrayLike], ArrayLike], 
+                     max_evaluations: Optional[int] = 100000) -> Tuple[np.ndarray, np.ndarray]:
+        evals = 0
+        while evals < max_evaluations:
+            xs = self.ask()
+            ys = np.array([fun(x) for x in xs])
+            self.tell(ys)
+            evals += self.popsize
+        return xs, ys
 
+        
+    def minimize_par(self, 
+                     fun: Callable[[ArrayLike], ArrayLike], 
+                     max_evaluations: Optional[int] = 100000, 
+                     workers: Optional[int] = mp.cpu_count()) -> Tuple[np.ndarray, np.ndarray]:
+        fit = parallel_mo(fun, self.nobj + self.ncon, workers)
+        evals = 0
+        while evals < max_evaluations:
+            xs = self.ask()
+            ys = fit(xs)
+            self.tell(ys)
+            evals += self.popsize
+        fit.stop()
+        return xs, ys
+                                    
     def pop_update(self):
-        domination = pareto(self.y, self.nobj, self.ncon)
+        y0 = self.y
+        x0 = self.x
+        if self.nobj == 1:
+            yi = np.flip(np.argsort(self.y[:,0]))
+            y0 = self.y[yi]
+            x0 = self.x[yi]    
+        domination, self.ycon, self.eps = pareto_domination(y0, self.nobj, self.ncon, self.ycon, self.eps)
         x = []
         y = []
         maxdom = int(max(domination))
         for dom in range(maxdom, -1, -1):
             domlevel = [p for p in range(len(domination)) if domination[p] == dom]
-            if dom == maxdom: # store pareto front in self.best_p
-                self.best_p =  domlevel
+            if len(domlevel) == 0:
+                continue
             if len(x) + len(domlevel) <= self.popsize:
                 # whole level fits
-                x = [*x, *self.x[domlevel]]
-                y = [*y, *self.y[domlevel]]
+                x = [*x, *x0[domlevel]]
+                y = [*y, *y0[domlevel]]
             else: # sort for crowding
-                nx = self.x[domlevel]
-                ny = self.y[domlevel]    
+                nx = x0[domlevel]
+                ny = y0[domlevel]    
                 si = [0]
                 if len(ny) > 1:                
                     cd = crowd_dist(ny)
@@ -384,7 +386,8 @@ class MODE(object):
         self.x[:self.popsize] = x[:self.popsize]
         self.y[:self.popsize] = y[:self.popsize]
         if self.nsga_update:
-            self.vx = variation(self.x[:self.popsize], self.lower, self.upper, self.rg) 
+            self.vx = variation(self.x[:self.popsize], self.lower, self.upper, self.rg, 
+                pro_c = self.pro_c, dis_c = self.dis_c, pro_m = self.pro_m, dis_m = self.dis_m) 
        
     def _next_x(self, p):
         if self.nsga_update: # use NSGA-II update strategy.
@@ -397,17 +400,15 @@ class MODE(object):
             self.Cr = 0.5*self.Cr0 if self.iterations % 2 == 0 else self.Cr0
             self.F = 0.5*self.F0 if self.iterations % 2 == 0 else self.F0
         while True:
-            if self.rg.random() < self.pareto_update and (not self.best_p is None) and len(self.best_p) > 3: 
-                # sample from pareto front
-                rb = self.rg.integers(0, len(self.best_p))
-                rb = self.best_p[rb]
+            if self.pareto_update > 0: # sample elite solutions
                 r1, r2 = self.rg.integers(0, self.popsize, 2)
+                rb = int(self.popsize * (self.rg.random() ** (1.0 + self.pareto_update)))
             else:
                 # sample from whole population
                 r1, r2, rb = self.rg.integers(0, self.popsize, 3)
-                if r1 != p and r1 != rb and r1 != r2 and r2 != rb \
-                        and r2 != p and rb != p:
-                    break
+            if r1 != p and r1 != rb and r1 != r2 and r2 != rb \
+                and r2 != p and rb != p:
+                break
         xp = self.x[p]
         xb = self.x[rb]
         x1 = self.x[r1]
@@ -416,8 +417,10 @@ class MODE(object):
         r = self.rg.integers(0, self.dim)
         tr = np.array(
             [i != r and self.rg.random() > self.Cr for i in range(self.dim)])    
-        x[tr] = xp[tr]       
-        return x
+        x[tr] = xp[tr]    
+        if not self.modifier is None:
+            x = self.modifier(x)   
+        return x.clip(self.lower, self.upper)
     
     def _sample(self):
         if self.upper is None:
@@ -429,7 +432,24 @@ class MODE(object):
         if self.upper is None:
             return x
         else:
-            return np.maximum(np.minimum(x, self.upper), self.lower)
+            return np.clip(x, self.lower, self.upper)
+        
+    # default modifier for integer variables
+    def _modifier(self, x):
+        x_ints = x[self.ints]
+        n_ints = len(self.ints)
+        lb = self.lower[self.ints]
+        ub = self.upper[self.ints]
+        to_mutate = self.rg.uniform(self.min_mutate, self.max_mutate)
+        # mututate some integer variables
+        x_ints = np.array([x if self.rg.random() > to_mutate/n_ints else 
+                           int(self.rg.uniform(lb[i], ub[i]))
+                           for i, x in enumerate(x_ints)])
+        return x   
+    
+    def _is_dominated(self, y, p):
+        return np.all(np.fromiter((y[i] >= self.y[p, i] for i in range(len(y))), dtype=bool))
+
                     
 def _check_bounds(bounds, dim):
     if bounds is None and dim is None:
@@ -438,8 +458,8 @@ def _check_bounds(bounds, dim):
         return dim, None, None
     else:
         return len(bounds.ub), np.asarray(bounds.lb), np.asarray(bounds.ub)
- 
-def filter(x, y):
+
+def _filter(x, y):
     ym = np.amax(y,axis=1)
     sorted = np.argsort(ym)
     x = x[sorted]
@@ -457,47 +477,79 @@ def objranks(objs):
     rank = np.sum(rank, axis=1)
     return rank
 
-def ranks(cons):
-    feasible = np.less_equal(cons, 0)
+def ranks(cons, feasible, eps):
     ci = cons.argsort(axis=0)
     rank = np.empty_like(ci)
     ar = np.arange(cons.shape[0])
     for i in range(cons.shape[1]): 
         rank[ci[:,i], i] = ar
     rank[feasible] = 0
-    alpha = np.sum(np.greater(cons, 0), axis=1) / cons.shape[1] # violations
+    alpha = np.sum(np.greater(cons, eps), axis=1) / cons.shape[1] # violations
     alpha = np.tile(alpha, (cons.shape[1],1)).T
     rank = rank*alpha
     rank = np.sum(rank, axis=1)
     return rank
-     
-def pareto(ys, nobj, ncon):
+
+def get_valid(xs, ys, nobj):
+    valid = (ys.T[nobj:].T <= 0).all(axis=1)
+    return xs[valid], ys[valid]
+
+def pareto_sort(x0, y0, nobj, ncon):
+    domination, _, _ = pareto_domination(y0, nobj, ncon)
+    x = []
+    y = []
+    maxdom = int(max(domination))
+    for dom in range(maxdom, -1, -1):
+        domlevel = [p for p in range(len(domination)) if domination[p] == dom]
+        if len(domlevel) == 0:
+            continue
+        nx = x0[domlevel]
+        ny = y0[domlevel]    
+        si = [0]
+        if len(ny) > 1:                
+            cd = crowd_dist(ny)
+            si = np.flip(np.argsort(cd))
+        for p in si:
+            x.append(nx[p])
+            y.append(ny[p])                             
+    return np.array(x), np.array(y)
+
+def pareto_domination(ys, nobj, ncon, last_ycon = None, last_eps = 0):
     if ncon == 0:
-        return pareto_levels(ys)
+        return pareto_levels(ys), None, 0
     else:
+        eps = 0 # adjust tolerance to small constraint violations
+        if not last_ycon is None and np.amax(last_ycon) < 1E90:
+            eps = 0.5*(last_eps + 0.5*np.mean(last_ycon, axis=0))
+            if np.amax(eps) < 1E-8: # ignore small eps
+                eps = 0
+        
         yobj = np.array([y[:nobj] for y in ys])
-        ycon = np.array([np.maximum(y[-ncon:], 0) for y in ys])    
-        csum = ranks(ycon)
-        feasible = np.less_equal(csum, 0)
+        ycon = np.array([np.maximum(y[-ncon:], 0) for y in ys])  
+        popn = len(ys)              
+        feasible = np.less_equal(ycon, eps).all(axis=1)
+        
+        csum = ranks(ycon, feasible, eps)
         if sum(feasible) > 0:
             csum += objranks(yobj)
+        
         ci = np.argsort(csum)
-        popn = len(ys)
         domination = np.zeros(popn)
         # first pareto front of feasible solutions
-        cy = np.array([i for i in ci if feasible[i]])
+        cy = np.fromiter((i for i in ci if feasible[i]), dtype=int)
         if len(cy) > 0:
             ypar = pareto_levels(yobj[cy])
-            domination[cy] += ypar        
+            domination[cy] = ypar        
+
         # then constraint violations   
-        ci = np.array([i for i in ci if not feasible[i]])  
+        ci = np.fromiter((i for i in ci if not feasible[i]), dtype=int) 
         if len(ci) > 0:    
-            maxcdom = len(ci)
-            cdom = np.arange(maxcdom, 0, -1)
+            cdom = np.arange(len(ci), 0, -1)
             domination[ci] += cdom
             if len(cy) > 0: # priorize feasible solutions
-                domination[cy] += maxcdom + 1
-    return domination   
+                domination[cy] += len(ci) + 1
+                
+        return domination, ycon, eps
  
 def pareto_levels(ys):
     popn = len(ys)
@@ -515,7 +567,7 @@ def pareto_levels(ys):
 
 def crowd_dist(y): # crowd distance for 1st objective
     n = len(y)
-    y0 = np.array([yi[0] for yi in y])
+    y0 = np.fromiter((yi[0] for yi in y), dtype=float)
     si = np.argsort(y0) # sort 1st objective
     y0_s = y0[si] # sorted
     d = y0_s[1:n] - y0_s[0:n-1] # neighbor distance
@@ -533,6 +585,8 @@ def crowd_dist(y): # crowd distance for 1st objective
 # derived from https://github.com/ChengHust/NSGA-II/blob/master/GLOBAL.py
 def variation(pop, lower, upper, rg, pro_c = 1, dis_c = 20, pro_m = 1, dis_m = 20):
     """Generate offspring individuals"""   
+    dis_c *= 0.5 + 0.5*rg.random() # vary spread factors randomly 
+    dis_m *= 0.5 + 0.5*rg.random() 
     pop = pop[:(len(pop) // 2) * 2][:]
     (n, d) = np.shape(pop)
     parent_1 = pop[:n // 2, :]
@@ -545,8 +599,9 @@ def variation(pop, lower, upper, rg, pro_c = 1, dis_c = 20, pro_m = 1, dis_m = 2
     beta[rg.random((n // 2, d)) < 0.5] = 1
     if pro_c < 1.0:
         beta[np.tile(rg.random((n // 2, 1)) > pro_c, (1, d))] = 1
-    offspring = np.vstack(((parent_1 + parent_2) / 2 + beta * (parent_1 - parent_2) / 2,
-                               (parent_1 + parent_2) / 2 - beta * (parent_1 - parent_2) / 2))
+    parent_mean = (parent_1 + parent_2) * 0.5
+    parent_diff = (parent_1 - parent_2) * 0.5
+    offspring = np.vstack((parent_mean + beta * parent_diff, parent_mean - beta * parent_diff))
     site = rg.random((n, d)) < pro_m / d
     mu = rg.random((n, d))
     temp = site & (mu <= 0.5)
@@ -561,16 +616,104 @@ def variation(pop, lower, upper, rg, pro_c = 1, dis_c = 20, pro_m = 1, dis_m = 2
                            (1. - np.power(
                                2. * (1. - mu[temp]) + 2. * (mu[temp] - 0.5) * np.power(1. - norm, dis_m + 1.),
                                1. / (dis_m + 1.)))
-    offspring = np.maximum(np.minimum(offspring, upper), lower)
+    offspring = np.clip(offspring, lower, upper)
     return offspring
 
-def minimize_plot(name, fun, nobj, ncon, bounds, popsize = 64, max_evaluations = 100000, nsga_update=False, 
-                  pareto_update=0, workers = mp.cpu_count(), logger=logger(), plot_name = None):
+def feasible(xs, ys, ncon, eps = 1E-2):
+    if ncon > 0: # select feasible
+        ycon = np.array([np.maximum(y[-ncon:], 0) for y in ys])  
+        con = np.sum(ycon, axis=1)
+        nobj = len(ys[0]) - ncon
+        feasible = np.fromiter((i for i in range(len(ys)) if con[i] < eps), dtype=int)
+        if len(feasible) > 0:
+            xs, ys = xs[feasible], np.array([y[:nobj] for y in ys[feasible]])
+        else:
+            print("no feasible")
+    return xs, ys
+
+def is_feasible(y, nobj, eps = 1E-2):
+    ncon = len(y) - nobj
+    if ncon == 0:
+        return True
+    else:
+        c = np.sum(np.maximum(y[-ncon:], 0))
+        return c < eps
+
+class wrapper(object):
+    """thread safe wrapper for objective function monitoring evaluation count and optimization result."""
+   
+    def __init__(self, 
+                 fun: Callable[[ArrayLike], ArrayLike], 
+                 nobj: int, 
+                 store: Optional[store] = None, 
+                 interval: Optional[int] = 100000, 
+                 plot: Optional[bool] = False, 
+                 name: Optional[str] = None):
+        self.fun = fun
+        self.nobj = nobj
+        self.n_evals = mp.RawValue(ct.c_long, 0)
+        self.time_0 = time.perf_counter()
+        self.best_y = mp.RawArray(ct.c_double, nobj)  
+        for i in range(nobj):
+            self.best_y[i] = sys.float_info.max
+        self.store = store
+        self.interval = interval
+        self.plot = plot
+        self.name = name
+        self.lock = mp.Lock()
+    
+    def __call__(self, x: ArrayLike) -> np.ndarray:
+        try:
+            y = self.fun(x)
+            with self.lock:
+                self.n_evals.value += 1
+            if not self.store is None and is_feasible(y, self.nobj):
+                self.store.create_views()
+                self.store.add_result(x, y[:self.nobj])
+            improve = False
+            for i in range(self.nobj):
+                if y[i] < self.best_y[i]:
+                    improve = True 
+                    self.best_y[i] = y[i] 
+            improve = improve# and self.n_evals.value > 10000
+            if self.n_evals.value % self.interval == 0 or improve:
+                constr = np.maximum(y[self.nobj:], 0) 
+                logger.info(
+                    str(dtime(self.time_0)) + ' ' + 
+                    str(self.n_evals.value) + ' ' + 
+                    str(round(self.n_evals.value/(1E-9 + dtime(self.time_0)),0)) + ' ' + 
+                    str(self.best_y[:]) + ' ' + str(list(constr)) + ' ' + str(list(x))) 
+                if (not self.store is None) and (not self.name is None):
+                    try:
+                        xs, ys = self.store.get_front()
+                        num = self.store.num_stored.value
+                        name = self.name + '_' + str(num)
+                        np.savez_compressed(name, xs=xs, ys=ys)
+                        if self.plot:
+                            moretry.plot(name, 0, xs, ys, all=False)
+                    except Exception as ex:
+                        print(str(ex))                                                
+            return y
+        except Exception as ex:
+            print(str(ex))  
+            return None  
+ 
+def minimize_plot(name: str, 
+                 fun: Callable[[ArrayLike], ArrayLike], 
+                 nobj: int,
+                 ncon: int, 
+                 bounds: Bounds,
+                 popsize: Optional[int] = 64, 
+                 max_evaluations: Optional[int] = 100000, 
+                 nsga_update: Optional[bool] = True,
+                 pareto_update: Optional[int] = 0,
+                 ints: Optional[ArrayLike] = None,
+                 workers: Optional[int] = mp.cpu_count()) -> Tuple[np.ndarray, np.ndarray]:     
     name += '_mode_' + str(popsize) + '_' + \
                 ('nsga_update' if nsga_update else ('de_update_' + str(pareto_update)))
     logger.info('optimize ' + name) 
     xs, ys = minimize(fun, nobj, ncon, bounds, popsize = popsize, max_evaluations = max_evaluations,
-                   nsga_update = nsga_update, pareto_update = pareto_update, workers=workers, 
-                   logger=logger, plot_name = plot_name)
+                   nsga_update = nsga_update, pareto_update = pareto_update, workers=workers, ints=ints)
     np.savez_compressed(name, xs=xs, ys=ys)
     moretry.plot(name, ncon, xs, ys)
+    return xs, ys