python-misc-utils 0.2__py3-none-any.whl

py_misc_utils/select_params.py

@@ -0,0 +1,300 @@
+import collections
+import functools
+import multiprocessing as mp
+import multiprocessing.pool as mp_pool
+import os
+import pickle
+import re
+import subprocess
+
+import numpy as np
+
+from . import alog
+from . import gfs
+from . import np_utils as npu
+from . import utils as ut
+
+
+Point = collections.namedtuple('Point', 'pid, idx')
+
+
+_SCORES_X_RUN = ut.getenv('SCORES_X_RUN', dtype=int, defval=10)
+_KGEN_EXTRA = ut.getenv('KGEN_EXTRA', dtype=float, defval=2)
+
+
+def _norm_params(params):
+  nparams = dict()
+  for k, v in params.items():
+    if not isinstance(v, np.ndarray):
+      v = np.array(v)
+    nparams[k] = np.sort(v)
+
+  return nparams
+
+
+def _get_space(params):
+  skeys = sorted(params.keys())
+
+  return skeys, [len(params[k]) for k in skeys]
+
+
+def _mkdelta(idx, space, delta_std):
+  # Sample around the index.
+  rng = np.random.default_rng()
+
+  aspace = np.array(space)
+  delta = np.array(idx) + rng.standard_normal(len(idx)) * aspace * delta_std
+  delta = np.rint(delta).astype(np.int32)
+
+  return np.clip(delta, np.zeros_like(delta), aspace - 1)
+
+
+def _mp_score_fn(score_fn, params):
+  return score_fn(**params)
+
+
+def _is_worth_gain(pscore, score, min_gain_pct):
+  pabs = np.abs(pscore)
+  if np.isclose(pabs, 0):
+    return score > pscore
+
+  delta = (score - pscore) / pabs
+
+  return delta >= min_gain_pct
+
+
+class Selector:
+
+  def __init__(self, params, seeds=None):
+    self.nparams = _norm_params(params)
+    self.processed = set()
+    self.scores_db = collections.defaultdict(list)
+    self.best_score, self.best_idx, self.best_param = None, None, None
+    self.pid_scores = dict()
+    self.blanks = 0
+    self.skeys, self.space = _get_space(self.nparams)
+    self.pts, self.cpid = [], 0
+    self.current_scores, self.processed_scores = [], 0
+    if seeds:
+      self._load_seeds(seeds)
+
+  def _load_seeds(self, seeds):
+    for sd in seeds:
+      idx = np.zeros(len(self.space), dtype=np.int32)
+      for i, k in enumerate(self.skeys):
+        v = sd[k]
+        idx[i] = np.argmin(np.abs(self.nparams[k] - v))
+
+      if not self._is_processed(idx):
+        self.pts.append(Point(self.cpid, idx))
+        self.cpid += 1
+      else:
+        alog.info(f'Seed already processed: {sd}')
+
+  def _register_scores(self, xparams, scores):
+    for params, score in zip(xparams, scores):
+      alog.debug0(f'SCORE: {score} -- {params}')
+
+      for k, v in params.items():
+        self.scores_db[k].append(v)
+      self.scores_db['SCORE'].append(score)
+
+  def _make_param(self, idx):
+    param = dict()
+    for i, k in enumerate(self.skeys):
+      # We keep parameters as numpy arrays, but when we pluck values we want to
+      # return them in Python scalar form.
+      pvalue = self.nparams[k][idx[i]]
+      param[k] = pvalue.item()
+
+    return param
+
+  def _score_slice(self, pts, score_fn, n_jobs=None, mp_ctx=None):
+    xparams = [self._make_param(pt.idx) for pt in pts]
+
+    n_jobs = os.cpu_count() if n_jobs is None else n_jobs
+    if n_jobs == 1:
+      scores = [score_fn(**p) for p in xparams]
+    else:
+      context = mp.get_context(mp_ctx if mp_ctx is not None else mp.get_start_method())
+      fn = functools.partial(_mp_score_fn, score_fn)
+      with mp_pool.Pool(processes=n_jobs if n_jobs > 0 else None, context=context) as pool:
+        scores = list(pool.map(fn, xparams))
+
+    self._register_scores(xparams, scores)
+
+    return scores
+
+  def _fetch_scores(self, score_fn, n_jobs=None, mp_ctx=None, scores_x_run=None,
+                    status_path=None):
+    scores_x_run = scores_x_run or _SCORES_X_RUN
+
+    for i in range(self.processed_scores, len(self.pts), scores_x_run):
+      current_points = self.pts[i: i + scores_x_run]
+
+      scores = self._score_slice(current_points, score_fn, n_jobs=n_jobs, mp_ctx=mp_ctx)
+
+      self.current_scores.extend(scores)
+      self.processed_scores += len(current_points)
+
+      alog.info(f'Processed {self.processed_scores}/{len(self.pts)}: ' \
+                f'{ut.format(sorted(self.current_scores, reverse=True), ".6e")}')
+
+      if status_path is not None:
+        self.save_status(status_path)
+
+  def _select_top_n(self, top_n, min_pid_gain_pct):
+    # The np.argsort() has no "reverse" option, so it's either np.flip() or negate
+    # the scores.
+    sidx = np.flip(np.argsort(self.current_scores))
+
+    pseen, fsidx = set(), []
+    for i in sidx:
+      pt = self.pts[i]
+      if pt.pid not in pseen:
+        pseen.add(pt.pid)
+        pscore = self.pid_scores.get(pt.pid)
+        if pscore is None or _is_worth_gain(pscore, self.current_scores[i], min_pid_gain_pct):
+          self.pid_scores[pt.pid] = self.current_scores[i]
+          fsidx.append(i)
+          # The sidx array contains indices mapping to a descending sort of the
+          # scores, so once we have top_n of them, we know we have selected the
+          # higher ones available.
+          if len(fsidx) >= top_n:
+            break
+
+    return fsidx
+
+  def _is_processed(self, idx):
+    return idx.tobytes() in self.processed
+
+  def _generating(self, dest, count):
+    max_attempts = round(count * _KGEN_EXTRA)
+    n = 0
+    while count > len(dest) and n < max_attempts:
+      yield n
+      n += 1
+
+  def _randgen(self, count):
+    rng = np.random.default_rng()
+    high = np.array(self.space, dtype=np.int32)
+    low = np.zeros_like(high)
+
+    rpoints = []
+    for _ in self._generating(rpoints, count):
+      idx = rng.integers(low, high)
+      if not self._is_processed(idx):
+        rpoints.append(Point(self.cpid, idx))
+        self.cpid += 1
+
+    return rpoints
+
+  def _select_deltas(self, pt, delta_spacek, delta_std):
+    if delta_spacek is None:
+      num_deltas = len(self.space)
+    elif delta_spacek > 0:
+      num_deltas = int(np.ceil(len(self.space) * delta_spacek))
+    else:
+      num_deltas = int(np.rint(-delta_spacek))
+
+    deltas = []
+    for _ in self._generating(deltas, num_deltas):
+      idx = _mkdelta(pt.idx, self.space, delta_std)
+      if not self._is_processed(idx):
+        deltas.append(Point(pt.pid, idx))
+
+    return deltas
+
+  def __call__(self, score_fn,
+               status_path=None,
+               delta_spacek=None,
+               delta_std=0.1,
+               top_n=10,
+               explore_pct=0.05,
+               rnd_pct=0.2,
+               min_pid_gain_pct=0.01,
+               max_blanks_pct=0.1,
+               scores_x_run=None,
+               n_jobs=None,
+               mp_ctx=None):
+    alog.debug0(f'{len(self.space)} parameters, {np.prod(self.space)} configurations')
+
+    if not self.pts:
+      self.pts.extend(self._randgen(top_n))
+
+    max_explore = int(np.prod(self.space) * explore_pct)
+    max_blanks = int(max_explore * max_blanks_pct)
+    while self.pts and len(self.processed) < max_explore and self.blanks < max_blanks:
+      alog.debug0(f'{len(self.pts)} points, {len(self.processed)} processed ' \
+                  f'(max {max_explore}), {self.blanks}/{max_blanks} blanks')
+
+      self._fetch_scores(score_fn,
+                         n_jobs=n_jobs,
+                         mp_ctx=mp_ctx,
+                         scores_x_run=scores_x_run,
+                         status_path=status_path)
+
+      fsidx = self._select_top_n(top_n, min_pid_gain_pct)
+
+      score = self.current_scores[fsidx[0]]
+      if self.best_score is None or score > self.best_score:
+        self.best_score = score
+        self.best_idx = self.pts[fsidx[0]].idx
+        self.best_param = self._make_param(self.best_idx)
+        self.blanks = 0
+
+        alog.debug0(f'BestScore = {self.best_score:.5e}\tParam = {self.best_param}')
+      else:
+        self.blanks += len(self.pts)
+        alog.info(f'Score not improved (current run top {score}, best {self.best_score})')
+
+      for pt in self.pts:
+        self.processed.add(pt.idx.tobytes())
+
+      # Sample around best points ...
+      next_pts = []
+      for i in fsidx:
+        next_pts.extend(self._select_deltas(self.pts[i], delta_spacek, delta_std))
+
+      # And randomly add ones in search of better scores.
+      rnd_count = max(top_n, int(rnd_pct * len(next_pts)))
+      next_pts.extend(self._randgen(rnd_count))
+
+      self.pts = next_pts
+      self.current_scores, self.processed_scores = [], 0
+
+  def save_status(self, path):
+    with gfs.open(path, mode='wb') as sfd:
+      pickle.dump(self, sfd, protocol=ut.pickle_proto())
+
+  @staticmethod
+  def load_status(path):
+    with gfs.open(path, mode='rb') as sfd:
+      return pickle.load(sfd)
+
+
+_SCORE_TAG = 'SPSCORE'
+_SCORE_FMT = os.getenv('SPSCORE_FMT', 'f')
+
+def format_score(s):
+  return f'[{_SCORE_TAG}={s:{_SCORE_FMT}}]'
+
+
+def match_score(data):
+  matches = re.findall(f'\[{_SCORE_TAG}=' + r'([^]]+)\]', data)
+
+  return [float(m) for m in matches]
+
+
+def run_score_process(cmdline):
+  try:
+    output = subprocess.check_output(cmdline, stderr=subprocess.STDOUT)
+  except subprocess.CalledProcessError as ex:
+    alog.exception(ex, exmsg=f'Error while running scoring process: {ex.output.decode()}')
+    raise
+
+  if isinstance(output, bytes):
+    output = output.decode()
+
+  return match_score(output), output
+

py_misc_utils/signal.py

@@ -0,0 +1,141 @@
+import collections
+import signal as sgn
+import threading
+
+from . import core_utils as cu
+from . import global_namespace as gns
+from . import traceback as tb
+
+
+_Handler = collections.namedtuple('Handler', 'handler, prio')
+
+MAX_PRIO = 0
+MIN_PRIO = 99
+STD_PRIO = MIN_PRIO
+CALL_NEXT = 0
+HANDLED = 1
+
+
+class _SignalRegistry:
+
+  def __init__(self):
+    self.lock = threading.Lock()
+    self.handlers = dict()
+    self.prev_handlers = dict()
+
+  def signal(self, sig, handler, prio=None):
+    prio = STD_PRIO if prio is None else prio
+
+    with self.lock:
+      handlers = self.handlers.get(sig, ())
+      handlers += (_Handler(handler, prio),)
+      self.handlers[sig] = tuple(sorted(handlers, key=lambda h: h.prio))
+
+      if sig not in self.prev_handlers:
+        self.prev_handlers[sig] = sgn.signal(sig, _handler)
+
+  def unsignal(self, sig, uhandler):
+    handlers, dropped = [], 0
+    with self.lock:
+      for handler in self.handlers.get(sig, ()):
+        if handler.handler != uhandler:
+          handlers.append(handler)
+        else:
+          dropped += 1
+
+      if dropped:
+        self.handlers[sig] = tuple(handlers)
+        if not handlers:
+          sgn.signal(sig, self.prev_handlers.pop(sig))
+
+    return dropped
+
+  def sig_handler(self, sig, frame):
+    with self.lock:
+      handlers = self.handlers.get(sig, ())
+      prev_handler = self.prev_handlers.get(sig)
+
+    for handler in handlers:
+      hres = handler.handler(sig, frame)
+      if hres == HANDLED:
+        return
+
+    if callable(prev_handler):
+      prev_handler(sig, frame)
+    else:
+      handler = sgn.getsignal(sig)
+      if callable(handler):
+        handler(sig, frame)
+
+
+def _parent_fn(sreg):
+  return sreg.prev_handlers
+
+
+def _child_fn(prev_handlers):
+  for sig, prev_handler in prev_handlers.items():
+    sgn.signal(sig, prev_handler)
+
+  return _SignalRegistry()
+
+
+def _create_fn():
+  return _SignalRegistry()
+
+
+_SIGREG = gns.Var(f'{__name__}.SIGREG',
+                  parent_fn=_parent_fn,
+                  child_fn=_child_fn,
+                  defval=_create_fn)
+
+def _sig_registry():
+  return gns.get(_SIGREG)
+
+
+def _handler(sig, frame):
+  sreg = _sig_registry()
+
+  sreg.sig_handler(sig, frame)
+
+
+def trigger(sig, frame=None):
+  _handler(sig, frame or tb.get_frame(n=1))
+
+
+def signal(sig, handler, prio=None):
+  sreg = _sig_registry()
+
+  sreg.signal(sig, handler, prio=prio)
+
+
+def unsignal(sig, uhandler):
+  sreg = _sig_registry()
+
+  return sreg.unsignal(sig, uhandler)
+
+
+class Signals:
+
+  def __init__(self, sig, handler, prio=None):
+    if isinstance(sig, str):
+      sig = [getattr(sgn, f'SIG{s.upper()}') for s in cu.splitstrip(sig, ',')]
+    elif not isinstance(sig, (list, tuple)):
+      sig = [sig]
+    if not isinstance(handler, (list, tuple)):
+      handler = [handler] * len(sig)
+
+    self._sigs = tuple(zip(sig, handler))
+    self._prio = prio
+
+  def __enter__(self):
+    for sig, handler in self._sigs:
+      signal(sig, handler, prio=self._prio)
+
+    return self
+
+  def __exit__(self, *exc):
+    for sig, handler in self._sigs:
+      unsignal(sig, handler)
+
+    return True
+

py_misc_utils/skl_utils.py

@@ -0,0 +1,270 @@
+import os
+
+import numpy as np
+import pandas as pd
+import sklearn.decomposition
+import sklearn.neighbors
+import sklearn.preprocessing
+import sklearn.random_projection
+import sklearn.utils.extmath
+
+from . import alog
+from . import assert_checks as tas
+from . import utils as pyu
+
+
+class Quantizer:
+
+  def __init__(self, nbins=3, std_k=2.0, eps=1e-6):
+    self._nbins = nbins
+    self._std_k = std_k
+    self._eps = eps
+    self._mean = None
+    self._std = None
+
+  def fit(self, X, *args):
+    self._mean = np.mean(X, axis=0, keepdims=True)
+    std = np.std(X, axis=0, keepdims=True)
+    self._std = np.where(std > self._eps, std, self._eps)
+
+    return self
+
+  def transform(self, X):
+    q = np.round(self._nbins * (X - self._mean) / (self._std_k * self._std))
+
+    return np.clip(q, -self._nbins, self._nbins)
+
+  def fit_transform(self, X, *args):
+    return self.fit(X).transform(X)
+
+
+_TRANSFORMERS = {
+  'QTZ': Quantizer,
+  'STD_SCALE': sklearn.preprocessing.StandardScaler,
+  'MIN_MAX': sklearn.preprocessing.MinMaxScaler,
+  'MAX_ABS': sklearn.preprocessing.MaxAbsScaler,
+  'ROBUST': sklearn.preprocessing.RobustScaler,
+  'QUANT': sklearn.preprocessing.QuantileTransformer,
+  'POWER': sklearn.preprocessing.PowerTransformer,
+  'NORM': sklearn.preprocessing.Normalizer,
+  'PCA': sklearn.decomposition.PCA,
+  'FICA': sklearn.decomposition.FastICA,
+  'INCPCA': sklearn.decomposition.IncrementalPCA,
+  'KBINDIS': sklearn.preprocessing.KBinsDiscretizer,
+  'POLYF': sklearn.preprocessing.PolynomialFeatures,
+  'PWR': sklearn.preprocessing.PowerTransformer,
+  'SPLN': sklearn.preprocessing.SplineTransformer,
+  'GRPRJ': sklearn.random_projection.GaussianRandomProjection,
+  'SRPRJ': sklearn.random_projection.SparseRandomProjection,
+}
+
+def parse_transform(trs_spec):
+  trs, *spec = trs_spec.split(':', 1)
+  spec_cfg = pyu.parse_config(spec[0]) if spec else dict()
+
+  alog.debug0(f'Parsed Transformer: {trs}\t{spec_cfg}')
+
+  trs_fn = _TRANSFORMERS.get(trs)
+  tas.check_is_not_none(trs_fn, msg=f'Unknown transformation requested: {trs_spec}')
+
+  return trs_fn(**spec_cfg)
+
+
+class TransformPipeline:
+
+  def __init__(self, transformers):
+    self._transformers = []
+    for trs in transformers or []:
+      if isinstance(trs, str):
+        trs = parse_transform(trs)
+      self._transformers.append(trs)
+
+  @property
+  def transformers(self):
+    return tuple(self._transformers)
+
+  @property
+  def supports_partial_fit(self):
+    return all([hasattr(trs, 'partial_fit') for trs in self._transformers])
+
+  @property
+  def supports_fit(self):
+    return len(self._transformers) <= 1
+
+  def add(self, transformer):
+    self._transformers.append(transformer)
+
+  def transform(self, x):
+    tx = x
+    for trs in self._transformers:
+      tx = trs.transform(tx)
+
+    return tx
+
+  def fit_transform(self, x):
+    tx = x
+    for trs in self._transformers:
+      tx = trs.fit_transform(tx)
+
+    return tx
+
+  def fit(self, x):
+    tas.check(self.supports_fit,
+              msg=f'Only pipelines with a single transformer can call fit()')
+    for trs in self._transformers:
+      trs.fit(x)
+
+    return self
+
+  def partial_fit_transform(self, x):
+    tx = x
+    for trs in self._transformers:
+      trs.partial_fit(tx)
+      tx = trs.transform(tx)
+
+    return tx
+
+
+def fit(m, x, y, **kwargs):
+  shape = y.shape
+  # Some SciKit Learn models (ie. KNeighborsClassifier/KNeighborsRegressor) although
+  # supporting multi-output targets, insist on special casing the 1-output case
+  # requiring the 1D (N,) vector instead of a (N, 1) tensor.
+  if len(shape) == 2 and shape[-1] == 1:
+    y = np.squeeze(pyu.to_numpy(y), axis=-1)
+
+  return m.fit(x, y, **kwargs)
+
+
+def predict(m, x, **kwargs):
+  y = m.predict(x, **kwargs)
+
+  # Some SciKit Learn models (ie. KNeighborsClassifier/KNeighborsRegressor) although
+  # supporting multi-output targets, insist on special casing the 1-output case
+  # emitting 1D (N,) vectors instead of a (N, 1) tensors.
+  return y if y.ndim > 1 else y.reshape(-1, 1)
+
+
+def predict_proba(m, x, classes=None):
+
+  def extract_probs(p, cls):
+    preds = [p[:, c].reshape(-1, 1) for c in cls]
+    return np.concatenate(preds, axis=-1)
+
+  probs = m.predict_proba(x)
+  if classes is None:
+    return probs
+
+  if isinstance(probs, (list, tuple)):
+    return [extract_probs(p, classes) for p in probs]
+
+  return extract_probs(probs, classes)
+
+
+def _get_weights(dist, weights):
+  if weights in (None, "uniform"):
+    return np.ones_like(dist)
+  elif weights == "distance":
+    # if user attempts to classify a point that was zero distance from one
+    # or more training points, those training points are weighted as 1.0
+    # and the other points as 0.0
+    if dist.dtype is np.dtype:
+      for point_dist_i, point_dist in enumerate(dist):
+        # check if point_dist is iterable
+        # (ex: RadiusNeighborClassifier.predict may set an element of
+        # dist to 1e-6 to represent an 'outlier')
+        if hasattr(point_dist, "__contains__") and 0.0 in point_dist:
+          dist[point_dist_i] = point_dist == 0.0
+        else:
+          dist[point_dist_i] = 1.0 / point_dist
+    else:
+      with np.errstate(divide="ignore"):
+        dist = 1.0 / dist
+      inf_mask = np.isinf(dist)
+      inf_row = np.any(inf_mask, axis=1)
+      dist[inf_row] = inf_mask[inf_row]
+      return dist
+  elif callable(weights):
+    return weights(dist)
+  else:
+    alog.xraise(ValueError, f'Unrecognized "weights" value: {weights}')
+
+
+class WeighedKNNClassifier:
+
+  def __init__(self, **kwargs):
+    self._sample_weight = None
+    self._neigh = sklearn.neighbors.KNeighborsClassifier(**kwargs)
+
+  def fit(self, X, y, sample_weight=None):
+    self._sample_weight = sample_weight
+    self._neigh.fit(X, y)
+
+    return self
+
+  def _prepare_predict(self, X):
+    neigh_dist, neigh_ind = self._neigh.kneighbors(X)
+
+    classes = self._neigh.classes_
+    y = self._neigh._y
+    if not self._neigh.outputs_2d_:
+      y = y.reshape(-1, 1)
+      classes = [classes]
+    cweights = _get_weights(neigh_dist, self._neigh.weights)
+    sample_weight = self._sample_weight
+    if sample_weight.ndim < y.ndim:
+      sample_weight = sample_weight.reshape(-1, 1)
+
+    return pyu.make_object(neigh_dist=neigh_dist,
+                           neigh_ind=neigh_ind,
+                           y=y,
+                           classes=classes,
+                           cweights=cweights,
+                           sample_weight=sample_weight)
+
+  def predict(self, X):
+    if self._sample_weight is None:
+      return self._neigh.predict(X)
+
+    pp = self._prepare_predict(X)
+
+    y_pred = np.empty((len(X), len(pp.classes)), dtype=pp.classes[0].dtype)
+    for k, k_classes in enumerate(pp.classes):
+      weights = pp.sample_weight[pp.neigh_ind, k] * pp.cweights
+      mode, _ = sklearn.utils.extmath.weighted_mode(pp.y[pp.neigh_ind, k], weights,
+                                                    axis=1)
+
+      mode = np.asarray(mode.ravel(), dtype=np.intp)
+      y_pred[:, k] = k_classes.take(mode)
+
+    if not self._neigh.outputs_2d_:
+      y_pred = y_pred.ravel()
+
+    return y_pred
+
+  def predict_proba(self, X):
+    if self._sample_weight is None:
+      return self._neigh.predict_proba(X)
+
+    pp = self._prepare_predict(X)
+
+    all_rows = np.arange(len(X))
+    probabilities = []
+    for k, k_classes in enumerate(pp.classes):
+      pred_labels = pp.y[:, k][pp.neigh_ind]
+      proba_k = np.zeros((len(X), k_classes.size))
+
+      ksweight = pp.sample_weight[:, k]
+      for i, idx in enumerate(pred_labels.T):
+        proba_k[all_rows, idx] += ksweight * pp.cweights[:, i]
+
+      normalizer = proba_k.sum(axis=1)[:, np.newaxis]
+      normalizer[normalizer == 0.0] = 1.0
+
+      probabilities.append(proba_k / normalizer)
+
+    if not self._neigh.outputs_2d_:
+      probabilities = probabilities[0]
+
+    return probabilities
+