easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/compat/queues.py

@@ -0,0 +1,311 @@
+#
+# Module implementing queues
+#
+# multiprocessing/queues.py
+#
+# Copyright (c) 2006-2008, R Oudkerk
+# Licensed to PSF under a Contributor Agreement.
+#
+
+import collections
+import errno
+import logging
+import os
+import sys
+import threading
+import time
+import weakref
+from multiprocessing import connection
+from multiprocessing.util import Finalize
+from multiprocessing.util import is_exiting
+from multiprocessing.util import register_after_fork
+from queue import Empty
+from queue import Full
+
+import six
+
+try:
+  from multiprocessing import context
+except ImportError:
+  context = None
+  pass
+
+if context is not None:
+  _ForkingPickler = context.reduction.ForkingPickler
+else:
+  _ForkingPickler = None
+
+#
+# Queue type using a pipe, buffer and thread
+#
+
+
+class Queue(object):
+
+  _sentinel = object()
+
+  def __init__(self, ctx, maxsize=0, name=''):
+    assert not six.PY2, 'python2 is not supported'
+    if maxsize <= 0:
+      # Can raise ImportError (see issues #3770 and #23400)
+      from multiprocessing.synchronize import SEM_VALUE_MAX as maxsize
+    self._maxsize = maxsize
+    self._reader, self._writer = connection.Pipe(duplex=False)
+    self._rlock = ctx.Lock()
+    self._opid = os.getpid()
+    if sys.platform == 'win32':
+      self._wlock = None
+    else:
+      self._wlock = ctx.Lock()
+    self._sem = ctx.BoundedSemaphore(maxsize)
+    # For use by concurrent.futures
+    self._ignore_epipe = False
+    self._reset()
+    self._name = name
+    self._run = True
+
+    if sys.platform != 'win32':
+      register_after_fork(self, Queue._after_fork)
+
+  def __getstate__(self):
+    context.assert_spawning(self)
+    return (self._ignore_epipe, self._maxsize, self._reader, self._writer,
+            self._rlock, self._wlock, self._sem, self._opid, self._name,
+            self._run)
+
+  def __setstate__(self, state):
+    (self._ignore_epipe, self._maxsize, self._reader, self._writer, self._rlock,
+     self._wlock, self._sem, self._opid, self._name, self._run) = state
+    self._reset()
+
+  def _after_fork(self):
+    logging.debug('Queue._after_fork()')
+    self._reset(after_fork=True)
+
+  def _reset(self, after_fork=False):
+    if after_fork:
+      self._notempty._at_fork_reinit()
+    else:
+      self._notempty = threading.Condition(threading.Lock())
+    self._buffer = collections.deque()
+    self._thread = None
+    self._jointhread = None
+    self._joincancelled = False
+    self._closed = False
+    self._close = None
+    self._send_bytes = self._writer.send_bytes
+    self._recv_bytes = self._reader.recv_bytes
+    self._poll = self._reader.poll
+
+  def put(self, obj, block=True, timeout=None):
+    if self._closed:
+      raise ValueError('Queue %s is closed' % self._name)
+    if not self._sem.acquire(block, timeout):
+      raise Full
+
+    with self._notempty:
+      if self._thread is None:
+        self._start_thread()
+      self._buffer.append(obj)
+      self._notempty.notify()
+
+  def get(self, block=True, timeout=None):
+    if self._closed:
+      raise ValueError('Queue %s is closed' % self._name)
+    if block and timeout is None:
+      with self._rlock:
+        res = self._recv_bytes()
+      self._sem.release()
+    else:
+      if block:
+        deadline = time.monotonic() + timeout
+      if not self._rlock.acquire(block, timeout):
+        raise Empty
+      try:
+        if block:
+          timeout = deadline - time.monotonic()
+          if not self._poll(timeout):
+            raise Empty
+        elif not self._poll():
+          raise Empty
+        res = self._recv_bytes()
+        self._sem.release()
+      finally:
+        self._rlock.release()
+    # unserialize the data after having released the lock
+    return _ForkingPickler.loads(res)
+
+  def qsize(self):
+    # Raises NotImplementedError on Mac OSX because of broken sem_getvalue()
+    return self._maxsize - self._sem._semlock._get_value()
+
+  def empty(self):
+    return not self._poll()
+
+  def full(self):
+    return self._sem._semlock._is_zero()
+
+  def get_nowait(self):
+    return self.get(False)
+
+  def put_nowait(self, obj):
+    return self.put(obj, False)
+
+  def close(self, wait_send_finish=True):
+    self._closed = True
+    close = self._close
+    if not wait_send_finish and self._thread is not None and self._thread.is_alive(
+    ):
+      try:
+        if self._reader is not None:
+          self._reader.close()
+      except Exception:
+        pass
+      self._run = False
+      # clear queue
+      # with self._rlock:
+      #   while self._thread.is_alive() and self._poll(1):
+      #     res = self._recv_bytes()
+      #     logging.info('Queue[name=' + self._name + '] clear one elem')
+      # logging.info('Queue[name=' + self._name + '] clear queue done')
+    if close:
+      self._close = None
+      close()
+
+  def join_thread(self):
+    logging.debug('Queue.join_thread()')
+    assert self._closed, 'Queue {0!r} not closed'.format(self)
+    if self._jointhread:
+      self._jointhread()
+
+  def cancel_join_thread(self):
+    logging.debug('Queue.cancel_join_thread()')
+    self._joincancelled = True
+    try:
+      self._jointhread.cancel()
+    except AttributeError:
+      pass
+
+  def _start_thread(self):
+    logging.debug('Queue._start_thread()')
+
+    # Start thread which transfers data from buffer to pipe
+    self._buffer.clear()
+    self._thread = threading.Thread(
+        target=self._feed,
+        args=(self._buffer, self._notempty, self._send_bytes, self._wlock,
+              self._reader.close, self._writer.close, self._ignore_epipe,
+              self._on_queue_feeder_error, self._sem),
+        name='QueueFeederThread')
+    self._thread.daemon = True
+
+    logging.debug('doing self._thread.start()')
+    self._thread.start()
+    logging.debug('... done self._thread.start()')
+
+    if not self._joincancelled:
+      self._jointhread = Finalize(
+          self._thread,
+          Queue._finalize_join, [weakref.ref(self._thread)],
+          exitpriority=-5)
+
+    # Send sentinel to the thread queue object when garbage collected
+    self._close = Finalize(
+        self,
+        Queue._finalize_close, [self._buffer, self._notempty],
+        exitpriority=10)
+
+  @staticmethod
+  def _finalize_join(twr):
+    logging.debug('joining queue thread')
+    thread = twr()
+    if thread is not None:
+      thread.join()
+      logging.debug('... queue thread joined')
+    else:
+      logging.debug('... queue thread already dead')
+
+  @staticmethod
+  def _finalize_close(buffer, notempty):
+    logging.debug('telling queue thread to quit')
+    with notempty:
+      buffer.append(Queue._sentinel)
+      notempty.notify()
+
+  def _feed(self, buffer, notempty, send_bytes, writelock, reader_close,
+            writer_close, ignore_epipe, onerror, queue_sem):
+    logging.debug('starting thread to feed data to pipe')
+    nacquire = notempty.acquire
+    nrelease = notempty.release
+    nwait = notempty.wait
+    bpopleft = buffer.popleft
+    sentinel = Queue._sentinel
+    if sys.platform != 'win32':
+      wacquire = writelock.acquire
+      wrelease = writelock.release
+    else:
+      wacquire = None
+
+    pid = os.getpid()
+    name = self._name
+    while self._run:
+      try:
+        nacquire()
+        try:
+          if not buffer:
+            nwait()
+        finally:
+          nrelease()
+        try:
+          while self._run:
+            obj = bpopleft()
+            if obj is sentinel:
+              # logging.info('Queue[' + self._name + '] feeder thread got sentinel -- exiting: ' + str(self._run))
+              reader_close()
+              writer_close()
+              return
+
+            # serialize the data before acquiring the lock
+            obj = _ForkingPickler.dumps(obj)
+            if wacquire is None:
+              send_bytes(obj)
+            else:
+              wacquire()
+              try:
+                send_bytes(obj)
+              finally:
+                wrelease()
+        except IndexError:
+          pass
+      except Exception as e:
+        if ignore_epipe and getattr(e, 'errno', 0) == errno.EPIPE:
+          logging.warning('Queue[' + name + '] exception: pid=' + str(pid) +
+                          ' run=' + str(self._run) + ' e=' + str(e))
+          return
+        # Since this runs in a daemon thread the resources it uses
+        # may be become unusable while the process is cleaning up.
+        # We ignore errors which happen after the process has
+        # started to cleanup.
+        if is_exiting():
+          logging.warning('Queue[' + name + '] thread error in exiting: pid=' +
+                          str(pid) + ' run=' + str(self._run) + ' e=' + str(e))
+          return
+        else:
+          # Since the object has not been sent in the queue, we need
+          # to decrease the size of the queue. The error acts as
+          # if the object had been silently removed from the queue
+          # and this step is necessary to have a properly working
+          # queue.
+          queue_sem.release()
+          onerror(e, obj)
+    # logging.info('Queue[' + name +  '] send thread finish: pid=' + str(pid)
+    #    +  ' run=' + str(self._run))
+
+  @staticmethod
+  def _on_queue_feeder_error(e, obj):
+    """Private API hook called when feeding data in the background thread raises an exception.
+
+    For overriding by concurrent.futures.
+    """
+    import traceback
+    traceback.print_exc()

easy_rec/python/compat/regularizers.py

@@ -0,0 +1,208 @@
+# -*- encoding:utf-8 -*-
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# from tf.contrib
+"""Regularizers for use with layers."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numbers
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
+from tensorflow.python.ops import standard_ops
+from tensorflow.python.platform import tf_logging as logging
+
+__all__ = [
+    'l1_regularizer', 'l2_regularizer', 'l1_l2_regularizer', 'sum_regularizer',
+    'apply_regularization'
+]
+
+
+def l1_regularizer(scale, scope=None):
+  """Returns a function that can be used to apply L1 regularization to weights.
+
+  L1 regularization encourages sparsity.
+
+  Args:
+    scale: A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+    scope: An optional scope name.
+
+  Returns:
+    A function with signature `l1(weights)` that apply L1 regularization.
+
+  Raises:
+    ValueError: If scale is negative or if scale is not a float.
+  """
+  if isinstance(scale, numbers.Integral):
+    raise ValueError('scale cannot be an integer: %s' % scale)
+  if isinstance(scale, numbers.Real):
+    if scale < 0.:
+      raise ValueError('Setting a scale less than 0 on a regularizer: %g' %
+                       scale)
+    if scale == 0.:
+      logging.info('Scale of 0 disables regularizer.')
+      return lambda _: None
+
+  def l1(weights, name=None):
+    """Applies L1 regularization to weights."""
+    with ops.name_scope(scope, 'l1_regularizer', [weights]) as name:
+      my_scale = ops.convert_to_tensor(
+          scale, dtype=weights.dtype.base_dtype, name='scale')
+      return standard_ops.multiply(
+          my_scale,
+          standard_ops.reduce_sum(standard_ops.abs(weights)),
+          name=name)
+
+  return l1
+
+
+def l2_regularizer(scale, scope=None):
+  """Returns a function that can be used to apply L2 regularization to weights.
+
+  Small values of L2 can help prevent overfitting the training data.
+
+  Args:
+    scale: A scalar multiplier `Tensor`. 0.0 disables the regularizer.
+    scope: An optional scope name.
+
+  Returns:
+    A function with signature `l2(weights)` that applies L2 regularization.
+
+  Raises:
+    ValueError: If scale is negative or if scale is not a float.
+  """
+  if isinstance(scale, numbers.Integral):
+    raise ValueError('scale cannot be an integer: %s' % (scale,))
+  if isinstance(scale, numbers.Real):
+    if scale < 0.:
+      raise ValueError('Setting a scale less than 0 on a regularizer: %g.' %
+                       scale)
+    if scale == 0.:
+      logging.info('Scale of 0 disables regularizer.')
+      return lambda _: None
+
+  def l2(weights):
+    """Applies l2 regularization to weights."""
+    with ops.name_scope(scope, 'l2_regularizer', [weights]) as name:
+      my_scale = ops.convert_to_tensor(
+          scale, dtype=weights.dtype.base_dtype, name='scale')
+      return standard_ops.multiply(my_scale, nn.l2_loss(weights), name=name)
+
+  return l2
+
+
+def l1_l2_regularizer(scale_l1=1.0, scale_l2=1.0, scope=None):
+  """Returns a function that can be used to apply L1 L2 regularizations.
+
+  Args:
+    scale_l1: A scalar multiplier `Tensor` for L1 regularization.
+    scale_l2: A scalar multiplier `Tensor` for L2 regularization.
+    scope: An optional scope name.
+
+  Returns:
+    A function with signature `l1_l2(weights)` that applies a weighted sum of
+    L1 L2 regularization.
+
+  Raises:
+    ValueError: If scale is negative or if scale is not a float.
+  """
+  if isinstance(scale_l1, numbers.Integral):
+    raise ValueError('scale_l1 cannot be an integer: %s' % (scale_l1,))
+  if isinstance(scale_l2, numbers.Integral):
+    raise ValueError('scale_l2 cannot be an integer: %s' % (scale_l2,))
+  scope = scope or 'l1_l2_regularizer'
+  if scale_l1 == 0.:
+    return l2_regularizer(scale_l2, scope)
+  if scale_l2 == 0.:
+    return l1_regularizer(scale_l1, scope)
+  return sum_regularizer([l1_regularizer(scale_l1),
+                          l2_regularizer(scale_l2)],
+                         scope=scope)
+
+
+def sum_regularizer(regularizer_list, scope=None):
+  """Returns a function that applies the sum of multiple regularizers.
+
+  Args:
+    regularizer_list: A list of regularizers to apply.
+    scope: An optional scope name
+
+  Returns:
+    A function with signature `sum_reg(weights)` that applies the
+    sum of all the input regularizers.
+  """
+  regularizer_list = [reg for reg in regularizer_list if reg is not None]
+  if not regularizer_list:
+    return None
+
+  def sum_reg(weights):
+    """Applies the sum of all the input regularizers."""
+    with ops.name_scope(scope, 'sum_regularizer', [weights]) as name:
+      regularizer_tensors = []
+      for reg in regularizer_list:
+        tensor = reg(weights)
+        if tensor is not None:
+          regularizer_tensors.append(tensor)
+      return math_ops.add_n(
+          regularizer_tensors, name=name) if regularizer_tensors else None
+
+  return sum_reg
+
+
+def apply_regularization(regularizer, weights_list=None):
+  """Returns the summed penalty by applying `regularizer` to the `weights_list`.
+
+  Adding a regularization penalty over the layer weights and embedding weights
+  can help prevent overfitting the training data. Regularization over layer
+  biases is less common/useful, but assuming proper data preprocessing/mean
+  subtraction, it usually shouldn't hurt much either.
+
+  Args:
+    regularizer: A function that takes a single `Tensor` argument and returns
+      a scalar `Tensor` output.
+    weights_list: List of weights `Tensors` or `Variables` to apply
+      `regularizer` over. Defaults to the `GraphKeys.WEIGHTS` collection if
+      `None`.
+
+  Returns:
+    A scalar representing the overall regularization penalty.
+
+  Raises:
+    ValueError: If `regularizer` does not return a scalar output, or if we find
+        no weights.
+  """
+  if not weights_list:
+    weights_list = ops.get_collection(ops.GraphKeys.WEIGHTS)
+  if not weights_list:
+    raise ValueError('No weights to regularize.')
+  with ops.name_scope(
+      'get_regularization_penalty', values=weights_list) as scope:
+    penalties = [regularizer(w) for w in weights_list]
+    penalties = [
+        p if p is not None else constant_op.constant(0.0) for p in penalties
+    ]
+    for p in penalties:
+      if p.get_shape().ndims != 0:
+        raise ValueError('regularizer must return a scalar Tensor instead of a '
+                         'Tensor with rank %d.' % p.get_shape().ndims)
+
+    summed_penalty = math_ops.add_n(penalties, name=scope)
+    ops.add_to_collection(ops.GraphKeys.REGULARIZATION_LOSSES, summed_penalty)
+    return summed_penalty