easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/loss/pairwise_loss.py

@@ -0,0 +1,307 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import tensorflow as tf
+from tensorflow.python.ops.losses.losses_impl import compute_weighted_loss
+
+from easy_rec.python.loss.focal_loss import sigmoid_focal_loss_with_logits
+from easy_rec.python.utils.shape_utils import get_shape_list
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def pairwise_loss(labels,
+                  logits,
+                  session_ids=None,
+                  margin=0,
+                  temperature=1.0,
+                  weights=1.0,
+                  name=''):
+  """Deprecated Pairwise loss.  Also see `pairwise_logistic_loss` below.
+
+  Args:
+    labels: a `Tensor` with shape [batch_size]. e.g. click or not click in the session.
+    logits: a `Tensor` with shape [batch_size]. e.g. the value of last neuron before activation.
+    session_ids: a `Tensor` with shape [batch_size]. Session ids of each sample, used to max GAUC metric. e.g. user_id
+    margin: the margin between positive and negative sample pair
+    temperature: (Optional) The temperature to use for scaling the logits.
+    weights: sample weights
+    name: the name of loss
+  """
+  logging.warning(
+      'The old `pairwise_loss` is being deprecated. '
+      'Please use the new `pairwise_logistic_loss` or `pairwise_focal_loss`')
+  loss_name = name if name else 'pairwise_loss'
+  logging.info('[{}] margin: {}, temperature: {}'.format(
+      loss_name, margin, temperature))
+
+  if temperature != 1.0:
+    logits /= temperature
+  pairwise_logits = tf.math.subtract(
+      tf.expand_dims(logits, -1), tf.expand_dims(logits, 0)) - margin
+  pairwise_mask = tf.greater(
+      tf.expand_dims(labels, -1), tf.expand_dims(labels, 0))
+  if session_ids is not None:
+    logging.info('[%s] use session ids' % loss_name)
+    group_equal = tf.equal(
+        tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0))
+    pairwise_mask = tf.logical_and(pairwise_mask, group_equal)
+
+  pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask)
+  num_pair = tf.size(pairwise_logits)
+  tf.summary.scalar('loss/%s_num_of_pairs' % loss_name, num_pair)
+
+  if tf.is_numeric_tensor(weights):
+    logging.info('[%s] use sample weight' % loss_name)
+    weights = tf.expand_dims(tf.cast(weights, tf.float32), -1)
+    batch_size, _ = get_shape_list(weights, 2)
+    pairwise_weights = tf.tile(weights, tf.stack([1, batch_size]))
+    pairwise_weights = tf.boolean_mask(pairwise_weights, pairwise_mask)
+  else:
+    pairwise_weights = weights
+
+  pairwise_pseudo_labels = tf.ones_like(pairwise_logits)
+  loss = tf.losses.sigmoid_cross_entropy(
+      pairwise_pseudo_labels, pairwise_logits, weights=pairwise_weights)
+  # set rank loss to zero if a batch has no positive sample.
+  # loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss)
+  return loss
+
+
+def pairwise_focal_loss(labels,
+                        logits,
+                        session_ids=None,
+                        hinge_margin=None,
+                        gamma=2,
+                        alpha=None,
+                        ohem_ratio=1.0,
+                        temperature=1.0,
+                        weights=1.0,
+                        name=''):
+  loss_name = name if name else 'pairwise_focal_loss'
+  assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]'
+  logging.info(
+      '[{}] hinge margin: {}, gamma: {}, alpha: {}, ohem_ratio: {}, temperature: {}'
+      .format(loss_name, hinge_margin, gamma, alpha, ohem_ratio, temperature))
+
+  if temperature != 1.0:
+    logits /= temperature
+  pairwise_logits = tf.expand_dims(logits, -1) - tf.expand_dims(logits, 0)
+
+  pairwise_mask = tf.greater(
+      tf.expand_dims(labels, -1), tf.expand_dims(labels, 0))
+  if hinge_margin is not None:
+    hinge_mask = tf.less(pairwise_logits, hinge_margin)
+    pairwise_mask = tf.logical_and(pairwise_mask, hinge_mask)
+  if session_ids is not None:
+    logging.info('[%s] use session ids' % loss_name)
+    group_equal = tf.equal(
+        tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0))
+    pairwise_mask = tf.logical_and(pairwise_mask, group_equal)
+
+  pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask)
+  num_pair = tf.size(pairwise_logits)
+  tf.summary.scalar('loss/%s_num_of_pairs' % loss_name, num_pair)
+
+  if tf.is_numeric_tensor(weights):
+    logging.info('[%s] use sample weight' % loss_name)
+    weights = tf.expand_dims(tf.cast(weights, tf.float32), -1)
+    batch_size, _ = get_shape_list(weights, 2)
+    pairwise_weights = tf.tile(weights, tf.stack([1, batch_size]))
+    pairwise_weights = tf.boolean_mask(pairwise_weights, pairwise_mask)
+  else:
+    pairwise_weights = weights
+
+  pairwise_pseudo_labels = tf.ones_like(pairwise_logits)
+  loss = sigmoid_focal_loss_with_logits(
+      pairwise_pseudo_labels,
+      pairwise_logits,
+      gamma=gamma,
+      alpha=alpha,
+      ohem_ratio=ohem_ratio,
+      sample_weights=pairwise_weights)
+  return loss
+
+
+def pairwise_logistic_loss(labels,
+                           logits,
+                           session_ids=None,
+                           temperature=1.0,
+                           hinge_margin=None,
+                           weights=1.0,
+                           ohem_ratio=1.0,
+                           use_label_margin=False,
+                           name=''):
+  r"""Computes pairwise logistic loss between `labels` and `logits`, equivalent to RankNet loss.
+
+  Definition:
+  $$
+  \mathcal{L}(\{y\}, \{s\}) =
+  \sum_i \sum_j I[y_i > y_j] \log(1 + \exp(-(s_i - s_j)))
+  $$
+
+  Args:
+    labels: A `Tensor` of the same shape as `logits` representing graded
+      relevance.
+    logits: A `Tensor` with shape [batch_size].
+    session_ids: a `Tensor` with shape [batch_size]. Session ids of each
+      sample, used to max GAUC metric. e.g. user_id
+    temperature: (Optional) The temperature to use for scaling the logits.
+    hinge_margin: the margin between positive and negative logits
+    weights: A scalar, a `Tensor` with shape [batch_size] for each sample
+    ohem_ratio: the percent of hard examples to be mined
+    use_label_margin: whether to use the diff `label[i]-label[j]` as margin
+    name: the name of loss
+  """
+  loss_name = name if name else 'pairwise_logistic_loss'
+  assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]'
+  logging.info('[{}] hinge margin: {}, ohem_ratio: {}, temperature: {}'.format(
+      loss_name, hinge_margin, ohem_ratio, temperature))
+
+  if temperature != 1.0:
+    logits /= temperature
+    if use_label_margin:
+      labels /= temperature
+
+  pairwise_logits = tf.math.subtract(
+      tf.expand_dims(logits, -1), tf.expand_dims(logits, 0))
+  if use_label_margin:
+    pairwise_logits -= tf.math.subtract(
+        tf.expand_dims(labels, -1), tf.expand_dims(labels, 0))
+  elif hinge_margin is not None:
+    pairwise_logits -= hinge_margin
+
+  pairwise_mask = tf.greater(
+      tf.expand_dims(labels, -1), tf.expand_dims(labels, 0))
+  if session_ids is not None:
+    logging.info('[%s] use session ids' % loss_name)
+    group_equal = tf.equal(
+        tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0))
+    pairwise_mask = tf.logical_and(pairwise_mask, group_equal)
+
+  pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask)
+  num_pair = tf.size(pairwise_logits)
+  tf.summary.scalar('loss/%s_num_of_pairs' % loss_name, num_pair)
+
+  # The following is the same as log(1 + exp(-pairwise_logits)).
+  losses = tf.nn.relu(-pairwise_logits) + tf.math.log1p(
+      tf.exp(-tf.abs(pairwise_logits)))
+
+  if tf.is_numeric_tensor(weights):
+    logging.info('[%s] use sample weight' % loss_name)
+    weights = tf.expand_dims(tf.cast(weights, tf.float32), -1)
+    batch_size, _ = get_shape_list(weights, 2)
+    pairwise_weights = tf.tile(weights, tf.stack([1, batch_size]))
+    pairwise_weights = tf.boolean_mask(pairwise_weights, pairwise_mask)
+  else:
+    pairwise_weights = weights
+
+  if ohem_ratio == 1.0:
+    return compute_weighted_loss(losses, pairwise_weights)
+
+  losses = compute_weighted_loss(
+      losses, pairwise_weights, reduction=tf.losses.Reduction.NONE)
+  k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio)
+  k = tf.to_int32(tf.math.rint(k))
+  topk = tf.nn.top_k(losses, k)
+  losses = tf.boolean_mask(topk.values, topk.values > 0)
+  return tf.reduce_mean(losses)
+
+
+def pairwise_hinge_loss(labels,
+                        logits,
+                        session_ids=None,
+                        temperature=1.0,
+                        margin=1.0,
+                        weights=1.0,
+                        ohem_ratio=1.0,
+                        label_is_logits=True,
+                        use_label_margin=True,
+                        use_exponent=False,
+                        name=''):
+  r"""Computes pairwise hinge loss between `labels` and `logits`.
+
+  Definition:
+  $$
+  \mathcal{L}(\{y\}, \{s\}) =
+  \sum_i \sum_j I[y_i > y_j] \max(0, 1 - (s_i - s_j))
+  $$
+
+  Args:
+    labels: A `Tensor` of the same shape as `logits` representing graded
+      relevance.
+    logits: A `Tensor` with shape [batch_size].
+    session_ids: a `Tensor` with shape [batch_size]. Session ids of each sample, used to max GAUC metric. e.g. user_id
+    temperature: (Optional) The temperature to use for scaling the logits.
+    margin: the margin between positive and negative logits
+    weights: A scalar, a `Tensor` with shape [batch_size] for each sample
+    ohem_ratio: the percent of hard examples to be mined
+    label_is_logits: Whether `labels` is expected to be a logits tensor.
+    use_label_margin: whether to use the diff `label[i]-label[j]` as margin
+    use_exponent: whether to use exponential difference
+    name: the name of loss
+  """
+  loss_name = name if name else 'pairwise_hinge_loss'
+  assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]'
+  logging.info(
+      '[{}] margin: {}, ohem_ratio: {}, temperature: {}, use_exponent: {}, label_is_logits: {}, use_label_margin: {}'
+      .format(loss_name, margin, ohem_ratio, temperature, use_exponent,
+              label_is_logits, use_label_margin))
+
+  if temperature != 1.0:
+    logits /= temperature
+    if label_is_logits:
+      labels /= temperature
+  if use_exponent:
+    labels = tf.nn.sigmoid(labels)
+    logits = tf.nn.sigmoid(labels)
+
+  pairwise_logits = tf.math.subtract(
+      tf.expand_dims(logits, -1), tf.expand_dims(logits, 0))
+  pairwise_labels = tf.math.subtract(
+      tf.expand_dims(labels, -1), tf.expand_dims(labels, 0))
+
+  pairwise_mask = tf.greater(pairwise_labels, 0)
+  if session_ids is not None:
+    logging.info('[%s] use session ids' % loss_name)
+    group_equal = tf.equal(
+        tf.expand_dims(session_ids, -1), tf.expand_dims(session_ids, 0))
+    pairwise_mask = tf.logical_and(pairwise_mask, group_equal)
+
+  pairwise_logits = tf.boolean_mask(pairwise_logits, pairwise_mask)
+  pairwise_labels = tf.boolean_mask(pairwise_labels, pairwise_mask)
+  num_pair = tf.size(pairwise_logits)
+  tf.summary.scalar('loss/%s_num_of_pairs' % loss_name, num_pair)
+
+  if use_label_margin:
+    diff = pairwise_labels - pairwise_logits
+  else:
+    diff = margin - pairwise_logits
+  if use_exponent:
+    threshold = 88.0  # the max value of float32 is 3.4028235e+38
+    safe_diff = tf.clip_by_value(diff, -threshold, threshold)
+    losses = tf.nn.relu(tf.exp(safe_diff) - 1.0)
+  else:
+    losses = tf.nn.relu(diff)
+
+  if tf.is_numeric_tensor(weights):
+    logging.info('[%s] use sample weight' % loss_name)
+    weights = tf.expand_dims(tf.cast(weights, tf.float32), -1)
+    batch_size, _ = get_shape_list(weights, 2)
+    pairwise_weights = tf.tile(weights, tf.stack([1, batch_size]))
+    pairwise_weights = tf.boolean_mask(pairwise_weights, pairwise_mask)
+  else:
+    pairwise_weights = weights
+
+  if ohem_ratio == 1.0:
+    return compute_weighted_loss(losses, pairwise_weights)
+
+  losses = compute_weighted_loss(
+      losses, pairwise_weights, reduction=tf.losses.Reduction.NONE)
+  k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio)
+  k = tf.to_int32(tf.math.rint(k))
+  topk = tf.nn.top_k(losses, k)
+  losses = tf.boolean_mask(topk.values, topk.values > 0)
+  return tf.reduce_mean(losses)

easy_rec/python/loss/softmax_loss_with_negative_mining.py

@@ -0,0 +1,110 @@
+# coding=utf-8
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import tensorflow as tf
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def support_vector_guided_softmax_loss(pos_score,
+                                       neg_scores,
+                                       margin=0,
+                                       t=1,
+                                       smooth=1.0,
+                                       threshold=0,
+                                       weights=1.0):
+  """Refer paper: Support Vector Guided Softmax Loss for Face Recognition (https://128.84.21.199/abs/1812.11317)."""
+  new_pos_score = pos_score - margin
+  cond = tf.greater_equal(new_pos_score - neg_scores, threshold)
+  mask = tf.where(cond, tf.zeros_like(cond, tf.float32),
+                  tf.ones_like(cond, tf.float32))  # I_k
+  new_neg_scores = mask * (neg_scores * t + t - 1) + (1 - mask) * neg_scores
+  logits = tf.concat([new_pos_score, new_neg_scores], axis=1)
+  if 1.0 != smooth:
+    logits *= smooth
+
+  loss = tf.losses.sparse_softmax_cross_entropy(
+      tf.zeros_like(pos_score, dtype=tf.int32), logits, weights=weights)
+  # set rank loss to zero if a batch has no positive sample.
+  loss = tf.where(tf.is_nan(loss), tf.zeros_like(loss), loss)
+  return loss
+
+
+def softmax_loss_with_negative_mining(user_emb,
+                                      item_emb,
+                                      labels,
+                                      num_negative_samples=4,
+                                      embed_normed=False,
+                                      weights=1.0,
+                                      gamma=1.0,
+                                      margin=0,
+                                      t=1,
+                                      seed=None):
+  """Compute the softmax loss based on the cosine distance explained below.
+
+  Given mini batches for `user_emb` and `item_emb`, this function computes for each element in `user_emb`
+  the cosine distance between it and the corresponding `item_emb`,
+  and additionally the cosine distance between `user_emb` and some other elements of `item_emb`
+   (referred to a negative samples).
+  The negative samples are formed on the fly by shifting the right side (`item_emb`).
+  Then the softmax loss will be computed based on these cosine distance.
+
+  Args:
+    user_emb: A `Tensor` with shape [batch_size, embedding_size]. The embedding of user.
+    item_emb: A `Tensor` with shape [batch_size, embedding_size]. The embedding of item.
+    labels: a `Tensor` with shape [batch_size]. e.g. click or not click in the session. It's values must be 0 or 1.
+    num_negative_samples: the num of negative samples, should be in range [1, batch_size).
+    embed_normed: bool, whether input embeddings l2 normalized
+    weights: `weights` acts as a coefficient for the loss. If a scalar is provided,
+      then the loss is simply scaled by the given value. If `weights` is a
+      tensor of shape `[batch_size]`, then the loss weights apply to each corresponding sample.
+    gamma: smooth coefficient of softmax
+    margin: the margin between positive pair and negative pair
+    t: coefficient of support vector guided softmax loss
+    seed: A Python integer. Used to create a random seed for the distribution.
+      See `tf.set_random_seed`
+      for behavior.
+
+  Return:
+    support vector guided softmax loss of positive labels
+  """
+  assert 0 < num_negative_samples, '`num_negative_samples` should be greater than 0'
+
+  batch_size = tf.shape(item_emb)[0]
+  is_valid = tf.assert_less(
+      num_negative_samples,
+      batch_size,
+      message='`num_negative_samples` should be less than batch_size')
+  with tf.control_dependencies([is_valid]):
+    if not embed_normed:
+      user_emb = tf.nn.l2_normalize(user_emb, axis=-1)
+      item_emb = tf.nn.l2_normalize(item_emb, axis=-1)
+
+    vectors = [item_emb]
+    for i in range(num_negative_samples):
+      shift = tf.random_uniform([], 1, batch_size, dtype=tf.int32, seed=seed)
+      neg_item_emb = tf.roll(item_emb, shift, axis=0)
+      vectors.append(neg_item_emb)
+    # all_embeddings's shape: (batch_size, num_negative_samples + 1, vec_dim)
+    all_embeddings = tf.stack(vectors, axis=1)
+
+    mask = tf.greater(labels, 0)
+    mask_user_emb = tf.boolean_mask(user_emb, mask)
+    mask_item_emb = tf.boolean_mask(all_embeddings, mask)
+    if isinstance(weights, tf.Tensor):
+      weights = tf.boolean_mask(weights, mask)
+
+    # sim_scores's shape: (num_of_pos_label_in_batch_size, num_negative_samples + 1)
+    sim_scores = tf.keras.backend.batch_dot(
+        mask_user_emb, mask_item_emb, axes=(1, 2))
+    pos_score = tf.slice(sim_scores, [0, 0], [-1, 1])
+    neg_scores = tf.slice(sim_scores, [0, 1], [-1, -1])
+
+    loss = support_vector_guided_softmax_loss(
+        pos_score,
+        neg_scores,
+        margin=margin,
+        t=t,
+        smooth=gamma,
+        weights=weights)
+  return loss

easy_rec/python/loss/zero_inflated_lognormal.py

@@ -0,0 +1,76 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+"""Zero-inflated lognormal loss for lifetime value prediction."""
+import tensorflow as tf
+import tensorflow_probability as tfp
+
+tfd = tfp.distributions
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def zero_inflated_lognormal_pred(logits):
+  """Calculates predicted mean of zero inflated lognormal logits.
+
+  Arguments:
+    logits: [batch_size, 3] tensor of logits.
+
+  Returns:
+    positive_probs: [batch_size, 1] tensor of positive probability.
+    preds: [batch_size, 1] tensor of predicted mean.
+  """
+  logits = tf.convert_to_tensor(logits, dtype=tf.float32)
+  positive_probs = tf.keras.backend.sigmoid(logits[..., :1])
+  loc = logits[..., 1:2]
+  scale = tf.keras.backend.softplus(logits[..., 2:])
+  preds = (
+      positive_probs *
+      tf.keras.backend.exp(loc + 0.5 * tf.keras.backend.square(scale)))
+  return positive_probs, preds
+
+
+def zero_inflated_lognormal_loss(labels, logits, name=''):
+  """Computes the zero inflated lognormal loss.
+
+  Usage with tf.keras API:
+
+  ```python
+  model = tf.keras.Model(inputs, outputs)
+  model.compile('sgd', loss=zero_inflated_lognormal)
+  ```
+
+  Arguments:
+    labels: True targets, tensor of shape [batch_size, 1].
+    logits: Logits of output layer, tensor of shape [batch_size, 3].
+    name: the name of loss
+
+  Returns:
+    Zero inflated lognormal loss value.
+  """
+  loss_name = name if name else 'ziln_loss'
+  labels = tf.cast(labels, dtype=tf.float32)
+  if labels.shape.ndims == 1:
+    labels = tf.expand_dims(labels, 1)  # [B, 1]
+  positive = tf.cast(labels > 0, tf.float32)
+
+  logits = tf.convert_to_tensor(logits, dtype=tf.float32)
+  logits.shape.assert_is_compatible_with(
+      tf.TensorShape(labels.shape[:-1].as_list() + [3]))
+
+  positive_logits = logits[..., :1]
+  classification_loss = tf.keras.backend.binary_crossentropy(
+      positive, positive_logits, from_logits=True)
+  classification_loss = tf.keras.backend.mean(classification_loss)
+  tf.summary.scalar('loss/%s_classify' % loss_name, classification_loss)
+
+  loc = logits[..., 1:2]
+  scale = tf.math.maximum(
+      tf.keras.backend.softplus(logits[..., 2:]),
+      tf.math.sqrt(tf.keras.backend.epsilon()))
+  safe_labels = positive * labels + (
+      1 - positive) * tf.keras.backend.ones_like(labels)
+  regression_loss = -tf.keras.backend.mean(
+      positive * tfd.LogNormal(loc=loc, scale=scale).log_prob(safe_labels))
+  tf.summary.scalar('loss/%s_regression' % loss_name, regression_loss)
+  return classification_loss + regression_loss