easy-cs-rec-custommodel 0.8.6__py2.py3-none-any.whl

easy_rec/python/loss/circle_loss.py

@@ -0,0 +1,82 @@
+# coding=utf-8
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import tensorflow as tf
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def circle_loss(embeddings,
+                labels,
+                sessions=None,
+                margin=0.25,
+                gamma=32,
+                embed_normed=False):
+  """Paper: Circle Loss: A Unified Perspective of Pair Similarity Optimization.
+
+  Link: http://arxiv.org/pdf/2002.10857.pdf
+
+  Args:
+    embeddings: A `Tensor` with shape [batch_size, embedding_size]. The embedding of each sample.
+    labels: a `Tensor` with shape [batch_size]. e.g. click or not click in the session.
+    sessions: a `Tensor` with shape [batch_size]. session ids of each sample.
+    margin: the margin between positive similarity and negative similarity
+    gamma: parameter of circle loss
+    embed_normed: bool, whether input embeddings l2 normalized
+  """
+  norm_embeddings = embeddings if embed_normed else tf.nn.l2_normalize(
+      embeddings, axis=-1)
+  pair_wise_cosine_matrix = tf.matmul(
+      norm_embeddings, norm_embeddings, transpose_b=True)
+
+  positive_mask = get_anchor_positive_triplet_mask(labels, sessions)
+  negative_mask = 1 - positive_mask - tf.eye(tf.shape(labels)[0])
+
+  delta_p = 1 - margin
+  delta_n = margin
+
+  ap = tf.nn.relu(-tf.stop_gradient(pair_wise_cosine_matrix * positive_mask) +
+                  1 + margin)
+  an = tf.nn.relu(
+      tf.stop_gradient(pair_wise_cosine_matrix * negative_mask) + margin)
+
+  logit_p = -ap * (pair_wise_cosine_matrix -
+                   delta_p) * gamma * positive_mask - (1 - positive_mask) * 1e12
+  logit_n = an * (pair_wise_cosine_matrix -
+                  delta_n) * gamma * negative_mask - (1 - negative_mask) * 1e12
+
+  joint_neg_loss = tf.reduce_logsumexp(logit_n, axis=-1)
+  joint_pos_loss = tf.reduce_logsumexp(logit_p, axis=-1)
+  loss = tf.nn.softplus(joint_neg_loss + joint_pos_loss)
+  return tf.reduce_mean(loss)
+
+
+def get_anchor_positive_triplet_mask(labels, sessions=None):
+  """Return a 2D mask where mask[a, p] is 1.0 iff a and p are distinct and have same session and label.
+
+  Args:
+    labels: a `Tensor` with shape [batch_size]
+    sessions: a `Tensor` with shape [batch_size]
+
+  Returns:
+    mask: tf.float32 `Tensor` with shape [batch_size, batch_size]
+  """
+  # Check that i and j are distinct
+  indices_equal = tf.cast(tf.eye(tf.shape(labels)[0]), tf.bool)
+  indices_not_equal = tf.logical_not(indices_equal)
+
+  # Check if labels[i] == labels[j]
+  # Uses broadcasting where the 1st argument has shape (1, batch_size) and the 2nd (batch_size, 1)
+  labels_equal = tf.equal(tf.expand_dims(labels, 0), tf.expand_dims(labels, 1))
+
+  # Check if labels[i] == labels[j]
+  if sessions is None or sessions is labels:
+    class_equal = labels_equal
+  else:
+    sessions_equal = tf.equal(
+        tf.expand_dims(sessions, 0), tf.expand_dims(sessions, 1))
+    class_equal = tf.logical_and(sessions_equal, labels_equal)
+
+  # Combine the three masks
+  mask = tf.logical_and(indices_not_equal, class_equal)
+  return tf.cast(mask, tf.float32)

easy_rec/python/loss/contrastive_loss.py

@@ -0,0 +1,79 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import tensorflow as tf
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def l2_loss(x1, x2):
+  """Compute euclidean distance of two embeddings."""
+  distance = tf.reduce_sum(tf.square(x1 - x2), axis=-1)
+  return tf.reduce_mean(distance)
+
+
+def info_nce_loss(query, positive, temperature=0.1):
+  """Calculates the InfoNCE loss for self-supervised learning.
+
+  This contrastive loss enforces the embeddings of similar (positive) samples to be close
+      and those of different (negative) samples to be distant.
+  A query embedding is compared with one positive key and with one or more negative keys.
+
+  References:
+      https://arxiv.org/abs/1807.03748v2
+      https://arxiv.org/abs/2010.05113
+  """
+  # Check input dimensionality.
+  if query.shape.ndims != 2:
+    raise ValueError('<query> must have 2 dimensions.')
+  if positive.shape.ndims != 2:
+    raise ValueError('<positive> must have 2 dimensions.')
+  # Embedding vectors should have same number of components.
+  if query.shape[-1] != positive.shape[-1]:
+    raise ValueError(
+        'Vectors of <query> and <positive> should have the same number of components.'
+    )
+
+  # Negative keys are implicitly off-diagonal positive keys.
+
+  # Cosine between all combinations
+  logits = tf.matmul(query, positive, transpose_b=True)
+  logits /= temperature
+
+  # Positive keys are the entries on the diagonal
+  batch_size = tf.shape(query)[0]
+  labels = tf.range(batch_size)
+
+  return tf.losses.sparse_softmax_cross_entropy(labels, logits)
+
+
+def get_mask_matrix(batch_size):
+  mat = tf.ones((batch_size, batch_size), dtype=tf.bool)
+  diag = tf.zeros([batch_size], dtype=tf.bool)
+  mask = tf.linalg.set_diag(mat, diag)
+  mask = tf.tile(mask, [2, 2])
+  return mask
+
+
+def nce_loss(z_i, z_j, temperature=1.0):
+  """Contrastive nce loss for homogeneous embeddings.
+
+  Refer paper: Contrastive Learning for Sequential Recommendation
+  """
+  batch_size = tf.shape(z_i)[0]
+  N = 2 * batch_size
+  z = tf.concat((z_i, z_j), axis=0)
+  sim = tf.matmul(z, tf.transpose(z)) / temperature
+  sim_i_j = tf.matrix_diag_part(
+      tf.slice(sim, [batch_size, 0], [batch_size, batch_size]))
+  sim_j_i = tf.matrix_diag_part(
+      tf.slice(sim, [0, batch_size], [batch_size, batch_size]))
+  positive_samples = tf.reshape(tf.concat((sim_i_j, sim_j_i), axis=0), (N, 1))
+  mask = get_mask_matrix(batch_size)
+  negative_samples = tf.reshape(tf.boolean_mask(sim, mask), (N, -1))
+
+  labels = tf.zeros(N, dtype=tf.int32)
+  logits = tf.concat((positive_samples, negative_samples), axis=1)
+
+  loss = tf.losses.sparse_softmax_cross_entropy(labels, logits)
+  return loss

easy_rec/python/loss/f1_reweight_loss.py

@@ -0,0 +1,38 @@
+# coding=utf-8
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import tensorflow as tf
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def f1_reweight_sigmoid_cross_entropy(labels,
+                                      logits,
+                                      beta_square,
+                                      label_smoothing=0,
+                                      weights=None):
+  """Refer paper: Adaptive Scaling for Sparse Detection in Information Extraction."""
+  probs = tf.nn.sigmoid(logits)
+  if len(logits.shape.as_list()) == 1:
+    logits = tf.expand_dims(logits, -1)
+  if len(labels.shape.as_list()) == 1:
+    labels = tf.expand_dims(labels, -1)
+  labels = tf.to_float(labels)
+  batch_size = tf.shape(labels)[0]
+  batch_size_float = tf.to_float(batch_size)
+  num_pos = tf.reduce_sum(labels, axis=0)
+  num_neg = batch_size_float - num_pos
+  tp = tf.reduce_sum(probs, axis=0)
+  tn = batch_size_float - tp
+  neg_weight = tp / (beta_square * num_pos + num_neg - tn + 1e-8)
+  neg_weight_tile = tf.tile(tf.expand_dims(neg_weight, 0), [batch_size, 1])
+  final_weights = tf.where(
+      tf.equal(labels, 1.0), tf.ones_like(labels), neg_weight_tile)
+  if weights is not None:
+    weights = tf.cast(weights, tf.float32)
+    if len(weights.shape.as_list()) == 1:
+      weights = tf.expand_dims(weights, -1)
+    final_weights *= weights
+  return tf.losses.sigmoid_cross_entropy(
+      labels, logits, final_weights, label_smoothing=label_smoothing)

easy_rec/python/loss/focal_loss.py

@@ -0,0 +1,93 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import tensorflow as tf
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def sigmoid_focal_loss_with_logits(labels,
+                                   logits,
+                                   gamma=2.0,
+                                   alpha=None,
+                                   ohem_ratio=1.0,
+                                   sample_weights=None,
+                                   label_smoothing=0,
+                                   name=''):
+  """Implements the focal loss function.
+
+  Focal loss was first introduced in the RetinaNet paper
+  (https://arxiv.org/pdf/1708.02002.pdf). Focal loss is extremely useful for
+  classification when you have highly imbalanced classes. It down-weights
+  well-classified examples and focuses on hard examples. The loss value is
+  much high for a sample which is misclassified by the classifier as compared
+  to the loss value corresponding to a well-classified example. One of the
+  best use-cases of focal loss is its usage in object detection where the
+  imbalance between the background class and other classes is extremely high.
+
+  Args
+      labels: `[batch_size]` target integer labels in `{0, 1}`.
+      logits: Float `[batch_size]` logits outputs of the network.
+      alpha: balancing factor.
+      gamma: modulating factor.
+      ohem_ratio: the percent of hard examples to be mined
+      sample_weights:  Optional `Tensor` whose rank is either 0, or the same rank as
+        `labels`, and must be broadcastable to `labels` (i.e., all dimensions must
+        be either `1`, or the same as the corresponding `losses` dimension).
+      label_smoothing: If greater than `0` then smooth the labels.
+      name: the name of loss
+
+  Returns:
+      Weighted loss float `Tensor`. If `reduction` is `NONE`,this has the
+      same shape as `y_true`; otherwise, it is scalar.
+
+  Raises:
+      ValueError: If the shape of `sample_weight` is invalid or value of
+        `gamma` is less than zero
+  """
+  loss_name = name if name else 'focal_loss'
+  assert 0 < ohem_ratio <= 1.0, loss_name + ' ohem_ratio must be in (0, 1]'
+  if gamma and gamma < 0:
+    raise ValueError('Value of gamma should be greater than or equal to zero')
+  logging.info(
+      '[{}] gamma: {}, alpha: {}, ohem_ratho: {}, label smoothing: {}'.format(
+          loss_name, gamma, alpha, ohem_ratio, label_smoothing))
+
+  y_true = tf.cast(labels, logits.dtype)
+
+  # convert the predictions into probabilities
+  y_pred = tf.nn.sigmoid(logits)
+  epsilon = 1e-7
+  y_pred = tf.clip_by_value(y_pred, epsilon, 1 - epsilon)
+  p_t = (y_true * y_pred) + ((1 - y_true) * (1 - y_pred))
+  weights = tf.pow((1 - p_t), gamma)
+
+  if alpha is not None:
+    alpha_factor = y_true * alpha + ((1 - alpha) * (1 - y_true))
+    weights *= alpha_factor
+
+  if sample_weights is not None:
+    if tf.is_numeric_tensor(sample_weights):
+      logging.info('[%s] use sample weight' % loss_name)
+      weights *= tf.cast(sample_weights, tf.float32)
+    elif sample_weights != 1.0:
+      logging.info('[%s] use sample weight: %f' % (loss_name, sample_weights))
+      weights *= sample_weights
+
+  if ohem_ratio == 1.0:
+    return tf.losses.sigmoid_cross_entropy(
+        y_true, logits, weights=weights, label_smoothing=label_smoothing)
+
+  losses = tf.losses.sigmoid_cross_entropy(
+      y_true,
+      logits,
+      weights=weights,
+      label_smoothing=label_smoothing,
+      reduction=tf.losses.Reduction.NONE)
+  k = tf.to_float(tf.size(losses)) * tf.convert_to_tensor(ohem_ratio)
+  k = tf.to_int32(tf.math.rint(k))
+  topk = tf.nn.top_k(losses, k)
+  losses = tf.boolean_mask(topk.values, topk.values > 0)
+  return tf.reduce_mean(losses)

easy_rec/python/loss/jrc_loss.py

@@ -0,0 +1,128 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import numpy as np
+import tensorflow as tf
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def jrc_loss(labels,
+             logits,
+             session_ids,
+             alpha=0.5,
+             loss_weight_strategy='fixed',
+             sample_weights=1.0,
+             same_label_loss=True,
+             name=''):
+  """Joint Optimization of Ranking and Calibration with Contextualized Hybrid Model.
+
+     https://arxiv.org/abs/2208.06164
+
+  Args:
+    labels: a `Tensor` with shape [batch_size]. e.g. click or not click in the session.
+    logits: a `Tensor` with shape [batch_size, 2]. e.g. the value of last neuron before activation.
+    session_ids: a `Tensor` with shape [batch_size]. Session ids of each sample, used to max GAUC metric. e.g. user_id
+    alpha: the weight to balance ranking loss and calibration loss
+    loss_weight_strategy: str, the loss weight strategy to balancing between ce_loss and ge_loss
+    sample_weights: Coefficients for the loss. This must be scalar or broadcastable to
+      `labels` (i.e. same rank and each dimension is either 1 or the same).
+    same_label_loss: enable ge_loss for sample with same label in a session or not.
+    name: the name of loss
+  """
+  loss_name = name if name else 'jrc_loss'
+  logging.info('[{}] alpha: {}, loss_weight_strategy: {}'.format(
+      loss_name, alpha, loss_weight_strategy))
+
+  ce_loss = tf.losses.sparse_softmax_cross_entropy(
+      labels, logits, weights=sample_weights)
+
+  labels = tf.expand_dims(labels, 1)  # [B, 1]
+  labels = tf.concat([1 - labels, labels], axis=1)  # [B, 2]
+
+  batch_size = tf.shape(logits)[0]
+
+  # Mask: shape [B, B], mask[i,j]=1 indicates the i-th sample
+  # and j-th sample are in the same context
+  mask = tf.equal(
+      tf.expand_dims(session_ids, 1), tf.expand_dims(session_ids, 0))
+  mask = tf.to_float(mask)
+
+  # Tile logits and label: [B, 2]->[B, B, 2]
+  logits = tf.tile(tf.expand_dims(logits, 1), [1, batch_size, 1])
+  y = tf.tile(tf.expand_dims(labels, 1), [1, batch_size, 1])
+
+  # Set logits that are not in the same context to -inf
+  mask3d = tf.expand_dims(mask, 2)
+  y = tf.to_float(y) * mask3d
+  logits = logits + (1 - mask3d) * -1e9
+  y_neg, y_pos = y[:, :, 0], y[:, :, 1]
+  l_neg, l_pos = logits[:, :, 0], logits[:, :, 1]
+
+  if tf.is_numeric_tensor(sample_weights):
+    logging.info('[%s] use sample weight' % loss_name)
+    weights = tf.expand_dims(tf.cast(sample_weights, tf.float32), 0)
+    pairwise_weights = tf.tile(weights, tf.stack([batch_size, 1]))
+    y_pos *= pairwise_weights
+    y_neg *= pairwise_weights
+
+  # Compute list-wise generative loss -log p(x|y, z)
+  if same_label_loss:
+    logging.info('[%s] enable same_label_loss' % loss_name)
+    loss_pos = -tf.reduce_sum(y_pos * tf.nn.log_softmax(l_pos, axis=0), axis=0)
+    loss_neg = -tf.reduce_sum(y_neg * tf.nn.log_softmax(l_neg, axis=0), axis=0)
+    ge_loss = tf.reduce_mean(
+        (loss_pos + loss_neg) / tf.reduce_sum(mask, axis=0))
+  else:
+    logging.info('[%s] disable same_label_loss' % loss_name)
+    diag = tf.one_hot(tf.range(batch_size), batch_size)
+    l_pos = l_pos + (1 - diag) * y_pos * -1e9
+    l_neg = l_neg + (1 - diag) * y_neg * -1e9
+    loss_pos = -tf.linalg.diag_part(y_pos * tf.nn.log_softmax(l_pos, axis=0))
+    loss_neg = -tf.linalg.diag_part(y_neg * tf.nn.log_softmax(l_neg, axis=0))
+    ge_loss = tf.reduce_mean(loss_pos + loss_neg)
+
+  tf.summary.scalar('loss/%s_ce' % loss_name, ce_loss)
+  tf.summary.scalar('loss/%s_ge' % loss_name, ge_loss)
+
+  # The final JRC model
+  if loss_weight_strategy == 'fixed':
+    loss = alpha * ce_loss + (1 - alpha) * ge_loss
+  elif loss_weight_strategy == 'random_uniform':
+    weight = tf.random_uniform([])
+    loss = weight * ce_loss + (1 - weight) * ge_loss
+    tf.summary.scalar('loss/%s_ce_weight' % loss_name, weight)
+    tf.summary.scalar('loss/%s_ge_weight' % loss_name, 1 - weight)
+  elif loss_weight_strategy == 'random_normal':
+    weights = tf.random_normal([2])
+    loss_weight = tf.nn.softmax(weights)
+    loss = loss_weight[0] * ce_loss + loss_weight[1] * ge_loss
+    tf.summary.scalar('loss/%s_ce_weight' % loss_name, loss_weight[0])
+    tf.summary.scalar('loss/%s_ge_weight' % loss_name, loss_weight[1])
+  elif loss_weight_strategy == 'random_bernoulli':
+    bern = tf.distributions.Bernoulli(probs=0.5, dtype=tf.float32)
+    weights = bern.sample(2)
+    loss_weight = tf.cond(
+        tf.equal(tf.reduce_sum(weights), 1), lambda: weights,
+        lambda: tf.convert_to_tensor([0.5, 0.5]))
+    loss = loss_weight[0] * ce_loss + loss_weight[1] * ge_loss
+    tf.summary.scalar('loss/%s_ce_weight' % loss_name, loss_weight[0])
+    tf.summary.scalar('loss/%s_ge_weight' % loss_name, loss_weight[1])
+  elif loss_weight_strategy == 'uncertainty':
+    uncertainty1 = tf.Variable(
+        0, name='%s_ranking_loss_weight' % loss_name, dtype=tf.float32)
+    tf.summary.scalar('loss/%s_ranking_uncertainty' % loss_name, uncertainty1)
+    uncertainty2 = tf.Variable(
+        0, name='%s_calibration_loss_weight' % loss_name, dtype=tf.float32)
+    tf.summary.scalar('loss/%s_calibration_uncertainty' % loss_name,
+                      uncertainty2)
+    loss = tf.exp(-uncertainty1) * ce_loss + 0.5 * uncertainty1
+    loss += tf.exp(-uncertainty2) * ge_loss + 0.5 * uncertainty2
+  else:
+    raise ValueError('Unsupported loss weight strategy `%s` for jrc loss' %
+                     loss_weight_strategy)
+  if np.isscalar(sample_weights) and sample_weights != 1.0:
+    return loss * sample_weights
+  return loss

easy_rec/python/loss/listwise_loss.py

@@ -0,0 +1,161 @@
+# -*- encoding:utf-8 -*-
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import logging
+
+import tensorflow as tf
+
+from easy_rec.python.utils.load_class import load_by_path
+
+
+def _list_wise_loss(x, labels, logits, session_ids, label_is_logits):
+  mask = tf.equal(x, session_ids)
+  logits = tf.boolean_mask(logits, mask)
+  labels = tf.boolean_mask(labels, mask)
+  y = tf.nn.softmax(labels) if label_is_logits else labels
+  y_hat = tf.nn.log_softmax(logits)
+  return -tf.reduce_sum(y * y_hat)
+
+
+def _list_prob_loss(x, labels, logits, session_ids):
+  mask = tf.equal(x, session_ids)
+  logits = tf.boolean_mask(logits, mask)
+  labels = tf.boolean_mask(labels, mask)
+  y = labels / tf.reduce_sum(labels)
+  y_hat = tf.nn.log_softmax(logits)
+  return -tf.reduce_sum(y * y_hat)
+
+
+def listwise_rank_loss(labels,
+                       logits,
+                       session_ids,
+                       transform_fn=None,
+                       temperature=1.0,
+                       label_is_logits=False,
+                       scale_logits=False,
+                       weights=1.0,
+                       name='listwise_loss'):
+  r"""Computes listwise softmax cross entropy loss between `labels` and `logits`.
+
+  Definition:
+  $$
+  \mathcal{L}(\{y\}, \{s\}) =
+  \sum_i y_j \log( \frac{\exp(s_i)}{\sum_j exp(s_j)} )
+  $$
+
+  Args:
+    labels: A `Tensor` of the same shape as `logits` representing graded
+      relevance.
+    logits: A `Tensor` with shape [batch_size].
+    session_ids: a `Tensor` with shape [batch_size]. Session ids of each sample, used to max GAUC metric. e.g. user_id
+    transform_fn: an affine transformation function of labels
+    temperature: (Optional) The temperature to use for scaling the logits.
+    label_is_logits: Whether `labels` is expected to be a logits tensor.
+          By default, we consider that `labels` encodes a probability distribution.
+    scale_logits: Whether to scale the logits.
+    weights: sample weights
+    name: the name of loss
+  """
+  loss_name = name if name else 'listwise_rank_loss'
+  logging.info('[{}] temperature: {}, scale logits: {}'.format(
+      loss_name, temperature, scale_logits))
+  labels = tf.to_float(labels)
+  if scale_logits:
+    with tf.variable_scope(loss_name):
+      w = tf.get_variable(
+          'scale_w',
+          dtype=tf.float32,
+          shape=(1,),
+          initializer=tf.ones_initializer())
+      b = tf.get_variable(
+          'scale_b',
+          dtype=tf.float32,
+          shape=(1,),
+          initializer=tf.zeros_initializer())
+    logits = logits * tf.abs(w) + b
+  if temperature != 1.0:
+    logits /= temperature
+    if label_is_logits:
+      labels /= temperature
+  if transform_fn is not None:
+    trans_fn = load_by_path(transform_fn)
+    labels = trans_fn(labels)
+
+  sessions, _ = tf.unique(tf.squeeze(session_ids))
+  tf.summary.scalar('loss/%s_num_of_group' % loss_name, tf.size(sessions))
+  losses = tf.map_fn(
+      lambda x: _list_wise_loss(x, labels, logits, session_ids, label_is_logits
+                                ),
+      sessions,
+      dtype=tf.float32)
+  if tf.is_numeric_tensor(weights):
+    logging.error('[%s] use unsupported sample weight' % loss_name)
+    return tf.reduce_mean(losses)
+  else:
+    return tf.reduce_mean(losses) * weights
+
+
+def listwise_distill_loss(labels,
+                          logits,
+                          session_ids,
+                          transform_fn=None,
+                          temperature=1.0,
+                          label_clip_max_value=512,
+                          scale_logits=False,
+                          weights=1.0,
+                          name='listwise_distill_loss'):
+  r"""Computes listwise softmax cross entropy loss between `labels` and `logits`.
+
+  Definition:
+  $$
+  \mathcal{L}(\{y\}, \{s\}) =
+  \sum_i y_j \log( \frac{\exp(s_i)}{\sum_j exp(s_j)} )
+  $$
+
+  Args:
+    labels: A `Tensor` of the same shape as `logits` representing the rank position of a base model.
+    logits: A `Tensor` with shape [batch_size].
+    session_ids: a `Tensor` with shape [batch_size]. Session ids of each sample, used to max GAUC metric. e.g. user_id
+    transform_fn: an transformation function of labels.
+    temperature: (Optional) The temperature to use for scaling the logits.
+    label_clip_max_value: clip the labels to this value.
+    scale_logits: Whether to scale the logits.
+    weights: sample weights
+    name: the name of loss
+  """
+  loss_name = name if name else 'listwise_rank_loss'
+  logging.info('[{}] temperature: {}'.format(loss_name, temperature))
+  labels = tf.to_float(labels)  # supposed to be positions of a teacher model
+  labels = tf.clip_by_value(labels, 1, label_clip_max_value)
+  if transform_fn is not None:
+    trans_fn = load_by_path(transform_fn)
+    labels = trans_fn(labels)
+  else:
+    labels = tf.log1p(label_clip_max_value) - tf.log(labels)
+
+  if scale_logits:
+    with tf.variable_scope(loss_name):
+      w = tf.get_variable(
+          'scale_w',
+          dtype=tf.float32,
+          shape=(1,),
+          initializer=tf.ones_initializer())
+      b = tf.get_variable(
+          'scale_b',
+          dtype=tf.float32,
+          shape=(1,),
+          initializer=tf.zeros_initializer())
+    logits = logits * tf.abs(w) + b
+  if temperature != 1.0:
+    logits /= temperature
+
+  sessions, _ = tf.unique(tf.squeeze(session_ids))
+  tf.summary.scalar('loss/%s_num_of_group' % loss_name, tf.size(sessions))
+  losses = tf.map_fn(
+      lambda x: _list_prob_loss(x, labels, logits, session_ids),
+      sessions,
+      dtype=tf.float32)
+  if tf.is_numeric_tensor(weights):
+    logging.error('[%s] use unsupported sample weight' % loss_name)
+    return tf.reduce_mean(losses)
+  else:
+    return tf.reduce_mean(losses) * weights

easy_rec/python/loss/multi_similarity.py

@@ -0,0 +1,68 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import tensorflow as tf
+
+from easy_rec.python.loss.circle_loss import get_anchor_positive_triplet_mask
+from easy_rec.python.utils.shape_utils import get_shape_list
+
+if tf.__version__ >= '2.0':
+  tf = tf.compat.v1
+
+
+def ms_loss(embeddings,
+            labels,
+            session_ids=None,
+            alpha=2.0,
+            beta=50.0,
+            lamb=1.0,
+            eps=0.1,
+            ms_mining=False,
+            embed_normed=False):
+  """Refer paper: Multi-Similarity Loss with General Pair Weighting for Deep Metric Learning.
+
+  ref: http://openaccess.thecvf.com/content_CVPR_2019/papers/
+    Wang_Multi-Similarity_Loss_With_General_Pair_Weighting_for_Deep_Metric_Learning_CVPR_2019_paper.pdf
+  """
+  # make sure embedding should be l2-normalized
+  if not embed_normed:
+    embeddings = tf.nn.l2_normalize(embeddings, axis=1)
+  labels = tf.reshape(labels, [-1, 1])
+
+  embed_shape = get_shape_list(embeddings)
+  batch_size = embed_shape[0]
+
+  mask_pos = get_anchor_positive_triplet_mask(labels, session_ids)
+  mask_neg = 1 - mask_pos - tf.eye(batch_size)
+
+  sim_mat = tf.matmul(
+      embeddings, embeddings, transpose_a=False, transpose_b=True)
+  sim_mat = tf.maximum(sim_mat, 0.0)
+
+  pos_mat = tf.multiply(sim_mat, mask_pos)
+  neg_mat = tf.multiply(sim_mat, mask_neg)
+
+  if ms_mining:
+    max_val = tf.reduce_max(neg_mat, axis=1, keepdims=True)
+    tmp_max_val = tf.reduce_max(pos_mat, axis=1, keepdims=True)
+    min_val = tf.reduce_min(
+        tf.multiply(sim_mat - tmp_max_val, mask_pos), axis=1,
+        keepdims=True) + tmp_max_val
+
+    max_val = tf.tile(max_val, [1, batch_size])
+    min_val = tf.tile(min_val, [1, batch_size])
+
+    mask_pos = tf.where(pos_mat < max_val + eps, mask_pos,
+                        tf.zeros_like(mask_pos))
+    mask_neg = tf.where(neg_mat > min_val - eps, mask_neg,
+                        tf.zeros_like(mask_neg))
+
+  pos_exp = tf.exp(-alpha * (pos_mat - lamb))
+  pos_exp = tf.where(mask_pos > 0.0, pos_exp, tf.zeros_like(pos_exp))
+
+  neg_exp = tf.exp(beta * (neg_mat - lamb))
+  neg_exp = tf.where(mask_neg > 0.0, neg_exp, tf.zeros_like(neg_exp))
+
+  pos_term = tf.log(1.0 + tf.reduce_sum(pos_exp, axis=1)) / alpha
+  neg_term = tf.log(1.0 + tf.reduce_sum(neg_exp, axis=1)) / beta
+
+  loss = tf.reduce_mean(pos_term + neg_term)
+  return loss