python-ldl 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

pyldl/algorithms/__init__.py

@@ -0,0 +1,30 @@
+from ._problem_transformation import PT_Bayes, PT_SVM, LDSVR
+from ._algorithm_adaptation import AA_BP, AA_KNN, CAD, QFD2, CJS, CPNN, BCPNN, ACPNN
+from ._specialized_algorithms import SA_BFGS, SA_IIS
+
+from ._incomplete import IncomLDL
+from ._classifier import LDL4C, LDL_HR, LDLM
+from ._ensemble import DF_LDL, AdaBoostLDL
+
+from ._ldlf import LDLF
+from ._ldl_scl import LDL_SCL
+from ._ldl_lrr import LDL_LRR
+from ._ldl_dpa import LDL_DPA
+
+from ._ssg_ldl import SSG_LDL
+
+from ._label_enhancement import FCM, KM, LP, ML, GLLE, LEVI, LIBLE
+
+from ._ldl_da import LDL_DA
+
+
+__all__ = ["SA_BFGS", "SA_IIS", "AA_KNN", "AA_BP", "PT_Bayes", "PT_SVM",
+           "CPNN", "BCPNN", "ACPNN", "LDSVR",
+           "LDLF", "LDL_SCL", "LDL_LRR", "LDL_DPA", "CAD", "QFD2", "CJS",
+           "DF_LDL", "AdaBoostLDL",
+           "LDL4C", "LDL_HR", "LDLM",
+           "IncomLDL",
+           "SSG_LDL",
+           "FCM", "KM", "LP", "ML", "GLLE",
+           "LEVI", "LIBLE",
+           "LDL_DA"]

pyldl/algorithms/_algorithm_adaptation.py

@@ -0,0 +1,198 @@
+import numpy as np
+from sklearn.neighbors import NearestNeighbors
+
+import keras
+import tensorflow as tf
+from keras import backend as K
+
+from pyldl.algorithms.base import BaseLDL, BaseDeepLDL, BaseGD, BaseAdam
+
+
+class AA_KNN(BaseLDL):
+
+    def fit(self, X, y, k=5):
+        super().fit(X, y)
+        self._model = NearestNeighbors(n_neighbors=k).fit(self._X)
+        return self
+
+    def predict(self, X):
+        _, inds = self._model.kneighbors(X)
+        return np.average(self._y[inds], axis=1)
+
+
+class AA_BP(BaseGD, BaseDeepLDL):
+    pass
+
+
+class CAD(BaseAdam, BaseDeepLDL):
+
+    @staticmethod
+    @tf.function
+    def loss_function(y, y_pred):
+        def _CAD(y, y_pred):
+            return tf.reduce_mean(tf.abs(
+                tf.cumsum(y, axis=1) - tf.cumsum(y_pred, axis=1)
+            ), axis=1)
+        return tf.math.reduce_sum(
+            tf.map_fn(lambda i: _CAD(y[:, :i], y_pred[:, :i]),
+                      tf.range(1, y.shape[1] + 1),
+                      fn_output_signature=tf.float32)
+        )
+
+
+class QFD2(BaseAdam, BaseDeepLDL):
+
+    @staticmethod
+    @tf.function
+    def _loss_function(y, y_pred):
+        Q = y - y_pred
+        j = tf.reshape(tf.range(y.shape[1]), [y.shape[1], 1])
+        k = tf.reshape(tf.range(y.shape[1]), [1, y.shape[1]])
+        A = tf.cast(1 - tf.abs(j - k) / (y.shape[1] - 1), dtype=tf.float32)
+        return tf.math.reduce_mean(
+            tf.linalg.diag_part(tf.matmul(tf.matmul(Q, A), tf.transpose(Q)))
+        )
+
+
+class CJS(BaseAdam, BaseDeepLDL):
+
+    @staticmethod
+    @tf.function
+    def loss_function(y, y_pred):
+        def _CJS(y, y_pred):
+            m = 0.5 * (y + y_pred)
+            js = 0.5 * (keras.losses.kl_divergence(y, m) + keras.losses.kl_divergence(y_pred, m))
+            return tf.reduce_mean(js)
+        return tf.math.reduce_sum(
+            tf.map_fn(lambda i: _CJS(y[:, :i], y_pred[:, :i]),
+                      tf.range(1, y.shape[1] + 1),
+                      fn_output_signature=tf.float32)
+        )
+
+
+class RProp(keras.optimizers.Optimizer):
+    
+    def __init__(self, init_alpha=1e-3, scale_up=1.2, scale_down=0.5, min_alpha=1e-6, max_alpha=50., **kwargs):
+        super(RProp, self).__init__(name='rprop', **kwargs)
+        self.init_alpha = K.variable(init_alpha, name='init_alpha')
+        self.scale_up = K.variable(scale_up, name='scale_up')
+        self.scale_down = K.variable(scale_down, name='scale_down')
+        self.min_alpha = K.variable(min_alpha, name='min_alpha')
+        self.max_alpha = K.variable(max_alpha, name='max_alpha')
+
+    def apply_gradients(self, grads_and_vars):
+        grads, trainable_variables = zip(*grads_and_vars)
+        self.get_updates(trainable_variables, grads)
+
+    def get_updates(self, params, gradients):
+        grads = gradients
+        shapes = [K.int_shape(p) for p in params]
+        alphas = [K.variable(np.ones(shape) * self.init_alpha) for shape in shapes]
+        old_grads = [K.zeros(shape) for shape in shapes]
+        prev_weight_deltas = [K.zeros(shape) for shape in shapes]
+        self.updates = []
+
+        for param, grad, old_grad, prev_weight_delta, alpha in zip(params, grads,
+                                                                   old_grads, prev_weight_deltas,
+                                                                   alphas):
+
+            new_alpha = K.switch(
+                K.greater(grad * old_grad, 0),
+                K.minimum(alpha * self.scale_up, self.max_alpha),
+                K.switch(K.less(grad * old_grad, 0), K.maximum(alpha * self.scale_down, self.min_alpha), alpha)
+            )
+
+            new_delta = K.switch(K.greater(grad, 0),
+                                 -new_alpha,
+                                 K.switch(K.less(grad, 0),
+                                          new_alpha,
+                                          K.zeros_like(new_alpha)))
+
+            weight_delta = K.switch(K.less(grad*old_grad, 0), -prev_weight_delta, new_delta)
+
+            new_param = param + weight_delta
+
+            grad = K.switch(K.less(grad*old_grad, 0), K.zeros_like(grad), grad)
+
+            self.updates.append(K.update(param, new_param))
+            self.updates.append(K.update(alpha, new_alpha))
+            self.updates.append(K.update(old_grad, grad))
+            self.updates.append(K.update(prev_weight_delta, weight_delta))
+
+        return self.updates
+
+    def get_config(self):
+        config = {
+            'init_alpha': float(K.get_value(self.init_alpha)),
+            'scale_up': float(K.get_value(self.scale_up)),
+            'scale_down': float(K.get_value(self.scale_down)),
+            'min_alpha': float(K.get_value(self.min_alpha)),
+            'max_alpha': float(K.get_value(self.max_alpha)),
+        }
+        base_config = super(RProp, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+class CPNN(BaseGD, BaseDeepLDL):
+
+    def _not_proper_mode(self):
+        raise ValueError("The argument 'mode' can only be 'none', 'binary' or 'augment'.")
+
+    def __init__(self, mode='none', v=5, n_hidden=64, n_latent=None, random_state=None):
+        super().__init__(n_hidden, n_latent, random_state)
+        if mode == 'none' or mode == 'binary' or mode == 'augment':
+            self._mode = mode
+        else:
+            self._not_proper_mode()
+        self._v = v
+
+    @staticmethod
+    @tf.function
+    def loss_function(y, y_pred):
+        return tf.math.reduce_mean(keras.losses.kl_divergence(y, y_pred))
+
+    def _get_default_model(self):
+        input_shape = (self._n_features + (1 if self._mode == 'none' else self._n_outputs),)
+        return keras.Sequential([keras.layers.InputLayer(input_shape=input_shape),
+                                 keras.layers.Dense(self._n_hidden, activation='sigmoid'),
+                                 keras.layers.Dense(1, activation=None)])
+
+    def _get_default_optimizer(self):
+        return RProp()
+
+    def _before_train(self):
+        if self._mode == 'augment':
+            n = self._X.shape[0]
+            one_hot = tf.one_hot(tf.math.argmax(self._y, axis=1), self.n_outputs)
+            self._X = tf.repeat(self._X, self._v, axis=0)
+            self._y = tf.repeat(self._y, self._v, axis=0)
+            one_hot = tf.repeat(one_hot, self._v, axis=0)
+            v = tf.reshape(tf.tile([1 / (i + 1) for i in range(self._v)], [n]), (-1, 1))
+            self._y += self._y * one_hot * v
+
+    def _make_inputs(self, X):
+        temp = tf.reshape(tf.tile([i + 1 for i in range(self._n_outputs)], [X.shape[0]]), (-1, 1))
+        if self._mode != 'none':
+            temp = tf.one_hot(tf.reshape(temp, (-1, )) - 1, depth=self._n_outputs)
+        return tf.concat([tf.cast(tf.repeat(X, self._n_outputs, axis=0), dtype=tf.float32),
+                          tf.cast(temp, dtype=tf.float32)],
+                          axis=1)
+
+    def _call(self, X):
+        inputs = self._make_inputs(X)
+        outputs = self._model(inputs)
+        results = tf.reshape(outputs, (X.shape[0], self._n_outputs))
+        b = tf.reshape(-tf.math.log(tf.math.reduce_sum(tf.math.exp(results), axis=1)), (-1, 1))
+        return tf.math.exp(b + results)
+
+
+class BCPNN(CPNN):
+
+    def __init__(self, **params):
+        super().__init__(mode='binary', **params)
+
+
+class ACPNN(CPNN):
+
+    def __init__(self, **params):
+        super().__init__(mode='augment', **params)

pyldl/algorithms/_classifier.py

@@ -0,0 +1,105 @@
+import keras
+import tensorflow as tf
+
+from pyldl.algorithms.base import BaseDeepLDLClassifier, BaseGD, BaseBFGS
+
+
+class LDL4C(BaseBFGS, BaseDeepLDLClassifier):
+    """LDL4C is proposed in paper Classification with Label Distribution Learning.
+    """
+
+    @tf.function
+    def _loss(self, params_1d):
+        y_pred = keras.activations.softmax(self._X @ self._params2model(params_1d)[0])
+        top2 = tf.gather(y_pred, self._top2, axis=1, batch_dims=1)
+        margin = tf.reduce_sum(tf.maximum(0., 1. - (top2[:, 0] - top2[:, 1]) / self._rho))
+        mae = keras.losses.mean_absolute_error(self._y, y_pred)
+        return tf.reduce_sum(self._entropy * mae) + self._alpha * margin + self._beta * self._l2_reg(self._model)
+
+    def _before_train(self):
+        self._top2 = tf.math.top_k(self._y, k=2)[1]
+        self._entropy = tf.cast(-tf.reduce_sum(self._y * tf.math.log(self._y) + 1e-7, axis=1), dtype=tf.float32)
+
+    def fit(self, X, y, alpha=1e-2, beta=1e-6, rho=1e-2, **kwargs):
+        self._alpha = alpha
+        self._beta = beta
+        self._rho = rho
+        return super().fit(X, y, **kwargs)
+
+
+class LDL_HR(BaseBFGS, BaseDeepLDLClassifier):
+    """LDL-HR is proposed in paper Learn the Highest Label and Rest Label Description Degrees.
+    """
+
+    @tf.function
+    def _loss(self, params_1d):
+        y_pred = keras.activations.softmax(self._X @ self._params2model(params_1d)[0])
+
+        highest = tf.gather(y_pred, self._highest, axis=1, batch_dims=1)
+        rest = tf.gather(y_pred, self._rest, axis=1, batch_dims=1)
+        margin = tf.reduce_sum(tf.maximum(0., 1. - (highest - rest) / self._rho))
+
+        real_rest = tf.gather(self._y, self._rest, axis=1, batch_dims=1)
+        rest_mae = tf.reduce_sum(keras.losses.mean_absolute_error(real_rest, rest))
+
+        mae = tf.reduce_sum(keras.losses.mean_absolute_error(self._l, y_pred))
+
+        return mae + self._alpha * margin + self._beta * rest_mae + self._gamma * self._l2_reg(self._model)
+
+    def _before_train(self):
+        temp = tf.math.top_k(self._y, k=self._n_outputs)[1]
+        self._highest = temp[:, 0:1]
+        self._rest = temp[:, 1:]
+        self._l = tf.one_hot(tf.reshape(self._highest, -1), self._n_outputs)
+
+    def fit(self, X, y, alpha=1e-2, beta=1e-2, gamma=1e-6, rho=1e-2, **kwargs):
+        self._alpha = alpha
+        self._beta = beta
+        self._gamma = gamma
+        self._rho = rho
+        return super().fit(X, y, **kwargs) 
+
+
+class LDLM(BaseGD, BaseDeepLDLClassifier):
+    """LDLM is proposed in paper Label Distribution Learning Machine.
+    """
+
+    @tf.function
+    def _loss(self, X, y, start, end):
+        y_pred = self._model(X)
+
+        pred_margin = tf.reduce_sum(tf.clip_by_value(
+            tf.reduce_sum(tf.abs(self._l[start:end] - y_pred), axis=1) - self._rho,
+            0., float('inf')))
+
+        highest = tf.gather(y_pred, self._highest[start:end], axis=1, batch_dims=1)
+        rest = tf.gather(y_pred, self._rest[start:end], axis=1, batch_dims=1)
+        label_margin = tf.reduce_sum(tf.maximum(0., 1. - (highest - rest) / self._rho))
+
+        second_margin = tf.reduce_sum(tf.clip_by_value(
+            tf.reduce_sum(tf.abs((y - y_pred) * self._neg_l[start:end]), axis=1) - self._second_margin[start:end],
+            0., float('inf')))
+
+        return pred_margin + self._alpha * label_margin + \
+            self._beta * second_margin + self._gamma * self._l2_reg(self._model)
+
+    def _before_train(self):
+        temp = tf.math.top_k(self._y, k=self._n_outputs)[1]
+        self._highest = temp[:, 0:1]
+        self._rest = temp[:, 1:]
+
+        temp = tf.math.top_k(tf.gather(self._y, self._rest, axis=1, batch_dims=1), k=2)[0]
+        self._second_margin = temp[:, 0] - temp[:, 1]
+
+        self._l = tf.one_hot(tf.reshape(self._highest, -1), self._n_outputs)
+        self._neg_l = tf.where(tf.equal(self._l, 0.), 1., 0.)
+
+    def _get_default_model(self):
+        return self.get_2layer_model(self._n_features, self._n_outputs)
+
+    def fit(self, X, y, alpha=1e-2, beta=1e-2, gamma=1e-6, rho=1e-2, **kwargs):
+        self._alpha = alpha
+        self._beta = beta
+        self._gamma = gamma
+        self._rho = rho
+        return super().fit(X, y, **kwargs)

pyldl/algorithms/_ensemble.py

@@ -0,0 +1,99 @@
+import copy
+
+import numpy as np
+
+from pyldl.algorithms.base import BaseEnsemble
+from pyldl.metrics import sort_loss
+from ._algorithm_adaptation import AA_KNN
+from ._specialized_algorithms import SA_BFGS
+
+
+class DF_LDL(BaseEnsemble):
+
+    def __init__(self, estimator=SA_BFGS(), random_state=None):
+        super().__init__(estimator, None, random_state)
+
+    def fit(self, X, y):
+        super().fit(X, y)
+
+        m, c = self._y.shape[0], self._y.shape[1]
+        L = {}
+
+        for i in range(c):
+            for j in range(i + 1, c):
+
+                ss1 = []
+                ss2 = []
+
+                for k in range(m):
+                    if self._y[k, i] >= self._y[k, j]:
+                        ss1.append(k)
+                    else:
+                        ss2.append(k)
+
+                l1 = copy.deepcopy(self._estimator)
+                l1.fit(self._X[ss1], self._y[ss1])
+                L[str(i)+","+str(j)] = copy.deepcopy(l1)
+
+                l2 = copy.deepcopy(self._estimator)
+                l2.fit(self._X[ss2], self._y[ss2])
+                L[str(j)+","+str(i)] = copy.deepcopy(l2)
+
+        self._estimators = L
+
+        self._knn = AA_KNN()
+        self._knn.fit(self._X, self._y)
+
+    def predict(self, X):
+
+        m, c = X.shape[0], self._y.shape[1]
+        p_knn = self._knn.predict(X)
+        p = np.zeros((m, c), dtype=np.float32)
+
+        for k in range(m):
+            for i in range(c):
+                for j in range(i + 1, c):
+
+                    if p_knn[k, i] >= p_knn[k, j]:
+                        l = self._estimators[str(i)+","+str(j)]
+                    else:
+                        l = self._estimators[str(j)+","+str(i)]
+
+                    p[k] += l.predict(X[k].reshape(1, -1)).reshape(-1)
+
+        return p / (c * (c - 1) / 2)
+
+
+class AdaBoostLDL(BaseEnsemble):
+
+    def __init__(self, estimator=SA_BFGS(), n_estimators=10, random_state=None):
+        super().__init__(estimator, n_estimators, random_state)
+
+    def fit(self, X, y, loss=sort_loss, alpha=1.):
+        super().fit(X, y)
+
+        m = self._X.shape[0]
+        p = np.ones((m,)) / m
+
+        self._loss = np.zeros((self._n_estimators, m))
+        self._estimators = []
+        for i in range(self._n_estimators):
+            select = np.random.choice(m, size=m, p=p)
+            X_train, y_train = self._X[select], self._y[select]
+
+            model = copy.deepcopy(self._estimator)
+            model.fit(X_train, y_train)
+            self._estimators.append(copy.deepcopy(model))
+
+            y_pred = model.predict(self._X)
+            self._loss[i] = loss(y, y_pred, reduction=None)
+            p += alpha * (self._loss[i] / np.sum(self._loss))
+            p /= np.sum(p)
+
+    def predict(self, X):
+        w = np.sum(self._loss, axis=1)
+        w /= np.sum(w)
+        y = np.zeros((X.shape[0], self._n_outputs))
+        for i in range(self._n_estimators):
+            y += w[i] * self._estimators[i].predict(X)
+        return y

pyldl/algorithms/_incomplete.py

@@ -0,0 +1,58 @@
+import numpy as np
+
+from qpsolvers import solve_qp
+
+from pyldl.algorithms.base import BaseLDL
+
+
+class IncomLDL(BaseLDL):
+    """IncomLDL is proposed in paper Incomplete Label Distribution Learning.
+    """
+
+    @staticmethod
+    def svt(A, tau):
+        U, S, VT = np.linalg.svd(A, full_matrices=False)
+        S_thresh = np.maximum(S - tau, 0)
+        return U @ np.diag(S_thresh) @ VT
+
+    def _update_Z(self):
+        A = self._X @ self._W + self._V / self._rho
+        tau = self._alpha / self._rho
+        self._Z = self.svt(A, tau)
+
+    def _update_W(self):
+   
+        G = -np.eye(self._n_outputs, dtype=np.float64)
+        h = np.zeros((self._n_outputs, 1), dtype=np.float64)
+        A = np.ones((1, self._n_outputs), dtype=np.float64)
+        b = np.array([1.], dtype=np.float64)
+
+        M = np.zeros_like(self._y)
+
+        for i in range(self._X.shape[0]):
+            P = np.diag((1 + self._rho) * self._mask[i] + self._rho * (1 - self._mask[i])).astype(np.float64)
+            ql = (self._V[i] - self._y[i] * self._mask[i] - self._rho * self._Z[i]) * self._mask[i]
+            qr = (self._V[i] - self._rho * self._Z[i]) * (1 - self._mask[i])
+            q = np.transpose(ql + qr).astype(np.float64)
+            M[i] = solve_qp(P, q, G, h, A, b, solver='quadprog')
+
+        self._W = np.linalg.pinv(np.transpose(self._X) @ self._X) @ np.transpose(self._X) @ M
+
+    def _update_V(self):
+        self._V = self._V + self._rho * (self._X @ self._W - self._Z)
+
+    def fit(self, X, y, mask, alpha=1e-3, rho=1., max_iterations=100):
+        super().fit(X, y)
+        self._alpha = alpha
+        self._rho = rho
+        self._mask = np.where(mask, 0., 1.)
+        self._W = np.ones((self._n_features, self._n_outputs))
+        self._Z = np.ones((self._X.shape[0], self._n_outputs))
+        self._V = np.ones((self._X.shape[0], self._n_outputs))
+        for _ in range(max_iterations):
+            self._update_W()
+            self._update_Z()
+            self._update_V()
+    
+    def predict(self, X):
+        return X @ self._W