python-ldl 0.0.2__py3-none-any.whl → 0.0.3__py3-none-any.whl

pyldl/algorithms/__init__.py

@@ -9,18 +9,22 @@ from ._ensemble import DF_LDL, AdaBoostLDL
 from ._ldlf import LDLF
 from ._ldl_scl import LDL_SCL
 from ._ldl_lrr import LDL_LRR
+from ._ldl_dpa import LDL_DPA
 
 from ._ssg_ldl import SSG_LDL
 
 from ._label_enhancement import FCM, KM, LP, ML, GLLE, LEVI, LIBLE
 
+from ._ldl_da import LDL_DA
+
 
 __all__ = ["SA_BFGS", "SA_IIS", "AA_KNN", "AA_BP", "PT_Bayes", "PT_SVM",
            "CPNN", "BCPNN", "ACPNN", "LDSVR",
-           "LDLF", "LDL_SCL", "LDL_LRR", "CAD", "QFD2", "CJS",
+           "LDLF", "LDL_SCL", "LDL_LRR", "LDL_DPA", "CAD", "QFD2", "CJS",
            "DF_LDL", "AdaBoostLDL",
            "LDL4C", "LDL_HR", "LDLM",
            "IncomLDL",
            "SSG_LDL",
            "FCM", "KM", "LP", "ML", "GLLE",
-           "LEVI", 'LIBLE']
+           "LEVI", "LIBLE",
+           "LDL_DA"]

pyldl/algorithms/_algorithm_adaptation.py

@@ -1,128 +1,74 @@
 import numpy as np
 from sklearn.neighbors import NearestNeighbors
-from sklearn.preprocessing import OneHotEncoder
 
 import keras
 import tensorflow as tf
 from keras import backend as K
 
-from pyldl.algorithms.base import BaseLDL, BaseDeepLDL
+from pyldl.algorithms.base import BaseLDL, BaseDeepLDL, BaseGD, BaseAdam
 
 
 class AA_KNN(BaseLDL):
 
-    def __init__(self,
-                 k=5,
-                 random_state=None):
-
-        super().__init__(random_state)
-
-        self.k = k
-        self._model = NearestNeighbors(n_neighbors=self.k)
-
-    def fit(self, X, y):
+    def fit(self, X, y, k=5):
         super().fit(X, y)
-        self._model.fit(self._X)
+        self._model = NearestNeighbors(n_neighbors=k).fit(self._X)
+        return self
 
     def predict(self, X):
         _, inds = self._model.kneighbors(X)
         return np.average(self._y[inds], axis=1)
 
 
-class AA_BP(BaseDeepLDL):
+class AA_BP(BaseGD, BaseDeepLDL):
+    pass
 
-    def __init__(self, n_hidden=None, n_latent=None, random_state=None):
-        super().__init__(n_hidden, n_latent, random_state)
 
-    @tf.function
-    def _loss_function(self, y, y_pred):
-        return tf.math.reduce_mean(keras.losses.mean_squared_error(y, y_pred))
-
-    def fit(self, X, y, learning_rate=5e-3, epochs=3000, batch_size=32,
-            model=None, activation='sigmoid', optimizer='SGD', X_test=None, y_test=None):
-        super().fit(X, y)
-
-        self._batch_size = batch_size
-
-        if self._n_hidden is None:
-            self._n_hidden = self._n_features * 3 // 2
-
-        self._model = model
-        if self._model is None:
-            self._model = keras.Sequential([keras.layers.InputLayer(input_shape=(self._n_features,)),
-                                        keras.layers.Dense(self._n_hidden, activation=activation),
-                                        keras.layers.Dense(self._n_outputs, activation='softmax')])
-        self._optimizer = eval(f'keras.optimizers.{optimizer}({learning_rate})')
-        data = tf.data.Dataset.from_tensor_slices((self._X, self._y)).batch(self._batch_size)
-
-        for _ in range(epochs):
-            total_loss = 0.
-            for batch in data:
-                with tf.GradientTape() as tape:
-                    y_pred = self._model(batch[0])
-                    loss = self._loss_function(batch[1], y_pred)
-                gradients = tape.gradient(loss, self.trainable_variables)
-                self._optimizer.apply_gradients(zip(gradients, self.trainable_variables))
-                total_loss += loss
-
-    def predict(self, X):
-        return self._model(X)
-
-
-class CAD(AA_BP):
+class CAD(BaseAdam, BaseDeepLDL):
 
+    @staticmethod
     @tf.function
-    def _loss_function(self, y, y_pred):
+    def loss_function(y, y_pred):
         def _CAD(y, y_pred):
             return tf.reduce_mean(tf.abs(
                 tf.cumsum(y, axis=1) - tf.cumsum(y_pred, axis=1)
             ), axis=1)
         return tf.math.reduce_sum(
             tf.map_fn(lambda i: _CAD(y[:, :i], y_pred[:, :i]),
-                      tf.range(1, self._n_outputs + 1),
+                      tf.range(1, y.shape[1] + 1),
                       fn_output_signature=tf.float32)
         )
 
-    def fit(self, X, y, learning_rate=1e-4, epochs=500,
-            activation='relu', optimizer='Adam'):
-        return super().fit(X, y, learning_rate, epochs, activation, optimizer)
-
 
-class QFD2(AA_BP):
+class QFD2(BaseAdam, BaseDeepLDL):
 
+    @staticmethod
     @tf.function
-    def _loss_function(self, y, y_pred):
+    def _loss_function(y, y_pred):
         Q = y - y_pred
-        j = tf.reshape(tf.range(self._n_outputs), [self._n_outputs, 1])
-        k = tf.reshape(tf.range(self._n_outputs), [1, self._n_outputs])
-        A = tf.cast(1 - tf.abs(j - k) / (self._n_outputs - 1), dtype=tf.float32)
+        j = tf.reshape(tf.range(y.shape[1]), [y.shape[1], 1])
+        k = tf.reshape(tf.range(y.shape[1]), [1, y.shape[1]])
+        A = tf.cast(1 - tf.abs(j - k) / (y.shape[1] - 1), dtype=tf.float32)
         return tf.math.reduce_mean(
             tf.linalg.diag_part(tf.matmul(tf.matmul(Q, A), tf.transpose(Q)))
         )
 
-    def fit(self, X, y, learning_rate=1e-4, epochs=500,
-            activation='relu', optimizer='Adam'):
-        return super().fit(X, y, learning_rate, epochs, activation, optimizer)
 
+class CJS(BaseAdam, BaseDeepLDL):
 
-class CJS(AA_BP):
-
+    @staticmethod
     @tf.function
-    def _loss_function(self, y, y_pred):
+    def loss_function(y, y_pred):
         def _CJS(y, y_pred):
             m = 0.5 * (y + y_pred)
             js = 0.5 * (keras.losses.kl_divergence(y, m) + keras.losses.kl_divergence(y_pred, m))
             return tf.reduce_mean(js)
         return tf.math.reduce_sum(
             tf.map_fn(lambda i: _CJS(y[:, :i], y_pred[:, :i]),
-                      tf.range(1, self._n_outputs + 1),
+                      tf.range(1, y.shape[1] + 1),
                       fn_output_signature=tf.float32)
         )
 
-    def fit(self, X, y, learning_rate=1e-4, epochs=500,
-            activation='relu', optimizer='Adam'):
-        return super().fit(X, y, learning_rate, epochs, activation, optimizer)
-
 
 class RProp(keras.optimizers.Optimizer):
     
@@ -134,6 +80,10 @@ class RProp(keras.optimizers.Optimizer):
         self.min_alpha = K.variable(min_alpha, name='min_alpha')
         self.max_alpha = K.variable(max_alpha, name='max_alpha')
 
+    def apply_gradients(self, grads_and_vars):
+        grads, trainable_variables = zip(*grads_and_vars)
+        self.get_updates(trainable_variables, grads)
+
     def get_updates(self, params, gradients):
         grads = gradients
         shapes = [K.int_shape(p) for p in params]
@@ -183,12 +133,12 @@ class RProp(keras.optimizers.Optimizer):
         return dict(list(base_config.items()) + list(config.items()))
 
 
-class CPNN(BaseDeepLDL):
+class CPNN(BaseGD, BaseDeepLDL):
 
     def _not_proper_mode(self):
         raise ValueError("The argument 'mode' can only be 'none', 'binary' or 'augment'.")
 
-    def __init__(self, mode='none', v=5, n_hidden=None, n_latent=None, random_state=None):
+    def __init__(self, mode='none', v=5, n_hidden=64, n_latent=None, random_state=None):
         super().__init__(n_hidden, n_latent, random_state)
         if mode == 'none' or mode == 'binary' or mode == 'augment':
             self._mode = mode
@@ -196,37 +146,34 @@ class CPNN(BaseDeepLDL):
             self._not_proper_mode()
         self._v = v
 
-    def fit(self, X, y, learning_rate=5e-3, epochs=3000):
-        super().fit(X, y)
+    @staticmethod
+    @tf.function
+    def loss_function(y, y_pred):
+        return tf.math.reduce_mean(keras.losses.kl_divergence(y, y_pred))
 
-        self._optimizer = RProp(init_alpha=learning_rate)
+    def _get_default_model(self):
+        input_shape = (self._n_features + (1 if self._mode == 'none' else self._n_outputs),)
+        return keras.Sequential([keras.layers.InputLayer(input_shape=input_shape),
+                                 keras.layers.Dense(self._n_hidden, activation='sigmoid'),
+                                 keras.layers.Dense(1, activation=None)])
 
-        if self._n_hidden is None:
-            self._n_hidden = self._n_features * 3 // 2
+    def _get_default_optimizer(self):
+        return RProp()
 
+    def _before_train(self):
         if self._mode == 'augment':
+            n = self._X.shape[0]
             one_hot = tf.one_hot(tf.math.argmax(self._y, axis=1), self.n_outputs)
             self._X = tf.repeat(self._X, self._v, axis=0)
             self._y = tf.repeat(self._y, self._v, axis=0)
             one_hot = tf.repeat(one_hot, self._v, axis=0)
-            v = tf.reshape(tf.tile([1 / (i + 1) for i in range(self._v)], [X.shape[0]]), (-1, 1))
+            v = tf.reshape(tf.tile([1 / (i + 1) for i in range(self._v)], [n]), (-1, 1))
             self._y += self._y * one_hot * v
 
-        input_shape = (self._n_features + (1 if self._mode == 'none' else self._n_outputs),)
-        self._model = keras.Sequential([keras.layers.InputLayer(input_shape=input_shape),
-                                        keras.layers.Dense(self._n_hidden, activation='sigmoid'),
-                                        keras.layers.Dense(1, activation=None)])
-
-        for _ in range(epochs):
-            with tf.GradientTape() as tape:
-                loss = self._loss(self._X, self._y)
-            gradients = tape.gradient(loss, self.trainable_variables)
-            self._optimizer.get_updates(self.trainable_variables, gradients)
-
     def _make_inputs(self, X):
         temp = tf.reshape(tf.tile([i + 1 for i in range(self._n_outputs)], [X.shape[0]]), (-1, 1))
         if self._mode != 'none':
-            temp = OneHotEncoder(sparse=False).fit_transform(temp)
+            temp = tf.one_hot(tf.reshape(temp, (-1, )) - 1, depth=self._n_outputs)
         return tf.concat([tf.cast(tf.repeat(X, self._n_outputs, axis=0), dtype=tf.float32),
                           tf.cast(temp, dtype=tf.float32)],
                           axis=1)
@@ -238,12 +185,6 @@ class CPNN(BaseDeepLDL):
         b = tf.reshape(-tf.math.log(tf.math.reduce_sum(tf.math.exp(results), axis=1)), (-1, 1))
         return tf.math.exp(b + results)
 
-    def _loss(self, X, y):
-        return tf.math.reduce_mean(keras.losses.kl_divergence(y, self._call(X)))
-
-    def predict(self, X):
-        return self._call(X)
-
 
 class BCPNN(CPNN):
 

pyldl/algorithms/_classifier.py

@@ -1,114 +1,89 @@
 import keras
 import tensorflow as tf
 
-from pyldl.metrics import score
-from pyldl.algorithms.base import BaseDeepLDL, BaseDeepLDLClassifier, DeepBFGS
+from pyldl.algorithms.base import BaseDeepLDLClassifier, BaseGD, BaseBFGS
 
 
-class LDL4C(BaseDeepLDLClassifier, DeepBFGS):
-
-    def __init__(self, n_hidden=None, n_latent=None, random_state=None):
-        super().__init__(n_hidden, n_latent, random_state)
+class LDL4C(BaseBFGS, BaseDeepLDLClassifier):
+    """LDL4C is proposed in paper Classification with Label Distribution Learning.
+    """
 
     @tf.function
-    def _loss(self, X, y):
-        y_pred = self._model(X)
+    def _loss(self, params_1d):
+        y_pred = keras.activations.softmax(self._X @ self._params2model(params_1d)[0])
         top2 = tf.gather(y_pred, self._top2, axis=1, batch_dims=1)
         margin = tf.reduce_sum(tf.maximum(0., 1. - (top2[:, 0] - top2[:, 1]) / self._rho))
-        mae = keras.losses.mean_absolute_error(y, y_pred)
-        return tf.reduce_sum(self._entropy * mae) + self._alpha * margin + self._beta * BaseDeepLDL._l2_reg(self._model)
-    
-    def fit(self, X, y, max_iterations=50, alpha=1e-2, beta=1e-6, rho=1e-2):
-        super().fit(X, y)
-
-        self._alpha = alpha
-        self._beta = beta
-        self._rho = rho
+        mae = keras.losses.mean_absolute_error(self._y, y_pred)
+        return tf.reduce_sum(self._entropy * mae) + self._alpha * margin + self._beta * self._l2_reg(self._model)
 
+    def _before_train(self):
         self._top2 = tf.math.top_k(self._y, k=2)[1]
         self._entropy = tf.cast(-tf.reduce_sum(self._y * tf.math.log(self._y) + 1e-7, axis=1), dtype=tf.float32)
 
-        self._model = keras.Sequential(
-            [keras.layers.InputLayer(input_shape=self._n_features),
-             keras.layers.Dense(self._n_outputs, activation="softmax")])
-
-        self._optimize_bfgs(self._model, self._loss, self._X, self._y, max_iterations=max_iterations)
-
+    def fit(self, X, y, alpha=1e-2, beta=1e-6, rho=1e-2, **kwargs):
+        self._alpha = alpha
+        self._beta = beta
+        self._rho = rho
+        return super().fit(X, y, **kwargs)
 
-class LDL_HR(BaseDeepLDLClassifier, DeepBFGS):
 
-    def __init__(self, n_hidden=None, n_latent=None, random_state=None):
-        super().__init__(n_hidden, n_latent, random_state)
+class LDL_HR(BaseBFGS, BaseDeepLDLClassifier):
+    """LDL-HR is proposed in paper Learn the Highest Label and Rest Label Description Degrees.
+    """
 
     @tf.function
-    def _loss(self, X, y):
-        y_pred = self._model(X)
-        
+    def _loss(self, params_1d):
+        y_pred = keras.activations.softmax(self._X @ self._params2model(params_1d)[0])
+
         highest = tf.gather(y_pred, self._highest, axis=1, batch_dims=1)
         rest = tf.gather(y_pred, self._rest, axis=1, batch_dims=1)
         margin = tf.reduce_sum(tf.maximum(0., 1. - (highest - rest) / self._rho))
 
-        real_rest = tf.gather(y, self._rest, axis=1, batch_dims=1)
+        real_rest = tf.gather(self._y, self._rest, axis=1, batch_dims=1)
         rest_mae = tf.reduce_sum(keras.losses.mean_absolute_error(real_rest, rest))
 
         mae = tf.reduce_sum(keras.losses.mean_absolute_error(self._l, y_pred))
 
-        return mae + self._alpha * margin + self._beta * rest_mae + self._gamma * BaseDeepLDL._l2_reg(self._model)
-    
-    def fit(self, X, y, max_iterations=50, alpha=1e-2, beta=1e-2, gamma=1e-6, rho=1e-2):
-        super().fit(X, y)
-
-        self._alpha = alpha
-        self._beta = beta
-        self._gamma = gamma
-        self._rho = rho
+        return mae + self._alpha * margin + self._beta * rest_mae + self._gamma * self._l2_reg(self._model)
 
+    def _before_train(self):
         temp = tf.math.top_k(self._y, k=self._n_outputs)[1]
         self._highest = temp[:, 0:1]
         self._rest = temp[:, 1:]
-
         self._l = tf.one_hot(tf.reshape(self._highest, -1), self._n_outputs)
 
-        self._model = keras.Sequential(
-            [keras.layers.InputLayer(input_shape=self._n_features),
-             keras.layers.Dense(self._n_outputs, activation="softmax")])
-
-        self._optimize_bfgs(self._model, self._loss, self._X, self._y, max_iterations=max_iterations)
-
+    def fit(self, X, y, alpha=1e-2, beta=1e-2, gamma=1e-6, rho=1e-2, **kwargs):
+        self._alpha = alpha
+        self._beta = beta
+        self._gamma = gamma
+        self._rho = rho
+        return super().fit(X, y, **kwargs) 
 
-class LDLM(BaseDeepLDLClassifier):
 
-    def __init__(self, n_hidden=None, n_latent=None, random_state=None):
-        super().__init__(n_hidden, n_latent, random_state)
+class LDLM(BaseGD, BaseDeepLDLClassifier):
+    """LDLM is proposed in paper Label Distribution Learning Machine.
+    """
 
     @tf.function
-    def _loss(self, X, y):
+    def _loss(self, X, y, start, end):
         y_pred = self._model(X)
 
         pred_margin = tf.reduce_sum(tf.clip_by_value(
-            tf.reduce_sum(tf.abs(self._l - y_pred), axis=1) - self._rho,
+            tf.reduce_sum(tf.abs(self._l[start:end] - y_pred), axis=1) - self._rho,
             0., float('inf')))
-        
-        highest = tf.gather(y_pred, self._highest, axis=1, batch_dims=1)
-        rest = tf.gather(y_pred, self._rest, axis=1, batch_dims=1)
+
+        highest = tf.gather(y_pred, self._highest[start:end], axis=1, batch_dims=1)
+        rest = tf.gather(y_pred, self._rest[start:end], axis=1, batch_dims=1)
         label_margin = tf.reduce_sum(tf.maximum(0., 1. - (highest - rest) / self._rho))
 
         second_margin = tf.reduce_sum(tf.clip_by_value(
-            tf.reduce_sum(tf.abs((y - y_pred) * self._neg_l), axis=1) - self._second_margin,
+            tf.reduce_sum(tf.abs((y - y_pred) * self._neg_l[start:end]), axis=1) - self._second_margin[start:end],
             0., float('inf')))
 
         return pred_margin + self._alpha * label_margin + \
-            self._beta * second_margin + self._gamma * BaseDeepLDL._l2_reg(self._model)
-
-    def fit(self, X, y, learning_rate=5e-4, epochs=1000,
-            alpha=1e-2, beta=1e-2, gamma=1e-6, rho=1e-2):
-        super().fit(X, y)
-
-        self._alpha = alpha
-        self._beta = beta
-        self._gamma = gamma
-        self._rho = rho
+            self._beta * second_margin + self._gamma * self._l2_reg(self._model)
 
+    def _before_train(self):
         temp = tf.math.top_k(self._y, k=self._n_outputs)[1]
         self._highest = temp[:, 0:1]
         self._rest = temp[:, 1:]
@@ -119,15 +94,12 @@ class LDLM(BaseDeepLDLClassifier):
         self._l = tf.one_hot(tf.reshape(self._highest, -1), self._n_outputs)
         self._neg_l = tf.where(tf.equal(self._l, 0.), 1., 0.)
 
-        self._model = keras.Sequential(
-            [keras.layers.InputLayer(input_shape=self._n_features),
-             keras.layers.Dense(self._n_outputs, activation="softmax")])
-        
-        self._optimizer = keras.optimizers.SGD(learning_rate)
-
-        for _ in range(epochs):
-            with tf.GradientTape() as tape:
-                loss = self._loss(self._X, self._y)
+    def _get_default_model(self):
+        return self.get_2layer_model(self._n_features, self._n_outputs)
 
-            gradients = tape.gradient(loss, self.trainable_variables)
-            self._optimizer.apply_gradients(zip(gradients, self.trainable_variables))
+    def fit(self, X, y, alpha=1e-2, beta=1e-2, gamma=1e-6, rho=1e-2, **kwargs):
+        self._alpha = alpha
+        self._beta = beta
+        self._gamma = gamma
+        self._rho = rho
+        return super().fit(X, y, **kwargs)

pyldl/algorithms/_incomplete.py

@@ -1,45 +1,58 @@
-import keras
-import tensorflow as tf
-import tensorflow_addons as tfa
+import numpy as np
 
-from pyldl.metrics import score
-from pyldl.algorithms.base import BaseDeepLDL
+from qpsolvers import solve_qp
 
+from pyldl.algorithms.base import BaseLDL
 
-class IncomLDL(BaseDeepLDL):
 
-    def __init__(self, n_hidden=None, n_latent=None, random_state=None):
-        super().__init__(n_hidden, n_latent, random_state)
+class IncomLDL(BaseLDL):
+    """IncomLDL is proposed in paper Incomplete Label Distribution Learning.
+    """
 
-    def _loss(self, X, y):
-        y_pred = self._model(X)
-        trace_norm = 0.
-        for i in self._model.trainable_variables:
-            trace_norm += tf.linalg.trace(tf.sqrt(tf.matmul(tf.transpose(i), i)))
-        fro_norm = tf.reduce_sum(tf.square(self._mask * (y_pred - y)))
-        return fro_norm / 2. + self._alpha * trace_norm
+    @staticmethod
+    def svt(A, tau):
+        U, S, VT = np.linalg.svd(A, full_matrices=False)
+        S_thresh = np.maximum(S - tau, 0)
+        return U @ np.diag(S_thresh) @ VT
 
-    def fit(self, X, y, mask, alpha=2., learning_rate=5e-2, epochs=5000):
-        super().fit(X, y)
+    def _update_Z(self):
+        A = self._X @ self._W + self._V / self._rho
+        tau = self._alpha / self._rho
+        self._Z = self.svt(A, tau)
 
-        self._alpha = alpha
-        self._mask = tf.where(mask, 0., 1.)
+    def _update_W(self):
+   
+        G = -np.eye(self._n_outputs, dtype=np.float64)
+        h = np.zeros((self._n_outputs, 1), dtype=np.float64)
+        A = np.ones((1, self._n_outputs), dtype=np.float64)
+        b = np.array([1.], dtype=np.float64)
 
-        self._model = keras.Sequential([keras.layers.InputLayer(input_shape=self._n_features),
-                                        keras.layers.Dense(self._n_outputs, activation='softmax', use_bias=False)])
-        self._optimizer = tfa.optimizers.ProximalAdagrad(learning_rate)
+        M = np.zeros_like(self._y)
 
-        for _ in range(epochs):
-            with tf.GradientTape() as tape:
-                loss = self._loss(self._X, self._y)
+        for i in range(self._X.shape[0]):
+            P = np.diag((1 + self._rho) * self._mask[i] + self._rho * (1 - self._mask[i])).astype(np.float64)
+            ql = (self._V[i] - self._y[i] * self._mask[i] - self._rho * self._Z[i]) * self._mask[i]
+            qr = (self._V[i] - self._rho * self._Z[i]) * (1 - self._mask[i])
+            q = np.transpose(ql + qr).astype(np.float64)
+            M[i] = solve_qp(P, q, G, h, A, b, solver='quadprog')
 
-            gradients = tape.gradient(loss, self.trainable_variables)
-            self._optimizer.apply_gradients(zip(gradients, self.trainable_variables))
-        
-    def predict(self, X):
-        return self._model(X).numpy()
+        self._W = np.linalg.pinv(np.transpose(self._X) @ self._X) @ np.transpose(self._X) @ M
+
+    def _update_V(self):
+        self._V = self._V + self._rho * (self._X @ self._W - self._Z)
+
+    def fit(self, X, y, mask, alpha=1e-3, rho=1., max_iterations=100):
+        super().fit(X, y)
+        self._alpha = alpha
+        self._rho = rho
+        self._mask = np.where(mask, 0., 1.)
+        self._W = np.ones((self._n_features, self._n_outputs))
+        self._Z = np.ones((self._X.shape[0], self._n_outputs))
+        self._V = np.ones((self._X.shape[0], self._n_outputs))
+        for _ in range(max_iterations):
+            self._update_W()
+            self._update_Z()
+            self._update_V()
     
-    def score(self, X, y, metrics=None):
-        if metrics is None:
-            metrics = ["chebyshev", "clark", "canberra", "cosine", "intersection"]
-        return score(y, self.predict(X), metrics=metrics)
+    def predict(self, X):
+        return X @ self._W