deepordinal 0.2.0__py3-none-any.whl

deepordinal/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.2.0"
+__all__ = ["tf", "torch"]

deepordinal/tf.py

@@ -0,0 +1,163 @@
+import tensorflow as tf
+from tensorflow.keras import initializers
+from tensorflow.keras.layers import Layer
+
+__all__ = ["OrdinalOutput", "SortedInitializer", "ordinal_loss", "ordistic_loss"]
+
+_INF = tf.constant(float("inf"))
+
+
+def _penalty(z, name):
+    if name == "hinge":
+        return tf.maximum(0.0, 1.0 - z)
+    elif name == "smooth_hinge":
+        return tf.where(
+            z >= 1.0,
+            tf.zeros_like(z),
+            tf.where(z > 0.0, (1.0 - z) ** 2 / 2.0, 0.5 - z),
+        )
+    elif name == "modified_least_squares":
+        return tf.where(z >= 1.0, tf.zeros_like(z), (1.0 - z) ** 2)
+    elif name == "logistic":
+        return tf.math.softplus(-z)
+    else:
+        raise ValueError(f"Unknown penalty: {name}")
+
+
+def ordinal_loss(logits, targets, thresholds, construction="all", penalty="logistic"):
+    """Rennie & Srebro ordinal loss (IJCAI 2005).
+
+    Args:
+        logits: (batch,) or (batch, 1) — raw predictor output z(x).
+        targets: (batch,) — integer labels in [0, K).
+        thresholds: (K-1,) — sorted interior thresholds.
+        construction: ``'all'`` or ``'immediate'``.
+        penalty: ``'hinge'``, ``'smooth_hinge'``, ``'modified_least_squares'``,
+            or ``'logistic'``.
+
+    Returns:
+        Scalar mean loss over the batch.
+    """
+    logits = tf.cast(tf.reshape(logits, [-1]), tf.float32)
+    targets = tf.cast(targets, tf.int32)
+    thresholds = tf.cast(thresholds, tf.float32)
+    K = thresholds.shape[0] + 1
+    y = targets + 1  # 1-indexed
+
+    if construction == "all":
+        l_idx = tf.cast(tf.range(1, K), tf.float32)  # (K-1,)
+        y_f = tf.cast(y, tf.float32)
+        signs = tf.where(
+            tf.expand_dims(l_idx, 0) < tf.expand_dims(y_f, 1), -1.0, 1.0
+        )  # (batch, K-1)
+        diff = tf.expand_dims(thresholds, 0) - tf.expand_dims(logits, 1)  # (batch, K-1)
+        loss = tf.reduce_sum(_penalty(signs * diff, penalty), axis=1)
+    elif construction == "immediate":
+        t_low = tf.concat([[float("-inf")], thresholds], axis=0)
+        t_high = tf.concat([thresholds, [float("inf")]], axis=0)
+        theta_low = tf.gather(t_low, targets)
+        theta_high = tf.gather(t_high, targets)
+        loss = _penalty(logits - theta_low, penalty) + _penalty(theta_high - logits, penalty)
+    else:
+        raise ValueError(f"Unknown construction: {construction}")
+
+    return tf.reduce_mean(loss)
+
+
+def ordistic_loss(logits, targets, means, log_priors=None):
+    """Ordistic loss (Rennie & Srebro, Section 4).
+
+    Args:
+        logits: (batch,) or (batch, 1) — raw predictor output z(x).
+        targets: (batch,) — integer labels in [0, K).
+        means: (K,) — class means.
+        log_priors: (K,) or None — log-prior terms. Defaults to zeros.
+
+    Returns:
+        Scalar mean negative log-likelihood over the batch.
+    """
+    logits = tf.cast(tf.reshape(logits, [-1]), tf.float32)
+    targets = tf.cast(targets, tf.int32)
+    means = tf.cast(means, tf.float32)
+    K = means.shape[0]
+    if log_priors is None:
+        log_priors = tf.zeros([K], dtype=tf.float32)
+    else:
+        log_priors = tf.cast(log_priors, tf.float32)
+    energy = (
+        tf.expand_dims(means, 0) * tf.expand_dims(logits, 1)
+        + tf.expand_dims(log_priors, 0)
+        - tf.expand_dims(means, 0) ** 2 / 2.0
+    )
+    batch_idx = tf.range(tf.shape(targets)[0])
+    indices = tf.stack([batch_idx, targets], axis=1)
+    target_energy = tf.gather_nd(energy, indices)
+    log_partition = tf.reduce_logsumexp(energy, axis=1)
+    return tf.reduce_mean(log_partition - target_energy)
+
+
+class SortedInitializer(initializers.Initializer):
+    """Wraps a Keras initializer and returns its output sorted along the last axis."""
+
+    def __init__(self, base="glorot_uniform"):
+        self.base = initializers.get(base)
+
+    def __call__(self, shape, dtype=None):
+        values = self.base(shape, dtype=dtype)
+        return tf.sort(values, axis=-1)
+
+    def get_config(self):
+        return {"base": initializers.serialize(self.base)}
+
+
+class OrdinalOutput(Layer):
+    """Ordinal regression output layer.
+
+    Projects an arbitrary input down to a single logit and converts it
+    into *output_dim* class probabilities via learned, sorted thresholds.
+
+    The layer learns ``output_dim - 1`` interior thresholds ``t(1)…t(K-1)``
+    (with ``t(0) = -∞`` and ``t(K) = +∞`` fixed) and computes::
+
+        P(y = k | x) = σ(t(k+1) - logit) - σ(t(k) - logit)
+    """
+
+    def __init__(self, output_dim, **kwargs):
+        super().__init__(**kwargs)
+        self.output_dim = output_dim
+
+    def build(self, input_shape):
+        self.kernel = self.add_weight(
+            name="kernel",
+            shape=(input_shape[-1], 1),
+            initializer="glorot_uniform",
+            trainable=True,
+        )
+        self.bias = self.add_weight(
+            name="bias",
+            shape=(1,),
+            initializer="zeros",
+            trainable=True,
+        )
+        self.interior_thresholds = self.add_weight(
+            name="thresholds",
+            shape=(1, self.output_dim - 1),
+            initializer=SortedInitializer("glorot_uniform"),
+            trainable=True,
+        )
+        super().build(input_shape)
+
+    def call(self, inputs):
+        logit = tf.matmul(inputs, self.kernel) + self.bias
+        t_low = tf.fill([1, 1], -_INF)
+        t_high = tf.fill([1, 1], _INF)
+        thresholds = tf.concat([t_low, self.interior_thresholds, t_high], axis=-1)
+        return tf.sigmoid(thresholds[:, 1:] - logit) - tf.sigmoid(thresholds[:, :-1] - logit)
+
+    def compute_output_shape(self, input_shape):
+        return (input_shape[0], self.output_dim)
+
+    def get_config(self):
+        config = super().get_config()
+        config["output_dim"] = self.output_dim
+        return config

deepordinal/torch.py

@@ -0,0 +1,123 @@
+import torch
+import torch.nn as nn
+
+__all__ = ["OrdinalOutput", "ordinal_loss", "ordistic_loss"]
+
+
+def _penalty(z, name):
+    if name == "hinge":
+        return torch.clamp(1 - z, min=0)
+    elif name == "smooth_hinge":
+        return torch.where(
+            z >= 1,
+            torch.zeros_like(z),
+            torch.where(z > 0, (1 - z) ** 2 / 2, 0.5 - z),
+        )
+    elif name == "modified_least_squares":
+        return torch.where(z >= 1, torch.zeros_like(z), (1 - z) ** 2)
+    elif name == "logistic":
+        return torch.nn.functional.softplus(-z)
+    else:
+        raise ValueError(f"Unknown penalty: {name}")
+
+
+def ordinal_loss(logits, targets, thresholds, construction="all", penalty="logistic"):
+    """Rennie & Srebro ordinal loss (IJCAI 2005).
+
+    Args:
+        logits: (batch,) or (batch, 1) — raw predictor output z(x).
+        targets: (batch,) — integer labels in [0, K).
+        thresholds: (K-1,) — sorted interior thresholds.
+        construction: ``'all'`` or ``'immediate'``.
+        penalty: ``'hinge'``, ``'smooth_hinge'``, ``'modified_least_squares'``,
+            or ``'logistic'``.
+
+    Returns:
+        Scalar mean loss over the batch.
+    """
+    logits = logits.reshape(-1)
+    targets = targets.long()
+    K = thresholds.shape[0] + 1
+    # Paper uses 1-indexed labels; convert 0-indexed targets
+    y = targets + 1  # (batch,) in [1, K]
+
+    if construction == "all":
+        # eq 13: sum over l=1..K-1 of f(s(l;y) * (theta_l - z))
+        # s(l;y) = -1 if l < y, +1 if l >= y
+        l_idx = torch.arange(1, K, device=logits.device).float()  # (K-1,)
+        signs = torch.where(l_idx.unsqueeze(0) < y.unsqueeze(1), -1.0, 1.0)  # (batch, K-1)
+        diff = thresholds.unsqueeze(0) - logits.unsqueeze(1)  # (batch, K-1)
+        loss = _penalty(signs * diff, penalty).sum(dim=1)  # (batch,)
+    elif construction == "immediate":
+        # eq 12: f(z - theta_{y-1}) + f(theta_y - z)
+        t_low = torch.cat([torch.tensor([float("-inf")], device=thresholds.device), thresholds])
+        t_high = torch.cat([thresholds, torch.tensor([float("inf")], device=thresholds.device)])
+        theta_low = t_low[targets]   # theta_{y-1} (0-indexed: targets maps to y-1)
+        theta_high = t_high[targets]  # theta_y
+        loss = _penalty(logits - theta_low, penalty) + _penalty(theta_high - logits, penalty)
+    else:
+        raise ValueError(f"Unknown construction: {construction}")
+
+    return loss.mean()
+
+
+def ordistic_loss(logits, targets, means, log_priors=None):
+    """Ordistic loss (Rennie & Srebro, Section 4).
+
+    Args:
+        logits: (batch,) or (batch, 1) — raw predictor output z(x).
+        targets: (batch,) — integer labels in [0, K).
+        means: (K,) — class means (mu_1=-1, mu_K=1 by convention; interior learned).
+        log_priors: (K,) or None — log-prior terms pi_i. Defaults to zeros.
+
+    Returns:
+        Scalar mean negative log-likelihood over the batch.
+    """
+    logits = logits.reshape(-1)
+    targets = targets.long()
+    K = means.shape[0]
+    if log_priors is None:
+        log_priors = torch.zeros(K, device=logits.device, dtype=logits.dtype)
+    # energy_ik = mu_k * z_i + pi_k - mu_k^2 / 2
+    energy = means.unsqueeze(0) * logits.unsqueeze(1) + log_priors.unsqueeze(0) - means.unsqueeze(0) ** 2 / 2
+    # loss = -log P(y|z) = -energy[y] + log(sum_k exp(energy[k]))
+    target_energy = energy[torch.arange(len(targets), device=targets.device), targets]
+    log_partition = torch.logsumexp(energy, dim=1)
+    return (log_partition - target_energy).mean()
+
+
+class OrdinalOutput(nn.Module):
+    """Ordinal regression output layer.
+
+    Projects an arbitrary input down to a single logit and converts it
+    into *output_dim* class probabilities via learned, sorted thresholds.
+
+    The layer learns ``output_dim - 1`` interior thresholds ``t(1)…t(K-1)``
+    (with ``t(0) = -∞`` and ``t(K) = +∞`` fixed) and computes::
+
+        P(y = k | x) = σ(t(k+1) - logit) - σ(t(k) - logit)
+
+    Args:
+        input_dim: Size of the input feature dimension.
+        output_dim: Number of ordinal classes.
+    """
+
+    def __init__(self, input_dim, output_dim):
+        super().__init__()
+        self.input_dim = input_dim
+        self.output_dim = output_dim
+        self.linear = nn.Linear(input_dim, 1)
+        self.interior_thresholds = nn.Parameter(torch.empty(output_dim - 1))
+        self._init_thresholds()
+
+    def _init_thresholds(self):
+        nn.init.xavier_uniform_(self.interior_thresholds.unsqueeze(0))
+        with torch.no_grad():
+            self.interior_thresholds.copy_(self.interior_thresholds.sort().values)
+
+    def forward(self, x):
+        logit = self.linear(x)  # (batch, 1)
+        t_low = torch.full((1,), float("-inf"), device=logit.device, dtype=logit.dtype)
+        t_high = torch.full((1,), float("inf"), device=logit.device, dtype=logit.dtype)
+        thresholds = torch.cat([t_low, self.interior_thresholds, t_high])  # (K+1,)
+        return torch.sigmoid(thresholds[1:] - logit) - torch.sigmoid(thresholds[:-1] - logit)

deepordinal-0.2.0.dist-info/METADATA

@@ -0,0 +1,185 @@
+Metadata-Version: 2.4
+Name: deepordinal
+Version: 0.2.0
+Summary: Ordinal output layers and loss functions (Rennie & Srebro, 2005) for PyTorch and TF/Keras
+Author: Nicholas Hirons
+License-Expression: MIT
+Project-URL: Repository, https://github.com/nhirons/deepordinal
+Keywords: ordinal-regression,deep-learning,pytorch,tensorflow,keras
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Provides-Extra: tf
+Requires-Dist: tensorflow>=2.0; extra == "tf"
+Provides-Extra: torch
+Requires-Dist: torch>=2.0; extra == "torch"
+Dynamic: license-file
+
+# DeepOrdinal
+
+Ordinal output layers and loss functions ([Rennie & Srebro, 2005](https://ttic.uchicago.edu/~nati/Publications/RennieSrebroIJCAI05.pdf)) for PyTorch and TF/Keras.
+
+DeepOrdinal provides an `OrdinalOutput` layer that converts a learned logit into ordinal class probabilities via sorted thresholds, plus loss functions designed specifically for ordinal regression.
+
+## Installation
+
+```bash
+pip install deepordinal
+```
+
+With a specific backend:
+
+```bash
+pip install ".[tf]"     # TensorFlow/Keras
+pip install ".[torch]"  # PyTorch
+```
+
+For development:
+
+```bash
+pip install -e ".[tf,torch]"
+```
+
+## Backends
+
+DeepOrdinal supports two backends with identical APIs:
+
+| | PyTorch | TensorFlow/Keras |
+|---|---|---|
+| Module | `deepordinal.torch` | `deepordinal.tf` |
+| Layer | `OrdinalOutput(input_dim=D, output_dim=K)` | `OrdinalOutput(output_dim=K)` |
+| Loss functions | `ordinal_loss`, `ordistic_loss` | `ordinal_loss`, `ordistic_loss` |
+
+## OrdinalOutput Layer
+
+The `OrdinalOutput` layer accepts any input size, projects to a single logit, and converts it into K class probabilities using K-1 learned, sorted thresholds:
+
+```
+P(y = k | x) = sigmoid(t(k+1) - logit) - sigmoid(t(k) - logit)
+```
+
+where `t(0) = -inf` and `t(K) = inf` are fixed, and interior thresholds are initialized sorted.
+
+```python
+from deepordinal.torch import OrdinalOutput  # or deepordinal.tf
+layer = OrdinalOutput(input_dim=16, output_dim=4)  # TF omits input_dim
+```
+
+## Loss Functions
+
+DeepOrdinal implements the threshold-based ordinal loss functions from Rennie & Srebro, "Loss Functions for Preference Levels" (IJCAI 2005). These operate on raw logits and thresholds rather than probability output.
+
+### `ordinal_loss`
+
+```python
+ordinal_loss(logits, targets, thresholds, construction='all', penalty='logistic')
+```
+
+- **logits**: `(batch,)` or `(batch, 1)` — raw predictor output
+- **targets**: `(batch,)` — integer labels in `[0, K)`
+- **thresholds**: `(K-1,)` — sorted interior thresholds
+- **construction**: `'all'` or `'immediate'`
+- **penalty**: `'hinge'`, `'smooth_hinge'`, `'modified_least_squares'`, or `'logistic'`
+- **Returns**: scalar mean loss over the batch
+
+#### Constructions
+
+- **All-threshold** (default, eq 13): penalizes violations of every threshold, weighted by direction. Bounds mean absolute error. Best performer in the paper's experiments.
+- **Immediate-threshold** (eq 12): only penalizes violations of the two thresholds bounding the correct class segment.
+
+#### Penalty functions
+
+| Name | Formula | Reference |
+|---|---|---|
+| `'hinge'` | `max(0, 1-z)` | eq 5 |
+| `'smooth_hinge'` | 0 if z≥1, (1-z)²/2 if 0<z<1, 0.5-z if z≤0 | eq 6 |
+| `'modified_least_squares'` | 0 if z≥1, (1-z)² if z<1 | eq 7 |
+| `'logistic'` | `log(1 + exp(-z))` | eq 9 |
+
+The paper recommends **all-threshold + logistic** as the best-performing combination.
+
+### `ordistic_loss`
+
+Probabilistic generalization of logistic regression to K-class ordinal problems (Section 4).
+
+```python
+ordistic_loss(logits, targets, means, log_priors=None)
+```
+
+- **logits**: `(batch,)` or `(batch, 1)` — raw predictor output
+- **targets**: `(batch,)` — integer labels in `[0, K)`
+- **means**: `(K,)` — class means (convention: μ₁=-1, μ_K=1; interior means learned)
+- **log_priors**: `(K,)` or `None` — optional log-prior terms π_i
+- **Returns**: scalar mean negative log-likelihood over the batch
+
+### Example usage
+
+#### PyTorch
+
+```python
+import torch
+from deepordinal.torch import OrdinalOutput, ordinal_loss
+
+layer = OrdinalOutput(input_dim=16, output_dim=4)
+h = torch.randn(8, 16)
+targets = torch.randint(0, 4, (8,))
+
+probs = layer(h)
+loss = ordinal_loss(layer.linear(h), targets, layer.interior_thresholds)
+loss.backward()
+```
+
+#### TensorFlow
+
+```python
+import tensorflow as tf
+from deepordinal.tf import OrdinalOutput, ordinal_loss
+
+layer = OrdinalOutput(output_dim=4)
+h = tf.random.normal((8, 16))
+targets = tf.random.uniform((8,), 0, 4, dtype=tf.int32)
+
+with tf.GradientTape() as tape:
+    probs = layer(h)
+    logit = tf.matmul(h, layer.kernel) + layer.bias
+    loss = ordinal_loss(logit, targets, tf.squeeze(layer.interior_thresholds))
+grads = tape.gradient(loss, layer.trainable_variables)
+```
+
+## Running Tests
+
+```bash
+pip install -e ".[tf,torch]"
+pytest -v
+```
+
+## Changelog
+
+### 0.2.0
+
+- Added `ordinal_loss` — Rennie & Srebro threshold-based ordinal loss with two constructions (all-threshold, immediate-threshold) and four penalty functions (hinge, smooth hinge, modified least squares, logistic)
+- Added `ordistic_loss` — ordistic negative log-likelihood loss (Rennie & Srebro, Section 4)
+- Both loss functions available in `deepordinal.torch` and `deepordinal.tf`
+
+### 0.1.0
+
+- Added PyTorch backend (`deepordinal.torch`) with `OrdinalOutput` module
+- Modernized TensorFlow backend to `tf.keras` with self-contained `OrdinalOutput` layer and `SortedInitializer`
+- Dual-backend support (TensorFlow/Keras and PyTorch) with matching APIs
+- `pyproject.toml` build configuration with optional `[tf]` and `[torch]` extras
+
+### Initial
+
+- `OrdinalOutput` Keras layer for deep ordinal regression
+- Example notebook with synthetic ordinal data
+
+## Examples
+
+- `examples/example_tf.ipynb` — TensorFlow/Keras with `ordinal_loss` and `GradientTape` training loop
+- `examples/example_torch.ipynb` — PyTorch with `ordinal_loss` and standard training loop
+

deepordinal-0.2.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+deepordinal/__init__.py,sha256=J_V0erDdvUobpcKnwcFY6Uvb_a5qF4J0dcE14znJWX4,48
+deepordinal/tf.py,sha256=FRZ-LuAjEvrOfAVXI_B-wGxxZiU4RlOvZc4sEpl4Z2o,5782
+deepordinal/torch.py,sha256=PLtkdhRKGfsuPg9Q52pPiu2Q2v5S2g0hoEzmPikadhE,5040
+deepordinal-0.2.0.dist-info/licenses/LICENSE,sha256=AWIoKzqfTVmlyfwzvydQoYGGiA6IKHnAdQzaaAGGQRw,1072
+deepordinal-0.2.0.dist-info/METADATA,sha256=NB-xBvi6v0oDVdrDd2SP8aCEWC1HB-zpueFXUhwQz68,6182
+deepordinal-0.2.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+deepordinal-0.2.0.dist-info/top_level.txt,sha256=fa0kRbTv8EIePUchQkt-TfBTcUfXnMSIXDNBiAZ8BGw,12
+deepordinal-0.2.0.dist-info/RECORD,,

deepordinal-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

deepordinal-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nicholas Hirons
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

deepordinal-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+deepordinal