pytextad 0.1.0__py3-none-any.whl

pytextad/__init__.py

@@ -0,0 +1,9 @@
+"""PyTextAD: a unified library for text anomaly detection, in the style of PyOD."""
+from .version import __version__
+from .models.cvdd import CVDD
+from .models.date import DATE
+from .models.fate import FATE
+from .models.rsrae import RSRAE
+from .utils.embeddings import TokenEmbedder, mean_pool, words_from_subwords
+
+__all__ = ["__version__", "CVDD", "DATE", "FATE", "RSRAE", "TokenEmbedder", "mean_pool", "words_from_subwords"]

pytextad/models/__init__.py

@@ -0,0 +1,7 @@
+from .base import BaseTextDetector
+from .cvdd import CVDD
+from .date import DATE
+from .fate import FATE
+from .rsrae import RSRAE
+
+__all__ = ["BaseTextDetector", "CVDD", "DATE", "FATE", "RSRAE"]

pytextad/models/base.py

@@ -0,0 +1,71 @@
+"""Common interface for all detectors, modelled on PyOD's BaseDetector.
+
+Conventions (identical to PyOD):
+  * fit(X, y=None) returns self and sets ``decision_scores_`` (scores of the training data),
+    ``threshold_`` and ``labels_``.
+  * decision_function(X) returns one score per sample; higher = more anomalous.
+  * predict(X) returns 0/1 using ``threshold_`` (the (1 - contamination) quantile of
+    the training scores).
+Detectors that can score tokens additionally implement ``token_scores``.
+"""
+
+import abc
+import random
+
+import numpy as np
+import torch
+
+
+class BaseTextDetector(abc.ABC):
+
+    def __init__(self, contamination=0.1, random_state=0, device=None, verbose=False):
+        if not 0.0 < contamination <= 0.5:
+            raise ValueError("contamination must be in (0, 0.5]")
+        self.contamination = contamination
+        self.random_state = random_state
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+        self.verbose = verbose
+
+    # ------------------------------------------------------------------ to implement
+    @abc.abstractmethod
+    def fit(self, X, y=None):
+        """Fit the detector on (mostly) normal data and set ``decision_scores_``,
+        ``threshold_`` and ``labels_``. ``y`` is ignored except by semi-supervised
+        detectors (FATE). Returns ``self``."""
+
+    @abc.abstractmethod
+    def decision_function(self, X):
+        """Anomaly score of every sample in ``X``; higher means more anomalous."""
+
+    # ------------------------------------------------------------------ shared
+    def _set_seed(self):
+        if self.random_state is None:
+            return
+        random.seed(self.random_state)
+        np.random.seed(self.random_state)
+        torch.manual_seed(self.random_state)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed_all(self.random_state)
+
+    def _process_decision_scores(self, scores):
+        self.decision_scores_ = np.asarray(scores, dtype=float)
+        self.threshold_ = float(np.percentile(self.decision_scores_, 100 * (1 - self.contamination)))
+        self.labels_ = (self.decision_scores_ > self.threshold_).astype(int)
+        return self
+
+    def predict(self, X):
+        """Binary labels (1 = anomaly) using ``threshold_`` from the training scores."""
+        self._check_fitted()
+        return (self.decision_function(X) > self.threshold_).astype(int)
+
+    def fit_predict(self, X, y=None):
+        """Fit on ``X`` and return the labels of the training data (``labels_``)."""
+        return self.fit(X, y).labels_
+
+    def _check_fitted(self):
+        if not hasattr(self, "decision_scores_"):
+            raise RuntimeError(f"{type(self).__name__} is not fitted yet; call fit() first.")
+
+    def _log(self, msg):
+        if self.verbose:
+            print(f"[{type(self).__name__}] {msg}")

pytextad/models/cvdd.py

@@ -0,0 +1,221 @@
+"""CVDD: Context Vector Data Description (Ruff et al., ACL 2019).
+
+Re-implementation for modern PyTorch. The model and training logic follow the
+official code, https://github.com/lukasruff/CVDD-PyTorch (MIT License,
+Copyright (c) 2019 lukasruff); see THIRD_PARTY_NOTICES.
+
+Input: a list of frozen token-embedding matrices, one [n_tokens, dim] array per
+document (GloVe/fastText vectors, or hidden states of a frozen PLM, e.g. from
+``pytextad.embeddings.TokenEmbedder``). Only the self-attention layer and the context
+vectors are trained, exactly as in the official code.
+
+Faithful to the official code
+  * self-attention  A = softmax_over_tokens(W2 tanh(W1 H)), r heads, no biases
+  * M = A H; cosine distance d_k = 0.5 (1 - cos(M_k, c_k))
+  * context vectors initialised by k-means on L2-normalised mean token embeddings
+    of the training set, then L2-normalised
+  * loss = mean_n sum_k softmax_k(-alpha d) d_k + lambda_p * mean((C C^T - I)^2)
+  * temperature alpha annealed at 5 equidistant milestones
+    (soft / linear / logarithmic / hard, same values as the official code)
+  * Adam with weight decay, gradient-norm clipping at 0.5, MultiStepLR(gamma=0.1)
+    stepped at the start of each epoch as in the official trainer (so with
+    PyTorch >= 1.1 the LR drop happens one epoch before the nominal milestone)
+  * anomaly score = mean_k d_k ("context_dist_mean", the official default)
+  * defaults = the settings in the official README for Reuters / 20 Newsgroups
+    (3 heads, attention size 150, lambda_p 1, logarithmic, 100 epochs, lr 0.01,
+    lr milestone 40, batch 64, weight decay 5e-7)
+
+Deliberate deviations
+  * Padding is masked out of the attention softmax. The official code pads with index
+    0 and does not mask. With static word vectors (GloVe, fastText) the pad vector is
+    zero, which only rescales M, so the cosine distances, the loss and the training
+    are identical (verified numerically with variable-length documents). With PLM
+    hidden states the official "bert" option feeds non-zero pad vectors into M, which
+    makes a document's score depend on the length of the others in its batch; we do
+    not reproduce that.
+  * Mini-batches are drawn uniformly at random and every document is scored. The
+    official loaders group documents by length (BucketBatchSampler) with
+    drop_last=True for both training and testing, so up to batch_size - 1 test
+    documents are never scored in the official evaluation.
+  * The official "context_best" score is NOT provided: it selects, per run, the
+    head with the highest AUROC on the labelled test set, which uses test labels.
+    ``head_scores()`` returns per-head distances if you need them.
+
+Extension (not in the CVDD paper)
+  * token_scores(): min over heads of the cosine distance between each token vector
+    and the context vectors. Comparable across documents. Use with care: it is our
+    definition, not part of CVDD.
+"""
+
+import warnings
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from sklearn.cluster import KMeans
+
+from .base import BaseTextDetector
+
+
+def _alpha_schedule(name, n_epochs):
+    milestones = list(np.arange(1, 6) * int(n_epochs / 5))
+    if name == "soft":
+        alphas = [0.0] * 5
+    elif name == "linear":
+        alphas = list(np.linspace(0.2, 1, 5))
+    elif name == "logarithmic":
+        alphas = list(np.logspace(-4, 0, 5))
+    elif name == "hard":
+        alphas = [100.0] * 4            # official list has 4 entries
+    else:
+        raise ValueError(f"unknown alpha_scheduler {name}")
+    return milestones, alphas
+
+
+class _CVDDNet(nn.Module):
+    def __init__(self, dim, attention_size, n_heads):
+        super().__init__()
+        self.W1 = nn.Linear(dim, attention_size, bias=False)
+        self.W2 = nn.Linear(attention_size, n_heads, bias=False)
+        self.c = nn.Parameter((torch.rand(n_heads, dim) - 0.5) * 2)
+        self.alpha = 0.0
+
+    def attend(self, H, mask):
+        # H [B, L, d], mask [B, L] (True = real token)
+        logits = self.W2(torch.tanh(self.W1(H)))                    # [B, L, r]
+        logits = logits.masked_fill(~mask.unsqueeze(-1), float("-inf"))
+        A = F.softmax(logits, dim=1).transpose(1, 2)                # [B, r, L]
+        return A @ H, A                                             # M [B, r, d]
+
+    def forward(self, H, mask):
+        M, A = self.attend(H, mask)
+        d = 0.5 * (1 - F.cosine_similarity(M, self.c.unsqueeze(0), dim=2))   # [B, r]
+        w = F.softmax(-self.alpha * d, dim=1)
+        return d, w, A
+
+
+class CVDD(BaseTextDetector):
+
+    def __init__(self, n_heads=3, attention_size=150, lambda_p=1.0,
+                 alpha_scheduler="logarithmic", n_epochs=100, lr=0.01, lr_milestones=(40,),
+                 batch_size=64, weight_decay=0.5e-6, contamination=0.1, random_state=0,
+                 device=None, verbose=False):
+        super().__init__(contamination, random_state, device, verbose)
+        self.n_heads = n_heads
+        self.attention_size = attention_size
+        self.lambda_p = lambda_p
+        self.alpha_scheduler = alpha_scheduler
+        self.n_epochs = n_epochs
+        self.lr = lr
+        self.lr_milestones = tuple(lr_milestones)
+        self.batch_size = batch_size
+        self.weight_decay = weight_decay
+
+    # ------------------------------------------------------------------ batching
+    @staticmethod
+    def _check(X):
+        X = [np.asarray(x, dtype=np.float32) for x in X]
+        if any(x.ndim != 2 or len(x) == 0 for x in X):
+            raise ValueError("X must be a list of non-empty [n_tokens, dim] arrays")
+        if len({x.shape[1] for x in X}) != 1:
+            raise ValueError("all token embeddings must have the same dimension")
+        return X
+
+    def _batch(self, X, idx):
+        L = max(len(X[i]) for i in idx)
+        H = np.zeros((len(idx), L, X[idx[0]].shape[1]), dtype=np.float32)
+        mask = np.zeros((len(idx), L), dtype=bool)
+        for j, i in enumerate(idx):
+            H[j, :len(X[i])] = X[i]
+            mask[j, :len(X[i])] = True
+        return torch.from_numpy(H).to(self.device), torch.from_numpy(mask).to(self.device)
+
+    def _batches(self, n, shuffle):
+        order = np.random.permutation(n) if shuffle else np.arange(n)
+        for s in range(0, n, self.batch_size):
+            yield order[s:s + self.batch_size]
+
+    # ------------------------------------------------------------------ training
+    def fit(self, X, y=None):
+        self._set_seed()
+        X = self._check(X)
+        dim = X[0].shape[1]
+        self.net_ = _CVDDNet(dim, self.attention_size, self.n_heads).to(self.device)
+
+        # context vector initialisation (official initialize_context_vectors)
+        means = np.stack([x.mean(0) for x in X])
+        means = means / np.clip(np.linalg.norm(means, axis=1, keepdims=True), 1e-8, None)
+        km = KMeans(n_clusters=self.n_heads, n_init=10, random_state=self.random_state).fit(means)
+        centers = km.cluster_centers_ / np.linalg.norm(km.cluster_centers_, axis=1, keepdims=True)
+        self.net_.c.data = torch.from_numpy(centers.astype(np.float32)).to(self.device)
+
+        opt = torch.optim.Adam(self.net_.parameters(), lr=self.lr, weight_decay=self.weight_decay)
+        sched = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=list(self.lr_milestones), gamma=0.1)
+        milestones, alphas = _alpha_schedule(self.alpha_scheduler, self.n_epochs)
+        alpha_i = 0
+        I = torch.eye(self.n_heads, device=self.device)
+
+        self.net_.alpha = 0.0
+        self.history_ = []
+        for epoch in range(self.n_epochs):
+            with warnings.catch_warnings():  # official order: step at the START of each epoch
+                warnings.simplefilter("ignore", UserWarning)
+                sched.step()
+            if epoch in milestones and alpha_i < len(alphas):     # official: one step per epoch
+                self.net_.alpha = float(alphas[alpha_i])
+                alpha_i += 1
+            self.net_.train()
+            tot, nb = 0.0, 0
+            for idx in self._batches(len(X), shuffle=True):
+                H, mask = self._batch(X, idx)
+                d, w, _ = self.net_(H, mask)
+                P = torch.mean((self.net_.c @ self.net_.c.t() - I) ** 2)
+                loss = torch.mean(torch.sum(w * d, dim=1)) + self.lambda_p * P
+                opt.zero_grad()
+                loss.backward()
+                torch.nn.utils.clip_grad_norm_(self.net_.parameters(), 0.5)   # as in the official trainer
+                opt.step()
+                tot += loss.item()
+                nb += 1
+            self.history_.append(tot / nb)
+            self._log(f"epoch {epoch + 1}/{self.n_epochs} loss {tot / nb:.6f} alpha {self.net_.alpha:g}")
+
+        return self._process_decision_scores(self.decision_function(X))
+
+    # ------------------------------------------------------------------ inference
+    @torch.no_grad()
+    def _forward_all(self, X):
+        X = self._check(X)
+        self.net_.eval()
+        D, A_all = [], []
+        for idx in self._batches(len(X), shuffle=False):
+            H, mask = self._batch(X, idx)
+            d, _, A = self.net_(H, mask)
+            D.append(d.cpu().numpy())
+            A = A.cpu().numpy()
+            A_all += [A[j, :, :len(X[i])] for j, i in enumerate(idx)]
+        return np.concatenate(D), A_all
+
+    def head_scores(self, X):
+        """Per-head cosine distances, shape [n_docs, n_heads]."""
+        return self._forward_all(X)[0]
+
+    def decision_function(self, X):
+        """Official 'context_dist_mean' score: mean cosine distance over heads."""
+        return self.head_scores(X).mean(1)
+
+    def attention(self, X):
+        """Attention weights per document, each [n_heads, n_tokens] (for inspection)."""
+        return self._forward_all(X)[1]
+
+    @torch.no_grad()
+    def token_scores(self, X):
+        """EXTENSION (not in the paper): per-token min_k 0.5 (1 - cos(h_t, c_k))."""
+        X = self._check(X)
+        C = F.normalize(self.net_.c.detach(), dim=1).cpu().numpy()
+        out = []
+        for x in X:
+            xn = x / np.clip(np.linalg.norm(x, axis=1, keepdims=True), 1e-8, None)
+            out.append((0.5 * (1 - xn @ C.T)).min(1))
+        return out

pytextad/models/date.py

@@ -0,0 +1,270 @@
+"""DATE: Detecting Anomalies in Text via Self-Supervision of Transformers
+(Manolache, Brad & Burceanu, NAACL 2021).
+
+Re-implementation for modern PyTorch / transformers (4.x and 5.x). Logic follows
+the official code, https://github.com/bit-ml/date (experiments/train_ag.py defaults
+and the modified simpletransformers fork it ships, Apache License 2.0).
+
+Faithful to the official code
+  * discriminator: ELECTRA encoder trained from scratch on inlier text
+    (embedding 128, hidden 256, 4 layers, 4 heads, FFN 1024, dropout 0.5, gelu),
+    RTD head dense-gelu-linear, RMD head on [CLS] linear-relu-linear
+  * pretext task: one of K=50 fixed masks (50% of the 128 positions, first position
+    always masked) chosen uniformly per sequence; masked positions get uniformly
+    random token ids in [5, vocab-1) ("random_generator=1", the default)
+  * loss = 50 * BCE(RTD) + 100 * CE(RMD); RTD labels are 0 where the random id
+    happens to equal the original token
+  * no attention mask is given to the encoder, and the RTD loss is taken over all
+    positions including [CLS] and padding, exactly as in the official training loop
+    (use_attention_mask=True changes this)
+  * AdamW (amsgrad, eps 1e-8), lr 1e-5, weight decay 0.1, and 0.01 (AdamW's default,
+    because the official parameter group omits the key) for bias/LayerNorm,
+    gradient-norm clipping 1.0, batch 16, 20 epochs; the official "plateau"
+    scheduler has patience 1e5 and therefore never changes the LR
+  * sliding windows of 128 tokens with stride int(0.8 * 130) = 104 for training and
+    testing; tokenizer bert-base-uncased
+  * test: no masking; window score = mean over positions 1..128 that are not padding
+    (so [SEP] is included unless it falls on position 129) of P(original token);
+    document score = mean over its windows. The official AUROC is computed on this
+    "PL_RTD" score with inliers as the positive class; decision_function returns
+    1 - score so that higher = more anomalous (identical AUROC).
+
+Removed relative to the official code (no effect with the default settings)
+  * The ELECTRA generator: with random_generator=1 its samples are overwritten by
+    random ids and its MLM loss is never added to the objective.
+
+Not reproduced
+  * The official code evaluates on the test set every 500 steps while training.
+    Here you get the model after the last epoch.
+
+Extension (not in the paper)
+  * token_scores(): per-subword P(replaced), max over overlapping windows, aggregated
+    to words with max.
+"""
+
+import pickle
+from typing import List, Sequence, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import ElectraConfig, ElectraModel
+
+from .base import BaseTextDetector
+from ..utils.embeddings import words_from_subwords
+
+Text = Union[str, Sequence[str]]
+
+
+class _DATENet(nn.Module):
+    def __init__(self, vocab_size, n_masks, hidden=256, layers=4, heads=4, intermediate=1024,
+                 emb=128, dropout=0.5):
+        super().__init__()
+        cfg = ElectraConfig(vocab_size=vocab_size, embedding_size=emb, hidden_size=hidden,
+                            num_hidden_layers=layers, num_attention_heads=heads,
+                            intermediate_size=intermediate, hidden_dropout_prob=dropout,
+                            attention_probs_dropout_prob=dropout, max_position_embeddings=512,
+                            hidden_act="gelu")
+        self.encoder = ElectraModel(cfg)
+        self.rtd_dense = nn.Linear(hidden, hidden)        # ElectraDiscriminatorPredictions
+        self.rtd_out = nn.Linear(hidden, 1)
+        self.rmd_fc1 = nn.Linear(hidden, hidden)          # ElectraRMD
+        self.rmd_fc2 = nn.Linear(hidden, n_masks)
+
+    def forward(self, input_ids, attention_mask=None):
+        h = self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state
+        rtd = self.rtd_out(F.gelu(self.rtd_dense(h))).squeeze(-1)    # [B, L]
+        rmd = self.rmd_fc2(F.relu(self.rmd_fc1(h[:, 0])))            # [B, K]
+        return rtd, rmd
+
+
+class DATE(BaseTextDetector):
+
+    def __init__(self, tokenizer="bert-base-uncased", max_len=128, stride=0.8,
+                 n_masks=50, mask_ratio=0.5, masks=None,
+                 hidden=256, layers=4, heads=4, intermediate=1024, emb=128, dropout=0.5,
+                 rtd_weight=50.0, rmd_weight=100.0, lr=1e-5, weight_decay=0.1,
+                 max_grad_norm=1.0, n_epochs=20, batch_size=16, use_attention_mask=False,
+                 contamination=0.1, random_state=0, device=None, verbose=False):
+        """
+        tokenizer : HF name or path, a vocab.txt path, or a fast tokenizer object.
+        masks     : None -> K random masks from random_state, or the path of the official
+                    experiments/pseudo_labels128_p50.pkl to use the exact official set.
+        """
+        super().__init__(contamination, random_state, device, verbose)
+        self.tokenizer = tokenizer
+        self.max_len = max_len
+        self.stride = stride
+        self.n_masks = n_masks
+        self.mask_ratio = mask_ratio
+        self.masks = masks
+        self.hidden, self.layers, self.heads = hidden, layers, heads
+        self.intermediate, self.emb, self.dropout = intermediate, emb, dropout
+        self.rtd_weight, self.rmd_weight = rtd_weight, rmd_weight
+        self.lr, self.weight_decay, self.max_grad_norm = lr, weight_decay, max_grad_norm
+        self.n_epochs, self.batch_size = n_epochs, batch_size
+        self.use_attention_mask = use_attention_mask
+
+    # ------------------------------------------------------------------ setup
+    def _load_tokenizer(self):
+        from transformers import AutoTokenizer, BertTokenizerFast
+        tok = self.tokenizer
+        if isinstance(tok, str):
+            if tok.endswith(".txt"):
+                vocab = {w.rstrip("\n"): i for i, w in enumerate(open(tok, encoding="utf-8"))}
+                try:                                   # transformers >= 5
+                    tok = BertTokenizerFast(vocab=vocab, do_lower_case=True)
+                except TypeError:
+                    tok = None
+                if tok is None or len(tok) < len(vocab):  # transformers 4.x
+                    tok = BertTokenizerFast(vocab_file=self.tokenizer, do_lower_case=True)
+            else:
+                tok = AutoTokenizer.from_pretrained(tok, use_fast=True)
+        if not getattr(tok, "is_fast", False):
+            raise ValueError("A fast tokenizer is required (for word alignment).")
+        self.tok_ = tok
+
+    def _build_masks(self):
+        L = self.max_len
+        if isinstance(self.masks, str):
+            raw = sorted(pickle.load(open(self.masks, "rb")), key=lambda m: m["label"])
+            M = np.array([m["mask"] for m in raw], dtype=bool)
+            if M.shape[1] != L:
+                raise ValueError(f"mask length {M.shape[1]} != max_len {L}")
+        else:
+            rng = np.random.RandomState(self.random_state)
+            k = int(self.mask_ratio * L)
+            M = np.zeros((self.n_masks, L), dtype=bool)
+            for i in range(self.n_masks):
+                M[i, rng.choice(L, k, replace=False)] = True
+        M = M.copy()
+        M[:, 0] = True                                  # official: first content position always masked
+        full = np.zeros((M.shape[0], L + 2), dtype=bool)
+        full[:, 1:L + 1] = M
+        self.masks_ = torch.from_numpy(full)
+        self.n_masks_ = M.shape[0]
+
+    # ------------------------------------------------------------------ windowing
+    def _encode(self, texts: List[Text]):
+        out = []
+        for t in texts:
+            split = not isinstance(t, str)
+            enc = self.tok_(list(t) if split else t, is_split_into_words=split,
+                            add_special_tokens=False, truncation=False)
+            out.append((enc["input_ids"], enc.word_ids()))
+        return out
+
+    def _windows(self, ids):
+        """encode_sliding_window_custom: windows of max_len, step int((max_len + 2) * stride)."""
+        L = self.max_len
+        if len(ids) <= L:
+            return [(0, ids)]
+        step = max(1, int((L + 2) * self.stride))
+        return [(s, ids[s:s + L]) for s in range(0, len(ids), step)]
+
+    def _pack(self, chunks):
+        cls, sep, pad = self.tok_.cls_token_id, self.tok_.sep_token_id, self.tok_.pad_token_id
+        X = torch.full((len(chunks), self.max_len + 2), pad, dtype=torch.long)
+        for i, c in enumerate(chunks):
+            seq = [cls] + list(c) + [sep]
+            X[i, :len(seq)] = torch.tensor(seq)
+        return X, X != pad
+
+    # ------------------------------------------------------------------ training
+    def _corrupt(self, X, nonpad):
+        labels = torch.randint(0, self.n_masks_, (X.shape[0],))
+        m = self.masks_[labels] & nonpad                # padding is never replaced
+        rand = torch.randint(5, self.vocab_size_ - 1, X.shape)
+        Xc = X.clone()
+        Xc[m] = rand[m]
+        return Xc, (m & (Xc != X)).float(), labels
+
+    def _loss(self, Xc, nonpad, rtd_y, rmd_y):
+        am = nonpad.long() if self.use_attention_mask else None
+        rtd, rmd = self.net_(Xc, am)
+        if self.use_attention_mask:
+            l_rtd = F.binary_cross_entropy_with_logits(rtd[nonpad], rtd_y[nonpad])
+        else:                                           # official: all positions
+            l_rtd = F.binary_cross_entropy_with_logits(rtd, rtd_y)
+        l_rmd = F.cross_entropy(rmd, rmd_y)
+        return self.rtd_weight * l_rtd + self.rmd_weight * l_rmd
+
+    def _make_optimizer(self):
+        no_decay = ("bias", "LayerNorm.weight")
+        named = list(self.net_.named_parameters())
+        return torch.optim.AdamW(
+            [{"params": [p for n, p in named if not any(k in n for k in no_decay)], "weight_decay": self.weight_decay},
+             # the official group for bias/LayerNorm has no "weight_decay" key, so it gets
+             # torch.optim.AdamW's default 0.01 (not 0)
+             {"params": [p for n, p in named if any(k in n for k in no_decay)], "weight_decay": 0.01}],
+            lr=self.lr, eps=1e-8, amsgrad=True)
+
+    def _train_step(self, Xc, nonpad, rtd_y, rmd_y, opt):
+        loss = self._loss(Xc, nonpad, rtd_y, rmd_y)
+        opt.zero_grad()
+        loss.backward()
+        torch.nn.utils.clip_grad_norm_(self.net_.parameters(), self.max_grad_norm)
+        opt.step()
+        return loss.item()
+
+    def fit(self, X: List[Text], y=None):
+        self._set_seed()
+        self._load_tokenizer()
+        self.vocab_size_ = len(self.tok_)
+        self._build_masks()
+        windows = [w for ids, _ in self._encode(X) for _, w in self._windows(ids) if len(w) > 0]
+        data, nonpad = self._pack(windows)
+
+        self.net_ = _DATENet(self.vocab_size_, self.n_masks_, self.hidden, self.layers, self.heads,
+                             self.intermediate, self.emb, self.dropout).to(self.device)
+        opt = self._make_optimizer()
+
+        n = data.shape[0]
+        self.history_ = []
+        for ep in range(self.n_epochs):
+            self.net_.train()
+            perm = torch.randperm(n)
+            tot = 0.0
+            for s in range(0, n, self.batch_size):
+                idx = perm[s:s + self.batch_size]
+                Xc, rtd_y, rmd_y = self._corrupt(data[idx], nonpad[idx])
+                loss = self._train_step(Xc.to(self.device), nonpad[idx].to(self.device),
+                                        rtd_y.to(self.device), rmd_y.to(self.device), opt)
+                tot += loss * len(idx)
+            self.history_.append(tot / n)
+            self._log(f"epoch {ep + 1}/{self.n_epochs} loss {tot / n:.4f} ({n} windows)")
+        return self._process_decision_scores(self.decision_function(X))
+
+    # ------------------------------------------------------------------ inference
+    @torch.no_grad()
+    def _score(self, X: List[Text], batch_size=64):
+        self.net_.eval()
+        enc = self._encode(X)
+        jobs = [(d, s, w) for d, (ids, _) in enumerate(enc) for s, w in self._windows(ids) if len(w) > 0]
+        sub = [np.full(len(ids), -np.inf) for ids, _ in enc]
+        win = [[] for _ in enc]
+        L2 = self.max_len + 2
+        for b in range(0, len(jobs), batch_size):
+            part = jobs[b:b + batch_size]
+            xb, nonpad = self._pack([j[2] for j in part])
+            am = nonpad.long().to(self.device) if self.use_attention_mask else None
+            rtd, _ = self.net_(xb.to(self.device), am)
+            p_rep = torch.sigmoid(rtd).cpu().numpy()
+            for (d, s, w), pr, npd in zip(part, p_rep, nonpad.numpy()):
+                keep = npd.copy()
+                keep[0] = False
+                keep[L2 - 1] = False                    # official skips positions 0 and 129
+                win[d].append(1.0 - pr[keep].mean())    # PL_RTD = mean P(original)
+                sub[d][s:s + len(w)] = np.maximum(sub[d][s:s + len(w)], pr[1:1 + len(w)])
+        return win, sub, [e[1] for e in enc]
+
+    def decision_function(self, X: List[Text]):
+        win, _, _ = self._score(X)
+        return np.array([1.0 - np.mean(w) if w else np.nan for w in win])
+
+    def token_scores(self, X: List[Text], agg="max"):
+        """EXTENSION: per-word P(replaced). For word lists the output has one score per word."""
+        _, sub, wids = self._score(X)
+        return [words_from_subwords(p, wid, len(x) if not isinstance(x, str) else None, agg)
+                for x, p, wid in zip(X, sub, wids)]

pytextad/models/fate.py

@@ -0,0 +1,171 @@
+"""FATE: Few-shot Anomaly detection in TExt (Das, Ajay, Saha & Bhuyan, ICONIP 2023).
+
+Re-implementation of the official code, https://github.com/arav1ndajay/fate
+(src/main.py, deviation_loss.py, balanced_sampler.py). The repository has no
+licence file, so no code is copied; the logic is re-written and was checked
+numerically against it (see tests/).
+
+Faithful to the official code
+  * encoder: a SentenceTransformer, fine-tuned end to end; default
+    sentence-transformers/all-MiniLM-L6-v2 (the one hard-coded in main.py);
+    h = its token embeddings (last transformer layer, before pooling)
+  * inputs padded to max_length=128 and the attention softmax runs over all 128
+    positions, padding included (mask_padding=True changes this)
+  * A = softmax_over_tokens(tanh(h W1) W2), attention size 150, 5 heads, no biases;
+    S = A h flattened (heads x dim); score = mean of the top 10 % of |S|
+    (the code takes absolute values; the paper's formula does not)
+  * deviation loss: a new reference sample of 5000 N(0, 1) draws at EVERY batch;
+    Z = (score - mean(ref)) / std(ref); loss = mean((1-y)|Z| + y max(0, 5 - Z))
+  * regulariser mean((A A^T - I)^2) (a mean, not the paper's Frobenius sum), weight 1
+  * balanced batches: 8 inliers + 8 labelled anomalies, each drawn from its own
+    endlessly re-shuffled list; steps per epoch = (n_inliers + n_anomalies) // 16
+  * Adam, lr 1e-6, batch 16; epochs in main.py: 4 for AG News, 50 for 20 Newsgroups,
+    40 for Reuters, 80 whenever there are < 500 training inliers (default here: 4)
+  * test score = the network output (no Z-transform), higher = more anomalous
+
+Not reproduced
+  * main.py evaluates on the test set every 600 steps during training; here you get
+    the model after the last epoch.
+
+Unsupervised use (y=None or no 1 in y)
+  The official sampler cannot run without labelled anomalies. We then fill whole
+  batches with inliers and train only the |Z| term: this is the "FATE*" variant of
+  NLP-ADBench, not the method of the paper, and should be reported as such.
+"""
+
+from typing import List, Optional
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .base import BaseTextDetector
+
+
+class _FATENet(nn.Module):
+    def __init__(self, encoder, attention_size, n_heads, top_k, mask_padding):
+        super().__init__()
+        self.encoder = encoder
+        d = encoder.config.hidden_size
+        self.W1 = nn.Linear(d, attention_size, bias=False)
+        self.W2 = nn.Linear(attention_size, n_heads, bias=False)
+        self.top_k = top_k
+        self.mask_padding = mask_padding
+
+    def forward(self, input_ids, attention_mask):
+        h = self.encoder(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state   # [B, T, d]
+        logits = self.W2(torch.tanh(self.W1(h)))                                                 # [B, T, m]
+        if self.mask_padding:
+            logits = logits.masked_fill(attention_mask.unsqueeze(-1) == 0, float("-inf"))
+        A = F.softmax(logits, dim=1).transpose(1, 2)                                              # [B, m, T]
+        S = (A @ h).flatten(1)                                                                    # [B, m*d]
+        k = max(int(S.size(1) * self.top_k), 1)
+        return torch.topk(S.abs(), k, dim=1)[0].mean(1).float(), A
+
+
+def _endless(idx):
+    while True:
+        for i in np.random.permutation(idx):
+            yield i
+
+
+class FATE(BaseTextDetector):
+
+    def __init__(self, encoder="sentence-transformers/all-MiniLM-L6-v2", max_length=128,
+                 attention_size=150, n_heads=5, top_k=0.1, margin=5.0, n_ref=5000,
+                 include_regularization=True, mask_padding=False, lr=1e-6, batch_size=16,
+                 n_epochs=4, contamination=0.1, random_state=0, device=None, verbose=False):
+        """encoder: Hugging Face id or local path of the (Sentence-)BERT encoder; it is fine-tuned."""
+        super().__init__(contamination, random_state, device, verbose)
+        self.encoder = encoder
+        self.max_length = max_length
+        self.attention_size = attention_size
+        self.n_heads = n_heads
+        self.top_k = top_k
+        self.margin = margin
+        self.n_ref = n_ref
+        self.include_regularization = include_regularization
+        self.mask_padding = mask_padding
+        self.lr = lr
+        self.batch_size = batch_size
+        self.n_epochs = n_epochs
+
+    # ------------------------------------------------------------------ pieces
+    def _tok(self, texts):
+        pad = "longest" if self.mask_padding else "max_length"     # official: pad to 128
+        return self.tokenizer_(list(texts), padding=pad, truncation=True, max_length=self.max_length,
+                               return_tensors="pt")
+
+    def _build(self):
+        from transformers import AutoModel, AutoTokenizer
+        self.tokenizer_ = AutoTokenizer.from_pretrained(self.encoder)
+        enc = AutoModel.from_pretrained(self.encoder)
+        self.net_ = _FATENet(enc, self.attention_size, self.n_heads, self.top_k, self.mask_padding).to(self.device)
+
+    def _loss(self, input_ids, attention_mask, yb):
+        psi, A = self.net_(input_ids, attention_mask)
+        ref = torch.normal(mean=0.0, std=torch.full([self.n_ref], 1.0)).to(self.device)   # redrawn every batch
+        z = (psi - ref.mean()) / ref.std()
+        loss = torch.mean((1 - yb) * z.abs() + yb * (self.margin - z).clamp(min=0.0))
+        if self.include_regularization:
+            I = torch.eye(self.n_heads, device=self.device)
+            loss = loss + torch.mean((A @ A.transpose(1, 2) - I) ** 2)
+        return loss
+
+    def _train_steps(self, batches, opt):
+        """batches: iterable of (input_ids, attention_mask, labels) tensors."""
+        self.net_.train()
+        tot, nb = 0.0, 0
+        for ids, am, yb in batches:
+            opt.zero_grad()
+            loss = self._loss(ids.to(self.device), am.to(self.device), yb.float().to(self.device))
+            loss.backward()
+            opt.step()
+            tot += loss.item()
+            nb += 1
+        return tot / max(nb, 1)
+
+    # ------------------------------------------------------------------ API
+    def fit(self, X: List[str], y: Optional[np.ndarray] = None):
+        """X: texts. y: 1 = labelled anomaly, 0 = inlier (None = all inliers -> FATE*)."""
+        self._set_seed()
+        X = list(X)
+        y = np.zeros(len(X), dtype=int) if y is None else np.asarray(y, dtype=int)
+        if len(y) != len(X):
+            raise ValueError("X and y have different lengths")
+        inl, out = np.where(y == 0)[0], np.where(y == 1)[0]
+        self.few_shot_ = len(out) > 0
+        if not self.few_shot_:
+            self._log("no labelled anomalies: training the unsupervised FATE* variant")
+        self._build()
+        enc = self._tok(X)
+        opt = torch.optim.Adam(self.net_.parameters(), lr=self.lr)
+
+        n_in = self.batch_size // 2 if self.few_shot_ else self.batch_size
+        gen_in, gen_out = _endless(inl), (_endless(out) if self.few_shot_ else None)
+        steps = len(X) // self.batch_size
+
+        def batches():
+            for _ in range(steps):
+                idx = [next(gen_in) for _ in range(n_in)]
+                if self.few_shot_:
+                    idx += [next(gen_out) for _ in range(self.batch_size - n_in)]
+                idx = torch.tensor(idx)
+                yield enc["input_ids"][idx], enc["attention_mask"][idx], torch.from_numpy(y[idx.numpy()])
+
+        self.history_ = []
+        for ep in range(self.n_epochs):
+            self.history_.append(self._train_steps(batches(), opt))
+            self._log(f"epoch {ep + 1}/{self.n_epochs} loss {self.history_[-1]:.4f}")
+        return self._process_decision_scores(self.decision_function(X))
+
+    @torch.no_grad()
+    def decision_function(self, X: List[str], batch_size=16):
+        self.net_.eval()
+        X = list(X)
+        out = []
+        for s in range(0, len(X), batch_size):
+            t = self._tok(X[s:s + batch_size])
+            out.append(self.net_(t["input_ids"].to(self.device), t["attention_mask"].to(self.device))[0].cpu().numpy())
+        return np.concatenate(out)

pytextad/models/rsrae.py

@@ -0,0 +1,213 @@
+"""RSRAE: Robust Subspace Recovery AutoEncoder (Lai, Zou & Lerman, ICLR 2020).
+
+PyTorch port of the official TensorFlow code, https://github.com/dmzou/RSRAE
+(MIT License, Copyright (c) 2019-present Chieh-Hsin Lai, Dongmian Zou and
+Gilad Lerman); see THIRD_PARTY_NOTICES. Input is a vector per document
+(sentence embeddings, TF-IDF, ...); RSRAE itself is not text-specific.
+
+Faithful to the official code (RSRAE/model.py + experiments.py defaults)
+  * encoder  d -> 32 -> 64 -> 128 (dense + activation + BN), RSR layer y A with
+    A ~ N(0, 1) of shape [128, intrinsic_size], L2 re-normalisation of z,
+    decoder 128 -> 64 -> 32 -> d (activation also on the output layer)
+  * activation chosen from the training data exactly as experiments.py does:
+    relu if min(X) >= 0, else tanh if |max(X)| <= 1, else leaky_relu(0.2)
+  * losses with the L2,1 norm (mean over samples of the non-squared L2 norm):
+      reconstruction ||x - x~||, PCA ||y - A A^T y||, projection mean((A^T A - I)^2)
+  * all_alt=True, enforce_proj=True (defaults): every mini-batch runs three Adam steps
+    with three separate optimisers: reconstruction on all weights (lr), projection on
+    A (10 lr), PCA error on encoder + A (10 lr)
+  * lr 2.5e-4, 200 epochs, batch 128, intrinsic size 10, Glorot-uniform weights,
+    zero biases, one shuffle before training (the official code does not reshuffle)
+  * anomaly score = -cos(x, x~)
+  * bn_mode="official": the official code calls Keras BatchNormalization in a TF1
+    graph without a training flag, so it always runs in inference mode with moving
+    statistics that are never updated (mean 0, variance 1). We verified this by
+    running the official code. The layer is therefore a trainable affine map
+    gamma * x / sqrt(1 + 1e-3) + beta, which is what "official" reproduces.
+    bn_mode="batch" uses real batch normalisation; bn_mode=None disables it.
+
+Note on protocol: the official experiments fit RSRAE on the (contaminated) test set
+and score the same data. Here fit() and decision_function() are separate, so use
+fit(X).decision_scores_ for that transductive protocol.
+"""
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .base import BaseTextDetector
+
+
+class _FrozenStatBN(nn.Module):
+    """Keras BatchNormalization in inference mode with untouched moving statistics."""
+
+    def __init__(self, n, eps=1e-3):
+        super().__init__()
+        self.gamma = nn.Parameter(torch.ones(n))
+        self.beta = nn.Parameter(torch.zeros(n))
+        self.scale = 1.0 / np.sqrt(1.0 + eps)
+
+    def forward(self, x):
+        return self.gamma * x * self.scale + self.beta
+
+
+def _bn(n, mode):
+    if mode is None:
+        return nn.Identity()
+    if mode == "official":
+        return _FrozenStatBN(n)
+    if mode == "batch":
+        return nn.BatchNorm1d(n, eps=1e-3, momentum=0.01)   # Keras momentum 0.99
+    raise ValueError(f"unknown bn_mode {mode}")
+
+
+def _dense(i, o):
+    layer = nn.Linear(i, o)
+    nn.init.xavier_uniform_(layer.weight)
+    nn.init.zeros_(layer.bias)
+    return layer
+
+
+class _RSRAENet(nn.Module):
+    def __init__(self, d, hidden, intrinsic, act, normalize, bn_mode):
+        super().__init__()
+        h0, h1, h2 = hidden
+        self.act = act
+        self.normalize = normalize
+        self.enc = nn.ModuleList([_dense(d, h0), _dense(h0, h1), _dense(h1, h2)])
+        self.enc_bn = nn.ModuleList([_bn(h0, bn_mode), _bn(h1, bn_mode), _bn(h2, bn_mode)])
+        self.A = nn.Parameter(torch.randn(h2, intrinsic))
+        self.dec = nn.ModuleList([_dense(intrinsic, h2), _dense(h2, h1), _dense(h1, h0)])
+        self.dec_bn = nn.ModuleList([_bn(h2, bn_mode), _bn(h1, bn_mode), _bn(h0, bn_mode)])
+        self.out = _dense(h0, d)
+
+    def encode(self, x):
+        for lin, bn in zip(self.enc, self.enc_bn):
+            x = bn(self.act(lin(x)))
+        return x
+
+    def forward(self, x):
+        y = self.encode(x)
+        y_rsr = y @ self.A
+        z = F.normalize(y_rsr, p=2, dim=-1, eps=1e-12) if self.normalize else y_rsr
+        h = z
+        for lin, bn in zip(self.dec, self.dec_bn):
+            h = bn(self.act(lin(h)))
+        return y, y_rsr, z, self.act(self.out(h))
+
+    def encoder_params(self):
+        return list(self.enc.parameters()) + list(self.enc_bn.parameters())
+
+
+def _norm_loss(diff, kind):
+    kind = kind.lower()
+    if kind in ("mse", "frob", "f"):
+        return diff.norm(dim=1).pow(2).mean()
+    if kind == "l1":
+        return diff.abs().sum(1).mean()
+    if kind in ("l21", "lad", "l2"):
+        return diff.norm(dim=1).mean()
+    raise ValueError(f"unknown norm {kind}")
+
+
+class RSRAE(BaseTextDetector):
+
+    def __init__(self, hidden_layer_sizes=(32, 64, 128), intrinsic_size=10,
+                 loss_norm_type="L21", norm_type="L21", all_alt=True, enforce_proj=True,
+                 lambda1=0.0025, lambda2=0.1, lr=2.5e-4, n_epochs=200, batch_size=128,
+                 normalize=True, bn_mode="official", activation="auto",
+                 contamination=0.1, random_state=0, device=None, verbose=False):
+        """
+        activation : "auto" (official rule), or "relu" / "tanh" / "leaky_relu".
+        lambda1, lambda2 are only used when all_alt=False (joint loss), as in the
+        official code.
+        """
+        super().__init__(contamination, random_state, device, verbose)
+        self.hidden_layer_sizes = tuple(hidden_layer_sizes)
+        self.intrinsic_size = intrinsic_size
+        self.loss_norm_type = loss_norm_type
+        self.norm_type = norm_type
+        self.all_alt = all_alt
+        self.enforce_proj = enforce_proj
+        self.lambda1 = lambda1
+        self.lambda2 = lambda2
+        self.lr = lr
+        self.n_epochs = n_epochs
+        self.batch_size = batch_size
+        self.normalize = normalize
+        self.bn_mode = bn_mode
+        self.activation = activation
+
+    def _pick_activation(self, X):
+        name = self.activation
+        if name == "auto":
+            if X.min() >= 0:
+                name = "relu"
+            elif abs(X.max()) <= 1:          # official rule checks max only
+                name = "tanh"
+            else:
+                name = "leaky_relu"
+        self.activation_ = name
+        return {"relu": F.relu, "tanh": torch.tanh,
+                "leaky_relu": lambda t: F.leaky_relu(t, 0.2)}[name]
+
+    def _pca_error(self, y, y_rsr):
+        return _norm_loss(y - y_rsr @ self.net_.A.t(), self.norm_type)
+
+    def _proj_error(self):
+        A = self.net_.A
+        return torch.mean((A.t() @ A - torch.eye(A.shape[1], device=A.device)) ** 2)
+
+    def fit(self, X, y=None):
+        self._set_seed()
+        X = np.asarray(X, dtype=np.float32)
+        act = self._pick_activation(X)
+        self.net_ = _RSRAENet(X.shape[1], self.hidden_layer_sizes, self.intrinsic_size, act,
+                              self.normalize, self.bn_mode).to(self.device)
+        Xt = torch.from_numpy(X).to(self.device)
+        idx = np.random.permutation(len(X))           # official: shuffled once
+        self._train_loop(Xt, idx)
+        return self._process_decision_scores(self.decision_function(X))
+
+    def _train_loop(self, Xt, idx):
+        net = self.net_
+        opt_main = torch.optim.Adam(net.parameters(), lr=self.lr, eps=1e-8)
+        opt_proj = torch.optim.Adam([net.A], lr=10 * self.lr, eps=1e-8)
+        opt_pca = torch.optim.Adam(net.encoder_params() + [net.A], lr=10 * self.lr, eps=1e-8)
+
+        n_batch = (len(Xt) - 1) // self.batch_size + 1
+        net.train()
+        for epoch in range(self.n_epochs):
+            for b in range(n_batch):
+                xb = Xt[idx[b * self.batch_size:(b + 1) * self.batch_size]]
+                yv, y_rsr, _, x_rec = net(xb)
+                loss = _norm_loss(xb - x_rec, self.loss_norm_type)
+                if not self.all_alt:
+                    loss = loss + self.lambda1 * self._pca_error(yv, y_rsr) + self.lambda2 * self._proj_error()
+                opt_main.zero_grad()
+                loss.backward()
+                opt_main.step()
+                if self.all_alt and self.enforce_proj:
+                    opt_proj.zero_grad()
+                    self._proj_error().backward()
+                    opt_proj.step()
+                if self.all_alt:
+                    yv, y_rsr, _, _ = net(xb)
+                    opt_pca.zero_grad()
+                    self._pca_error(yv, y_rsr).backward()
+                    opt_pca.step()
+            if self.verbose and (epoch + 1) % 20 == 0:
+                self._log(f"epoch {epoch + 1}/{self.n_epochs} last-batch loss {loss.item():.4f}")
+
+    @torch.no_grad()
+    def reconstruct(self, X):
+        self.net_.eval()
+        Xt = torch.as_tensor(np.asarray(X, dtype=np.float32), device=self.device)
+        return torch.cat([self.net_(Xt[s:s + 1024])[3] for s in range(0, len(Xt), 1024)]).cpu().numpy()
+
+    def decision_function(self, X):
+        X = np.asarray(X, dtype=np.float32)
+        R = self.reconstruct(X)
+        cos = (X * R).sum(1) / (np.linalg.norm(R, axis=1) + 1e-6) / (np.linalg.norm(X, axis=1) + 1e-6)
+        return -cos

pytextad/utils/__init__.py

@@ -0,0 +1,3 @@
+from .embeddings import TokenEmbedder, mean_pool, words_from_subwords
+
+__all__ = ["TokenEmbedder", "mean_pool", "words_from_subwords"]

pytextad/utils/embeddings.py

@@ -0,0 +1,98 @@
+"""Frozen token-embedding extraction (input for CVDD and other token-sequence detectors).
+
+Returns, per document, the last-layer hidden states of a frozen Hugging Face encoder,
+as a float32 array of shape [n_tokens, dim], plus the word index of every subword
+(so that token scores can be aggregated back to words).
+"""
+
+from typing import List, Sequence, Union
+
+import numpy as np
+import torch
+
+Text = Union[str, Sequence[str]]
+
+
+class TokenEmbedder:
+
+    def __init__(self, model_name_or_path, layer=-1, max_length=512, batch_size=32,
+                 keep_special_tokens=False, device=None, dtype=torch.float32, cache_dir=None):
+        """
+        model_name_or_path : any Hugging Face encoder (bert-base-uncased, roberta-base, ...).
+        layer              : hidden layer to return (-1 = last, as in official CVDD-BERT).
+        max_length         : longer documents are truncated; the number truncated is
+                             stored in ``n_truncated_`` after each call.
+        keep_special_tokens: keep [CLS]/[SEP] (<s>/</s>) vectors. Default False.
+        cache_dir          : Hugging Face cache folder, as in from_pretrained(..., cache_dir=...).
+        """
+        from transformers import AutoModel, AutoTokenizer
+        self._path, self._cache = model_name_or_path, cache_dir
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, use_fast=True, cache_dir=cache_dir)
+        if not self.tokenizer.is_fast:
+            raise ValueError("A fast tokenizer is required for word alignment.")
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+        try:                                   # transformers >= 4.56 / 5.x
+            model = AutoModel.from_pretrained(model_name_or_path, dtype=dtype, cache_dir=cache_dir)
+        except TypeError:
+            model = AutoModel.from_pretrained(model_name_or_path, torch_dtype=dtype, cache_dir=cache_dir)
+        self.model = model.to(self.device).eval()
+        for p in self.model.parameters():
+            p.requires_grad = False
+        self.layer = layer
+        self.max_length = max_length
+        self.batch_size = batch_size
+        self.keep_special_tokens = keep_special_tokens
+
+    @torch.no_grad()
+    def transform(self, texts: List[Text]):
+        """Returns (embeddings, word_ids): two lists with one entry per document."""
+        embs, wids = [], []
+        self.n_truncated_ = 0
+        for b in range(0, len(texts), self.batch_size):
+            batch = list(texts[b:b + self.batch_size])
+            split = not isinstance(batch[0], str)
+            if split and getattr(self.tokenizer, "add_prefix_space", None) is False:
+                # byte-level BPE tokenizers (RoBERTa, GPT-2) need this for pre-split words
+                from transformers import AutoTokenizer
+                self.tokenizer = AutoTokenizer.from_pretrained(self._path, use_fast=True, add_prefix_space=True,
+                                                               cache_dir=self._cache)
+            enc = self.tokenizer(batch, is_split_into_words=split, padding=True, truncation=True,
+                                 max_length=self.max_length, return_tensors="pt",
+                                 return_special_tokens_mask=True, return_overflowing_tokens=False)
+            full = self.tokenizer(batch, is_split_into_words=split, add_special_tokens=True)["input_ids"]
+            self.n_truncated_ += sum(len(f) > self.max_length for f in full)
+            special = enc.pop("special_tokens_mask")
+            inputs = {k: v.to(self.device) for k, v in enc.items()}
+            out = self.model(**inputs, output_hidden_states=True)
+            H = out.hidden_states[self.layer].float().cpu().numpy()
+            for i in range(len(batch)):
+                keep = enc["attention_mask"][i].bool()
+                if not self.keep_special_tokens:
+                    keep &= ~special[i].bool()
+                idx = keep.nonzero().squeeze(-1).numpy()
+                embs.append(H[i, idx].astype(np.float32))
+                wi = enc.word_ids(i)
+                wids.append([wi[j] for j in idx])
+        return embs, wids
+
+
+def mean_pool(token_embeddings):
+    """Document vector = mean of its token vectors (for RSRAE and other vector detectors)."""
+    return np.stack([e.mean(0) for e in token_embeddings])
+
+
+def words_from_subwords(scores, word_ids, n_words=None, agg="max"):
+    """Aggregate one document's subword scores to word scores (max or mean)."""
+    valid = [w for w in word_ids if w is not None]
+    n = n_words if n_words is not None else (max(valid) + 1 if valid else 0)
+    out = np.full(n, -np.inf) if agg == "max" else np.zeros(n)
+    cnt = np.zeros(n)
+    for s, w in zip(scores, word_ids):
+        if w is None:
+            continue
+        out[w] = max(out[w], s) if agg == "max" else out[w] + s
+        cnt[w] += 1
+    if agg == "mean":
+        out = out / np.maximum(cnt, 1)
+    out[cnt == 0] = np.nan          # words lost to truncation have no score
+    return out

pytextad/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

pytextad-0.1.0.dist-info/METADATA

@@ -0,0 +1,108 @@
+Metadata-Version: 2.4
+Name: pytextad
+Version: 0.1.0
+Summary: A unified library for text anomaly detection (document- and token-level), in the style of PyOD
+Author: Yang Cao
+License-Expression: BSD-2-Clause
+Project-URL: Homepage, https://github.com/charles-cao/pytextad
+Project-URL: Documentation, https://pytextad.readthedocs.io
+Keywords: anomaly detection,outlier detection,text,NLP,token-level
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: THIRD_PARTY_NOTICES.md
+Requires-Dist: numpy>=1.21
+Requires-Dist: scikit-learn>=1.0
+Requires-Dist: torch>=1.13
+Requires-Dist: transformers>=4.30
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+Provides-Extra: docs
+Requires-Dist: sphinx>=7; extra == "docs"
+Requires-Dist: furo; extra == "docs"
+Requires-Dist: myst-parser; extra == "docs"
+Dynamic: license-file
+
+# PyTextAD: Text Anomaly Detection in Python
+
+[![PyPI](https://img.shields.io/pypi/v/pytextad.svg)](https://pypi.org/project/pytextad/)
+[![Documentation](https://readthedocs.org/projects/pytextad/badge/?version=latest)](https://pytextad.readthedocs.io)
+[![Tests](https://github.com/charles-cao/pytextad/actions/workflows/tests.yml/badge.svg)](https://github.com/charles-cao/pytextad/actions/workflows/tests.yml)
+[![License](https://img.shields.io/badge/license-BSD--2--Clause-blue.svg)](LICENSE)
+
+**PyTextAD** is a Python library for detecting anomalies in text, at the document
+and at the token level. Every detector follows the PyOD interface
+(`fit`, `decision_function`, `predict`, `decision_scores_`, `labels_`), and every
+re-implemented method is checked numerically against its original code.
+
+## Installation
+
+```bash
+pip install pytextad
+```
+
+From source:
+
+```bash
+git clone https://github.com/charles-cao/pytextad.git
+cd pytextad
+pip install -e ".[test]"
+```
+
+Requires Python >= 3.9, PyTorch >= 1.13 and transformers >= 4.30. Install the
+PyTorch build that matches your CUDA version first (https://pytorch.org).
+
+## Quick start
+
+```python
+from pytextad import CVDD, DATE, FATE, RSRAE, TokenEmbedder, mean_pool
+
+# frozen token embeddings from any Hugging Face encoder
+emb = TokenEmbedder("bert-base-uncased")
+H_train, _ = emb.transform(train_texts)
+H_test, _ = emb.transform(test_texts)
+
+scores = CVDD().fit(H_train).decision_function(H_test)                       # higher = more anomalous
+scores = RSRAE().fit(mean_pool(H_train)).decision_function(mean_pool(H_test))
+scores = DATE().fit(train_texts).decision_function(test_texts)               # raw text in, trains its own model
+scores = FATE().fit(texts, y).decision_function(test_texts)                  # few-shot: y = 1 for labelled anomalies
+
+word_scores = DATE().fit(train_texts).token_scores([t.split() for t in test_texts])
+```
+
+A runnable example on AG News: `python examples/quickstart.py`.
+
+## Implemented methods
+
+| Method | Year | Input | Token scores | Reference |
+|---|---|---|---|---|
+| CVDD | 2019 | frozen token embeddings | yes | Ruff et al., *Self-Attentive, Multi-Context One-Class Classification for Unsupervised Anomaly Detection on Text*, ACL 2019 |
+| RSRAE | 2020 | document vectors | no | Lai et al., *Robust Subspace Recovery Layer for Unsupervised Anomaly Detection*, ICLR 2020 |
+| DATE | 2021 | raw text | yes | Manolache et al., *DATE: Detecting Anomalies in Text via Self-Supervision of Transformers*, NAACL 2021 |
+| FATE | 2023 | raw text (+ few labelled anomalies) | no | Das et al., *Few-shot Anomaly Detection in Text with Deviation Learning*, ICONIP 2023 |
+
+Default hyperparameters are those of the official code. Each module's docstring
+lists where the official code and the paper disagree and which one we follow.
+
+## Faithfulness to the original implementations
+
+`tests/verification/` runs each original implementation next to ours with the same
+weights, inputs and random seeds and compares the results (DATE against the original
+transformers 3.0.2 code, RSRAE against the original TensorFlow code). All checks
+pass; see [tests/verification/README.md](tests/verification/README.md).
+
+## Running the tests
+
+```bash
+pytest                               # fast API tests, a few seconds
+PYTEXTAD_DEVICE=cuda pytest            # same, on GPU (PowerShell: $env:PYTEXTAD_DEVICE="cuda"; pytest)
+```
+
+## License
+
+BSD 2-Clause. Third-party notices for the original implementations are in
+[THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md).

pytextad-0.1.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+pytextad/__init__.py,sha256=naJhcjwsw2ovsXhp7iClkqnZ-PYFUuBbM7ikF6PAoek,428
+pytextad/version.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
+pytextad/models/__init__.py,sha256=RxAI0AKf2DexbxVA-MNWIxk2VQ-BFc32LBswFfdKy5w,194
+pytextad/models/base.py,sha256=lOJeOfpb2pXv4CmsDvbz0qC0D86iGe-ycpzyKMFE9No,2841
+pytextad/models/cvdd.py,sha256=LtJ3fERHIsgr2KCRAA5ilGGNiqMUx3o61SuU5XV5Ds8,10207
+pytextad/models/date.py,sha256=zr74ZF_tCPPbS8AXfcm_KxaKAKCIAHAXK66l_UOYl0U,13431
+pytextad/models/fate.py,sha256=fnwUjj1tWtH2zKpDsCErN1-1zOikTRqDpcLExckKPyM,8144
+pytextad/models/rsrae.py,sha256=l4oyTZhhHhQLuuWmy16CGi4-LOchN0c1SEXyO9TYT8Y,9026
+pytextad/utils/__init__.py,sha256=tGNt0TSrBTDdAqOFtN6lcCMuTl9O-4dX_a1Vr0FG8iU,135
+pytextad/utils/embeddings.py,sha256=k-ftLzwaez6mMdEUHu8H1WdPF_pWtdaAxvLJ1Xjn_7I,5010
+pytextad-0.1.0.dist-info/licenses/LICENSE,sha256=QNelLymU93gvw0MdZm0Bt02p4XCxk9XvBoGdszabVts,1297
+pytextad-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md,sha256=8fUXxny_phNs7YLBfzphwK1DdSuCPxkAjbfXTgGDNG0,2839
+pytextad-0.1.0.dist-info/METADATA,sha256=Sxk3SUqBKCxLyBpFD6-kyoolSvZcOh3pYCG04X0Ke8s,4611
+pytextad-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+pytextad-0.1.0.dist-info/top_level.txt,sha256=sQuDfOeL6h0hBxFVdIeSG5efM45WYZBjqusAtvMoSrU,9
+pytextad-0.1.0.dist-info/RECORD,,

pytextad-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pytextad-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,24 @@
+BSD 2-Clause License
+
+Copyright (c) 2026, Yang Cao
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pytextad-0.1.0.dist-info/licenses/THIRD_PARTY_NOTICES.md

@@ -0,0 +1,36 @@
+# Third-party notices
+
+## CVDD (pytextad/cvdd.py)
+Re-implemented from https://github.com/lukasruff/CVDD-PyTorch
+
+MIT License
+
+Copyright (c) 2019 lukasruff
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+## RSRAE (pytextad/rsrae.py)
+Ported to PyTorch from https://github.com/dmzou/RSRAE
+
+MIT License
+
+Copyright (c) 2019-present Chieh-Hsin Lai, Dongmian Zou and Gilad Lerman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+## DATE (pytextad/date.py)
+Re-implemented from https://github.com/bit-ml/date, whose model code lives in a
+modified copy of simpletransformers distributed under the Apache License 2.0.
+No source code is copied; the logic was re-written against modern transformers.
+
+## FATE (pytextad/fate.py)
+Re-implemented from https://github.com/arav1ndajay/fate, which has no licence file.
+No source code is copied; the logic was re-written and checked numerically against it.

pytextad-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+pytextad