singlebehaviorlab 2.3.1__tar.gz → 2.3.3__tar.gz

{singlebehaviorlab-2.3.1/singlebehaviorlab.egg-info → singlebehaviorlab-2.3.3}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: singlebehaviorlab
-Version: 2.3.1
-Summary: Semi-automated behavioral video annotation, training, and analysis tool
+Version: 2.3.3
+Summary: Behavioral sequencing and phenotyping with lightweight task specific adaptation
 Author: Almir Aljovic
 Maintainer: Almir Aljovic
 License: MIT License
@@ -59,7 +59,6 @@ Requires-Dist: einshape
 Requires-Dist: huggingface-hub
 Requires-Dist: sentencepiece
 Requires-Dist: absl-py
-Requires-Dist: tensorflow-cpu
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"

{singlebehaviorlab-2.3.1 → singlebehaviorlab-2.3.3}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "singlebehaviorlab"
-version = "2.3.1"
-description = "Semi-automated behavioral video annotation, training, and analysis tool"
+version = "2.3.3"
+description = "Behavioral sequencing and phenotyping with lightweight task specific adaptation"
 readme = "README.md"
 license = { file = "LICENSE" }
 requires-python = ">=3.10"
@@ -43,7 +43,6 @@ dependencies = [
     "huggingface-hub",
     "sentencepiece",
     "absl-py",
-    "tensorflow-cpu",
 ]
 
 [project.urls]

{singlebehaviorlab-2.3.1 → singlebehaviorlab-2.3.3}/singlebehaviorlab/__init__.py

@@ -19,7 +19,7 @@ or videoprism. Each symbol triggers its underlying backend module only on
 first access.
 """
 
-__version__ = "2.3.1"
+__version__ = "2.3.2"
 __author__ = "Almir Aljovic"
 
 # Mapping of public name → (backend module, attribute name).
@@ -35,6 +35,7 @@ _PUBLIC_API = {
     "infer": ("singlebehaviorlab.backend.inference", "run_inference_on_video"),
     "train": ("singlebehaviorlab.backend.training_runner", "run_training_session"),
     "load_config": ("singlebehaviorlab.config", "load_config"),
+    "learn_behavior_features": ("singlebehaviorlab.backend.contrastive", "learn_behavior_features"),
     "load_demo": ("singlebehaviorlab.demo", "load_demo"),
     "DEMOS": ("singlebehaviorlab.demo", "DEMOS"),
 }

{singlebehaviorlab-2.3.1 → singlebehaviorlab-2.3.3}/singlebehaviorlab/backend/clustering.py

@@ -28,7 +28,8 @@ class ClusteringParams:
     n_components: int = 2
     n_neighbors: int = 15
     min_dist: float = 0.1
-    normalization: str = "standard"  # standard | minmax | l2 | none
+    normalization: str = "standard"
+    subtract_video_mean: bool = False
     leiden_resolution: float = 1.0
     leiden_k: int = 15
     min_cluster_size: int = 10
@@ -157,8 +158,29 @@ def run_clustering(
     matrix_df, metadata_df = _load_matrix_metadata(matrix_path_str, metadata_path_str)
     _log(f"Matrix shape: {matrix_df.shape[0]} features × {matrix_df.shape[1]} samples")
 
-    processed = _normalize(matrix_df.T, params.normalization)
-    _log(f"Processed shape: {processed.shape} (samples × features); normalization={params.normalization}")
+    X = matrix_df.T
+    X = X.replace([np.inf, -np.inf], np.nan).fillna(0.0)
+
+    if params.subtract_video_mean and metadata_df is not None:
+        group_col = None
+        for col in ("group", "video_id"):
+            if col in metadata_df.columns:
+                group_col = col
+                break
+        snippet_col = "snippet" if "snippet" in metadata_df.columns else None
+        if group_col and snippet_col:
+            for grp in metadata_df[group_col].unique():
+                grp_snippets = metadata_df.loc[metadata_df[group_col] == grp, snippet_col].values
+                mask = X.index.isin(grp_snippets)
+                if mask.sum() > 1:
+                    X.loc[mask] -= X.loc[mask].mean(axis=0)
+            _log("Applied per-video mean subtraction")
+
+
+    processed = _normalize(X, params.normalization)
+
+
+    _log(f"Processed shape: {processed.shape} (samples × features)")
 
     _log(
         f"Running UMAP (n_neighbors={params.n_neighbors}, "

singlebehaviorlab-2.3.3/singlebehaviorlab/backend/contrastive.py

@@ -0,0 +1,202 @@
+"""Temporal contrastive projection for behavior-focused embeddings.
+
+Trains a lightweight MLP on pre-computed VideoPrism embeddings using
+temporal proximity as the supervision signal: clips close in time within
+the same video should map nearby; clips far apart should map far away.
+The projected embeddings suppress static visual factors (lighting,
+background, camera) and amplify behavioral dynamics.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+import numpy as np
+import pandas as pd
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+__all__ = ["learn_behavior_features"]
+
+_DEFAULT_DIM = 128
+_DEFAULT_EPOCHS = 30
+_DEFAULT_LR = 3e-4
+_POSITIVE_WINDOW = 5
+_TEMPERATURE = 0.07
+
+
+class _ProjectionHead(nn.Module):
+    def __init__(self, in_dim: int, out_dim: int):
+        super().__init__()
+        hidden = max(out_dim, in_dim // 2)
+        self.net = nn.Sequential(
+            nn.Linear(in_dim, hidden),
+            nn.ReLU(),
+            nn.Linear(hidden, out_dim),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return F.normalize(self.net(x), dim=-1)
+
+
+def _build_pairs(
+    metadata: pd.DataFrame,
+    n_samples: int,
+    positive_window: int,
+    rng: np.random.Generator,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+    group_col = None
+    for col in ("group", "video_id"):
+        if col in metadata.columns:
+            group_col = col
+            break
+    snippet_col = "snippet" if "snippet" in metadata.columns else None
+    if not group_col or not snippet_col:
+        indices = np.arange(len(metadata))
+        rng.shuffle(indices)
+        anchors = indices[:n_samples]
+        positives = np.clip(anchors + rng.integers(-positive_window, positive_window + 1, size=n_samples), 0, len(metadata) - 1)
+        negatives = rng.integers(0, len(metadata), size=n_samples)
+        return anchors, positives, negatives
+
+    groups = metadata[group_col].values
+    unique_groups = np.unique(groups)
+    group_indices: dict[Any, np.ndarray] = {}
+    for g in unique_groups:
+        group_indices[g] = np.where(groups == g)[0]
+
+    anchors = []
+    positives = []
+    negatives = []
+    per_group = max(1, n_samples // len(unique_groups))
+
+    for g in unique_groups:
+        idx = group_indices[g]
+        if len(idx) < 2:
+            continue
+        a = rng.choice(idx, size=min(per_group, len(idx)), replace=len(idx) < per_group)
+        for ai in a:
+            pos_in_group = np.where(idx == ai)[0][0]
+            lo = max(0, pos_in_group - positive_window)
+            hi = min(len(idx), pos_in_group + positive_window + 1)
+            candidates = idx[lo:hi]
+            candidates = candidates[candidates != ai]
+            if len(candidates) == 0:
+                continue
+            pi = rng.choice(candidates)
+
+            other_groups = [og for og in unique_groups if og != g]
+            if other_groups:
+                ng = rng.choice(other_groups)
+                ni = rng.choice(group_indices[ng])
+            else:
+                far_lo = max(0, pos_in_group - 3 * positive_window)
+                far_hi = min(len(idx), pos_in_group + 3 * positive_window + 1)
+                far_candidates = np.setdiff1d(idx, idx[far_lo:far_hi])
+                if len(far_candidates) == 0:
+                    far_candidates = idx
+                ni = rng.choice(far_candidates)
+
+            anchors.append(ai)
+            positives.append(pi)
+            negatives.append(ni)
+
+    return np.array(anchors), np.array(positives), np.array(negatives)
+
+
+def _info_nce_loss(anchor, positive, negative, temperature):
+    pos_sim = (anchor * positive).sum(dim=-1) / temperature
+    neg_sim = (anchor * negative).sum(dim=-1) / temperature
+    logits = torch.stack([pos_sim, neg_sim], dim=-1)
+    labels = torch.zeros(len(anchor), dtype=torch.long, device=anchor.device)
+    return F.cross_entropy(logits, labels)
+
+
+def learn_behavior_features(
+    matrix_path: str | os.PathLike[str],
+    output_path: str | os.PathLike[str],
+    *,
+    metadata_path: Optional[str | os.PathLike[str]] = None,
+    out_dim: int = _DEFAULT_DIM,
+    epochs: int = _DEFAULT_EPOCHS,
+    lr: float = _DEFAULT_LR,
+    positive_window: int = _POSITIVE_WINDOW,
+    temperature: float = _TEMPERATURE,
+    log_fn: Optional[Callable[[str], None]] = None,
+) -> dict[str, str]:
+    """Train a contrastive projection and write the projected embedding matrix.
+
+    Returns dict with ``matrix`` and ``metadata`` output paths.
+    """
+    from singlebehaviorlab.backend.clustering import _load_matrix_metadata
+
+    matrix_path = str(Path(matrix_path).expanduser().resolve())
+    output_path_obj = Path(output_path).expanduser().resolve()
+    output_path_obj.parent.mkdir(parents=True, exist_ok=True)
+
+    metadata_path_str = str(Path(metadata_path).expanduser().resolve()) if metadata_path else None
+
+    def _log(msg: str) -> None:
+        if log_fn:
+            log_fn(msg)
+
+    matrix_df, metadata_df = _load_matrix_metadata(matrix_path, metadata_path_str)
+    X = matrix_df.T
+    embeddings = X.values.astype(np.float32)
+    n_samples, in_dim = embeddings.shape
+    _log(f"Loaded {n_samples} embeddings ({in_dim}-dim)")
+
+    if metadata_df is None:
+        metadata_df = pd.DataFrame({"snippet": X.index, "group": "video_0"})
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = _ProjectionHead(in_dim, out_dim).to(device)
+    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
+    all_emb = torch.from_numpy(embeddings).to(device)
+
+    rng = np.random.default_rng(42)
+    pairs_per_epoch = max(1024, min(n_samples * 4, 65536))
+
+    _log(f"Training projection head ({in_dim} → {out_dim}) for {epochs} epochs on {device}")
+    for epoch in range(epochs):
+        anchors, positives, negatives = _build_pairs(metadata_df, pairs_per_epoch, positive_window, rng)
+        if len(anchors) == 0:
+            _log("No valid pairs found — check metadata has group/video_id column")
+            break
+        a_emb = model(all_emb[anchors])
+        p_emb = model(all_emb[positives])
+        n_emb = model(all_emb[negatives])
+        loss = _info_nce_loss(a_emb, p_emb, n_emb, temperature)
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+        if (epoch + 1) % 10 == 0 or epoch == 0:
+            _log(f"  epoch {epoch + 1}/{epochs}  loss={loss.item():.4f}")
+
+    model.eval()
+    with torch.no_grad():
+        projected = model(all_emb).cpu().numpy()
+    _log(f"Projected embeddings: {projected.shape}")
+
+    snippet_ids = np.array(X.index.tolist())
+    feature_names = np.array([f"behavior_feat_{i}" for i in range(out_dim)])
+
+    out_matrix = str(output_path_obj)
+    if out_matrix.endswith("_matrix.npz"):
+        out_metadata = out_matrix.replace("_matrix.npz", "_metadata.npz")
+    elif out_matrix.endswith(".npz"):
+        out_metadata = out_matrix[:-4] + "_metadata.npz"
+    else:
+        out_metadata = out_matrix + "_metadata.npz"
+
+    np.savez_compressed(out_matrix, matrix=projected.T, feature_names=feature_names, snippet_ids=snippet_ids)
+    _log(f"Wrote projected matrix: {out_matrix}")
+
+    if metadata_df is not None:
+        np.savez_compressed(out_metadata, metadata=metadata_df.values, columns=np.array(metadata_df.columns))
+        _log(f"Wrote metadata: {out_metadata}")
+
+    return {"matrix": out_matrix, "metadata": out_metadata}

singlebehaviorlab-2.3.3/singlebehaviorlab/backend/embedding_refine.py

@@ -0,0 +1,158 @@
+"""Embedding-based timeline refinement.
+
+Uses per-frame embeddings from the inference model to correct predictions
+via semi-supervised label propagation on a nearest-neighbor graph, then
+detects true behavior boundaries from embedding distance spikes.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from typing import Optional
+
+__all__ = ["refine_with_embeddings"]
+
+
+def _cosine_distance_adjacent(embeddings: np.ndarray) -> np.ndarray:
+    norms = np.maximum(np.linalg.norm(embeddings, axis=1, keepdims=True), 1e-8)
+    normed = embeddings / norms
+    return 1.0 - np.sum(normed[:-1] * normed[1:], axis=1)
+
+
+def _detect_boundaries(distances: np.ndarray, threshold_factor: float) -> list[int]:
+    if len(distances) == 0:
+        return []
+    median = float(np.median(distances))
+    mad = float(np.median(np.abs(distances - median)))
+    threshold = median + threshold_factor * max(mad, 1e-6)
+    boundaries = [0]
+    for i, d in enumerate(distances):
+        if d > threshold:
+            boundaries.append(i + 1)
+    return boundaries
+
+
+def _majority_label(labels: np.ndarray, weights: Optional[np.ndarray] = None) -> int:
+    valid_mask = labels >= 0
+    valid = labels[valid_mask]
+    if len(valid) == 0:
+        return -1
+    if weights is not None:
+        w = weights[valid_mask]
+        counts: dict[int, float] = {}
+        for lbl, wt in zip(valid, w):
+            counts[int(lbl)] = counts.get(int(lbl), 0.0) + float(wt)
+        return max(counts, key=counts.get)
+    vals, cnts = np.unique(valid, return_counts=True)
+    return int(vals[np.argmax(cnts)])
+
+
+def _label_propagation_correction(
+    frame_labels: np.ndarray,
+    frame_embeddings: np.ndarray,
+    frame_confidences: np.ndarray,
+    confidence_threshold: float,
+) -> np.ndarray:
+    from sklearn.semi_supervised import LabelSpreading
+
+    n_frames = len(frame_labels)
+    labels_for_propagation = frame_labels.copy()
+
+    for i in range(n_frames):
+        if frame_confidences[i] < confidence_threshold:
+            labels_for_propagation[i] = -1
+
+    n_labeled = np.sum(labels_for_propagation >= 0)
+    if n_labeled < 2 or n_labeled == n_frames:
+        return frame_labels.copy()
+
+    n_neighbors = min(7, n_frames - 1)
+    lp = LabelSpreading(kernel="knn", n_neighbors=n_neighbors, max_iter=30, alpha=0.2)
+    lp.fit(frame_embeddings, labels_for_propagation)
+    propagated = lp.transduction_
+
+    result = frame_labels.copy()
+    for i in range(n_frames):
+        if frame_confidences[i] < confidence_threshold and propagated[i] >= 0:
+            result[i] = int(propagated[i])
+
+    return result
+
+
+def refine_with_embeddings(
+    frame_labels: np.ndarray,
+    frame_embeddings: np.ndarray,
+    frame_confidences: Optional[np.ndarray] = None,
+    n_classes: int = 0,
+    boundary_sensitivity: float = 1.5,
+    min_segment_frames: int = 3,
+    confidence_threshold: float = 0.7,
+) -> np.ndarray:
+    """Refine per-frame predictions using label propagation and boundary detection.
+
+    High-confidence predictions seed a nearest-neighbor graph. Labels
+    propagate to uncertain frames through embedding similarity. Boundary
+    detection then snaps segment edges to real embedding transitions.
+    """
+    n_frames = len(frame_labels)
+    if n_frames < 4 or frame_embeddings.shape[0] != n_frames:
+        return frame_labels.copy()
+
+    if frame_confidences is None:
+        frame_confidences = np.ones(n_frames, dtype=np.float32)
+
+    corrected = _label_propagation_correction(
+        frame_labels, frame_embeddings, frame_confidences, confidence_threshold,
+    )
+
+    distances = _cosine_distance_adjacent(frame_embeddings)
+    boundaries = _detect_boundaries(distances, boundary_sensitivity)
+    boundaries.append(n_frames)
+
+    refined = corrected.copy()
+    segments: list[tuple[int, int]] = []
+    for i in range(len(boundaries) - 1):
+        start, end = boundaries[i], boundaries[i + 1]
+        if end <= start:
+            continue
+        majority = _majority_label(corrected[start:end], frame_confidences[start:end])
+        refined[start:end] = majority
+        segments.append((start, end))
+
+    changed = True
+    while changed:
+        changed = False
+        new_segments = []
+        i = 0
+        while i < len(segments):
+            start, end = segments[i]
+            if (end - start) < min_segment_frames and len(segments) > 1:
+                mean_emb = frame_embeddings[start:end].mean(axis=0)
+                mean_emb /= max(np.linalg.norm(mean_emb), 1e-8)
+                best_sim, merge_with = -1.0, -1
+                for j in [i - 1, i + 1]:
+                    if 0 <= j < len(segments):
+                        ns, ne = segments[j]
+                        ne_emb = frame_embeddings[ns:ne].mean(axis=0)
+                        ne_emb /= max(np.linalg.norm(ne_emb), 1e-8)
+                        sim = float(np.dot(mean_emb, ne_emb))
+                        if sim > best_sim:
+                            best_sim, merge_with = sim, j
+                if merge_with >= 0:
+                    ms, me = segments[merge_with]
+                    ms2, me2 = min(start, ms), max(end, me)
+                    majority = _majority_label(corrected[ms2:me2], frame_confidences[ms2:me2])
+                    refined[ms2:me2] = majority
+                    if merge_with < i:
+                        new_segments[-1] = (ms2, me2)
+                    else:
+                        new_segments.append((ms2, me2))
+                        i += 1
+                    changed = True
+                    i += 1
+                    continue
+            new_segments.append((start, end))
+            i += 1
+        segments = new_segments
+
+    return refined

{singlebehaviorlab-2.3.1 → singlebehaviorlab-2.3.3}/singlebehaviorlab/backend/registration.py

@@ -40,6 +40,7 @@ class RegistrationParams:
     clip_length_frames: int = 16
     step_frames: Optional[int] = None
     backbone_model: str = "videoprism_public_v1_base"
+    flip_invariant: bool = False
     experiment_name: Optional[str] = None
 
     @property
@@ -71,6 +72,7 @@ def _extract_embedding(
     backbone: VideoPrismBackbone,
     frames: np.ndarray,
     target_size: int,
+    flip_invariant: bool = False,
 ) -> Optional[np.ndarray]:
     try:
         resized = np.array([cv2.resize(f, (target_size, target_size)) for f in frames])
@@ -79,6 +81,12 @@ def _extract_embedding(
         with torch.no_grad():
             tokens = backbone(tensor)
             embedding = tokens.mean(dim=1).squeeze(0).cpu().numpy()
+            if flip_invariant:
+                embs = [embedding]
+                for dims in [[-1], [-2], [-1, -2]]:
+                    t_flip = torch.flip(tensor, dims=dims)
+                    embs.append(backbone(t_flip).mean(dim=1).squeeze(0).cpu().numpy())
+                embedding = np.mean(embs, axis=0)
         return embedding.astype(np.float32)
     except Exception:
         return None
@@ -178,7 +186,7 @@ def run_registration(
         if frames is None or len(frames) == 0:
             _log(f"Skipping {os.path.basename(clip_path)}: no frames")
             continue
-        embedding = _extract_embedding(backbone, frames, params.target_size)
+        embedding = _extract_embedding(backbone, frames, params.target_size, params.flip_invariant)
         del frames
         if embedding is None:
             _log(f"Skipping {os.path.basename(clip_path)}: embedding failed")

{singlebehaviorlab-2.3.1 → singlebehaviorlab-2.3.3}/singlebehaviorlab/cli.py

@@ -178,6 +178,10 @@ def _build_parser() -> argparse.ArgumentParser:
         "--no-clahe", dest="clahe", action="store_false", default=None,
         help="Disable CLAHE contrast normalization.",
     )
+    register_parser.add_argument(
+        "--flip-invariant", action="store_true",
+        help="Average original + horizontally flipped embeddings to remove facing-direction bias. 2x extraction time.",
+    )
     _add_common_runtime_flags(register_parser)
 
     segment_parser = subparsers.add_parser(
@@ -347,6 +351,8 @@ def cmd_register(args: argparse.Namespace) -> int:
         params.target_fps = int(args.target_fps)
     if args.clahe is False:
         params.normalization_method = "None"
+    if args.flip_invariant:
+        params.flip_invariant = True
 
     bar = {"pbar": None}