singlebehaviorlab 2.3.2__tar.gz → 2.3.4__tar.gz

{singlebehaviorlab-2.3.2/singlebehaviorlab.egg-info → singlebehaviorlab-2.3.4}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: singlebehaviorlab
-Version: 2.3.2
-Summary: Semi-automated behavioral video annotation, training, and analysis tool
+Version: 2.3.4
+Summary: Behavioral sequencing and phenotyping with lightweight task specific adaptation
 Author: Almir Aljovic
 Maintainer: Almir Aljovic
 License: MIT License
@@ -59,7 +59,6 @@ Requires-Dist: einshape
 Requires-Dist: huggingface-hub
 Requires-Dist: sentencepiece
 Requires-Dist: absl-py
-Requires-Dist: tensorflow-cpu
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"

{singlebehaviorlab-2.3.2 → singlebehaviorlab-2.3.4}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "singlebehaviorlab"
-version = "2.3.2"
-description = "Semi-automated behavioral video annotation, training, and analysis tool"
+version = "2.3.4"
+description = "Behavioral sequencing and phenotyping with lightweight task specific adaptation"
 readme = "README.md"
 license = { file = "LICENSE" }
 requires-python = ">=3.10"
@@ -43,7 +43,6 @@ dependencies = [
     "huggingface-hub",
     "sentencepiece",
     "absl-py",
-    "tensorflow-cpu",
 ]
 
 [project.urls]

singlebehaviorlab-2.3.4/singlebehaviorlab/backend/embedding_refine.py

@@ -0,0 +1,158 @@
+"""Embedding-based timeline refinement.
+
+Uses per-frame embeddings from the inference model to correct predictions
+via semi-supervised label propagation on a nearest-neighbor graph, then
+detects true behavior boundaries from embedding distance spikes.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from typing import Optional
+
+__all__ = ["refine_with_embeddings"]
+
+
+def _cosine_distance_adjacent(embeddings: np.ndarray) -> np.ndarray:
+    norms = np.maximum(np.linalg.norm(embeddings, axis=1, keepdims=True), 1e-8)
+    normed = embeddings / norms
+    return 1.0 - np.sum(normed[:-1] * normed[1:], axis=1)
+
+
+def _detect_boundaries(distances: np.ndarray, threshold_factor: float) -> list[int]:
+    if len(distances) == 0:
+        return []
+    median = float(np.median(distances))
+    mad = float(np.median(np.abs(distances - median)))
+    threshold = median + threshold_factor * max(mad, 1e-6)
+    boundaries = [0]
+    for i, d in enumerate(distances):
+        if d > threshold:
+            boundaries.append(i + 1)
+    return boundaries
+
+
+def _majority_label(labels: np.ndarray, weights: Optional[np.ndarray] = None) -> int:
+    valid_mask = labels >= 0
+    valid = labels[valid_mask]
+    if len(valid) == 0:
+        return -1
+    if weights is not None:
+        w = weights[valid_mask]
+        counts: dict[int, float] = {}
+        for lbl, wt in zip(valid, w):
+            counts[int(lbl)] = counts.get(int(lbl), 0.0) + float(wt)
+        return max(counts, key=counts.get)
+    vals, cnts = np.unique(valid, return_counts=True)
+    return int(vals[np.argmax(cnts)])
+
+
+def _label_propagation_correction(
+    frame_labels: np.ndarray,
+    frame_embeddings: np.ndarray,
+    frame_confidences: np.ndarray,
+    confidence_threshold: float,
+) -> np.ndarray:
+    from sklearn.semi_supervised import LabelSpreading
+
+    n_frames = len(frame_labels)
+    labels_for_propagation = frame_labels.copy()
+
+    for i in range(n_frames):
+        if frame_confidences[i] < confidence_threshold:
+            labels_for_propagation[i] = -1
+
+    n_labeled = np.sum(labels_for_propagation >= 0)
+    if n_labeled < 2 or n_labeled == n_frames:
+        return frame_labels.copy()
+
+    n_neighbors = min(7, n_frames - 1)
+    lp = LabelSpreading(kernel="knn", n_neighbors=n_neighbors, max_iter=30, alpha=0.2)
+    lp.fit(frame_embeddings, labels_for_propagation)
+    propagated = lp.transduction_
+
+    result = frame_labels.copy()
+    for i in range(n_frames):
+        if frame_confidences[i] < confidence_threshold and propagated[i] >= 0:
+            result[i] = int(propagated[i])
+
+    return result
+
+
+def refine_with_embeddings(
+    frame_labels: np.ndarray,
+    frame_embeddings: np.ndarray,
+    frame_confidences: Optional[np.ndarray] = None,
+    n_classes: int = 0,
+    boundary_sensitivity: float = 1.5,
+    min_segment_frames: int = 3,
+    confidence_threshold: float = 0.7,
+) -> np.ndarray:
+    """Refine per-frame predictions using label propagation and boundary detection.
+
+    High-confidence predictions seed a nearest-neighbor graph. Labels
+    propagate to uncertain frames through embedding similarity. Boundary
+    detection then snaps segment edges to real embedding transitions.
+    """
+    n_frames = len(frame_labels)
+    if n_frames < 4 or frame_embeddings.shape[0] != n_frames:
+        return frame_labels.copy()
+
+    if frame_confidences is None:
+        frame_confidences = np.ones(n_frames, dtype=np.float32)
+
+    corrected = _label_propagation_correction(
+        frame_labels, frame_embeddings, frame_confidences, confidence_threshold,
+    )
+
+    distances = _cosine_distance_adjacent(frame_embeddings)
+    boundaries = _detect_boundaries(distances, boundary_sensitivity)
+    boundaries.append(n_frames)
+
+    refined = corrected.copy()
+    segments: list[tuple[int, int]] = []
+    for i in range(len(boundaries) - 1):
+        start, end = boundaries[i], boundaries[i + 1]
+        if end <= start:
+            continue
+        majority = _majority_label(corrected[start:end], frame_confidences[start:end])
+        refined[start:end] = majority
+        segments.append((start, end))
+
+    changed = True
+    while changed:
+        changed = False
+        new_segments = []
+        i = 0
+        while i < len(segments):
+            start, end = segments[i]
+            if (end - start) < min_segment_frames and len(segments) > 1:
+                mean_emb = frame_embeddings[start:end].mean(axis=0)
+                mean_emb /= max(np.linalg.norm(mean_emb), 1e-8)
+                best_sim, merge_with = -1.0, -1
+                for j in [i - 1, i + 1]:
+                    if 0 <= j < len(segments):
+                        ns, ne = segments[j]
+                        ne_emb = frame_embeddings[ns:ne].mean(axis=0)
+                        ne_emb /= max(np.linalg.norm(ne_emb), 1e-8)
+                        sim = float(np.dot(mean_emb, ne_emb))
+                        if sim > best_sim:
+                            best_sim, merge_with = sim, j
+                if merge_with >= 0:
+                    ms, me = segments[merge_with]
+                    ms2, me2 = min(start, ms), max(end, me)
+                    majority = _majority_label(corrected[ms2:me2], frame_confidences[ms2:me2])
+                    refined[ms2:me2] = majority
+                    if merge_with < i:
+                        new_segments[-1] = (ms2, me2)
+                    else:
+                        new_segments.append((ms2, me2))
+                        i += 1
+                    changed = True
+                    i += 1
+                    continue
+            new_segments.append((start, end))
+            i += 1
+        segments = new_segments
+
+    return refined

{singlebehaviorlab-2.3.2 → singlebehaviorlab-2.3.4}/singlebehaviorlab/gui/inference_widget.py

@@ -320,6 +320,27 @@ class InferenceWidget(QWidget):
         self.timeline_theme_combo.currentIndexChanged.connect(self._on_theme_changed)
         timeline_controls_layout.addWidget(self.timeline_theme_combo)
         
+        self.embedding_refine_check = QCheckBox("Embedding refinement")
+        self.embedding_refine_check.setToolTip(
+            "Use the model's internal frame embeddings to detect true behavior\n"
+            "boundaries and correct predictions via label propagation."
+        )
+        self.embedding_refine_check.stateChanged.connect(self._on_embedding_refine_changed)
+        timeline_controls_layout.addWidget(self.embedding_refine_check)
+
+        self.embedding_refine_threshold = QDoubleSpinBox()
+        self.embedding_refine_threshold.setRange(0.1, 0.99)
+        self.embedding_refine_threshold.setSingleStep(0.05)
+        self.embedding_refine_threshold.setValue(0.70)
+        self.embedding_refine_threshold.setToolTip(
+            "Confidence threshold for embedding refinement.\n"
+            "Frames above this are trusted as seed labels.\n"
+            "Frames below defer to their embedding neighbors.\n"
+            "Lower = more aggressive correction."
+        )
+        self.embedding_refine_threshold.valueChanged.connect(self._on_embedding_refine_changed)
+        timeline_controls_layout.addWidget(self.embedding_refine_threshold)
+
         self._advanced_toggle = QPushButton("Advanced")
         self._advanced_toggle.setCheckable(True)
         self._advanced_toggle.setChecked(False)
@@ -2910,6 +2931,11 @@ class InferenceWidget(QWidget):
         self._interactive_timeline.set_model(model, colors)
         self._minimap.set_model(model, colors)
     
+    def _on_embedding_refine_changed(self, *_args):
+        if self.predictions and self.frame_aggregation_check.isChecked():
+            self._build_timeline_segments()
+            self._display_results()
+
     def _toggle_advanced_controls(self, show: bool):
         for w in self._advanced_widgets:
             w.setVisible(show)
@@ -3247,10 +3273,22 @@ class InferenceWidget(QWidget):
                 if float(fs[fi, ci]) < thr:
                     frame_labels[fi] = -1
 
-        # Merge-gap and min-segment run ONCE as the final cleanup, after all
-        # other preprocessing (smoothing + threshold) is done.
         frame_labels = self._apply_gap_merge_and_min_seg(frame_labels, T)
 
+        if self.embedding_refine_check.isChecked():
+            frame_embs = None
+            if hasattr(self, 'video_path') and self.video_path and self.video_path in self.results_cache:
+                frame_embs = self.results_cache[self.video_path].get("aggregated_frame_embs")
+            if isinstance(frame_embs, np.ndarray) and frame_embs.shape[0] >= T:
+                from singlebehaviorlab.backend.embedding_refine import refine_with_embeddings
+                frame_conf = np.max(fs, axis=1)
+                frame_labels = refine_with_embeddings(
+                    frame_labels[:T], frame_embs[:T], frame_conf,
+                    n_classes=C,
+                    min_segment_frames=max(3, self._min_segment_frames),
+                    confidence_threshold=float(self.embedding_refine_threshold.value()),
+                )
+
         segments = []
         cur_cls = int(frame_labels[0])
         cur_start = 0

{singlebehaviorlab-2.3.2 → singlebehaviorlab-2.3.4}/singlebehaviorlab/gui/inference_worker.py

@@ -598,6 +598,7 @@ class InferenceWorker(QThread):
                         "orig_fps": video_orig_fps,
                         "frame_interval": video_frame_interval,
                         "aggregated_frame_probs": aggregated_frame_probs,
+                        "aggregated_frame_embs": aggregated_frame_embs,
                     }
                     if sample_ranges:
                         res_entry["sample_ranges"] = sample_ranges

{singlebehaviorlab-2.3.2 → singlebehaviorlab-2.3.4}/singlebehaviorlab/gui/registration_widget.py

@@ -326,7 +326,7 @@ class EmbeddingExtractionWorker(QThread):
     error = pyqtSignal(str)
     log_message = pyqtSignal(str)
     
-    def __init__(self, clip_paths: list, output_dir: str, experiment_name: str = None, model_name: str = 'videoprism_public_v1_base', clip_frame_ranges: dict = None, append_to_existing: bool = False, flip_invariant: bool = False, align_orientation: bool = False, mask_path: str = None):
+    def __init__(self, clip_paths: list, output_dir: str, experiment_name: str = None, model_name: str = 'videoprism_public_v1_base', clip_frame_ranges: dict = None, append_to_existing: bool = False, flip_invariant: bool = False):
         super().__init__()
         self.clip_paths = clip_paths
         self.clip_frame_ranges = clip_frame_ranges or {}
@@ -336,8 +336,6 @@ class EmbeddingExtractionWorker(QThread):
         self.should_stop = False
         self.append_to_existing = append_to_existing
         self.flip_invariant = flip_invariant
-        self.align_orientation = align_orientation
-        self.mask_path = mask_path
     
     def stop(self):
         self.should_stop = True
@@ -1256,11 +1254,6 @@ class RegistrationWidget(QWidget):
         experiment_name = self.config.get("experiment_name", None)
         
         # Start extraction worker with frame ranges if available
-        mask_path = None
-        if self.align_orientation_check.isChecked() and self.video_mask_pairs:
-            mask_path = self.video_mask_pairs[0][1] if len(self.video_mask_pairs) > 0 else None
-            self.log_text.append(f"Align orientation: mask_path={mask_path}, pairs={len(self.video_mask_pairs)}, frame_ranges={len(self.clip_frame_ranges)}")
-
         self.embedding_worker = EmbeddingExtractionWorker(
             clip_paths,
             self.output_dir,
@@ -1269,8 +1262,6 @@ class RegistrationWidget(QWidget):
             clip_frame_ranges=self.clip_frame_ranges if hasattr(self, 'clip_frame_ranges') else None,
             append_to_existing=self.append_embeddings_check.isChecked(),
             flip_invariant=self.flip_invariant_check.isChecked(),
-            align_orientation=self.align_orientation_check.isChecked(),
-            mask_path=mask_path,
         )
         self.embedding_worker.progress.connect(self._on_embedding_progress)
         self.embedding_worker.finished.connect(self._on_embedding_finished)

{singlebehaviorlab-2.3.2 → singlebehaviorlab-2.3.4/singlebehaviorlab.egg-info}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: singlebehaviorlab
-Version: 2.3.2
-Summary: Semi-automated behavioral video annotation, training, and analysis tool
+Version: 2.3.4
+Summary: Behavioral sequencing and phenotyping with lightweight task specific adaptation
 Author: Almir Aljovic
 Maintainer: Almir Aljovic
 License: MIT License
@@ -59,7 +59,6 @@ Requires-Dist: einshape
 Requires-Dist: huggingface-hub
 Requires-Dist: sentencepiece
 Requires-Dist: absl-py
-Requires-Dist: tensorflow-cpu
 Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: pytest-cov; extra == "test"

{singlebehaviorlab-2.3.2 → singlebehaviorlab-2.3.4}/singlebehaviorlab.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ singlebehaviorlab/backend/augmentations.py
 singlebehaviorlab/backend/clustering.py
 singlebehaviorlab/backend/contrastive.py
 singlebehaviorlab/backend/data_store.py
+singlebehaviorlab/backend/embedding_refine.py
 singlebehaviorlab/backend/inference.py
 singlebehaviorlab/backend/model.py
 singlebehaviorlab/backend/registration.py

{singlebehaviorlab-2.3.2 → singlebehaviorlab-2.3.4}/singlebehaviorlab.egg-info/requires.txt

@@ -26,7 +26,6 @@ einshape
 huggingface-hub
 sentencepiece
 absl-py
-tensorflow-cpu
 
 [test]
 pytest

{singlebehaviorlab-2.3.2 → singlebehaviorlab-2.3.4}/third_party/videoprism_backend/videoprism/models.py

@@ -45,7 +45,6 @@ import jax.numpy as jnp
 import huggingface_hub
 import numpy as np
 from videoprism import encoders
-from videoprism import tokenizers
 from videoprism import utils
 
 K400_NUM_CLASSES: int = 400
@@ -336,7 +335,8 @@ def load_pretrained_weights(
   return jax.tree_util.tree_map(jnp.asarray, variables)
 
 
-def load_text_tokenizer(name: str) -> tokenizers.Tokenizer:
+def load_text_tokenizer(name: str):
+  from videoprism import tokenizers  # lazy: avoids top-level TF dependency
   """Loads a text tokenizer by name.
 
   Args:
@@ -353,7 +353,7 @@ def load_text_tokenizer(name: str) -> tokenizers.Tokenizer:
 
 
 def tokenize_texts(
-    tokenizer: tokenizers.Tokenizer,
+    tokenizer,
     inputs: Sequence[str],
     max_length: int = TEXT_MAX_LEN,
     add_bos: bool | None = None,

{singlebehaviorlab-2.3.2 → singlebehaviorlab-2.3.4}/third_party/videoprism_backend/videoprism/tokenizers.py

@@ -17,9 +17,6 @@
 from collections.abc import Sequence
 from typing import Protocol
 
-import tensorflow as tf
-from tensorflow.io import gfile
-
 import sentencepiece
 
 SentencePieceProcessor = sentencepiece.SentencePieceProcessor
@@ -44,7 +41,7 @@ class Tokenizer(Protocol):
 
   def to_int_tf_op(
       self, text: str | Sequence[str], *, bos: bool = False, eos: bool = False
-  ) -> tf.Tensor | tf.RaggedTensor:
+  ):
     """Same as `to_int()`, but as TF ops to be used in data pipelines.
 
     Args:
@@ -55,6 +52,7 @@ class Tokenizer(Protocol):
     Returns:
       A tf.Tensor of tokens.
     """
+    import tensorflow as tf
 
   @property
   def pad_token(self) -> int:
@@ -82,7 +80,7 @@ class SentencePieceTokenizer(Tokenizer):
     Args:
       model_path: A path to load the SentencePiece model.
     """
-    with gfile.GFile(model_path, "rb") as f:
+    with open(model_path, "rb") as f:
       model_bytes = f.read()
 
     self._model = SentencePieceProcessor()
@@ -115,7 +113,7 @@ class SentencePieceTokenizer(Tokenizer):
 
   def to_int_tf_op(
       self, text: str | Sequence[str], *, bos: bool = False, eos: bool = False
-  ) -> tf.Tensor | tf.RaggedTensor:
+  ):
     """Same as `to_int()`, but as TF ops to be used in data pipelines.
 
     Args:

{singlebehaviorlab-2.3.2 → singlebehaviorlab-2.3.4}/third_party/videoprism_backend/videoprism/utils.py

@@ -16,13 +16,10 @@
 
 import collections
 from collections.abc import Mapping, Sequence
-import io
-import os
 import string
 
 import jax
 import numpy as np
-from tensorflow.io import gfile
 
 
 def traverse_with_names(tree, with_inner_nodes=False):
@@ -106,13 +103,7 @@ def recover_tree(keys, values):
 def npload(fname):
   """Loads `fname` and returns an np.ndarray or dict thereof."""
   # Load the data; use local paths directly if possible:
-  if os.path.exists(fname):
-    loaded = np.load(fname, allow_pickle=False)
-  else:
-    # For other (remote) paths go via gfile+BytesIO as np.load requires seeks.
-    with gfile.GFile(fname, "rb") as f:
-      data = f.read()
-    loaded = np.load(io.BytesIO(data), allow_pickle=False)
+  loaded = np.load(fname, allow_pickle=False)
 
   # Support loading both single-array files (np.save) and zips (np.savez).
   if isinstance(loaded, np.ndarray):