opensportslib 0.2.0__tar.gz → 0.2.0.dev2__tar.gz

{opensportslib-0.2.0/opensportslib.egg-info → opensportslib-0.2.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.2.0
+Version: 0.2.0.dev2
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12
@@ -42,6 +42,7 @@ It provides a unified framework to **train, evaluate, and run inference** for ke
 
 - **Action classification**
 - **Action localization / spotting**
+- **Visual Question Answering (VQA)**
 - **Action retrieval**
 - **Action description / captioning**
 
@@ -257,7 +258,7 @@ print(metrics)
 from opensportslib.apis import LocalizationModel
 
 my_model = LocalizationModel(
-    config="/path/to/localization.yaml",
+    config="/path/to/localization_video_dali.yaml",
     weights=None,  # optional: path or Hugging Face model ID
 )
 
@@ -280,6 +281,32 @@ metrics_from_file = my_model.evaluate(
 )
 ```
 
+### VQA example
+
+```python
+from opensportslib.apis import VQAModel
+
+my_model = VQAModel(
+    config="/path/to/vqa.yaml",
+    weights=None,  # optional: path or Hugging Face model ID
+)
+
+predictions = my_model.infer(
+    test_set="/path/to/test_annotations.json",
+)
+
+# Headless single-video VQA uses the same prediction payload shape.
+single_prediction = my_model.infer(
+    video_path="/path/to/video.mp4",
+    question="What card would you give? Why?",
+)
+
+metrics = my_model.evaluate(
+    test_set="/path/to/test_annotations.json",
+    predictions=predictions,
+)
+```
+
 
 ---
 
@@ -316,6 +343,9 @@ Classify clips or event centered samples into predefined categories.
 ### Action Localization / Spotting
 Predict when key events happen in long untrimmed sports videos.
 
+### Visual Question Answering (VQA)
+Answer natural-language questions about sports video clips.
+
 ### Action Retrieval
 Search and retrieve relevant clips or moments from a collection of sports videos.
 This is part of the roadmap and OSL data model, not a first-class OpenSportsLib

{opensportslib-0.2.0 → opensportslib-0.2.0.dev2}/README.md

@@ -7,6 +7,7 @@ It provides a unified framework to **train, evaluate, and run inference** for ke
 
 - **Action classification**
 - **Action localization / spotting**
+- **Visual Question Answering (VQA)**
 - **Action retrieval**
 - **Action description / captioning**
 
@@ -222,7 +223,7 @@ print(metrics)
 from opensportslib.apis import LocalizationModel
 
 my_model = LocalizationModel(
-    config="/path/to/localization.yaml",
+    config="/path/to/localization_video_dali.yaml",
     weights=None,  # optional: path or Hugging Face model ID
 )
 
@@ -245,6 +246,32 @@ metrics_from_file = my_model.evaluate(
 )
 ```
 
+### VQA example
+
+```python
+from opensportslib.apis import VQAModel
+
+my_model = VQAModel(
+    config="/path/to/vqa.yaml",
+    weights=None,  # optional: path or Hugging Face model ID
+)
+
+predictions = my_model.infer(
+    test_set="/path/to/test_annotations.json",
+)
+
+# Headless single-video VQA uses the same prediction payload shape.
+single_prediction = my_model.infer(
+    video_path="/path/to/video.mp4",
+    question="What card would you give? Why?",
+)
+
+metrics = my_model.evaluate(
+    test_set="/path/to/test_annotations.json",
+    predictions=predictions,
+)
+```
+
 
 ---
 
@@ -281,6 +308,9 @@ Classify clips or event centered samples into predefined categories.
 ### Action Localization / Spotting
 Predict when key events happen in long untrimmed sports videos.
 
+### Visual Question Answering (VQA)
+Answer natural-language questions about sports video clips.
+
 ### Action Retrieval
 Search and retrieve relevant clips or moments from a collection of sports videos.
 This is part of the roadmap and OSL data model, not a first-class OpenSportsLib

{opensportslib-0.2.0 → opensportslib-0.2.0.dev2}/examples/quickstart/basic_localization.py

@@ -8,7 +8,7 @@ def main():
     """
 
     my_model = LocalizationModel(
-        config="examples/configs/localization.yaml",
+        config="examples/configs/localization_video_dali.yaml",
         weights=None,  # optional: path or Hugging Face model ID
     )
 

{opensportslib-0.2.0 → opensportslib-0.2.0.dev2}/opensportslib/apis/__init__.py

@@ -4,6 +4,7 @@
 from opensportslib.apis.base_task_model import BaseTaskModel
 from opensportslib.apis.classification import ClassificationModel
 from opensportslib.apis.localization import LocalizationModel
+from opensportslib.apis.vqa import VQAModel
 import warnings
 warnings.filterwarnings("ignore")
 
@@ -12,4 +13,5 @@ __all__ = [
     "BaseTaskModel",
     "ClassificationModel",
     "LocalizationModel",
+    "VQAModel",
 ]

opensportslib-0.2.0.dev2/opensportslib/apis/vqa.py

@@ -0,0 +1,313 @@
+"""Public API for VQA tasks."""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any
+
+from opensportslib.apis.base_task_model import BaseTaskModel
+from opensportslib.core.config.accessors import get_split_annotation_path, get_system_gpu_count, get_train_execution, get_vqa_backend
+from opensportslib.core.utils.config import expand, resolve_config_omega
+
+
+def _set_model_checkpoint_path(config, weights: str | None) -> None:
+    if weights is None:
+        return
+    model = getattr(config, "MODEL", None)
+    if model is None:
+        return
+    load = getattr(model, "load", None)
+    if load is None:
+        from types import SimpleNamespace
+
+        load = SimpleNamespace()
+        setattr(model, "load", load)
+    if isinstance(load, dict):
+        load["checkpoint_path"] = weights
+    else:
+        setattr(load, "checkpoint_path", weights)
+
+
+class VQAModel(BaseTaskModel):
+    """Top-level task wrapper for VQA."""
+
+    def _init_wandb(self, use_wandb: bool) -> None:
+        from opensportslib.core.utils.wandb import init_wandb
+
+        init_wandb(
+            self.config_path,
+            self.config,
+            run_id=os.environ["RUN_ID"],
+            use_wandb=use_wandb,
+        )
+
+    @staticmethod
+    def _worker_ddp(
+        rank,
+        world_size,
+        config_path,
+        config,
+        return_queue=None,
+        train_set=None,
+        valid_set=None,
+        use_wandb=False,
+    ):
+        import torch
+        from opensportslib.core.trainer.vqa_trainer import Trainer_VQA
+        from opensportslib.core.utils.ddp import ddp_cleanup, ddp_setup
+        from opensportslib.core.utils.wandb import init_wandb
+        from opensportslib.datasets.builder import build_dataset
+
+        logging.basicConfig(
+            level=logging.INFO,
+            format=f"[RANK {rank}] %(asctime)s | %(levelname)s | %(message)s",
+            force=True,
+        )
+        if rank != 0:
+            logging.getLogger().setLevel(logging.ERROR)
+        # Keep INFO on all ranks while debugging multi-GPU startup/hangs.
+        # This makes DDP failures visible instead of appearing as a silent stall.
+        #logging.getLogger().setLevel(logging.INFO)
+
+        is_ddp = world_size > 1
+        if is_ddp:
+            os.environ["RANK"] = str(rank)
+            os.environ["WORLD_SIZE"] = str(world_size)
+            os.environ["LOCAL_RANK"] = str(rank)
+            if "TORCH_DISTRIBUTED_DEBUG" not in os.environ:
+                os.environ["TORCH_DISTRIBUTED_DEBUG"] = "INFO"
+            torch.cuda.set_device(rank)
+            ddp_setup(rank, world_size)
+            logging.info(
+                "Initialized VQA DDP worker | rank=%s | world_size=%s | torch_distributed_debug=%s",
+                rank,
+                world_size,
+                os.environ.get("TORCH_DISTRIBUTED_DEBUG"),
+            )
+
+        try:
+            if rank == 0:
+                init_wandb(
+                    config_path,
+                    config,
+                    run_id=os.environ["RUN_ID"],
+                    use_wandb=use_wandb,
+                )
+            train_data = build_dataset(config, train_set, None, split="train")
+            valid_data = build_dataset(config, valid_set, None, split="valid")
+            trainer = Trainer_VQA(config)
+            ckpt = trainer.train(
+                None,
+                train_data,
+                valid_data,
+                rank=rank,
+                world_size=world_size,
+                use_wandb=use_wandb,
+            )
+            if rank == 0 and return_queue is not None:
+                return_queue.put(ckpt)
+        finally:
+            if is_ddp:
+                ddp_cleanup()
+
+    def _resolve_split_path(self, split: str, override: str | None = None) -> str:
+        if override is not None:
+            return expand(override)
+        path = get_split_annotation_path(self.config, split)
+        if not path:
+            raise ValueError(
+                f"Could not resolve path for split '{split}'. "
+                f"Expected DATA.common.splits.{split}.annotation_path."
+            )
+        return expand(path)
+
+    def load_weights(self, weights: str | None = None, **kwargs) -> None:
+        del kwargs
+        if weights is None:
+            raise ValueError("`weights` must be provided to load_weights().")
+        from opensportslib.core.trainer.vqa_trainer import Trainer_VQA
+
+        _set_model_checkpoint_path(self.config, weights)
+        self.trainer = Trainer_VQA(self.config)
+        self.trainer.load(weights)
+        self.last_loaded_weights = weights
+        self.best_checkpoint = weights
+
+    def train(
+        self,
+        train_set: str | None = None,
+        valid_set: str | None = None,
+        weights: str | None = None,
+        use_wandb: bool = True,
+        **kwargs,
+    ) -> str | None:
+        del kwargs
+        import torch
+        import torch.multiprocessing as mp
+
+        self.config = resolve_config_omega(self.config, weights=weights)
+        if get_vqa_backend(self.config) == "qwen_xvars_infer":
+            raise ValueError("The 'qwen_xvars_infer' backend is inference-only and does not support train().")
+        train_set = self._resolve_split_path("train", train_set)
+        valid_set = self._resolve_split_path("valid", valid_set)
+        execution = get_train_execution(self.config)
+        backend = str(execution.get("training_backend", "placeholder")).lower()
+        if backend == "xvars_videochatgpt_lora":
+            world_size = torch.cuda.device_count() or get_system_gpu_count(self.config)
+            requested_gpus = get_system_gpu_count(self.config)
+            use_ddp = world_size > 1 and int(requested_gpus) > 1
+            logging.info(
+                "VQA train launch | mode=%s | world_size=%s",
+                "ddp" if use_ddp else "single",
+                world_size if use_ddp else 1,
+            )
+
+            ctx = mp.get_context("spawn")
+            queue = ctx.SimpleQueue()
+            if use_ddp:
+                mp.spawn(
+                    VQAModel._worker_ddp,
+                    args=(world_size, self.config_path, self.config, queue, train_set, valid_set, use_wandb),
+                    nprocs=world_size,
+                )
+            else:
+                VQAModel._worker_ddp(
+                    rank=0,
+                    world_size=1,
+                    config_path=self.config_path,
+                    config=self.config,
+                    return_queue=queue,
+                    train_set=train_set,
+                    valid_set=valid_set,
+                    use_wandb=use_wandb,
+                )
+            ckpt = queue.get()
+            self.best_checkpoint = ckpt
+            self.last_loaded_weights = ckpt
+            return ckpt
+
+        raise ValueError(
+            f"Unsupported VQA training backend '{backend}'. "
+            "Only 'xvars_videochatgpt_lora' is supported."
+        )
+
+    def infer(
+        self,
+        test_set: str | None = None,
+        weights: str | None = None,
+        use_wandb: bool = True,
+        video_path: str | None = None,
+        question: str | None = None,
+        **kwargs,
+    ) -> dict:
+        del kwargs
+        from opensportslib.core.trainer.vqa_trainer import Trainer_VQA
+        from opensportslib.datasets.builder import build_dataset
+        from opensportslib.models.builder import build_model
+        from opensportslib.core.utils.config import select_device
+
+        direct_requested = video_path is not None or question is not None
+        if direct_requested and test_set is not None:
+            raise ValueError("Provide either `test_set` or `video_path`/`question`, not both.")
+        if direct_requested and (not video_path or not str(question or "").strip()):
+            raise ValueError("Direct VQA inference requires both `video_path` and a non-empty `question`.")
+
+        self.config = resolve_config_omega(self.config, weights=weights)
+        backend = get_vqa_backend(self.config)
+        effective_weights = weights if weights is not None else self.last_loaded_weights
+        if backend == "qwen_xvars_infer" and effective_weights is not None:
+            raise ValueError("The 'qwen_xvars_infer' backend does not support adapter weights for infer().")
+        _set_model_checkpoint_path(self.config, effective_weights)
+        self.trainer = Trainer_VQA(self.config)
+        if effective_weights is not None:
+            # Validate OpenSportsLib adapter metadata before allocating the base model.
+            self.trainer.load(effective_weights)
+        resolved_video_path = None
+        if direct_requested:
+            resolved_video_path = expand(str(video_path))
+            if not os.path.isfile(resolved_video_path):
+                raise FileNotFoundError(f"Video file not found: {resolved_video_path}")
+        device = select_device(self.config.SYSTEM)
+        model, _ = build_model(self.config, device)
+        if direct_requested:
+            test_data = [
+                {
+                    "id": os.path.splitext(os.path.basename(resolved_video_path))[0],
+                    "question": str(question).strip(),
+                    "references": [],
+                    "video_path": resolved_video_path,
+                    "video_spatio_temporal_features": None,
+                    "prior_prediction_text": "",
+                    "labels": {},
+                    "metadata": {},
+                    "_xvars_demo_parity_direct_infer": True,
+                }
+            ]
+        else:
+            test_set = self._resolve_split_path("test", test_set)
+            test_data = build_dataset(self.config, test_set, None, split="test")
+        self._init_wandb(use_wandb=use_wandb)
+        return self.trainer.infer(model, test_data, use_wandb=use_wandb)
+
+    def evaluate(
+        self,
+        test_set: str | None = None,
+        weights: str | None = None,
+        predictions: str | dict | None = None,
+        use_wandb: bool = True,
+        **kwargs,
+    ) -> dict | str | None:
+        del kwargs
+        from opensportslib.core.trainer.vqa_trainer import Trainer_VQA
+        from opensportslib.datasets.builder import build_dataset
+
+        self.config = resolve_config_omega(self.config, weights=weights)
+        test_set = self._resolve_split_path("test", test_set)
+        test_data = build_dataset(self.config, test_set, None, split="test")
+        self._init_wandb(use_wandb=use_wandb)
+        if predictions is None:
+            predictions = self.infer(test_set=test_set, weights=weights, use_wandb=use_wandb)
+        elif isinstance(predictions, str):
+            import json
+
+            with open(expand(predictions), encoding="utf-8") as f:
+                predictions = json.load(f)
+        self.trainer = self.trainer or Trainer_VQA(self.config)
+        return self.trainer.evaluate(predictions, test_data, use_wandb=use_wandb)
+
+    def save_predictions(
+        self,
+        output_path: str,
+        predictions: dict,
+        output_format: str = "osl",
+    ) -> str:
+        """Persist VQA predictions, optionally as X-VARS-compatible rows."""
+
+        if str(output_format).lower() != "xvars":
+            return super().save_predictions(output_path, predictions)
+
+        import json
+
+        payload = self._to_xvars_prediction_rows(predictions)
+        dst = expand(output_path)
+        os.makedirs(os.path.dirname(dst) or ".", exist_ok=True)
+        with open(dst, "w", encoding="utf-8") as f:
+            json.dump(payload, f)
+        return dst
+
+    @staticmethod
+    def _to_xvars_prediction_rows(predictions: dict[str, Any]) -> list[dict[str, Any]]:
+        rows = []
+        for item in predictions.get("data", []) if isinstance(predictions, dict) else []:
+            video_path = str(item.get("video_path") or "")
+            video_name = os.path.splitext(os.path.basename(video_path))[0] if video_path else str(item.get("id"))
+            rows.append(
+                {
+                    "id": item.get("id"),
+                    "video_name": video_name,
+                    "Q": item.get("question"),
+                    "pred": item.get("answer_text"),
+                }
+            )
+        return rows

{opensportslib-0.2.0 → opensportslib-0.2.0.dev2}/opensportslib/cli.py

@@ -11,6 +11,8 @@ def main(argv: Optional[list[str]] = None) -> int:
     parser.add_argument("command", choices=["setup"])
     parser.add_argument("--pyg", action="store_true")
     parser.add_argument("--dali", action="store_true")
+    parser.add_argument("--xvars", action="store_true")
+    parser.add_argument("--qwen", action="store_true")
 
     args = parser.parse_args(argv)
 
@@ -18,6 +20,8 @@ def main(argv: Optional[list[str]] = None) -> int:
         setup(
             pyg=args.pyg,
             dali=args.dali,
+            xvars=args.xvars,
+            qwen=args.qwen
         )
         return 0
 

{opensportslib-0.2.0 → opensportslib-0.2.0.dev2}/opensportslib/configs/classification/default.yaml

@@ -2,23 +2,13 @@ TASK: classification
 VERSION: 2
 
 SYSTEM:
-  paths:
-    log_dir: ./logs
-    save_dir: ./checkpoints
-  device: cuda
   gpu:
-    count: 1
-    id: 0
-  reproducibility:
-    use_seed: true
-    seed: 42
+    count: 4
 
 DATA:
   common:
     dataset_name: dataset_name
     data_root: /path/to/data
-    runtime:
-      loader_backend: opencv
     splits:
       train:
         annotation_path: /path/to/train.json
@@ -61,17 +51,6 @@ DATA:
         color_mode: rgb
 
 MODEL:
-  runtime:
-    dtype: fp32
-    device: auto
-    compile: false
-    freeze: false
-  load:
-    checkpoint_path: null
-    pretrained: false
-    strict: true
-    map_location: null
-    format: auto
   components:
     video_encoder:
       kind: encoder
@@ -81,6 +60,14 @@ MODEL:
         name: null
       params: {}
       overrides: {}
+    video_adapter:
+      kind: adapter
+      source:
+        provider: opensportslib
+        registry: neck
+        name: null
+      params: {}
+      overrides: {}
     task_head:
       kind: head
       source:
@@ -91,6 +78,8 @@ MODEL:
       overrides: {}
   topology:
     - from: video_encoder
+      to: video_adapter
+    - from: video_adapter
       to: task_head
 
 IO:

opensportslib-0.2.0/opensportslib/configs/classification/frames_npy/sngar-frames.yaml → opensportslib-0.2.0.dev2/opensportslib/configs/classification/sngar_frames.yaml

@@ -1,24 +1,11 @@
-TASK: classification
-VERSION: 2
-
 SYSTEM:
   paths:
-    log_dir: ./logs
     save_dir: ./checkpoints_video
-  device: cuda
-  gpu:
-    count: 4
-    id: 0
-  reproducibility:
-    use_seed: true
-    seed: 42
 
 DATA:
   common:
     dataset_name: sngar
     data_root: /home/vorajv/dataset/sngar-frames
-    runtime:
-      loader_backend: opencv
     splits:
       train:
         annotation_path: ${DATA.common.data_root}/train.json
@@ -69,20 +56,8 @@ DATA:
           training_matches: 45
 
 MODEL:
-  runtime:
-    dtype: fp32
-    device: auto
-    compile: false
-    freeze: false
-  load:
-    checkpoint_path: null
-    pretrained: false
-    strict: true
-    map_location: null
-    format: auto
   components:
     video_encoder:
-      kind: encoder
       source:
         provider: opensportslib
         registry: backbone
@@ -97,7 +72,6 @@ MODEL:
       overrides:
         unfreeze_last_n_layers: 0
     video_adapter:
-      kind: adapter
       source:
         provider: opensportslib
         registry: neck
@@ -108,9 +82,7 @@ MODEL:
         dropout: 0.1
         lstm_dropout: 0.3
         num_attention_heads: 8
-      overrides: {}
     task_head:
-      kind: head
       source:
         provider: opensportslib
         registry: head
@@ -119,26 +91,9 @@ MODEL:
         hidden_dim: 64
         num_classes: 10
         dropout: 0.1
-      overrides: {}
-  topology:
-    - from: video_encoder
-      to: video_adapter
-    - from: video_adapter
-      to: task_head
-
-IO:
-  inputs:
-    pixel_values: video_encoder
-    label: task_head
-  outputs:
-    logits: task_head
 
 TRAIN:
-  trainer:
-    type: classification
   epochs: 100
-  criterion:
-    type: CrossEntropyLoss
   optimizer:
     type: AdamW
     lr: 0.00005
@@ -153,19 +108,14 @@ TRAIN:
     factor: 0.1
     min_lr: 1e-8
   execution:
-    enabled: true
     use_amp: true
     mixup_alpha: 0.2
-    log_interval: 10
     detailed_results: true
   sampling:
     use_weighted_sampler: true
-    use_weighted_loss: false
     samples_per_class: 4000
   selection:
     monitor: loss
-    mode: min
     patience: 5
   checkpoint:
     save_every: 5
-    save_best: true