opensportslib 0.2.0.dev5__tar.gz → 0.2.0.dev7__tar.gz

{opensportslib-0.2.0.dev5/opensportslib.egg-info → opensportslib-0.2.0.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.2.0.dev5
+Version: 0.2.0.dev7
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12
@@ -26,7 +26,7 @@ Requires-Dist: tabulate
 Requires-Dist: pytorch-lightning
 Requires-Dist: pandas
 Requires-Dist: pyarrow
-Requires-Dist: h5py
+Requires-Dist: h5py==3.16.0
 Requires-Dist: huggingface_hub
 Requires-Dist: easydict
 Provides-Extra: test
@@ -317,10 +317,25 @@ single_prediction = my_model.infer(
 ```
 
 Use `opensportslib/configs/vqa/xvars.yaml` with `opensportslib setup --vqa_xvars`
-for the X-VARS-compatible backend, or `opensportslib/configs/vqa/qwen.yaml` with
-`opensportslib setup --vqa_qwen` for the Qwen-compatible backend. The Qwen
-backend currently supports `Qwen/Qwen2.5-7B-Instruct` and
-`Qwen/Qwen3.5-9B-Base`.
+for the X-VARS backend. OpenSportsLib supports three VQA options:
+
+- `opensportslib/configs/vqa/xvars.yaml`
+  Original X-VARS / Video-ChatGPT path.
+- CLIP features + Qwen
+  Use `opensportslib/configs/vqa/qwen.yaml` for inference and
+  `opensportslib/configs/vqa/qwen_lora.yaml` for LoRA training.
+- `opensportslib/configs/vqa/qwen3_vl_native.yaml`
+  Full end-to-end native QwenVL path. This is the single canonical QwenVL
+  config; change `MODEL.components.llm_decoder.params.repo_id` to switch model
+  IDs.
+
+Use `opensportslib setup --vqa_qwen` for both the CLIP+Qwen and native QwenVL
+paths. The CLIP+Qwen configs support `Qwen/Qwen2.5-7B-Instruct` and
+`Qwen/Qwen3.5-9B-Base`. The native QwenVL config defaults to
+`Qwen/Qwen3-VL-8B-Instruct` and supports:
+
+- `Qwen/Qwen3-VL-8B-Instruct`
+- `Qwen/Qwen2.5-VL-7B-Instruct`
 
 For X-VARS, `feature_source: indexed_or_raw_clip` prefers indexed CLIP features
 when available and falls back to extracting CLIP features from raw video during

{opensportslib-0.2.0.dev5 → opensportslib-0.2.0.dev7}/README.md

@@ -281,10 +281,25 @@ single_prediction = my_model.infer(
 ```
 
 Use `opensportslib/configs/vqa/xvars.yaml` with `opensportslib setup --vqa_xvars`
-for the X-VARS-compatible backend, or `opensportslib/configs/vqa/qwen.yaml` with
-`opensportslib setup --vqa_qwen` for the Qwen-compatible backend. The Qwen
-backend currently supports `Qwen/Qwen2.5-7B-Instruct` and
-`Qwen/Qwen3.5-9B-Base`.
+for the X-VARS backend. OpenSportsLib supports three VQA options:
+
+- `opensportslib/configs/vqa/xvars.yaml`
+  Original X-VARS / Video-ChatGPT path.
+- CLIP features + Qwen
+  Use `opensportslib/configs/vqa/qwen.yaml` for inference and
+  `opensportslib/configs/vqa/qwen_lora.yaml` for LoRA training.
+- `opensportslib/configs/vqa/qwen3_vl_native.yaml`
+  Full end-to-end native QwenVL path. This is the single canonical QwenVL
+  config; change `MODEL.components.llm_decoder.params.repo_id` to switch model
+  IDs.
+
+Use `opensportslib setup --vqa_qwen` for both the CLIP+Qwen and native QwenVL
+paths. The CLIP+Qwen configs support `Qwen/Qwen2.5-7B-Instruct` and
+`Qwen/Qwen3.5-9B-Base`. The native QwenVL config defaults to
+`Qwen/Qwen3-VL-8B-Instruct` and supports:
+
+- `Qwen/Qwen3-VL-8B-Instruct`
+- `Qwen/Qwen2.5-VL-7B-Instruct`
 
 For X-VARS, `feature_source: indexed_or_raw_clip` prefers indexed CLIP features
 when available and falls back to extracting CLIP features from raw video during

{opensportslib-0.2.0.dev5 → opensportslib-0.2.0.dev7}/opensportslib/apis/localization.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import time
+from types import SimpleNamespace
 
 from opensportslib.apis.base_task_model import BaseTaskModel
 from opensportslib.core.config.accessors import (
@@ -14,6 +15,7 @@ from opensportslib.core.config.accessors import (
     get_split_annotation_path,
     get_split_cfg,
     set_split_annotation_path,
+    set_loader_backend,
     get_model_family,
 )
 from opensportslib.core.utils.config import expand
@@ -22,6 +24,17 @@ from opensportslib.core.utils.config import expand
 class LocalizationModel(BaseTaskModel):
     """Top-level task wrapper for localization / spotting."""
 
+    _HF_BACKEND_SPLIT_TYPES = {
+        "dali": {
+            "VideoGameWithOpencv": "VideoGameWithDali",
+            "VideoGameWithOpencvVideo": "VideoGameWithDaliVideo",
+        },
+        "opencv": {
+            "VideoGameWithDali": "VideoGameWithOpencv",
+            "VideoGameWithDaliVideo": "VideoGameWithOpencvVideo",
+        },
+    }
+
     # def __init__(self, config=None, weights=None):
     #     super().__init__(config=config, weights=None)
     #     if weights is not None:
@@ -62,6 +75,82 @@ class LocalizationModel(BaseTaskModel):
                 device,
             )
 
+    @staticmethod
+    def _device_type(device) -> str:
+        device_type = getattr(device, "type", device)
+        return str(device_type).split(":", 1)[0].lower()
+
+    @staticmethod
+    def _is_hf_repo_weights(weights: str | None) -> bool:
+        if not weights:
+            return False
+        from opensportslib.core.utils.config import is_local_path
+
+        return not is_local_path(weights)
+
+    @staticmethod
+    def _iter_split_items(splits):
+        if splits is None:
+            return []
+        if isinstance(splits, dict):
+            return list(splits.items())
+        return list(vars(splits).items())
+
+    @staticmethod
+    def _ensure_namespace(parent, attr: str):
+        current = getattr(parent, attr, None)
+        if current is None:
+            current = SimpleNamespace()
+            setattr(parent, attr, current)
+        return current
+
+    def _normalize_opencv_dataloader(self, split_name: str, split_cfg) -> None:
+        dataloader = getattr(split_cfg, "dataloader", None)
+        if dataloader is None:
+            dataloader = SimpleNamespace()
+            setattr(split_cfg, "dataloader", dataloader)
+
+        if getattr(dataloader, "batch_size", None) is None:
+            dataloader.batch_size = 1
+        if getattr(dataloader, "shuffle", None) is None:
+            dataloader.shuffle = split_name == "train"
+        if getattr(dataloader, "num_workers", None) is None:
+            dataloader.num_workers = 0
+        if getattr(dataloader, "pin_memory", None) is None:
+            dataloader.pin_memory = False
+
+    def _adapt_hf_backend_for_device(self, weights: str | None) -> None:
+        if not self._is_hf_repo_weights(weights):
+            return
+
+        from opensportslib.core.utils.config import select_device
+
+        device = select_device(self.config.SYSTEM)
+        backend = "dali" if self._device_type(device) == "cuda" else "opencv"
+        current_backend = get_loader_backend(self.config)
+        set_loader_backend(self.config, backend)
+
+        common = getattr(getattr(self.config, "DATA", None), "common", None)
+        if common is None:
+            return
+        splits = self._ensure_namespace(common, "splits")
+        remap = self._HF_BACKEND_SPLIT_TYPES[backend]
+
+        for split_name, split_cfg in self._iter_split_items(splits):
+            split_type = getattr(split_cfg, "type", None)
+            if split_type in remap:
+                setattr(split_cfg, "type", remap[split_type])
+            if backend == "opencv":
+                self._normalize_opencv_dataloader(split_name, split_cfg)
+
+        if current_backend != backend:
+            logging.info(
+                "HF localization runtime backend override: %s -> %s for weights=%s",
+                current_backend,
+                backend,
+                weights,
+            )
+
     def load_weights(
         self,
         weights: str | None = None,
@@ -80,6 +169,7 @@ class LocalizationModel(BaseTaskModel):
         if weights is None:
             raise ValueError("`weights` must be provided to load_weights().")
 
+        self._adapt_hf_backend_for_device(weights)
         device = select_device(self.config.SYSTEM)
         self._gate_multi_gpu_by_device(device)
         if self.model is None:
@@ -183,8 +273,10 @@ class LocalizationModel(BaseTaskModel):
         # E2E validation mAP uses the `valid_data_frames` split; keep it in sync
         # with explicit valid annotation overrides.
         self._set_split_path("valid_data_frames", valid_set)
-        
+
         self.config = resolve_config_omega(self.config, weights=weights)
+        effective_weights = weights if weights is not None else self.last_loaded_weights
+        self._adapt_hf_backend_for_device(effective_weights)
         check_config(self.config, split="train")
         init_wandb(
             self.config_path,
@@ -196,8 +288,6 @@ class LocalizationModel(BaseTaskModel):
         logging.info("Configuration:")
         logging.info(self.config)
 
-        effective_weights = weights if weights is not None else self.last_loaded_weights
-
         def set_seed(seed):
             random.seed(seed)
             np.random.seed(seed)
@@ -298,6 +388,8 @@ class LocalizationModel(BaseTaskModel):
         self._set_split_path("test", test_set)
 
         self.config = resolve_config_omega(self.config, weights=weights)
+        effective_weights = weights if weights is not None else self.last_loaded_weights
+        self._adapt_hf_backend_for_device(effective_weights)
         check_config(self.config, split="test")
         self.config.infer_split = whether_infer_split(get_split_cfg(self.config, "test"))
 
@@ -313,8 +405,6 @@ class LocalizationModel(BaseTaskModel):
 
         start = time.time()
 
-        effective_weights = weights if weights is not None else self.last_loaded_weights
-
         if effective_weights is not None:
             if self.model is None or self.last_loaded_weights != effective_weights:
                 self.load_weights(weights=effective_weights)
@@ -368,6 +458,8 @@ class LocalizationModel(BaseTaskModel):
         test_set = self._resolve_split_path("test", test_set)
         self._set_split_path("test", test_set)
         self.config = resolve_config_omega(self.config, weights=weights)
+        effective_weights = weights if weights is not None else self.last_loaded_weights
+        self._adapt_hf_backend_for_device(effective_weights)
         check_config(self.config, split="test")
         self.config.infer_split = whether_infer_split(get_split_cfg(self.config, "test"))
 

{opensportslib-0.2.0.dev5 → opensportslib-0.2.0.dev7}/opensportslib/apis/vqa.py

@@ -51,6 +51,7 @@ class VQAModel(BaseTaskModel):
         return_queue=None,
         train_set=None,
         valid_set=None,
+        resume_from_checkpoint=None,
         use_wandb=False,
     ):
         import torch
@@ -97,14 +98,10 @@ class VQAModel(BaseTaskModel):
             train_data = build_dataset(config, train_set, None, split="train")
             valid_data = build_dataset(config, valid_set, None, split="valid")
             trainer = Trainer_VQA(config)
-            ckpt = trainer.train(
-                None,
-                train_data,
-                valid_data,
-                rank=rank,
-                world_size=world_size,
-                use_wandb=use_wandb,
-            )
+            train_kwargs = {"rank": rank, "world_size": world_size, "use_wandb": use_wandb}
+            if resume_from_checkpoint is not None:
+                train_kwargs["resume_from_checkpoint"] = resume_from_checkpoint
+            ckpt = trainer.train(None, train_data, valid_data, **train_kwargs)
             if rank == 0 and return_queue is not None:
                 return_queue.put(ckpt)
         finally:
@@ -139,37 +136,73 @@ class VQAModel(BaseTaskModel):
         train_set: str | None = None,
         valid_set: str | None = None,
         weights: str | None = None,
+        resume_from_checkpoint: str | None = None,
+        use_ddp: bool | None = None,
         use_wandb: bool = True,
         **kwargs,
     ) -> str | None:
         del kwargs
+
+        self.config = resolve_config_omega(self.config, weights=weights)
+        execution = get_train_execution(self.config)
+        backend = str(execution.get("training_backend", "placeholder")).lower()
+        vqa_backend = get_vqa_backend(self.config)
+        if vqa_backend == "qwen_xvars_infer" and backend != "qwen_xvars_lora":
+            raise ValueError(
+                "The 'qwen_xvars_infer' backend requires TRAIN.execution.training_backend='qwen_xvars_lora' for train()."
+            )
+        if vqa_backend == "qwen_vl_native_infer" and backend != "qwen_vl_native_lora":
+            raise ValueError(
+                "The 'qwen_vl_native_infer' backend requires TRAIN.execution.training_backend='qwen_vl_native_lora' for train()."
+            )
         import torch
         import torch.multiprocessing as mp
 
-        self.config = resolve_config_omega(self.config, weights=weights)
-        if get_vqa_backend(self.config) == "qwen_xvars_infer":
-            raise ValueError("The 'qwen_xvars_infer' backend is inference-only and does not support train().")
         train_set = self._resolve_split_path("train", train_set)
         valid_set = self._resolve_split_path("valid", valid_set)
-        execution = get_train_execution(self.config)
-        backend = str(execution.get("training_backend", "placeholder")).lower()
-        if backend == "xvars_videochatgpt_lora":
-            world_size = torch.cuda.device_count() or get_system_gpu_count(self.config)
+        if backend in {"xvars_videochatgpt_lora", "qwen_xvars_lora", "qwen_vl_native_lora"}:
             requested_gpus = get_system_gpu_count(self.config)
-            use_ddp = world_size > 1 and int(requested_gpus) > 1
+            available_gpus = int(torch.cuda.device_count() or 0)
+            if requested_gpus > 0:
+                world_size = min(available_gpus, int(requested_gpus)) if available_gpus > 0 else int(requested_gpus)
+            else:
+                world_size = available_gpus
+            effective_world_size = max(int(world_size), 1)
+            if use_ddp is None:
+                launch_ddp = effective_world_size > 1
+            else:
+                launch_ddp = bool(use_ddp) and effective_world_size > 1
+            if requested_gpus > 0 and available_gpus > 0 and requested_gpus != available_gpus:
+                logging.info(
+                    "VQA train GPU selection | requested=%s | available=%s | launching=%s",
+                    requested_gpus,
+                    available_gpus,
+                    effective_world_size if launch_ddp else 1,
+                )
             logging.info(
-                "VQA train launch | mode=%s | world_size=%s",
-                "ddp" if use_ddp else "single",
-                world_size if use_ddp else 1,
+                "VQA train launch | mode=%s | requested_gpus=%s | available_gpus=%s | world_size=%s",
+                "ddp" if launch_ddp else "single",
+                requested_gpus,
+                available_gpus,
+                effective_world_size if launch_ddp else 1,
             )
 
             ctx = mp.get_context("spawn")
             queue = ctx.SimpleQueue()
-            if use_ddp:
+            if launch_ddp:
                 mp.spawn(
                     VQAModel._worker_ddp,
-                    args=(world_size, self.config_path, self.config, queue, train_set, valid_set, use_wandb),
-                    nprocs=world_size,
+                    args=(
+                        effective_world_size,
+                        self.config_path,
+                        self.config,
+                        queue,
+                        train_set,
+                        valid_set,
+                        resume_from_checkpoint,
+                        use_wandb,
+                    ),
+                    nprocs=effective_world_size,
                 )
             else:
                 VQAModel._worker_ddp(
@@ -180,6 +213,7 @@ class VQAModel(BaseTaskModel):
                     return_queue=queue,
                     train_set=train_set,
                     valid_set=valid_set,
+                    resume_from_checkpoint=resume_from_checkpoint,
                     use_wandb=use_wandb,
                 )
             ckpt = queue.get()
@@ -189,7 +223,7 @@ class VQAModel(BaseTaskModel):
 
         raise ValueError(
             f"Unsupported VQA training backend '{backend}'. "
-            "Only 'xvars_videochatgpt_lora' is supported."
+            "Only 'xvars_videochatgpt_lora' and 'qwen_xvars_lora' are supported."
         )
 
     def infer(
@@ -216,8 +250,6 @@ class VQAModel(BaseTaskModel):
         self.config = resolve_config_omega(self.config, weights=weights)
         backend = get_vqa_backend(self.config)
         effective_weights = weights if weights is not None else self.last_loaded_weights
-        if backend == "qwen_xvars_infer" and effective_weights is not None:
-            raise ValueError("The 'qwen_xvars_infer' backend does not support adapter weights for infer().")
         _set_model_checkpoint_path(self.config, effective_weights)
         self.trainer = Trainer_VQA(self.config)
         if effective_weights is not None:
@@ -237,6 +269,8 @@ class VQAModel(BaseTaskModel):
                     "question": str(question).strip(),
                     "references": [],
                     "video_path": resolved_video_path,
+                    "frame_paths": [],
+                    "video_frames": [],
                     "video_spatio_temporal_features": None,
                     "prior_prediction_text": "",
                     "labels": {},

{opensportslib-0.2.0.dev5 → opensportslib-0.2.0.dev7}/opensportslib/configs/default.yaml

@@ -2,7 +2,7 @@ SYSTEM:
   paths:
     log_dir: ./logs
     save_dir: ./checkpoints
-  device: cuda
+  device: auto
   gpu:
     id: 0
   reproducibility:

{opensportslib-0.2.0.dev5 → opensportslib-0.2.0.dev7}/opensportslib/configs/localization/h5_header_distance_speed_angle.yaml

@@ -11,14 +11,14 @@ SYSTEM:
 DATA:
   common:
     dataset_name: h5_headers
-    data_root: ./data
+    data_root: /home/vorajv/dataset/silvio-h5/
     classes:
       - header
     splits:
       test:
         type: H5OSLJsonSpotting
-        annotation_path: ./data/h5.json
-        source_path: ./data
+        annotation_path: /home/vorajv/dataset/silvio-h5/h5_tiny.json
+        source_path: /home/vorajv/dataset/silvio-h5/
         dataloader:
           batch_size: 1
           shuffle: false

{opensportslib-0.2.0.dev5 → opensportslib-0.2.0.dev7}/opensportslib/configs/vqa/default.yaml

@@ -5,7 +5,7 @@ SYSTEM:
   paths:
     save_dir: ./checkpoints_vqa
     work_dir: ${SYSTEM.paths.save_dir}
-  device: cuda
+  device: auto
   gpu:
     count: 1
     id: 0

opensportslib-0.2.0.dev7/opensportslib/configs/vqa/qwen3_vl_native.yaml

@@ -0,0 +1,81 @@
+SYSTEM:
+  paths:
+    save_dir: ./checkpoints_vqa_qwen3_vl_native
+  gpu:
+    count: 4
+    id: null
+
+MODEL:
+  topology: []
+  components:
+    video_encoder:
+      kind: encoder
+      source:
+        provider: opensportslib
+        name: qwen_vl_native
+      params:
+        feature_source: raw_video
+        native_vl:
+          visual_input_mode: frames
+          num_frames: 8
+          min_pixels: 50176 # 28*28*64
+          max_pixels: 401408 # 28*28*512
+      overrides: {}
+    llm_decoder:
+      kind: decoder
+      source:
+        provider: huggingface
+        # Canonical end-to-end QwenVL config.
+        # Swap this model ID if you want another supported QwenVL checkpoint.
+        # - Qwen/Qwen3-VL-8B-Instruct
+        # - Qwen/Qwen2.5-VL-7B-Instruct
+        name: Qwen/Qwen3-VL-8B-Instruct
+      params:
+        # Supported examples:
+        # - Qwen/Qwen3-VL-8B-Instruct
+        # - Qwen/Qwen2.5-VL-7B-Instruct
+        repo_id: Qwen/Qwen3-VL-8B-Instruct
+      overrides: {}
+  metadata:
+    backend: qwen_vl_native_infer
+
+TRAIN:
+  optimizer:
+    type: adamw_torch
+    lr: 0.0002
+    weight_decay: 0.001
+  execution:
+    acc_grad_iter: 4
+    training_backend: qwen_vl_native_lora
+    prompt:
+      system_prompt: You are a football video assistant. Answer the VQA question using the provided visual evidence and referee priors.
+    native_vl:
+      visual_input_mode: frames
+      num_frames: 8
+      min_pixels: 50176 # 28*28*64
+      max_pixels: 401408 # 28*28*512
+    hf:
+      tokenizer_id: ${MODEL.components.llm_decoder.params.repo_id}
+      prefer_cuda: true
+      local_files_only: false
+      device_map: null
+    sft:
+      per_device_train_batch_size: 1
+      per_device_eval_batch_size: 1
+      max_seq_length: 1024
+      max_grad_norm: 1.0
+      memory_cache_rows: 0
+      gradient_checkpointing: true
+      ddp_find_unused_parameters: false
+      ddp_broadcast_buffers: false
+      resume_optimizer_state: false
+      cache_tokenized_rows: false
+      disable_tqdm: false
+      save_strategy: epoch
+      # save_steps: 100
+      # save_total_limit: 3
+      evaluation_strategy: "no"
+    lora:
+      prepare_kbit: false
+    quantization:
+      enabled: false

opensportslib-0.2.0.dev7/opensportslib/configs/vqa/qwen_lora.yaml

@@ -0,0 +1,69 @@
+SYSTEM:
+  paths:
+    save_dir: ./checkpoints_vqa_qwen_lora
+  gpu:
+    count: 4
+
+MODEL:
+  components:
+    video_encoder:
+      kind: encoder
+      source:
+        provider: opensportslib
+        name: xvars_clip_features
+      load:
+        weights_path: /home/vorajv/X-VARS/weights/14_model.pth.tar
+      params:
+        feature_source: indexed_or_raw_clip
+        vision_tower: openai/clip-vit-large-patch14
+        feature_dim: 1024
+      overrides: {}
+    mm_projector:
+      kind: projector
+      source:
+        provider: opensportslib
+      params:
+        input_dim: 1024
+      overrides: {}
+    llm_decoder:
+      kind: decoder
+      source:
+        provider: huggingface
+        name: Qwen/Qwen3.5-9B-Base
+      params:
+        repo_id: Qwen/Qwen3.5-9B-Base
+      overrides: {}
+  metadata:
+    backend: qwen_xvars_infer
+
+TRAIN:
+  execution:
+    training_backend: qwen_xvars_lora
+    prompt:
+      system_prompt: You are a football video assistant. Answer the VQA question using the provided video context and referee priors.
+    hf:
+      tokenizer_id: ${MODEL.components.llm_decoder.params.repo_id}
+      prefer_cuda: true
+      local_files_only: false
+    sft:
+      per_device_train_batch_size: 1
+      per_device_eval_batch_size: 1
+      gradient_accumulation_steps: 8
+      max_seq_length: 480
+      gradient_checkpointing: true
+      disable_tqdm: true
+      save_strategy: epoch
+      evaluation_strategy: "no"
+    lora:
+      r: 16
+      alpha: 32
+      dropout: 0.05
+      bias: none
+      prepare_kbit: false
+      target_modules: [q_proj, v_proj, k_proj, o_proj]
+    quantization:
+      enabled: false
+      load_in_4bit: true
+      bnb_4bit_quant_type: nf4
+      compute_dtype: float16
+      bnb_4bit_use_double_quant: true