opensportslib 0.2.0.dev5__tar.gz → 0.2.0.dev6__tar.gz

{opensportslib-0.2.0.dev5/opensportslib.egg-info → opensportslib-0.2.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.2.0.dev5
+Version: 0.2.0.dev6
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12

{opensportslib-0.2.0.dev5 → opensportslib-0.2.0.dev6}/opensportslib/apis/vqa.py

@@ -51,6 +51,7 @@ class VQAModel(BaseTaskModel):
         return_queue=None,
         train_set=None,
         valid_set=None,
+        resume_from_checkpoint=None,
         use_wandb=False,
     ):
         import torch
@@ -97,14 +98,10 @@ class VQAModel(BaseTaskModel):
             train_data = build_dataset(config, train_set, None, split="train")
             valid_data = build_dataset(config, valid_set, None, split="valid")
             trainer = Trainer_VQA(config)
-            ckpt = trainer.train(
-                None,
-                train_data,
-                valid_data,
-                rank=rank,
-                world_size=world_size,
-                use_wandb=use_wandb,
-            )
+            train_kwargs = {"rank": rank, "world_size": world_size, "use_wandb": use_wandb}
+            if resume_from_checkpoint is not None:
+                train_kwargs["resume_from_checkpoint"] = resume_from_checkpoint
+            ckpt = trainer.train(None, train_data, valid_data, **train_kwargs)
             if rank == 0 and return_queue is not None:
                 return_queue.put(ckpt)
         finally:
@@ -139,37 +136,73 @@ class VQAModel(BaseTaskModel):
         train_set: str | None = None,
         valid_set: str | None = None,
         weights: str | None = None,
+        resume_from_checkpoint: str | None = None,
+        use_ddp: bool | None = None,
         use_wandb: bool = True,
         **kwargs,
     ) -> str | None:
         del kwargs
+
+        self.config = resolve_config_omega(self.config, weights=weights)
+        execution = get_train_execution(self.config)
+        backend = str(execution.get("training_backend", "placeholder")).lower()
+        vqa_backend = get_vqa_backend(self.config)
+        if vqa_backend == "qwen_xvars_infer" and backend != "qwen_xvars_lora":
+            raise ValueError(
+                "The 'qwen_xvars_infer' backend requires TRAIN.execution.training_backend='qwen_xvars_lora' for train()."
+            )
+        if vqa_backend == "qwen_vl_native_infer" and backend != "qwen_vl_native_lora":
+            raise ValueError(
+                "The 'qwen_vl_native_infer' backend requires TRAIN.execution.training_backend='qwen_vl_native_lora' for train()."
+            )
         import torch
         import torch.multiprocessing as mp
 
-        self.config = resolve_config_omega(self.config, weights=weights)
-        if get_vqa_backend(self.config) == "qwen_xvars_infer":
-            raise ValueError("The 'qwen_xvars_infer' backend is inference-only and does not support train().")
         train_set = self._resolve_split_path("train", train_set)
         valid_set = self._resolve_split_path("valid", valid_set)
-        execution = get_train_execution(self.config)
-        backend = str(execution.get("training_backend", "placeholder")).lower()
-        if backend == "xvars_videochatgpt_lora":
-            world_size = torch.cuda.device_count() or get_system_gpu_count(self.config)
+        if backend in {"xvars_videochatgpt_lora", "qwen_xvars_lora", "qwen_vl_native_lora"}:
             requested_gpus = get_system_gpu_count(self.config)
-            use_ddp = world_size > 1 and int(requested_gpus) > 1
+            available_gpus = int(torch.cuda.device_count() or 0)
+            if requested_gpus > 0:
+                world_size = min(available_gpus, int(requested_gpus)) if available_gpus > 0 else int(requested_gpus)
+            else:
+                world_size = available_gpus
+            effective_world_size = max(int(world_size), 1)
+            if use_ddp is None:
+                launch_ddp = effective_world_size > 1
+            else:
+                launch_ddp = bool(use_ddp) and effective_world_size > 1
+            if requested_gpus > 0 and available_gpus > 0 and requested_gpus != available_gpus:
+                logging.info(
+                    "VQA train GPU selection | requested=%s | available=%s | launching=%s",
+                    requested_gpus,
+                    available_gpus,
+                    effective_world_size if launch_ddp else 1,
+                )
             logging.info(
-                "VQA train launch | mode=%s | world_size=%s",
-                "ddp" if use_ddp else "single",
-                world_size if use_ddp else 1,
+                "VQA train launch | mode=%s | requested_gpus=%s | available_gpus=%s | world_size=%s",
+                "ddp" if launch_ddp else "single",
+                requested_gpus,
+                available_gpus,
+                effective_world_size if launch_ddp else 1,
             )
 
             ctx = mp.get_context("spawn")
             queue = ctx.SimpleQueue()
-            if use_ddp:
+            if launch_ddp:
                 mp.spawn(
                     VQAModel._worker_ddp,
-                    args=(world_size, self.config_path, self.config, queue, train_set, valid_set, use_wandb),
-                    nprocs=world_size,
+                    args=(
+                        effective_world_size,
+                        self.config_path,
+                        self.config,
+                        queue,
+                        train_set,
+                        valid_set,
+                        resume_from_checkpoint,
+                        use_wandb,
+                    ),
+                    nprocs=effective_world_size,
                 )
             else:
                 VQAModel._worker_ddp(
@@ -180,6 +213,7 @@ class VQAModel(BaseTaskModel):
                     return_queue=queue,
                     train_set=train_set,
                     valid_set=valid_set,
+                    resume_from_checkpoint=resume_from_checkpoint,
                     use_wandb=use_wandb,
                 )
             ckpt = queue.get()
@@ -189,7 +223,7 @@ class VQAModel(BaseTaskModel):
 
         raise ValueError(
             f"Unsupported VQA training backend '{backend}'. "
-            "Only 'xvars_videochatgpt_lora' is supported."
+            "Only 'xvars_videochatgpt_lora' and 'qwen_xvars_lora' are supported."
         )
 
     def infer(
@@ -216,8 +250,6 @@ class VQAModel(BaseTaskModel):
         self.config = resolve_config_omega(self.config, weights=weights)
         backend = get_vqa_backend(self.config)
         effective_weights = weights if weights is not None else self.last_loaded_weights
-        if backend == "qwen_xvars_infer" and effective_weights is not None:
-            raise ValueError("The 'qwen_xvars_infer' backend does not support adapter weights for infer().")
         _set_model_checkpoint_path(self.config, effective_weights)
         self.trainer = Trainer_VQA(self.config)
         if effective_weights is not None:
@@ -237,6 +269,8 @@ class VQAModel(BaseTaskModel):
                     "question": str(question).strip(),
                     "references": [],
                     "video_path": resolved_video_path,
+                    "frame_paths": [],
+                    "video_frames": [],
                     "video_spatio_temporal_features": None,
                     "prior_prediction_text": "",
                     "labels": {},

{opensportslib-0.2.0.dev5 → opensportslib-0.2.0.dev6}/opensportslib/configs/localization/h5_header_distance_speed_angle.yaml

@@ -11,14 +11,14 @@ SYSTEM:
 DATA:
   common:
     dataset_name: h5_headers
-    data_root: ./data
+    data_root: /home/vorajv/dataset/silvio-h5/
     classes:
       - header
     splits:
       test:
         type: H5OSLJsonSpotting
-        annotation_path: ./data/h5.json
-        source_path: ./data
+        annotation_path: /home/vorajv/dataset/silvio-h5/h5_tiny.json
+        source_path: /home/vorajv/dataset/silvio-h5/
         dataloader:
           batch_size: 1
           shuffle: false

opensportslib-0.2.0.dev6/opensportslib/configs/vqa/qwen3_vl_native.yaml

@@ -0,0 +1,74 @@
+SYSTEM:
+  paths:
+    save_dir: ./checkpoints_vqa_qwen3_vl_native
+  gpu:
+    count: 4
+    id: null
+
+MODEL:
+  topology: []
+  components:
+    video_encoder:
+      kind: encoder
+      source:
+        provider: opensportslib
+        name: qwen_vl_native
+      params:
+        feature_source: raw_video
+        native_vl:
+          visual_input_mode: frames
+          num_frames: 8
+          min_pixels: 50176 # 28*28*64
+          max_pixels: 401408 # 28*28*512
+      overrides: {}
+    llm_decoder:
+      kind: decoder
+      source:
+        provider: huggingface
+        name:  Qwen/Qwen3-VL-8B-Instruct
+      params:
+        repo_id: Qwen/Qwen3-VL-8B-Instruct
+      overrides: {}
+  metadata:
+    backend: qwen_vl_native_infer
+
+TRAIN:
+  optimizer:
+    type: adamw_torch
+    lr: 0.0002
+    weight_decay: 0.001
+  execution:
+    acc_grad_iter: 4
+    training_backend: qwen_vl_native_lora
+    prompt:
+      system_prompt: You are a football video assistant. Answer the VQA question using the provided visual evidence and referee priors.
+    native_vl:
+      visual_input_mode: frames
+      num_frames: 8
+      min_pixels: 50176 # 28*28*64
+      max_pixels: 401408 # 28*28*512
+    hf:
+      tokenizer_id: ${MODEL.components.llm_decoder.params.repo_id}
+      prefer_cuda: true
+      local_files_only: false
+      device_map: null
+    sft:
+      per_device_train_batch_size: 1
+      per_device_eval_batch_size: 1
+      max_seq_length: 1024
+      max_grad_norm: 1.0
+      memory_cache_rows: 0
+      gradient_checkpointing: true
+      ddp_find_unused_parameters: false
+      ddp_broadcast_buffers: false
+      resume_optimizer_state: false
+      cache_tokenized_rows: false
+      disable_tqdm: false
+      save_strategy: epoch
+      # save_steps: 100
+      # save_total_limit: 3
+      evaluation_strategy: "no"
+    lora:
+      prepare_kbit: false
+    quantization:
+      enabled: false

opensportslib-0.2.0.dev6/opensportslib/configs/vqa/qwen_lora.yaml

@@ -0,0 +1,69 @@
+SYSTEM:
+  paths:
+    save_dir: ./checkpoints_vqa_qwen_lora
+  gpu:
+    count: 4
+
+MODEL:
+  components:
+    video_encoder:
+      kind: encoder
+      source:
+        provider: opensportslib
+        name: xvars_clip_features
+      load:
+        weights_path: /home/vorajv/X-VARS/weights/14_model.pth.tar
+      params:
+        feature_source: indexed_or_raw_clip
+        vision_tower: openai/clip-vit-large-patch14
+        feature_dim: 1024
+      overrides: {}
+    mm_projector:
+      kind: projector
+      source:
+        provider: opensportslib
+      params:
+        input_dim: 1024
+      overrides: {}
+    llm_decoder:
+      kind: decoder
+      source:
+        provider: huggingface
+        name: Qwen/Qwen3.5-9B-Base
+      params:
+        repo_id: Qwen/Qwen3.5-9B-Base
+      overrides: {}
+  metadata:
+    backend: qwen_xvars_infer
+
+TRAIN:
+  execution:
+    training_backend: qwen_xvars_lora
+    prompt:
+      system_prompt: You are a football video assistant. Answer the VQA question using the provided video context and referee priors.
+    hf:
+      tokenizer_id: ${MODEL.components.llm_decoder.params.repo_id}
+      prefer_cuda: true
+      local_files_only: false
+    sft:
+      per_device_train_batch_size: 1
+      per_device_eval_batch_size: 1
+      gradient_accumulation_steps: 8
+      max_seq_length: 480
+      gradient_checkpointing: true
+      disable_tqdm: true
+      save_strategy: epoch
+      evaluation_strategy: "no"
+    lora:
+      r: 16
+      alpha: 32
+      dropout: 0.05
+      bias: none
+      prepare_kbit: false
+      target_modules: [q_proj, v_proj, k_proj, o_proj]
+    quantization:
+      enabled: false
+      load_in_4bit: true
+      bnb_4bit_quant_type: nf4
+      compute_dtype: float16
+      bnb_4bit_use_double_quant: true

opensportslib-0.2.0.dev6/opensportslib/configs/vqa/qwen_sngar_frames.yaml

@@ -0,0 +1,133 @@
+SYSTEM:
+  paths:
+    save_dir: ./checkpoints_vqa_qwen_sngar_frames
+  gpu:
+    count: 4
+    id: null
+
+DATA:
+  common:
+    dataset_name: sngar-frames-vqa
+    data_root: /home/vorajv/dataset/sngar-frames
+    splits:
+      test:
+        annotation_path: ${DATA.common.data_root}/test.json
+        source_path: ${DATA.common.data_root}
+        dataloader:
+          batch_size: 1
+          shuffle: false
+          num_workers: 0
+          pin_memory: false
+  inputs:
+    video:
+      modality: frames_npy
+      representation: frames_npy
+      source:
+        format: npy
+      sampling:
+        num_frames: 16
+      transform: {}
+      augmentations: {}
+      params: {}
+    question:
+      modality: text
+      representation: raw
+      source:
+        format: json
+      sampling: {}
+      transform: {}
+      augmentations: {}
+      params: {}
+
+MODEL:
+  topology: []
+  components:
+    video_encoder:
+      kind: encoder
+      source:
+        provider: opensportslib
+        name: qwen_vl_native
+      params:
+        feature_source: raw_video
+        native_vl:
+          visual_input_mode: frames
+          num_frames: 8
+          min_pixels: 50176 # 28*28*64
+          max_pixels: 401408 # 28*28*512
+      overrides: {}
+    llm_decoder:
+      kind: decoder
+      source:
+        provider: huggingface
+        name: Qwen/Qwen3-VL-8B-Instruct
+      params:
+        repo_id: Qwen/Qwen3-VL-8B-Instruct
+      overrides: {}
+  metadata:
+    backend: qwen_vl_native_infer
+
+TRAIN:
+  optimizer:
+    hf_optim: adamw_torch
+  execution:
+    acc_grad_iter: 4
+    training_backend: qwen_vl_native_lora
+    prompt:
+      include_priors: false
+      system_prompt: >-
+        You are a soccer action classifier for short soccer clips.
+        Your task is to choose exactly one label from this closed set:
+        PASS, HEADER, HIGH PASS, OUT, CROSS, THROW IN, SHOT,
+        PLAYER SUCCESSFUL TACKLE, FREE KICK, GOAL.
+        Use these label meanings:
+        PASS = a standard grounded or short pass between teammates.
+        HEADER = the ball is intentionally played with the head.
+        HIGH PASS = a lofted or aerial pass to a teammate that is not mainly a cross.
+        OUT = the ball goes out of play.
+        CROSS = the ball is delivered from a wide area into the box or a dangerous central area.
+        THROW IN = play is restarted with a throw-in from the sideline.
+        SHOT = a player attempts to score.
+        PLAYER SUCCESSFUL TACKLE = a defender successfully wins or disrupts possession with a tackle.
+        FREE KICK = play is restarted with a free kick.
+        GOAL = the ball enters the goal and a goal is scored.
+        Decision rules:
+        Prefer the main action visible in the clip.
+        Choose the closest valid label if the clip is ambiguous.
+        Do not output any words other than the single label.
+        Do not explain.
+        Do not output a sentence.
+        Do not output punctuation.
+        Do not output 'The' or any extra text.
+    generation:
+      max_new_tokens: 3
+      temperature: 0.0
+      repetition_penalty: 1.0
+      no_repeat_ngram_size: 2
+    native_vl:
+      visual_input_mode: frames
+      num_frames: 8
+      min_pixels: 50176 # 28*28*64
+      max_pixels: 401408 # 28*28*512
+    hf:
+      tokenizer_id: ${MODEL.components.llm_decoder.params.repo_id}
+      prefer_cuda: true
+      local_files_only: false
+      device_map: auto
+      offload_folder: ./hf_offload_qwen
+    sft:
+      per_device_train_batch_size: 1
+      per_device_eval_batch_size: 1
+      max_seq_length: 1024
+      max_grad_norm: 1.0
+      gradient_checkpointing: true
+      ddp_find_unused_parameters: false
+      ddp_broadcast_buffers: false
+      resume_optimizer_state: false
+      cache_tokenized_rows: false
+      disable_tqdm: false
+      save_strategy: epoch
+      evaluation_strategy: "no"
+    lora:
+      prepare_kbit: false
+    quantization:
+      enabled: false

opensportslib-0.2.0.dev6/opensportslib/configs/vqa/qwen_vl_native.yaml

@@ -0,0 +1,70 @@
+SYSTEM:
+  paths:
+    save_dir: ./checkpoints_vqa_qwen_vl_native
+  gpu:
+    count: 4
+    id: null
+
+MODEL:
+  topology: []
+  components:
+    video_encoder:
+      kind: encoder
+      source:
+        provider: opensportslib
+        name: qwen_vl_native
+      params:
+        feature_source: raw_video
+        native_vl:
+          visual_input_mode: frames
+          num_frames: 8
+          min_pixels: 50176 # 28*28*64
+          max_pixels: 401408 # 28*28*512
+      overrides: {}
+    llm_decoder:
+      kind: decoder
+      source:
+        provider: huggingface
+        name: Qwen/Qwen2.5-VL-7B-Instruct
+      params:
+        repo_id: Qwen/Qwen2.5-VL-7B-Instruct
+      overrides: {}
+  metadata:
+    backend: qwen_vl_native_infer
+
+TRAIN:
+  optimizer:
+    type: adamw_torch
+    lr: 0.0002
+    weight_decay: 0.001
+  execution:
+    acc_grad_iter: 4
+    training_backend: qwen_vl_native_lora
+    prompt:
+      system_prompt: You are a football video assistant. Answer the VQA question using the provided visual evidence and referee priors.
+    native_vl:
+      visual_input_mode: frames
+      num_frames: 8
+      min_pixels: 50176 # 28*28*64
+      max_pixels: 401408 # 28*28*512
+    hf:
+      tokenizer_id: ${MODEL.components.llm_decoder.params.repo_id}
+      prefer_cuda: true
+      local_files_only: false
+      device_map: null
+    sft:
+      per_device_train_batch_size: 1
+      per_device_eval_batch_size: 1
+      max_seq_length: 1024
+      memory_cache_rows: 0
+      gradient_checkpointing: true
+      ddp_find_unused_parameters: false
+      resume_optimizer_state: false
+      cache_tokenized_rows: false
+      disable_tqdm: false
+      save_strategy: epoch
+      evaluation_strategy: "no"
+    lora:
+      prepare_kbit: false
+    quantization:
+      enabled: false

opensportslib-0.2.0.dev6/opensportslib/configs/vqa/qwen_vl_native_awq.yaml

@@ -0,0 +1,45 @@
+SYSTEM:
+  paths:
+    save_dir: ./checkpoints_vqa_qwen_vl_native_awq
+
+MODEL:
+  topology: []
+  components:
+    video_encoder:
+      kind: encoder
+      source:
+        provider: opensportslib
+        name: qwen_vl_native
+      params:
+        feature_source: raw_video
+        native_vl:
+          visual_input_mode: frames
+          num_frames: 8
+          min_pixels: 200704
+          max_pixels: 1003520
+      overrides: {}
+    llm_decoder:
+      kind: decoder
+      source:
+        provider: huggingface
+        name: Qwen/Qwen2.5-VL-7B-Instruct-AWQ
+      params:
+        repo_id: Qwen/Qwen2.5-VL-7B-Instruct-AWQ
+      overrides: {}
+  metadata:
+    backend: qwen_vl_native_infer
+
+TRAIN:
+  execution:
+    prompt:
+      system_prompt: You are a football video assistant. Answer the VQA question using the provided visual evidence and referee priors.
+    native_vl:
+      visual_input_mode: frames
+      num_frames: 8
+      min_pixels: 200704
+      max_pixels: 1003520
+    hf:
+      tokenizer_id: ${MODEL.components.llm_decoder.params.repo_id}
+      prefer_cuda: true
+      local_files_only: false
+      device_map: auto

{opensportslib-0.2.0.dev5 → opensportslib-0.2.0.dev6}/opensportslib/core/config/accessors.py

@@ -466,6 +466,10 @@ def is_xvars_videochatgpt_backend(cfg: Any) -> bool:
     return get_vqa_backend(cfg) == "xvars_videochatgpt"
 
 
+def is_qwen_vl_native_backend(cfg: Any) -> bool:
+    return get_vqa_backend(cfg) == "qwen_vl_native_infer"
+
+
 def normalize_xvars_feature_mode(mode: Any, default: str = "strict_xvars") -> str:
     value = str(mode or "").strip().lower()
     if not value:
@@ -612,6 +616,50 @@ def get_vqa_mm_hidden_size(cfg: Any, default: int = 1024) -> int:
     return int(default)
 
 
+def get_vqa_native_visual_cfg(cfg: Any) -> dict[str, Any]:
+    execution = get_train_execution(cfg)
+    native_cfg = _as_dict(execution.get("native_vl"))
+    if native_cfg:
+        return native_cfg
+
+    encoder_params = get_component_params_by_kind(cfg, "encoder")
+    return _as_dict(encoder_params.get("native_vl"))
+
+
+def get_vqa_native_visual_input_mode(cfg: Any, default: str = "frames") -> str:
+    native_cfg = get_vqa_native_visual_cfg(cfg)
+    mode = str(native_cfg.get("visual_input_mode", default) or default).strip().lower()
+    if mode not in {"frames", "video_with_frames_fallback"}:
+        raise ValueError(
+            f"Unsupported native VL visual_input_mode '{mode}'. "
+            "Expected 'frames' or 'video_with_frames_fallback'."
+        )
+    return mode
+
+
+def get_vqa_native_num_frames(cfg: Any, default: int = 8) -> int:
+    native_cfg = get_vqa_native_visual_cfg(cfg)
+    if native_cfg.get("num_frames") is not None:
+        return max(1, int(native_cfg["num_frames"]))
+
+    sampling = get_data_sampling(cfg)
+    if sampling.get("num_frames") is not None:
+        return max(1, int(sampling["num_frames"]))
+    return int(default)
+
+
+def get_vqa_native_min_pixels(cfg: Any) -> int | None:
+    native_cfg = get_vqa_native_visual_cfg(cfg)
+    value = native_cfg.get("min_pixels")
+    return int(value) if value is not None else None
+
+
+def get_vqa_native_max_pixels(cfg: Any) -> int | None:
+    native_cfg = get_vqa_native_visual_cfg(cfg)
+    value = native_cfg.get("max_pixels")
+    return int(value) if value is not None else None
+
+
 def get_model_runtime_dtype(cfg: Any, default: str = "fp32") -> str:
     runtime = get_model_runtime(cfg)
     dtype = runtime.get("dtype", default)