opensportslib 0.2.0.dev3__py3-none-any.whl → 0.2.0.dev4__py3-none-any.whl

examples/quickstart/basic_vqa.py

@@ -0,0 +1,37 @@
+from opensportslib.apis import VQAModel
+
+
+def main():
+    """
+    Minimal VQA example.
+    Update config, question, and dataset paths before running.
+    """
+
+    my_model = VQAModel(
+        config="examples/configs/vqa_qwen.yaml",
+        weights=None,  # optional: path or Hugging Face model ID
+    )
+
+    predictions = my_model.infer(
+        test_set="/path/to/test_annotations.json",
+    )
+
+    print(predictions)
+
+    single_prediction = my_model.infer(
+        video_path="/path/to/video.mp4",
+        question="What card would you give? Why?",
+    )
+
+    print(single_prediction)
+
+    metrics = my_model.evaluate(
+        test_set="/path/to/test_annotations.json",
+        predictions=predictions,
+    )
+
+    print(metrics)
+
+
+if __name__ == "__main__":
+    main()

opensportslib/configs/vqa/default.yaml

@@ -0,0 +1,172 @@
+TASK: vqa
+VERSION: 2
+
+SYSTEM:
+  paths:
+    save_dir: ./checkpoints_vqa
+    work_dir: ${SYSTEM.paths.save_dir}
+  device: cuda
+  gpu:
+    count: 1
+    id: 0
+  reproducibility:
+    use_seed: true
+    seed: 42
+
+DATA:
+  common:
+    dataset_name: OSL-XFoul
+    data_root: /home/vorajv/dataset/OSL-XFoul
+    feature_index: /home/vorajv/dataset/OSL-XFoul/feature_index.json
+    prediction_index: /home/vorajv/dataset/OSL-XFoul/prediction_index.json
+    runtime:
+      loader_backend: opencv
+    splits:
+      train:
+        annotation_path: /home/vorajv/dataset/OSL-XFoul/train.json
+        source_path: /home/vorajv/dataset/OSL-XFoul
+        dataloader:
+          batch_size: 1
+          shuffle: true
+          num_workers: 0
+          pin_memory: false
+          mp_context: spawn
+          persistent_workers: false
+      valid:
+        annotation_path: /home/vorajv/dataset/OSL-XFoul/valid.json
+        source_path: /home/vorajv/dataset/OSL-XFoul
+        dataloader:
+          batch_size: 1
+          shuffle: false
+          num_workers: 0
+          pin_memory: false
+          mp_context: spawn
+          persistent_workers: false
+      test:
+        annotation_path: /home/vorajv/dataset/OSL-XFoul/test.json
+        source_path: /home/vorajv/dataset/OSL-XFoul
+        dataloader:
+          batch_size: 1
+          shuffle: false
+          num_workers: 0
+          pin_memory: false
+          mp_context: spawn
+          persistent_workers: false
+  inputs:
+    video:
+      modality: video
+      representation: raw
+      source:
+        format: mp4
+      sampling:
+        num_frames: 100
+        input_fps: 25
+        target_fps: 17
+        start_frame: 63
+        end_frame: 87
+      transform: {}
+      augmentations: {}
+      params: {}
+    question:
+      modality: text
+      representation: raw
+      source:
+        format: json
+      sampling: {}
+      transform: {}
+      augmentations: {}
+      params: {}
+
+MODEL:
+  runtime:
+    dtype: fp16
+    device: auto
+    compile: false
+    freeze: false
+  load:
+    checkpoint_path: null
+    pretrained: false
+    strict: true
+    map_location: null
+    format: auto
+  topology:
+    - from: video_encoder
+      to: mm_projector
+    - from: mm_projector
+      to: llm_decoder
+
+IO:
+  inputs:
+    video: video_encoder
+    question: llm_decoder
+  outputs:
+    answer_text: llm_decoder
+    explanation_text: llm_decoder
+
+TRAIN:
+  trainer:
+    type: vqa
+  epochs: 3
+  criterion:
+    type: CrossEntropyLoss
+  optimizer:
+    type: AdamW
+    lr: 0.0002
+    weight_decay: 0.001
+  scheduler:
+    type: constant
+  execution:
+    enabled: true
+    training_backend: xvars_videochatgpt_lora
+    feature_backend: xvars_clip
+    view_sampling_policy: random_train_deterministic_eval
+    acc_grad_iter: 8
+    log_interval: 1
+    dry_run: false
+    xvars:
+      feature_mode: strict_xvars
+      projection_path: null
+    prompt:
+      style: detailed
+      include_priors: true
+      prediction_prior_adapter: xvars_referee
+      prior_fields: [action, offence, contact, bodypart]
+      video_token_len: 300
+    generation:
+      max_new_tokens: 128
+      temperature: 0.0
+    eval_profile:
+      metric_set: [exact_match, contains_match, token_f1, referee_semantic]
+      aggregation: mean
+      exclusions: []
+    sft:
+      max_seq_length: 480
+      include_video_tokens: true
+      disable_tqdm: false
+      reference_mode: all
+      append_eos_token: true
+      gradient_checkpointing: true
+      save_strategy: epoch
+    lora:
+      r: 16
+      alpha: 32
+      dropout: 0.05
+      bias: none
+      prepare_kbit: true
+      target_modules: [mm_projector, upsample_features, up_proj, down_proj, gate_proj, k_proj, q_proj, v_proj, o_proj]
+      exclude_modules: '^base_lm\.model\.mm_projector$'
+    quantization:
+      enabled: false
+      load_in_4bit: true
+      bnb_4bit_quant_type: nf4
+      compute_dtype: float16
+      bnb_4bit_use_double_quant: true
+    checkpoint:
+      save_adapter: true
+      merge_and_save: false
+  selection:
+    monitor: loss
+    mode: min
+  checkpoint:
+    save_every: 1
+    save_best: true

opensportslib/configs/vqa/xvars.yaml

@@ -0,0 +1,58 @@
+SYSTEM:
+  paths:
+    save_dir: ./checkpoints_vqa_lora
+  gpu:
+    count: 4
+
+MODEL:
+  components:
+    video_encoder:
+      kind: encoder
+      source:
+        provider: opensportslib
+      # XVARS-trained classifier weights to load into that architecture
+      load:
+        weights_path: /home/vorajv/X-VARS/weights/14_model.pth.tar
+      params:
+        # CLIP architecture and image processor to instantiate.
+        feature_source: indexed_or_raw_clip
+        vision_tower: openai/clip-vit-large-patch14
+        feature_dim: 1024
+      overrides: {}
+    mm_projector:
+      kind: projector
+      source:
+        provider: opensportslib
+      params:
+        input_dim: 1024
+      overrides: {}
+    llm_decoder:
+      kind: decoder
+      source:
+        provider: opensportslib
+      params:
+        repo_id: /home/vorajv/X-VARS/weights/base_model_videoChatGPT
+      overrides: {}
+  metadata:
+    backend: xvars_videochatgpt
+
+TRAIN:
+  execution:
+    prompt:
+      system_prompt: You are Video-ChatGPT, a large vision-language assistant. You are able to understand the video content that the user provides, and assist the user with a variety of tasks using natural language.Follow the instructions carefully and explain your answers in detail based on the provided video.
+
+    # Optional XFoul-only generated-answer smoke test.
+    # During LoRA training, this runs generation on one known training sample and
+    # checks that the answer still contains referee-domain terms and avoids known
+    # code-like failure strings. Disable or replace these values for non-XFoul data.
+    generated_validation:
+      enabled: true
+      sample_id: action_0
+      every_steps: 25
+      max_new_tokens: 128
+      require_relevance: true
+      required_terms: [foul, card, challenge, spa, dogso, advantage]
+      forbidden_terms: [get_children, django, httpclient, "```python", "```php"]
+    
+    hf:
+      tokenizer_id: ${MODEL.components.llm_decoder.params.repo_id}

{opensportslib-0.2.0.dev3.dist-info → opensportslib-0.2.0.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.2.0.dev3
+Version: 0.2.0.dev4
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12
@@ -324,8 +324,8 @@ backend currently supports `Qwen/Qwen2.5-7B-Instruct` and
 For X-VARS, `feature_source: indexed_or_raw_clip` prefers indexed CLIP features
 when available and falls back to extracting CLIP features from raw video during
 `infer()`. Pre-extracted features remain the preferred path for parity, speed,
-and reproducibility. See [docs/xvars_integration_phases.md](docs/xvars_integration_phases.md)
-for the full X-VARS setup workflow.
+and reproducibility. See [docs/tools/vqa.md](docs/tools/vqa.md) for the full
+VQA setup workflow.
 
 
 ---

{opensportslib-0.2.0.dev3.dist-info → opensportslib-0.2.0.dev4.dist-info}/RECORD

@@ -1,5 +1,6 @@
 examples/quickstart/basic_classification.py,sha256=byefVwdVS6yHuHOSBRHB3czrm5ya4ZpIM_qI72UTpv4,1048
 examples/quickstart/basic_localization.py,sha256=ZTwVIBFcIuIcctW9OJNHctpsIKAgtVsRzn2HxmYYLaY,1049
+examples/quickstart/basic_vqa.py,sha256=q8qe5X3J3jHatUcykod4LFhvhEYwwXGFXzg0Qd_oO_o,773
 opensportslib/__init__.py,sha256=yl-0Zd38CgUGwn-ZUpT_rbaJx1nMwUPjod5U1TcAL7U,619
 opensportslib/cli.py,sha256=45tA92cctPQuruJACAm4TqEbdFwLp-AEknvsOgszxUM,829
 opensportslib/apis/__init__.py,sha256=4PCdg5owC2HWoTnPYiTl140ddz1rxipexoOmxHKL2yE,460
@@ -17,7 +18,9 @@ opensportslib/configs/localization/default.yaml,sha256=CCsR1ewdVn5aBTo6nBPkBnwCt
 opensportslib/configs/localization/netvladpp_resnetpca512.yaml,sha256=i-bB-LGekwdha761xuBjIs_6LJpTmQ2MH6Vusc1WSXI,3169
 opensportslib/configs/localization/video_dali.yaml,sha256=GhbqlNLJNY3U2Vwd1TOBtO--yRNpk0wbt8xFoxTibbg,2802
 opensportslib/configs/localization/video_ocv.yaml,sha256=sbUpACassaVK6ND4ChogiK9DfMvJisPM9SUpSh8olvg,3055
+opensportslib/configs/vqa/default.yaml,sha256=wH9eOJIwxZVRvXvgyT-0iTIN7IlN29a_GFTIkKB_9mw,4060
 opensportslib/configs/vqa/qwen.yaml,sha256=yQ7RSCa3JKN3POrgZWjfYdsG_BsQeo0oDekojUe1jqg,1318
+opensportslib/configs/vqa/xvars.yaml,sha256=JA299ps1yQ5upvnX9P-iC7dbvUb4rcBTjGnQQUVRY14,1957
 opensportslib/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 opensportslib/core/config/__init__.py,sha256=joSTWPBuLQDSFQIbAVpmOyeFLTxhIZRR_-K4DgEK8MM,635
 opensportslib/core/config/accessors.py,sha256=I6oi2lhqr9HChci90iHV3SnJAw6ykxeTLUVoecS-xhk,19613
@@ -111,8 +114,8 @@ opensportslib/tools/_common.py,sha256=CNSADAK8odNS0xUd2tNExoukrgKEbfNNwGbwIF5JJZ
 opensportslib/tools/hf_transfer.py,sha256=wkVJc0iyJojJSV6XBAPtPVQ58NbrGNoEZG5c3-bkMEE,31275
 opensportslib/tools/osl_json_to_parquet.py,sha256=VsEy7KF7x6PA5GPdiAFDgYlyfW9Cey-SvnOiRca2s74,15132
 opensportslib/tools/parquet_to_osl_json.py,sha256=CcD4oYF8HTwJnANr81PerJXLRqTc6BZ6mVsKu7YNJw0,9259
-opensportslib-0.2.0.dev3.dist-info/licenses/LICENSE,sha256=TfPDBt3ar0uv_f9cqCDMZ5rIzW3CY8anRRd4PkL6ejs,34522
-opensportslib-0.2.0.dev3.dist-info/licenses/LICENSE-COMMERCIAL,sha256=dg9GjCyCFMcfDJPkZ-IFDDt3Z5x_IdTY3YY-yOC1Eo0,163
+opensportslib-0.2.0.dev4.dist-info/licenses/LICENSE,sha256=TfPDBt3ar0uv_f9cqCDMZ5rIzW3CY8anRRd4PkL6ejs,34522
+opensportslib-0.2.0.dev4.dist-info/licenses/LICENSE-COMMERCIAL,sha256=dg9GjCyCFMcfDJPkZ-IFDDt3Z5x_IdTY3YY-yOC1Eo0,163
 tests/conftest.py,sha256=M10-WTFqwR8n4GvaHh2uDzeknxOWSwCoWslpHA4VQ3I,26806
 tests/test_classification_dataset_paths.py,sha256=UMKj52WcWIayPAYVm7ZAmsF2W9yiYecEJ_F7lzmejEs,3636
 tests/test_classification_trainer_dataloader.py,sha256=T-1ukT7UOjnoAG-9zD4oLHijTRy7qD_vAlMFmavU5oI,5118
@@ -126,9 +129,12 @@ tests/test_localization_dali_filenames.py,sha256=6bUH0UHFXNHsuo93GlYf8vZAjTwnXgw
 tests/test_package_smoke.py,sha256=-FDsRc4tdeGJwDMXBJT4Dlcsnsx6LvmeUXq9vnFm4YI,666
 tests/test_pretrained_config_merge_policy.py,sha256=GzCJbxxYfrqTrrUhoWuMVzF5Swb7WmqfruXE0M8UXVo,6677
 tests/test_public_apis_smoke.py,sha256=SJhAbMPQB-UzvTKPOlA-EoYB6pptJdcRnWRU-YhsOZg,1746
+tests/test_setup_cli.py,sha256=1LFyPZZTsf5xvoGyTME0cZ9-Aog_wnSjDFLIIODVVL4,3157
 tests/test_subset_train_infer_integration.py,sha256=ACw-PJRCQO2KCd4gjFBU4cgMb2Ka6MRABai_w-0HqJU,8238
 tests/test_task_model_api_contract.py,sha256=5L5q-eZ2QrL0HhW4EXBihWUevpJ3r3HYbvYuVqLR3A0,32025
+tests/test_vqa_api.py,sha256=7CM8_Is6mDAjldIWVKE8xIVGazclf5R41IJ1s5CG24c,7654
 tests/test_vqa_metrics_semantic.py,sha256=xAiVRuJi21olgqYcxP3p2ekD-EHqiKzkO-6dZAg2yvo,3973
+tests/test_vqa_qwen_xvars.py,sha256=KpifJb5Gr3my86CQ9VVmxV494Nmz2VVy5t7Tid0JFyU,9805
 tests/test_vqa_training_lora.py,sha256=AdA-WevoNkdSaikOTuYnqXYyLL-LcPGiZ95AWMyo2EA,20742
 tests/test_vqa_xvars_videochatgpt.py,sha256=1xCEJRrM0O-CYeQ0GX4rot4WSeEi0eD3w0HTR5LfoPs,50945
 tools/convert/build_soccernet_gar.py,sha256=XJ9esTiYjFkbhmETgucYa4I9Blb39moxyO74iIh3wBE,42540
@@ -142,8 +148,8 @@ tools/download/download_osl_hf.py,sha256=4dd-Yei2g_lp7NKom8FQlR99PQkQO3DZpAyKmdY
 tools/download/upload_osl_hf.py,sha256=YO-1uqxs-_6wdEdVYrJ7nqelpRDUOYsEmZW5HhpseiI,5777
 tools/training/classification.py,sha256=cSMIMNfP08McYUOt17ONo4WVKjAmLDl20WYOidYjJ8k,1277
 tools/training/localization.py,sha256=UC7liIEF_BrVtN8lnLZ5DOGEqSuwB_rNQgjoN_xEKuI,1271
-opensportslib-0.2.0.dev3.dist-info/METADATA,sha256=4xf0tx6TSpzdV9FmI2saESQd7DUmSLqe6ByoC7Fr7Rg,12875
-opensportslib-0.2.0.dev3.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
-opensportslib-0.2.0.dev3.dist-info/entry_points.txt,sha256=8haQLjVcah3IRKrlsyeecr0cUB5eXH9UPgJ_DDdT4Zk,57
-opensportslib-0.2.0.dev3.dist-info/top_level.txt,sha256=IJ6LnztNOeuEKQTiMTPPokBJr9ZxUCEIvbCjlPwUVEs,35
-opensportslib-0.2.0.dev3.dist-info/RECORD,,
+opensportslib-0.2.0.dev4.dist-info/METADATA,sha256=fjastTzSf7e0mf-iHBBzfgDYtvLG7zdjNuFLx39VwzE,12842
+opensportslib-0.2.0.dev4.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+opensportslib-0.2.0.dev4.dist-info/entry_points.txt,sha256=8haQLjVcah3IRKrlsyeecr0cUB5eXH9UPgJ_DDdT4Zk,57
+opensportslib-0.2.0.dev4.dist-info/top_level.txt,sha256=IJ6LnztNOeuEKQTiMTPPokBJr9ZxUCEIvbCjlPwUVEs,35
+opensportslib-0.2.0.dev4.dist-info/RECORD,,

tests/test_setup_cli.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from opensportslib import cli
+from opensportslib.setup import setup as setup_lib
+
+
+def test_cli_setup_forwards_xvars_flag(monkeypatch):
+    captured: dict[str, object] = {}
+
+    def fake_setup(*, dali: bool, pyg: bool, vqa_xvars: bool, vqa_qwen: bool):
+        captured["dali"] = dali
+        captured["pyg"] = pyg
+        captured["vqa_xvars"] = vqa_xvars
+        captured["vqa_qwen"] = vqa_qwen
+
+    monkeypatch.setattr(cli, "setup", fake_setup)
+
+    rc = cli.main(["setup", "--vqa_xvars", "--dali"])
+
+    assert rc == 0
+    assert captured == {"dali": True, "pyg": False, "vqa_xvars": True, "vqa_qwen": False}
+
+
+def test_cli_setup_forwards_qwen_flag(monkeypatch):
+    captured: dict[str, object] = {}
+
+    def fake_setup(*, dali: bool, pyg: bool, vqa_xvars: bool, vqa_qwen: bool):
+        captured["dali"] = dali
+        captured["pyg"] = pyg
+        captured["vqa_xvars"] = vqa_xvars
+        captured["vqa_qwen"] = vqa_qwen
+
+    monkeypatch.setattr(cli, "setup", fake_setup)
+
+    rc = cli.main(["setup", "--vqa_qwen", "--pyg"])
+
+    assert rc == 0
+    assert captured == {"dali": False, "pyg": True, "vqa_xvars": False, "vqa_qwen": True}
+
+
+def test_install_xvars_dependencies_uninstalls_then_reinstalls(monkeypatch):
+    calls: list[tuple[str, list[str]]] = []
+
+    def fake_call(cmd: list[str]):
+        calls.append(("call", cmd))
+        return 0
+
+    def fake_check_call(cmd: list[str]):
+        calls.append(("check_call", cmd))
+        return 0
+
+    monkeypatch.setattr(setup_lib.subprocess, "call", fake_call)
+    monkeypatch.setattr(setup_lib.subprocess, "check_call", fake_check_call)
+    monkeypatch.setattr(setup_lib.sys, "executable", "/usr/bin/python3")
+
+    setup_lib.install_xvars_dependencies(setup_lib.XVARS_DEPENDENCY_PINS)
+
+    assert calls == [
+        (
+            "call",
+            [
+                "/usr/bin/python3",
+                "-m",
+                "pip",
+                "uninstall",
+                "-y",
+                "transformers",
+                "peft",
+                "tokenizers",
+                "accelerate",
+                "trl",
+            ],
+        ),
+        (
+            "check_call",
+            [
+                "/usr/bin/python3",
+                "-m",
+                "pip",
+                "install",
+                "transformers==4.38.2",
+                "peft==0.9.0",
+                "tokenizers==0.15.2",
+                "accelerate==0.27.2",
+                "trl==0.10.1",
+            ],
+        ),
+    ]
+
+
+def test_setup_skips_vqa_dependency_install_when_flags_are_false(monkeypatch):
+    calls: list[str] = []
+
+    monkeypatch.setattr(setup_lib, "install_extras", lambda dali=False, pyg=False: calls.append(f"extras:{dali}:{pyg}"))
+    monkeypatch.setattr(setup_lib, "install_xvars_dependencies", lambda pins: calls.append(f"deps:{sorted(pins)}"))
+    monkeypatch.setattr(setup_lib, "verify", lambda: calls.append("verify"))
+    monkeypatch.setattr(setup_lib, "install_torch", lambda: calls.append("torch"))
+
+    setup_lib.setup(dali=True, pyg=False, vqa_xvars=False, vqa_qwen=False)
+
+    assert calls == ["torch", "extras:True:False", "verify"]

tests/test_vqa_api.py

@@ -0,0 +1,240 @@
+from pathlib import Path
+from types import SimpleNamespace
+import inspect
+import json
+import sys
+
+import pytest
+
+from opensportslib.apis import VQAModel
+from opensportslib.apis._xvars_runtime import (
+    XVarsHeadlessRuntime,
+    XVarsPrediction,
+    _dependency_error,
+    build_xvars_prompt,
+)
+
+
+class _FakeRuntime:
+    def __init__(self):
+        self.model = object()
+        self.image_processor = object()
+        self.calls = []
+
+    def infer_one(self, *, video_path, question, conv_mode, temperature, max_new_tokens):
+        self.calls.append(
+            {
+                "video_path": video_path,
+                "question": question,
+                "conv_mode": conv_mode,
+                "temperature": temperature,
+                "max_new_tokens": max_new_tokens,
+            }
+        )
+        return XVarsPrediction(
+            answer_text=f"answer::{question}",
+            action_class="Tackling",
+            offence_class="Offence",
+            severity_class="3.0",
+        )
+
+
+def test_vqa_method_signatures_expose_weights(vqa_config_path):
+    api = VQAModel(config=vqa_config_path)
+
+    for method_name in ("load_weights", "train", "infer", "evaluate"):
+        sig = inspect.signature(getattr(api, method_name))
+        assert "weights" in sig.parameters
+        assert any(
+            p.kind == inspect.Parameter.VAR_KEYWORD
+            for p in sig.parameters.values()
+        )
+
+    infer_sig = inspect.signature(api.infer)
+    assert "test_set" in infer_sig.parameters
+
+
+def test_vqa_constructor_weights_are_default_for_infer(vqa_config_path, monkeypatch):
+    calls = []
+    runtime = _FakeRuntime()
+
+    def fake_get_runtime(self, weights=None):
+        calls.append(weights)
+        self.last_loaded_weights = weights
+        self.best_checkpoint = weights
+        return runtime
+
+    monkeypatch.setattr(VQAModel, "_get_runtime", fake_get_runtime)
+    api = VQAModel(config=vqa_config_path, weights="default-adapter")
+
+    predictions = api.infer(
+        video_path="/tmp/sample.mp4",
+        question="Is it a foul or not? Why?",
+        use_wandb=False,
+    )
+
+    assert predictions["task"] == "vqa"
+    assert predictions["data"][0]["answer_text"] == "answer::Is it a foul or not? Why?"
+    assert calls[-1] == "default-adapter"
+
+    api.infer(
+        video_path="/tmp/sample.mp4",
+        question="What card would you give? Why?",
+        weights="override-adapter",
+        use_wandb=False,
+    )
+    assert calls[-1] == "override-adapter"
+
+
+def test_vqa_single_video_multi_question_infer(vqa_config_path, monkeypatch):
+    runtime = _FakeRuntime()
+    monkeypatch.setattr(VQAModel, "_get_runtime", lambda self, weights=None: runtime)
+
+    api = VQAModel(config=vqa_config_path)
+    predictions = api.infer(
+        video_path="/tmp/clip_0.mp4",
+        questions=[
+            "Is it a foul or not? Why?",
+            "What card would you give? Why?",
+        ],
+        use_wandb=False,
+    )
+
+    assert predictions["task"] == "vqa"
+    assert [row["id"] for row in predictions["data"]] == ["clip_0:0", "clip_0:1"]
+    assert [row["question"] for row in predictions["data"]] == [
+        "Is it a foul or not? Why?",
+        "What card would you give? Why?",
+    ]
+    assert predictions["data"][0]["video_path"] == "/tmp/clip_0.mp4"
+
+
+def test_vqa_dataset_infer_expands_all_questions(vqa_config_path, tmp_path, monkeypatch):
+    runtime = _FakeRuntime()
+    monkeypatch.setattr(VQAModel, "_get_runtime", lambda self, weights=None: runtime)
+
+    dataset_path = tmp_path / "test.json"
+    dataset_path.write_text(
+        json.dumps(
+            {
+                "data": [
+                    {
+                        "id": "action_0",
+                        "video_path": "/tmp/action_0.mp4",
+                        "questions": [
+                            "Is it a foul or not? Why?",
+                            "What card would you give? Why?",
+                        ],
+                    },
+                    {
+                        "id": "action_1",
+                        "inputs": [{"type": "video", "path": "relative/action_1.mp4"}],
+                        "question": "Could the referee have given advantage? Why?",
+                    },
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    api = VQAModel(config=vqa_config_path)
+    predictions = api.infer(test_set=str(dataset_path), use_wandb=False)
+
+    assert predictions["task"] == "vqa"
+    assert len(predictions["data"]) == 3
+    assert [row["id"] for row in predictions["data"]] == [
+        "action_0:0",
+        "action_0:1",
+        "action_1",
+    ]
+    assert predictions["data"][2]["video_path"] == str(
+        (dataset_path.parent / "relative/action_1.mp4").resolve()
+    )
+
+
+def test_build_xvars_prompt_injects_classifier_priors():
+    class FakeState:
+        def __init__(self):
+            self.roles = ("USER", "ASSISTANT")
+            self.messages = []
+            self.sep = " "
+            self.sep2 = "</s>"
+            self.sep_style = SimpleNamespace(name="TWO")
+            self.action_prompt = ""
+            self.offence_prompt = ""
+
+        def copy(self):
+            return FakeState()
+
+        def append_message(self, role, message):
+            self.messages.append([role, message])
+
+        def set_predictions(self, action_class, offence_class):
+            self.action_prompt = action_class
+            self.offence_prompt = offence_class
+
+        def get_prompt(self):
+            return (
+                f"{self.messages[0][0]}:{self.messages[0][1]}"
+                f"|priors:{self.action_prompt}{self.offence_prompt}"
+            )
+
+    prompt, stop_str = build_xvars_prompt(
+        conv_templates={"video-chatgpt_v1": FakeState()},
+        conv_mode="video-chatgpt_v1",
+        question="What card would you give? Why?",
+        replace_token="<vid_tokens>",
+        video_token="<video>",
+        action_prompt="a tackle",
+        offence_prompt=", foul and a yellow card",
+    )
+
+    assert "<vid_tokens>" in prompt
+    assert "a tackle, foul and a yellow card" in prompt
+    assert "<video>" not in prompt
+    assert stop_str == "</s>"
+
+
+def test_vqa_save_predictions_round_trip(vqa_config_path, tmp_path, monkeypatch):
+    runtime = _FakeRuntime()
+    monkeypatch.setattr(VQAModel, "_get_runtime", lambda self, weights=None: runtime)
+
+    api = VQAModel(config=vqa_config_path)
+    predictions = api.infer(
+        video_path="/tmp/clip_0.mp4",
+        question="Is it a foul or not? Why?",
+        use_wandb=False,
+    )
+
+    out_path = tmp_path / "predictions.json"
+    saved = api.save_predictions(str(out_path), predictions)
+
+    assert saved == str(out_path)
+    assert json.loads(out_path.read_text(encoding="utf-8"))["task"] == "vqa"
+
+
+def test_vqa_train_and_evaluate_raise_not_implemented(vqa_config_path):
+    api = VQAModel(config=vqa_config_path)
+
+    with pytest.raises(NotImplementedError):
+        api.train(use_wandb=False)
+
+    with pytest.raises(NotImplementedError):
+        api.evaluate(use_wandb=False)
+
+
+def test_vqa_import_does_not_pull_gradio_or_demo_modules():
+    assert "gradio" not in sys.modules
+    assert "video_chatgpt.demo.chat" not in sys.modules
+    assert "video_chatgpt.demo" not in sys.modules
+
+
+def test_vqa_runtime_dependency_error_names_missing_package(monkeypatch):
+    runtime = XVarsHeadlessRuntime()
+
+    def fail_bootstrap():
+        raise ModuleNotFoundError("No module named 'numpy'", name="numpy")
+
+    monkeypatch.setattr(runtime, "_bootstrap_components", fail_bootstrap)
+    error = _dependency_error(ModuleNotFoundError("No module named 'numpy'", name="numpy"))
+    assert "numpy" in str(error)

tests/test_vqa_qwen_xvars.py

@@ -0,0 +1,251 @@
+from types import SimpleNamespace
+
+import torch
+
+from opensportslib.models.base.qwen_xvars import QwenXVarsCausalLM, QwenXVarsModel
+from opensportslib.models.builder import build_model
+
+
+class TinyTokenizer:
+    pad_token_id = 0
+    eos_token_id = 1
+    pad_token = "<pad>"
+    eos_token = "</s>"
+
+    def __init__(self):
+        self.vocab = {"<pad>": 0, "</s>": 1, "<vid_patch>": 2, "<vid_start>": 3, "<vid_end>": 4}
+
+    def get_vocab(self):
+        return dict(self.vocab)
+
+    def add_special_tokens(self, payload):
+        added = 0
+        for token in payload.get("additional_special_tokens", []):
+            if token not in self.vocab:
+                self.vocab[token] = len(self.vocab)
+                added += 1
+        return added
+
+    def convert_tokens_to_ids(self, token):
+        return self.vocab[token]
+
+    def __len__(self):
+        return len(self.vocab)
+
+    def __call__(self, text, return_tensors=None, **kwargs):
+        del kwargs
+        if isinstance(text, list):
+            text = text[0]
+        tokens = []
+        i = 0
+        while i < len(text):
+            matched = False
+            for token in ("<vid_start>", "<vid_patch>", "<vid_end>", "</s>"):
+                if text.startswith(token, i):
+                    tokens.append(token)
+                    i += len(token)
+                    matched = True
+                    break
+            if matched:
+                continue
+            if text[i].isspace():
+                i += 1
+                continue
+            j = i
+            while j < len(text) and not text[j].isspace() and not any(
+                text.startswith(token, j) for token in ("<vid_start>", "<vid_patch>", "<vid_end>", "</s>")
+            ):
+                j += 1
+            tokens.append(text[i:j])
+            i = j
+        ids = []
+        for token in tokens:
+            if token not in self.vocab:
+                self.vocab[token] = len(self.vocab)
+            ids.append(self.vocab[token])
+        attn = [1] * len(ids)
+        if return_tensors == "pt":
+            return {"input_ids": torch.tensor([ids]), "attention_mask": torch.tensor([attn])}
+        return SimpleNamespace(input_ids=ids, attention_mask=attn)
+
+    def batch_decode(self, sequences, skip_special_tokens=True):
+        del sequences, skip_special_tokens
+        return ["qwen answer </s>"]
+
+
+class TinyGenerateLM(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.config = SimpleNamespace(hidden_size=4, vocab_size=64, use_cache=True)
+        self.generation_config = SimpleNamespace(use_cache=False)
+        self.emb = torch.nn.Embedding(128, 4)
+        self.lm = torch.nn.Linear(4, 64)
+        self.seen_inputs_embeds = None
+        self.generate_kwargs = None
+
+    def get_input_embeddings(self):
+        return self.emb
+
+    def resize_token_embeddings(self, size):
+        return self.emb
+
+    def forward(self, input_ids=None, inputs_embeds=None, attention_mask=None, labels=None, **kwargs):
+        del input_ids, attention_mask, kwargs
+        self.seen_inputs_embeds = inputs_embeds.detach().clone()
+        logits = self.lm(inputs_embeds)
+        loss = logits.sum() * 0
+        if labels is not None:
+            loss = loss + 0.123
+        return SimpleNamespace(loss=loss, logits=logits)
+
+    def generate(self, input_ids=None, inputs_embeds=None, attention_mask=None, **kwargs):
+        del attention_mask
+        self.generate_kwargs = {"input_ids": input_ids, "inputs_embeds": inputs_embeds, **kwargs}
+        extra = torch.tensor([[9, 1]], device=input_ids.device if input_ids is not None else inputs_embeds.device)
+        if input_ids is None:
+            prefix = torch.zeros((1, inputs_embeds.shape[1]), dtype=torch.long, device=extra.device)
+            return torch.cat((prefix, extra), dim=1)
+        return torch.cat((input_ids, extra), dim=1)
+
+
+def _cfg(tmp_path):
+    return SimpleNamespace(
+        TASK="VQA",
+        SYSTEM=SimpleNamespace(device="cpu", gpu=SimpleNamespace(count=0, id=0)),
+        MODEL=SimpleNamespace(
+            runtime=SimpleNamespace(dtype="fp32"),
+            load=SimpleNamespace(checkpoint_path=None),
+            components=SimpleNamespace(
+                video_encoder=SimpleNamespace(
+                    kind="encoder",
+                    source=SimpleNamespace(provider="opensportslib", name="xvars_clip_features"),
+                    load=SimpleNamespace(weights_path=str(tmp_path / "vision.pt")),
+                    params=SimpleNamespace(feature_source="indexed_or_raw_clip", vision_tower="clip", feature_dim=1024),
+                    overrides=SimpleNamespace(),
+                ),
+                mm_projector=SimpleNamespace(
+                    kind="projector",
+                    source=SimpleNamespace(provider="opensportslib"),
+                    params=SimpleNamespace(input_dim=1024),
+                    overrides=SimpleNamespace(),
+                ),
+                llm_decoder=SimpleNamespace(
+                    kind="decoder",
+                    source=SimpleNamespace(provider="huggingface", name="Qwen/Qwen3.5-9B-Base"),
+                    params=SimpleNamespace(repo_id="Qwen/Qwen3.5-9B-Base"),
+                    overrides=SimpleNamespace(),
+                ),
+            ),
+            metadata=SimpleNamespace(backend="qwen_xvars_infer"),
+        ),
+        DATA=SimpleNamespace(
+            inputs=SimpleNamespace(video=SimpleNamespace(sampling=SimpleNamespace(num_frames=100, start_frame=63, end_frame=87, input_fps=25, target_fps=17))),
+        ),
+        TRAIN=SimpleNamespace(
+            execution={
+                "prompt": {"include_priors": True, "video_token_len": 3},
+                "generation": {"max_new_tokens": 8, "temperature": 0.0},
+                "xvars": {"feature_mode": "strict_xvars"},
+                "hf": {"local_files_only": True, "prefer_cuda": False, "tokenizer_id": "Qwen/Qwen3.5-9B-Base"},
+                "quantization": {"enabled": False},
+            }
+        ),
+    )
+
+
+def test_qwen_xvars_wrapper_projects_features_into_hidden_size():
+    tokenizer = TinyTokenizer()
+    base = TinyGenerateLM()
+    model = QwenXVarsCausalLM(base, mm_hidden_size=1024)
+    input_ids = torch.tensor([[tokenizer.convert_tokens_to_ids("<vid_start>"), tokenizer.convert_tokens_to_ids("<vid_patch>"), tokenizer.convert_tokens_to_ids("<vid_patch>"), tokenizer.convert_tokens_to_ids("<vid_end>")]])
+    features = torch.ones((1, 2, 1024))
+
+    out = model(
+        input_ids=input_ids,
+        attention_mask=torch.ones_like(input_ids),
+        labels=input_ids.clone(),
+        video_spatio_temporal_features=features,
+        tokenizer=tokenizer,
+    )
+
+    assert abs(out.loss.item() - 0.123) < 1e-6
+    assert model.mm_projector.out_features == base.config.hidden_size
+    assert tuple(base.seen_inputs_embeds.shape) == (1, 4, base.config.hidden_size)
+
+
+def test_qwen_xvars_model_generate_answer_uses_existing_feature_contract(tmp_path, monkeypatch):
+    cfg = _cfg(tmp_path)
+    tokenizer = TinyTokenizer()
+    base_model = TinyGenerateLM()
+
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    monkeypatch.setattr(AutoTokenizer, "from_pretrained", lambda *args, **kwargs: tokenizer)
+    monkeypatch.setattr(AutoModelForCausalLM, "from_pretrained", lambda *args, **kwargs: base_model)
+
+    model = QwenXVarsModel(cfg, model_id="Qwen/Qwen3.5-9B-Base", projector_params={"input_dim": 1024})
+    answer = model.generate_answer(
+        {
+            "id": "action_0",
+            "question": "Was this a foul?",
+            "references": [],
+            "labels": {},
+            "metadata": {},
+            "prior_prediction_text": "",
+            "video_spatio_temporal_features": torch.ones((3, 1024)),
+        },
+        prompt_cfg={"video_token_len": 3},
+        generation_cfg={"max_new_tokens": 8, "temperature": 0.0},
+    )
+
+    assert model._ready is True
+    assert answer == "qwen answer"
+    assert base_model.generate_kwargs is not None
+    assert base_model.generate_kwargs["inputs_embeds"].shape[-1] == base_model.config.hidden_size
+    assert base_model.generate_kwargs["inputs_embeds"].shape[-2] == base_model.generate_kwargs["input_ids"].shape[-1]
+
+
+def test_qwen_loader_does_not_pass_use_cache_to_from_pretrained(tmp_path, monkeypatch):
+    cfg = _cfg(tmp_path)
+    tokenizer = TinyTokenizer()
+    base_model = TinyGenerateLM()
+    captured = {}
+
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    def _load_tokenizer(*args, **kwargs):
+        del args, kwargs
+        return tokenizer
+
+    def _load_model(model_id, **kwargs):
+        captured["model_id"] = model_id
+        captured["model_kwargs"] = dict(kwargs)
+        if "use_cache" in kwargs:
+            raise TypeError("unexpected load kwarg")
+        return base_model
+
+    monkeypatch.setattr(AutoTokenizer, "from_pretrained", _load_tokenizer)
+    monkeypatch.setattr(AutoModelForCausalLM, "from_pretrained", _load_model)
+
+    model = QwenXVarsModel(cfg, model_id="Qwen/Qwen3.5-9B-Base", projector_params={"input_dim": 1024})
+
+    assert model._ready is True
+    assert captured["model_id"] == "Qwen/Qwen3.5-9B-Base"
+    assert "use_cache" not in captured["model_kwargs"]
+    assert base_model.config.use_cache is True
+    assert base_model.generation_config.use_cache is True
+
+
+def test_builder_routes_qwen_backend(tmp_path, monkeypatch):
+    cfg = _cfg(tmp_path)
+    tokenizer = TinyTokenizer()
+    base_model = TinyGenerateLM()
+
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    monkeypatch.setattr(AutoTokenizer, "from_pretrained", lambda *args, **kwargs: tokenizer)
+    monkeypatch.setattr(AutoModelForCausalLM, "from_pretrained", lambda *args, **kwargs: base_model)
+
+    model, _ = build_model(cfg, device="cpu")
+
+    assert isinstance(model, QwenXVarsModel)