opensportslib 0.3.0.dev17__tar.gz → 0.3.0.dev19__tar.gz

{opensportslib-0.3.0.dev17/opensportslib.egg-info → opensportslib-0.3.0.dev19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.3.0.dev17
+Version: 0.3.0.dev19
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12
@@ -385,6 +385,8 @@ python tools/download/upload_osl_hf.py --repo-id <org/repo> --json-path <local_d
 ```
 
 Downloads are placed under `<output-dir>/<revision>/<split>`.
+For Parquet/WebDataset downloads, an existing `<split>.json` in that directory
+is reused without downloading or converting the split again.
 
 ---
 

{opensportslib-0.3.0.dev17 → opensportslib-0.3.0.dev19}/README.md

@@ -349,6 +349,8 @@ python tools/download/upload_osl_hf.py --repo-id <org/repo> --json-path <local_d
 ```
 
 Downloads are placed under `<output-dir>/<revision>/<split>`.
+For Parquet/WebDataset downloads, an existing `<split>.json` in that directory
+is reused without downloading or converting the split again.
 
 ---
 

{opensportslib-0.3.0.dev17 → opensportslib-0.3.0.dev19}/opensportslib/setup/setup.py

@@ -140,10 +140,10 @@ def select_cuda_wheel(cuda_version, compute_capabilities):
 
 
 def select_torch_packages(compute_capabilities):
-    if compute_capabilities and any(
-        capability <= LEGACY_GPU_MAX_COMPUTE_CAPABILITY for capability in compute_capabilities
-    ):
-        return LEGACY_GPU_TORCH_PACKAGES
+    # if compute_capabilities and any(
+    #     capability <= LEGACY_GPU_MAX_COMPUTE_CAPABILITY for capability in compute_capabilities
+    # ):
+    #     return LEGACY_GPU_TORCH_PACKAGES
     return ("torch", "torchvision", "torchaudio")
 
 

{opensportslib-0.3.0.dev17 → opensportslib-0.3.0.dev19}/opensportslib/tools/hf_transfer.py

@@ -211,7 +211,6 @@ def _download_parquet_split_and_convert(
     progress_cb: ProgressCallback | None = None,
     is_cancelled: CancelCheck | None = None,
 ) -> dict[str, Any]:
-    _, _, snapshot_download = _import_hf_hub()
     cleaned_repo_id = str(repo_id or "").strip()
     cleaned_revision = str(revision or "").strip() or "main"
     cleaned_split = _clean_hf_split(split)
@@ -219,9 +218,36 @@ def _download_parquet_split_and_convert(
         raise ValueError("repo_id is required.")
 
     os.makedirs(output_dir, exist_ok=True)
+    output_json_path = Path(output_dir) / f"{cleaned_split}.json"
+    if output_json_path.is_file():
+        _emit_progress(
+            progress_cb,
+            f"JSON already exists at {output_json_path}; skipping Parquet/WebDataset download and conversion.",
+        )
+        return {
+            "repo_id": cleaned_repo_id,
+            "revision": cleaned_revision,
+            "split": cleaned_split,
+            "folder_path": cleaned_split,
+            "output_dir": output_dir,
+            "json_path": str(output_json_path),
+            "source": "parquet_split",
+            "download_kind": "parquet",
+            "downloaded_file_count": 0,
+            "download_skipped": True,
+            "extracted_media": True,
+            "extracted_media_count": 0,
+            "hf_source_metadata": {
+                "repo_id": cleaned_repo_id,
+                "branch": cleaned_revision,
+                "split": cleaned_split,
+            },
+        }
+
     _ensure_not_cancelled(is_cancelled)
     _emit_progress(progress_cb, f"Downloading Parquet split '{cleaned_split}' from {cleaned_repo_id}@{cleaned_revision}...")
 
+    _, _, snapshot_download = _import_hf_hub()
     tmp_dir = tempfile.mkdtemp(prefix="hf_parquet_dl_", dir=output_dir)
     try:
         snapshot_download(
@@ -235,8 +261,6 @@ def _download_parquet_split_and_convert(
         _ensure_not_cancelled(is_cancelled)
 
         parquet_dataset_dir = Path(tmp_dir) / cleaned_split
-        output_json_path = Path(output_dir) / f"{cleaned_split}.json"
-
         _emit_progress(progress_cb, f"Converting Parquet split to JSON and extracting media into {output_dir}...")
         conversion_result = convert_parquet_to_json(
             dataset_dir=parquet_dataset_dir,
@@ -269,6 +293,7 @@ def _download_parquet_split_and_convert(
         "json_path": str(output_json_path),
         "source": "parquet_split",
         "download_kind": "parquet",
+        "download_skipped": False,
         "num_samples": int(conversion_result.get("num_samples") or 0),
         "extracted_media": True,
         "extracted_media_count": int(conversion_result.get("extracted_media_files") or 0),

{opensportslib-0.3.0.dev17 → opensportslib-0.3.0.dev19/opensportslib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.3.0.dev17
+Version: 0.3.0.dev19
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12
@@ -385,6 +385,8 @@ python tools/download/upload_osl_hf.py --repo-id <org/repo> --json-path <local_d
 ```
 
 Downloads are placed under `<output-dir>/<revision>/<split>`.
+For Parquet/WebDataset downloads, an existing `<split>.json` in that directory
+is reused without downloading or converting the split again.
 
 ---
 

{opensportslib-0.3.0.dev17 → opensportslib-0.3.0.dev19}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "opensportslib"
-version = "0.3.0.dev17"
+version = "0.3.0.dev19"
 description = "OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data."
 readme = "README.md"
 requires-python = ">=3.12"

{opensportslib-0.3.0.dev17 → opensportslib-0.3.0.dev19}/tests/release/test_classification_release.py

@@ -104,6 +104,19 @@ os.environ.setdefault("OSL_PRETRAINED_WEIGHTS", "0")
 # Dataset prep
 # --------------------------------------------------------------------------
 
+# opensportslib.datasets.classification_dataset.ClassificationDataset
+# hardcodes exclude_labels = ["Unknown", "Dont know"] and drops them from
+# the label space it actually builds (label_map, class weights, etc). If
+# DATA.common.classes / num_classes don't apply the same exclusion, the
+# model is built with the wrong output size and weighted-loss class weights
+# come back the wrong shape ("weight tensor should be defined for all N
+# classes but got shape [N-k]"). Keep this in sync with that hardcoded list.
+_DATASET_EXCLUDED_LABELS = {"Unknown", "Dont know"}
+
+
+def _exclude_dataset_labels(classes: list[str]) -> list[str]:
+    return [c for c in classes if c not in _DATASET_EXCLUDED_LABELS]
+
 
 def _resolve_label(record: dict) -> str | None:
     for key in ("label", "action_class", "foul_type", "class", "action"):
@@ -225,8 +238,21 @@ def classification_dataset():
             split: download_shard_split(repo_id, split, root, revision=revision)
             for split in ("train", "valid", "test")
         }
-        classes = classes_from_osl_json(json.loads(split_paths["test"].read_text(encoding="utf-8")), head="action")
-        return {"data_root": root, "classes": classes, "split_paths": split_paths}
+        classes = _exclude_dataset_labels(
+            classes_from_osl_json(json.loads(split_paths["test"].read_text(encoding="utf-8")), head="action")
+        )
+        # Each split's media is extracted under its own split directory (see
+        # download_dataset_split_from_hf's split_output_dir), and the json's
+        # "inputs[].path" values (e.g. "train/action_0/clip_0.mp4") are
+        # relative to that directory -- NOT to a single shared data_root, so
+        # source_path must be resolved per split.
+        source_paths = {split: path.parent for split, path in split_paths.items()}
+        return {
+            "data_root": root,
+            "classes": classes,
+            "split_paths": split_paths,
+            "source_paths": source_paths,
+        }
 
     # Fallback: known-good, real (gated) dataset -- not sharded, but small
     # (13 clips), so downloaded and split locally rather than capped.
@@ -246,15 +272,20 @@ def classification_dataset():
         path.write_text(json.dumps(split_payload, indent=2), encoding="utf-8")
         split_paths[split] = path
 
-    classes = payload["labels"]["action"]["labels"]
+    classes = _exclude_dataset_labels(payload["labels"]["action"]["labels"])
     counts = Counter(item["labels"]["action"]["label"] for item in payload["data"])
     print(f"Classes ({len(classes)}): {classes}")
     print(f"Label distribution: {dict(counts)}")
 
+    # All splits share one media root here (processed_videos/<id>.mp4
+    # relative to `root`), unlike the sharded branch above.
+    source_paths = {split: root for split in split_paths}
+
     return {
         "data_root": root,
         "classes": classes,
         "split_paths": split_paths,
+        "source_paths": source_paths,
     }
 
 
@@ -312,7 +343,7 @@ def test_classification_mvnetwork_backbone(classification_dataset, backbone):
             "splits": {
                 split: {
                     "annotation_path": str(path),
-                    "source_path": str(dataset["data_root"]),
+                    "source_path": str(dataset["source_paths"][split]),
                 }
                 for split, path in dataset["split_paths"].items()
             },
@@ -356,7 +387,7 @@ def test_classification_video_mae_huggingface_backend(classification_dataset):
             "splits": {
                 split: {
                     "annotation_path": str(path),
-                    "source_path": str(dataset["data_root"]),
+                    "source_path": str(dataset["source_paths"][split]),
                 }
                 for split, path in dataset["split_paths"].items()
             },

{opensportslib-0.3.0.dev17 → opensportslib-0.3.0.dev19}/tests/test_hf_transfer_tools.py

@@ -607,6 +607,41 @@ def test_download_dataset_split_from_hf_parquet_downloads_split_folder(monkeypat
     assert result["output_dir"] == str(tmp_path / "dev" / "test")
     assert result["json_path"] == str(tmp_path / "dev" / "test" / "test.json")
     assert result["num_samples"] == 3
+    assert result["download_skipped"] is False
+
+
+def test_download_dataset_split_from_hf_parquet_skips_download_when_json_exists(
+    monkeypatch, tmp_path
+):
+    output_json_path = tmp_path / "dev" / "test" / "test.json"
+    output_json_path.parent.mkdir(parents=True)
+    output_json_path.write_text(json.dumps({"data": []}), encoding="utf-8")
+    progress_messages = []
+
+    def _fail_hf_import():
+        raise AssertionError("Hugging Face must not be imported for an existing JSON")
+
+    def _fail_conversion(**kwargs):
+        raise AssertionError("Existing JSON must not be converted again")
+
+    monkeypatch.setattr("opensportslib.tools.hf_transfer._import_hf_hub", _fail_hf_import)
+    monkeypatch.setattr("opensportslib.tools.hf_transfer.convert_parquet_to_json", _fail_conversion)
+
+    result = download_dataset_split_from_hf(
+        "OpenSportsLab/repo",
+        "dev",
+        "test",
+        str(tmp_path),
+        download_format="parquet",
+        progress_cb=progress_messages.append,
+    )
+
+    assert result["json_path"] == str(output_json_path)
+    assert result["downloaded_file_count"] == 0
+    assert result["download_skipped"] is True
+    assert progress_messages == [
+        f"JSON already exists at {output_json_path}; skipping Parquet/WebDataset download and conversion."
+    ]
 
 
 def test_download_dataset_split_from_hf_json_writes_hf_metadata_on_non_dry_run(monkeypatch, tmp_path):