opensportslib 0.3.0.dev18__tar.gz → 0.3.0.dev20__tar.gz

{opensportslib-0.3.0.dev18/opensportslib.egg-info → opensportslib-0.3.0.dev20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.3.0.dev18
+Version: 0.3.0.dev20
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12
@@ -385,6 +385,8 @@ python tools/download/upload_osl_hf.py --repo-id <org/repo> --json-path <local_d
 ```
 
 Downloads are placed under `<output-dir>/<revision>/<split>`.
+For Parquet/WebDataset downloads, an existing `<split>.json` in that directory
+is reused without downloading or converting the split again.
 
 ---
 

{opensportslib-0.3.0.dev18 → opensportslib-0.3.0.dev20}/README.md

@@ -349,6 +349,8 @@ python tools/download/upload_osl_hf.py --repo-id <org/repo> --json-path <local_d
 ```
 
 Downloads are placed under `<output-dir>/<revision>/<split>`.
+For Parquet/WebDataset downloads, an existing `<split>.json` in that directory
+is reused without downloading or converting the split again.
 
 ---
 

{opensportslib-0.3.0.dev18 → opensportslib-0.3.0.dev20}/opensportslib/setup/setup.py

@@ -13,11 +13,6 @@ LEGACY_GPU_MAX_COMPUTE_CAPABILITY = (7, 4)
 LEGACY_GPU_CUDA_WHEEL = "cu126"
 LEGACY_GPU_CUDA_WHEEL_MAX_COMPUTE_CAPABILITY = (9, 0)
 CUDA13_REQUIRED_MIN_COMPUTE_CAPABILITY = (10, 0)
-LEGACY_GPU_TORCH_PACKAGES = (
-    "torch==2.10.0",
-    "torchvision==0.25.0",
-    "torchaudio==2.10.0",
-)
 
 XVARS_DEPENDENCY_PINS = {
     "transformers": "4.38.2",
@@ -140,10 +135,6 @@ def select_cuda_wheel(cuda_version, compute_capabilities):
 
 
 def select_torch_packages(compute_capabilities):
-    # if compute_capabilities and any(
-    #     capability <= LEGACY_GPU_MAX_COMPUTE_CAPABILITY for capability in compute_capabilities
-    # ):
-    #     return LEGACY_GPU_TORCH_PACKAGES
     return ("torch", "torchvision", "torchaudio")
 
 

{opensportslib-0.3.0.dev18 → opensportslib-0.3.0.dev20}/opensportslib/tools/hf_transfer.py

@@ -211,7 +211,6 @@ def _download_parquet_split_and_convert(
     progress_cb: ProgressCallback | None = None,
     is_cancelled: CancelCheck | None = None,
 ) -> dict[str, Any]:
-    _, _, snapshot_download = _import_hf_hub()
     cleaned_repo_id = str(repo_id or "").strip()
     cleaned_revision = str(revision or "").strip() or "main"
     cleaned_split = _clean_hf_split(split)
@@ -219,9 +218,36 @@ def _download_parquet_split_and_convert(
         raise ValueError("repo_id is required.")
 
     os.makedirs(output_dir, exist_ok=True)
+    output_json_path = Path(output_dir) / f"{cleaned_split}.json"
+    if output_json_path.is_file():
+        _emit_progress(
+            progress_cb,
+            f"JSON already exists at {output_json_path}; skipping Parquet/WebDataset download and conversion.",
+        )
+        return {
+            "repo_id": cleaned_repo_id,
+            "revision": cleaned_revision,
+            "split": cleaned_split,
+            "folder_path": cleaned_split,
+            "output_dir": output_dir,
+            "json_path": str(output_json_path),
+            "source": "parquet_split",
+            "download_kind": "parquet",
+            "downloaded_file_count": 0,
+            "download_skipped": True,
+            "extracted_media": True,
+            "extracted_media_count": 0,
+            "hf_source_metadata": {
+                "repo_id": cleaned_repo_id,
+                "branch": cleaned_revision,
+                "split": cleaned_split,
+            },
+        }
+
     _ensure_not_cancelled(is_cancelled)
     _emit_progress(progress_cb, f"Downloading Parquet split '{cleaned_split}' from {cleaned_repo_id}@{cleaned_revision}...")
 
+    _, _, snapshot_download = _import_hf_hub()
     tmp_dir = tempfile.mkdtemp(prefix="hf_parquet_dl_", dir=output_dir)
     try:
         snapshot_download(
@@ -235,8 +261,6 @@ def _download_parquet_split_and_convert(
         _ensure_not_cancelled(is_cancelled)
 
         parquet_dataset_dir = Path(tmp_dir) / cleaned_split
-        output_json_path = Path(output_dir) / f"{cleaned_split}.json"
-
         _emit_progress(progress_cb, f"Converting Parquet split to JSON and extracting media into {output_dir}...")
         conversion_result = convert_parquet_to_json(
             dataset_dir=parquet_dataset_dir,
@@ -269,6 +293,7 @@ def _download_parquet_split_and_convert(
         "json_path": str(output_json_path),
         "source": "parquet_split",
         "download_kind": "parquet",
+        "download_skipped": False,
         "num_samples": int(conversion_result.get("num_samples") or 0),
         "extracted_media": True,
         "extracted_media_count": int(conversion_result.get("extracted_media_files") or 0),
@@ -410,6 +435,131 @@ def _download_json_path_from_hf(
     return result
 
 
+_PREFERRED_SPLIT_ORDER = ["train", "valid", "val", "validation", "test", "challenge"]
+_NON_SPLIT_JSON_FILES = {"dataset_infos.json", "dataset_dict.json"}
+
+
+def _sort_splits(splits: set[str]) -> list[str]:
+    def _sort_key(name: str) -> tuple[int, str]:
+        try:
+            rank = _PREFERRED_SPLIT_ORDER.index(name.lower())
+        except ValueError:
+            rank = len(_PREFERRED_SPLIT_ORDER)
+        return (rank, name.lower())
+
+    return sorted(splits, key=_sort_key)
+
+
+def list_dataset_branches_on_hf(
+    repo_id: str,
+    *,
+    token: str | None = None,
+) -> list[str]:
+    cleaned_repo_id = str(repo_id or "").strip()
+    if not cleaned_repo_id:
+        raise ValueError("repo_id is required.")
+
+    HfApi, _, _ = _import_hf_hub()
+    api = HfApi(token=token or None)
+    refs = api.list_repo_refs(cleaned_repo_id, repo_type="dataset")
+    branch_names = [str(branch.name) for branch in getattr(refs, "branches", [])]
+
+    unique_names = sorted(set(branch_names))
+    if "main" in unique_names:
+        unique_names.remove("main")
+        return ["main"] + unique_names
+    return unique_names
+
+
+def list_dataset_splits_on_hf(
+    repo_id: str,
+    revision: str,
+    *,
+    token: str | None = None,
+) -> dict[str, Any]:
+    cleaned_repo_id = str(repo_id or "").strip()
+    cleaned_revision = str(revision or "").strip() or "main"
+    if not cleaned_repo_id:
+        raise ValueError("repo_id is required.")
+
+    HfApi, _, _ = _import_hf_hub()
+    api = HfApi(token=token or None)
+    repo_files = api.list_repo_files(
+        cleaned_repo_id,
+        revision=cleaned_revision,
+        repo_type="dataset",
+    )
+
+    parquet_splits: set[str] = set()
+    json_splits: set[str] = set()
+    for path in repo_files:
+        normalized = _normalize_repo_path(path)
+        if "/" in normalized:
+            folder, filename = normalized.split("/", 1)
+            # Only the canonical Parquet+WebDataset export layout counts as a
+            # parquet split (produced by convert_json_to_parquet / expected by
+            # convert_parquet_to_json): `{split}/metadata.parquet` plus TAR
+            # shards under `{split}/shards/`. A JSON-format dataset can also
+            # reference arbitrary `.parquet` media files (e.g. tensor-encoded
+            # videos) under a folder that happens to share the split's name,
+            # so a loose "any .parquet/.tar anywhere under this folder" check
+            # would misclassify those as Parquet+WebDataset splits.
+            if folder and (
+                filename == "metadata.parquet"
+                or (filename.startswith("shards/") and filename.lower().endswith(".tar"))
+            ):
+                parquet_splits.add(folder)
+        elif normalized.lower().endswith(".json") and normalized not in _NON_SPLIT_JSON_FILES:
+            json_splits.add(normalized[: -len(".json")])
+
+    if parquet_splits:
+        return {"format": "parquet", "splits": _sort_splits(parquet_splits)}
+    if json_splits:
+        return {"format": "json", "splits": _sort_splits(json_splits)}
+    return {"format": None, "splits": []}
+
+
+def download_dataset_splits_from_hf(
+    repo_id: str,
+    revision: str,
+    splits: list[str],
+    output_dir: str,
+    *,
+    download_format: str = "parquet",
+    dry_run: bool = False,
+    token: str | None = None,
+    progress_cb: ProgressCallback | None = None,
+    is_cancelled: CancelCheck | None = None,
+) -> list[dict[str, Any]]:
+    cleaned_splits = [str(split or "").strip() for split in (splits or [])]
+    cleaned_splits = [split for split in cleaned_splits if split]
+    if not cleaned_splits:
+        raise ValueError("At least one split is required.")
+
+    total = len(cleaned_splits)
+    results: list[dict[str, Any]] = []
+    for idx, split in enumerate(cleaned_splits, start=1):
+        _ensure_not_cancelled(is_cancelled)
+
+        def _scoped_progress(message: str, _idx: int = idx, _split: str = split) -> None:
+            _emit_progress(progress_cb, f"[{_idx}/{total}] {_split}: {message}")
+
+        result = download_dataset_split_from_hf(
+            repo_id,
+            revision,
+            split,
+            output_dir,
+            download_format=download_format,
+            dry_run=dry_run,
+            token=token,
+            progress_cb=_scoped_progress,
+            is_cancelled=is_cancelled,
+        )
+        results.append(result)
+
+    return results
+
+
 def download_dataset_split_from_hf(
     repo_id: str,
     revision: str,

{opensportslib-0.3.0.dev18 → opensportslib-0.3.0.dev20/opensportslib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.3.0.dev18
+Version: 0.3.0.dev20
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12
@@ -385,6 +385,8 @@ python tools/download/upload_osl_hf.py --repo-id <org/repo> --json-path <local_d
 ```
 
 Downloads are placed under `<output-dir>/<revision>/<split>`.
+For Parquet/WebDataset downloads, an existing `<split>.json` in that directory
+is reused without downloading or converting the split again.
 
 ---
 

{opensportslib-0.3.0.dev18 → opensportslib-0.3.0.dev20}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "opensportslib"
-version = "0.3.0.dev18"
+version = "0.3.0.dev20"
 description = "OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data."
 readme = "README.md"
 requires-python = ">=3.12"

{opensportslib-0.3.0.dev18 → opensportslib-0.3.0.dev20}/tests/release/test_classification_release.py

@@ -104,6 +104,19 @@ os.environ.setdefault("OSL_PRETRAINED_WEIGHTS", "0")
 # Dataset prep
 # --------------------------------------------------------------------------
 
+# opensportslib.datasets.classification_dataset.ClassificationDataset
+# hardcodes exclude_labels = ["Unknown", "Dont know"] and drops them from
+# the label space it actually builds (label_map, class weights, etc). If
+# DATA.common.classes / num_classes don't apply the same exclusion, the
+# model is built with the wrong output size and weighted-loss class weights
+# come back the wrong shape ("weight tensor should be defined for all N
+# classes but got shape [N-k]"). Keep this in sync with that hardcoded list.
+_DATASET_EXCLUDED_LABELS = {"Unknown", "Dont know"}
+
+
+def _exclude_dataset_labels(classes: list[str]) -> list[str]:
+    return [c for c in classes if c not in _DATASET_EXCLUDED_LABELS]
+
 
 def _resolve_label(record: dict) -> str | None:
     for key in ("label", "action_class", "foul_type", "class", "action"):
@@ -225,8 +238,21 @@ def classification_dataset():
             split: download_shard_split(repo_id, split, root, revision=revision)
             for split in ("train", "valid", "test")
         }
-        classes = classes_from_osl_json(json.loads(split_paths["test"].read_text(encoding="utf-8")), head="action")
-        return {"data_root": root, "classes": classes, "split_paths": split_paths}
+        classes = _exclude_dataset_labels(
+            classes_from_osl_json(json.loads(split_paths["test"].read_text(encoding="utf-8")), head="action")
+        )
+        # Each split's media is extracted under its own split directory (see
+        # download_dataset_split_from_hf's split_output_dir), and the json's
+        # "inputs[].path" values (e.g. "train/action_0/clip_0.mp4") are
+        # relative to that directory -- NOT to a single shared data_root, so
+        # source_path must be resolved per split.
+        source_paths = {split: path.parent for split, path in split_paths.items()}
+        return {
+            "data_root": root,
+            "classes": classes,
+            "split_paths": split_paths,
+            "source_paths": source_paths,
+        }
 
     # Fallback: known-good, real (gated) dataset -- not sharded, but small
     # (13 clips), so downloaded and split locally rather than capped.
@@ -246,15 +272,20 @@ def classification_dataset():
         path.write_text(json.dumps(split_payload, indent=2), encoding="utf-8")
         split_paths[split] = path
 
-    classes = payload["labels"]["action"]["labels"]
+    classes = _exclude_dataset_labels(payload["labels"]["action"]["labels"])
     counts = Counter(item["labels"]["action"]["label"] for item in payload["data"])
     print(f"Classes ({len(classes)}): {classes}")
     print(f"Label distribution: {dict(counts)}")
 
+    # All splits share one media root here (processed_videos/<id>.mp4
+    # relative to `root`), unlike the sharded branch above.
+    source_paths = {split: root for split in split_paths}
+
     return {
         "data_root": root,
         "classes": classes,
         "split_paths": split_paths,
+        "source_paths": source_paths,
     }
 
 
@@ -312,7 +343,7 @@ def test_classification_mvnetwork_backbone(classification_dataset, backbone):
             "splits": {
                 split: {
                     "annotation_path": str(path),
-                    "source_path": str(dataset["data_root"]),
+                    "source_path": str(dataset["source_paths"][split]),
                 }
                 for split, path in dataset["split_paths"].items()
             },
@@ -356,7 +387,7 @@ def test_classification_video_mae_huggingface_backend(classification_dataset):
             "splits": {
                 split: {
                     "annotation_path": str(path),
-                    "source_path": str(dataset["data_root"]),
+                    "source_path": str(dataset["source_paths"][split]),
                 }
                 for split, path in dataset["split_paths"].items()
             },

{opensportslib-0.3.0.dev18 → opensportslib-0.3.0.dev20}/tests/test_hf_transfer_tools.py

@@ -12,11 +12,14 @@ from opensportslib.tools.hf_transfer import (
     create_dataset_repo_on_hf,
     dataset_repo_exists_on_hf,
     download_dataset_split_from_hf,
+    download_dataset_splits_from_hf,
     extract_local_input_upload_entries_from_json,
     extract_repo_paths_from_json,
     is_hf_download_url_not_found_error,
     is_hf_repo_not_found_error,
     is_hf_revision_not_found_error,
+    list_dataset_branches_on_hf,
+    list_dataset_splits_on_hf,
     read_hf_source_metadata_from_dataset,
     upload_dataset_as_parquet_to_hf,
     upload_dataset_inputs_from_json_to_hf,
@@ -226,6 +229,164 @@ def test_dataset_repo_exists_on_hf_returns_false_for_repo_not_found(monkeypatch)
     assert dataset_repo_exists_on_hf("OpenSportsLab/missing-repo", token="hf_token") is False
 
 
+def test_list_dataset_branches_on_hf_puts_main_first_then_alphabetical(monkeypatch):
+    class _FakeRefs:
+        branches = [
+            type("_Ref", (), {"name": "zeta"})(),
+            type("_Ref", (), {"name": "main"})(),
+            type("_Ref", (), {"name": "alpha"})(),
+        ]
+
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def list_repo_refs(self, repo_id, repo_type=None):
+            return _FakeRefs()
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+
+    assert list_dataset_branches_on_hf("OpenSportsLab/repo") == ["main", "alpha", "zeta"]
+
+
+def test_list_dataset_splits_on_hf_detects_parquet_layout(monkeypatch):
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def list_repo_files(self, repo_id, revision=None, repo_type=None):
+            return [
+                "README.md",
+                "train/metadata.parquet",
+                "train/shard_manifest.parquet",
+                "train/shards/shard-000000.tar",
+                "test/metadata.parquet",
+                "test/shards/shard-000000.tar",
+            ]
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+
+    result = list_dataset_splits_on_hf("OpenSportsLab/repo", "main")
+
+    assert result == {"format": "parquet", "splits": ["train", "test"]}
+
+
+def test_list_dataset_splits_on_hf_treats_json_dataset_with_parquet_media_as_json(monkeypatch):
+    """
+    A JSON-format dataset can reference arbitrary media files (e.g. tensor-encoded
+    videos serialized as .parquet) inside a folder that happens to share a split's
+    name. That must not be misdetected as the canonical Parquet+WebDataset export
+    layout, which requires `{split}/metadata.parquet` + `{split}/shards/*.tar`.
+    Regression test for OpenSportsLab/SNGAR-Action-Spotting-Tracking.
+    """
+
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def list_repo_files(self, repo_id, revision=None, repo_type=None):
+            return [
+                "README.md",
+                "train.json",
+                "valid.json",
+                "test.json",
+                "train/videos/10502.parquet",
+                "train/videos/10503.parquet",
+                "valid/videos/3841.parquet",
+                "test/videos/3850.parquet",
+            ]
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+
+    result = list_dataset_splits_on_hf("OpenSportsLab/repo", "main")
+
+    assert result == {"format": "json", "splits": ["train", "valid", "test"]}
+
+
+def test_list_dataset_splits_on_hf_detects_json_layout(monkeypatch):
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def list_repo_files(self, repo_id, revision=None, repo_type=None):
+            return [
+                "dataset_infos.json",
+                "train.json",
+                "challenge.json",
+                "valid.json",
+            ]
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+
+    result = list_dataset_splits_on_hf("OpenSportsLab/repo", "main")
+
+    assert result == {"format": "json", "splits": ["train", "valid", "challenge"]}
+
+
+def test_list_dataset_splits_on_hf_returns_none_format_when_nothing_matches(monkeypatch):
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def list_repo_files(self, repo_id, revision=None, repo_type=None):
+            return ["README.md", "dataset_infos.json"]
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+
+    assert list_dataset_splits_on_hf("OpenSportsLab/repo", "main") == {"format": None, "splits": []}
+
+
+def test_download_dataset_splits_from_hf_downloads_each_split_with_prefixed_progress(monkeypatch, tmp_path):
+    progress_messages = []
+    calls = []
+
+    def _fake_download_dataset_split_from_hf(repo_id, revision, split, output_dir, **kwargs):
+        calls.append((repo_id, revision, split, output_dir))
+        kwargs["progress_cb"](f"working on {split}")
+        return {"split": split, "json_path": str(tmp_path / f"{split}.json")}
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer.download_dataset_split_from_hf",
+        _fake_download_dataset_split_from_hf,
+    )
+
+    results = download_dataset_splits_from_hf(
+        "OpenSportsLab/repo",
+        "main",
+        ["train", "valid"],
+        str(tmp_path),
+        download_format="json",
+        progress_cb=progress_messages.append,
+    )
+
+    assert [call[2] for call in calls] == ["train", "valid"]
+    assert [result["split"] for result in results] == ["train", "valid"]
+    assert progress_messages == [
+        "[1/2] train: working on train",
+        "[2/2] valid: working on valid",
+    ]
+
+
+def test_download_dataset_splits_from_hf_requires_at_least_one_split():
+    with pytest.raises(ValueError):
+        download_dataset_splits_from_hf("OpenSportsLab/repo", "main", [], "/tmp/out")
+
+
 def test_upload_dataset_inputs_from_json_to_hf_uploads_inputs_and_json(monkeypatch, tmp_path):
     clip_path = tmp_path / "train" / "clip_0.mp4"
     clip_path.parent.mkdir(parents=True)
@@ -607,6 +768,41 @@ def test_download_dataset_split_from_hf_parquet_downloads_split_folder(monkeypat
     assert result["output_dir"] == str(tmp_path / "dev" / "test")
     assert result["json_path"] == str(tmp_path / "dev" / "test" / "test.json")
     assert result["num_samples"] == 3
+    assert result["download_skipped"] is False
+
+
+def test_download_dataset_split_from_hf_parquet_skips_download_when_json_exists(
+    monkeypatch, tmp_path
+):
+    output_json_path = tmp_path / "dev" / "test" / "test.json"
+    output_json_path.parent.mkdir(parents=True)
+    output_json_path.write_text(json.dumps({"data": []}), encoding="utf-8")
+    progress_messages = []
+
+    def _fail_hf_import():
+        raise AssertionError("Hugging Face must not be imported for an existing JSON")
+
+    def _fail_conversion(**kwargs):
+        raise AssertionError("Existing JSON must not be converted again")
+
+    monkeypatch.setattr("opensportslib.tools.hf_transfer._import_hf_hub", _fail_hf_import)
+    monkeypatch.setattr("opensportslib.tools.hf_transfer.convert_parquet_to_json", _fail_conversion)
+
+    result = download_dataset_split_from_hf(
+        "OpenSportsLab/repo",
+        "dev",
+        "test",
+        str(tmp_path),
+        download_format="parquet",
+        progress_cb=progress_messages.append,
+    )
+
+    assert result["json_path"] == str(output_json_path)
+    assert result["downloaded_file_count"] == 0
+    assert result["download_skipped"] is True
+    assert progress_messages == [
+        f"JSON already exists at {output_json_path}; skipping Parquet/WebDataset download and conversion."
+    ]
 
 
 def test_download_dataset_split_from_hf_json_writes_hf_metadata_on_non_dry_run(monkeypatch, tmp_path):