opensportslib 0.3.0.dev19__tar.gz → 0.3.0.dev20__tar.gz

{opensportslib-0.3.0.dev19/opensportslib.egg-info → opensportslib-0.3.0.dev20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.3.0.dev19
+Version: 0.3.0.dev20
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12

{opensportslib-0.3.0.dev19 → opensportslib-0.3.0.dev20}/opensportslib/setup/setup.py

@@ -13,11 +13,6 @@ LEGACY_GPU_MAX_COMPUTE_CAPABILITY = (7, 4)
 LEGACY_GPU_CUDA_WHEEL = "cu126"
 LEGACY_GPU_CUDA_WHEEL_MAX_COMPUTE_CAPABILITY = (9, 0)
 CUDA13_REQUIRED_MIN_COMPUTE_CAPABILITY = (10, 0)
-LEGACY_GPU_TORCH_PACKAGES = (
-    "torch==2.10.0",
-    "torchvision==0.25.0",
-    "torchaudio==2.10.0",
-)
 
 XVARS_DEPENDENCY_PINS = {
     "transformers": "4.38.2",
@@ -140,10 +135,6 @@ def select_cuda_wheel(cuda_version, compute_capabilities):
 
 
 def select_torch_packages(compute_capabilities):
-    # if compute_capabilities and any(
-    #     capability <= LEGACY_GPU_MAX_COMPUTE_CAPABILITY for capability in compute_capabilities
-    # ):
-    #     return LEGACY_GPU_TORCH_PACKAGES
     return ("torch", "torchvision", "torchaudio")
 
 

{opensportslib-0.3.0.dev19 → opensportslib-0.3.0.dev20}/opensportslib/tools/hf_transfer.py

@@ -435,6 +435,131 @@ def _download_json_path_from_hf(
     return result
 
 
+_PREFERRED_SPLIT_ORDER = ["train", "valid", "val", "validation", "test", "challenge"]
+_NON_SPLIT_JSON_FILES = {"dataset_infos.json", "dataset_dict.json"}
+
+
+def _sort_splits(splits: set[str]) -> list[str]:
+    def _sort_key(name: str) -> tuple[int, str]:
+        try:
+            rank = _PREFERRED_SPLIT_ORDER.index(name.lower())
+        except ValueError:
+            rank = len(_PREFERRED_SPLIT_ORDER)
+        return (rank, name.lower())
+
+    return sorted(splits, key=_sort_key)
+
+
+def list_dataset_branches_on_hf(
+    repo_id: str,
+    *,
+    token: str | None = None,
+) -> list[str]:
+    cleaned_repo_id = str(repo_id or "").strip()
+    if not cleaned_repo_id:
+        raise ValueError("repo_id is required.")
+
+    HfApi, _, _ = _import_hf_hub()
+    api = HfApi(token=token or None)
+    refs = api.list_repo_refs(cleaned_repo_id, repo_type="dataset")
+    branch_names = [str(branch.name) for branch in getattr(refs, "branches", [])]
+
+    unique_names = sorted(set(branch_names))
+    if "main" in unique_names:
+        unique_names.remove("main")
+        return ["main"] + unique_names
+    return unique_names
+
+
+def list_dataset_splits_on_hf(
+    repo_id: str,
+    revision: str,
+    *,
+    token: str | None = None,
+) -> dict[str, Any]:
+    cleaned_repo_id = str(repo_id or "").strip()
+    cleaned_revision = str(revision or "").strip() or "main"
+    if not cleaned_repo_id:
+        raise ValueError("repo_id is required.")
+
+    HfApi, _, _ = _import_hf_hub()
+    api = HfApi(token=token or None)
+    repo_files = api.list_repo_files(
+        cleaned_repo_id,
+        revision=cleaned_revision,
+        repo_type="dataset",
+    )
+
+    parquet_splits: set[str] = set()
+    json_splits: set[str] = set()
+    for path in repo_files:
+        normalized = _normalize_repo_path(path)
+        if "/" in normalized:
+            folder, filename = normalized.split("/", 1)
+            # Only the canonical Parquet+WebDataset export layout counts as a
+            # parquet split (produced by convert_json_to_parquet / expected by
+            # convert_parquet_to_json): `{split}/metadata.parquet` plus TAR
+            # shards under `{split}/shards/`. A JSON-format dataset can also
+            # reference arbitrary `.parquet` media files (e.g. tensor-encoded
+            # videos) under a folder that happens to share the split's name,
+            # so a loose "any .parquet/.tar anywhere under this folder" check
+            # would misclassify those as Parquet+WebDataset splits.
+            if folder and (
+                filename == "metadata.parquet"
+                or (filename.startswith("shards/") and filename.lower().endswith(".tar"))
+            ):
+                parquet_splits.add(folder)
+        elif normalized.lower().endswith(".json") and normalized not in _NON_SPLIT_JSON_FILES:
+            json_splits.add(normalized[: -len(".json")])
+
+    if parquet_splits:
+        return {"format": "parquet", "splits": _sort_splits(parquet_splits)}
+    if json_splits:
+        return {"format": "json", "splits": _sort_splits(json_splits)}
+    return {"format": None, "splits": []}
+
+
+def download_dataset_splits_from_hf(
+    repo_id: str,
+    revision: str,
+    splits: list[str],
+    output_dir: str,
+    *,
+    download_format: str = "parquet",
+    dry_run: bool = False,
+    token: str | None = None,
+    progress_cb: ProgressCallback | None = None,
+    is_cancelled: CancelCheck | None = None,
+) -> list[dict[str, Any]]:
+    cleaned_splits = [str(split or "").strip() for split in (splits or [])]
+    cleaned_splits = [split for split in cleaned_splits if split]
+    if not cleaned_splits:
+        raise ValueError("At least one split is required.")
+
+    total = len(cleaned_splits)
+    results: list[dict[str, Any]] = []
+    for idx, split in enumerate(cleaned_splits, start=1):
+        _ensure_not_cancelled(is_cancelled)
+
+        def _scoped_progress(message: str, _idx: int = idx, _split: str = split) -> None:
+            _emit_progress(progress_cb, f"[{_idx}/{total}] {_split}: {message}")
+
+        result = download_dataset_split_from_hf(
+            repo_id,
+            revision,
+            split,
+            output_dir,
+            download_format=download_format,
+            dry_run=dry_run,
+            token=token,
+            progress_cb=_scoped_progress,
+            is_cancelled=is_cancelled,
+        )
+        results.append(result)
+
+    return results
+
+
 def download_dataset_split_from_hf(
     repo_id: str,
     revision: str,

{opensportslib-0.3.0.dev19 → opensportslib-0.3.0.dev20/opensportslib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.3.0.dev19
+Version: 0.3.0.dev20
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12

{opensportslib-0.3.0.dev19 → opensportslib-0.3.0.dev20}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "opensportslib"
-version = "0.3.0.dev19"
+version = "0.3.0.dev20"
 description = "OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data."
 readme = "README.md"
 requires-python = ">=3.12"

{opensportslib-0.3.0.dev19 → opensportslib-0.3.0.dev20}/tests/test_hf_transfer_tools.py

@@ -12,11 +12,14 @@ from opensportslib.tools.hf_transfer import (
     create_dataset_repo_on_hf,
     dataset_repo_exists_on_hf,
     download_dataset_split_from_hf,
+    download_dataset_splits_from_hf,
     extract_local_input_upload_entries_from_json,
     extract_repo_paths_from_json,
     is_hf_download_url_not_found_error,
     is_hf_repo_not_found_error,
     is_hf_revision_not_found_error,
+    list_dataset_branches_on_hf,
+    list_dataset_splits_on_hf,
     read_hf_source_metadata_from_dataset,
     upload_dataset_as_parquet_to_hf,
     upload_dataset_inputs_from_json_to_hf,
@@ -226,6 +229,164 @@ def test_dataset_repo_exists_on_hf_returns_false_for_repo_not_found(monkeypatch)
     assert dataset_repo_exists_on_hf("OpenSportsLab/missing-repo", token="hf_token") is False
 
 
+def test_list_dataset_branches_on_hf_puts_main_first_then_alphabetical(monkeypatch):
+    class _FakeRefs:
+        branches = [
+            type("_Ref", (), {"name": "zeta"})(),
+            type("_Ref", (), {"name": "main"})(),
+            type("_Ref", (), {"name": "alpha"})(),
+        ]
+
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def list_repo_refs(self, repo_id, repo_type=None):
+            return _FakeRefs()
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+
+    assert list_dataset_branches_on_hf("OpenSportsLab/repo") == ["main", "alpha", "zeta"]
+
+
+def test_list_dataset_splits_on_hf_detects_parquet_layout(monkeypatch):
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def list_repo_files(self, repo_id, revision=None, repo_type=None):
+            return [
+                "README.md",
+                "train/metadata.parquet",
+                "train/shard_manifest.parquet",
+                "train/shards/shard-000000.tar",
+                "test/metadata.parquet",
+                "test/shards/shard-000000.tar",
+            ]
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+
+    result = list_dataset_splits_on_hf("OpenSportsLab/repo", "main")
+
+    assert result == {"format": "parquet", "splits": ["train", "test"]}
+
+
+def test_list_dataset_splits_on_hf_treats_json_dataset_with_parquet_media_as_json(monkeypatch):
+    """
+    A JSON-format dataset can reference arbitrary media files (e.g. tensor-encoded
+    videos serialized as .parquet) inside a folder that happens to share a split's
+    name. That must not be misdetected as the canonical Parquet+WebDataset export
+    layout, which requires `{split}/metadata.parquet` + `{split}/shards/*.tar`.
+    Regression test for OpenSportsLab/SNGAR-Action-Spotting-Tracking.
+    """
+
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def list_repo_files(self, repo_id, revision=None, repo_type=None):
+            return [
+                "README.md",
+                "train.json",
+                "valid.json",
+                "test.json",
+                "train/videos/10502.parquet",
+                "train/videos/10503.parquet",
+                "valid/videos/3841.parquet",
+                "test/videos/3850.parquet",
+            ]
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+
+    result = list_dataset_splits_on_hf("OpenSportsLab/repo", "main")
+
+    assert result == {"format": "json", "splits": ["train", "valid", "test"]}
+
+
+def test_list_dataset_splits_on_hf_detects_json_layout(monkeypatch):
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def list_repo_files(self, repo_id, revision=None, repo_type=None):
+            return [
+                "dataset_infos.json",
+                "train.json",
+                "challenge.json",
+                "valid.json",
+            ]
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+
+    result = list_dataset_splits_on_hf("OpenSportsLab/repo", "main")
+
+    assert result == {"format": "json", "splits": ["train", "valid", "challenge"]}
+
+
+def test_list_dataset_splits_on_hf_returns_none_format_when_nothing_matches(monkeypatch):
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def list_repo_files(self, repo_id, revision=None, repo_type=None):
+            return ["README.md", "dataset_infos.json"]
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+
+    assert list_dataset_splits_on_hf("OpenSportsLab/repo", "main") == {"format": None, "splits": []}
+
+
+def test_download_dataset_splits_from_hf_downloads_each_split_with_prefixed_progress(monkeypatch, tmp_path):
+    progress_messages = []
+    calls = []
+
+    def _fake_download_dataset_split_from_hf(repo_id, revision, split, output_dir, **kwargs):
+        calls.append((repo_id, revision, split, output_dir))
+        kwargs["progress_cb"](f"working on {split}")
+        return {"split": split, "json_path": str(tmp_path / f"{split}.json")}
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer.download_dataset_split_from_hf",
+        _fake_download_dataset_split_from_hf,
+    )
+
+    results = download_dataset_splits_from_hf(
+        "OpenSportsLab/repo",
+        "main",
+        ["train", "valid"],
+        str(tmp_path),
+        download_format="json",
+        progress_cb=progress_messages.append,
+    )
+
+    assert [call[2] for call in calls] == ["train", "valid"]
+    assert [result["split"] for result in results] == ["train", "valid"]
+    assert progress_messages == [
+        "[1/2] train: working on train",
+        "[2/2] valid: working on valid",
+    ]
+
+
+def test_download_dataset_splits_from_hf_requires_at_least_one_split():
+    with pytest.raises(ValueError):
+        download_dataset_splits_from_hf("OpenSportsLab/repo", "main", [], "/tmp/out")
+
+
 def test_upload_dataset_inputs_from_json_to_hf_uploads_inputs_and_json(monkeypatch, tmp_path):
     clip_path = tmp_path / "train" / "clip_0.mp4"
     clip_path.parent.mkdir(parents=True)