opensportslib 0.3.1.dev2__tar.gz → 0.3.1.dev3__tar.gz

{opensportslib-0.3.1.dev2/opensportslib.egg-info → opensportslib-0.3.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.3.1.dev2
+Version: 0.3.1.dev3
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12
@@ -392,6 +392,19 @@ The JSON records the resolved Hugging Face commit and can later be passed to
 Parquet/WebDataset download always completes the local split even when a
 metadata-only `<split>.json` already exists.
 
+Download APIs accept `byte_progress_cb(filename, downloaded_bytes,
+total_bytes)`. When the repository file is Xet-backed, OpenSportsLib keeps the
+accelerated Xet transfer and adapts Xet's byte updates to this callback. It
+falls back to classic HTTP progress when Xet is unavailable, disabled, or not
+used by the file.
+When byte progress is enabled, Parquet downloads also emit `[current/total]`
+file messages through `progress_cb` so clients can present file-count progress.
+High-level split downloads also accept `file_plan_cb(filenames)`,
+`file_completed_cb(filename, local_path)`, and
+`json_ready_cb(split, json_path)`. These are transfer lifecycle notifications;
+callers remain responsible for queue policy and presentation. For non-dry-run
+JSON datasets, pinned source metadata is persisted before `json_ready_cb` runs.
+
 JSON uploads support partially downloaded datasets: the JSON and all
 referenced files available locally are committed, while missing referenced
 files are skipped and reported. Remote files not included in that commit are

{opensportslib-0.3.1.dev2 → opensportslib-0.3.1.dev3}/README.md

@@ -356,6 +356,19 @@ The JSON records the resolved Hugging Face commit and can later be passed to
 Parquet/WebDataset download always completes the local split even when a
 metadata-only `<split>.json` already exists.
 
+Download APIs accept `byte_progress_cb(filename, downloaded_bytes,
+total_bytes)`. When the repository file is Xet-backed, OpenSportsLib keeps the
+accelerated Xet transfer and adapts Xet's byte updates to this callback. It
+falls back to classic HTTP progress when Xet is unavailable, disabled, or not
+used by the file.
+When byte progress is enabled, Parquet downloads also emit `[current/total]`
+file messages through `progress_cb` so clients can present file-count progress.
+High-level split downloads also accept `file_plan_cb(filenames)`,
+`file_completed_cb(filename, local_path)`, and
+`json_ready_cb(split, json_path)`. These are transfer lifecycle notifications;
+callers remain responsible for queue policy and presentation. For non-dry-run
+JSON datasets, pinned source metadata is persisted before `json_ready_cb` runs.
+
 JSON uploads support partially downloaded datasets: the JSON and all
 referenced files available locally are committed, while missing referenced
 files are skipped and reported. Remote files not included in that commit are

{opensportslib-0.3.1.dev2 → opensportslib-0.3.1.dev3}/opensportslib/tools/hf_transfer.py

@@ -9,6 +9,9 @@ from typing import Any, Callable
 
 ProgressCallback = Callable[[str], None]
 ByteProgressCallback = Callable[[str, int, int], None]
+FilePlanCallback = Callable[[list[str]], None]
+FileCompletedCallback = Callable[[str, str], None]
+JsonReadyCallback = Callable[[str, str], None]
 CancelCheck = Callable[[], bool]
 
 HF_REPO_ID_KEY = "hf_repo_id"
@@ -101,6 +104,29 @@ def _emit_progress(progress_cb: ProgressCallback | None, message: str) -> None:
         progress_cb(message)
 
 
+def _emit_file_plan(
+    file_plan_cb: FilePlanCallback | None, filenames: list[str]
+) -> None:
+    if file_plan_cb and filenames:
+        file_plan_cb(list(filenames))
+
+
+def _emit_file_completed(
+    file_completed_cb: FileCompletedCallback | None,
+    filename: str,
+    local_path: str,
+) -> None:
+    if file_completed_cb:
+        file_completed_cb(filename, local_path)
+
+
+def _emit_json_ready(
+    json_ready_cb: JsonReadyCallback | None, split: str, json_path: str
+) -> None:
+    if json_ready_cb:
+        json_ready_cb(split, json_path)
+
+
 def _ensure_not_cancelled(is_cancelled: CancelCheck | None) -> None:
     if is_cancelled and is_cancelled():
         raise HfTransferCancelled("Transfer cancelled by user.")
@@ -128,6 +154,23 @@ def _import_hf_file_system():
     return HfFileSystem, TqdmCallback
 
 
+def _import_hf_xet_download():
+    try:
+        from huggingface_hub import get_hf_file_metadata, hf_hub_url
+        from huggingface_hub.file_download import xet_get
+        from huggingface_hub.utils import build_hf_headers
+        from huggingface_hub.utils._runtime import is_xet_available
+    except ImportError:
+        return None
+    return (
+        get_hf_file_metadata,
+        hf_hub_url,
+        xet_get,
+        build_hf_headers,
+        is_xet_available,
+    )
+
+
 def _download_hf_file(
     hf_hub_download,
     *,
@@ -169,7 +212,6 @@ def _download_hf_file(
             token=token or None,
         )
 
-    HfFileSystem, TqdmCallback = _import_hf_file_system()
     destination.parent.mkdir(parents=True, exist_ok=True)
     file_descriptor, temporary_path = tempfile.mkstemp(
         prefix=f".{destination.name}.",
@@ -198,9 +240,43 @@ def _download_hf_file(
         def close(self):
             pass
 
-    callback = TqdmCallback(tqdm_cls=_CallbackProgress)
     remote_path = f"datasets/{repo_id}/{normalized_filename}"
     try:
+        xet_download = _import_hf_xet_download()
+        if xet_download is not None:
+            (
+                get_hf_file_metadata,
+                hf_hub_url,
+                xet_get,
+                build_hf_headers,
+                is_xet_available,
+            ) = xet_download
+            if is_xet_available():
+                metadata = get_hf_file_metadata(
+                    hf_hub_url(
+                        repo_id=repo_id,
+                        repo_type="dataset",
+                        filename=normalized_filename,
+                        revision=revision,
+                    ),
+                    token=token or None,
+                )
+                if metadata.xet_file_data is not None:
+                    progress = _CallbackProgress(total=metadata.size)
+                    xet_get(
+                        incomplete_path=Path(temporary_path),
+                        xet_file_data=metadata.xet_file_data,
+                        headers=build_hf_headers(token=token or None),
+                        expected_size=metadata.size,
+                        displayed_filename=normalized_filename,
+                        _tqdm_bar=progress,
+                    )
+                    _ensure_not_cancelled(is_cancelled)
+                    os.replace(temporary_path, destination)
+                    return str(destination)
+
+        HfFileSystem, TqdmCallback = _import_hf_file_system()
+        callback = TqdmCallback(tqdm_cls=_CallbackProgress)
         with callback:
             HfFileSystem(token=token or None).get_file(
                 remote_path,
@@ -391,6 +467,9 @@ def _download_parquet_split_and_convert(
     token: str | None = None,
     progress_cb: ProgressCallback | None = None,
     byte_progress_cb: ByteProgressCallback | None = None,
+    file_plan_cb: FilePlanCallback | None = None,
+    file_completed_cb: FileCompletedCallback | None = None,
+    json_ready_cb: JsonReadyCallback | None = None,
     is_cancelled: CancelCheck | None = None,
 ) -> dict[str, Any]:
     cleaned_repo_id = str(repo_id or "").strip()
@@ -413,16 +492,21 @@ def _download_parquet_split_and_convert(
     tmp_dir = tempfile.mkdtemp(prefix="hf_parquet_dl_", dir=output_dir)
     try:
         if annotations_only:
+            metadata_filename = f"{cleaned_split}/metadata.parquet"
+            _emit_file_plan(file_plan_cb, [metadata_filename])
             metadata_path = _download_hf_file(
                 hf_hub_download,
                 repo_id=cleaned_repo_id,
-                filename=f"{cleaned_split}/metadata.parquet",
+                filename=metadata_filename,
                 revision=commit,
                 local_dir=tmp_dir,
                 token=token or None,
                 byte_progress_cb=byte_progress_cb,
                 is_cancelled=is_cancelled,
             )
+            _emit_file_completed(
+                file_completed_cb, metadata_filename, metadata_path
+            )
         else:
             if byte_progress_cb is None:
                 snapshot_download(
@@ -447,9 +531,14 @@ def _download_parquet_split_and_convert(
                     raise FileNotFoundError(
                         f"No files found for Parquet split: {cleaned_split}"
                     )
-                for remote_path in split_files:
+                _emit_file_plan(file_plan_cb, split_files)
+                for index, remote_path in enumerate(split_files, start=1):
                     _ensure_not_cancelled(is_cancelled)
-                    _download_hf_file(
+                    _emit_progress(
+                        progress_cb,
+                        f"[{index}/{len(split_files)}] Downloading {remote_path}",
+                    )
+                    downloaded_path = _download_hf_file(
                         hf_hub_download,
                         repo_id=cleaned_repo_id,
                         filename=remote_path,
@@ -459,6 +548,9 @@ def _download_parquet_split_and_convert(
                         byte_progress_cb=byte_progress_cb,
                         is_cancelled=is_cancelled,
                     )
+                    _emit_file_completed(
+                        file_completed_cb, remote_path, downloaded_path
+                    )
         _ensure_not_cancelled(is_cancelled)
 
         if annotations_only:
@@ -484,6 +576,9 @@ def _download_parquet_split_and_convert(
             source_format="parquet",
             commit=commit,
         )
+        _emit_json_ready(
+            json_ready_cb, cleaned_split, str(output_json_path)
+        )
     finally:
         shutil.rmtree(tmp_dir, ignore_errors=True)
 
@@ -532,6 +627,9 @@ def _download_json_path_from_hf(
     token: str | None = None,
     progress_cb: ProgressCallback | None = None,
     byte_progress_cb: ByteProgressCallback | None = None,
+    file_plan_cb: FilePlanCallback | None = None,
+    file_completed_cb: FileCompletedCallback | None = None,
+    json_ready_cb: JsonReadyCallback | None = None,
     is_cancelled: CancelCheck | None = None,
 ) -> dict[str, Any]:
     HfApi, hf_hub_download, _ = _import_hf_hub()
@@ -543,6 +641,7 @@ def _download_json_path_from_hf(
     os.makedirs(output_dir, exist_ok=True)
     _ensure_not_cancelled(is_cancelled)
     _emit_progress(progress_cb, f"Downloading JSON from {repo_id}@{commit}: {path_in_repo}")
+    _emit_file_plan(file_plan_cb, [path_in_repo])
 
     json_path = _download_hf_file(
         hf_hub_download,
@@ -554,6 +653,7 @@ def _download_json_path_from_hf(
         byte_progress_cb=byte_progress_cb,
         is_cancelled=is_cancelled,
     )
+    _emit_file_completed(file_completed_cb, path_in_repo, json_path)
 
     _ensure_not_cancelled(is_cancelled)
     with open(json_path, "r", encoding="utf-8") as handle:
@@ -564,6 +664,8 @@ def _download_json_path_from_hf(
     except ValueError:
         repo_paths = []
     allow_patterns = _build_allow_patterns(repo_paths, repo_json_folder)
+    if not annotations_only:
+        _emit_file_plan(file_plan_cb, allow_patterns)
 
     result: dict[str, Any] = {
         "repo_id": repo_id,
@@ -580,8 +682,11 @@ def _download_json_path_from_hf(
         "num_samples": len(osl_json.get("data", [])) if isinstance(osl_json.get("data"), list) else 0,
     }
 
-    if annotations_only:
-        _emit_progress(progress_cb, "Persisting Hugging Face source metadata into downloaded JSON.")
+    if not dry_run:
+        _emit_progress(
+            progress_cb,
+            "Persisting Hugging Face source metadata into downloaded JSON.",
+        )
         hf_source_metadata = write_hf_source_metadata_to_dataset_json(
             json_path,
             repo_id=repo_id,
@@ -590,9 +695,12 @@ def _download_json_path_from_hf(
             source_format="json",
             commit=commit,
         )
+        result["hf_source_metadata"] = hf_source_metadata
+        _emit_json_ready(json_ready_cb, cleaned_split, json_path)
+
+    if annotations_only:
         result["download_kind"] = "json"
         result["downloaded_file_count"] = 0
-        result["hf_source_metadata"] = hf_source_metadata
         _emit_progress(progress_cb, "Annotation-only download completed.")
         return result
 
@@ -651,7 +759,7 @@ def _download_json_path_from_hf(
     for idx, full_repo_path in enumerate(allow_patterns, start=1):
         _ensure_not_cancelled(is_cancelled)
         _emit_progress(progress_cb, f"[{idx}/{len(allow_patterns)}] Downloading {full_repo_path}")
-        _download_hf_file(
+        downloaded_path = _download_hf_file(
             hf_hub_download,
             repo_id=repo_id,
             filename=full_repo_path,
@@ -661,21 +769,13 @@ def _download_json_path_from_hf(
             byte_progress_cb=byte_progress_cb,
             is_cancelled=is_cancelled,
         )
+        _emit_file_completed(
+            file_completed_cb, full_repo_path, downloaded_path
+        )
         downloaded_count += 1
 
-    _emit_progress(progress_cb, "Persisting Hugging Face source metadata into downloaded JSON.")
-    hf_source_metadata = write_hf_source_metadata_to_dataset_json(
-        json_path,
-        repo_id=repo_id,
-        branch=revision,
-        split=cleaned_split,
-        source_format="json",
-        commit=commit,
-    )
-
     result["download_kind"] = "json"
     result["downloaded_file_count"] = downloaded_count
-    result["hf_source_metadata"] = hf_source_metadata
     _emit_progress(progress_cb, "Download completed.")
     return result
 
@@ -776,6 +876,9 @@ def download_dataset_splits_from_hf(
     token: str | None = None,
     progress_cb: ProgressCallback | None = None,
     byte_progress_cb: ByteProgressCallback | None = None,
+    file_plan_cb: FilePlanCallback | None = None,
+    file_completed_cb: FileCompletedCallback | None = None,
+    json_ready_cb: JsonReadyCallback | None = None,
     is_cancelled: CancelCheck | None = None,
 ) -> list[dict[str, Any]]:
     cleaned_splits = [str(split or "").strip() for split in (splits or [])]
@@ -802,6 +905,9 @@ def download_dataset_splits_from_hf(
             token=token,
             progress_cb=_scoped_progress,
             byte_progress_cb=byte_progress_cb,
+            file_plan_cb=file_plan_cb,
+            file_completed_cb=file_completed_cb,
+            json_ready_cb=json_ready_cb,
             is_cancelled=is_cancelled,
         )
         results.append(result)
@@ -821,6 +927,9 @@ def download_dataset_split_from_hf(
     token: str | None = None,
     progress_cb: ProgressCallback | None = None,
     byte_progress_cb: ByteProgressCallback | None = None,
+    file_plan_cb: FilePlanCallback | None = None,
+    file_completed_cb: FileCompletedCallback | None = None,
+    json_ready_cb: JsonReadyCallback | None = None,
     is_cancelled: CancelCheck | None = None,
 ) -> dict[str, Any]:
     cleaned_repo_id = str(repo_id or "").strip()
@@ -849,6 +958,9 @@ def download_dataset_split_from_hf(
             token=token,
             progress_cb=progress_cb,
             byte_progress_cb=byte_progress_cb,
+            file_plan_cb=file_plan_cb,
+            file_completed_cb=file_completed_cb,
+            json_ready_cb=json_ready_cb,
             is_cancelled=is_cancelled,
         )
 
@@ -863,6 +975,9 @@ def download_dataset_split_from_hf(
         token=token,
         progress_cb=progress_cb,
         byte_progress_cb=byte_progress_cb,
+        file_plan_cb=file_plan_cb,
+        file_completed_cb=file_completed_cb,
+        json_ready_cb=json_ready_cb,
         is_cancelled=is_cancelled,
     )
 

{opensportslib-0.3.1.dev2 → opensportslib-0.3.1.dev3/opensportslib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.3.1.dev2
+Version: 0.3.1.dev3
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12
@@ -392,6 +392,19 @@ The JSON records the resolved Hugging Face commit and can later be passed to
 Parquet/WebDataset download always completes the local split even when a
 metadata-only `<split>.json` already exists.
 
+Download APIs accept `byte_progress_cb(filename, downloaded_bytes,
+total_bytes)`. When the repository file is Xet-backed, OpenSportsLib keeps the
+accelerated Xet transfer and adapts Xet's byte updates to this callback. It
+falls back to classic HTTP progress when Xet is unavailable, disabled, or not
+used by the file.
+When byte progress is enabled, Parquet downloads also emit `[current/total]`
+file messages through `progress_cb` so clients can present file-count progress.
+High-level split downloads also accept `file_plan_cb(filenames)`,
+`file_completed_cb(filename, local_path)`, and
+`json_ready_cb(split, json_path)`. These are transfer lifecycle notifications;
+callers remain responsible for queue policy and presentation. For non-dry-run
+JSON datasets, pinned source metadata is persisted before `json_ready_cb` runs.
+
 JSON uploads support partially downloaded datasets: the JSON and all
 referenced files available locally are committed, while missing referenced
 files are skipped and reported. Remote files not included in that commit are

{opensportslib-0.3.1.dev2 → opensportslib-0.3.1.dev3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "opensportslib"
-version = "0.3.1.dev2"
+version = "0.3.1.dev3"
 description = "OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data."
 readme = "README.md"
 requires-python = ">=3.12"

{opensportslib-0.3.1.dev2 → opensportslib-0.3.1.dev3}/tests/test_hf_transfer_tools.py

@@ -70,6 +70,7 @@ def test_download_hf_file_reports_transferred_and_total_bytes(monkeypatch, tmp_p
         "_import_hf_file_system",
         lambda: (_FakeFileSystem, TqdmCallback),
     )
+    monkeypatch.setattr(hf_transfer_module, "_import_hf_xet_download", lambda: None)
     progress = []
 
     result = hf_transfer_module._download_hf_file(
@@ -115,6 +116,7 @@ def test_download_hf_file_cancels_during_transfer_and_removes_partial(
         "_import_hf_file_system",
         lambda: (_FakeFileSystem, TqdmCallback),
     )
+    monkeypatch.setattr(hf_transfer_module, "_import_hf_xet_download", lambda: None)
 
     with pytest.raises(HfTransferCancelled):
         hf_transfer_module._download_hf_file(
@@ -132,6 +134,62 @@ def test_download_hf_file_cancels_during_transfer_and_removes_partial(
     assert list((tmp_path / "clips").glob("*.part")) == []
 
 
+def test_download_hf_file_reports_bytes_while_using_xet(monkeypatch, tmp_path):
+    xet_file_data = object()
+
+    class _Metadata:
+        size = 6
+
+    _Metadata.xet_file_data = xet_file_data
+
+    def _fake_xet_get(**kwargs):
+        assert kwargs["xet_file_data"] is xet_file_data
+        assert kwargs["headers"] == {"authorization": "Bearer hf_test"}
+        assert kwargs["expected_size"] == 6
+        assert kwargs["displayed_filename"] == "clips/large.mp4"
+        Path(kwargs["incomplete_path"]).write_bytes(b"abcdef")
+        kwargs["_tqdm_bar"].update(2)
+        kwargs["_tqdm_bar"].update(4)
+
+    monkeypatch.setattr(
+        hf_transfer_module,
+        "_import_hf_xet_download",
+        lambda: (
+            lambda url, token: _Metadata(),
+            lambda **kwargs: "https://huggingface.test/file",
+            _fake_xet_get,
+            lambda token: {"authorization": f"Bearer {token}"},
+            lambda: True,
+        ),
+    )
+    monkeypatch.setattr(
+        hf_transfer_module,
+        "_import_hf_file_system",
+        lambda: pytest.fail("classic HTTP fallback should not be used"),
+    )
+    progress = []
+
+    result = hf_transfer_module._download_hf_file(
+        pytest.fail,
+        repo_id="OpenSportsLab/repo",
+        filename="clips/large.mp4",
+        revision="pinned",
+        local_dir=str(tmp_path),
+        token="hf_test",
+        byte_progress_cb=lambda filename, current, total: progress.append(
+            (filename, current, total)
+        ),
+    )
+
+    assert result == str(tmp_path / "clips" / "large.mp4")
+    assert Path(result).read_bytes() == b"abcdef"
+    assert progress == [
+        ("clips/large.mp4", 0, 6),
+        ("clips/large.mp4", 2, 6),
+        ("clips/large.mp4", 6, 6),
+    ]
+
+
 def test_download_hf_file_rejects_unsafe_destination(tmp_path):
     with pytest.raises(ValueError, match="Unsafe Hugging Face file path"):
         hf_transfer_module._download_hf_file(
@@ -1053,6 +1111,9 @@ def test_download_dataset_split_from_hf_json_downloads_split_json_and_all_inputs
         ]
     }
     downloaded = []
+    planned = []
+    completed = []
+    json_ready = []
 
     class _FakeApi:
         def __init__(self, token=None):
@@ -1082,6 +1143,11 @@ def test_download_dataset_split_from_hf_json_downloads_split_json_and_all_inputs
         "test",
         str(tmp_path),
         download_format="json",
+        file_plan_cb=planned.append,
+        file_completed_cb=lambda filename, path: completed.append(
+            (filename, path)
+        ),
+        json_ready_cb=lambda split, path: json_ready.append((split, path)),
     )
 
     assert downloaded == ["test.json", "test/captions.json", "test/clip_0.mp4"]
@@ -1094,6 +1160,12 @@ def test_download_dataset_split_from_hf_json_downloads_split_json_and_all_inputs
     assert result["output_dir"] == str(expected_output_dir)
     assert result["json_path"] == str(expected_output_dir / "test.json")
     assert result["downloaded_file_count"] == 2
+    assert planned == [
+        ["test.json"],
+        ["test/captions.json", "test/clip_0.mp4"],
+    ]
+    assert [filename for filename, _path in completed] == downloaded
+    assert json_ready == [("test", str(expected_output_dir / "test.json"))]
 
 
 def test_download_dataset_split_from_hf_parquet_downloads_split_folder(monkeypatch, tmp_path):
@@ -1140,6 +1212,78 @@ def test_download_dataset_split_from_hf_parquet_downloads_split_folder(monkeypat
     assert result["download_skipped"] is False
 
 
+def test_parquet_byte_download_reports_file_count_progress(monkeypatch, tmp_path):
+    progress_messages = []
+    downloaded = []
+    planned = []
+    completed = []
+    json_ready = []
+
+    class _FakeApi:
+        def __init__(self, token=None):
+            pass
+
+        def repo_info(self, **kwargs):
+            return type("_Info", (), {"sha": "pinned"})()
+
+        def list_repo_files(self, *args, **kwargs):
+            return [
+                "test/metadata.parquet",
+                "test/shards/shard-000000.tar",
+            ]
+
+    def _fake_download_file(_hf_hub_download, **kwargs):
+        downloaded.append(kwargs["filename"])
+        return str(Path(kwargs["local_dir"]) / kwargs["filename"])
+
+    def _fake_conversion(**kwargs):
+        kwargs["output_json_path"].write_text(
+            json.dumps({"data": []}), encoding="utf-8"
+        )
+        return {"num_samples": 0, "extracted_media_files": 0}
+
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._import_hf_hub",
+        lambda: (_FakeApi, object(), object()),
+    )
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer._download_hf_file",
+        _fake_download_file,
+    )
+    monkeypatch.setattr(
+        "opensportslib.tools.hf_transfer.convert_parquet_to_json",
+        _fake_conversion,
+    )
+
+    download_dataset_split_from_hf(
+        "OpenSportsLab/repo",
+        "main",
+        "test",
+        str(tmp_path),
+        download_format="parquet",
+        progress_cb=progress_messages.append,
+        byte_progress_cb=lambda *_args: None,
+        file_plan_cb=planned.append,
+        file_completed_cb=lambda filename, path: completed.append(
+            (filename, path)
+        ),
+        json_ready_cb=lambda split, path: json_ready.append((split, path)),
+    )
+
+    assert downloaded == [
+        "test/metadata.parquet",
+        "test/shards/shard-000000.tar",
+    ]
+    assert "[1/2] Downloading test/metadata.parquet" in progress_messages
+    assert "[2/2] Downloading test/shards/shard-000000.tar" in progress_messages
+    assert planned == [[
+        "test/metadata.parquet",
+        "test/shards/shard-000000.tar",
+    ]]
+    assert [filename for filename, _path in completed] == downloaded
+    assert json_ready == [("test", str(tmp_path / "main" / "test" / "test.json"))]
+
+
 def test_download_dataset_split_from_hf_parquet_completes_existing_json(
     monkeypatch, tmp_path
 ):
@@ -1196,6 +1340,7 @@ def test_json_annotations_only_downloads_json_and_persists_pinned_source(monkeyp
         encoding="utf-8",
     )
     downloaded = []
+    planned = []
 
     class _FakeApi:
         def __init__(self, token=None):
@@ -1215,9 +1360,11 @@ def test_json_annotations_only_downloads_json_and_persists_pinned_source(monkeyp
         str(tmp_path / "output"),
         download_format="json",
         annotations_only=True,
+        file_plan_cb=planned.append,
     )
 
     assert downloaded == ["test.json"]
+    assert planned == [["test.json"]]
     payload = json.loads(Path(result["json_path"]).read_text(encoding="utf-8"))
     assert payload[HF_FORMAT_KEY] == "json"
     assert payload[HF_COMMIT_KEY] == "pinned-json"