opensportslib 0.3.1.dev1__tar.gz → 0.3.1.dev3__tar.gz

{opensportslib-0.3.1.dev1/opensportslib.egg-info → opensportslib-0.3.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opensportslib
-Version: 0.3.1.dev1
+Version: 0.3.1.dev3
 Summary: OpenSportsLib is the professional library, designed for advanced video understanding in sports. It provides state-of-the-art tools for action recognition, spotting, retrieval, and captioning, making it ideal for researchers, analysts, and developers working with sports video data.
 Author: Jeet Vora
 Requires-Python: >=3.12
@@ -372,6 +372,7 @@ OpenSportsLib provides APIs and scripts for downloading and uploading OSL datase
 ```python
 from opensportslib.tools import (
     download_dataset_split_from_hf,
+    download_dataset_sample_inputs_from_hf,
     upload_dataset_inputs_from_json_to_hf,
     upload_dataset_as_parquet_to_hf,
 )
@@ -380,13 +381,35 @@ from opensportslib.tools import (
 ### Scripts
 
 ```bash
-python tools/download/download_osl_hf.py --repo-id <org/repo> --revision main --split test --format parquet --output-dir downloaded_data
+python tools/download/download_osl_hf.py --repo-id <org/repo> --revision main --split test --format parquet --output-dir downloaded_data --annotations-only
 python tools/download/upload_osl_hf.py --repo-id <org/repo> --json-path <local_dataset.json> --split test --revision main
 ```
 
 Downloads are placed under `<output-dir>/<revision>/<split>`.
-For Parquet/WebDataset downloads, an existing `<split>.json` in that directory
-is reused without downloading or converting the split again.
+Pass `annotations_only=True` to download or reconstruct only `<split>.json`.
+The JSON records the resolved Hugging Face commit and can later be passed to
+`download_dataset_sample_inputs_from_hf()` to fetch one sample or input. A full
+Parquet/WebDataset download always completes the local split even when a
+metadata-only `<split>.json` already exists.
+
+Download APIs accept `byte_progress_cb(filename, downloaded_bytes,
+total_bytes)`. When the repository file is Xet-backed, OpenSportsLib keeps the
+accelerated Xet transfer and adapts Xet's byte updates to this callback. It
+falls back to classic HTTP progress when Xet is unavailable, disabled, or not
+used by the file.
+When byte progress is enabled, Parquet downloads also emit `[current/total]`
+file messages through `progress_cb` so clients can present file-count progress.
+High-level split downloads also accept `file_plan_cb(filenames)`,
+`file_completed_cb(filename, local_path)`, and
+`json_ready_cb(split, json_path)`. These are transfer lifecycle notifications;
+callers remain responsible for queue policy and presentation. For non-dry-run
+JSON datasets, pinned source metadata is persisted before `json_ready_cb` runs.
+
+JSON uploads support partially downloaded datasets: the JSON and all
+referenced files available locally are committed, while missing referenced
+files are skipped and reported. Remote files not included in that commit are
+left untouched. Parquet/WebDataset uploads remain strict and require every
+referenced file locally before conversion.
 
 ---
 

{opensportslib-0.3.1.dev1 → opensportslib-0.3.1.dev3}/README.md

@@ -336,6 +336,7 @@ OpenSportsLib provides APIs and scripts for downloading and uploading OSL datase
 ```python
 from opensportslib.tools import (
     download_dataset_split_from_hf,
+    download_dataset_sample_inputs_from_hf,
     upload_dataset_inputs_from_json_to_hf,
     upload_dataset_as_parquet_to_hf,
 )
@@ -344,13 +345,35 @@ from opensportslib.tools import (
 ### Scripts
 
 ```bash
-python tools/download/download_osl_hf.py --repo-id <org/repo> --revision main --split test --format parquet --output-dir downloaded_data
+python tools/download/download_osl_hf.py --repo-id <org/repo> --revision main --split test --format parquet --output-dir downloaded_data --annotations-only
 python tools/download/upload_osl_hf.py --repo-id <org/repo> --json-path <local_dataset.json> --split test --revision main
 ```
 
 Downloads are placed under `<output-dir>/<revision>/<split>`.
-For Parquet/WebDataset downloads, an existing `<split>.json` in that directory
-is reused without downloading or converting the split again.
+Pass `annotations_only=True` to download or reconstruct only `<split>.json`.
+The JSON records the resolved Hugging Face commit and can later be passed to
+`download_dataset_sample_inputs_from_hf()` to fetch one sample or input. A full
+Parquet/WebDataset download always completes the local split even when a
+metadata-only `<split>.json` already exists.
+
+Download APIs accept `byte_progress_cb(filename, downloaded_bytes,
+total_bytes)`. When the repository file is Xet-backed, OpenSportsLib keeps the
+accelerated Xet transfer and adapts Xet's byte updates to this callback. It
+falls back to classic HTTP progress when Xet is unavailable, disabled, or not
+used by the file.
+When byte progress is enabled, Parquet downloads also emit `[current/total]`
+file messages through `progress_cb` so clients can present file-count progress.
+High-level split downloads also accept `file_plan_cb(filenames)`,
+`file_completed_cb(filename, local_path)`, and
+`json_ready_cb(split, json_path)`. These are transfer lifecycle notifications;
+callers remain responsible for queue policy and presentation. For non-dry-run
+JSON datasets, pinned source metadata is persisted before `json_ready_cb` runs.
+
+JSON uploads support partially downloaded datasets: the JSON and all
+referenced files available locally are committed, while missing referenced
+files are skipped and reported. Remote files not included in that commit are
+left untouched. Parquet/WebDataset uploads remain strict and require every
+referenced file locally before conversion.
 
 ---
 

{opensportslib-0.3.1.dev1 → opensportslib-0.3.1.dev3}/opensportslib/tools/__init__.py

@@ -10,13 +10,29 @@ _EXPORTS = {
     "DEFAULT_SHARD_SIZE": ("opensportslib.tools.osl_json_to_parquet", "DEFAULT_SHARD_SIZE"),
     "parse_shard_size": ("opensportslib.tools.osl_json_to_parquet", "parse_shard_size"),
     "convert_parquet_to_json": ("opensportslib.tools.parquet_to_osl_json", "convert_parquet_to_json"),
+    "convert_parquet_metadata_to_json": (
+        "opensportslib.tools.parquet_to_osl_json",
+        "convert_parquet_metadata_to_json",
+    ),
     "convert_sn_vqa_2026_to_osl": ("opensportslib.tools.sn_vqa_2026_qwen_native", "convert_sn_vqa_2026_to_osl"),
     "evaluate_sn_vqa_predictions": ("opensportslib.tools.sn_vqa_2026_qwen_native", "evaluate_sn_vqa_predictions"),
     "HfTransferCancelled": ("opensportslib.tools.hf_transfer", "HfTransferCancelled"),
+    "MissingDatasetInputsError": ("opensportslib.tools.hf_transfer", "MissingDatasetInputsError"),
     "HF_REPO_ID_KEY": ("opensportslib.tools.hf_transfer", "HF_REPO_ID_KEY"),
     "HF_BRANCH_KEY": ("opensportslib.tools.hf_transfer", "HF_BRANCH_KEY"),
     "HF_SPLIT_KEY": ("opensportslib.tools.hf_transfer", "HF_SPLIT_KEY"),
+    "HF_FORMAT_KEY": ("opensportslib.tools.hf_transfer", "HF_FORMAT_KEY"),
+    "HF_COMMIT_KEY": ("opensportslib.tools.hf_transfer", "HF_COMMIT_KEY"),
     "download_dataset_split_from_hf": ("opensportslib.tools.hf_transfer", "download_dataset_split_from_hf"),
+    "download_dataset_sample_inputs_from_hf": (
+        "opensportslib.tools.hf_transfer",
+        "download_dataset_sample_inputs_from_hf",
+    ),
+    "download_dataset_missing_inputs_from_hf": (
+        "opensportslib.tools.hf_transfer",
+        "download_dataset_missing_inputs_from_hf",
+    ),
+    "find_missing_dataset_inputs": ("opensportslib.tools.hf_transfer", "find_missing_dataset_inputs"),
     "upload_dataset_inputs_from_json_to_hf": ("opensportslib.tools.hf_transfer", "upload_dataset_inputs_from_json_to_hf"),
     "upload_dataset_as_parquet_to_hf": ("opensportslib.tools.hf_transfer", "upload_dataset_as_parquet_to_hf"),
     "create_dataset_repo_on_hf": ("opensportslib.tools.hf_transfer", "create_dataset_repo_on_hf"),
@@ -37,13 +53,20 @@ __all__ = [
     "DEFAULT_SHARD_SIZE",
     "parse_shard_size",
     "convert_parquet_to_json",
+    "convert_parquet_metadata_to_json",
     "convert_sn_vqa_2026_to_osl",
     "evaluate_sn_vqa_predictions",
     "HfTransferCancelled",
+    "MissingDatasetInputsError",
     "HF_REPO_ID_KEY",
     "HF_BRANCH_KEY",
     "HF_SPLIT_KEY",
+    "HF_FORMAT_KEY",
+    "HF_COMMIT_KEY",
     "download_dataset_split_from_hf",
+    "download_dataset_sample_inputs_from_hf",
+    "download_dataset_missing_inputs_from_hf",
+    "find_missing_dataset_inputs",
     "upload_dataset_inputs_from_json_to_hf",
     "upload_dataset_as_parquet_to_hf",
     "create_dataset_repo_on_hf",