hfutils 0.3.5__tar.gz → 0.3.6__tar.gz

{hfutils-0.3.5/hfutils.egg-info → hfutils-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.3.5
+Version: 0.3.6
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -29,28 +29,6 @@ Requires-Dist: tzlocal
 Requires-Dist: natsort
 Requires-Dist: urlobject
 Requires-Dist: fsspec>=2024
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
-Provides-Extra: 7z
-Requires-Dist: py7zr; extra == "7z"
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
-Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -71,8 +49,30 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
+Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: rar
 Requires-Dist: rarfile; extra == "rar"
 

{hfutils-0.3.5 → hfutils-0.3.6}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.3.5'
+__VERSION__ = '0.3.6'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.3.5 → hfutils-0.3.6}/hfutils/index/fetch.py

@@ -1,5 +1,7 @@
 import json
 import os.path
+import threading
+from collections import defaultdict
 from typing import Optional, Dict, Union, List
 
 from huggingface_hub.file_download import http_get, hf_hub_url
@@ -21,6 +23,9 @@ class ArchiveStandaloneFileHashNotMatch(Exception):
     """
 
 
+_HF_TAR_IDX_LOCKS = defaultdict(threading.Lock)
+
+
 def hf_tar_get_index(repo_id: str, archive_in_repo: str,
                      repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
                      idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
@@ -83,13 +88,18 @@ def hf_tar_get_index(repo_id: str, archive_in_repo: str,
     hf_client = get_hf_client(hf_token)
     body, _ = os.path.splitext(archive_in_repo)
     default_index_file = f'{body}.json'
-    with open(hf_client.hf_hub_download(
-            repo_id=idx_repo_id or repo_id,
-            repo_type=idx_repo_type or repo_type,
-            filename=idx_file_in_repo or default_index_file,
-            revision=idx_revision or revision,
-    ), 'r') as f:
-        return json.load(f)
+    f_repo_id = idx_repo_id or repo_id
+    f_repo_type = idx_repo_type or repo_type
+    f_filename = idx_file_in_repo or default_index_file
+    f_revision = idx_revision or revision
+    with _HF_TAR_IDX_LOCKS[(f_repo_id, f_repo_type, f_filename, f_revision)]:
+        with open(hf_client.hf_hub_download(
+                repo_id=f_repo_id,
+                repo_type=f_repo_type,
+                filename=f_filename,
+                revision=f_revision,
+        ), 'r') as f:
+            return json.load(f)
 
 
 def hf_tar_list_files(repo_id: str, archive_in_repo: str,

{hfutils-0.3.5 → hfutils-0.3.6/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.3.5
+Version: 0.3.6
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -29,28 +29,6 @@ Requires-Dist: tzlocal
 Requires-Dist: natsort
 Requires-Dist: urlobject
 Requires-Dist: fsspec>=2024
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
-Provides-Extra: 7z
-Requires-Dist: py7zr; extra == "7z"
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
-Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -71,8 +49,30 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
+Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: rar
 Requires-Dist: rarfile; extra == "rar"