hfutils 0.0.2__tar.gz → 0.0.4__tar.gz

{hfutils-0.0.2/hfutils.egg-info → hfutils-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.0.2
+Version: 0.0.4
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -24,6 +24,8 @@ Requires-Dist: huggingface_hub>=0.16
 Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: click>=7
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
 Provides-Extra: test
 Requires-Dist: coverage>=5; extra == "test"
 Requires-Dist: mock>=4.0.3; extra == "test"
@@ -61,6 +63,10 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
+Provides-Extra: transfer
+Requires-Dist: hf-transfer; extra == "transfer"
 Provides-Extra: build
 Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 
@@ -221,3 +227,42 @@ Explore additional options for downloading:
 hfutils download -h
 
 ```
+
+## Supported Formats
+
+By default, we support the `zip` and `tar` formats, including `.zip`, `.tar`, `.tar.gz`, `.tar.bz2`, and `.tar.xz`.
+
+If you require support for `rar` and `7z` files, install the extra dependencies using the following command:
+
+```shell
+pip install hfutils[rar,7z]
+```
+
+**NOTE:** Creating RAR archive files is not supported. We use the [rarfile](https://github.com/markokr/rarfile) library,
+which lacks the functionality for creating RAR files.
+
+## How to Access Private Repositories
+
+Simply configure the `HF_TOKEN` environment variable by using your HuggingFace access token.
+Note that write permissions are required if you plan to upload any content.
+
+## How to Use Hf-Transfer for Acceleration
+
+If you are using the PyPI CLI, you need to install `hfutils` with the following command:
+
+```shell
+pip install hfutils[transfer]
+
+```
+
+If you are using a precompiled executable file, the transfer module is integrated inside; simply use it.
+
+Enable Hf-Transfer acceleration by setting the environment variable `HF_HUB_ENABLE_HF_TRANSFER` to `1`.
+
+## How to Change the Temporary Directory
+
+The temporary directory is utilized for storing partially downloaded files, consuming a considerable amount of disk
+space.
+
+If your disk, especially the C drive on Windows, does not have sufficient space, simply set the `TMPDIR` to designate
+another directory as the temporary directory.

{hfutils-0.0.2 → hfutils-0.0.4}/README.md

@@ -155,3 +155,42 @@ Explore additional options for downloading:
 hfutils download -h
 
 ```
+
+## Supported Formats
+
+By default, we support the `zip` and `tar` formats, including `.zip`, `.tar`, `.tar.gz`, `.tar.bz2`, and `.tar.xz`.
+
+If you require support for `rar` and `7z` files, install the extra dependencies using the following command:
+
+```shell
+pip install hfutils[rar,7z]
+```
+
+**NOTE:** Creating RAR archive files is not supported. We use the [rarfile](https://github.com/markokr/rarfile) library,
+which lacks the functionality for creating RAR files.
+
+## How to Access Private Repositories
+
+Simply configure the `HF_TOKEN` environment variable by using your HuggingFace access token.
+Note that write permissions are required if you plan to upload any content.
+
+## How to Use Hf-Transfer for Acceleration
+
+If you are using the PyPI CLI, you need to install `hfutils` with the following command:
+
+```shell
+pip install hfutils[transfer]
+
+```
+
+If you are using a precompiled executable file, the transfer module is integrated inside; simply use it.
+
+Enable Hf-Transfer acceleration by setting the environment variable `HF_HUB_ENABLE_HF_TRANSFER` to `1`.
+
+## How to Change the Temporary Directory
+
+The temporary directory is utilized for storing partially downloaded files, consuming a considerable amount of disk
+space.
+
+If your disk, especially the C drive on Windows, does not have sufficient space, simply set the `TMPDIR` to designate
+another directory as the temporary directory.

hfutils-0.0.4/hfutils/archive/__init__.py

@@ -0,0 +1,24 @@
+"""
+Overview:
+    Archive pack and unpack management.
+
+Supported Formats:
+
+    .. include:: supported_types.demo.py.txt
+
+.. note::
+    If you require support for 7z and RAR formats, simply install ``hfutils`` using the following code:
+
+    .. code:: shell
+
+        pip install hfutils[7z]
+        pip install hfutils[rar]
+
+.. warning::
+    The creation of archive files in the RAR format is not supported, as we utilize the `rarfile <https://github.com/markokr/rarfile>`_ library, which does not offer functionality for creating RAR files.
+"""
+from .base import register_archive_type, archive_pack, archive_unpack, get_archive_type, get_archive_extname
+from .rar import _rar_pack, _rar_unpack
+from .sevenz import _7z_pack, _7z_unpack
+from .tar import _tarfile_pack, _tarfile_unpack
+from .zip import _zip_pack, _zip_unpack

hfutils-0.0.4/hfutils/archive/rar.py

@@ -0,0 +1,28 @@
+import os
+
+from .base import register_archive_type
+from ..utils import tqdm
+
+try:
+    import rarfile
+except ImportError:  # pragma: no cover
+    rarfile = None
+
+
+def _rar_pack(directory, zip_file, silent: bool = False):
+    _ = directory, zip_file, silent
+    raise RuntimeError('RAR format packing is not supported.')
+
+
+def _rar_unpack(rar_file, directory, silent: bool = False):
+    directory = os.fspath(directory)
+    os.makedirs(directory, exist_ok=True)
+    with rarfile.RarFile(rar_file, 'r') as zf:
+        progress = tqdm(zf.namelist(), silent=silent, desc=f'Unpacking {directory!r} ...')
+        for rarinfo in progress:
+            progress.set_description(rarinfo)
+            zf.extract(rarinfo, directory)
+
+
+if rarfile is not None:
+    register_archive_type('rar', ['.rar'], _rar_pack, _rar_unpack)

hfutils-0.0.4/hfutils/archive/sevenz.py

@@ -0,0 +1,32 @@
+import os
+
+from .base import register_archive_type
+from ..utils import tqdm, walk_files
+
+try:
+    import py7zr
+except ImportError:  # pragma: no cover
+    py7zr = None
+
+
+def _7z_pack(directory, sz_file, silent: bool = False):
+    with py7zr.SevenZipFile(sz_file, 'w') as zf:
+        progress = tqdm(walk_files(directory), silent=silent, desc=f'Packing {directory!r} ...')
+        for file in progress:
+            progress.set_description(file)
+            zf.write(os.path.join(directory, file), file)
+
+
+def _7z_unpack(sz_file, directory, silent: bool = False):
+    directory = os.fspath(directory)
+    os.makedirs(directory, exist_ok=True)
+    with py7zr.SevenZipFile(sz_file, 'r') as zf:
+        progress = tqdm(zf.getnames(), silent=silent, desc=f'Unpacking {directory!r} ...')
+        for name in progress:
+            progress.set_description(name)
+            zf.extract(path=directory, targets=[name])
+            zf.reset()
+
+
+if py7zr is not None:
+    register_archive_type('7z', ['.7z'], _7z_pack, _7z_unpack)

{hfutils-0.0.2 → hfutils-0.0.4}/hfutils/archive/zip.py

@@ -25,7 +25,7 @@ def _zip_unpack(zip_file, directory, silent: bool = False):
     directory = os.fspath(directory)
     os.makedirs(directory, exist_ok=True)
     with zipfile.ZipFile(zip_file, 'r') as zf:
-        progress = tqdm(zf.namelist(), silent=silent, desc=f'Packing {directory!r} ...')
+        progress = tqdm(zf.namelist(), silent=silent, desc=f'Unpacking {directory!r} ...')
         for zipinfo in progress:
             progress.set_description(zipinfo)
             zf.extract(zipinfo, directory)

{hfutils-0.0.2 → hfutils-0.0.4}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.0.2'
+__VERSION__ = '0.0.4'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.0.2 → hfutils-0.0.4}/hfutils/entry/download.py

@@ -42,10 +42,12 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
                   help='Output path for download.')
     @click.option('-R', '--revision', 'revision', type=str, default='main',
                   help='Revision of repository.', show_default=True)
+    @click.option('-n', '--max_workers', 'max_workers', type=int, default=8,
+                  help='Max threads to download.', show_default=True)
     @command_wrap()
     def download(repo_id: str, repo_type: RepoTypeTyping,
                  file_in_repo: Optional[str], archive_in_repo: Optional[str], dir_in_repo: Optional[str],
-                 output_path: str, revision: str):
+                 output_path: str, revision: str, max_workers: int):
         """
         Download data from HuggingFace repositories.
 
@@ -63,10 +65,12 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
         :type output_path: str
         :param revision: Revision of repository.
         :type revision: str
+        :param max_workers: Max workers to download
+        :type max_workers: int
         """
         if not file_in_repo and not archive_in_repo and not dir_in_repo:
             raise NoRemotePathAssignedWithDownload('No remote path in repository assigned.\n'
-                                       'One of the -f, -a, or -d option is required.')
+                                                   'One of the -f, -a, or -d option is required.')
 
         if file_in_repo:
             if archive_in_repo:
@@ -79,7 +83,6 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
                 file_in_repo=file_in_repo,
                 repo_type=repo_type,
                 revision=revision,
-                silent=False,
             )
 
         elif archive_in_repo:
@@ -91,7 +94,6 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
                 file_in_repo=archive_in_repo,
                 repo_type=repo_type,
                 revision=revision,
-                silent=False,
             )
 
         elif dir_in_repo:
@@ -102,6 +104,7 @@ def _add_download_subcommand(cli: click.Group) -> click.Group:
                 repo_type=repo_type,
                 revision=revision,
                 silent=False,
+                max_workers=max_workers,
             )
 
         else:

{hfutils-0.0.2 → hfutils-0.0.4}/hfutils/operate/download.py

@@ -1,17 +1,16 @@
 import os.path
+import shutil
 from concurrent.futures import ThreadPoolExecutor
 from typing import List
 
-from hbutils.system import TemporaryDirectory
-from huggingface_hub import hf_hub_url
-
-from .base import RepoTypeTyping, _get_hf_token, list_files_in_repository, _IGNORE_PATTERN_UNSET
+from .base import RepoTypeTyping, list_files_in_repository, _IGNORE_PATTERN_UNSET, get_hf_client
 from ..archive import archive_unpack
-from ..utils import download_file, tqdm
+from ..utils import tqdm, TemporaryDirectory
 
 
 def download_file_to_file(local_file: str, repo_id: str, file_in_repo: str,
-                          repo_type: RepoTypeTyping = 'dataset', revision: str = 'main', silent: bool = False):
+                          repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                          resume_download: bool = True):
     """
     Download a file from a Hugging Face repository and save it to a local file.
 
@@ -25,27 +24,33 @@ def download_file_to_file(local_file: str, repo_id: str, file_in_repo: str,
     :type repo_type: RepoTypeTyping
     :param revision: The revision of the repository (e.g., branch, tag, commit hash).
     :type revision: str
-    :param silent: If True, suppress progress bar output.
-    :type silent: bool
+    :param resume_download: Resume the existing download.
+    :type resume_download: bool
     """
-    headers = {}
-    token = _get_hf_token()
-    if token:
-        headers['Authorization'] = f'Bearer {token}'
-
-    parent_dir = os.path.dirname(local_file)
-    if parent_dir:
-        os.makedirs(parent_dir, exist_ok=True)
-    download_file(hf_hub_url(
-        repo_id=repo_id,
-        repo_type=repo_type,
-        filename=file_in_repo,
-        revision=revision,
-    ), local_file, headers=headers, silent=silent)
+    hf_client = get_hf_client()
+    relative_filename = os.path.join(*file_in_repo.split("/"))
+    with TemporaryDirectory() as td:
+        temp_path = os.path.join(td, relative_filename)
+        try:
+            hf_client.hf_hub_download(
+                repo_id=repo_id,
+                repo_type=repo_type,
+                filename=file_in_repo,
+                revision=revision,
+                local_dir=td,
+                force_download=True,
+                local_dir_use_symlinks=False,
+                resume_download=resume_download,
+            )
+        finally:
+            if os.path.exists(temp_path):
+                if os.path.dirname(local_file):
+                    os.makedirs(os.path.dirname(local_file), exist_ok=True)
+                shutil.move(temp_path, local_file)
 
 
 def download_archive_as_directory(local_directory: str, repo_id: str, file_in_repo: str,
-                                  repo_type: RepoTypeTyping = 'dataset', revision: str = 'main', silent: bool = False):
+                                  repo_type: RepoTypeTyping = 'dataset', revision: str = 'main'):
     """
     Download an archive file from a Hugging Face repository and extract it to a local directory.
 
@@ -59,18 +64,17 @@ def download_archive_as_directory(local_directory: str, repo_id: str, file_in_re
     :type repo_type: RepoTypeTyping
     :param revision: The revision of the repository (e.g., branch, tag, commit hash).
     :type revision: str
-    :param silent: If True, suppress progress bar output.
-    :type silent: bool
     """
     with TemporaryDirectory() as td:
         archive_file = os.path.join(td, os.path.basename(file_in_repo))
-        download_file_to_file(archive_file, repo_id, file_in_repo, repo_type, revision, silent=silent)
-        archive_unpack(archive_file, local_directory, silent=silent)
+        download_file_to_file(archive_file, repo_id, file_in_repo, repo_type, revision)
+        archive_unpack(archive_file, local_directory)
 
 
 def download_directory_as_directory(local_directory: str, repo_id: str, dir_in_repo: str = '.',
                                     repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
-                                    silent: bool = False, ignore_patterns: List[str] = _IGNORE_PATTERN_UNSET):
+                                    silent: bool = False, ignore_patterns: List[str] = _IGNORE_PATTERN_UNSET,
+                                    resume_download: bool = True, max_workers: int = 8):
     """
     Download all files in a directory from a Hugging Face repository to a local directory.
 
@@ -88,6 +92,10 @@ def download_directory_as_directory(local_directory: str, repo_id: str, dir_in_r
     :type silent: bool
     :param ignore_patterns: List of file patterns to ignore.
     :type ignore_patterns: List[str]
+    :param max_workers: Max workers when downloading. Default is ``8``.
+    :type max_workers: int
+    :param resume_download: Resume the existing download.
+    :type resume_download: bool
     """
     files = list_files_in_repository(repo_id, repo_type, dir_in_repo, revision, ignore_patterns)
     progress = tqdm(files, silent=silent, desc=f'Downloading {dir_in_repo!r} ...')
@@ -99,11 +107,11 @@ def download_directory_as_directory(local_directory: str, repo_id: str, dir_in_r
             file_in_repo=f'{dir_in_repo}/{rel_file}',
             repo_type=repo_type,
             revision=revision,
-            silent=silent
+            resume_download=resume_download,
         )
         progress.update()
 
-    tp = ThreadPoolExecutor()
+    tp = ThreadPoolExecutor(max_workers=max_workers)
     for file in files:
         tp.submit(_download_one_file, file)
     tp.shutdown(wait=True)

{hfutils-0.0.2 → hfutils-0.0.4}/hfutils/operate/upload.py

@@ -3,12 +3,11 @@ import os.path
 import re
 from typing import Optional, List
 
-from hbutils.system import TemporaryDirectory
 from huggingface_hub import CommitOperationAdd, CommitOperationDelete
 
 from .base import RepoTypeTyping, get_hf_client, list_files_in_repository, _IGNORE_PATTERN_UNSET
 from ..archive import get_archive_type, archive_pack
-from ..utils import walk_files
+from ..utils import walk_files, TemporaryDirectory
 
 
 def upload_file_to_file(local_file, repo_id: str, file_in_repo: str,

{hfutils-0.0.2 → hfutils-0.0.4}/hfutils/utils/__init__.py

@@ -1,4 +1,5 @@
 from .binary import is_binary_file
 from .download import download_file
+from .temp import TemporaryDirectory
 from .tqdm_ import tqdm
 from .walk import walk_files

hfutils-0.0.4/hfutils/utils/temp.py

@@ -0,0 +1,18 @@
+import os
+from functools import lru_cache
+
+from hbutils.system import TemporaryDirectory as _OriginTemporaryDirectory
+
+
+@lru_cache()
+def _create_tmp_dir(dir_):
+    if dir_:
+        os.makedirs(dir_, exist_ok=True)
+
+
+class TemporaryDirectory(_OriginTemporaryDirectory):
+    # noinspection PyShadowingBuiltins
+    def __init__(self, suffix=None, prefix=None, dir=None, ignore_cleanup_errors=False):
+        dir = dir or os.environ.get('TMPDIR') or None
+        _create_tmp_dir(dir)
+        _OriginTemporaryDirectory.__init__(self, suffix, prefix, dir, ignore_cleanup_errors)

{hfutils-0.0.2 → hfutils-0.0.4/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.0.2
+Version: 0.0.4
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -24,6 +24,8 @@ Requires-Dist: huggingface_hub>=0.16
 Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: click>=7
+Provides-Extra: 7z
+Requires-Dist: py7zr; extra == "7z"
 Provides-Extra: test
 Requires-Dist: coverage>=5; extra == "test"
 Requires-Dist: mock>=4.0.3; extra == "test"
@@ -61,6 +63,10 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
+Provides-Extra: transfer
+Requires-Dist: hf-transfer; extra == "transfer"
 Provides-Extra: build
 Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 
@@ -221,3 +227,42 @@ Explore additional options for downloading:
 hfutils download -h
 
 ```
+
+## Supported Formats
+
+By default, we support the `zip` and `tar` formats, including `.zip`, `.tar`, `.tar.gz`, `.tar.bz2`, and `.tar.xz`.
+
+If you require support for `rar` and `7z` files, install the extra dependencies using the following command:
+
+```shell
+pip install hfutils[rar,7z]
+```
+
+**NOTE:** Creating RAR archive files is not supported. We use the [rarfile](https://github.com/markokr/rarfile) library,
+which lacks the functionality for creating RAR files.
+
+## How to Access Private Repositories
+
+Simply configure the `HF_TOKEN` environment variable by using your HuggingFace access token.
+Note that write permissions are required if you plan to upload any content.
+
+## How to Use Hf-Transfer for Acceleration
+
+If you are using the PyPI CLI, you need to install `hfutils` with the following command:
+
+```shell
+pip install hfutils[transfer]
+
+```
+
+If you are using a precompiled executable file, the transfer module is integrated inside; simply use it.
+
+Enable Hf-Transfer acceleration by setting the environment variable `HF_HUB_ENABLE_HF_TRANSFER` to `1`.
+
+## How to Change the Temporary Directory
+
+The temporary directory is utilized for storing partially downloaded files, consuming a considerable amount of disk
+space.
+
+If your disk, especially the C drive on Windows, does not have sufficient space, simply set the `TMPDIR` to designate
+another directory as the temporary directory.

{hfutils-0.0.2 → hfutils-0.0.4}/hfutils.egg-info/SOURCES.txt

@@ -1,9 +1,12 @@
 LICENSE
 MANIFEST.in
 README.md
+requirements-7z.txt
 requirements-build.txt
 requirements-doc.txt
+requirements-rar.txt
 requirements-test.txt
+requirements-transfer.txt
 requirements.txt
 setup.py
 hfutils/__init__.py
@@ -15,6 +18,8 @@ hfutils.egg-info/requires.txt
 hfutils.egg-info/top_level.txt
 hfutils/archive/__init__.py
 hfutils/archive/base.py
+hfutils/archive/rar.py
+hfutils/archive/sevenz.py
 hfutils/archive/tar.py
 hfutils/archive/zip.py
 hfutils/config/__init__.py
@@ -32,5 +37,6 @@ hfutils/operate/upload.py
 hfutils/utils/__init__.py
 hfutils/utils/binary.py
 hfutils/utils/download.py
+hfutils/utils/temp.py
 hfutils/utils/tqdm_.py
 hfutils/utils/walk.py

{hfutils-0.0.2 → hfutils-0.0.4}/hfutils.egg-info/requires.txt

@@ -4,6 +4,9 @@ tqdm
 requests
 click>=7
 
+[7z]
+py7zr
+
 [build]
 pyinstaller<5,>=4.7
 
@@ -28,6 +31,9 @@ py-cpuinfo>=8.0.0
 click>=7.0.0
 pandas
 
+[rar]
+rarfile
+
 [test]
 coverage>=5
 mock>=4.0.3
@@ -45,3 +51,6 @@ testtools>=2
 where>=1.0.2
 responses>=0.20.0
 natsort
+
+[transfer]
+hf-transfer

hfutils-0.0.4/requirements-7z.txt

@@ -0,0 +1 @@
+py7zr

hfutils-0.0.4/requirements-rar.txt

@@ -0,0 +1 @@
+rarfile

hfutils-0.0.4/requirements-transfer.txt

@@ -0,0 +1 @@
+hf-transfer

hfutils-0.0.2/hfutils/archive/__init__.py

@@ -1,11 +0,0 @@
-"""
-Overview:
-    Archive pack and unpack management.
-
-Supported Formats:
-
-    .. include:: supported_types.demo.py.txt
-"""
-from .base import register_archive_type, archive_pack, archive_unpack, get_archive_type, get_archive_extname
-from .tar import _tarfile_pack, _tarfile_unpack
-from .zip import _zip_pack, _zip_unpack