hfutils 0.2.6__tar.gz → 0.2.8__tar.gz

{hfutils-0.2.6/hfutils.egg-info → hfutils-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.6
+Version: 0.2.8
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -20,8 +20,8 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hbutils>=0.9.0
-Requires-Dist: huggingface_hub>=0.22
+Requires-Dist: hbutils>=0.9.3
+Requires-Dist: huggingface_hub>=0.23
 Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: click>=7
@@ -29,6 +29,7 @@ Requires-Dist: tzlocal
 Requires-Dist: natsort
 Provides-Extra: build
 Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
+Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"

{hfutils-0.2.6 → hfutils-0.2.8}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.2.6'
+__VERSION__ = '0.2.8'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.2.6 → hfutils-0.2.8}/hfutils/index/__init__.py

@@ -1,4 +1,4 @@
 from .fetch import hf_tar_list_files, hf_tar_file_download, hf_tar_get_index, hf_tar_file_exists, \
-    ArchiveStandaloneFileIncompleteDownload, ArchiveStandaloneFileHashNotMatch
+    ArchiveStandaloneFileIncompleteDownload, ArchiveStandaloneFileHashNotMatch, hf_tar_file_size, hf_tar_file_info
 from .make import tar_create_index, hf_tar_create_index, tar_get_index_info, hf_tar_create_from_directory
 from .validate import hf_tar_item_validate, hf_tar_validate

{hfutils-0.2.6 → hfutils-0.2.8}/hfutils/index/fetch.py

@@ -49,6 +49,36 @@ def hf_tar_get_index(repo_id: str, archive_in_repo: str,
     :type hf_token: str, optional
     :return: The index of the tar archive file.
     :rtype: Dict
+
+    Examples::
+        >>> from hfutils.index import hf_tar_get_index
+        >>>
+        >>> idx = hf_tar_get_index(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ... )
+        >>> idx.keys()
+        dict_keys(['filesize', 'hash', 'hash_lfs', 'files'])
+        >>> idx['files'].keys()
+        dict_keys(['7507000.jpg', '7506000.jpg', '7505000.jpg', ...])
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        get the index information by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_get_index
+        >>>
+        >>> idx = hf_tar_get_index(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ... )
+        >>> idx.keys()
+        dict_keys(['filesize', 'hash', 'hash_lfs', 'files'])
+        >>> idx['files'].keys()
+        dict_keys(['./1000.png', './10000.jpg', './100000.jpg', ...])
+
     """
     hf_client = get_hf_client(hf_token)
     body, _ = os.path.splitext(archive_in_repo)
@@ -90,6 +120,30 @@ def hf_tar_list_files(repo_id: str, archive_in_repo: str,
     :type hf_token: str, optional
     :return: The list of files inside the tar archive.
     :rtype: List[str]
+
+    Examples::
+        >>> from hfutils.index import hf_tar_list_files
+        >>>
+        >>> hf_tar_list_files(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ... )
+        ['7507000.jpg', '7506000.jpg', '7505000.jpg', ...]
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        list all the files by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_list_files
+        >>>
+        >>> hf_tar_list_files(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ... )
+        ['./1000.png', './10000.jpg', './100000.jpg', ...]
+
     """
     index_data = hf_tar_get_index(
         repo_id=repo_id,
@@ -138,6 +192,45 @@ def hf_tar_file_exists(repo_id: str, archive_in_repo: str, file_in_archive: str,
     :type hf_token: str, optional
     :return: True if the file exists, False otherwise.
     :rtype: bool
+
+    Examples::
+        >>> from hfutils.index import hf_tar_file_exists
+        >>>
+        >>> hf_tar_file_exists(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ...     file_in_archive='7506000.jpg',
+        ... )
+        True
+        >>> hf_tar_file_exists(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ...     file_in_archive='17506000.jpg',
+        ... )
+        False
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        check the file existence by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_file_exists
+        >>>
+        >>> hf_tar_file_exists(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ...     file_in_archive='1000.png'
+        ... )
+        True
+        >>> hf_tar_file_exists(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ...     file_in_archive='10000000001000.png'
+        ... )
+        False
+
     """
     index = hf_tar_get_index(
         repo_id=repo_id,
@@ -180,6 +273,155 @@ def _hf_files_process(files: Dict[str, dict]):
     return {_n_path(key): value for key, value in files.items()}
 
 
+def hf_tar_file_info(repo_id: str, archive_in_repo: str, file_in_archive: str,
+                     repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                     idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                     idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                     hf_token: Optional[str] = None) -> dict:
+    """
+    Get a file's detailed information in index tars, including offset, sha256 and size.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param file_in_archive: The path to the file inside the archive.
+    :type file_in_archive: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    :return: Return a dictionary object with meta information of this file.
+    :rtype: dict
+    :raises FileNotFoundError: Raise this when file not exist in tar archive.
+
+    Examples::
+        >>> from hfutils.index import hf_tar_file_info
+        >>>
+        >>> hf_tar_file_info(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ...     file_in_archive='7506000.jpg',
+        ... )
+        {'offset': 265728, 'size': 435671, 'sha256': 'ef6a4e031fdffb705c8ce2c64e8cb8d993f431a887d7c1c0b1e6fa56e6107fcd'}
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        get the file information by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_file_info
+        >>>
+        >>> hf_tar_file_info(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ...     file_in_archive='1000.png'
+        ... )
+        {'offset': 1024, 'size': 11966, 'sha256': '478d3313860519372f6a75ede287d4a7c18a2d851bbc79b3dd65caff4c716858'}
+    """
+    index = hf_tar_get_index(
+        repo_id=repo_id,
+        archive_in_repo=archive_in_repo,
+        repo_type=repo_type,
+        revision=revision,
+
+        idx_repo_id=idx_repo_id,
+        idx_file_in_repo=idx_file_in_repo,
+        idx_repo_type=idx_repo_type,
+        idx_revision=idx_revision,
+
+        hf_token=hf_token,
+    )
+    files = _hf_files_process(index['files'])
+    if _n_path(file_in_archive) not in files:
+        raise FileNotFoundError(f'File {file_in_archive!r} not found '
+                                f'in {repo_type}s/{repo_id}@{revision}/{archive_in_repo}.')
+    else:
+        return files[_n_path(file_in_archive)]
+
+
+def hf_tar_file_size(repo_id: str, archive_in_repo: str, file_in_archive: str,
+                     repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                     idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                     idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                     hf_token: Optional[str] = None) -> int:
+    """
+    Get a file's size in index tars.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param file_in_archive: The path to the file inside the archive.
+    :type file_in_archive: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    :return: Return an integer which represents the size of this file.
+    :rtype: int
+    :raises FileNotFoundError: Raise this when file not exist in tar archive.
+
+    Examples::
+        >>> from hfutils.index import hf_tar_file_size
+        >>>
+        >>> hf_tar_file_size(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ...     file_in_archive='7506000.jpg',
+        ... )
+        435671
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        get the file size by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_file_size
+        >>>
+        >>> hf_tar_file_size(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ...     file_in_archive='1000.png'
+        ... )
+        11966
+    """
+    return hf_tar_file_info(
+        repo_id=repo_id,
+        archive_in_repo=archive_in_repo,
+        file_in_archive=file_in_archive,
+        repo_type=repo_type,
+        revision=revision,
+        idx_repo_id=idx_repo_id,
+        idx_file_in_repo=idx_file_in_repo,
+        idx_repo_type=idx_repo_type,
+        idx_revision=idx_revision,
+        hf_token=hf_token
+    )['size']
+
+
 def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: str, local_file: str,
                          repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
                          idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
@@ -220,6 +462,34 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
     :type endpoint: str, optional
     :param hf_token: The Hugging Face access token.
     :type hf_token: str, optional
+    :raises FileNotFoundError: Raise this when file not exist in tar archive.
+    :raises ArchiveStandaloneFileIncompleteDownload: Raise when download incomplete.
+    :raises ArchiveStandaloneFileHashNotMatch: Raise when download hash not match.
+
+    Examples::
+        >>> from hfutils.index import hf_tar_file_download
+        >>>
+        >>> hf_tar_file_download(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ...     file_in_archive='7506000.jpg',
+        ...     local_file='test_example.jpg'  # download destination
+        ... )
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        download the given file by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_file_download
+        >>>
+        >>> hf_tar_file_download(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ...     file_in_archive='1000.png',
+        ...     local_file='test_example.png'  # download destination
+        ... )
     """
     index = hf_tar_get_index(
         repo_id=repo_id,

{hfutils-0.2.6 → hfutils-0.2.8}/hfutils/index/make.py

@@ -16,6 +16,9 @@ def tar_get_index_info(src_tar_file, chunk_for_hash: int = 1 << 20, with_hash: b
     """
     Get the index information of a tar archive file.
 
+    .. note::
+        The return value of this function will be directly used as the index json file.
+
     :param src_tar_file: The path to the source tar archive file.
     :type src_tar_file: str
     :param chunk_for_hash: The chunk size for hashing, defaults to 1 << 20 (1 MB).

{hfutils-0.2.6 → hfutils-0.2.8}/hfutils/index/validate.py

@@ -48,6 +48,9 @@ def hf_tar_validate(repo_id: str, archive_in_repo: str, repo_type: RepoTypeTypin
 
     This function validates if the tar archive in the Hugging Face repository matches the expected size and hash.
 
+    .. note::
+        This function is based on Huggingface API and hash information in index files, no tar file will be downloaded.
+
     :param repo_id: The ID of the Hugging Face repository.
     :type repo_id: str
     :param archive_in_repo: The path to the tar archive in the repository.
@@ -70,6 +73,13 @@ def hf_tar_validate(repo_id: str, archive_in_repo: str, repo_type: RepoTypeTypin
     :raises IsADirectoryError: If the specified entry is a directory.
     :return: True if the tar archive is valid, False otherwise.
     :rtype: bool
+
+    .. note::
+
+        If this function returns `False`, it means the json index is expired and need to be re-generated.
+
+        So this function and :func:`hfutils.index.make.hf_tar_create_index` can be used together to gracefully
+        refresh an indexed tar dataset.
     """
     hf_client = get_hf_client(hf_token)
 

{hfutils-0.2.6 → hfutils-0.2.8}/hfutils/operate/download.py

@@ -44,8 +44,6 @@ def download_file_to_file(local_file: str, repo_id: str, file_in_repo: str,
                 filename=hf_normpath(file_in_repo),
                 revision=revision,
                 local_dir=td,
-                force_download=True,
-                local_dir_use_symlinks=False,
                 resume_download=resume_download,
             )
         finally:

hfutils-0.2.8/hfutils/utils/irregular_repo.json

@@ -0,0 +1,5 @@
+{
+    "datasets": [],
+    "models": [],
+    "spaces": []
+}

{hfutils-0.2.6 → hfutils-0.2.8/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.6
+Version: 0.2.8
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -20,8 +20,8 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hbutils>=0.9.0
-Requires-Dist: huggingface_hub>=0.22
+Requires-Dist: hbutils>=0.9.3
+Requires-Dist: huggingface_hub>=0.23
 Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: click>=7
@@ -29,6 +29,7 @@ Requires-Dist: tzlocal
 Requires-Dist: natsort
 Provides-Extra: build
 Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
+Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"

{hfutils-0.2.6 → hfutils-0.2.8}/hfutils.egg-info/requires.txt

@@ -1,5 +1,5 @@
-hbutils>=0.9.0
-huggingface_hub>=0.22
+hbutils>=0.9.3
+huggingface_hub>=0.23
 tqdm
 requests
 click>=7
@@ -11,6 +11,7 @@ py7zr
 
 [build]
 pyinstaller<5,>=4.7
+setuptools<70
 
 [doc]
 Jinja2>=3.0.0

hfutils-0.2.8/requirements-build.txt

@@ -0,0 +1,2 @@
+pyinstaller>=4.7,<5
+setuptools<70

hfutils-0.2.8/requirements.txt

@@ -0,0 +1,7 @@
+hbutils>=0.9.3
+huggingface_hub>=0.23
+tqdm
+requests
+click>=7
+tzlocal
+natsort

hfutils-0.2.6/hfutils/utils/irregular_repo.json

@@ -1,662 +0,0 @@
-{
-    "datasets": [
-        "acronym_identification",
-        "ade_corpus_v2",
-        "adv_glue",
-        "aeslc",
-        "afrikaans_ner_corpus",
-        "ag_news",
-        "air_dialogue",
-        "ajgt_twitter_ar",
-        "allegro_reviews",
-        "allocine",
-        "alt",
-        "amazon_polarity",
-        "amazon_reviews_multi",
-        "amazon_us_reviews",
-        "ambig_qa",
-        "ami",
-        "amttl",
-        "app_reviews",
-        "aqua_rat",
-        "aquamuse",
-        "ar_res_reviews",
-        "ar_sarcasm",
-        "arabic_billion_words",
-        "arabic_pos_dialect",
-        "arabic_speech_corpus",
-        "arcd",
-        "arsentd_lev",
-        "art",
-        "arxiv_dataset",
-        "ascent_kb",
-        "aslg_pc12",
-        "asnq",
-        "assin",
-        "assin2",
-        "atomic",
-        "autshumato",
-        "banking77",
-        "bbaw_egyptian",
-        "bbc_hindi_nli",
-        "bc2gm_corpus",
-        "best2009",
-        "bible_para",
-        "big_patent",
-        "bigbench",
-        "billsum",
-        "bing_coronavirus_query_set",
-        "biomrc",
-        "biosses",
-        "biwi_kinect_head_pose",
-        "blended_skill_talk",
-        "blog_authorship_corpus",
-        "bn_hate_speech",
-        "bnl_newspapers",
-        "bookcorpus",
-        "bookcorpusopen",
-        "bprec",
-        "break_data",
-        "brwac",
-        "bsd_ja_en",
-        "bswac",
-        "c3",
-        "c4",
-        "cail2018",
-        "caner",
-        "casino",
-        "catalonia_independence",
-        "cats_vs_dogs",
-        "cawac",
-        "cbt",
-        "cc100",
-        "ccaligned_multilingual",
-        "cdsc",
-        "cdt",
-        "cedr",
-        "cfq",
-        "chr_en",
-        "cifar10",
-        "cifar100",
-        "circa",
-        "clickbait_news_bg",
-        "climate_fever",
-        "clinc_oos",
-        "clue",
-        "cmrc2018",
-        "cmu_hinglish_dog",
-        "cnn_dailymail",
-        "coached_conv_pref",
-        "coarse_discourse",
-        "codah",
-        "code_search_net",
-        "code_x_glue_cc_clone_detection_big_clone_bench",
-        "code_x_glue_cc_clone_detection_poj104",
-        "code_x_glue_cc_cloze_testing_all",
-        "code_x_glue_cc_cloze_testing_maxmin",
-        "code_x_glue_cc_code_completion_line",
-        "code_x_glue_cc_code_completion_token",
-        "code_x_glue_cc_code_refinement",
-        "code_x_glue_cc_code_to_code_trans",
-        "code_x_glue_cc_defect_detection",
-        "code_x_glue_ct_code_to_text",
-        "code_x_glue_tc_nl_code_search_adv",
-        "code_x_glue_tc_text_to_code",
-        "code_x_glue_tt_text_to_text",
-        "com_qa",
-        "common_language",
-        "common_voice",
-        "compguesswhat",
-        "conceptnet5",
-        "conceptual_12m",
-        "conceptual_captions",
-        "conll2000",
-        "conll2002",
-        "conll2003",
-        "conll2012_ontonotesv5",
-        "conllpp",
-        "consumer-finance-complaints",
-        "conv_ai",
-        "conv_ai_2",
-        "conv_ai_3",
-        "conv_questions",
-        "cornell_movie_dialog",
-        "cos_e",
-        "cosmos_qa",
-        "counter",
-        "covid_qa_castorini",
-        "covid_qa_deepset",
-        "covid_qa_ucsd",
-        "covid_tweets_japanese",
-        "covost2",
-        "cppe-5",
-        "craigslist_bargains",
-        "crawl_domain",
-        "crd3",
-        "crime_and_punish",
-        "crows_pairs",
-        "cryptonite",
-        "cs_restaurants",
-        "cuad",
-        "curiosity_dialogs",
-        "daily_dialog",
-        "dane",
-        "danish_political_comments",
-        "dart",
-        "datacommons_factcheck",
-        "dbrd",
-        "deal_or_no_dialog",
-        "definite_pronoun_resolution",
-        "dengue_filipino",
-        "dialog_re",
-        "diplomacy_detection",
-        "disaster_response_messages",
-        "discofuse",
-        "discovery",
-        "disfl_qa",
-        "doc2dial",
-        "docred",
-        "doqa",
-        "dream",
-        "dutch_social",
-        "dyk",
-        "e2e_nlg",
-        "e2e_nlg_cleaned",
-        "ecb",
-        "ecthr_cases",
-        "eduge",
-        "ehealth_kd",
-        "electricity_load_diagrams",
-        "eli5",
-        "eli5_category",
-        "elkarhizketak",
-        "emea",
-        "emo",
-        "emotone_ar",
-        "empathetic_dialogues",
-        "enriched_web_nlg",
-        "enwik8",
-        "eraser_multi_rc",
-        "esnli",
-        "eth_py150_open",
-        "ethos",
-        "ett",
-        "eu_regulatory_ir",
-        "eurlex",
-        "euronews",
-        "europa_eac_tm",
-        "europa_ecdc_tm",
-        "event2Mind",
-        "evidence_infer_treatment",
-        "factckbr",
-        "fake_news_english",
-        "fake_news_filipino",
-        "farsi_news",
-        "fashion_mnist",
-        "fever",
-        "few_rel",
-        "financial_phrasebank",
-        "finer",
-        "flores",
-        "flue",
-        "food101",
-        "fquad",
-        "freebase_qa",
-        "gap",
-        "gem",
-        "generated_reviews_enth",
-        "generics_kb",
-        "german_legal_entity_recognition",
-        "germaner",
-        "germeval_14",
-        "giga_fren",
-        "gigaword",
-        "glucose",
-        "gnad10",
-        "go_emotions",
-        "gooaq",
-        "google_wellformed_query",
-        "grail_qa",
-        "great_code",
-        "gsm8k",
-        "guardian_authorship",
-        "gutenberg_time",
-        "hans",
-        "hansards",
-        "hard",
-        "harem",
-        "has_part",
-        "hate_offensive",
-        "hate_speech18",
-        "hate_speech_filipino",
-        "hate_speech_pl",
-        "hate_speech_portuguese",
-        "hatexplain",
-        "hausa_voa_ner",
-        "hausa_voa_topics",
-        "hda_nli_hindi",
-        "head_qa",
-        "health_fact",
-        "hebrew_projectbenyehuda",
-        "hebrew_sentiment",
-        "hebrew_this_world",
-        "hind_encorp",
-        "hindi_discourse",
-        "hippocorpus",
-        "hkcancor",
-        "hlgd",
-        "hope_edi",
-        "hotpot_qa",
-        "hover",
-        "hrenwac_para",
-        "hrwac",
-        "humicroedit",
-        "hybrid_qa",
-        "hyperpartisan_news_detection",
-        "iapp_wiki_qa_squad",
-        "id_clickbait",
-        "id_liputan6",
-        "id_nergrit_corpus",
-        "id_newspapers_2018",
-        "id_panl_bppt",
-        "id_puisi",
-        "igbo_english_machine_translation",
-        "igbo_monolingual",
-        "igbo_ner",
-        "ilist",
-        "imagenet-1k",
-        "imagenet_sketch",
-        "imdb_urdu_reviews",
-        "imppres",
-        "indic_glue",
-        "indonli",
-        "inquisitive_qg",
-        "interpress_news_category_tr",
-        "interpress_news_category_tr_lite",
-        "irc_disentangle",
-        "isixhosa_ner_corpus",
-        "isizulu_ner_corpus",
-        "iwslt2017",
-        "jeopardy",
-        "jnlpba",
-        "journalists_questions",
-        "kan_hope",
-        "kannada_news",
-        "kd_conv",
-        "kde4",
-        "kelm",
-        "kilt_tasks",
-        "kilt_wikipedia",
-        "kinnews_kirnews",
-        "klue",
-        "kor_3i4k",
-        "kor_hate",
-        "kor_ner",
-        "kor_nli",
-        "kor_nlu",
-        "kor_qpair",
-        "kor_sae",
-        "kor_sarcasm",
-        "labr",
-        "lama",
-        "large_spanish_corpus",
-        "laroseda",
-        "lc_quad",
-        "lccc",
-        "lener_br",
-        "liar",
-        "librispeech_asr",
-        "librispeech_lm",
-        "limit",
-        "lince",
-        "linnaeus",
-        "liveqa",
-        "lj_speech",
-        "lm1b",
-        "lst20",
-        "m_lama",
-        "mac_morpho",
-        "makhzan",
-        "masakhaner",
-        "math_dataset",
-        "math_qa",
-        "matinf",
-        "mbpp",
-        "mc4",
-        "mc_taco",
-        "md_gender_bias",
-        "mdd",
-        "med_hop",
-        "medal",
-        "medical_dialog",
-        "medical_questions_pairs",
-        "menyo20k_mt",
-        "meta_woz",
-        "metashift",
-        "metooma",
-        "metrec",
-        "miam",
-        "mkb",
-        "mkqa",
-        "mlqa",
-        "mlsum",
-        "mnist",
-        "mocha",
-        "monash_tsf",
-        "moroco",
-        "movie_rationales",
-        "mrqa",
-        "ms_marco",
-        "ms_terms",
-        "msr_genomics_kbcomp",
-        "msr_sqa",
-        "msr_text_compression",
-        "msr_zhen_translation_parity",
-        "msra_ner",
-        "mt_eng_vietnamese",
-        "muchocine",
-        "multi_booked",
-        "multi_news",
-        "multi_nli_mismatch",
-        "multi_para_crawl",
-        "multi_re_qa",
-        "multi_woz_v22",
-        "multi_x_science_sum",
-        "multidoc2dial",
-        "multilingual_librispeech",
-        "mutual_friends",
-        "mwsc",
-        "myanmar_news",
-        "narrativeqa",
-        "narrativeqa_manual",
-        "natural_questions",
-        "ncbi_disease",
-        "nchlt",
-        "ncslgr",
-        "nell",
-        "neural_code_search",
-        "newsgroup",
-        "newsph",
-        "newsph_nli",
-        "newspop",
-        "newsqa",
-        "newsroom",
-        "nkjp-ner",
-        "nli_tr",
-        "nlu_evaluation_data",
-        "norec",
-        "norne",
-        "norwegian_ner",
-        "nq_open",
-        "nsmc",
-        "numer_sense",
-        "numeric_fused_head",
-        "oclar",
-        "offcombr",
-        "offenseval2020_tr",
-        "offenseval_dravidian",
-        "ofis_publik",
-        "ohsumed",
-        "ollie",
-        "omp",
-        "onestop_english",
-        "onestop_qa",
-        "open_subtitles",
-        "openai_humaneval",
-        "openslr",
-        "opinosis",
-        "orange_sum",
-        "oscar",
-        "para_crawl",
-        "para_pat",
-        "parsinlu_reading_comprehension",
-        "pass",
-        "paws",
-        "paws-x",
-        "pec",
-        "peoples_daily_ner",
-        "per_sent",
-        "persian_ner",
-        "pg19",
-        "php",
-        "pib",
-        "piqa",
-        "pn_summary",
-        "poem_sentiment",
-        "polemo2",
-        "poleval2019_cyberbullying",
-        "poleval2019_mt",
-        "polsum",
-        "polyglot_ner",
-        "prachathai67k",
-        "pragmeval",
-        "proto_qa",
-        "psc",
-        "ptb_text_only",
-        "pubmed",
-        "py_ast",
-        "qa4mre",
-        "qa_srl",
-        "qa_zre",
-        "qangaroo",
-        "qanta",
-        "qed",
-        "qed_amara",
-        "quac",
-        "quail",
-        "quarel",
-        "quickdraw",
-        "quora",
-        "quoref",
-        "re_dial",
-        "reasoning_bg",
-        "recipe_nlg",
-        "reclor",
-        "red_caps",
-        "reddit_tifu",
-        "refresd",
-        "reuters21578",
-        "riddle_sense",
-        "ro_sent",
-        "ro_sts",
-        "ro_sts_parallel",
-        "roman_urdu",
-        "roman_urdu_hate_speech",
-        "ronec",
-        "rotten_tomatoes",
-        "samsum",
-        "sanskrit_classic",
-        "saudinewsnet",
-        "sberquad",
-        "sbu_captions",
-        "scan",
-        "scb_mt_enth_2020",
-        "scene_parse_150",
-        "schema_guided_dstc8",
-        "scielo",
-        "scientific_papers",
-        "search_qa",
-        "sede",
-        "selqa",
-        "sem_eval_2010_task_8",
-        "sem_eval_2014_task_1",
-        "sem_eval_2018_task_1",
-        "sem_eval_2020_task_11",
-        "sent_comp",
-        "senti_lex",
-        "senti_ws",
-        "sentiment140",
-        "sepedi_ner",
-        "sesotho_ner_corpus",
-        "setimes",
-        "setswana_ner_corpus",
-        "sharc_modified",
-        "sick",
-        "silicone",
-        "simple_questions_v2",
-        "siswati_ner_corpus",
-        "smartdata",
-        "sms_spam",
-        "snips_built_in_intents",
-        "snow_simplified_japanese_corpus",
-        "so_stacksample",
-        "social_bias_frames",
-        "social_i_qa",
-        "sofc_materials_articles",
-        "sogou_news",
-        "spanish_billion_words",
-        "spc",
-        "species_800",
-        "speech_commands",
-        "squad_adversarial",
-        "squad_es",
-        "squad_it",
-        "squad_kor_v1",
-        "squad_kor_v2",
-        "squad_v1_pt",
-        "squadshifts",
-        "srwac",
-        "sst",
-        "story_cloze",
-        "stsb_mt_sv",
-        "style_change_detection",
-        "subjqa",
-        "super_glue",
-        "superb",
-        "svhn",
-        "swag",
-        "swahili",
-        "swahili_news",
-        "swda",
-        "swedish_medical_ner",
-        "swedish_ner_corpus",
-        "swedish_reviews",
-        "tab_fact",
-        "tamilmixsentiment",
-        "tanzil",
-        "tapaco",
-        "tashkeela",
-        "taskmaster1",
-        "taskmaster2",
-        "taskmaster3",
-        "tatoeba",
-        "ted_hrlr",
-        "ted_iwlst2013",
-        "ted_multi",
-        "ted_talks_iwslt",
-        "telugu_books",
-        "telugu_news",
-        "tep_en_fa_para",
-        "text2log",
-        "textvqa",
-        "thai_toxicity_tweet",
-        "thainer",
-        "thaiqa_squad",
-        "thaisum",
-        "the_pile_books3",
-        "the_pile_openwebtext2",
-        "the_pile_stack_exchange",
-        "tilde_model",
-        "time_dial",
-        "times_of_india_news_headlines",
-        "timit_asr",
-        "tlc",
-        "tmu_gfm_dataset",
-        "tne",
-        "told-br",
-        "totto",
-        "trec",
-        "truthful_qa",
-        "tsac",
-        "ttc4900",
-        "tunizi",
-        "tuple_ie",
-        "turk",
-        "turkic_xwmt",
-        "turkish_movie_sentiment",
-        "turkish_ner",
-        "turkish_product_reviews",
-        "turkish_shrinked_ner",
-        "turku_ner_corpus",
-        "tweet_eval",
-        "tweet_qa",
-        "tweets_ar_en_parallel",
-        "tweets_hate_speech_detection",
-        "twi_text_c3",
-        "twi_wordsim353",
-        "tydiqa",
-        "ubuntu_dialogs_corpus",
-        "udhr",
-        "um005",
-        "universal_dependencies",
-        "universal_morphologies",
-        "urdu_fake_news",
-        "urdu_sentiment_corpus",
-        "vctk",
-        "visual_genome",
-        "vivos",
-        "web_nlg",
-        "web_of_science",
-        "web_questions",
-        "weibo_ner",
-        "wi_locness",
-        "wider_face",
-        "wiki40b",
-        "wiki_asp",
-        "wiki_atomic_edits",
-        "wiki_auto",
-        "wiki_bio",
-        "wiki_dpr",
-        "wiki_hop",
-        "wiki_lingua",
-        "wiki_movies",
-        "wiki_qa",
-        "wiki_qa_ar",
-        "wiki_snippets",
-        "wiki_source",
-        "wiki_split",
-        "wiki_summary",
-        "wikiann",
-        "wikicorpus",
-        "wikihow",
-        "wikipedia",
-        "wikisql",
-        "wikitablequestions",
-        "wikitext",
-        "wikitext_tl39",
-        "wili_2018",
-        "wino_bias",
-        "winograd_wsc",
-        "winogrande",
-        "wiqa",
-        "wisesight1000",
-        "wisesight_sentiment",
-        "wnut_17",
-        "wongnai_reviews",
-        "woz_dialogue",
-        "wrbsc",
-        "x_stance",
-        "xcopa",
-        "xcsr",
-        "xed_en_fi",
-        "xglue",
-        "xnli",
-        "xor_tydi_qa",
-        "xquad",
-        "xquad_r",
-        "xsum_factuality",
-        "xtreme",
-        "yahoo_answers_qa",
-        "yahoo_answers_topics",
-        "yelp_polarity",
-        "yelp_review_full",
-        "yoruba_bbc_topics",
-        "yoruba_gv_ner",
-        "yoruba_text_c3",
-        "yoruba_wordsim353",
-        "youtube_caption_corrections",
-        "zest"
-    ],
-    "models": [],
-    "spaces": []
-}

hfutils-0.2.6/requirements-build.txt

@@ -1 +0,0 @@
-pyinstaller>=4.7,<5

hfutils-0.2.6/requirements.txt

@@ -1,7 +0,0 @@
-hbutils>=0.9.0
-huggingface_hub>=0.22
-tqdm
-requests
-click>=7
-tzlocal
-natsort