hfutils 0.2.7__tar.gz → 0.2.8__tar.gz

{hfutils-0.2.7/hfutils.egg-info → hfutils-0.2.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.7
+Version: 0.2.8
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -20,7 +20,7 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hbutils>=0.9.0
+Requires-Dist: hbutils>=0.9.3
 Requires-Dist: huggingface_hub>=0.23
 Requires-Dist: tqdm
 Requires-Dist: requests

{hfutils-0.2.7 → hfutils-0.2.8}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.2.7'
+__VERSION__ = '0.2.8'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.2.7 → hfutils-0.2.8}/hfutils/index/__init__.py

@@ -1,4 +1,4 @@
 from .fetch import hf_tar_list_files, hf_tar_file_download, hf_tar_get_index, hf_tar_file_exists, \
-    ArchiveStandaloneFileIncompleteDownload, ArchiveStandaloneFileHashNotMatch
+    ArchiveStandaloneFileIncompleteDownload, ArchiveStandaloneFileHashNotMatch, hf_tar_file_size, hf_tar_file_info
 from .make import tar_create_index, hf_tar_create_index, tar_get_index_info, hf_tar_create_from_directory
 from .validate import hf_tar_item_validate, hf_tar_validate

{hfutils-0.2.7 → hfutils-0.2.8}/hfutils/index/fetch.py

@@ -49,6 +49,36 @@ def hf_tar_get_index(repo_id: str, archive_in_repo: str,
     :type hf_token: str, optional
     :return: The index of the tar archive file.
     :rtype: Dict
+
+    Examples::
+        >>> from hfutils.index import hf_tar_get_index
+        >>>
+        >>> idx = hf_tar_get_index(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ... )
+        >>> idx.keys()
+        dict_keys(['filesize', 'hash', 'hash_lfs', 'files'])
+        >>> idx['files'].keys()
+        dict_keys(['7507000.jpg', '7506000.jpg', '7505000.jpg', ...])
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        get the index information by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_get_index
+        >>>
+        >>> idx = hf_tar_get_index(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ... )
+        >>> idx.keys()
+        dict_keys(['filesize', 'hash', 'hash_lfs', 'files'])
+        >>> idx['files'].keys()
+        dict_keys(['./1000.png', './10000.jpg', './100000.jpg', ...])
+
     """
     hf_client = get_hf_client(hf_token)
     body, _ = os.path.splitext(archive_in_repo)
@@ -90,6 +120,30 @@ def hf_tar_list_files(repo_id: str, archive_in_repo: str,
     :type hf_token: str, optional
     :return: The list of files inside the tar archive.
     :rtype: List[str]
+
+    Examples::
+        >>> from hfutils.index import hf_tar_list_files
+        >>>
+        >>> hf_tar_list_files(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ... )
+        ['7507000.jpg', '7506000.jpg', '7505000.jpg', ...]
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        list all the files by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_list_files
+        >>>
+        >>> hf_tar_list_files(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ... )
+        ['./1000.png', './10000.jpg', './100000.jpg', ...]
+
     """
     index_data = hf_tar_get_index(
         repo_id=repo_id,
@@ -138,6 +192,45 @@ def hf_tar_file_exists(repo_id: str, archive_in_repo: str, file_in_archive: str,
     :type hf_token: str, optional
     :return: True if the file exists, False otherwise.
     :rtype: bool
+
+    Examples::
+        >>> from hfutils.index import hf_tar_file_exists
+        >>>
+        >>> hf_tar_file_exists(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ...     file_in_archive='7506000.jpg',
+        ... )
+        True
+        >>> hf_tar_file_exists(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ...     file_in_archive='17506000.jpg',
+        ... )
+        False
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        check the file existence by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_file_exists
+        >>>
+        >>> hf_tar_file_exists(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ...     file_in_archive='1000.png'
+        ... )
+        True
+        >>> hf_tar_file_exists(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ...     file_in_archive='10000000001000.png'
+        ... )
+        False
+
     """
     index = hf_tar_get_index(
         repo_id=repo_id,
@@ -180,6 +273,155 @@ def _hf_files_process(files: Dict[str, dict]):
     return {_n_path(key): value for key, value in files.items()}
 
 
+def hf_tar_file_info(repo_id: str, archive_in_repo: str, file_in_archive: str,
+                     repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                     idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                     idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                     hf_token: Optional[str] = None) -> dict:
+    """
+    Get a file's detailed information in index tars, including offset, sha256 and size.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param file_in_archive: The path to the file inside the archive.
+    :type file_in_archive: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    :return: Return a dictionary object with meta information of this file.
+    :rtype: dict
+    :raises FileNotFoundError: Raise this when file not exist in tar archive.
+
+    Examples::
+        >>> from hfutils.index import hf_tar_file_info
+        >>>
+        >>> hf_tar_file_info(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ...     file_in_archive='7506000.jpg',
+        ... )
+        {'offset': 265728, 'size': 435671, 'sha256': 'ef6a4e031fdffb705c8ce2c64e8cb8d993f431a887d7c1c0b1e6fa56e6107fcd'}
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        get the file information by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_file_info
+        >>>
+        >>> hf_tar_file_info(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ...     file_in_archive='1000.png'
+        ... )
+        {'offset': 1024, 'size': 11966, 'sha256': '478d3313860519372f6a75ede287d4a7c18a2d851bbc79b3dd65caff4c716858'}
+    """
+    index = hf_tar_get_index(
+        repo_id=repo_id,
+        archive_in_repo=archive_in_repo,
+        repo_type=repo_type,
+        revision=revision,
+
+        idx_repo_id=idx_repo_id,
+        idx_file_in_repo=idx_file_in_repo,
+        idx_repo_type=idx_repo_type,
+        idx_revision=idx_revision,
+
+        hf_token=hf_token,
+    )
+    files = _hf_files_process(index['files'])
+    if _n_path(file_in_archive) not in files:
+        raise FileNotFoundError(f'File {file_in_archive!r} not found '
+                                f'in {repo_type}s/{repo_id}@{revision}/{archive_in_repo}.')
+    else:
+        return files[_n_path(file_in_archive)]
+
+
+def hf_tar_file_size(repo_id: str, archive_in_repo: str, file_in_archive: str,
+                     repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
+                     idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
+                     idx_repo_type: Optional[RepoTypeTyping] = None, idx_revision: Optional[str] = None,
+                     hf_token: Optional[str] = None) -> int:
+    """
+    Get a file's size in index tars.
+
+    :param repo_id: The identifier of the repository.
+    :type repo_id: str
+    :param archive_in_repo: The path to the archive file in the repository.
+    :type archive_in_repo: str
+    :param file_in_archive: The path to the file inside the archive.
+    :type file_in_archive: str
+    :param repo_type: The type of the Hugging Face repository.
+    :type repo_type: RepoTypeTyping, optional
+    :param revision: The revision of the repository.
+    :type revision: str, optional
+    :param idx_repo_id: The identifier of the index repository.
+    :type idx_repo_id: str, optional
+    :param idx_file_in_repo: The path to the index file in the index repository.
+    :type idx_file_in_repo: str, optional
+    :param idx_repo_type: The type of the index repository.
+    :type idx_repo_type: RepoTypeTyping, optional
+    :param idx_revision: The revision of the index repository.
+    :type idx_revision: str, optional
+    :param hf_token: The Hugging Face access token.
+    :type hf_token: str, optional
+    :return: Return an integer which represents the size of this file.
+    :rtype: int
+    :raises FileNotFoundError: Raise this when file not exist in tar archive.
+
+    Examples::
+        >>> from hfutils.index import hf_tar_file_size
+        >>>
+        >>> hf_tar_file_size(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ...     file_in_archive='7506000.jpg',
+        ... )
+        435671
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        get the file size by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_file_size
+        >>>
+        >>> hf_tar_file_size(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ...     file_in_archive='1000.png'
+        ... )
+        11966
+    """
+    return hf_tar_file_info(
+        repo_id=repo_id,
+        archive_in_repo=archive_in_repo,
+        file_in_archive=file_in_archive,
+        repo_type=repo_type,
+        revision=revision,
+        idx_repo_id=idx_repo_id,
+        idx_file_in_repo=idx_file_in_repo,
+        idx_repo_type=idx_repo_type,
+        idx_revision=idx_revision,
+        hf_token=hf_token
+    )['size']
+
+
 def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: str, local_file: str,
                          repo_type: RepoTypeTyping = 'dataset', revision: str = 'main',
                          idx_repo_id: Optional[str] = None, idx_file_in_repo: Optional[str] = None,
@@ -220,6 +462,34 @@ def hf_tar_file_download(repo_id: str, archive_in_repo: str, file_in_archive: st
     :type endpoint: str, optional
     :param hf_token: The Hugging Face access token.
     :type hf_token: str, optional
+    :raises FileNotFoundError: Raise this when file not exist in tar archive.
+    :raises ArchiveStandaloneFileIncompleteDownload: Raise when download incomplete.
+    :raises ArchiveStandaloneFileHashNotMatch: Raise when download hash not match.
+
+    Examples::
+        >>> from hfutils.index import hf_tar_file_download
+        >>>
+        >>> hf_tar_file_download(
+        ...     repo_id='deepghs/danbooru_newest',
+        ...     archive_in_repo='images/0000.tar',
+        ...     file_in_archive='7506000.jpg',
+        ...     local_file='test_example.jpg'  # download destination
+        ... )
+
+    .. note::
+
+        Besides, if the tar and index files are in different repositories, you can also use this function to
+        download the given file by explicitly assigning the ``idx_repo_id`` argument.
+
+        >>> from hfutils.index import hf_tar_file_download
+        >>>
+        >>> hf_tar_file_download(
+        ...     repo_id='nyanko7/danbooru2023',
+        ...     idx_repo_id='deepghs/danbooru2023_index',
+        ...     archive_in_repo='original/data-0000.tar',
+        ...     file_in_archive='1000.png',
+        ...     local_file='test_example.png'  # download destination
+        ... )
     """
     index = hf_tar_get_index(
         repo_id=repo_id,

{hfutils-0.2.7 → hfutils-0.2.8}/hfutils/index/make.py

@@ -16,6 +16,9 @@ def tar_get_index_info(src_tar_file, chunk_for_hash: int = 1 << 20, with_hash: b
     """
     Get the index information of a tar archive file.
 
+    .. note::
+        The return value of this function will be directly used as the index json file.
+
     :param src_tar_file: The path to the source tar archive file.
     :type src_tar_file: str
     :param chunk_for_hash: The chunk size for hashing, defaults to 1 << 20 (1 MB).

{hfutils-0.2.7 → hfutils-0.2.8}/hfutils/index/validate.py

@@ -48,6 +48,9 @@ def hf_tar_validate(repo_id: str, archive_in_repo: str, repo_type: RepoTypeTypin
 
     This function validates if the tar archive in the Hugging Face repository matches the expected size and hash.
 
+    .. note::
+        This function is based on Huggingface API and hash information in index files, no tar file will be downloaded.
+
     :param repo_id: The ID of the Hugging Face repository.
     :type repo_id: str
     :param archive_in_repo: The path to the tar archive in the repository.
@@ -70,6 +73,13 @@ def hf_tar_validate(repo_id: str, archive_in_repo: str, repo_type: RepoTypeTypin
     :raises IsADirectoryError: If the specified entry is a directory.
     :return: True if the tar archive is valid, False otherwise.
     :rtype: bool
+
+    .. note::
+
+        If this function returns `False`, it means the json index is expired and need to be re-generated.
+
+        So this function and :func:`hfutils.index.make.hf_tar_create_index` can be used together to gracefully
+        refresh an indexed tar dataset.
     """
     hf_client = get_hf_client(hf_token)
 

hfutils-0.2.8/hfutils/utils/irregular_repo.json

@@ -0,0 +1,5 @@
+{
+    "datasets": [],
+    "models": [],
+    "spaces": []
+}

{hfutils-0.2.7 → hfutils-0.2.8/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.7
+Version: 0.2.8
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -20,7 +20,7 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hbutils>=0.9.0
+Requires-Dist: hbutils>=0.9.3
 Requires-Dist: huggingface_hub>=0.23
 Requires-Dist: tqdm
 Requires-Dist: requests

{hfutils-0.2.7 → hfutils-0.2.8}/hfutils.egg-info/requires.txt

@@ -1,4 +1,4 @@
-hbutils>=0.9.0
+hbutils>=0.9.3
 huggingface_hub>=0.23
 tqdm
 requests

{hfutils-0.2.7 → hfutils-0.2.8}/requirements.txt

@@ -1,4 +1,4 @@
-hbutils>=0.9.0
+hbutils>=0.9.3
 huggingface_hub>=0.23
 tqdm
 requests

hfutils-0.2.7/hfutils/utils/irregular_repo.json

@@ -1,247 +0,0 @@
-{
-    "datasets": [
-        "ade_corpus_v2",
-        "afrikaans_ner_corpus",
-        "app_reviews",
-        "arabic_billion_words",
-        "arabic_speech_corpus",
-        "arxiv_dataset",
-        "aslg_pc12",
-        "autshumato",
-        "banking77",
-        "bigbench",
-        "billsum",
-        "blended_skill_talk",
-        "bnl_newspapers",
-        "bookcorpus",
-        "bswac",
-        "c3",
-        "c4",
-        "cail2018",
-        "caner",
-        "catalonia_independence",
-        "cawac",
-        "cbt",
-        "cc100",
-        "cdsc",
-        "cifar10",
-        "cifar100",
-        "clickbait_news_bg",
-        "clinc_oos",
-        "clue",
-        "cmu_hinglish_dog",
-        "code_search_net",
-        "conceptnet5",
-        "conll2012_ontonotesv5",
-        "consumer-finance-complaints",
-        "conv_ai",
-        "conv_ai_2",
-        "conv_ai_3",
-        "cornell_movie_dialog",
-        "cos_e",
-        "covid_qa_ucsd",
-        "covid_tweets_japanese",
-        "covost2",
-        "cppe-5",
-        "crime_and_punish",
-        "cs_restaurants",
-        "daily_dialog",
-        "danish_political_comments",
-        "definite_pronoun_resolution",
-        "dialog_re",
-        "diplomacy_detection",
-        "disaster_response_messages",
-        "discofuse",
-        "doqa",
-        "dream",
-        "dutch_social",
-        "dyk",
-        "ehealth_kd",
-        "elkarhizketak",
-        "emo",
-        "emotone_ar",
-        "enwik8",
-        "esnli",
-        "ett",
-        "eu_regulatory_ir",
-        "eurlex",
-        "euronews",
-        "europa_eac_tm",
-        "europa_ecdc_tm",
-        "factckbr",
-        "fake_news_english",
-        "farsi_news",
-        "fashion_mnist",
-        "food101",
-        "generics_kb",
-        "germaner",
-        "germeval_14",
-        "glucose",
-        "gnad10",
-        "go_emotions",
-        "grail_qa",
-        "gsm8k",
-        "gutenberg_time",
-        "hansards",
-        "harem",
-        "has_part",
-        "hate_speech_filipino",
-        "hate_speech_pl",
-        "hate_speech_portuguese",
-        "hebrew_projectbenyehuda",
-        "hebrew_sentiment",
-        "hebrew_this_world",
-        "hope_edi",
-        "hotpot_qa",
-        "hover",
-        "hrenwac_para",
-        "hrwac",
-        "humicroedit",
-        "hyperpartisan_news_detection",
-        "id_clickbait",
-        "id_nergrit_corpus",
-        "id_newspapers_2018",
-        "ilist",
-        "imagenet-1k",
-        "imppres",
-        "indic_glue",
-        "isixhosa_ner_corpus",
-        "isizulu_ner_corpus",
-        "jeopardy",
-        "jnlpba",
-        "journalists_questions",
-        "kilt_tasks",
-        "klue",
-        "kor_ner",
-        "kor_sarcasm",
-        "librispeech_asr",
-        "lince",
-        "liveqa",
-        "lm1b",
-        "makhzan",
-        "matinf",
-        "mbpp",
-        "mc4",
-        "med_hop",
-        "medical_questions_pairs",
-        "movie_rationales",
-        "mrqa",
-        "ms_marco",
-        "muchocine",
-        "narrativeqa",
-        "nchlt",
-        "ncslgr",
-        "nell",
-        "newsqa",
-        "nkjp-ner",
-        "nq_open",
-        "oclar",
-        "offenseval_dravidian",
-        "ohsumed",
-        "onestop_english",
-        "openai_humaneval",
-        "para_crawl",
-        "para_pat",
-        "parsinlu_reading_comprehension",
-        "paws",
-        "paws-x",
-        "peoples_daily_ner",
-        "per_sent",
-        "proto_qa",
-        "psc",
-        "ptb_text_only",
-        "qa4mre",
-        "qa_zre",
-        "qangaroo",
-        "qanta",
-        "quail",
-        "quarel",
-        "quickdraw",
-        "quora",
-        "re_dial",
-        "reclor",
-        "reddit_tifu",
-        "roman_urdu",
-        "roman_urdu_hate_speech",
-        "ronec",
-        "rotten_tomatoes",
-        "scan",
-        "scielo",
-        "sede",
-        "selqa",
-        "sem_eval_2010_task_8",
-        "sem_eval_2014_task_1",
-        "sem_eval_2018_task_1",
-        "sem_eval_2020_task_11",
-        "sent_comp",
-        "senti_lex",
-        "senti_ws",
-        "sepedi_ner",
-        "sesotho_ner_corpus",
-        "setimes",
-        "setswana_ner_corpus",
-        "siswati_ner_corpus",
-        "snips_built_in_intents",
-        "snow_simplified_japanese_corpus",
-        "so_stacksample",
-        "sofc_materials_articles",
-        "sogou_news",
-        "squad_kor_v1",
-        "squad_kor_v2",
-        "srwac",
-        "story_cloze",
-        "style_change_detection",
-        "svhn",
-        "swahili",
-        "swahili_news",
-        "swedish_medical_ner",
-        "tamilmixsentiment",
-        "tapaco",
-        "tashkeela",
-        "telugu_books",
-        "telugu_news",
-        "thai_toxicity_tweet",
-        "times_of_india_news_headlines",
-        "timit_asr",
-        "tmu_gfm_dataset",
-        "truthful_qa",
-        "turk",
-        "turkic_xwmt",
-        "turkish_shrinked_ner",
-        "tweet_eval",
-        "tweet_qa",
-        "tweets_ar_en_parallel",
-        "tweets_hate_speech_detection",
-        "ubuntu_dialogs_corpus",
-        "udhr",
-        "um005",
-        "universal_dependencies",
-        "universal_morphologies",
-        "urdu_fake_news",
-        "urdu_sentiment_corpus",
-        "vctk",
-        "vivos",
-        "web_nlg",
-        "web_of_science",
-        "web_questions",
-        "wi_locness",
-        "wiki40b",
-        "wiki_hop",
-        "wiki_lingua",
-        "wiki_qa",
-        "wiki_snippets",
-        "wikipedia",
-        "wikitext",
-        "wikitext_tl39",
-        "wongnai_reviews",
-        "xnli",
-        "xquad_r",
-        "yahoo_answers_qa",
-        "yahoo_answers_topics",
-        "yelp_review_full",
-        "yoruba_bbc_topics",
-        "youtube_caption_corrections"
-    ],
-    "models": [],
-    "spaces": []
-}