hfutils 0.2.5__tar.gz → 0.2.7__tar.gz

{hfutils-0.2.5/hfutils.egg-info → hfutils-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.5
+Version: 0.2.7
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -21,31 +21,15 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hbutils>=0.9.0
-Requires-Dist: huggingface_hub>=0.22
+Requires-Dist: huggingface_hub>=0.23
 Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: click>=7
 Requires-Dist: tzlocal
 Requires-Dist: natsort
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
+Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -66,12 +50,29 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Provides-Extra: 7z
 Requires-Dist: py7zr; extra == "7z"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 
 # hfutils
 

hfutils-0.2.7/hfutils/__main__.py

@@ -0,0 +1,4 @@
+from .entry import hfutilscli
+
+if __name__ == '__main__':
+    hfutilscli()

{hfutils-0.2.5 → hfutils-0.2.7}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.2.5'
+__VERSION__ = '0.2.7'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.2.5 → hfutils-0.2.7}/hfutils/entry/cli.py

@@ -1,6 +1,7 @@
 from .dispatch import hfutilcli
 from .download import _add_download_subcommand
 from .ls import _add_ls_subcommand
+from .ls_repo import _add_ls_repo_subcommand
 from .upload import _add_upload_subcommand
 from .whoami import _add_whoami_subcommand
 
@@ -9,6 +10,7 @@ _DECORATORS = [
     _add_upload_subcommand,
     _add_ls_subcommand,
     _add_whoami_subcommand,
+    _add_ls_repo_subcommand,
 ]
 
 cli = hfutilcli

hfutils-0.2.7/hfutils/entry/ls_repo.py

@@ -0,0 +1,74 @@
+import fnmatch
+from typing import Optional
+
+import click
+from huggingface_hub.utils import LocalTokenNotFoundError
+
+from .base import CONTEXT_SETTINGS, ClickErrorException
+from ..operate.base import REPO_TYPES, get_hf_client
+
+
+class NoLocalAuthentication(ClickErrorException):
+    """
+    Exception raised when there is no local authentication token.
+    """
+    exit_code = 0x31
+
+
+def _add_ls_repo_subcommand(cli: click.Group) -> click.Group:
+    """
+    Add the ls_repo subcommand to the CLI.
+
+    :param cli: The click Group object.
+    :type cli: click.Group
+
+    :return: The updated click Group object.
+    :rtype: click.Group
+    """
+
+    @cli.command('ls_repo', help='List repositories from HuggingFace.\n\n'
+                                 'Set environment $HF_TOKEN to use your own access token.',
+                 context_settings=CONTEXT_SETTINGS)
+    @click.option('-a', '--author', 'author', type=str, default=None,
+                  help='Author of the repositories. Search my repositories when not given.')
+    @click.option('-t', '--type', 'repo_type', type=click.Choice(REPO_TYPES), default='dataset',
+                  help='Type of the HuggingFace repository.', show_default=True)
+    @click.option('-p', '--pattern', 'pattern', type=str, default='*',
+                  help='Pattern of the repository names.', show_default=True)
+    def ls(author: Optional[str], repo_type: str, pattern: str):
+        """
+        List repositories from HuggingFace.
+
+        :param author: Author of the repositories.
+        :type author: Optional[str]
+        :param repo_type: Type of the HuggingFace repository.
+        :type repo_type: str
+        :param pattern: Pattern of the repository names.
+        :type pattern: str
+        """
+        hf_client = get_hf_client()
+        if not author:
+            try:
+                info = hf_client.whoami()
+                author = author or info['name']
+            except LocalTokenNotFoundError:
+                raise NoLocalAuthentication(
+                    'Authentication failed.\n'
+                    'Make sure you have set the correct Huggingface token.\n'
+                    'Or if need to use this with guest mode, please explicitly set the `-a` option.'
+                )
+
+        if repo_type == 'model':
+            r = hf_client.list_models(author=author)
+        elif repo_type == 'dataset':
+            r = hf_client.list_datasets(author=author)
+        elif repo_type == 'space':
+            r = hf_client.list_spaces(author=author)
+        else:
+            raise ValueError(f'Unknown repository type - {repo_type!r}.')  # pragma: no cover
+
+        for repo_item in r:
+            if fnmatch.fnmatch(repo_item.id, pattern):
+                print(repo_item.id)
+
+    return cli

{hfutils-0.2.5 → hfutils-0.2.7}/hfutils/operate/download.py

@@ -44,8 +44,6 @@ def download_file_to_file(local_file: str, repo_id: str, file_in_repo: str,
                 filename=hf_normpath(file_in_repo),
                 revision=revision,
                 local_dir=td,
-                force_download=True,
-                local_dir_use_symlinks=False,
                 resume_download=resume_download,
             )
         finally:

hfutils-0.2.7/hfutils/utils/irregular_repo.json

@@ -0,0 +1,247 @@
+{
+    "datasets": [
+        "ade_corpus_v2",
+        "afrikaans_ner_corpus",
+        "app_reviews",
+        "arabic_billion_words",
+        "arabic_speech_corpus",
+        "arxiv_dataset",
+        "aslg_pc12",
+        "autshumato",
+        "banking77",
+        "bigbench",
+        "billsum",
+        "blended_skill_talk",
+        "bnl_newspapers",
+        "bookcorpus",
+        "bswac",
+        "c3",
+        "c4",
+        "cail2018",
+        "caner",
+        "catalonia_independence",
+        "cawac",
+        "cbt",
+        "cc100",
+        "cdsc",
+        "cifar10",
+        "cifar100",
+        "clickbait_news_bg",
+        "clinc_oos",
+        "clue",
+        "cmu_hinglish_dog",
+        "code_search_net",
+        "conceptnet5",
+        "conll2012_ontonotesv5",
+        "consumer-finance-complaints",
+        "conv_ai",
+        "conv_ai_2",
+        "conv_ai_3",
+        "cornell_movie_dialog",
+        "cos_e",
+        "covid_qa_ucsd",
+        "covid_tweets_japanese",
+        "covost2",
+        "cppe-5",
+        "crime_and_punish",
+        "cs_restaurants",
+        "daily_dialog",
+        "danish_political_comments",
+        "definite_pronoun_resolution",
+        "dialog_re",
+        "diplomacy_detection",
+        "disaster_response_messages",
+        "discofuse",
+        "doqa",
+        "dream",
+        "dutch_social",
+        "dyk",
+        "ehealth_kd",
+        "elkarhizketak",
+        "emo",
+        "emotone_ar",
+        "enwik8",
+        "esnli",
+        "ett",
+        "eu_regulatory_ir",
+        "eurlex",
+        "euronews",
+        "europa_eac_tm",
+        "europa_ecdc_tm",
+        "factckbr",
+        "fake_news_english",
+        "farsi_news",
+        "fashion_mnist",
+        "food101",
+        "generics_kb",
+        "germaner",
+        "germeval_14",
+        "glucose",
+        "gnad10",
+        "go_emotions",
+        "grail_qa",
+        "gsm8k",
+        "gutenberg_time",
+        "hansards",
+        "harem",
+        "has_part",
+        "hate_speech_filipino",
+        "hate_speech_pl",
+        "hate_speech_portuguese",
+        "hebrew_projectbenyehuda",
+        "hebrew_sentiment",
+        "hebrew_this_world",
+        "hope_edi",
+        "hotpot_qa",
+        "hover",
+        "hrenwac_para",
+        "hrwac",
+        "humicroedit",
+        "hyperpartisan_news_detection",
+        "id_clickbait",
+        "id_nergrit_corpus",
+        "id_newspapers_2018",
+        "ilist",
+        "imagenet-1k",
+        "imppres",
+        "indic_glue",
+        "isixhosa_ner_corpus",
+        "isizulu_ner_corpus",
+        "jeopardy",
+        "jnlpba",
+        "journalists_questions",
+        "kilt_tasks",
+        "klue",
+        "kor_ner",
+        "kor_sarcasm",
+        "librispeech_asr",
+        "lince",
+        "liveqa",
+        "lm1b",
+        "makhzan",
+        "matinf",
+        "mbpp",
+        "mc4",
+        "med_hop",
+        "medical_questions_pairs",
+        "movie_rationales",
+        "mrqa",
+        "ms_marco",
+        "muchocine",
+        "narrativeqa",
+        "nchlt",
+        "ncslgr",
+        "nell",
+        "newsqa",
+        "nkjp-ner",
+        "nq_open",
+        "oclar",
+        "offenseval_dravidian",
+        "ohsumed",
+        "onestop_english",
+        "openai_humaneval",
+        "para_crawl",
+        "para_pat",
+        "parsinlu_reading_comprehension",
+        "paws",
+        "paws-x",
+        "peoples_daily_ner",
+        "per_sent",
+        "proto_qa",
+        "psc",
+        "ptb_text_only",
+        "qa4mre",
+        "qa_zre",
+        "qangaroo",
+        "qanta",
+        "quail",
+        "quarel",
+        "quickdraw",
+        "quora",
+        "re_dial",
+        "reclor",
+        "reddit_tifu",
+        "roman_urdu",
+        "roman_urdu_hate_speech",
+        "ronec",
+        "rotten_tomatoes",
+        "scan",
+        "scielo",
+        "sede",
+        "selqa",
+        "sem_eval_2010_task_8",
+        "sem_eval_2014_task_1",
+        "sem_eval_2018_task_1",
+        "sem_eval_2020_task_11",
+        "sent_comp",
+        "senti_lex",
+        "senti_ws",
+        "sepedi_ner",
+        "sesotho_ner_corpus",
+        "setimes",
+        "setswana_ner_corpus",
+        "siswati_ner_corpus",
+        "snips_built_in_intents",
+        "snow_simplified_japanese_corpus",
+        "so_stacksample",
+        "sofc_materials_articles",
+        "sogou_news",
+        "squad_kor_v1",
+        "squad_kor_v2",
+        "srwac",
+        "story_cloze",
+        "style_change_detection",
+        "svhn",
+        "swahili",
+        "swahili_news",
+        "swedish_medical_ner",
+        "tamilmixsentiment",
+        "tapaco",
+        "tashkeela",
+        "telugu_books",
+        "telugu_news",
+        "thai_toxicity_tweet",
+        "times_of_india_news_headlines",
+        "timit_asr",
+        "tmu_gfm_dataset",
+        "truthful_qa",
+        "turk",
+        "turkic_xwmt",
+        "turkish_shrinked_ner",
+        "tweet_eval",
+        "tweet_qa",
+        "tweets_ar_en_parallel",
+        "tweets_hate_speech_detection",
+        "ubuntu_dialogs_corpus",
+        "udhr",
+        "um005",
+        "universal_dependencies",
+        "universal_morphologies",
+        "urdu_fake_news",
+        "urdu_sentiment_corpus",
+        "vctk",
+        "vivos",
+        "web_nlg",
+        "web_of_science",
+        "web_questions",
+        "wi_locness",
+        "wiki40b",
+        "wiki_hop",
+        "wiki_lingua",
+        "wiki_qa",
+        "wiki_snippets",
+        "wikipedia",
+        "wikitext",
+        "wikitext_tl39",
+        "wongnai_reviews",
+        "xnli",
+        "xquad_r",
+        "yahoo_answers_qa",
+        "yahoo_answers_topics",
+        "yelp_review_full",
+        "yoruba_bbc_topics",
+        "youtube_caption_corrections"
+    ],
+    "models": [],
+    "spaces": []
+}

{hfutils-0.2.5 → hfutils-0.2.7/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.5
+Version: 0.2.7
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -21,31 +21,15 @@ Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: hbutils>=0.9.0
-Requires-Dist: huggingface_hub>=0.22
+Requires-Dist: huggingface_hub>=0.23
 Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: click>=7
 Requires-Dist: tzlocal
 Requires-Dist: natsort
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
+Requires-Dist: setuptools<70; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -66,12 +50,29 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Provides-Extra: 7z
 Requires-Dist: py7zr; extra == "7z"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 
 # hfutils
 

{hfutils-0.2.5 → hfutils-0.2.7}/hfutils.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ requirements-transfer.txt
 requirements.txt
 setup.py
 hfutils/__init__.py
+hfutils/__main__.py
 hfutils.egg-info/PKG-INFO
 hfutils.egg-info/SOURCES.txt
 hfutils.egg-info/dependency_links.txt
@@ -30,6 +31,7 @@ hfutils/entry/cli.py
 hfutils/entry/dispatch.py
 hfutils/entry/download.py
 hfutils/entry/ls.py
+hfutils/entry/ls_repo.py
 hfutils/entry/upload.py
 hfutils/entry/whoami.py
 hfutils/index/__init__.py

{hfutils-0.2.5 → hfutils-0.2.7}/hfutils.egg-info/requires.txt

@@ -1,5 +1,5 @@
 hbutils>=0.9.0
-huggingface_hub>=0.22
+huggingface_hub>=0.23
 tqdm
 requests
 click>=7
@@ -11,6 +11,7 @@ py7zr
 
 [build]
 pyinstaller<5,>=4.7
+setuptools<70
 
 [doc]
 Jinja2>=3.0.0

hfutils-0.2.7/requirements-build.txt

@@ -0,0 +1,2 @@
+pyinstaller>=4.7,<5
+setuptools<70

{hfutils-0.2.5 → hfutils-0.2.7}/requirements.txt

@@ -1,5 +1,5 @@
 hbutils>=0.9.0
-huggingface_hub>=0.22
+huggingface_hub>=0.23
 tqdm
 requests
 click>=7