hfutils 0.2.5__tar.gz → 0.2.6__tar.gz

{hfutils-0.2.5/hfutils.egg-info → hfutils-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.5
+Version: 0.2.6
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -27,25 +27,8 @@ Requires-Dist: requests
 Requires-Dist: click>=7
 Requires-Dist: tzlocal
 Requires-Dist: natsort
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -66,12 +49,29 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Provides-Extra: 7z
 Requires-Dist: py7zr; extra == "7z"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 
 # hfutils
 

hfutils-0.2.6/hfutils/__main__.py

@@ -0,0 +1,4 @@
+from .entry import hfutilscli
+
+if __name__ == '__main__':
+    hfutilscli()

{hfutils-0.2.5 → hfutils-0.2.6}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.2.5'
+__VERSION__ = '0.2.6'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.2.5 → hfutils-0.2.6}/hfutils/entry/cli.py

@@ -1,6 +1,7 @@
 from .dispatch import hfutilcli
 from .download import _add_download_subcommand
 from .ls import _add_ls_subcommand
+from .ls_repo import _add_ls_repo_subcommand
 from .upload import _add_upload_subcommand
 from .whoami import _add_whoami_subcommand
 
@@ -9,6 +10,7 @@ _DECORATORS = [
     _add_upload_subcommand,
     _add_ls_subcommand,
     _add_whoami_subcommand,
+    _add_ls_repo_subcommand,
 ]
 
 cli = hfutilcli

hfutils-0.2.6/hfutils/entry/ls_repo.py

@@ -0,0 +1,74 @@
+import fnmatch
+from typing import Optional
+
+import click
+from huggingface_hub.utils import LocalTokenNotFoundError
+
+from .base import CONTEXT_SETTINGS, ClickErrorException
+from ..operate.base import REPO_TYPES, get_hf_client
+
+
+class NoLocalAuthentication(ClickErrorException):
+    """
+    Exception raised when there is no local authentication token.
+    """
+    exit_code = 0x31
+
+
+def _add_ls_repo_subcommand(cli: click.Group) -> click.Group:
+    """
+    Add the ls_repo subcommand to the CLI.
+
+    :param cli: The click Group object.
+    :type cli: click.Group
+
+    :return: The updated click Group object.
+    :rtype: click.Group
+    """
+
+    @cli.command('ls_repo', help='List repositories from HuggingFace.\n\n'
+                                 'Set environment $HF_TOKEN to use your own access token.',
+                 context_settings=CONTEXT_SETTINGS)
+    @click.option('-a', '--author', 'author', type=str, default=None,
+                  help='Author of the repositories. Search my repositories when not given.')
+    @click.option('-t', '--type', 'repo_type', type=click.Choice(REPO_TYPES), default='dataset',
+                  help='Type of the HuggingFace repository.', show_default=True)
+    @click.option('-p', '--pattern', 'pattern', type=str, default='*',
+                  help='Pattern of the repository names.', show_default=True)
+    def ls(author: Optional[str], repo_type: str, pattern: str):
+        """
+        List repositories from HuggingFace.
+
+        :param author: Author of the repositories.
+        :type author: Optional[str]
+        :param repo_type: Type of the HuggingFace repository.
+        :type repo_type: str
+        :param pattern: Pattern of the repository names.
+        :type pattern: str
+        """
+        hf_client = get_hf_client()
+        if not author:
+            try:
+                info = hf_client.whoami()
+                author = author or info['name']
+            except LocalTokenNotFoundError:
+                raise NoLocalAuthentication(
+                    'Authentication failed.\n'
+                    'Make sure you have set the correct Huggingface token.\n'
+                    'Or if need to use this with guest mode, please explicitly set the `-a` option.'
+                )
+
+        if repo_type == 'model':
+            r = hf_client.list_models(author=author)
+        elif repo_type == 'dataset':
+            r = hf_client.list_datasets(author=author)
+        elif repo_type == 'space':
+            r = hf_client.list_spaces(author=author)
+        else:
+            raise ValueError(f'Unknown repository type - {repo_type!r}.')  # pragma: no cover
+
+        for repo_item in r:
+            if fnmatch.fnmatch(repo_item.id, pattern):
+                print(repo_item.id)
+
+    return cli

{hfutils-0.2.5 → hfutils-0.2.6}/hfutils/utils/irregular_repo.json

@@ -2,6 +2,7 @@
     "datasets": [
         "acronym_identification",
         "ade_corpus_v2",
+        "adv_glue",
         "aeslc",
         "afrikaans_ner_corpus",
         "ag_news",
@@ -42,10 +43,12 @@
         "best2009",
         "bible_para",
         "big_patent",
+        "bigbench",
         "billsum",
         "bing_coronavirus_query_set",
         "biomrc",
         "biosses",
+        "biwi_kinect_head_pose",
         "blended_skill_talk",
         "blog_authorship_corpus",
         "bn_hate_speech",
@@ -105,9 +108,12 @@
         "common_voice",
         "compguesswhat",
         "conceptnet5",
+        "conceptual_12m",
+        "conceptual_captions",
         "conll2000",
         "conll2002",
         "conll2003",
+        "conll2012_ontonotesv5",
         "conllpp",
         "consumer-finance-complaints",
         "conv_ai",
@@ -163,15 +169,18 @@
         "electricity_load_diagrams",
         "eli5",
         "eli5_category",
+        "elkarhizketak",
         "emea",
         "emo",
         "emotone_ar",
         "empathetic_dialogues",
         "enriched_web_nlg",
+        "enwik8",
         "eraser_multi_rc",
         "esnli",
         "eth_py150_open",
         "ethos",
+        "ett",
         "eu_regulatory_ir",
         "eurlex",
         "euronews",
@@ -209,6 +218,7 @@
         "google_wellformed_query",
         "grail_qa",
         "great_code",
+        "gsm8k",
         "guardian_authorship",
         "gutenberg_time",
         "hans",
@@ -254,6 +264,8 @@
         "igbo_monolingual",
         "igbo_ner",
         "ilist",
+        "imagenet-1k",
+        "imagenet_sketch",
         "imdb_urdu_reviews",
         "imppres",
         "indic_glue",
@@ -290,6 +302,7 @@
         "large_spanish_corpus",
         "laroseda",
         "lc_quad",
+        "lccc",
         "lener_br",
         "liar",
         "librispeech_asr",
@@ -319,6 +332,7 @@
         "medical_questions_pairs",
         "menyo20k_mt",
         "meta_woz",
+        "metashift",
         "metooma",
         "metrec",
         "miam",
@@ -328,6 +342,7 @@
         "mlsum",
         "mnist",
         "mocha",
+        "monash_tsf",
         "moroco",
         "movie_rationales",
         "mrqa",
@@ -396,8 +411,8 @@
         "para_pat",
         "parsinlu_reading_comprehension",
         "pass",
-        "paws-x",
         "paws",
+        "paws-x",
         "pec",
         "peoples_daily_ner",
         "per_sent",
@@ -430,6 +445,7 @@
         "quac",
         "quail",
         "quarel",
+        "quickdraw",
         "quora",
         "quoref",
         "re_dial",
@@ -445,12 +461,14 @@
         "ro_sts",
         "ro_sts_parallel",
         "roman_urdu",
+        "roman_urdu_hate_speech",
         "ronec",
         "rotten_tomatoes",
         "samsum",
         "sanskrit_classic",
         "saudinewsnet",
         "sberquad",
+        "sbu_captions",
         "scan",
         "scb_mt_enth_2020",
         "scene_parse_150",
@@ -530,6 +548,7 @@
         "telugu_news",
         "tep_en_fa_para",
         "text2log",
+        "textvqa",
         "thai_toxicity_tweet",
         "thainer",
         "thaiqa_squad",
@@ -543,9 +562,11 @@
         "timit_asr",
         "tlc",
         "tmu_gfm_dataset",
+        "tne",
         "told-br",
         "totto",
         "trec",
+        "truthful_qa",
         "tsac",
         "ttc4900",
         "tunizi",
@@ -572,6 +593,7 @@
         "urdu_fake_news",
         "urdu_sentiment_corpus",
         "vctk",
+        "visual_genome",
         "vivos",
         "web_nlg",
         "web_of_science",
@@ -599,6 +621,7 @@
         "wikihow",
         "wikipedia",
         "wikisql",
+        "wikitablequestions",
         "wikitext",
         "wikitext_tl39",
         "wili_2018",
@@ -632,30 +655,7 @@
         "yoruba_text_c3",
         "yoruba_wordsim353",
         "youtube_caption_corrections",
-        "zest",
-        "elkarhizketak",
-        "wikitablequestions",
-        "conll2012_ontonotesv5",
-        "monash_tsf",
-        "roman_urdu_hate_speech",
-        "adv_glue",
-        "metashift",
-        "gsm8k",
-        "sbu_captions",
-        "conceptual_captions",
-        "conceptual_12m",
-        "visual_genome",
-        "imagenet-1k",
-        "tne",
-        "textvqa",
-        "ett",
-        "imagenet_sketch",
-        "biwi_kinect_head_pose",
-        "enwik8",
-        "truthful_qa",
-        "bigbench",
-        "quickdraw",
-        "lccc"
+        "zest"
     ],
     "models": [],
     "spaces": []

{hfutils-0.2.5 → hfutils-0.2.6/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.5
+Version: 0.2.6
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -27,25 +27,8 @@ Requires-Dist: requests
 Requires-Dist: click>=7
 Requires-Dist: tzlocal
 Requires-Dist: natsort
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -66,12 +49,29 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Provides-Extra: 7z
 Requires-Dist: py7zr; extra == "7z"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 
 # hfutils
 

{hfutils-0.2.5 → hfutils-0.2.6}/hfutils.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ requirements-transfer.txt
 requirements.txt
 setup.py
 hfutils/__init__.py
+hfutils/__main__.py
 hfutils.egg-info/PKG-INFO
 hfutils.egg-info/SOURCES.txt
 hfutils.egg-info/dependency_links.txt
@@ -30,6 +31,7 @@ hfutils/entry/cli.py
 hfutils/entry/dispatch.py
 hfutils/entry/download.py
 hfutils/entry/ls.py
+hfutils/entry/ls_repo.py
 hfutils/entry/upload.py
 hfutils/entry/whoami.py
 hfutils/index/__init__.py