hfutils 0.2.4__tar.gz → 0.2.6__tar.gz

{hfutils-0.2.4/hfutils.egg-info → hfutils-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.4
+Version: 0.2.6
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -26,25 +26,9 @@ Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: click>=7
 Requires-Dist: tzlocal
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
+Requires-Dist: natsort
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -65,12 +49,29 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Provides-Extra: 7z
 Requires-Dist: py7zr; extra == "7z"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 
 # hfutils
 

hfutils-0.2.6/hfutils/__main__.py

@@ -0,0 +1,4 @@
+from .entry import hfutilscli
+
+if __name__ == '__main__':
+    hfutilscli()

{hfutils-0.2.4 → hfutils-0.2.6}/hfutils/config/meta.py

@@ -7,7 +7,7 @@ Overview:
 __TITLE__ = 'hfutils'
 
 #: Version of this project.
-__VERSION__ = '0.2.4'
+__VERSION__ = '0.2.6'
 
 #: Short description of the project, will be included in ``setup.py``.
 __DESCRIPTION__ = 'Useful utilities for huggingface'

{hfutils-0.2.4 → hfutils-0.2.6}/hfutils/entry/cli.py

@@ -1,6 +1,7 @@
 from .dispatch import hfutilcli
 from .download import _add_download_subcommand
 from .ls import _add_ls_subcommand
+from .ls_repo import _add_ls_repo_subcommand
 from .upload import _add_upload_subcommand
 from .whoami import _add_whoami_subcommand
 
@@ -9,6 +10,7 @@ _DECORATORS = [
     _add_upload_subcommand,
     _add_ls_subcommand,
     _add_whoami_subcommand,
+    _add_ls_repo_subcommand,
 ]
 
 cli = hfutilcli

hfutils-0.2.6/hfutils/entry/ls_repo.py

@@ -0,0 +1,74 @@
+import fnmatch
+from typing import Optional
+
+import click
+from huggingface_hub.utils import LocalTokenNotFoundError
+
+from .base import CONTEXT_SETTINGS, ClickErrorException
+from ..operate.base import REPO_TYPES, get_hf_client
+
+
+class NoLocalAuthentication(ClickErrorException):
+    """
+    Exception raised when there is no local authentication token.
+    """
+    exit_code = 0x31
+
+
+def _add_ls_repo_subcommand(cli: click.Group) -> click.Group:
+    """
+    Add the ls_repo subcommand to the CLI.
+
+    :param cli: The click Group object.
+    :type cli: click.Group
+
+    :return: The updated click Group object.
+    :rtype: click.Group
+    """
+
+    @cli.command('ls_repo', help='List repositories from HuggingFace.\n\n'
+                                 'Set environment $HF_TOKEN to use your own access token.',
+                 context_settings=CONTEXT_SETTINGS)
+    @click.option('-a', '--author', 'author', type=str, default=None,
+                  help='Author of the repositories. Search my repositories when not given.')
+    @click.option('-t', '--type', 'repo_type', type=click.Choice(REPO_TYPES), default='dataset',
+                  help='Type of the HuggingFace repository.', show_default=True)
+    @click.option('-p', '--pattern', 'pattern', type=str, default='*',
+                  help='Pattern of the repository names.', show_default=True)
+    def ls(author: Optional[str], repo_type: str, pattern: str):
+        """
+        List repositories from HuggingFace.
+
+        :param author: Author of the repositories.
+        :type author: Optional[str]
+        :param repo_type: Type of the HuggingFace repository.
+        :type repo_type: str
+        :param pattern: Pattern of the repository names.
+        :type pattern: str
+        """
+        hf_client = get_hf_client()
+        if not author:
+            try:
+                info = hf_client.whoami()
+                author = author or info['name']
+            except LocalTokenNotFoundError:
+                raise NoLocalAuthentication(
+                    'Authentication failed.\n'
+                    'Make sure you have set the correct Huggingface token.\n'
+                    'Or if need to use this with guest mode, please explicitly set the `-a` option.'
+                )
+
+        if repo_type == 'model':
+            r = hf_client.list_models(author=author)
+        elif repo_type == 'dataset':
+            r = hf_client.list_datasets(author=author)
+        elif repo_type == 'space':
+            r = hf_client.list_spaces(author=author)
+        else:
+            raise ValueError(f'Unknown repository type - {repo_type!r}.')  # pragma: no cover
+
+        for repo_item in r:
+            if fnmatch.fnmatch(repo_item.id, pattern):
+                print(repo_item.id)
+
+    return cli

{hfutils-0.2.4 → hfutils-0.2.6}/hfutils/operate/base.py

@@ -5,6 +5,8 @@ from typing import Literal, List, Optional
 
 from huggingface_hub import HfApi, HfFileSystem
 
+from hfutils.utils import parse_hf_fs_path
+
 RepoTypeTyping = Literal['dataset', 'model', 'space']
 REPO_TYPES = ['dataset', 'model', 'space']
 
@@ -113,12 +115,15 @@ def list_files_in_repository(repo_id: str, repo_type: RepoTypeTyping = 'dataset'
 
     try:
         _exist_files = [
-            os.path.relpath(file, repo_root_path)
+            parse_hf_fs_path(file).filename
             for file in hf_fs.glob(f'{repo_root_path}/**', revision=revision)
         ]
     except FileNotFoundError:
         return []
-    _exist_ps = sorted([(file, file.split(os.sep)) for file in _exist_files], key=lambda x: x[1])
+    if subdir and subdir != '.':
+        _exist_files = [os.path.relpath(file, subdir) for file in _exist_files]
+
+    _exist_ps = sorted([(file, file.split('/')) for file in _exist_files], key=lambda x: x[1])
     retval = []
     for i, (file, segments) in enumerate(_exist_ps):
         if i < len(_exist_ps) - 1 and segments == _exist_ps[i + 1][1][:len(segments)]:

{hfutils-0.2.4 → hfutils-0.2.6}/hfutils/utils/__init__.py

@@ -1,6 +1,6 @@
 from .binary import is_binary_file
 from .download import download_file
-from .path import hf_normpath
+from .path import hf_normpath, hf_fs_path, parse_hf_fs_path, HfFileSystemPath
 from .temp import TemporaryDirectory
 from .tqdm_ import tqdm
 from .walk import walk_files

hfutils-0.2.6/hfutils/utils/irregular_repo.json

@@ -0,0 +1,662 @@
+{
+    "datasets": [
+        "acronym_identification",
+        "ade_corpus_v2",
+        "adv_glue",
+        "aeslc",
+        "afrikaans_ner_corpus",
+        "ag_news",
+        "air_dialogue",
+        "ajgt_twitter_ar",
+        "allegro_reviews",
+        "allocine",
+        "alt",
+        "amazon_polarity",
+        "amazon_reviews_multi",
+        "amazon_us_reviews",
+        "ambig_qa",
+        "ami",
+        "amttl",
+        "app_reviews",
+        "aqua_rat",
+        "aquamuse",
+        "ar_res_reviews",
+        "ar_sarcasm",
+        "arabic_billion_words",
+        "arabic_pos_dialect",
+        "arabic_speech_corpus",
+        "arcd",
+        "arsentd_lev",
+        "art",
+        "arxiv_dataset",
+        "ascent_kb",
+        "aslg_pc12",
+        "asnq",
+        "assin",
+        "assin2",
+        "atomic",
+        "autshumato",
+        "banking77",
+        "bbaw_egyptian",
+        "bbc_hindi_nli",
+        "bc2gm_corpus",
+        "best2009",
+        "bible_para",
+        "big_patent",
+        "bigbench",
+        "billsum",
+        "bing_coronavirus_query_set",
+        "biomrc",
+        "biosses",
+        "biwi_kinect_head_pose",
+        "blended_skill_talk",
+        "blog_authorship_corpus",
+        "bn_hate_speech",
+        "bnl_newspapers",
+        "bookcorpus",
+        "bookcorpusopen",
+        "bprec",
+        "break_data",
+        "brwac",
+        "bsd_ja_en",
+        "bswac",
+        "c3",
+        "c4",
+        "cail2018",
+        "caner",
+        "casino",
+        "catalonia_independence",
+        "cats_vs_dogs",
+        "cawac",
+        "cbt",
+        "cc100",
+        "ccaligned_multilingual",
+        "cdsc",
+        "cdt",
+        "cedr",
+        "cfq",
+        "chr_en",
+        "cifar10",
+        "cifar100",
+        "circa",
+        "clickbait_news_bg",
+        "climate_fever",
+        "clinc_oos",
+        "clue",
+        "cmrc2018",
+        "cmu_hinglish_dog",
+        "cnn_dailymail",
+        "coached_conv_pref",
+        "coarse_discourse",
+        "codah",
+        "code_search_net",
+        "code_x_glue_cc_clone_detection_big_clone_bench",
+        "code_x_glue_cc_clone_detection_poj104",
+        "code_x_glue_cc_cloze_testing_all",
+        "code_x_glue_cc_cloze_testing_maxmin",
+        "code_x_glue_cc_code_completion_line",
+        "code_x_glue_cc_code_completion_token",
+        "code_x_glue_cc_code_refinement",
+        "code_x_glue_cc_code_to_code_trans",
+        "code_x_glue_cc_defect_detection",
+        "code_x_glue_ct_code_to_text",
+        "code_x_glue_tc_nl_code_search_adv",
+        "code_x_glue_tc_text_to_code",
+        "code_x_glue_tt_text_to_text",
+        "com_qa",
+        "common_language",
+        "common_voice",
+        "compguesswhat",
+        "conceptnet5",
+        "conceptual_12m",
+        "conceptual_captions",
+        "conll2000",
+        "conll2002",
+        "conll2003",
+        "conll2012_ontonotesv5",
+        "conllpp",
+        "consumer-finance-complaints",
+        "conv_ai",
+        "conv_ai_2",
+        "conv_ai_3",
+        "conv_questions",
+        "cornell_movie_dialog",
+        "cos_e",
+        "cosmos_qa",
+        "counter",
+        "covid_qa_castorini",
+        "covid_qa_deepset",
+        "covid_qa_ucsd",
+        "covid_tweets_japanese",
+        "covost2",
+        "cppe-5",
+        "craigslist_bargains",
+        "crawl_domain",
+        "crd3",
+        "crime_and_punish",
+        "crows_pairs",
+        "cryptonite",
+        "cs_restaurants",
+        "cuad",
+        "curiosity_dialogs",
+        "daily_dialog",
+        "dane",
+        "danish_political_comments",
+        "dart",
+        "datacommons_factcheck",
+        "dbrd",
+        "deal_or_no_dialog",
+        "definite_pronoun_resolution",
+        "dengue_filipino",
+        "dialog_re",
+        "diplomacy_detection",
+        "disaster_response_messages",
+        "discofuse",
+        "discovery",
+        "disfl_qa",
+        "doc2dial",
+        "docred",
+        "doqa",
+        "dream",
+        "dutch_social",
+        "dyk",
+        "e2e_nlg",
+        "e2e_nlg_cleaned",
+        "ecb",
+        "ecthr_cases",
+        "eduge",
+        "ehealth_kd",
+        "electricity_load_diagrams",
+        "eli5",
+        "eli5_category",
+        "elkarhizketak",
+        "emea",
+        "emo",
+        "emotone_ar",
+        "empathetic_dialogues",
+        "enriched_web_nlg",
+        "enwik8",
+        "eraser_multi_rc",
+        "esnli",
+        "eth_py150_open",
+        "ethos",
+        "ett",
+        "eu_regulatory_ir",
+        "eurlex",
+        "euronews",
+        "europa_eac_tm",
+        "europa_ecdc_tm",
+        "event2Mind",
+        "evidence_infer_treatment",
+        "factckbr",
+        "fake_news_english",
+        "fake_news_filipino",
+        "farsi_news",
+        "fashion_mnist",
+        "fever",
+        "few_rel",
+        "financial_phrasebank",
+        "finer",
+        "flores",
+        "flue",
+        "food101",
+        "fquad",
+        "freebase_qa",
+        "gap",
+        "gem",
+        "generated_reviews_enth",
+        "generics_kb",
+        "german_legal_entity_recognition",
+        "germaner",
+        "germeval_14",
+        "giga_fren",
+        "gigaword",
+        "glucose",
+        "gnad10",
+        "go_emotions",
+        "gooaq",
+        "google_wellformed_query",
+        "grail_qa",
+        "great_code",
+        "gsm8k",
+        "guardian_authorship",
+        "gutenberg_time",
+        "hans",
+        "hansards",
+        "hard",
+        "harem",
+        "has_part",
+        "hate_offensive",
+        "hate_speech18",
+        "hate_speech_filipino",
+        "hate_speech_pl",
+        "hate_speech_portuguese",
+        "hatexplain",
+        "hausa_voa_ner",
+        "hausa_voa_topics",
+        "hda_nli_hindi",
+        "head_qa",
+        "health_fact",
+        "hebrew_projectbenyehuda",
+        "hebrew_sentiment",
+        "hebrew_this_world",
+        "hind_encorp",
+        "hindi_discourse",
+        "hippocorpus",
+        "hkcancor",
+        "hlgd",
+        "hope_edi",
+        "hotpot_qa",
+        "hover",
+        "hrenwac_para",
+        "hrwac",
+        "humicroedit",
+        "hybrid_qa",
+        "hyperpartisan_news_detection",
+        "iapp_wiki_qa_squad",
+        "id_clickbait",
+        "id_liputan6",
+        "id_nergrit_corpus",
+        "id_newspapers_2018",
+        "id_panl_bppt",
+        "id_puisi",
+        "igbo_english_machine_translation",
+        "igbo_monolingual",
+        "igbo_ner",
+        "ilist",
+        "imagenet-1k",
+        "imagenet_sketch",
+        "imdb_urdu_reviews",
+        "imppres",
+        "indic_glue",
+        "indonli",
+        "inquisitive_qg",
+        "interpress_news_category_tr",
+        "interpress_news_category_tr_lite",
+        "irc_disentangle",
+        "isixhosa_ner_corpus",
+        "isizulu_ner_corpus",
+        "iwslt2017",
+        "jeopardy",
+        "jnlpba",
+        "journalists_questions",
+        "kan_hope",
+        "kannada_news",
+        "kd_conv",
+        "kde4",
+        "kelm",
+        "kilt_tasks",
+        "kilt_wikipedia",
+        "kinnews_kirnews",
+        "klue",
+        "kor_3i4k",
+        "kor_hate",
+        "kor_ner",
+        "kor_nli",
+        "kor_nlu",
+        "kor_qpair",
+        "kor_sae",
+        "kor_sarcasm",
+        "labr",
+        "lama",
+        "large_spanish_corpus",
+        "laroseda",
+        "lc_quad",
+        "lccc",
+        "lener_br",
+        "liar",
+        "librispeech_asr",
+        "librispeech_lm",
+        "limit",
+        "lince",
+        "linnaeus",
+        "liveqa",
+        "lj_speech",
+        "lm1b",
+        "lst20",
+        "m_lama",
+        "mac_morpho",
+        "makhzan",
+        "masakhaner",
+        "math_dataset",
+        "math_qa",
+        "matinf",
+        "mbpp",
+        "mc4",
+        "mc_taco",
+        "md_gender_bias",
+        "mdd",
+        "med_hop",
+        "medal",
+        "medical_dialog",
+        "medical_questions_pairs",
+        "menyo20k_mt",
+        "meta_woz",
+        "metashift",
+        "metooma",
+        "metrec",
+        "miam",
+        "mkb",
+        "mkqa",
+        "mlqa",
+        "mlsum",
+        "mnist",
+        "mocha",
+        "monash_tsf",
+        "moroco",
+        "movie_rationales",
+        "mrqa",
+        "ms_marco",
+        "ms_terms",
+        "msr_genomics_kbcomp",
+        "msr_sqa",
+        "msr_text_compression",
+        "msr_zhen_translation_parity",
+        "msra_ner",
+        "mt_eng_vietnamese",
+        "muchocine",
+        "multi_booked",
+        "multi_news",
+        "multi_nli_mismatch",
+        "multi_para_crawl",
+        "multi_re_qa",
+        "multi_woz_v22",
+        "multi_x_science_sum",
+        "multidoc2dial",
+        "multilingual_librispeech",
+        "mutual_friends",
+        "mwsc",
+        "myanmar_news",
+        "narrativeqa",
+        "narrativeqa_manual",
+        "natural_questions",
+        "ncbi_disease",
+        "nchlt",
+        "ncslgr",
+        "nell",
+        "neural_code_search",
+        "newsgroup",
+        "newsph",
+        "newsph_nli",
+        "newspop",
+        "newsqa",
+        "newsroom",
+        "nkjp-ner",
+        "nli_tr",
+        "nlu_evaluation_data",
+        "norec",
+        "norne",
+        "norwegian_ner",
+        "nq_open",
+        "nsmc",
+        "numer_sense",
+        "numeric_fused_head",
+        "oclar",
+        "offcombr",
+        "offenseval2020_tr",
+        "offenseval_dravidian",
+        "ofis_publik",
+        "ohsumed",
+        "ollie",
+        "omp",
+        "onestop_english",
+        "onestop_qa",
+        "open_subtitles",
+        "openai_humaneval",
+        "openslr",
+        "opinosis",
+        "orange_sum",
+        "oscar",
+        "para_crawl",
+        "para_pat",
+        "parsinlu_reading_comprehension",
+        "pass",
+        "paws",
+        "paws-x",
+        "pec",
+        "peoples_daily_ner",
+        "per_sent",
+        "persian_ner",
+        "pg19",
+        "php",
+        "pib",
+        "piqa",
+        "pn_summary",
+        "poem_sentiment",
+        "polemo2",
+        "poleval2019_cyberbullying",
+        "poleval2019_mt",
+        "polsum",
+        "polyglot_ner",
+        "prachathai67k",
+        "pragmeval",
+        "proto_qa",
+        "psc",
+        "ptb_text_only",
+        "pubmed",
+        "py_ast",
+        "qa4mre",
+        "qa_srl",
+        "qa_zre",
+        "qangaroo",
+        "qanta",
+        "qed",
+        "qed_amara",
+        "quac",
+        "quail",
+        "quarel",
+        "quickdraw",
+        "quora",
+        "quoref",
+        "re_dial",
+        "reasoning_bg",
+        "recipe_nlg",
+        "reclor",
+        "red_caps",
+        "reddit_tifu",
+        "refresd",
+        "reuters21578",
+        "riddle_sense",
+        "ro_sent",
+        "ro_sts",
+        "ro_sts_parallel",
+        "roman_urdu",
+        "roman_urdu_hate_speech",
+        "ronec",
+        "rotten_tomatoes",
+        "samsum",
+        "sanskrit_classic",
+        "saudinewsnet",
+        "sberquad",
+        "sbu_captions",
+        "scan",
+        "scb_mt_enth_2020",
+        "scene_parse_150",
+        "schema_guided_dstc8",
+        "scielo",
+        "scientific_papers",
+        "search_qa",
+        "sede",
+        "selqa",
+        "sem_eval_2010_task_8",
+        "sem_eval_2014_task_1",
+        "sem_eval_2018_task_1",
+        "sem_eval_2020_task_11",
+        "sent_comp",
+        "senti_lex",
+        "senti_ws",
+        "sentiment140",
+        "sepedi_ner",
+        "sesotho_ner_corpus",
+        "setimes",
+        "setswana_ner_corpus",
+        "sharc_modified",
+        "sick",
+        "silicone",
+        "simple_questions_v2",
+        "siswati_ner_corpus",
+        "smartdata",
+        "sms_spam",
+        "snips_built_in_intents",
+        "snow_simplified_japanese_corpus",
+        "so_stacksample",
+        "social_bias_frames",
+        "social_i_qa",
+        "sofc_materials_articles",
+        "sogou_news",
+        "spanish_billion_words",
+        "spc",
+        "species_800",
+        "speech_commands",
+        "squad_adversarial",
+        "squad_es",
+        "squad_it",
+        "squad_kor_v1",
+        "squad_kor_v2",
+        "squad_v1_pt",
+        "squadshifts",
+        "srwac",
+        "sst",
+        "story_cloze",
+        "stsb_mt_sv",
+        "style_change_detection",
+        "subjqa",
+        "super_glue",
+        "superb",
+        "svhn",
+        "swag",
+        "swahili",
+        "swahili_news",
+        "swda",
+        "swedish_medical_ner",
+        "swedish_ner_corpus",
+        "swedish_reviews",
+        "tab_fact",
+        "tamilmixsentiment",
+        "tanzil",
+        "tapaco",
+        "tashkeela",
+        "taskmaster1",
+        "taskmaster2",
+        "taskmaster3",
+        "tatoeba",
+        "ted_hrlr",
+        "ted_iwlst2013",
+        "ted_multi",
+        "ted_talks_iwslt",
+        "telugu_books",
+        "telugu_news",
+        "tep_en_fa_para",
+        "text2log",
+        "textvqa",
+        "thai_toxicity_tweet",
+        "thainer",
+        "thaiqa_squad",
+        "thaisum",
+        "the_pile_books3",
+        "the_pile_openwebtext2",
+        "the_pile_stack_exchange",
+        "tilde_model",
+        "time_dial",
+        "times_of_india_news_headlines",
+        "timit_asr",
+        "tlc",
+        "tmu_gfm_dataset",
+        "tne",
+        "told-br",
+        "totto",
+        "trec",
+        "truthful_qa",
+        "tsac",
+        "ttc4900",
+        "tunizi",
+        "tuple_ie",
+        "turk",
+        "turkic_xwmt",
+        "turkish_movie_sentiment",
+        "turkish_ner",
+        "turkish_product_reviews",
+        "turkish_shrinked_ner",
+        "turku_ner_corpus",
+        "tweet_eval",
+        "tweet_qa",
+        "tweets_ar_en_parallel",
+        "tweets_hate_speech_detection",
+        "twi_text_c3",
+        "twi_wordsim353",
+        "tydiqa",
+        "ubuntu_dialogs_corpus",
+        "udhr",
+        "um005",
+        "universal_dependencies",
+        "universal_morphologies",
+        "urdu_fake_news",
+        "urdu_sentiment_corpus",
+        "vctk",
+        "visual_genome",
+        "vivos",
+        "web_nlg",
+        "web_of_science",
+        "web_questions",
+        "weibo_ner",
+        "wi_locness",
+        "wider_face",
+        "wiki40b",
+        "wiki_asp",
+        "wiki_atomic_edits",
+        "wiki_auto",
+        "wiki_bio",
+        "wiki_dpr",
+        "wiki_hop",
+        "wiki_lingua",
+        "wiki_movies",
+        "wiki_qa",
+        "wiki_qa_ar",
+        "wiki_snippets",
+        "wiki_source",
+        "wiki_split",
+        "wiki_summary",
+        "wikiann",
+        "wikicorpus",
+        "wikihow",
+        "wikipedia",
+        "wikisql",
+        "wikitablequestions",
+        "wikitext",
+        "wikitext_tl39",
+        "wili_2018",
+        "wino_bias",
+        "winograd_wsc",
+        "winogrande",
+        "wiqa",
+        "wisesight1000",
+        "wisesight_sentiment",
+        "wnut_17",
+        "wongnai_reviews",
+        "woz_dialogue",
+        "wrbsc",
+        "x_stance",
+        "xcopa",
+        "xcsr",
+        "xed_en_fi",
+        "xglue",
+        "xnli",
+        "xor_tydi_qa",
+        "xquad",
+        "xquad_r",
+        "xsum_factuality",
+        "xtreme",
+        "yahoo_answers_qa",
+        "yahoo_answers_topics",
+        "yelp_polarity",
+        "yelp_review_full",
+        "yoruba_bbc_topics",
+        "yoruba_gv_ner",
+        "yoruba_text_c3",
+        "yoruba_wordsim353",
+        "youtube_caption_corrections",
+        "zest"
+    ],
+    "models": [],
+    "spaces": []
+}

hfutils-0.2.6/hfutils/utils/path.py

@@ -0,0 +1,144 @@
+import json
+import os
+import re
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Optional, Dict, Set, Literal
+
+RepoTypeTyping = Literal['dataset', 'model', 'space']
+
+
+def hf_normpath(path) -> str:
+    """
+    Normalize a given path.
+
+    :param path: The path to normalize.
+    :type path: Any
+
+    :return: The normalized path.
+    :rtype: str
+    """
+    return re.sub(
+        r'[\\/]+', '/',
+        os.path.relpath(os.path.normpath(os.path.join(os.sep, path)), os.sep)
+    )
+
+
+def hf_fs_path(repo_id: str, filename: str,
+               repo_type: RepoTypeTyping = 'dataset', revision: Optional[str] = None):
+    """
+    Get the huggingface filesystem path.
+
+    :param repo_id: The repository ID.
+    :type repo_id: str
+
+    :param filename: The filename.
+    :type filename: str
+
+    :param repo_type: The type of repository. (default: 'dataset')
+    :type repo_type: RepoTypeTyping
+
+    :param revision: The revision of the repository. (default: None)
+    :type revision: Optional[str]
+
+    :return: The huggingface filesystem path.
+    :rtype: str
+    """
+    filename = hf_normpath(filename)
+    if repo_type == 'dataset':
+        prefix = 'datasets/'
+    elif repo_type == 'space':
+        prefix = 'spaces/'
+    else:
+        prefix = ''
+
+    if revision is not None:
+        revision_text = f'@{revision}'
+    else:
+        revision_text = ''
+
+    return f'{prefix}{repo_id}{revision_text}/{filename}'
+
+
+@lru_cache()
+def _irregular_repos() -> Dict[RepoTypeTyping, Set[str]]:
+    """
+    Get irregular repositories.
+
+    :return: A dictionary containing irregular repositories.
+    :rtype: Dict[RepoTypeTyping, Set[str]]
+    """
+    with open(os.path.join(os.path.dirname(__file__), 'irregular_repo.json'), 'r') as f:
+        data = json.load(f)
+        return {
+            'model': set(data['models']),
+            'dataset': set(data['datasets']),
+            'space': set(data['spaces']),
+        }
+
+
+_RE_IR_PATH = re.compile(
+    r'^(?P<repo_id>[^@/]+)(@(?P<revision>[^@/]+))?(/(?P<filename>[\s\S]+))?$')
+_RE_PATH = re.compile(
+    r'^(?P<repo_id>[^@/]+/[^@/]+)(@(?P<revision>[^@/]+))?(/(?P<filename>[\s\S]+))?$')
+
+
+@dataclass
+class HfFileSystemPath:
+    """
+    Huggingface FileSystem Path.
+
+    :param repo_id: The repository ID.
+    :type repo_id: str
+
+    :param filename: The filename.
+    :type filename: str
+
+    :param repo_type: The type of repository.
+    :type repo_type: RepoTypeTyping
+
+    :param revision: The revision of the repository.
+    :type revision: Optional[str]
+    """
+    repo_id: str
+    filename: str
+    repo_type: RepoTypeTyping
+    revision: Optional[str]
+
+
+def parse_hf_fs_path(path: str) -> HfFileSystemPath:
+    """
+    Parse the huggingface filesystem path.
+
+    :param path: The path to parse.
+    :type path: str
+
+    :return: The parsed huggingface filesystem path.
+    :rtype: HfFileSystemPath
+    :raises ValueError: If this path is invalid.
+    """
+    origin_path = path
+    repo_type: RepoTypeTyping
+    if path.startswith('datasets/'):
+        repo_type = 'dataset'
+        path = path[len('datasets/'):]
+    elif path.startswith('spaces/'):
+        repo_type = 'space'
+        path = path[len('spaces/'):]
+    else:
+        repo_type = 'model'
+
+    matching = _RE_IR_PATH.fullmatch(path)
+    if matching:
+        if matching.group('repo_id') not in _irregular_repos()[repo_type]:
+            matching = None
+    if not matching:
+        matching = _RE_PATH.fullmatch(path)
+
+    if matching:
+        repo_id = matching.group('repo_id')
+        revision = matching.group('revision') or None
+        filename = hf_normpath(matching.group('filename') or '.')
+        return HfFileSystemPath(repo_id, filename, repo_type, revision)
+    else:
+        raise ValueError(f'Invalid huggingface filesystem path - {origin_path!r}.')

{hfutils-0.2.4 → hfutils-0.2.6/hfutils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hfutils
-Version: 0.2.4
+Version: 0.2.6
 Summary: Useful utilities for huggingface
 Home-page: https://github.com/deepghs/hfutils
 Author: narugo1992
@@ -26,25 +26,9 @@ Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: click>=7
 Requires-Dist: tzlocal
-Provides-Extra: rar
-Requires-Dist: rarfile; extra == "rar"
-Provides-Extra: test
-Requires-Dist: coverage>=5; extra == "test"
-Requires-Dist: mock>=4.0.3; extra == "test"
-Requires-Dist: flake8~=3.5; extra == "test"
-Requires-Dist: testfixtures>=6.18.5; extra == "test"
-Requires-Dist: pytest~=6.2.5; extra == "test"
-Requires-Dist: pytest-cov~=3.0.0; extra == "test"
-Requires-Dist: pytest-mock~=3.6.1; extra == "test"
-Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
-Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
-Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
-Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
-Requires-Dist: easydict<2,>=1.7; extra == "test"
-Requires-Dist: testtools>=2; extra == "test"
-Requires-Dist: where>=1.0.2; extra == "test"
-Requires-Dist: responses>=0.20.0; extra == "test"
-Requires-Dist: natsort; extra == "test"
+Requires-Dist: natsort
+Provides-Extra: build
+Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Provides-Extra: doc
 Requires-Dist: Jinja2>=3.0.0; extra == "doc"
 Requires-Dist: sphinx>=3.2.0; extra == "doc"
@@ -65,12 +49,29 @@ Requires-Dist: ipykernel>=6.15; extra == "doc"
 Requires-Dist: py-cpuinfo>=8.0.0; extra == "doc"
 Requires-Dist: click>=7.0.0; extra == "doc"
 Requires-Dist: pandas; extra == "doc"
-Provides-Extra: build
-Requires-Dist: pyinstaller<5,>=4.7; extra == "build"
 Provides-Extra: 7z
 Requires-Dist: py7zr; extra == "7z"
 Provides-Extra: transfer
 Requires-Dist: hf-transfer; extra == "transfer"
+Provides-Extra: test
+Requires-Dist: coverage>=5; extra == "test"
+Requires-Dist: mock>=4.0.3; extra == "test"
+Requires-Dist: flake8~=3.5; extra == "test"
+Requires-Dist: testfixtures>=6.18.5; extra == "test"
+Requires-Dist: pytest~=6.2.5; extra == "test"
+Requires-Dist: pytest-cov~=3.0.0; extra == "test"
+Requires-Dist: pytest-mock~=3.6.1; extra == "test"
+Requires-Dist: pytest-xdist>=1.34.0; extra == "test"
+Requires-Dist: pytest-rerunfailures~=10.2; extra == "test"
+Requires-Dist: pytest-timeout~=2.0.2; extra == "test"
+Requires-Dist: pytest-benchmark~=3.4.0; extra == "test"
+Requires-Dist: easydict<2,>=1.7; extra == "test"
+Requires-Dist: testtools>=2; extra == "test"
+Requires-Dist: where>=1.0.2; extra == "test"
+Requires-Dist: responses>=0.20.0; extra == "test"
+Requires-Dist: natsort; extra == "test"
+Provides-Extra: rar
+Requires-Dist: rarfile; extra == "rar"
 
 # hfutils
 

{hfutils-0.2.4 → hfutils-0.2.6}/hfutils.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ requirements-transfer.txt
 requirements.txt
 setup.py
 hfutils/__init__.py
+hfutils/__main__.py
 hfutils.egg-info/PKG-INFO
 hfutils.egg-info/SOURCES.txt
 hfutils.egg-info/dependency_links.txt
@@ -30,6 +31,7 @@ hfutils/entry/cli.py
 hfutils/entry/dispatch.py
 hfutils/entry/download.py
 hfutils/entry/ls.py
+hfutils/entry/ls_repo.py
 hfutils/entry/upload.py
 hfutils/entry/whoami.py
 hfutils/index/__init__.py
@@ -45,6 +47,7 @@ hfutils/operate/validate.py
 hfutils/utils/__init__.py
 hfutils/utils/binary.py
 hfutils/utils/download.py
+hfutils/utils/irregular_repo.json
 hfutils/utils/path.py
 hfutils/utils/temp.py
 hfutils/utils/tqdm_.py

{hfutils-0.2.4 → hfutils-0.2.6}/hfutils.egg-info/requires.txt

@@ -4,6 +4,7 @@ tqdm
 requests
 click>=7
 tzlocal
+natsort
 
 [7z]
 py7zr

{hfutils-0.2.4 → hfutils-0.2.6}/requirements.txt

@@ -4,3 +4,4 @@ tqdm
 requests
 click>=7
 tzlocal
+natsort

hfutils-0.2.4/hfutils/utils/path.py

@@ -1,5 +0,0 @@
-import os
-
-
-def hf_normpath(path):
-    return os.path.normpath(path).replace('\\', '/')