megfile 3.0.6.post1__py3-none-any.whl → 3.1.0.post1__py3-none-any.whl

scripts/generate_file.py

@@ -0,0 +1,268 @@
+import importlib
+import re
+
+ALL_IGNORE_FUNC_LIST = dict(
+    s3=[
+        "open", "readlink", "iterdir", "is_mount", "is_socket", "is_fifo",
+        "is_block_device", "is_char_device", "owner", "absolute", "rmdir",
+        "glob", "iglob", "glob_stat", "rename", "cwd", "mkdir", "parts",
+        "path_without_protocol", "path_with_protocol"
+    ],
+    fs=[
+        "open", "from_uri", "path_with_protocol", "joinpath", "readlink",
+        "iterdir", "chmod", "group", "is_socket", "is_fifo", "is_block_device",
+        "is_char_device", "rmdir", "owner", "absolute", "resolve", "cwd",
+        "home", "glob", "iglob", "glob_stat", "rename", "parts", "root",
+        "anchor", "drive", "replace", "hardlink_to", "mkdir", "utime"
+    ],
+    http=["open"],
+    sftp=[
+        "path_without_protocol", "expanduser", "iterdir", "readlink", "cwd",
+        "glob", "iglob", "glob_stat", "resolve", "relpath", "utime", "parts"
+    ],
+    hdfs=[
+        "iterdir", "absolute", "rmdir", "glob", "iglob", "glob_stat", "rename",
+        "mkdir", "path_without_protocol", "path_with_protocol", "parts"
+    ],
+)
+
+ALL_IMPORT_LINES = dict(
+    s3=[
+        "from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple",
+        "from megfile.interfaces import Access, FileEntry, PathLike, StatResult",
+    ],
+    fs=[
+        "from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple",
+        "from megfile.interfaces import Access, FileEntry, PathLike, StatResult",
+    ],
+    http=[
+        "from megfile.interfaces import PathLike, StatResult",
+    ],
+    stdio=[
+        "from typing import IO, AnyStr, Optional",
+        "from megfile.interfaces import PathLike",
+    ],
+    sftp=[
+        "from typing import IO, AnyStr, BinaryIO, Iterator, List, Tuple, Callable, Optional",
+        "from megfile.interfaces import FileEntry, PathLike, StatResult",
+    ],
+    hdfs=[
+        "from typing import IO, AnyStr, BinaryIO, Iterator, List, Optional, Tuple",
+        "from megfile.interfaces import FileEntry, PathLike, StatResult",
+    ],
+)
+
+ALL_FUNC_NAME_MAPPING = dict(
+    s3=dict(
+        is_dir="isdir",
+        is_file="isfile",
+        load="load_from",
+        mkdir="makedirs",
+        md5="getmd5",
+        symlink_to="symlink",
+        is_symlink="islink",
+        save="save_as",
+    ),
+    fs=dict(
+        is_dir="isdir",
+        is_file="isfile",
+        md5="getmd5",
+        load="load_from",
+        mkdir="makedirs",
+        symlink_to="symlink",
+        is_symlink="islink",
+        is_mount="ismount",
+        save="save_as",
+        joinpath="path_join",
+        is_absolute="isabs",
+        replace="move",
+    ),
+    http=dict(),
+    sftp=dict(
+        is_dir="isdir",
+        is_file="isfile",
+        md5="getmd5",
+        load="load_from",
+        mkdir="makedirs",
+        symlink_to="symlink",
+        is_symlink="islink",
+        save="save_as",
+        is_absolute="isabs",
+        replace="move",
+    ),
+    hdfs=dict(
+        is_dir="isdir",
+        is_file="isfile",
+        load="load_from",
+        mkdir="makedirs",
+        md5="getmd5",
+        symlink_to="symlink",
+        is_symlink="islink",
+        save="save_as",
+    ),
+)
+PARAMETER_PATTERN = re.compile(r'\[[^:]*\]')
+
+
+def get_class_name(current_file_type: str):
+    if current_file_type == 'fs':
+        return 'FSPath'
+    return f'{current_file_type.capitalize()}Path'
+
+
+def insert_class_method_lines(
+        func_params: list, annotation_lines: list, current_file_type: str):
+    ignore_func_list = ALL_IGNORE_FUNC_LIST.get(current_file_type, [])
+    func_name_mapping = ALL_FUNC_NAME_MAPPING.get(current_file_type, {})
+
+    real_func_name, func_content_lines = None, []
+    if func_params:
+        func_first_line = "".join(func_params)
+        path_param_name = "path"
+        current_params_line = PARAMETER_PATTERN.sub(
+            "",
+            func_first_line.split("(", maxsplit=1)[1].split(")", maxsplit=1)[0])
+        current_params = []
+        kwargs_mode = False
+        for params_words in current_params_line.split(","):
+            if ":" in params_words:
+                param = params_words.split(":", maxsplit=1)[0].strip()
+            elif "=" in params_words:
+                param = params_words.split("=", maxsplit=1)[0].strip()
+            else:
+                param = params_words.strip()
+            if param == '*':
+                kwargs_mode = True
+                continue
+
+            if param and param != '**kwargs':
+                if 'dst' in param:
+                    path_param_name = param.replace("dst", "src")
+                if kwargs_mode:
+                    param = f"{param}={param}"
+                current_params.append(param)
+        func_name = func_first_line.strip().split(
+            "def ", maxsplit=1)[1].split(
+                "(", maxsplit=1)[0]
+        if func_name == 'save':
+            func_first_line = func_first_line.replace(
+                "self", f"{path_param_name}: PathLike")
+            special_order_params = [
+                param.strip() for param in func_first_line.split('(', 1)
+                [1].split(')', 1)[0].split(',')
+            ]
+            special_order_params[0], special_order_params[
+                1] = special_order_params[1], special_order_params[0]
+            func_first_line = "".join(
+                [
+                    func_first_line.split('(', 1)[0], "(",
+                    ", ".join(special_order_params), ")",
+                    func_first_line.split(')', 1)[1]
+                ])
+        else:
+            func_first_line = func_first_line.replace(
+                "self", f"{path_param_name}: PathLike")
+
+        if not func_name.startswith("_") and func_name not in ignore_func_list:
+            real_func_name = f"{current_file_type}_{func_name_mapping.get(func_name, func_name)}"
+            func_content_lines.append(
+                func_first_line.replace(func_name, real_func_name).replace(
+                    ', **kwargs', ''))
+
+            insert_log = False
+            for annotation_line in annotation_lines:
+                if insert_log is False and annotation_line.strip().startswith(
+                        ":"):
+                    func_content_lines.append(
+                        f"    :param {path_param_name}: Given path")
+                    insert_log = True
+                func_content_lines.append(annotation_line)
+
+            class_name = get_class_name(current_file_type)
+            if class_name == 'StdioPath':
+                func_content_lines.append(
+                    f"    return {class_name}({path_param_name}).{func_name}({', '.join(current_params[1:])})  # pyre-ignore[6]\n\n"
+                )
+            else:
+                func_content_lines.append(
+                    f"    return {class_name}({path_param_name}).{func_name}({', '.join(current_params[1:])})\n\n"
+                )
+    return real_func_name, func_content_lines
+
+
+def get_methods_from_path_file(current_file_type: str):
+    all_func_list = importlib.import_module(
+        f'megfile.{current_file_type}_path').__all__
+    methods_content = []
+    import_lines = ALL_IMPORT_LINES.get(current_file_type, [])
+    import_lines.append(
+        f"from megfile.{current_file_type}_path import {', '.join(all_func_list)}"
+    )
+    with open(f'megfile/{current_file_type}_path.py', 'r') as f:
+        class_start = False
+        func_start = False
+        func_params = []
+        annotation_lines = []
+        annotation_start = False
+        for line in f.readlines():
+            if line.strip().startswith(
+                    f'class {get_class_name(current_file_type)}('):
+                class_start = True
+            elif class_start is True:
+                if line.strip() and not line.startswith(" " * 4):
+                    break
+                elif func_start is True:
+                    if line.rsplit('#', maxsplit=1)[0].strip().endswith(":"):
+                        func_start = False
+                    func_params.append(line.strip())
+                elif "'''" in line or '"""' in line:
+                    if line.count("'''") <= 1 and line.count('"""') <= 1:
+                        annotation_start = not annotation_start
+                    annotation_lines.append(line[4:].rstrip())
+                elif annotation_start is True:
+                    annotation_lines.append(line[4:].rstrip())
+                elif line.startswith("    def"):
+                    if line.rsplit('#', maxsplit=1)[0].strip().endswith(":"):
+                        func_start = False
+                    else:
+                        func_start = True
+                    func_name, func_content_lines = insert_class_method_lines(
+                        func_params, annotation_lines, current_file_type)
+                    if func_name:
+                        all_func_list.append(func_name)
+                    if func_content_lines:
+                        methods_content.extend(func_content_lines)
+                    func_params = [line.strip()]
+                    annotation_lines = []
+        func_name, func_content_lines = insert_class_method_lines(
+            func_params, annotation_lines, current_file_type)
+        if func_name:
+            all_func_list.append(func_name)
+        if func_content_lines:
+            methods_content.extend(func_content_lines)
+        return import_lines, all_func_list, methods_content
+
+
+def generate_file(current_file_type: str):
+    current_class_name = get_class_name(current_file_type)
+    import_lines, all_func_list, methods_content = get_methods_from_path_file(
+        current_file_type)
+    with open(f"megfile/{current_file_type}.py", 'w') as f:
+        for line in import_lines:
+            f.write('\n')
+            f.write(line)
+
+        f.write("\n\n__all__ = [\n")
+        for func_name in all_func_list:
+            if func_name != current_class_name and func_name != 'HttpsPath':
+                f.write(f"    '{func_name}',\n")
+        f.write("]\n\n")
+
+        for line in methods_content:
+            f.write('\n')
+            f.write(line)
+
+
+if __name__ == "__main__":
+    for t in ['s3', 'fs', 'http', 'stdio', 'sftp', 'hdfs']:
+        generate_file(t)

megfile-3.0.6.post1.dist-info/RECORD

@@ -1,52 +0,0 @@
-megfile/__init__.py,sha256=MT8SIXsmEUvtSpd1GHv6e3fFfR1gRnlEdkNNqv3gngo,6534
-megfile/cli.py,sha256=vPrFVWVx1iQOzv6QE1BChix5pCpbzkiGlB2S4eBW8gM,22445
-megfile/config.py,sha256=YeLuxsFoJz6ZmZq7Pufe_021PeRWs4H6GN-v0MvsV6E,1136
-megfile/errors.py,sha256=jVkQAv3XAmAxh2l2gGc5wptNsx61uGCHwX02ugQynns,13733
-megfile/fs.py,sha256=xLVSS4VWp2xcA7-vlVPbgJBG6s0zxH3JGaINwd9x57U,11850
-megfile/fs_path.py,sha256=2GbJYB4qbRhThWedM7VgbhjaNTKCENYUfsPBFp9JGfU,40651
-megfile/hdfs.py,sha256=9kEf6b-TVy6I3B_j5jufODzjNA5ngdTqW_mEForGFas,9200
-megfile/hdfs_path.py,sha256=iQGIS81PhV7bl9xAZkV14WjCp7O9Fb9yz12RfL2Z41w,27402
-megfile/http.py,sha256=a3oAuARSSaIU8VMx86Mui0N5Vh-EI0AoHnwxRU5DSMU,2032
-megfile/http_path.py,sha256=nCMx_8ETZynfDbNOnNkwY8vzHzCnFCx_H33PHVJQlug,16261
-megfile/interfaces.py,sha256=h3tWE8hVt5S-HopaMAX6lunPJ97vzhv6jH_2HubcDNc,6219
-megfile/pathlike.py,sha256=JhSP_bxRI7bwVGkubVvIkAqSYDcxdJr9h1DfZXEgc-g,29785
-megfile/s3.py,sha256=RmfUsmMVUD9-7A_O2d55dHBZ8_QX2FiFV4QAd_a1DZs,12830
-megfile/s3_path.py,sha256=S0rsYibPxcLOTOnz3nBS4OmsPcbL23570UsrN4NrJrM,92664
-megfile/sftp.py,sha256=dp0x84RYnxEX43NNIHqNZBCd69HgPqVg0EZPKT6vsCg,12978
-megfile/sftp_path.py,sha256=B_uOcKBdhZaYKh7mTieY1v-cvAheixrE3UFhA_JF6Z0,53253
-megfile/smart.py,sha256=vnm1sgxYShPSMVUl0qbVFnXu7I8NgCLHcxAXyPxXRr8,35967
-megfile/smart_path.py,sha256=Y0UFh4J2ccydRY2W-wX2ubaf9zzJx1M2nf-VLBGe4mk,6749
-megfile/stdio.py,sha256=yRhlfUA2DHi3bq-9cXsSlbLCnHvS_zvglO2IYYyPsGc,707
-megfile/stdio_path.py,sha256=eQulTXUwHvUKA-5PKCGfVNiEPkJhG9YtVhtU58OcmoM,2873
-megfile/version.py,sha256=r4nKj4bMHJNHtUJ8_7SBr2SUSvpgIaweKjVUnOyADzg,25
-megfile/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-megfile/lib/base_prefetch_reader.py,sha256=rrHB9FdVX_MKOJD2lGZ00LruTjRY9u0Hj4UdI-CxNO0,13231
-megfile/lib/combine_reader.py,sha256=XFSqEY5A5X5Uf7eQ6AXAzrvNteESSXvKNVPktGjo3KY,4546
-megfile/lib/compare.py,sha256=yG2fZve_gMg32rQVCdwixBdqgYRsjn-24TqhALQaOrA,2233
-megfile/lib/compat.py,sha256=0wt3_atcYhSLCxUj_WuDlQa3E1atjZfwJQ12thiFh5Q,234
-megfile/lib/fnmatch.py,sha256=HgdlnEWBsdFUOZqnW_v1kj1jeH_9lMcCqW85pyMu4vM,4054
-megfile/lib/glob.py,sha256=7i9dIput9rI9JIPyTZX-JDmFS7IP_THlX1k-35foAfw,9732
-megfile/lib/hdfs_prefetch_reader.py,sha256=50oVBNmEB8KRNwHnyDzxpxvwNABWxbOaIjyiTkmcemU,2080
-megfile/lib/hdfs_tools.py,sha256=t4GeoBxO0HPahIQDrsK17WBsLZtcfAaNwWfappzZ5q8,442
-megfile/lib/http_prefetch_reader.py,sha256=1F94fFY7ZtyP7Bm9Wphu0iLtvOt771rehhouflqVojk,4328
-megfile/lib/joinpath.py,sha256=D4Px6-lnDDpYs1LMUHkTIGqMPJQ0oCBGfTzREs373iU,929
-megfile/lib/lazy_handler.py,sha256=f1rip2_T57vVo0WRNXve2bAa4LArvVheMfQg1S0vFzg,1915
-megfile/lib/s3_buffered_writer.py,sha256=ZvCtaJsGZdFy-8K6EMHFakzu3oIKXnKof-a6nBa6AY4,7017
-megfile/lib/s3_cached_handler.py,sha256=xuWiThi6pJtGL_ErSBmcu8rDv1XyXNmEhiFBnRF4NWU,1412
-megfile/lib/s3_limited_seekable_writer.py,sha256=14gySEdbS349hpSdUcX5uvRr93HACv7BRZbgIki4b34,6150
-megfile/lib/s3_memory_handler.py,sha256=6Tj89xzc8z-FycVggGpjF_8lEbPsqRVB6undZwWsugo,3971
-megfile/lib/s3_pipe_handler.py,sha256=hG8sEajO9dv9bLTeXsERxDioHHhzi4t8NC61lSbYk94,3557
-megfile/lib/s3_prefetch_reader.py,sha256=iLvPaJGlJHaZ6jcyNPf-8A8URJgoJ0RN0C1YD--6wZ4,4285
-megfile/lib/s3_share_cache_reader.py,sha256=jyx6f3LAR05qtf-qq_RTu8wE6wqTrqg4qkwWRSdPGQ4,3600
-megfile/lib/shadow_handler.py,sha256=IbFyTw107t-yWH0cGrDjAJX-CS3xeEr77_PTGsnSgk4,2683
-megfile/lib/stdio_handler.py,sha256=QDWtcZxz-hzi-rqQUiSlR3NrihX1fjK_Rj9T2mdTFEg,2044
-megfile/lib/url.py,sha256=VbQLjo0s4AaV0iSk66BcjI68aUTcN9zBZ5x6-cM4Qvs,103
-megfile/utils/__init__.py,sha256=xrBIJcVJTb_yR68ekAyd9u4HQ46s3xgIjUZoH_Lx7hU,9531
-megfile/utils/mutex.py,sha256=-2KH3bNovKRd9zvsXq9n3bWM7rQdoG9hO7tUPxVG_Po,2538
-megfile-3.0.6.post1.dist-info/LICENSE,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
-megfile-3.0.6.post1.dist-info/LICENSE.pyre,sha256=9lf5nT-5ZH25JijpYAequ0bl8E8z5JmZB1qrjiUMp84,1080
-megfile-3.0.6.post1.dist-info/METADATA,sha256=AijH38cyPATFlFh284KHA-c6PWGUF1y3mterInTjark,9093
-megfile-3.0.6.post1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-megfile-3.0.6.post1.dist-info/entry_points.txt,sha256=M6ZWSSv5_5_QtIpZafy3vq7WuOJ_5dSGQQnEZbByt2Q,49
-megfile-3.0.6.post1.dist-info/top_level.txt,sha256=i3rMgdU1ZAJekAceojhA-bkm3749PzshtRmLTbeLUPQ,8
-megfile-3.0.6.post1.dist-info/RECORD,,

megfile-3.0.6.post1.dist-info/top_level.txt

@@ -1 +0,0 @@
-megfile