megfile 2.2.7__py3-none-any.whl → 2.2.8.post1__py3-none-any.whl

megfile/__init__.py

@@ -1,5 +1,7 @@
 from megfile.fs import fs_abspath, fs_access, fs_cwd, fs_exists, fs_expanduser, fs_getmd5, fs_getmtime, fs_getsize, fs_glob, fs_glob_stat, fs_home, fs_iglob, fs_isabs, fs_isdir, fs_isfile, fs_islink, fs_ismount, fs_listdir, fs_load_from, fs_lstat, fs_makedirs, fs_move, fs_readlink, fs_realpath, fs_relpath, fs_remove, fs_rename, fs_resolve, fs_save_as, fs_scan, fs_scan_stat, fs_scandir, fs_stat, fs_symlink, fs_sync, fs_unlink, fs_walk, is_fs
 from megfile.fs_path import FSPath
+from megfile.hdfs import hdfs_exists, hdfs_getmd5, hdfs_getmtime, hdfs_getsize, hdfs_glob, hdfs_glob_stat, hdfs_iglob, hdfs_isdir, hdfs_isfile, hdfs_listdir, hdfs_load_from, hdfs_makedirs, hdfs_move, hdfs_open, hdfs_remove, hdfs_save_as, hdfs_scan, hdfs_scan_stat, hdfs_scandir, hdfs_stat, hdfs_unlink, hdfs_walk, is_hdfs
+from megfile.hdfs_path import HdfsPath
 from megfile.http import http_exists, http_getmtime, http_getsize, http_open, http_stat, is_http
 from megfile.http_path import HttpPath, HttpsPath
 from megfile.s3 import is_s3, s3_access, s3_buffered_open, s3_cached_open, s3_concat, s3_copy, s3_download, s3_exists, s3_getmd5, s3_getmtime, s3_getsize, s3_glob, s3_glob_stat, s3_hasbucket, s3_iglob, s3_isdir, s3_isfile, s3_listdir, s3_load_content, s3_load_from, s3_lstat, s3_makedirs, s3_memory_open, s3_move, s3_open, s3_path_join, s3_pipe_open, s3_prefetch_open, s3_readlink, s3_remove, s3_rename, s3_save_as, s3_scan, s3_scan_stat, s3_scandir, s3_stat, s3_symlink, s3_sync, s3_unlink, s3_upload, s3_walk
@@ -178,6 +180,29 @@ __all__ = [
     'sftp_copy',
     'sftp_sync',
     'sftp_concat',
+    'is_hdfs',
+    'hdfs_exists',
+    'hdfs_stat',
+    'hdfs_getmtime',
+    'hdfs_getsize',
+    'hdfs_isdir',
+    'hdfs_isfile',
+    'hdfs_listdir',
+    'hdfs_load_from',
+    'hdfs_move',
+    'hdfs_remove',
+    'hdfs_scan',
+    'hdfs_scan_stat',
+    'hdfs_scandir',
+    'hdfs_unlink',
+    'hdfs_walk',
+    'hdfs_getmd5',
+    'hdfs_save_as',
+    'hdfs_open',
+    'hdfs_glob',
+    'hdfs_glob_stat',
+    'hdfs_iglob',
+    'hdfs_makedirs',
     'S3Path',
     'FSPath',
     'HttpPath',
@@ -185,4 +210,5 @@ __all__ = [
     'StdioPath',
     'SmartPath',
     'SftpPath',
+    'HdfsPath',
 ]

megfile/cli.py

@@ -1,3 +1,4 @@
+import configparser
 import logging
 import os
 import shutil
@@ -9,9 +10,10 @@ from functools import partial
 import click
 from tqdm import tqdm
 
+from megfile.hdfs_path import DEFAULT_HDFS_TIMEOUT
 from megfile.interfaces import FileEntry
 from megfile.lib.glob import get_non_glob_dir, has_magic
-from megfile.smart import _smart_sync_single_file, smart_copy, smart_getmd5, smart_getmtime, smart_getsize, smart_glob_stat, smart_isdir, smart_isfile, smart_makedirs, smart_move, smart_open, smart_path_join, smart_remove, smart_rename, smart_scan_stat, smart_scandir, smart_stat, smart_sync, smart_sync_with_progress, smart_touch, smart_unlink
+from megfile.smart import _smart_sync_single_file, smart_copy, smart_exists, smart_getmd5, smart_getmtime, smart_getsize, smart_glob_stat, smart_isdir, smart_isfile, smart_makedirs, smart_move, smart_open, smart_path_join, smart_remove, smart_rename, smart_scan_stat, smart_scandir, smart_stat, smart_sync, smart_sync_with_progress, smart_touch, smart_unlink
 from megfile.smart_path import SmartPath
 from megfile.utils import get_human_size
 from megfile.version import VERSION
@@ -286,7 +288,7 @@ def sync(
             src_root_path = get_non_glob_dir(src_path)
 
             def scan_func(path):
-                for glob_file_entry in smart_glob_stat(path):
+                for glob_file_entry in smart_glob_stat(path, missing_ok=False):
                     if glob_file_entry.is_file():
                         yield glob_file_entry
                     else:
@@ -295,7 +297,8 @@ def sync(
                             yield file_entry
         else:
             src_root_path = src_path
-            scan_func = partial(smart_scan_stat, followlinks=True)
+            scan_func = partial(
+                smart_scan_stat, followlinks=True, missing_ok=False)
 
         if progress_bar and not quiet:
             print('building progress bar', end='\r')
@@ -493,7 +496,8 @@ def config():
     '--path',
     type=str,
     default='~/.aws/credentials',
-    help='s3 config file')
+    help='s3 config file, default is $HOME/.aws/credentials',
+)
 @click.option(
     '-n', '--profile-name', type=str, default='default', help='s3 config file')
 @click.argument('aws_access_key_id')
@@ -504,6 +508,8 @@ def config():
 def s3(
         path, profile_name, aws_access_key_id, aws_secret_access_key,
         endpoint_url, addressing_style, no_cover):
+    path = os.path.expanduser(path)
+
     config_dict = {
         'name': profile_name,
         'aws_access_key_id': aws_access_key_id,
@@ -533,6 +539,7 @@ def s3(
                     s3['addressing_style'])
         return content
 
+    os.makedirs(os.path.dirname(path), exist_ok=True)  # make sure dirpath exist
     if not os.path.exists(path):  #If this file doesn't exist.
         content_str = dumps(config_dict)
         with open(path, 'w') as fp:
@@ -556,7 +563,7 @@ def s3(
         # Given profile_name has been used.
         if cur_name == profile_name:
             if no_cover:  # default True(cover the same-name config).
-                raise NameError(f'{profile_name} has been used.')
+                raise NameError(f'profile-name has been used: {profile_name}')
             used = True
             sections[i] = dumps(config_dict)
             continue
@@ -570,6 +577,51 @@ def s3(
     click.echo(f'Your oss config has been saved into {path}')
 
 
+@config.command(short_help='Return the config file for s3')
+@click.argument('url')
+@click.option(
+    '-p',
+    '--path',
+    default='~/.hdfscli.cfg',
+    help='s3 config file, default is $HOME/.hdfscli.cfg',
+)
+@click.option('-n', '--profile-name', default='default', help='s3 config file')
+@click.option('-u', '--user', help='user name')
+@click.option('-r', '--root', help="hdfs path's root dir")
+@click.option('-t', '--token', help="token for requesting hdfs server")
+@click.option(
+    '-o',
+    '--timeout',
+    help=f"request hdfs server timeout, default {DEFAULT_HDFS_TIMEOUT}")
+@click.option('--no-cover', is_flag=True, help='Not cover the same-name config')
+def hdfs(url, path, profile_name, user, root, token, timeout, no_cover):
+    path = os.path.expanduser(path)
+    current_config = {
+        'url': url,
+        'user': user,
+        'root': root,
+        'token': token,
+        'timeout': timeout,
+    }
+    profile_name = f"{profile_name}.alias"
+    config = configparser.ConfigParser()
+    if os.path.exists(path):
+        config.read(path)
+    if 'global' not in config.sections():
+        config['global'] = {'default.alias': 'default'}
+    if profile_name in config.sections():
+        if no_cover:
+            raise NameError(f'profile-name has been used: {profile_name[:-6]}')
+    else:
+        config[profile_name] = {}
+    for key, value in current_config.items():
+        if value:
+            config[profile_name][key] = value
+    with open(path, 'w') as fp:
+        config.write(fp)
+    click.echo(f'Your hdfs config has been saved into {path}')
+
+
 if __name__ == '__main__':
     # Usage: python -m megfile.cli
     safe_cli()  # pragma: no cover

megfile/errors.py

@@ -384,3 +384,28 @@ def s3_error_code_should_retry(error: str) -> bool:
     if error in ['InternalError', 'ServiceUnavailable', 'SlowDown']:
         return True
     return False
+
+
+def translate_hdfs_error(hdfs_error: Exception, hdfs_path: PathLike):
+    from megfile.lib.hdfs_tools import hdfs_api
+
+    if hdfs_api and isinstance(hdfs_error, hdfs_api.HdfsError):
+        if hdfs_error.message and 'Path is not a file' in hdfs_error.message:  # pytype: disable=attribute-error
+            return IsADirectoryError('Is a directory: %r' % hdfs_path)
+        elif hdfs_error.message and 'Path is not a directory' in hdfs_error.message:  # pytype: disable=attribute-error
+            return NotADirectoryError('Not a directory: %r' % hdfs_path)
+        elif hdfs_error.status_code in (401, 403):  # pytype: disable=attribute-error
+            return PermissionError('Permission denied: %r' % hdfs_path)
+        elif hdfs_error.status_code == 400:  # pytype: disable=attribute-error
+            return ValueError(f'{hdfs_error.message}, path: {hdfs_path}')  # pytype: disable=attribute-error
+        elif hdfs_error.status_code == 404:  # pytype: disable=attribute-error
+            return FileNotFoundError(f'No match file: {hdfs_path}')
+    return hdfs_error
+
+
+@contextmanager
+def raise_hdfs_error(hdfs_path: PathLike):
+    try:
+        yield
+    except Exception as error:
+        raise translate_hdfs_error(error, hdfs_path)

megfile/fs.py

@@ -1,6 +1,6 @@
 from typing import BinaryIO, Callable, Iterator, List, Optional, Tuple
 
-from megfile.fs_path import FSPath, StatResult, _make_stat, fs_cwd, fs_glob, fs_glob_stat, fs_home, fs_iglob, fs_makedirs, fs_move, fs_path_join, fs_readlink, fs_rename, fs_resolve, is_fs
+from megfile.fs_path import FSPath, StatResult, _make_stat, fs_cwd, fs_glob, fs_glob_stat, fs_home, fs_iglob, fs_lstat, fs_makedirs, fs_move, fs_path_join, fs_readlink, fs_rename, fs_resolve, is_fs
 from megfile.interfaces import Access, FileEntry, PathLike, StatResult
 
 __all__ = [
@@ -18,6 +18,7 @@ __all__ = [
     'fs_resolve',
     'fs_move',
     'fs_makedirs',
+    'fs_lstat',
     'fs_isabs',
     'fs_abspath',
     'fs_access',
@@ -36,7 +37,6 @@ __all__ = [
     'fs_scan_stat',
     'fs_scandir',
     'fs_stat',
-    'fs_lstat',
     'fs_unlink',
     'fs_walk',
     'fs_getmd5',
@@ -260,16 +260,6 @@ def fs_stat(path: PathLike, follow_symlinks=True) -> StatResult:
     return FSPath(path).stat(follow_symlinks)
 
 
-def fs_lstat(path: PathLike) -> StatResult:
-    '''
-    Like Path.stat() but, if the path points to a symbolic link, return the symbolic link’s information rather than its target’s.
-
-    :param path: Given path
-    :returns: StatResult
-    '''
-    return FSPath(path).lstat()
-
-
 def fs_unlink(path: PathLike, missing_ok: bool = False) -> None:
     '''
     Remove the file on fs

megfile/fs_path.py

@@ -37,6 +37,7 @@ __all__ = [
     'fs_resolve',
     'fs_move',
     'fs_makedirs',
+    'fs_lstat',
 ]
 
 
@@ -200,6 +201,16 @@ def fs_makedirs(path: PathLike, exist_ok: bool = False):
     return FSPath(path).mkdir(parents=True, exist_ok=exist_ok)
 
 
+def fs_lstat(path: PathLike) -> StatResult:
+    '''
+    Like Path.stat() but, if the path points to a symbolic link, return the symbolic link’s information rather than its target’s.
+
+    :param path: Given path
+    :returns: StatResult
+    '''
+    return FSPath(path).lstat()
+
+
 @SmartPath.register
 class FSPath(URIPath):
     """file protocol
@@ -601,14 +612,6 @@ class FSPath(URIPath):
                     mtime = stat.st_mtime
         return result._replace(size=size, ctime=ctime, mtime=mtime)
 
-    def lstat(self) -> StatResult:
-        '''
-        Like Path.stat() but, if the path points to a symbolic link, return the symbolic link’s information rather than its target’s.
-
-        :returns: StatResult
-        '''
-        return self.stat(follow_symlinks=False)
-
     def unlink(self, missing_ok: bool = False) -> None:
         '''
         Remove the file on fs

megfile/hdfs.py

@@ -0,0 +1,269 @@
+from typing import IO, AnyStr, BinaryIO, Iterator, List, Optional, Tuple
+
+from megfile.hdfs_path import HdfsPath, hdfs_glob, hdfs_glob_stat, hdfs_iglob, hdfs_makedirs, is_hdfs
+from megfile.interfaces import FileEntry, PathLike, StatResult
+
+__all__ = [
+    'is_hdfs',
+    'hdfs_glob',
+    'hdfs_glob_stat',
+    'hdfs_iglob',
+    'hdfs_makedirs',
+    'hdfs_exists',
+    'hdfs_stat',
+    'hdfs_getmtime',
+    'hdfs_getsize',
+    'hdfs_isdir',
+    'hdfs_isfile',
+    'hdfs_listdir',
+    'hdfs_load_from',
+    'hdfs_move',
+    'hdfs_remove',
+    'hdfs_scan',
+    'hdfs_scan_stat',
+    'hdfs_scandir',
+    'hdfs_unlink',
+    'hdfs_walk',
+    'hdfs_getmd5',
+    'hdfs_save_as',
+    'hdfs_open',
+]
+
+
+def hdfs_exists(path: PathLike, followlinks: bool = False) -> bool:
+    '''
+    Test if path exists
+
+    If the bucket of path are not permitted to read, return False
+
+    :param path: Given path
+    :returns: True if path eixsts, else False
+    '''
+    return HdfsPath(path).exists(followlinks)
+
+
+def hdfs_stat(path: PathLike, follow_symlinks=True) -> StatResult:
+    '''
+    Get StatResult of path file, including file size and mtime, referring to hdfs_getsize and hdfs_getmtime
+
+    If path is not an existent path, which means hdfs_exist(path) returns False, then raise FileNotFoundError
+    If attempt to get StatResult of complete hdfs, such as hdfs_dir_url == 'hdfs://', raise BucketNotFoundError
+
+    :param path: Given path
+    :returns: StatResult
+    :raises: FileNotFoundError
+    '''
+    return HdfsPath(path).stat(follow_symlinks)
+
+
+def hdfs_getmtime(path: PathLike, follow_symlinks: bool = False) -> float:
+    '''
+    Get last-modified time of the file on the given path path (in Unix timestamp format).
+    If the path is an existent directory, return the latest modified time of all file in it. The mtime of empty directory is 1970-01-01 00:00:00
+
+    If path is not an existent path, which means hdfs_exist(path) returns False, then raise FileNotFoundError
+
+    :param path: Given path
+    :returns: Last-modified time
+    :raises: FileNotFoundError
+    '''
+    return HdfsPath(path).getmtime(follow_symlinks)
+
+
+def hdfs_getsize(path: PathLike, follow_symlinks: bool = False) -> int:
+    '''
+    Get file size on the given path path (in bytes).
+    If the path in a directory, return the sum of all file size in it, including file in subdirectories (if exist).
+    The result excludes the size of directory itself. In other words, return 0 Byte on an empty directory path.
+
+    If path is not an existent path, which means hdfs_exist(path) returns False, then raise FileNotFoundError
+
+    :param path: Given path
+    :returns: File size
+    :raises: FileNotFoundError
+    '''
+    return HdfsPath(path).getsize(follow_symlinks)
+
+
+def hdfs_isdir(path: PathLike, followlinks: bool = False) -> bool:
+    '''
+    Test if an hdfs url is directory
+    Specific procedures are as follows:
+    If there exists a suffix, of which ``os.path.join(path, suffix)`` is a file
+    If the url is empty bucket or hdfs://
+
+    :param path: Given path
+    :param followlinks: whether followlinks is True or False, result is the same. Because hdfs symlink not support dir.
+    :returns: True if path is hdfs directory, else False
+    '''
+    return HdfsPath(path).is_dir(followlinks)
+
+
+def hdfs_isfile(path: PathLike, followlinks: bool = False) -> bool:
+    '''
+    Test if an path is file
+
+    :param path: Given path
+    :returns: True if path is hdfs file, else False
+    '''
+    return HdfsPath(path).is_file(followlinks)
+
+
+def hdfs_listdir(path: PathLike, followlinks: bool = False) -> List[str]:
+    '''
+    Get all contents of given path.
+
+    :param path: Given path
+    :returns: All contents have prefix of path.
+    :raises: FileNotFoundError, NotADirectoryError
+    '''
+    return HdfsPath(path).listdir(followlinks)
+
+
+def hdfs_load_from(path: PathLike, followlinks: bool = False) -> BinaryIO:
+    '''Read all content in binary on specified path and write into memory
+
+    User should close the BinaryIO manually
+
+    :param path: Given path
+    :returns: BinaryIO
+    '''
+    return HdfsPath(path).load(followlinks)
+
+
+def hdfs_move(src_path: PathLike, dst_path: PathLike) -> None:
+    '''
+    Move file/directory path from src_path to dst_path
+
+    :param src_path: Given path
+    :param dst_path: Given destination path
+    '''
+    return HdfsPath(src_path).move(dst_path)
+
+
+def hdfs_remove(path: PathLike, missing_ok: bool = False) -> None:
+    '''
+    Remove the file or directory on hdfs, `hdfs://` and `hdfs://bucket` are not permitted to remove
+
+    :param path: Given path
+    :param missing_ok: if False and target file/directory not exists, raise FileNotFoundError
+    :raises: FileNotFoundError, UnsupportedError
+    '''
+    return HdfsPath(path).remove(missing_ok)
+
+
+def hdfs_scan(
+        path: PathLike, missing_ok: bool = True,
+        followlinks: bool = False) -> Iterator[str]:
+    '''
+    Iteratively traverse only files in given hdfs directory.
+    Every iteration on generator yields a path string.
+
+    If path is a file path, yields the file only
+    If path is a non-existent path, return an empty generator
+    If path is a bucket path, return all file paths in the bucket
+    If path is an empty bucket, return an empty generator
+    If path doesn't contain any bucket, which is path == 'hdfs://', raise UnsupportedError. walk() on complete hdfs is not supported in megfile
+
+    :param path: Given path
+    :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+    :raises: UnsupportedError
+    :returns: A file path generator
+    '''
+    return HdfsPath(path).scan(missing_ok, followlinks)
+
+
+def hdfs_scan_stat(
+        path: PathLike, missing_ok: bool = True,
+        followlinks: bool = False) -> Iterator[FileEntry]:
+    '''
+    Iteratively traverse only files in given directory.
+    Every iteration on generator yields a tuple of path string and file stat
+
+    :param path: Given path
+    :param missing_ok: If False and there's no file in the directory, raise FileNotFoundError
+    :raises: UnsupportedError
+    :returns: A file path generator
+    '''
+    return HdfsPath(path).scan_stat(missing_ok, followlinks)
+
+
+def hdfs_scandir(path: PathLike,
+                 followlinks: bool = False) -> Iterator[FileEntry]:
+    '''
+    Get all contents of given path, the order of result is not guaranteed.
+
+    :param path: Given path
+    :returns: All contents have prefix of path
+    :raises: FileNotFoundError, NotADirectoryError
+    '''
+    return HdfsPath(path).scandir(followlinks)
+
+
+def hdfs_unlink(path: PathLike, missing_ok: bool = False) -> None:
+    '''
+    Remove the file on hdfs
+
+    :param path: Given path
+    :param missing_ok: if False and target file not exists, raise FileNotFoundError
+    :raises: FileNotFoundError, IsADirectoryError
+    '''
+    return HdfsPath(path).unlink(missing_ok)
+
+
+def hdfs_walk(path: PathLike, followlinks: bool = False
+             ) -> Iterator[Tuple[str, List[str], List[str]]]:
+    '''
+    Iteratively traverse the given hdfs directory, in top-bottom order. In other words, firstly traverse parent directory, if subdirectories exist, traverse the subdirectories.
+    Every iteration on generator yields a 3-tuple: (root, dirs, files)
+
+    - root: Current hdfs path;
+    - dirs: Name list of subdirectories in current directory.
+    - files: Name list of files in current directory.
+
+    If path is a file path, return an empty generator
+    If path is a non-existent path, return an empty generator
+    If path is a bucket path, bucket will be the top directory, and will be returned at first iteration of generator
+    If path is an empty bucket, only yield one 3-tuple (notes: hdfs doesn't have empty directory)
+    If path doesn't contain any bucket, which is path == 'hdfs://', raise UnsupportedError. walk() on complete hdfs is not supported in megfile
+
+    :param path: Given path
+    :param followlinks: whether followlinks is True or False, result is the same. Because hdfs not support symlink.
+    :returns: A 3-tuple generator
+    '''
+    return HdfsPath(path).walk(followlinks)
+
+
+def hdfs_getmd5(
+        path: PathLike, recalculate: bool = False,
+        followlinks: bool = False) -> str:
+    '''
+    Get checksum of the file or dir.
+
+    :param path: Given path
+    :param recalculate: Ignore this parameter, just for compatibility
+    :param followlinks: Ignore this parameter, just for compatibility
+    :returns: checksum
+    '''
+    return HdfsPath(path).md5(recalculate, followlinks)
+
+
+def hdfs_save_as(file_object: BinaryIO, path: PathLike):
+    '''Write the opened binary stream to specified path, but the stream won't be closed
+
+    :param path: Given path
+    :param file_object: Stream to be read
+    '''
+    return HdfsPath(path).save(file_object)
+
+
+def hdfs_open(
+        path: PathLike,
+        mode: str = 'r',
+        *,
+        buffering: Optional[int] = None,
+        encoding: Optional[str] = None,
+        errors: Optional[str] = None,
+        **kwargs) -> IO[AnyStr]:  # pytype: disable=signature-mismatch
+    return HdfsPath(path).open(
+        mode, buffering=buffering, encoding=encoding, errors=errors)