paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/ppdet/utils/cli.py

@@ -1,151 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from argparse import ArgumentParser, RawDescriptionHelpFormatter
-
-import yaml
-import re
-from paddlex.ppdet.core.workspace import get_registered_modules, dump_value
-
-__all__ = ['ColorTTY', 'ArgsParser']
-
-
-class ColorTTY(object):
-    def __init__(self):
-        super(ColorTTY, self).__init__()
-        self.colors = ['red', 'green', 'yellow', 'blue', 'magenta', 'cyan']
-
-    def __getattr__(self, attr):
-        if attr in self.colors:
-            color = self.colors.index(attr) + 31
-
-            def color_message(message):
-                return "[{}m{}".format(color, message)
-
-            setattr(self, attr, color_message)
-            return color_message
-
-    def bold(self, message):
-        return self.with_code('01', message)
-
-    def with_code(self, code, message):
-        return "[{}m{}".format(code, message)
-
-
-class ArgsParser(ArgumentParser):
-    def __init__(self):
-        super(ArgsParser, self).__init__(
-            formatter_class=RawDescriptionHelpFormatter)
-        self.add_argument("-c", "--config", help="configuration file to use")
-        self.add_argument(
-            "-o", "--opt", nargs='*', help="set configuration options")
-
-    def parse_args(self, argv=None):
-        args = super(ArgsParser, self).parse_args(argv)
-        assert args.config is not None, \
-            "Please specify --config=configure_file_path."
-        args.opt = self._parse_opt(args.opt)
-        return args
-
-    def _parse_opt(self, opts):
-        config = {}
-        if not opts:
-            return config
-        for s in opts:
-            s = s.strip()
-            k, v = s.split('=', 1)
-            if '.' not in k:
-                config[k] = yaml.load(v, Loader=yaml.Loader)
-            else:
-                keys = k.split('.')
-                if keys[0] not in config:
-                    config[keys[0]] = {}
-                cur = config[keys[0]]
-                for idx, key in enumerate(keys[1:]):
-                    if idx == len(keys) - 2:
-                        cur[key] = yaml.load(v, Loader=yaml.Loader)
-                    else:
-                        cur[key] = {}
-                        cur = cur[key]
-        return config
-
-
-def print_total_cfg(config):
-    modules = get_registered_modules()
-    color_tty = ColorTTY()
-    green = '___{}___'.format(color_tty.colors.index('green') + 31)
-
-    styled = {}
-    for key in config.keys():
-        if not config[key]:  # empty schema
-            continue
-
-        if key not in modules and not hasattr(config[key], '__dict__'):
-            styled[key] = config[key]
-            continue
-        elif key in modules:
-            module = modules[key]
-        else:
-            type_name = type(config[key]).__name__
-            if type_name in modules:
-                module = modules[type_name].copy()
-                module.update({
-                    k: v
-                    for k, v in config[key].__dict__.items()
-                    if k in module.schema
-                })
-                key += " ({})".format(type_name)
-        default = module.find_default_keys()
-        missing = module.find_missing_keys()
-        mismatch = module.find_mismatch_keys()
-        extra = module.find_extra_keys()
-        dep_missing = []
-        for dep in module.inject:
-            if isinstance(module[dep], str) and module[dep] != '<value>':
-                if module[dep] not in modules:  # not a valid module
-                    dep_missing.append(dep)
-                else:
-                    dep_mod = modules[module[dep]]
-                    # empty dict but mandatory
-                    if not dep_mod and dep_mod.mandatory():
-                        dep_missing.append(dep)
-        override = list(
-            set(module.keys()) - set(default) - set(extra) - set(dep_missing))
-        replacement = {}
-        for name in set(override + default + extra + mismatch + missing):
-            new_name = name
-            if name in missing:
-                value = "<missing>"
-            else:
-                value = module[name]
-
-            if name in extra:
-                value = dump_value(value) + " <extraneous>"
-            elif name in mismatch:
-                value = dump_value(value) + " <type mismatch>"
-            elif name in dep_missing:
-                value = dump_value(value) + " <module config missing>"
-            elif name in override and value != '<missing>':
-                mark = green
-                new_name = mark + name
-            replacement[new_name] = value
-        styled[key] = replacement
-    buffer = yaml.dump(styled, default_flow_style=False, default_style='')
-    buffer = (re.sub(r"<missing>", r"[31m<missing>[0m", buffer))
-    buffer = (re.sub(r"<extraneous>", r"[33m<extraneous>[0m", buffer))
-    buffer = (re.sub(r"<type mismatch>", r"[31m<type mismatch>[0m", buffer))
-    buffer = (re.sub(r"<module config missing>",
-                     r"[31m<module config missing>[0m", buffer))
-    buffer = re.sub(r"___(\d+)___(.*?):", r"[\1m\2[0m:", buffer)
-    print(buffer)

paddlex/ppdet/utils/colormap.py

@@ -1,56 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import numpy as np
-
-
-def colormap(rgb=False):
-    """
-    Get colormap
-    """
-    color_list = np.array([
-        0.000, 0.447, 0.741, 0.850, 0.325, 0.098, 0.929, 0.694, 0.125, 0.494,
-        0.184, 0.556, 0.466, 0.674, 0.188, 0.301, 0.745, 0.933, 0.635, 0.078,
-        0.184, 0.300, 0.300, 0.300, 0.600, 0.600, 0.600, 1.000, 0.000, 0.000,
-        1.000, 0.500, 0.000, 0.749, 0.749, 0.000, 0.000, 1.000, 0.000, 0.000,
-        0.000, 1.000, 0.667, 0.000, 1.000, 0.333, 0.333, 0.000, 0.333, 0.667,
-        0.000, 0.333, 1.000, 0.000, 0.667, 0.333, 0.000, 0.667, 0.667, 0.000,
-        0.667, 1.000, 0.000, 1.000, 0.333, 0.000, 1.000, 0.667, 0.000, 1.000,
-        1.000, 0.000, 0.000, 0.333, 0.500, 0.000, 0.667, 0.500, 0.000, 1.000,
-        0.500, 0.333, 0.000, 0.500, 0.333, 0.333, 0.500, 0.333, 0.667, 0.500,
-        0.333, 1.000, 0.500, 0.667, 0.000, 0.500, 0.667, 0.333, 0.500, 0.667,
-        0.667, 0.500, 0.667, 1.000, 0.500, 1.000, 0.000, 0.500, 1.000, 0.333,
-        0.500, 1.000, 0.667, 0.500, 1.000, 1.000, 0.500, 0.000, 0.333, 1.000,
-        0.000, 0.667, 1.000, 0.000, 1.000, 1.000, 0.333, 0.000, 1.000, 0.333,
-        0.333, 1.000, 0.333, 0.667, 1.000, 0.333, 1.000, 1.000, 0.667, 0.000,
-        1.000, 0.667, 0.333, 1.000, 0.667, 0.667, 1.000, 0.667, 1.000, 1.000,
-        1.000, 0.000, 1.000, 1.000, 0.333, 1.000, 1.000, 0.667, 1.000, 0.167,
-        0.000, 0.000, 0.333, 0.000, 0.000, 0.500, 0.000, 0.000, 0.667, 0.000,
-        0.000, 0.833, 0.000, 0.000, 1.000, 0.000, 0.000, 0.000, 0.167, 0.000,
-        0.000, 0.333, 0.000, 0.000, 0.500, 0.000, 0.000, 0.667, 0.000, 0.000,
-        0.833, 0.000, 0.000, 1.000, 0.000, 0.000, 0.000, 0.167, 0.000, 0.000,
-        0.333, 0.000, 0.000, 0.500, 0.000, 0.000, 0.667, 0.000, 0.000, 0.833,
-        0.000, 0.000, 1.000, 0.000, 0.000, 0.000, 0.143, 0.143, 0.143, 0.286,
-        0.286, 0.286, 0.429, 0.429, 0.429, 0.571, 0.571, 0.571, 0.714, 0.714,
-        0.714, 0.857, 0.857, 0.857, 1.000, 1.000, 1.000
-    ]).astype(np.float32)
-    color_list = color_list.reshape((-1, 3)) * 255
-    if not rgb:
-        color_list = color_list[:, ::-1]
-    return color_list

paddlex/ppdet/utils/download.py

@@ -1,477 +0,0 @@
-#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import os.path as osp
-import sys
-import yaml
-import shutil
-import requests
-import tqdm
-import hashlib
-import base64
-import binascii
-import tarfile
-import zipfile
-
-from .voc_utils import create_list
-from paddlex.ppdet.core.workspace import BASE_KEY
-
-from .logger import setup_logger
-logger = setup_logger(__name__)
-
-__all__ = [
-    'get_weights_path', 'get_dataset_path', 'get_config_path',
-    'download_dataset', 'create_voc_list'
-]
-
-WEIGHTS_HOME = osp.expanduser("~/.cache/paddle/weights")
-DATASET_HOME = osp.expanduser("~/.cache/paddle/dataset")
-CONFIGS_HOME = osp.expanduser("~/.cache/paddle/configs")
-
-# dict of {dataset_name: (download_info, sub_dirs)}
-# download info: [(url, md5sum)]
-DATASETS = {
-    'coco': ([
-        (
-            'http://images.cocodataset.org/zips/train2017.zip',
-            'cced6f7f71b7629ddf16f17bbcfab6b2', ),
-        (
-            'http://images.cocodataset.org/zips/val2017.zip',
-            '442b8da7639aecaf257c1dceb8ba8c80', ),
-        (
-            'http://images.cocodataset.org/annotations/annotations_trainval2017.zip',
-            'f4bbac642086de4f52a3fdda2de5fa2c', ),
-    ], ["annotations", "train2017", "val2017"]),
-    'voc': ([
-        (
-            'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar',
-            '6cd6e144f989b92b3379bac3b3de84fd', ),
-        (
-            'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar',
-            'c52e279531787c972589f7e41ab4ae64', ),
-        (
-            'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar',
-            'b6e924de25625d8de591ea690078ad9f', ),
-    ], ["VOCdevkit/VOC2012", "VOCdevkit/VOC2007"]),
-    'wider_face': ([
-        (
-            'https://dataset.bj.bcebos.com/wider_face/WIDER_train.zip',
-            '3fedf70df600953d25982bcd13d91ba2', ),
-        (
-            'https://dataset.bj.bcebos.com/wider_face/WIDER_val.zip',
-            'dfa7d7e790efa35df3788964cf0bbaea', ),
-        (
-            'https://dataset.bj.bcebos.com/wider_face/wider_face_split.zip',
-            'a4a898d6193db4b9ef3260a68bad0dc7', ),
-    ], ["WIDER_train", "WIDER_val", "wider_face_split"]),
-    'fruit': ([(
-        'https://dataset.bj.bcebos.com/PaddleDetection_demo/fruit.tar',
-        'baa8806617a54ccf3685fa7153388ae6', ), ],
-              ['Annotations', 'JPEGImages']),
-    'roadsign_voc': ([(
-        'https://paddlemodels.bj.bcebos.com/object_detection/roadsign_voc.tar',
-        '8d629c0f880dd8b48de9aeff44bf1f3e', ), ], ['annotations', 'images']),
-    'roadsign_coco': ([(
-        'https://paddlemodels.bj.bcebos.com/object_detection/roadsign_coco.tar',
-        '49ce5a9b5ad0d6266163cd01de4b018e', ), ], ['annotations', 'images']),
-    'mot': (),
-    'objects365': ()
-}
-
-DOWNLOAD_RETRY_LIMIT = 3
-
-PPDET_WEIGHTS_DOWNLOAD_URL_PREFIX = 'https://paddledet.bj.bcebos.com/'
-
-
-def parse_url(url):
-    url = url.replace("ppdet://", PPDET_WEIGHTS_DOWNLOAD_URL_PREFIX)
-    return url
-
-
-def get_weights_path(url):
-    """Get weights path from WEIGHTS_HOME, if not exists,
-    download it from url.
-    """
-    url = parse_url(url)
-    path, _ = get_path(url, WEIGHTS_HOME)
-    return path
-
-
-def get_config_path(url):
-    """Get weights path from CONFIGS_HOME, if not exists,
-    download it from url.
-    """
-    url = parse_url(url)
-    path, _ = get_path(url, CONFIGS_HOME)
-    _download_config(path, url, CONFIGS_HOME)
-
-    return path
-
-
-def _download_config(cfg_path, cfg_url, cur_dir):
-    with open(cfg_path) as f:
-        cfg = yaml.load(f, Loader=yaml.Loader)
-
-    # download dependence base ymls
-    if BASE_KEY in cfg:
-        base_ymls = list(cfg[BASE_KEY])
-        for base_yml in base_ymls:
-            if base_yml.startswith("~"):
-                base_yml = os.path.expanduser(base_yml)
-                relpath = osp.relpath(base_yml, cfg_path)
-            if not base_yml.startswith('/'):
-                relpath = base_yml
-                base_yml = os.path.join(os.path.dirname(cfg_path), base_yml)
-
-            if osp.isfile(base_yml):
-                logger.debug("Found _BASE_ config: {}".format(base_yml))
-                continue
-
-            # download to CONFIGS_HOME firstly
-            base_yml_url = osp.join(osp.split(cfg_url)[0], relpath)
-            path, _ = get_path(base_yml_url, CONFIGS_HOME)
-
-            # move from CONFIGS_HOME to dst_path to restore config directory structure
-            dst_path = osp.join(cur_dir, relpath)
-            dst_dir = osp.split(dst_path)[0]
-            if not osp.isdir(dst_dir):
-                os.makedirs(dst_dir)
-            shutil.move(path, dst_path)
-
-            # perfrom download base yml recursively
-            _download_config(dst_path, base_yml_url, osp.split(dst_path)[0])
-
-
-def get_dataset_path(path, annotation, image_dir):
-    """
-    If path exists, return path.
-    Otherwise, get dataset path from DATASET_HOME, if not exists,
-    download it.
-    """
-    if _dataset_exists(path, annotation, image_dir):
-        return path
-
-    logger.info(
-        "Dataset {} is not valid for reason above, try searching {} or "
-        "downloading dataset...".format(osp.realpath(path), DATASET_HOME))
-
-    data_name = os.path.split(path.strip().lower())[-1]
-    for name, dataset in DATASETS.items():
-        if data_name == name:
-            logger.debug("Parse dataset_dir {} as dataset "
-                         "{}".format(path, name))
-            if name == 'objects365':
-                raise NotImplementedError(
-                    "Dataset {} is not valid for download automatically. "
-                    "Please apply and download the dataset from "
-                    "https://www.objects365.org/download.html".format(name))
-            data_dir = osp.join(DATASET_HOME, name)
-
-            if name == 'mot':
-                if osp.exists(path) or osp.exists(data_dir):
-                    return data_dir
-                else:
-                    raise NotImplementedError(
-                        "Dataset {} is not valid for download automatically. "
-                        "Please apply and download the dataset following docs/tutorials/PrepareMOTDataSet.md".
-                        format(name))
-
-            # For voc, only check dir VOCdevkit/VOC2012, VOCdevkit/VOC2007
-            if name in ['voc', 'fruit', 'roadsign_voc']:
-                exists = True
-                for sub_dir in dataset[1]:
-                    check_dir = osp.join(data_dir, sub_dir)
-                    if osp.exists(check_dir):
-                        logger.info("Found {}".format(check_dir))
-                    else:
-                        exists = False
-                if exists:
-                    return data_dir
-
-            # voc exist is checked above, voc is not exist here
-            check_exist = name != 'voc' and name != 'fruit' and name != 'roadsign_voc'
-            for url, md5sum in dataset[0]:
-                get_path(url, data_dir, md5sum, check_exist)
-
-            # voc should create list after download
-            if name == 'voc':
-                create_voc_list(data_dir)
-            return data_dir
-
-    # not match any dataset in DATASETS
-    raise ValueError(
-        "Dataset {} is not valid and cannot parse dataset type "
-        "'{}' for automaticly downloading, which only supports "
-        "'voc' , 'coco', 'wider_face', 'fruit', 'roadsign_voc' and 'mot' currently".
-        format(path, osp.split(path)[-1]))
-
-
-def create_voc_list(data_dir, devkit_subdir='VOCdevkit'):
-    logger.debug("Create voc file list...")
-    devkit_dir = osp.join(data_dir, devkit_subdir)
-    years = ['2007', '2012']
-
-    # NOTE: since using auto download VOC
-    # dataset, VOC default label list should be used,
-    # do not generate label_list.txt here. For default
-    # label, see ../data/source/voc.py
-    create_list(devkit_dir, years, data_dir)
-    logger.debug("Create voc file list finished")
-
-
-def map_path(url, root_dir):
-    # parse path after download to decompress under root_dir
-    fname = osp.split(url)[-1]
-    zip_formats = ['.zip', '.tar', '.gz']
-    fpath = fname
-    for zip_format in zip_formats:
-        fpath = fpath.replace(zip_format, '')
-    return osp.join(root_dir, fpath)
-
-
-def get_path(url, root_dir, md5sum=None, check_exist=True):
-    """ Download from given url to root_dir.
-    if file or directory specified by url is exists under
-    root_dir, return the path directly, otherwise download
-    from url and decompress it, return the path.
-
-    url (str): download url
-    root_dir (str): root dir for downloading, it should be
-                    WEIGHTS_HOME or DATASET_HOME
-    md5sum (str): md5 sum of download package
-    """
-    # parse path after download to decompress under root_dir
-    fullpath = map_path(url, root_dir)
-
-    # For same zip file, decompressed directory name different
-    # from zip file name, rename by following map
-    decompress_name_map = {
-        "VOCtrainval_11-May-2012": "VOCdevkit/VOC2012",
-        "VOCtrainval_06-Nov-2007": "VOCdevkit/VOC2007",
-        "VOCtest_06-Nov-2007": "VOCdevkit/VOC2007",
-        "annotations_trainval": "annotations"
-    }
-    for k, v in decompress_name_map.items():
-        if fullpath.find(k) >= 0:
-            fullpath = osp.join(osp.split(fullpath)[0], v)
-
-    if osp.exists(fullpath) and check_exist:
-        if not osp.isfile(fullpath) or \
-                _check_exist_file_md5(fullpath, md5sum, url):
-            logger.debug("Found {}".format(fullpath))
-            return fullpath, True
-        else:
-            os.remove(fullpath)
-
-    fullname = _download(url, root_dir, md5sum)
-
-    # new weights format which postfix is 'pdparams' not
-    # need to decompress
-    if osp.splitext(fullname)[-1] not in ['.pdparams', '.yml']:
-        _decompress(fullname)
-
-    return fullpath, False
-
-
-def download_dataset(path, dataset=None):
-    if dataset not in DATASETS.keys():
-        logger.error("Unknown dataset {}, it should be "
-                     "{}".format(dataset, DATASETS.keys()))
-        return
-    dataset_info = DATASETS[dataset][0]
-    for info in dataset_info:
-        get_path(info[0], path, info[1], False)
-    logger.debug("Download dataset {} finished.".format(dataset))
-
-
-def _dataset_exists(path, annotation, image_dir):
-    """
-    Check if user define dataset exists
-    """
-    if not osp.exists(path):
-        logger.warning("Config dataset_dir {} is not exits, "
-                       "dataset config is not valid".format(path))
-        return False
-
-    if annotation:
-        annotation_path = osp.join(path, annotation)
-        if not osp.isfile(annotation_path):
-            logger.warning("Config annotation {} is not a "
-                           "file, dataset config is not "
-                           "valid".format(annotation_path))
-            return False
-    if image_dir:
-        image_path = osp.join(path, image_dir)
-        if not osp.isdir(image_path):
-            logger.warning("Config image_dir {} is not a "
-                           "directory, dataset config is not "
-                           "valid".format(image_path))
-            return False
-    return True
-
-
-def _download(url, path, md5sum=None):
-    """
-    Download from url, save to path.
-
-    url (str): download url
-    path (str): download to given path
-    """
-    if not osp.exists(path):
-        os.makedirs(path)
-
-    fname = osp.split(url)[-1]
-    fullname = osp.join(path, fname)
-    retry_cnt = 0
-
-    while not (osp.exists(fullname) and _check_exist_file_md5(fullname, md5sum,
-                                                              url)):
-        if retry_cnt < DOWNLOAD_RETRY_LIMIT:
-            retry_cnt += 1
-        else:
-            raise RuntimeError("Download from {} failed. "
-                               "Retry limit reached".format(url))
-
-        logger.info("Downloading {} from {}".format(fname, url))
-
-        # NOTE: windows path join may incur \, which is invalid in url
-        if sys.platform == "win32":
-            url = url.replace('\\', '/')
-
-        req = requests.get(url, stream=True)
-        if req.status_code != 200:
-            raise RuntimeError("Downloading from {} failed with code "
-                               "{}!".format(url, req.status_code))
-
-        # For protecting download interupted, download to
-        # tmp_fullname firstly, move tmp_fullname to fullname
-        # after download finished
-        tmp_fullname = fullname + "_tmp"
-        total_size = req.headers.get('content-length')
-        with open(tmp_fullname, 'wb') as f:
-            if total_size:
-                for chunk in tqdm.tqdm(
-                        req.iter_content(chunk_size=1024),
-                        total=(int(total_size) + 1023) // 1024,
-                        unit='KB'):
-                    f.write(chunk)
-            else:
-                for chunk in req.iter_content(chunk_size=1024):
-                    if chunk:
-                        f.write(chunk)
-        shutil.move(tmp_fullname, fullname)
-        return fullname
-
-
-def _check_exist_file_md5(filename, md5sum, url):
-    # if md5sum is None, and file to check is weights file,
-    # read md5um from url and check, else check md5sum directly
-    return _md5check_from_url(filename, url) if md5sum is None \
-            and filename.endswith('pdparams') \
-            else _md5check(filename, md5sum)
-
-
-def _md5check_from_url(filename, url):
-    # For weights in bcebos URLs, MD5 value is contained
-    # in request header as 'content_md5'
-    req = requests.get(url, stream=True)
-    content_md5 = req.headers.get('content-md5')
-    req.close()
-    if not content_md5 or _md5check(
-            filename,
-            binascii.hexlify(base64.b64decode(content_md5.strip('"'))).decode(
-            )):
-        return True
-    else:
-        return False
-
-
-def _md5check(fullname, md5sum=None):
-    if md5sum is None:
-        return True
-
-    logger.debug("File {} md5 checking...".format(fullname))
-    md5 = hashlib.md5()
-    with open(fullname, 'rb') as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            md5.update(chunk)
-    calc_md5sum = md5.hexdigest()
-
-    if calc_md5sum != md5sum:
-        logger.warning("File {} md5 check failed, {}(calc) != "
-                       "{}(base)".format(fullname, calc_md5sum, md5sum))
-        return False
-    return True
-
-
-def _decompress(fname):
-    """
-    Decompress for zip and tar file
-    """
-    logger.info("Decompressing {}...".format(fname))
-
-    # For protecting decompressing interupted,
-    # decompress to fpath_tmp directory firstly, if decompress
-    # successed, move decompress files to fpath and delete
-    # fpath_tmp and remove download compress file.
-    fpath = osp.split(fname)[0]
-    fpath_tmp = osp.join(fpath, 'tmp')
-    if osp.isdir(fpath_tmp):
-        shutil.rmtree(fpath_tmp)
-        os.makedirs(fpath_tmp)
-
-    if fname.find('tar') >= 0:
-        with tarfile.open(fname) as tf:
-            tf.extractall(path=fpath_tmp)
-    elif fname.find('zip') >= 0:
-        with zipfile.ZipFile(fname) as zf:
-            zf.extractall(path=fpath_tmp)
-    else:
-        raise TypeError("Unsupport compress file type {}".format(fname))
-
-    for f in os.listdir(fpath_tmp):
-        src_dir = osp.join(fpath_tmp, f)
-        dst_dir = osp.join(fpath, f)
-        _move_and_merge_tree(src_dir, dst_dir)
-
-    shutil.rmtree(fpath_tmp)
-    os.remove(fname)
-
-
-def _move_and_merge_tree(src, dst):
-    """
-    Move src directory to dst, if dst is already exists,
-    merge src to dst
-    """
-    if not osp.exists(dst):
-        shutil.move(src, dst)
-    elif osp.isfile(src):
-        shutil.move(src, dst)
-    else:
-        for fp in os.listdir(src):
-            src_fp = osp.join(src, fp)
-            dst_fp = osp.join(dst, fp)
-            if osp.isdir(src_fp):
-                if osp.isdir(dst_fp):
-                    _move_and_merge_tree(src_fp, dst_fp)
-                else:
-                    shutil.move(src_fp, dst_fp)
-            elif osp.isfile(src_fp) and \
-                    not osp.isfile(dst_fp):
-                shutil.move(src_fp, dst_fp)