paddlex 2.1.0__py3-none-any.whl → 3.0.0rc0__py3-none-any.whl

paddlex/ppcls/utils/check.py

@@ -1,151 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import sys
-
-import paddle
-from paddle import is_compiled_with_cuda
-
-from paddlex.ppcls.arch import get_architectures
-from paddlex.ppcls.arch import similar_architectures
-from paddlex.ppcls.arch import get_blacklist_model_in_static_mode
-from paddlex.ppcls.utils import logger
-
-
-def check_version():
-    """
-    Log error and exit when the installed version of paddlepaddle is
-    not satisfied.
-    """
-    err = "PaddlePaddle version 1.8.0 or higher is required, " \
-          "or a suitable develop version is satisfied as well. \n" \
-          "Please make sure the version is good with your code."
-    try:
-        pass
-        # paddle.utils.require_version('0.0.0')
-    except Exception:
-        logger.error(err)
-        sys.exit(1)
-
-
-def check_gpu():
-    """
-    Log error and exit when using paddlepaddle cpu version.
-    """
-    err = "You are using paddlepaddle cpu version! Please try to " \
-          "install paddlepaddle-gpu to run model on GPU."
-
-    try:
-        assert is_compiled_with_cuda()
-    except AssertionError:
-        logger.error(err)
-        sys.exit(1)
-
-
-def check_architecture(architecture):
-    """
-    check architecture and recommend similar architectures
-    """
-    assert isinstance(architecture, dict), \
-        ("the type of architecture({}) should be dict". format(architecture))
-    assert "name" in architecture, \
-        ("name must be in the architecture keys, just contains: {}". format(
-            architecture.keys()))
-
-    similar_names = similar_architectures(architecture["name"],
-                                          get_architectures())
-    model_list = ', '.join(similar_names)
-    err = "Architecture [{}] is not exist! Maybe you want: [{}]" \
-          "".format(architecture["name"], model_list)
-    try:
-        assert architecture["name"] in similar_names
-    except AssertionError:
-        logger.error(err)
-        sys.exit(1)
-
-
-def check_model_with_running_mode(architecture):
-    """
-    check whether the model is consistent with the operating mode
-    """
-    # some model are not supported in the static mode
-    blacklist = get_blacklist_model_in_static_mode()
-    if not paddle.in_dynamic_mode() and architecture["name"] in blacklist:
-        logger.error("Model: {} is not supported in the staic mode.".format(
-            architecture["name"]))
-        sys.exit(1)
-    return
-
-
-def check_mix(architecture, use_mix=False):
-    """
-    check mix parameter
-    """
-    err = "Cannot use mix processing in GoogLeNet, " \
-          "please set use_mix = False."
-    try:
-        if architecture["name"] == "GoogLeNet":
-            assert use_mix is not True
-    except AssertionError:
-        logger.error(err)
-        sys.exit(1)
-
-
-def check_classes_num(classes_num):
-    """
-    check classes_num
-    """
-    err = "classes_num({}) should be a positive integer" \
-        "and larger than 1".format(classes_num)
-    try:
-        assert isinstance(classes_num, int)
-        assert classes_num > 1
-    except AssertionError:
-        logger.error(err)
-        sys.exit(1)
-
-
-def check_data_dir(path):
-    """
-    check cata_dir
-    """
-    err = "Data path is not exist, please given a right path" \
-          "".format(path)
-    try:
-        assert os.isdir(path)
-    except AssertionError:
-        logger.error(err)
-        sys.exit(1)
-
-
-def check_function_params(config, key):
-    """
-    check specify config
-    """
-    k_config = config.get(key)
-    assert k_config is not None, \
-        ('{} is required in config'.format(key))
-
-    assert k_config.get('function'), \
-        ('function is required {} config'.format(key))
-    params = k_config.get('params')
-    assert params is not None, \
-        ('params is required in {} config'.format(key))
-    assert isinstance(params, dict), \
-        ('the params in {} config should be a dict'.format(key))

paddlex/ppcls/utils/config.py

@@ -1,210 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import copy
-import argparse
-import yaml
-from paddlex.ppcls.utils import logger
-from paddlex.ppcls.utils import check
-__all__ = ['get_config']
-
-
-class AttrDict(dict):
-    def __getattr__(self, key):
-        return self[key]
-
-    def __setattr__(self, key, value):
-        if key in self.__dict__:
-            self.__dict__[key] = value
-        else:
-            self[key] = value
-
-    def __deepcopy__(self, content):
-        return copy.deepcopy(dict(self))
-
-
-def create_attr_dict(yaml_config):
-    from ast import literal_eval
-    for key, value in yaml_config.items():
-        if type(value) is dict:
-            yaml_config[key] = value = AttrDict(value)
-        if isinstance(value, str):
-            try:
-                value = literal_eval(value)
-            except BaseException:
-                pass
-        if isinstance(value, AttrDict):
-            create_attr_dict(yaml_config[key])
-        else:
-            yaml_config[key] = value
-
-
-def parse_config(cfg_file):
-    """Load a config file into AttrDict"""
-    with open(cfg_file, 'r') as fopen:
-        yaml_config = AttrDict(yaml.load(fopen, Loader=yaml.SafeLoader))
-    create_attr_dict(yaml_config)
-    return yaml_config
-
-
-def print_dict(d, delimiter=0):
-    """
-    Recursively visualize a dict and
-    indenting acrrording by the relationship of keys.
-    """
-    placeholder = "-" * 60
-    for k, v in sorted(d.items()):
-        if isinstance(v, dict):
-            logger.info("{}{} : ".format(delimiter * " ", k))
-            print_dict(v, delimiter + 4)
-        elif isinstance(v, list) and len(v) >= 1 and isinstance(v[0], dict):
-            logger.info("{}{} : ".format(delimiter * " ", k))
-            for value in v:
-                print_dict(value, delimiter + 4)
-        else:
-            logger.info("{}{} : {}".format(delimiter * " ", k, v))
-        if k.isupper():
-            logger.info(placeholder)
-
-
-def print_config(config):
-    """
-    visualize configs
-    Arguments:
-        config: configs
-    """
-    logger.advertise()
-    print_dict(config)
-
-
-def check_config(config):
-    """
-    Check config
-    """
-    check.check_version()
-    use_gpu = config.get('use_gpu', True)
-    if use_gpu:
-        check.check_gpu()
-    architecture = config.get('ARCHITECTURE')
-    #check.check_architecture(architecture)
-    use_mix = config.get('use_mix', False)
-    check.check_mix(architecture, use_mix)
-    classes_num = config.get('classes_num')
-    check.check_classes_num(classes_num)
-    mode = config.get('mode', 'train')
-    if mode.lower() == 'train':
-        check.check_function_params(config, 'LEARNING_RATE')
-        check.check_function_params(config, 'OPTIMIZER')
-
-
-def override(dl, ks, v):
-    """
-    Recursively replace dict of list
-    Args:
-        dl(dict or list): dict or list to be replaced
-        ks(list): list of keys
-        v(str): value to be replaced
-    """
-
-    def str2num(v):
-        try:
-            return eval(v)
-        except Exception:
-            return v
-
-    assert isinstance(dl, (list, dict)), ("{} should be a list or a dict")
-    assert len(ks) > 0, ('lenght of keys should larger than 0')
-    if isinstance(dl, list):
-        k = str2num(ks[0])
-        if len(ks) == 1:
-            assert k < len(dl), ('index({}) out of range({})'.format(k, dl))
-            dl[k] = str2num(v)
-        else:
-            override(dl[k], ks[1:], v)
-    else:
-        if len(ks) == 1:
-            # assert ks[0] in dl, ('{} is not exist in {}'.format(ks[0], dl))
-            if not ks[0] in dl:
-                print('A new filed ({}) detected!'.format(ks[0], dl))
-            dl[ks[0]] = str2num(v)
-        else:
-            override(dl[ks[0]], ks[1:], v)
-
-
-def override_config(config, options=None):
-    """
-    Recursively override the config
-    Args:
-        config(dict): dict to be replaced
-        options(list): list of pairs(key0.key1.idx.key2=value)
-            such as: [
-                'topk=2',
-                'VALID.transforms.1.ResizeImage.resize_short=300'
-            ]
-    Returns:
-        config(dict): replaced config
-    """
-    if options is not None:
-        for opt in options:
-            assert isinstance(opt, str), (
-                "option({}) should be a str".format(opt))
-            assert "=" in opt, (
-                "option({}) should contain a ="
-                "to distinguish between key and value".format(opt))
-            pair = opt.split('=')
-            assert len(pair) == 2, ("there can be only a = in the option")
-            key, value = pair
-            keys = key.split('.')
-            override(config, keys, value)
-    return config
-
-
-def get_config(fname, overrides=None, show=False):
-    """
-    Read config from file
-    """
-    assert os.path.exists(fname), (
-        'config file({}) is not exist'.format(fname))
-    config = parse_config(fname)
-    override_config(config, overrides)
-    if show:
-        print_config(config)
-    # check_config(config)
-    return config
-
-
-def parse_args():
-    parser = argparse.ArgumentParser("generic-image-rec train script")
-    parser.add_argument(
-        '-c',
-        '--config',
-        type=str,
-        default='configs/config.yaml',
-        help='config file path')
-    parser.add_argument(
-        '-o',
-        '--override',
-        action='append',
-        default=[],
-        help='config options to be overridden')
-    parser.add_argument(
-        '-p',
-        '--profiler_options',
-        type=str,
-        default=None,
-        help='The option of profiler, which should be in format \"key1=value1;key2=value2;key3=value3\".'
-    )
-    args = parser.parse_args()
-    return args

paddlex/ppcls/utils/download.py

@@ -1,319 +0,0 @@
-#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import sys
-import os.path as osp
-import shutil
-import requests
-import hashlib
-import tarfile
-import zipfile
-import time
-from collections import OrderedDict
-from tqdm import tqdm
-
-from paddlex.ppcls.utils import logger
-
-__all__ = ['get_weights_path_from_url']
-
-WEIGHTS_HOME = osp.expanduser("~/.paddleclas/weights")
-
-DOWNLOAD_RETRY_LIMIT = 3
-
-
-def is_url(path):
-    """
-    Whether path is URL.
-    Args:
-        path (string): URL string or not.
-    """
-    return path.startswith('http://') or path.startswith('https://')
-
-
-def get_weights_path_from_url(url, md5sum=None):
-    """Get weights path from WEIGHT_HOME, if not exists,
-    download it from url.
-
-    Args:
-        url (str): download url
-        md5sum (str): md5 sum of download package
-
-    Returns:
-        str: a local path to save downloaded weights.
-
-    Examples:
-        .. code-block:: python
-
-            from paddle.utils.download import get_weights_path_from_url
-
-            resnet18_pretrained_weight_url = 'https://paddle-hapi.bj.bcebos.com/models/resnet18.pdparams'
-            local_weight_path = get_weights_path_from_url(resnet18_pretrained_weight_url)
-
-    """
-    path = get_path_from_url(url, WEIGHTS_HOME, md5sum)
-    return path
-
-
-def _map_path(url, root_dir):
-    # parse path after download under root_dir
-    fname = osp.split(url)[-1]
-    fpath = fname
-    return osp.join(root_dir, fpath)
-
-
-def _get_unique_endpoints(trainer_endpoints):
-    # Sorting is to avoid different environmental variables for each card
-    trainer_endpoints.sort()
-    ips = set()
-    unique_endpoints = set()
-    for endpoint in trainer_endpoints:
-        ip = endpoint.split(":")[0]
-        if ip in ips:
-            continue
-        ips.add(ip)
-        unique_endpoints.add(endpoint)
-    logger.info("unique_endpoints {}".format(unique_endpoints))
-    return unique_endpoints
-
-
-def get_path_from_url(url,
-                      root_dir,
-                      md5sum=None,
-                      check_exist=True,
-                      decompress=True):
-    """ Download from given url to root_dir.
-    if file or directory specified by url is exists under
-    root_dir, return the path directly, otherwise download
-    from url and decompress it, return the path.
-
-    Args:
-        url (str): download url
-        root_dir (str): root dir for downloading, it should be
-                        WEIGHTS_HOME or DATASET_HOME
-        md5sum (str): md5 sum of download package
-
-    Returns:
-        str: a local path to save downloaded models & weights & datasets.
-    """
-
-    from paddle.fluid.dygraph.parallel import ParallelEnv
-
-    assert is_url(url), "downloading from {} not a url".format(url)
-    # parse path after download to decompress under root_dir
-    fullpath = _map_path(url, root_dir)
-    # Mainly used to solve the problem of downloading data from different
-    # machines in the case of multiple machines. Different ips will download
-    # data, and the same ip will only download data once.
-    unique_endpoints = _get_unique_endpoints(ParallelEnv()
-                                             .trainer_endpoints[:])
-    if osp.exists(fullpath) and check_exist and _md5check(fullpath, md5sum):
-        logger.info("Found {}".format(fullpath))
-    else:
-        if ParallelEnv().current_endpoint in unique_endpoints:
-            fullpath = _download(url, root_dir, md5sum)
-        else:
-            while not os.path.exists(fullpath):
-                time.sleep(1)
-
-    if ParallelEnv().current_endpoint in unique_endpoints:
-        if decompress and (tarfile.is_tarfile(fullpath) or
-                           zipfile.is_zipfile(fullpath)):
-            fullpath = _decompress(fullpath)
-
-    return fullpath
-
-
-def _download(url, path, md5sum=None):
-    """
-    Download from url, save to path.
-
-    url (str): download url
-    path (str): download to given path
-    """
-    if not osp.exists(path):
-        os.makedirs(path)
-
-    fname = osp.split(url)[-1]
-    fullname = osp.join(path, fname)
-    retry_cnt = 0
-
-    while not (osp.exists(fullname) and _md5check(fullname, md5sum)):
-        if retry_cnt < DOWNLOAD_RETRY_LIMIT:
-            retry_cnt += 1
-        else:
-            raise RuntimeError("Download from {} failed. "
-                               "Retry limit reached".format(url))
-
-        logger.info("Downloading {} from {}".format(fname, url))
-
-        try:
-            req = requests.get(url, stream=True)
-        except Exception as e:  # requests.exceptions.ConnectionError
-            logger.info(
-                "Downloading {} from {} failed {} times with exception {}".
-                format(fname, url, retry_cnt + 1, str(e)))
-            time.sleep(1)
-            continue
-
-        if req.status_code != 200:
-            raise RuntimeError("Downloading from {} failed with code "
-                               "{}!".format(url, req.status_code))
-
-        # For protecting download interupted, download to
-        # tmp_fullname firstly, move tmp_fullname to fullname
-        # after download finished
-        tmp_fullname = fullname + "_tmp"
-        total_size = req.headers.get('content-length')
-        with open(tmp_fullname, 'wb') as f:
-            if total_size:
-                with tqdm(total=(int(total_size) + 1023) // 1024) as pbar:
-                    for chunk in req.iter_content(chunk_size=1024):
-                        f.write(chunk)
-                        pbar.update(1)
-            else:
-                for chunk in req.iter_content(chunk_size=1024):
-                    if chunk:
-                        f.write(chunk)
-        shutil.move(tmp_fullname, fullname)
-
-    return fullname
-
-
-def _md5check(fullname, md5sum=None):
-    if md5sum is None:
-        return True
-
-    logger.info("File {} md5 checking...".format(fullname))
-    md5 = hashlib.md5()
-    with open(fullname, 'rb') as f:
-        for chunk in iter(lambda: f.read(4096), b""):
-            md5.update(chunk)
-    calc_md5sum = md5.hexdigest()
-
-    if calc_md5sum != md5sum:
-        logger.info("File {} md5 check failed, {}(calc) != "
-                    "{}(base)".format(fullname, calc_md5sum, md5sum))
-        return False
-    return True
-
-
-def _decompress(fname):
-    """
-    Decompress for zip and tar file
-    """
-    logger.info("Decompressing {}...".format(fname))
-
-    # For protecting decompressing interupted,
-    # decompress to fpath_tmp directory firstly, if decompress
-    # successed, move decompress files to fpath and delete
-    # fpath_tmp and remove download compress file.
-
-    if tarfile.is_tarfile(fname):
-        uncompressed_path = _uncompress_file_tar(fname)
-    elif zipfile.is_zipfile(fname):
-        uncompressed_path = _uncompress_file_zip(fname)
-    else:
-        raise TypeError("Unsupport compress file type {}".format(fname))
-
-    return uncompressed_path
-
-
-def _uncompress_file_zip(filepath):
-    files = zipfile.ZipFile(filepath, 'r')
-    file_list = files.namelist()
-
-    file_dir = os.path.dirname(filepath)
-
-    if _is_a_single_file(file_list):
-        rootpath = file_list[0]
-        uncompressed_path = os.path.join(file_dir, rootpath)
-
-        for item in file_list:
-            files.extract(item, file_dir)
-
-    elif _is_a_single_dir(file_list):
-        rootpath = os.path.splitext(file_list[0])[0].split(os.sep)[-1]
-        uncompressed_path = os.path.join(file_dir, rootpath)
-
-        for item in file_list:
-            files.extract(item, file_dir)
-
-    else:
-        rootpath = os.path.splitext(filepath)[0].split(os.sep)[-1]
-        uncompressed_path = os.path.join(file_dir, rootpath)
-        if not os.path.exists(uncompressed_path):
-            os.makedirs(uncompressed_path)
-        for item in file_list:
-            files.extract(item, os.path.join(file_dir, rootpath))
-
-    files.close()
-
-    return uncompressed_path
-
-
-def _uncompress_file_tar(filepath, mode="r:*"):
-    files = tarfile.open(filepath, mode)
-    file_list = files.getnames()
-
-    file_dir = os.path.dirname(filepath)
-
-    if _is_a_single_file(file_list):
-        rootpath = file_list[0]
-        uncompressed_path = os.path.join(file_dir, rootpath)
-        for item in file_list:
-            files.extract(item, file_dir)
-    elif _is_a_single_dir(file_list):
-        rootpath = os.path.splitext(file_list[0])[0].split(os.sep)[-1]
-        uncompressed_path = os.path.join(file_dir, rootpath)
-        for item in file_list:
-            files.extract(item, file_dir)
-    else:
-        rootpath = os.path.splitext(filepath)[0].split(os.sep)[-1]
-        uncompressed_path = os.path.join(file_dir, rootpath)
-        if not os.path.exists(uncompressed_path):
-            os.makedirs(uncompressed_path)
-
-        for item in file_list:
-            files.extract(item, os.path.join(file_dir, rootpath))
-
-    files.close()
-
-    return uncompressed_path
-
-
-def _is_a_single_file(file_list):
-    if len(file_list) == 1 and file_list[0].find(os.sep) < -1:
-        return True
-    return False
-
-
-def _is_a_single_dir(file_list):
-    new_file_list = []
-    for file_path in file_list:
-        if '/' in file_path:
-            file_path = file_path.replace('/', os.sep)
-        elif '\\' in file_path:
-            file_path = file_path.replace('\\', os.sep)
-        new_file_list.append(file_path)
-
-    file_name = new_file_list[0].split(os.sep)[0]
-    for i in range(1, len(new_file_list)):
-        if file_name != new_file_list[i].split(os.sep)[0]:
-            return False
-    return True