paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/ppcls/utils/logger.py

@@ -1,120 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-import os
-import datetime
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s %(levelname)s: %(message)s",
-    datefmt="%Y-%m-%d %H:%M:%S")
-
-
-def time_zone(sec, fmt):
-    real_time = datetime.datetime.now()
-    return real_time.timetuple()
-
-
-logging.Formatter.converter = time_zone
-_logger = logging.getLogger(__name__)
-
-Color = {
-    'RED': '\033[31m',
-    'HEADER': '\033[35m',  # deep purple
-    'PURPLE': '\033[95m',  # purple
-    'OKBLUE': '\033[94m',
-    'OKGREEN': '\033[92m',
-    'WARNING': '\033[93m',
-    'FAIL': '\033[91m',
-    'ENDC': '\033[0m'
-}
-
-
-def coloring(message, color="OKGREEN"):
-    assert color in Color.keys()
-    if os.environ.get('PADDLECLAS_COLORING', False):
-        return Color[color] + str(message) + Color["ENDC"]
-    else:
-        return message
-
-
-def anti_fleet(log):
-    """
-    logs will print multi-times when calling Fleet API.
-    Only display single log and ignore the others.
-    """
-
-    def wrapper(fmt, *args):
-        if int(os.getenv("PADDLE_TRAINER_ID", 0)) == 0:
-            log(fmt, *args)
-
-    return wrapper
-
-
-@anti_fleet
-def info(fmt, *args):
-    _logger.info(fmt, *args)
-
-
-@anti_fleet
-def warning(fmt, *args):
-    _logger.warning(coloring(fmt, "RED"), *args)
-
-
-@anti_fleet
-def error(fmt, *args):
-    _logger.error(coloring(fmt, "FAIL"), *args)
-
-
-def scaler(name, value, step, writer):
-    """
-    This function will draw a scalar curve generated by the visualdl.
-    Usage: Install visualdl: pip3 install visualdl==2.0.0b4
-           and then:
-           visualdl --logdir ./scalar --host 0.0.0.0 --port 8830
-           to preview loss corve in real time.
-    """
-    writer.add_scalar(tag=name, step=step, value=value)
-
-
-def advertise():
-    """
-    Show the advertising message like the following:
-
-    ===========================================================
-    ==        PaddleClas is powered by PaddlePaddle !        ==
-    ===========================================================
-    ==                                                       ==
-    ==   For more info please go to the following website.   ==
-    ==                                                       ==
-    ==       https://github.com/PaddlePaddle/PaddleClas      ==
-    ===========================================================
-
-    """
-    copyright = "PaddleClas is powered by PaddlePaddle !"
-    ad = "For more info please go to the following website."
-    website = "https://github.com/PaddlePaddle/PaddleClas"
-    AD_LEN = 6 + len(max([copyright, ad, website], key=len))
-
-    info(
-        coloring("\n{0}\n{1}\n{2}\n{3}\n{4}\n{5}\n{6}\n{7}\n".format(
-            "=" * (AD_LEN + 4),
-            "=={}==".format(copyright.center(AD_LEN)),
-            "=" * (AD_LEN + 4),
-            "=={}==".format(' ' * AD_LEN),
-            "=={}==".format(ad.center(AD_LEN)),
-            "=={}==".format(' ' * AD_LEN),
-            "=={}==".format(website.center(AD_LEN)),
-            "=" * (AD_LEN + 4), ), "RED"))

paddlex/ppcls/utils/metrics.py

@@ -1,112 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from sklearn.metrics import hamming_loss
-from sklearn.metrics import accuracy_score as accuracy_metric
-from sklearn.metrics import multilabel_confusion_matrix
-from sklearn.metrics import precision_recall_fscore_support
-from sklearn.metrics import average_precision_score
-from sklearn.preprocessing import binarize
-
-import numpy as np
-
-__all__ = [
-    "multi_hot_encode", "hamming_distance", "accuracy_score",
-    "precision_recall_fscore", "mean_average_precision"
-]
-
-
-def multi_hot_encode(logits, threshold=0.5):
-    """
-    Encode logits to multi-hot by elementwise for multilabel
-    """
-
-    return binarize(logits, threshold)
-
-
-def hamming_distance(output, target):
-    """
-    Soft metric based label for multilabel classification
-    Returns:
-        The smaller the return value is, the better model is.
-    """
-
-    return hamming_loss(target, output)
-
-
-def accuracy_score(output, target, base="sample"):
-    """
-    Hard metric for multilabel classification
-    Args:
-        output:
-        target:
-        base: ["sample", "label"], default="sample"
-            if "sample", return metric score based sample,
-            if "label", return metric score based label.
-    Returns:
-        accuracy:
-    """
-
-    assert base in ["sample", "label"], 'must be one of ["sample", "label"]'
-
-    if base == "sample":
-        accuracy = accuracy_metric(target, output)
-    elif base == "label":
-        mcm = multilabel_confusion_matrix(target, output)
-        tns = mcm[:, 0, 0]
-        fns = mcm[:, 1, 0]
-        tps = mcm[:, 1, 1]
-        fps = mcm[:, 0, 1]
-
-        accuracy = (sum(tps) + sum(tns)) / (
-            sum(tps) + sum(tns) + sum(fns) + sum(fps))
-
-    return accuracy
-
-
-def precision_recall_fscore(output, target):
-    """
-    Metric based label for multilabel classification
-    Returns:
-        precisions:
-        recalls:
-        fscores:
-    """
-
-    precisions, recalls, fscores, _ = precision_recall_fscore_support(target,
-                                                                      output)
-
-    return precisions, recalls, fscores
-
-
-def mean_average_precision(logits, target):
-    """
-    Calculate average precision
-    Args:
-        logits: probability from network before sigmoid or softmax
-        target: ground truth, 0 or 1
-    """
-    if not (isinstance(logits, np.ndarray) and isinstance(target, np.ndarray)):
-        raise TypeError("logits and target should be np.ndarray.")
-
-    aps = []
-    for i in range(target.shape[1]):
-        ap = average_precision_score(target[:, i], logits[:, i])
-        aps.append(ap)
-
-    return np.mean(aps)

paddlex/ppcls/utils/misc.py

@@ -1,62 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-__all__ = ['AverageMeter']
-
-
-class AverageMeter(object):
-    """
-    Computes and stores the average and current value
-    """
-
-    def __init__(self, name='', fmt='f', postfix="", need_avg=True):
-        self.name = name
-        self.fmt = fmt
-        self.postfix = postfix
-        self.need_avg = need_avg
-        self.reset()
-
-    def reset(self):
-        """ reset """
-        self.val = 0
-        self.avg = 0
-        self.sum = 0
-        self.count = 0
-
-    def update(self, val, n=1):
-        """ update """
-        self.val = val
-        self.sum += val * n
-        self.count += n
-        self.avg = self.sum / self.count
-
-    @property
-    def total(self):
-        return '{self.name}_sum: {self.sum:{self.fmt}}{self.postfix}'.format(
-            self=self)
-
-    @property
-    def total_minute(self):
-        return '{self.name} {s:{self.fmt}}{self.postfix} min'.format(
-            s=self.sum / 60, self=self)
-
-    @property
-    def mean(self):
-        return '{self.name}: {self.avg:{self.fmt}}{self.postfix}'.format(
-            self=self) if self.need_avg else ''
-
-    @property
-    def value(self):
-        return '{self.name}: {self.val:{self.fmt}}{self.postfix}'.format(
-            self=self)

paddlex/ppcls/utils/model_zoo.py

@@ -1,213 +0,0 @@
-#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import requests
-import shutil
-import tarfile
-import tqdm
-import zipfile
-
-from paddlex.ppcls.modeling import similar_architectures
-from paddlex.ppcls.utils import logger
-
-__all__ = ['get']
-
-DOWNLOAD_RETRY_LIMIT = 3
-
-
-class UrlError(Exception):
-    """ UrlError
-    """
-
-    def __init__(self, url='', code=''):
-        message = "Downloading from {} failed with code {}!".format(url, code)
-        super(UrlError, self).__init__(message)
-
-
-class ModelNameError(Exception):
-    """ ModelNameError
-    """
-
-    def __init__(self, message=''):
-        super(ModelNameError, self).__init__(message)
-
-
-class RetryError(Exception):
-    """ RetryError
-    """
-
-    def __init__(self, url='', times=''):
-        message = "Download from {} failed. Retry({}) limit reached".format(
-            url, times)
-        super(RetryError, self).__init__(message)
-
-
-def _get_url(architecture, postfix="pdparams"):
-    prefix = "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/"
-    fname = architecture + "_pretrained." + postfix
-    return prefix + fname
-
-
-def _move_and_merge_tree(src, dst):
-    """
-    Move src directory to dst, if dst is already exists,
-    merge src to dst
-    """
-    if not os.path.exists(dst):
-        shutil.move(src, dst)
-    elif os.path.isfile(src):
-        shutil.move(src, dst)
-    else:
-        for fp in os.listdir(src):
-            src_fp = os.path.join(src, fp)
-            dst_fp = os.path.join(dst, fp)
-            if os.path.isdir(src_fp):
-                if os.path.isdir(dst_fp):
-                    _move_and_merge_tree(src_fp, dst_fp)
-                else:
-                    shutil.move(src_fp, dst_fp)
-            elif os.path.isfile(src_fp) and \
-                    not os.path.isfile(dst_fp):
-                shutil.move(src_fp, dst_fp)
-
-
-def _download(url, path):
-    """
-    Download from url, save to path.
-    url (str): download url
-    path (str): download to given path
-    """
-    if not os.path.exists(path):
-        os.makedirs(path)
-
-    fname = os.path.split(url)[-1]
-    fullname = os.path.join(path, fname)
-    retry_cnt = 0
-
-    while not os.path.exists(fullname):
-        if retry_cnt < DOWNLOAD_RETRY_LIMIT:
-            retry_cnt += 1
-        else:
-            raise RetryError(url, DOWNLOAD_RETRY_LIMIT)
-
-        logger.info("Downloading {} from {}".format(fname, url))
-
-        req = requests.get(url, stream=True)
-        if req.status_code != 200:
-            raise UrlError(url, req.status_code)
-
-        # For protecting download interupted, download to
-        # tmp_fullname firstly, move tmp_fullname to fullname
-        # after download finished
-        tmp_fullname = fullname + "_tmp"
-        total_size = req.headers.get('content-length')
-        with open(tmp_fullname, 'wb') as f:
-            if total_size:
-                for chunk in tqdm.tqdm(
-                        req.iter_content(chunk_size=1024),
-                        total=(int(total_size) + 1023) // 1024,
-                        unit='KB'):
-                    f.write(chunk)
-            else:
-                for chunk in req.iter_content(chunk_size=1024):
-                    if chunk:
-                        f.write(chunk)
-        shutil.move(tmp_fullname, fullname)
-
-    return fullname
-
-
-def _decompress(fname):
-    """
-    Decompress for zip and tar file
-    """
-    logger.info("Decompressing {}...".format(fname))
-
-    # For protecting decompressing interupted,
-    # decompress to fpath_tmp directory firstly, if decompress
-    # successed, move decompress files to fpath and delete
-    # fpath_tmp and remove download compress file.
-    fpath = os.path.split(fname)[0]
-    fpath_tmp = os.path.join(fpath, 'tmp')
-    if os.path.isdir(fpath_tmp):
-        shutil.rmtree(fpath_tmp)
-        os.makedirs(fpath_tmp)
-
-    if fname.find('tar') >= 0:
-        with tarfile.open(fname) as tf:
-            tf.extractall(path=fpath_tmp)
-    elif fname.find('zip') >= 0:
-        with zipfile.ZipFile(fname) as zf:
-            zf.extractall(path=fpath_tmp)
-    else:
-        raise TypeError("Unsupport compress file type {}".format(fname))
-
-    fs = os.listdir(fpath_tmp)
-    assert len(
-        fs
-    ) == 1, "There should just be 1 pretrained path in an archive file but got {}.".format(
-        len(fs))
-
-    f = fs[0]
-    src_dir = os.path.join(fpath_tmp, f)
-    dst_dir = os.path.join(fpath, f)
-    _move_and_merge_tree(src_dir, dst_dir)
-
-    shutil.rmtree(fpath_tmp)
-    os.remove(fname)
-
-    return f
-
-
-def _get_pretrained():
-    with open('./ppcls/utils/pretrained.list') as flist:
-        pretrained = [line.strip() for line in flist]
-    return pretrained
-
-
-def _check_pretrained_name(architecture):
-    assert isinstance(architecture, str), \
-        ("the type of architecture({}) should be str". format(architecture))
-    pretrained = _get_pretrained()
-    similar_names = similar_architectures(architecture, pretrained)
-    model_list = ', '.join(similar_names)
-    err = "{} is not exist! Maybe you want: [{}]" \
-          "".format(architecture, model_list)
-    if architecture not in similar_names:
-        raise ModelNameError(err)
-
-
-def list_models():
-    pretrained = _get_pretrained()
-    msg = "All avialable pretrained models are as follows: {}".format(
-        pretrained)
-    logger.info(msg)
-    return
-
-
-def get(architecture, path, decompress=False, postfix="pdparams"):
-    """
-    Get the pretrained model.
-    """
-    _check_pretrained_name(architecture)
-    url = _get_url(architecture, postfix=postfix)
-    fname = _download(url, path)
-    if postfix == "tar" and decompress:
-        _decompress(fname)
-    logger.info("download {} finished ".format(fname))

paddlex/ppcls/utils/save_load.py

@@ -1,163 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import errno
-import os
-import re
-import shutil
-import tempfile
-
-import paddle
-from paddle.static import load_program_state
-
-from paddlex.ppcls.utils import logger
-
-__all__ = ['init_model', 'save_model', 'load_dygraph_pretrain']
-
-
-def _mkdir_if_not_exist(path):
-    """
-    mkdir if not exists, ignore the exception when multiprocess mkdir together
-    """
-    if not os.path.exists(path):
-        try:
-            os.makedirs(path)
-        except OSError as e:
-            if e.errno == errno.EEXIST and os.path.isdir(path):
-                logger.warning(
-                    'be happy if some process has already created {}'.format(
-                        path))
-            else:
-                raise OSError('Failed to mkdir {}'.format(path))
-
-
-def load_dygraph_pretrain(model, path=None, load_static_weights=False):
-    if not (os.path.isdir(path) or os.path.exists(path + '.pdparams')):
-        raise ValueError("Model pretrain path {} does not "
-                         "exists.".format(path))
-    if load_static_weights:
-        pre_state_dict = load_program_state(path)
-        param_state_dict = {}
-        model_dict = model.state_dict()
-        for key in model_dict.keys():
-            weight_name = model_dict[key].name
-            if weight_name in pre_state_dict.keys():
-                logger.info('Load weight: {}, shape: {}'.format(
-                    weight_name, pre_state_dict[weight_name].shape))
-                param_state_dict[key] = pre_state_dict[weight_name]
-            else:
-                param_state_dict[key] = model_dict[key]
-        model.set_dict(param_state_dict)
-        return
-
-    param_state_dict = paddle.load(path + ".pdparams")
-    model.set_dict(param_state_dict)
-    return
-
-
-def load_distillation_model(model, pretrained_model, load_static_weights):
-    logger.info("In distillation mode, teacher model will be "
-                "loaded firstly before student model.")
-
-    if not isinstance(pretrained_model, list):
-        pretrained_model = [pretrained_model]
-
-    if not isinstance(load_static_weights, list):
-        load_static_weights = [load_static_weights] * len(pretrained_model)
-
-    teacher = model.teacher if hasattr(model,
-                                       "teacher") else model._layers.teacher
-    student = model.student if hasattr(model,
-                                       "student") else model._layers.student
-    load_dygraph_pretrain(
-        teacher,
-        path=pretrained_model[0],
-        load_static_weights=load_static_weights[0])
-    logger.info("Finish initing teacher model from {}".format(
-        pretrained_model))
-    # load student model
-    if len(pretrained_model) >= 2:
-        load_dygraph_pretrain(
-            student,
-            path=pretrained_model[1],
-            load_static_weights=load_static_weights[1])
-        logger.info("Finish initing student model from {}".format(
-            pretrained_model))
-
-
-def init_model(config, net, optimizer=None):
-    """
-    load model from checkpoint or pretrained_model
-    """
-    checkpoints = config.get('checkpoints')
-    if checkpoints and optimizer is not None:
-        assert os.path.exists(checkpoints + ".pdparams"), \
-            "Given dir {}.pdparams not exist.".format(checkpoints)
-        assert os.path.exists(checkpoints + ".pdopt"), \
-            "Given dir {}.pdopt not exist.".format(checkpoints)
-        para_dict = paddle.load(checkpoints + ".pdparams")
-        opti_dict = paddle.load(checkpoints + ".pdopt")
-        net.set_dict(para_dict)
-        optimizer.set_state_dict(opti_dict)
-        logger.info("Finish load checkpoints from {}".format(checkpoints))
-        return
-
-    pretrained_model = config.get('pretrained_model')
-    load_static_weights = config.get('load_static_weights', False)
-    use_distillation = config.get('use_distillation', False)
-    if pretrained_model:
-        if use_distillation:
-            load_distillation_model(net, pretrained_model, load_static_weights)
-        else:  # common load
-            load_dygraph_pretrain(
-                net,
-                path=pretrained_model,
-                load_static_weights=load_static_weights)
-            logger.info(
-                logger.coloring("Finish load pretrained model from {}".format(
-                    pretrained_model), "HEADER"))
-
-
-def _save_student_model(net, model_prefix):
-    """
-    save student model if the net is the network contains student
-    """
-    student_model_prefix = model_prefix + "_student.pdparams"
-    if hasattr(net, "_layers"):
-        net = net._layers
-    if hasattr(net, "student"):
-        paddle.save(net.student.state_dict(), student_model_prefix)
-        logger.info("Already save student model in {}".format(
-            student_model_prefix))
-
-
-def save_model(net, optimizer, model_path, epoch_id, prefix='ppcls'):
-    """
-    save model to the target path
-    """
-    if paddle.distributed.get_rank() != 0:
-        return
-    model_path = os.path.join(model_path, str(epoch_id))
-    _mkdir_if_not_exist(model_path)
-    model_prefix = os.path.join(model_path, prefix)
-
-    _save_student_model(net, model_prefix)
-
-    paddle.save(net.state_dict(), model_prefix + ".pdparams")
-    paddle.save(optimizer.state_dict(), model_prefix + ".pdopt")
-    logger.info("Already save model in {}".format(model_path))

paddlex/ppdet/__init__.py

@@ -1,16 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from . import (core, data, engine, modeling, model_zoo, optimizer, metrics,
-               utils, slim)

paddlex/ppdet/core/__init__.py

@@ -1,15 +0,0 @@
-#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from . import config