paddlex 2.1.0__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/ppdet/slim/unstructured_prune.py

@@ -1,66 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from paddle.utils import try_import
-
-from paddlex.ppdet.core.workspace import register, serializable
-from paddlex.ppdet.utils.logger import setup_logger
-logger = setup_logger(__name__)
-
-
-@register
-@serializable
-class UnstructuredPruner(object):
-    def __init__(self,
-                 stable_epochs,
-                 pruning_epochs,
-                 tunning_epochs,
-                 pruning_steps,
-                 ratio,
-                 initial_ratio,
-                 prune_params_type=None):
-        self.stable_epochs = stable_epochs
-        self.pruning_epochs = pruning_epochs
-        self.tunning_epochs = tunning_epochs
-        self.ratio = ratio
-        self.prune_params_type = prune_params_type
-        self.initial_ratio = initial_ratio
-        self.pruning_steps = pruning_steps
-
-    def __call__(self, model, steps_per_epoch, skip_params_func=None):
-        paddleslim = try_import('paddleslim')
-        from paddleslim import GMPUnstructuredPruner
-        configs = {
-            'pruning_strategy': 'gmp',
-            'stable_iterations': self.stable_epochs * steps_per_epoch,
-            'pruning_iterations': self.pruning_epochs * steps_per_epoch,
-            'tunning_iterations': self.tunning_epochs * steps_per_epoch,
-            'resume_iteration': 0,
-            'pruning_steps': self.pruning_steps,
-            'initial_ratio': self.initial_ratio,
-        }
-
-        pruner = GMPUnstructuredPruner(
-            model,
-            ratio=self.ratio,
-            skip_params_func=skip_params_func,
-            prune_params_type=self.prune_params_type,
-            local_sparsity=True,
-            configs=configs)
-
-        return pruner

paddlex/ppdet/utils/__init__.py

@@ -1,13 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

paddlex/ppdet/utils/check.py

@@ -1,112 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import sys
-
-import paddle
-import six
-import paddle.version as fluid_version
-
-from .logger import setup_logger
-logger = setup_logger(__name__)
-
-__all__ = ['check_gpu', 'check_npu', 'check_version', 'check_config']
-
-
-def check_npu(use_npu):
-    """
-    Log error and exit when set use_npu=true in paddlepaddle
-    cpu/gpu/xpu version.
-    """
-    err = "Config use_npu cannot be set as true while you are " \
-          "using paddlepaddle cpu/gpu/xpu version ! \nPlease try: \n" \
-          "\t1. Install paddlepaddle-npu to run model on NPU \n" \
-          "\t2. Set use_npu as false in config file to run " \
-          "model on CPU/GPU/XPU"
-
-    try:
-        if use_npu and not paddle.is_compiled_with_npu():
-            logger.error(err)
-            sys.exit(1)
-    except Exception as e:
-        pass
-
-
-def check_gpu(use_gpu):
-    """
-    Log error and exit when set use_gpu=true in paddlepaddle
-    cpu version.
-    """
-    err = "Config use_gpu cannot be set as true while you are " \
-          "using paddlepaddle cpu version ! \nPlease try: \n" \
-          "\t1. Install paddlepaddle-gpu to run model on GPU \n" \
-          "\t2. Set use_gpu as false in config file to run " \
-          "model on CPU"
-
-    try:
-        if use_gpu and not paddle.is_compiled_with_cuda():
-            logger.error(err)
-            sys.exit(1)
-    except Exception as e:
-        pass
-
-
-def check_version(version='2.0'):
-    """
-    Log error and exit when the installed version of paddlepaddle is
-    not satisfied.
-    """
-    err = "PaddlePaddle version {} or higher is required, " \
-          "or a suitable develop version is satisfied as well. \n" \
-          "Please make sure the version is good with your code.".format(version)
-
-    version_installed = [
-        fluid_version.major, fluid_version.minor, fluid_version.patch,
-        fluid_version.rc
-    ]
-    if version_installed == ['0', '0', '0', '0']:
-        return
-    version_split = version.split('.')
-
-    length = min(len(version_installed), len(version_split))
-    for i in six.moves.range(length):
-        if version_installed[i] > version_split[i]:
-            return
-        if version_installed[i] < version_split[i]:
-            raise Exception(err)
-
-
-def check_config(cfg):
-    """
-    Check the correctness of the configuration file. Log error and exit
-    when Config is not compliant.
-    """
-    err = "'{}' not specified in config file. Please set it in config file."
-    check_list = ['architecture', 'num_classes']
-    try:
-        for var in check_list:
-            if not var in cfg:
-                logger.error(err.format(var))
-                sys.exit(1)
-    except Exception as e:
-        pass
-
-    if 'log_iter' not in cfg:
-        cfg.log_iter = 20
-
-    return cfg

paddlex/ppdet/utils/checkpoint.py

@@ -1,226 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import errno
-import os
-import time
-import numpy as np
-import paddle
-import paddle.nn as nn
-from .download import get_weights_path
-
-from .logger import setup_logger
-logger = setup_logger(__name__)
-
-
-def is_url(path):
-    """
-    Whether path is URL.
-    Args:
-        path (string): URL string or not.
-    """
-    return path.startswith('http://') \
-            or path.startswith('https://') \
-            or path.startswith('ppdet://')
-
-
-def _get_unique_endpoints(trainer_endpoints):
-    # Sorting is to avoid different environmental variables for each card
-    trainer_endpoints.sort()
-    ips = set()
-    unique_endpoints = set()
-    for endpoint in trainer_endpoints:
-        ip = endpoint.split(":")[0]
-        if ip in ips:
-            continue
-        ips.add(ip)
-        unique_endpoints.add(endpoint)
-    logger.info("unique_endpoints {}".format(unique_endpoints))
-    return unique_endpoints
-
-
-def _strip_postfix(path):
-    path, ext = os.path.splitext(path)
-    assert ext in ['', '.pdparams', '.pdopt', '.pdmodel'], \
-            "Unknown postfix {} from weights".format(ext)
-    return path
-
-
-def load_weight(model, weight, optimizer=None):
-    if is_url(weight):
-        weight = get_weights_path(weight)
-
-    path = _strip_postfix(weight)
-    pdparam_path = path + '.pdparams'
-    if not os.path.exists(pdparam_path):
-        raise ValueError("Model pretrain path {} does not "
-                         "exists.".format(pdparam_path))
-
-    param_state_dict = paddle.load(pdparam_path)
-    model_dict = model.state_dict()
-    model_weight = {}
-    incorrect_keys = 0
-
-    for key in model_dict.keys():
-        if key in param_state_dict.keys():
-            model_weight[key] = param_state_dict[key]
-        else:
-            logger.info('Unmatched key: {}'.format(key))
-            incorrect_keys += 1
-
-    assert incorrect_keys == 0, "Load weight {} incorrectly, \
-            {} keys unmatched, please check again.".format(weight,
-                                                           incorrect_keys)
-    logger.info('Finish resuming model weights: {}'.format(pdparam_path))
-
-    model.set_dict(model_weight)
-
-    last_epoch = 0
-    if optimizer is not None and os.path.exists(path + '.pdopt'):
-        optim_state_dict = paddle.load(path + '.pdopt')
-        # to solve resume bug, will it be fixed in paddle 2.0
-        for key in optimizer.state_dict().keys():
-            if not key in optim_state_dict.keys():
-                optim_state_dict[key] = optimizer.state_dict()[key]
-        if 'last_epoch' in optim_state_dict:
-            last_epoch = optim_state_dict.pop('last_epoch')
-        optimizer.set_state_dict(optim_state_dict)
-
-    return last_epoch
-
-
-def match_state_dict(model_state_dict, weight_state_dict):
-    """
-    Match between the model state dict and pretrained weight state dict.
-    Return the matched state dict.
-
-    The method supposes that all the names in pretrained weight state dict are
-    subclass of the names in models`, if the prefix 'backbone.' in pretrained weight
-    keys is stripped. And we could get the candidates for each model key. Then we
-    select the name with the longest matched size as the final match result. For
-    example, the model state dict has the name of
-    'backbone.res2.res2a.branch2a.conv.weight' and the pretrained weight as
-    name of 'res2.res2a.branch2a.conv.weight' and 'branch2a.conv.weight'. We
-    match the 'res2.res2a.branch2a.conv.weight' to the model key.
-    """
-
-    model_keys = sorted(model_state_dict.keys())
-    weight_keys = sorted(weight_state_dict.keys())
-
-    def match(a, b):
-        if a.startswith('backbone.res5'):
-            # In Faster RCNN, res5 pretrained weights have prefix of backbone,
-            # however, the corresponding model weights have difficult prefix,
-            # bbox_head.
-            b = b[9:]
-        return a == b or a.endswith("." + b)
-
-    match_matrix = np.zeros([len(model_keys), len(weight_keys)])
-    for i, m_k in enumerate(model_keys):
-        for j, w_k in enumerate(weight_keys):
-            if match(m_k, w_k):
-                match_matrix[i, j] = len(w_k)
-    max_id = match_matrix.argmax(1)
-    max_len = match_matrix.max(1)
-    max_id[max_len == 0] = -1
-    not_load_weight_name = []
-    for match_idx in range(len(max_id)):
-        if match_idx < len(weight_keys) and max_id[match_idx] == -1:
-            not_load_weight_name.append(weight_keys[match_idx])
-    if len(not_load_weight_name) > 0:
-        logger.info('{} in pretrained weight is not used in the model, '
-                    'and its will not be loaded'.format(not_load_weight_name))
-    matched_keys = {}
-    result_state_dict = {}
-    for model_id, weight_id in enumerate(max_id):
-        if weight_id == -1:
-            continue
-        model_key = model_keys[model_id]
-        weight_key = weight_keys[weight_id]
-        weight_value = weight_state_dict[weight_key]
-        model_value_shape = list(model_state_dict[model_key].shape)
-
-        if list(weight_value.shape) != model_value_shape:
-            logger.info(
-                'The shape {} in pretrained weight {} is unmatched with '
-                'the shape {} in model {}. And the weight {} will not be '
-                'loaded'.format(weight_value.shape, weight_key,
-                                model_value_shape, model_key, weight_key))
-            continue
-
-        assert model_key not in result_state_dict
-        result_state_dict[model_key] = weight_value
-        if weight_key in matched_keys:
-            raise ValueError('Ambiguity weight {} loaded, it matches at least '
-                             '{} and {} in the model'.format(
-                                 weight_key, model_key, matched_keys[
-                                     weight_key]))
-        matched_keys[weight_key] = model_key
-    return result_state_dict
-
-
-def load_pretrain_weight(model, pretrain_weight):
-    if is_url(pretrain_weight):
-        pretrain_weight = get_weights_path(pretrain_weight)
-
-    path = _strip_postfix(pretrain_weight)
-    if not (os.path.isdir(path) or os.path.isfile(path) or
-            os.path.exists(path + '.pdparams')):
-        raise ValueError("Model pretrain path `{}` does not exists. "
-                         "If you don't want to load pretrain model, "
-                         "please delete `pretrain_weights` field in "
-                         "config file.".format(path))
-
-    model_dict = model.state_dict()
-
-    weights_path = path + '.pdparams'
-    param_state_dict = paddle.load(weights_path)
-    param_state_dict = match_state_dict(model_dict, param_state_dict)
-
-    model.set_dict(param_state_dict)
-    logger.info('Finish loading model weights: {}'.format(weights_path))
-
-
-def save_model(model, optimizer, save_dir, save_name, last_epoch):
-    """
-    save model into disk.
-
-    Args:
-        model (paddle.nn.Layer): the Layer instalce to save parameters.
-        optimizer (paddle.optimizer.Optimizer): the Optimizer instance to
-            save optimizer states.
-        save_dir (str): the directory to be saved.
-        save_name (str): the path to be saved.
-        last_epoch (int): the epoch index.
-    """
-    if paddle.distributed.get_rank() != 0:
-        return
-    if not os.path.exists(save_dir):
-        os.makedirs(save_dir)
-    save_path = os.path.join(save_dir, save_name)
-    if isinstance(model, nn.Layer):
-        paddle.save(model.state_dict(), save_path + ".pdparams")
-    else:
-        assert isinstance(model,
-                          dict), 'model is not a instance of nn.layer or dict'
-        paddle.save(model, save_path + ".pdparams")
-    state_dict = optimizer.state_dict()
-    state_dict['last_epoch'] = last_epoch
-    paddle.save(state_dict, save_path + ".pdopt")
-    logger.info("Save checkpoint: {}".format(save_dir))

paddlex/ppdet/utils/cli.py

@@ -1,151 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from argparse import ArgumentParser, RawDescriptionHelpFormatter
-
-import yaml
-import re
-from paddlex.ppdet.core.workspace import get_registered_modules, dump_value
-
-__all__ = ['ColorTTY', 'ArgsParser']
-
-
-class ColorTTY(object):
-    def __init__(self):
-        super(ColorTTY, self).__init__()
-        self.colors = ['red', 'green', 'yellow', 'blue', 'magenta', 'cyan']
-
-    def __getattr__(self, attr):
-        if attr in self.colors:
-            color = self.colors.index(attr) + 31
-
-            def color_message(message):
-                return "[{}m{}".format(color, message)
-
-            setattr(self, attr, color_message)
-            return color_message
-
-    def bold(self, message):
-        return self.with_code('01', message)
-
-    def with_code(self, code, message):
-        return "[{}m{}".format(code, message)
-
-
-class ArgsParser(ArgumentParser):
-    def __init__(self):
-        super(ArgsParser, self).__init__(
-            formatter_class=RawDescriptionHelpFormatter)
-        self.add_argument("-c", "--config", help="configuration file to use")
-        self.add_argument(
-            "-o", "--opt", nargs='*', help="set configuration options")
-
-    def parse_args(self, argv=None):
-        args = super(ArgsParser, self).parse_args(argv)
-        assert args.config is not None, \
-            "Please specify --config=configure_file_path."
-        args.opt = self._parse_opt(args.opt)
-        return args
-
-    def _parse_opt(self, opts):
-        config = {}
-        if not opts:
-            return config
-        for s in opts:
-            s = s.strip()
-            k, v = s.split('=', 1)
-            if '.' not in k:
-                config[k] = yaml.load(v, Loader=yaml.Loader)
-            else:
-                keys = k.split('.')
-                if keys[0] not in config:
-                    config[keys[0]] = {}
-                cur = config[keys[0]]
-                for idx, key in enumerate(keys[1:]):
-                    if idx == len(keys) - 2:
-                        cur[key] = yaml.load(v, Loader=yaml.Loader)
-                    else:
-                        cur[key] = {}
-                        cur = cur[key]
-        return config
-
-
-def print_total_cfg(config):
-    modules = get_registered_modules()
-    color_tty = ColorTTY()
-    green = '___{}___'.format(color_tty.colors.index('green') + 31)
-
-    styled = {}
-    for key in config.keys():
-        if not config[key]:  # empty schema
-            continue
-
-        if key not in modules and not hasattr(config[key], '__dict__'):
-            styled[key] = config[key]
-            continue
-        elif key in modules:
-            module = modules[key]
-        else:
-            type_name = type(config[key]).__name__
-            if type_name in modules:
-                module = modules[type_name].copy()
-                module.update({
-                    k: v
-                    for k, v in config[key].__dict__.items()
-                    if k in module.schema
-                })
-                key += " ({})".format(type_name)
-        default = module.find_default_keys()
-        missing = module.find_missing_keys()
-        mismatch = module.find_mismatch_keys()
-        extra = module.find_extra_keys()
-        dep_missing = []
-        for dep in module.inject:
-            if isinstance(module[dep], str) and module[dep] != '<value>':
-                if module[dep] not in modules:  # not a valid module
-                    dep_missing.append(dep)
-                else:
-                    dep_mod = modules[module[dep]]
-                    # empty dict but mandatory
-                    if not dep_mod and dep_mod.mandatory():
-                        dep_missing.append(dep)
-        override = list(
-            set(module.keys()) - set(default) - set(extra) - set(dep_missing))
-        replacement = {}
-        for name in set(override + default + extra + mismatch + missing):
-            new_name = name
-            if name in missing:
-                value = "<missing>"
-            else:
-                value = module[name]
-
-            if name in extra:
-                value = dump_value(value) + " <extraneous>"
-            elif name in mismatch:
-                value = dump_value(value) + " <type mismatch>"
-            elif name in dep_missing:
-                value = dump_value(value) + " <module config missing>"
-            elif name in override and value != '<missing>':
-                mark = green
-                new_name = mark + name
-            replacement[new_name] = value
-        styled[key] = replacement
-    buffer = yaml.dump(styled, default_flow_style=False, default_style='')
-    buffer = (re.sub(r"<missing>", r"[31m<missing>[0m", buffer))
-    buffer = (re.sub(r"<extraneous>", r"[33m<extraneous>[0m", buffer))
-    buffer = (re.sub(r"<type mismatch>", r"[31m<type mismatch>[0m", buffer))
-    buffer = (re.sub(r"<module config missing>",
-                     r"[31m<module config missing>[0m", buffer))
-    buffer = re.sub(r"___(\d+)___(.*?):", r"[\1m\2[0m:", buffer)
-    print(buffer)

paddlex/ppdet/utils/colormap.py

@@ -1,58 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import numpy as np
-
-
-def colormap(rgb=False):
-    """
-    Get colormap
-
-    The code of this function is copied from https://github.com/facebookresearch/Detectron/blob/main/detectron/utils/colormap.py
-    """
-    color_list = np.array([
-        0.000, 0.447, 0.741, 0.850, 0.325, 0.098, 0.929, 0.694, 0.125, 0.494,
-        0.184, 0.556, 0.466, 0.674, 0.188, 0.301, 0.745, 0.933, 0.635, 0.078,
-        0.184, 0.300, 0.300, 0.300, 0.600, 0.600, 0.600, 1.000, 0.000, 0.000,
-        1.000, 0.500, 0.000, 0.749, 0.749, 0.000, 0.000, 1.000, 0.000, 0.000,
-        0.000, 1.000, 0.667, 0.000, 1.000, 0.333, 0.333, 0.000, 0.333, 0.667,
-        0.000, 0.333, 1.000, 0.000, 0.667, 0.333, 0.000, 0.667, 0.667, 0.000,
-        0.667, 1.000, 0.000, 1.000, 0.333, 0.000, 1.000, 0.667, 0.000, 1.000,
-        1.000, 0.000, 0.000, 0.333, 0.500, 0.000, 0.667, 0.500, 0.000, 1.000,
-        0.500, 0.333, 0.000, 0.500, 0.333, 0.333, 0.500, 0.333, 0.667, 0.500,
-        0.333, 1.000, 0.500, 0.667, 0.000, 0.500, 0.667, 0.333, 0.500, 0.667,
-        0.667, 0.500, 0.667, 1.000, 0.500, 1.000, 0.000, 0.500, 1.000, 0.333,
-        0.500, 1.000, 0.667, 0.500, 1.000, 1.000, 0.500, 0.000, 0.333, 1.000,
-        0.000, 0.667, 1.000, 0.000, 1.000, 1.000, 0.333, 0.000, 1.000, 0.333,
-        0.333, 1.000, 0.333, 0.667, 1.000, 0.333, 1.000, 1.000, 0.667, 0.000,
-        1.000, 0.667, 0.333, 1.000, 0.667, 0.667, 1.000, 0.667, 1.000, 1.000,
-        1.000, 0.000, 1.000, 1.000, 0.333, 1.000, 1.000, 0.667, 1.000, 0.167,
-        0.000, 0.000, 0.333, 0.000, 0.000, 0.500, 0.000, 0.000, 0.667, 0.000,
-        0.000, 0.833, 0.000, 0.000, 1.000, 0.000, 0.000, 0.000, 0.167, 0.000,
-        0.000, 0.333, 0.000, 0.000, 0.500, 0.000, 0.000, 0.667, 0.000, 0.000,
-        0.833, 0.000, 0.000, 1.000, 0.000, 0.000, 0.000, 0.167, 0.000, 0.000,
-        0.333, 0.000, 0.000, 0.500, 0.000, 0.000, 0.667, 0.000, 0.000, 0.833,
-        0.000, 0.000, 1.000, 0.000, 0.000, 0.000, 0.143, 0.143, 0.143, 0.286,
-        0.286, 0.286, 0.429, 0.429, 0.429, 0.571, 0.571, 0.571, 0.714, 0.714,
-        0.714, 0.857, 0.857, 0.857, 1.000, 1.000, 1.000
-    ]).astype(np.float32)
-    color_list = color_list.reshape((-1, 3)) * 255
-    if not rgb:
-        color_list = color_list[:, ::-1]
-    return color_list