paddlex 2.0.0rc4__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/paddleseg/cvlibs/callbacks.py

@@ -1,279 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import time
-
-import numpy as np
-import paddle
-from paddle.distributed.parallel import ParallelEnv
-from visualdl import LogWriter
-from paddlex.paddleseg.utils.progbar import Progbar
-import paddlex.paddleseg.utils.logger as logger
-
-
-class CallbackList(object):
-    """
-    Container abstracting a list of callbacks.
-
-    Args:
-        callbacks (list[Callback]): List of `Callback` instances.
-    """
-
-    def __init__(self, callbacks=None):
-        callbacks = callbacks or []
-        self.callbacks = [c for c in callbacks]
-
-    def append(self, callback):
-        self.callbacks.append(callback)
-
-    def set_params(self, params):
-        for callback in self.callbacks:
-            callback.set_params(params)
-
-    def set_model(self, model):
-        for callback in self.callbacks:
-            callback.set_model(model)
-
-    def set_optimizer(self, optimizer):
-        for callback in self.callbacks:
-            callback.set_optimizer(optimizer)
-
-    def on_iter_begin(self, iter, logs=None):
-        """Called right before processing a batch.
-        """
-        logs = logs or {}
-        for callback in self.callbacks:
-            callback.on_iter_begin(iter, logs)
-        self._t_enter_iter = time.time()
-
-    def on_iter_end(self, iter, logs=None):
-        """Called at the end of a batch.
-        """
-        logs = logs or {}
-        for callback in self.callbacks:
-            callback.on_iter_end(iter, logs)
-        self._t_exit_iter = time.time()
-
-    def on_train_begin(self, logs=None):
-        """Called at the beginning of training.
-        """
-        logs = logs or {}
-        for callback in self.callbacks:
-            callback.on_train_begin(logs)
-
-    def on_train_end(self, logs=None):
-        """Called at the end of training.
-        """
-        logs = logs or {}
-        for callback in self.callbacks:
-            callback.on_train_end(logs)
-
-    def __iter__(self):
-        return iter(self.callbacks)
-
-
-class Callback(object):
-    """Abstract base class used to build new callbacks.
-    """
-
-    def __init__(self):
-        self.validation_data = None
-
-    def set_params(self, params):
-        self.params = params
-
-    def set_model(self, model):
-        self.model = model
-
-    def set_optimizer(self, optimizer):
-        self.optimizer = optimizer
-
-    def on_iter_begin(self, iter, logs=None):
-        pass
-
-    def on_iter_end(self, iter, logs=None):
-        pass
-
-    def on_train_begin(self, logs=None):
-        pass
-
-    def on_train_end(self, logs=None):
-        pass
-
-
-class BaseLogger(Callback):
-    def __init__(self, period=10):
-        super(BaseLogger, self).__init__()
-        self.period = period
-
-    def _reset(self):
-        self.totals = {}
-
-    def on_train_begin(self, logs=None):
-        self.totals = {}
-
-    def on_iter_end(self, iter, logs=None):
-        logs = logs or {}
-        #(iter - 1) // iters_per_epoch + 1
-        for k, v in logs.items():
-            if k in self.totals.keys():
-                self.totals[k] += v
-            else:
-                self.totals[k] = v
-
-        if iter % self.period == 0 and ParallelEnv().local_rank == 0:
-
-            for k in self.totals:
-                logs[k] = self.totals[k] / self.period
-            self._reset()
-
-
-class TrainLogger(Callback):
-    def __init__(self, log_freq=10):
-        self.log_freq = log_freq
-
-    def _calculate_eta(self, remaining_iters, speed):
-        if remaining_iters < 0:
-            remaining_iters = 0
-        remaining_time = int(remaining_iters * speed)
-        result = "{:0>2}:{:0>2}:{:0>2}"
-        arr = []
-        for i in range(2, -1, -1):
-            arr.append(int(remaining_time / 60**i))
-            remaining_time %= 60**i
-        return result.format(*arr)
-
-    def on_iter_end(self, iter, logs=None):
-
-        if iter % self.log_freq == 0 and ParallelEnv().local_rank == 0:
-            total_iters = self.params["total_iters"]
-            iters_per_epoch = self.params["iters_per_epoch"]
-            remaining_iters = total_iters - iter
-            eta = self._calculate_eta(remaining_iters, logs["batch_cost"])
-            current_epoch = (iter - 1) // self.params["iters_per_epoch"] + 1
-            loss = logs["loss"]
-            lr = self.optimizer.get_lr()
-            batch_cost = logs["batch_cost"]
-            reader_cost = logs["reader_cost"]
-
-            logger.info(
-                "[TRAIN] epoch={}, iter={}/{}, loss={:.4f}, lr={:.6f}, batch_cost={:.4f}, reader_cost={:.4f} | ETA {}"
-                .format(current_epoch, iter, total_iters, loss, lr, batch_cost,
-                        reader_cost, eta))
-
-
-class ProgbarLogger(Callback):
-    def __init__(self):
-        super(ProgbarLogger, self).__init__()
-
-    def on_train_begin(self, logs=None):
-        self.verbose = self.params["verbose"]
-        self.total_iters = self.params["total_iters"]
-        self.target = self.params["total_iters"]
-        self.progbar = Progbar(target=self.target, verbose=self.verbose)
-        self.seen = 0
-        self.log_values = []
-
-    def on_iter_begin(self, iter, logs=None):
-        #self.seen = 0
-        if self.seen < self.target:
-            self.log_values = []
-
-    def on_iter_end(self, iter, logs=None):
-        logs = logs or {}
-        self.seen += 1
-        for k in self.params['metrics']:
-            if k in logs:
-                self.log_values.append((k, logs[k]))
-
-        #if self.verbose and self.seen < self.target and ParallelEnv.local_rank == 0:
-        #print(self.log_values)
-        if self.seen < self.target:
-            self.progbar.update(self.seen, self.log_values)
-
-
-class ModelCheckpoint(Callback):
-    def __init__(self,
-                 save_dir,
-                 monitor="miou",
-                 save_best_only=False,
-                 save_params_only=True,
-                 mode="max",
-                 period=1):
-
-        super(ModelCheckpoint, self).__init__()
-        self.monitor = monitor
-        self.save_dir = save_dir
-        self.save_best_only = save_best_only
-        self.save_params_only = save_params_only
-        self.period = period
-        self.iters_since_last_save = 0
-
-        if mode == "min":
-            self.monitor_op = np.less
-            self.best = np.Inf
-        elif mode == "max":
-            self.monitor_op = np.greater
-            self.best = -np.Inf
-        else:
-            raise RuntimeError("`mode` is neither \"min\" nor \"max\"!")
-
-    def on_train_begin(self, logs=None):
-        self.verbose = self.params["verbose"]
-        save_dir = self.save_dir
-        if not os.path.isdir(save_dir):
-            if os.path.exists(save_dir):
-                os.remove(save_dir)
-            os.makedirs(save_dir)
-
-    def on_iter_end(self, iter, logs=None):
-        logs = logs or {}
-        self.iters_since_last_save += 1
-        current_save_dir = os.path.join(self.save_dir, "iter_{}".format(iter))
-        current_save_dir = os.path.abspath(current_save_dir)
-        #if self.iters_since_last_save % self.period and ParallelEnv().local_rank == 0:
-        #self.iters_since_last_save = 0
-        if iter % self.period == 0 and ParallelEnv().local_rank == 0:
-            if self.verbose > 0:
-                print("iter {iter_num}: saving model to {path}".format(
-                    iter_num=iter, path=current_save_dir))
-
-            paddle.save(self.model.state_dict(),
-                        os.path.join(current_save_dir, 'model.pdparams'))
-
-            if not self.save_params_only:
-                paddle.save(self.optimizer.state_dict(),
-                            os.path.join(current_save_dir, 'model.pdopt'))
-
-
-class VisualDL(Callback):
-    def __init__(self, log_dir="./log", freq=1):
-        super(VisualDL, self).__init__()
-        self.log_dir = log_dir
-        self.freq = freq
-
-    def on_train_begin(self, logs=None):
-        self.writer = LogWriter(self.log_dir)
-
-    def on_iter_end(self, iter, logs=None):
-        logs = logs or {}
-        if iter % self.freq == 0 and ParallelEnv().local_rank == 0:
-            for k, v in logs.items():
-                self.writer.add_scalar("Train/{}".format(k), v, iter)
-
-        self.writer.flush()
-
-    def on_train_end(self, logs=None):
-        self.writer.close()

paddlex/paddleseg/cvlibs/config.py

@@ -1,359 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import codecs
-import os
-from typing import Any, Dict, Generic
-
-import paddle
-import yaml
-
-from paddlex.paddleseg.cvlibs import manager
-from paddlex.paddleseg.utils import logger
-
-
-class Config(object):
-    '''
-    Training configuration parsing. The only yaml/yml file is supported.
-
-    The following hyper-parameters are available in the config file:
-        batch_size: The number of samples per gpu.
-        iters: The total training steps.
-        train_dataset: A training data config including type/data_root/transforms/mode.
-            For data type, please refer to paddleseg.datasets.
-            For specific transforms, please refer to paddleseg.transforms.transforms.
-        val_dataset: A validation data config including type/data_root/transforms/mode.
-        optimizer: A optimizer config, but currently PaddleSeg only supports sgd with momentum in config file.
-            In addition, weight_decay could be set as a regularization.
-        learning_rate: A learning rate config. If decay is configured, learning _rate value is the starting learning rate,
-             where only poly decay is supported using the config file. In addition, decay power and end_lr are tuned experimentally.
-        loss: A loss config. Multi-loss config is available. The loss type order is consistent with the seg model outputs,
-            where the coef term indicates the weight of corresponding loss. Note that the number of coef must be the same as the number of
-            model outputs, and there could be only one loss type if using the same loss type among the outputs, otherwise the number of
-            loss type must be consistent with coef.
-        model: A model config including type/backbone and model-dependent arguments.
-            For model type, please refer to paddleseg.models.
-            For backbone, please refer to paddleseg.models.backbones.
-
-    Args:
-        path (str) : The path of config file, supports yaml format only.
-
-    Examples:
-
-        from paddlex.paddleseg.cvlibs.config import Config
-
-        # Create a cfg object with yaml file path.
-        cfg = Config(yaml_cfg_path)
-
-        # Parsing the argument when its property is used.
-        train_dataset = cfg.train_dataset
-
-        # the argument of model should be parsed after dataset,
-        # since the model builder uses some properties in dataset.
-        model = cfg.model
-        ...
-    '''
-
-    def __init__(self,
-                 path: str,
-                 learning_rate: float=None,
-                 batch_size: int=None,
-                 iters: int=None):
-        if not path:
-            raise ValueError('Please specify the configuration file path.')
-
-        if not os.path.exists(path):
-            raise FileNotFoundError('File {} does not exist'.format(path))
-
-        self._model = None
-        self._losses = None
-        if path.endswith('yml') or path.endswith('yaml'):
-            self.dic = self._parse_from_yaml(path)
-        else:
-            raise RuntimeError('Config file should in yaml format!')
-
-        self.update(
-            learning_rate=learning_rate, batch_size=batch_size, iters=iters)
-
-    def _update_dic(self, dic, base_dic):
-        """
-        Update config from dic based base_dic
-        """
-        base_dic = base_dic.copy()
-        for key, val in dic.items():
-            if isinstance(val, dict) and key in base_dic:
-                base_dic[key] = self._update_dic(val, base_dic[key])
-            else:
-                base_dic[key] = val
-        dic = base_dic
-        return dic
-
-    def _parse_from_yaml(self, path: str):
-        '''Parse a yaml file and build config'''
-        with codecs.open(path, 'r', 'utf-8') as file:
-            dic = yaml.load(file, Loader=yaml.FullLoader)
-
-        if '_base_' in dic:
-            cfg_dir = os.path.dirname(path)
-            base_path = dic.pop('_base_')
-            base_path = os.path.join(cfg_dir, base_path)
-            base_dic = self._parse_from_yaml(base_path)
-            dic = self._update_dic(dic, base_dic)
-        return dic
-
-    def update(self,
-               learning_rate: float=None,
-               batch_size: int=None,
-               iters: int=None):
-        '''Update config'''
-        if learning_rate:
-            if 'lr_scheduler' in self.dic:
-                self.dic['lr_scheduler']['learning_rate'] = learning_rate
-            else:
-                self.dic['learning_rate']['value'] = learning_rate
-
-        if batch_size:
-            self.dic['batch_size'] = batch_size
-
-        if iters:
-            self.dic['iters'] = iters
-
-    @property
-    def batch_size(self) -> int:
-        return self.dic.get('batch_size', 1)
-
-    @property
-    def iters(self) -> int:
-        iters = self.dic.get('iters')
-        if not iters:
-            raise RuntimeError('No iters specified in the configuration file.')
-        return iters
-
-    @property
-    def lr_scheduler(self) -> paddle.optimizer.lr.LRScheduler:
-        if 'lr_scheduler' not in self.dic:
-            raise RuntimeError(
-                'No `lr_scheduler` specified in the configuration file.')
-        params = self.dic.get('lr_scheduler')
-
-        lr_type = params.pop('type')
-        if lr_type == 'PolynomialDecay':
-            params.setdefault('decay_steps', self.iters)
-            params.setdefault('end_lr', 0)
-            params.setdefault('power', 0.9)
-
-        return getattr(paddle.optimizer.lr, lr_type)(**params)
-
-    @property
-    def learning_rate(self) -> paddle.optimizer.lr.LRScheduler:
-        logger.warning(
-            '''`learning_rate` in configuration file will be deprecated, please use `lr_scheduler` instead. E.g
-            lr_scheduler:
-                type: PolynomialDecay
-                learning_rate: 0.01''')
-        _learning_rate = self.dic.get('learning_rate', {}).get('value')
-        if not _learning_rate:
-            raise RuntimeError(
-                'No learning rate specified in the configuration file.')
-
-        args = self.decay_args
-        decay_type = args.pop('type')
-
-        if decay_type == 'poly':
-            lr = _learning_rate
-            return paddle.optimizer.lr.PolynomialDecay(lr, **args)
-        elif decay_type == 'piecewise':
-            values = _learning_rate
-            return paddle.optimizer.lr.PiecewiseDecay(values=values, **args)
-        else:
-            raise RuntimeError('Only poly and piecewise decay support.')
-
-    @property
-    def optimizer(self) -> paddle.optimizer.Optimizer:
-        if 'lr_scheduler' in self.dic:
-            lr = self.lr_scheduler
-        else:
-            lr = self.learning_rate
-        args = self.optimizer_args
-        optimizer_type = args.pop('type')
-
-        if optimizer_type == 'sgd':
-            return paddle.optimizer.Momentum(
-                lr, parameters=self.model.parameters(), **args)
-        elif optimizer_type == 'adam':
-            return paddle.optimizer.Adam(
-                lr, parameters=self.model.parameters(), **args)
-        else:
-            raise RuntimeError('Only sgd and adam optimizer support.')
-
-    @property
-    def optimizer_args(self) -> dict:
-        args = self.dic.get('optimizer', {}).copy()
-        if args['type'] == 'sgd':
-            args.setdefault('momentum', 0.9)
-
-        return args
-
-    @property
-    def decay_args(self) -> dict:
-        args = self.dic.get('learning_rate', {}).get(
-            'decay', {'type': 'poly',
-                      'power': 0.9}).copy()
-
-        if args['type'] == 'poly':
-            args.setdefault('decay_steps', self.iters)
-            args.setdefault('end_lr', 0)
-
-        return args
-
-    @property
-    def loss(self) -> dict:
-        args = self.dic.get('loss', {}).copy()
-        if 'types' in args and 'coef' in args:
-            len_types = len(args['types'])
-            len_coef = len(args['coef'])
-            if len_types != len_coef:
-                if len_types == 1:
-                    args['types'] = args['types'] * len_coef
-                else:
-                    raise ValueError(
-                        'The length of types should equal to coef or equal to 1 in loss config, but they are {} and {}.'
-                        .format(len_types, len_coef))
-        else:
-            raise ValueError(
-                'Loss config should contain keys of "types" and "coef"')
-
-        if not self._losses:
-            self._losses = dict()
-            for key, val in args.items():
-                if key == 'types':
-                    self._losses['types'] = []
-                    for item in args['types']:
-                        if item['type'] != 'MixedLoss':
-                            item['ignore_index'] = \
-                                self.train_dataset.ignore_index
-                        self._losses['types'].append(self._load_object(item))
-                else:
-                    self._losses[key] = val
-            if len(self._losses['coef']) != len(self._losses['types']):
-                raise RuntimeError(
-                    'The length of coef should equal to types in loss config: {} != {}.'
-                    .format(
-                        len(self._losses['coef']), len(self._losses['types'])))
-        return self._losses
-
-    @property
-    def model(self) -> paddle.nn.Layer:
-        model_cfg = self.dic.get('model').copy()
-        if not model_cfg:
-            raise RuntimeError('No model specified in the configuration file.')
-        if not 'num_classes' in model_cfg:
-            num_classes = None
-            if self.train_dataset_config:
-                if hasattr(self.train_dataset_class, 'NUM_CLASSES'):
-                    num_classes = self.train_dataset_class.NUM_CLASSES
-                elif hasattr(self.train_dataset, 'num_classes'):
-                    num_classes = self.train_dataset.num_classes
-            elif self.val_dataset_config:
-                if hasattr(self.val_dataset_class, 'NUM_CLASSES'):
-                    num_classes = self.val_dataset_class.NUM_CLASSES
-                elif hasattr(self.val_dataset, 'num_classes'):
-                    num_classes = self.val_dataset.num_classes
-
-            if not num_classes:
-                raise ValueError(
-                    '`num_classes` is not found. Please set it in model, train_dataset or val_dataset'
-                )
-
-            model_cfg['num_classes'] = num_classes
-
-        if not self._model:
-            self._model = self._load_object(model_cfg)
-        return self._model
-
-    @property
-    def train_dataset_config(self) -> Dict:
-        return self.dic.get('train_dataset', {}).copy()
-
-    @property
-    def val_dataset_config(self) -> Dict:
-        return self.dic.get('val_dataset', {}).copy()
-
-    @property
-    def train_dataset_class(self) -> Generic:
-        dataset_type = self.train_dataset_config['type']
-        return self._load_component(dataset_type)
-
-    @property
-    def val_dataset_class(self) -> Generic:
-        dataset_type = self.val_dataset_config['type']
-        return self._load_component(dataset_type)
-
-    @property
-    def train_dataset(self) -> paddle.io.Dataset:
-        _train_dataset = self.train_dataset_config
-        if not _train_dataset:
-            return None
-        return self._load_object(_train_dataset)
-
-    @property
-    def val_dataset(self) -> paddle.io.Dataset:
-        _val_dataset = self.val_dataset_config
-        if not _val_dataset:
-            return None
-        return self._load_object(_val_dataset)
-
-    def _load_component(self, com_name: str) -> Any:
-        com_list = [
-            manager.MODELS, manager.BACKBONES, manager.DATASETS,
-            manager.TRANSFORMS, manager.LOSSES
-        ]
-
-        for com in com_list:
-            if com_name in com.components_dict:
-                return com[com_name]
-        else:
-            raise RuntimeError(
-                'The specified component was not found {}.'.format(com_name))
-
-    def _load_object(self, cfg: dict) -> Any:
-        cfg = cfg.copy()
-        if 'type' not in cfg:
-            raise RuntimeError('No object information in {}.'.format(cfg))
-
-        component = self._load_component(cfg.pop('type'))
-
-        params = {}
-        for key, val in cfg.items():
-            if self._is_meta_type(val):
-                params[key] = self._load_object(val)
-            elif isinstance(val, list):
-                params[key] = [
-                    self._load_object(item)
-                    if self._is_meta_type(item) else item for item in val
-                ]
-            else:
-                params[key] = val
-
-        return component(**params)
-
-    @property
-    def export_config(self) -> Dict:
-        return self.dic.get('export', {})
-
-    def _is_meta_type(self, item: Any) -> bool:
-        return isinstance(item, dict) and 'type' in item
-
-    def __str__(self) -> str:
-        return yaml.dump(self.dic)