paddlex 2.1.0__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/ppdet/engine/__init__.py

@@ -1,30 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from . import trainer
-from .trainer import *
-
-from . import callbacks
-from .callbacks import *
-
-from . import env
-from .env import *
-
-__all__ = trainer.__all__ \
-        + callbacks.__all__ \
-        + env.__all__
-
-from . import tracker
-from .tracker import *
-__all__ = __all__ + tracker.__all__

paddlex/ppdet/engine/callbacks.py

@@ -1,340 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import sys
-import datetime
-import six
-import copy
-import json
-
-import paddle
-import paddle.distributed as dist
-
-from paddlex.ppdet.utils.checkpoint import save_model
-from paddlex.ppdet.metrics import get_infer_results
-
-from paddlex.ppdet.utils.logger import setup_logger
-logger = setup_logger('ppdet.engine')
-
-__all__ = [
-    'Callback', 'ComposeCallback', 'LogPrinter', 'Checkpointer',
-    'VisualDLWriter', 'SniperProposalsGenerator'
-]
-
-
-class Callback(object):
-    def __init__(self, model):
-        self.model = model
-
-    def on_step_begin(self, status):
-        pass
-
-    def on_step_end(self, status):
-        pass
-
-    def on_epoch_begin(self, status):
-        pass
-
-    def on_epoch_end(self, status):
-        pass
-
-    def on_train_begin(self, status):
-        pass
-
-    def on_train_end(self, status):
-        pass
-
-
-class ComposeCallback(object):
-    def __init__(self, callbacks):
-        callbacks = [c for c in list(callbacks) if c is not None]
-        for c in callbacks:
-            assert isinstance(
-                c, Callback), "callback should be subclass of Callback"
-        self._callbacks = callbacks
-
-    def on_step_begin(self, status):
-        for c in self._callbacks:
-            c.on_step_begin(status)
-
-    def on_step_end(self, status):
-        for c in self._callbacks:
-            c.on_step_end(status)
-
-    def on_epoch_begin(self, status):
-        for c in self._callbacks:
-            c.on_epoch_begin(status)
-
-    def on_epoch_end(self, status):
-        for c in self._callbacks:
-            c.on_epoch_end(status)
-
-    def on_train_begin(self, status):
-        for c in self._callbacks:
-            c.on_train_begin(status)
-
-    def on_train_end(self, status):
-        for c in self._callbacks:
-            c.on_train_end(status)
-
-
-class LogPrinter(Callback):
-    def __init__(self, model):
-        super(LogPrinter, self).__init__(model)
-
-    def on_step_end(self, status):
-        if dist.get_world_size() < 2 or dist.get_rank() == 0:
-            mode = status['mode']
-            if mode == 'train':
-                epoch_id = status['epoch_id']
-                step_id = status['step_id']
-                steps_per_epoch = status['steps_per_epoch']
-                training_staus = status['training_staus']
-                batch_time = status['batch_time']
-                data_time = status['data_time']
-
-                epoches = self.model.cfg.epoch
-                batch_size = self.model.cfg['{}Reader'.format(mode.capitalize(
-                ))]['batch_size']
-
-                logs = training_staus.log()
-                space_fmt = ':' + str(len(str(steps_per_epoch))) + 'd'
-                if step_id % self.model.cfg.log_iter == 0:
-                    eta_steps = (epoches - epoch_id
-                                 ) * steps_per_epoch - step_id
-                    eta_sec = eta_steps * batch_time.global_avg
-                    eta_str = str(datetime.timedelta(seconds=int(eta_sec)))
-                    ips = float(batch_size) / batch_time.avg
-                    fmt = ' '.join([
-                        'Epoch: [{}]',
-                        '[{' + space_fmt + '}/{}]',
-                        'learning_rate: {lr:.6f}',
-                        '{meters}',
-                        'eta: {eta}',
-                        'batch_cost: {btime}',
-                        'data_cost: {dtime}',
-                        'ips: {ips:.4f} images/s',
-                    ])
-                    fmt = fmt.format(
-                        epoch_id,
-                        step_id,
-                        steps_per_epoch,
-                        lr=status['learning_rate'],
-                        meters=logs,
-                        eta=eta_str,
-                        btime=str(batch_time),
-                        dtime=str(data_time),
-                        ips=ips)
-                    logger.info(fmt)
-            if mode == 'eval':
-                step_id = status['step_id']
-                if step_id % 100 == 0:
-                    logger.info("Eval iter: {}".format(step_id))
-
-    def on_epoch_end(self, status):
-        if dist.get_world_size() < 2 or dist.get_rank() == 0:
-            mode = status['mode']
-            if mode == 'eval':
-                sample_num = status['sample_num']
-                cost_time = status['cost_time']
-                logger.info('Total sample number: {}, averge FPS: {}'.format(
-                    sample_num, sample_num / cost_time))
-
-
-class Checkpointer(Callback):
-    def __init__(self, model):
-        super(Checkpointer, self).__init__(model)
-        cfg = self.model.cfg
-        self.best_ap = 0.
-        self.save_dir = os.path.join(self.model.cfg.save_dir,
-                                     self.model.cfg.filename)
-        if hasattr(self.model.model, 'student_model'):
-            self.weight = self.model.model.student_model
-        else:
-            self.weight = self.model.model
-
-    def on_epoch_end(self, status):
-        # Checkpointer only performed during training
-        mode = status['mode']
-        epoch_id = status['epoch_id']
-        weight = None
-        save_name = None
-        if dist.get_world_size() < 2 or dist.get_rank() == 0:
-            if mode == 'train':
-                end_epoch = self.model.cfg.epoch
-                if (
-                        epoch_id + 1
-                ) % self.model.cfg.snapshot_epoch == 0 or epoch_id == end_epoch - 1:
-                    save_name = str(
-                        epoch_id
-                    ) if epoch_id != end_epoch - 1 else "model_final"
-                    weight = self.weight
-            elif mode == 'eval':
-                if 'save_best_model' in status and status['save_best_model']:
-                    for metric in self.model._metrics:
-                        map_res = metric.get_results()
-                        if 'bbox' in map_res:
-                            key = 'bbox'
-                        elif 'keypoint' in map_res:
-                            key = 'keypoint'
-                        else:
-                            key = 'mask'
-                        if key not in map_res:
-                            logger.warning("Evaluation results empty, this may be due to " \
-                                        "training iterations being too few or not " \
-                                        "loading the correct weights.")
-                            return
-                        if map_res[key][0] > self.best_ap:
-                            self.best_ap = map_res[key][0]
-                            save_name = 'best_model'
-                            weight = self.weight
-                        logger.info("Best test {} ap is {:0.3f}.".format(
-                            key, self.best_ap))
-            if weight:
-                save_model(weight, self.model.optimizer, self.save_dir,
-                           save_name, epoch_id + 1)
-
-
-class WiferFaceEval(Callback):
-    def __init__(self, model):
-        super(WiferFaceEval, self).__init__(model)
-
-    def on_epoch_begin(self, status):
-        assert self.model.mode == 'eval', \
-            "WiferFaceEval can only be set during evaluation"
-        for metric in self.model._metrics:
-            metric.update(self.model.model)
-        sys.exit()
-
-
-class VisualDLWriter(Callback):
-    """
-    Use VisualDL to log data or image
-    """
-
-    def __init__(self, model):
-        super(VisualDLWriter, self).__init__(model)
-
-        assert six.PY3, "VisualDL requires Python >= 3.5"
-        try:
-            from visualdl import LogWriter
-        except Exception as e:
-            logger.error('visualdl not found, plaese install visualdl. '
-                         'for example: `pip install visualdl`.')
-            raise e
-        self.vdl_writer = LogWriter(
-            model.cfg.get('vdl_log_dir', 'vdl_log_dir/scalar'))
-        self.vdl_loss_step = 0
-        self.vdl_mAP_step = 0
-        self.vdl_image_step = 0
-        self.vdl_image_frame = 0
-
-    def on_step_end(self, status):
-        mode = status['mode']
-        if dist.get_world_size() < 2 or dist.get_rank() == 0:
-            if mode == 'train':
-                training_staus = status['training_staus']
-                for loss_name, loss_value in training_staus.get().items():
-                    self.vdl_writer.add_scalar(loss_name, loss_value,
-                                               self.vdl_loss_step)
-                    self.vdl_loss_step += 1
-            elif mode == 'test':
-                ori_image = status['original_image']
-                result_image = status['result_image']
-                self.vdl_writer.add_image(
-                    "original/frame_{}".format(self.vdl_image_frame),
-                    ori_image, self.vdl_image_step)
-                self.vdl_writer.add_image(
-                    "result/frame_{}".format(self.vdl_image_frame),
-                    result_image, self.vdl_image_step)
-                self.vdl_image_step += 1
-                # each frame can display ten pictures at most.
-                if self.vdl_image_step % 10 == 0:
-                    self.vdl_image_step = 0
-                    self.vdl_image_frame += 1
-
-    def on_epoch_end(self, status):
-        mode = status['mode']
-        if dist.get_world_size() < 2 or dist.get_rank() == 0:
-            if mode == 'eval':
-                for metric in self.model._metrics:
-                    for key, map_value in metric.get_results().items():
-                        self.vdl_writer.add_scalar("{}-mAP".format(key),
-                                                   map_value[0],
-                                                   self.vdl_mAP_step)
-                self.vdl_mAP_step += 1
-
-
-class SniperProposalsGenerator(Callback):
-    def __init__(self, model):
-        super(SniperProposalsGenerator, self).__init__(model)
-        ori_dataset = self.model.dataset
-        self.dataset = self._create_new_dataset(ori_dataset)
-        self.loader = self.model.loader
-        self.cfg = self.model.cfg
-        self.infer_model = self.model.model
-
-    def _create_new_dataset(self, ori_dataset):
-        dataset = copy.deepcopy(ori_dataset)
-        # init anno_cropper
-        dataset.init_anno_cropper()
-        # generate infer roidbs
-        ori_roidbs = dataset.get_ori_roidbs()
-        roidbs = dataset.anno_cropper.crop_infer_anno_records(ori_roidbs)
-        # set new roidbs
-        dataset.set_roidbs(roidbs)
-
-        return dataset
-
-    def _eval_with_loader(self, loader):
-        results = []
-        with paddle.no_grad():
-            self.infer_model.eval()
-            for step_id, data in enumerate(loader):
-                outs = self.infer_model(data)
-                for key in ['im_shape', 'scale_factor', 'im_id']:
-                    outs[key] = data[key]
-                for key, value in outs.items():
-                    if hasattr(value, 'numpy'):
-                        outs[key] = value.numpy()
-
-                results.append(outs)
-
-        return results
-
-    def on_train_end(self, status):
-        self.loader.dataset = self.dataset
-        results = self._eval_with_loader(self.loader)
-        results = self.dataset.anno_cropper.aggregate_chips_detections(results)
-        # sniper
-        proposals = []
-        clsid2catid = {v: k for k, v in self.dataset.catid2clsid.items()}
-        for outs in results:
-            batch_res = get_infer_results(outs, clsid2catid)
-            start = 0
-            for i, im_id in enumerate(outs['im_id']):
-                bbox_num = outs['bbox_num']
-                end = start + bbox_num[i]
-                bbox_res = batch_res['bbox'][start:end] \
-                    if 'bbox' in batch_res else None
-                if bbox_res:
-                    proposals += bbox_res
-        logger.info("save proposals in {}".format(self.cfg.proposals_path))
-        with open(self.cfg.proposals_path, 'w') as f:
-            json.dump(proposals, f)

paddlex/ppdet/engine/env.py

@@ -1,50 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import random
-import numpy as np
-
-import paddle
-from paddle.distributed import fleet
-
-__all__ = ['init_parallel_env', 'set_random_seed', 'init_fleet_env']
-
-
-def init_fleet_env(find_unused_parameters=False):
-    strategy = fleet.DistributedStrategy()
-    strategy.find_unused_parameters = find_unused_parameters
-    fleet.init(is_collective=True, strategy=strategy)
-
-
-def init_parallel_env():
-    env = os.environ
-    dist = 'PADDLE_TRAINER_ID' in env and 'PADDLE_TRAINERS_NUM' in env
-    if dist:
-        trainer_id = int(env['PADDLE_TRAINER_ID'])
-        local_seed = (99 + trainer_id)
-        random.seed(local_seed)
-        np.random.seed(local_seed)
-
-    paddle.distributed.init_parallel_env()
-
-
-def set_random_seed(seed):
-    paddle.seed(seed)
-    random.seed(seed)
-    np.random.seed(seed)

paddlex/ppdet/engine/export_utils.py

@@ -1,177 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import os
-import yaml
-from collections import OrderedDict
-
-import paddle
-from paddlex.ppdet.data.source.category import get_categories
-
-from paddlex.ppdet.utils.logger import setup_logger
-logger = setup_logger('ppdet.engine')
-
-# Global dictionary
-TRT_MIN_SUBGRAPH = {
-    'YOLO': 3,
-    'SSD': 60,
-    'RCNN': 40,
-    'RetinaNet': 40,
-    'S2ANet': 80,
-    'EfficientDet': 40,
-    'Face': 3,
-    'TTFNet': 60,
-    'FCOS': 16,
-    'SOLOv2': 60,
-    'HigherHRNet': 3,
-    'HRNet': 3,
-    'DeepSORT': 3,
-    'JDE': 10,
-    'FairMOT': 5,
-    'GFL': 16,
-    'PicoDet': 3,
-    'CenterNet': 5,
-    'TOOD': 5,
-}
-
-KEYPOINT_ARCH = ['HigherHRNet', 'TopDownHRNet']
-MOT_ARCH = ['DeepSORT', 'JDE', 'FairMOT']
-
-
-def _prune_input_spec(input_spec, program, targets):
-    # try to prune static program to figure out pruned input spec
-    # so we perform following operations in static mode
-    paddle.enable_static()
-    pruned_input_spec = [{}]
-    program = program.clone()
-    program = program._prune(targets=targets)
-    global_block = program.global_block()
-    for name, spec in input_spec[0].items():
-        try:
-            v = global_block.var(name)
-            pruned_input_spec[0][name] = spec
-        except Exception:
-            pass
-    paddle.disable_static()
-    return pruned_input_spec
-
-
-def _parse_reader(reader_cfg, dataset_cfg, metric, arch, image_shape):
-    preprocess_list = []
-
-    anno_file = dataset_cfg.get_anno()
-
-    clsid2catid, catid2name = get_categories(metric, anno_file, arch)
-
-    label_list = [str(cat) for cat in catid2name.values()]
-
-    fuse_normalize = reader_cfg.get('fuse_normalize', False)
-    sample_transforms = reader_cfg['sample_transforms']
-    for st in sample_transforms[1:]:
-        for key, value in st.items():
-            p = {'type': key}
-            if key == 'Resize':
-                if int(image_shape[1]) != -1:
-                    value['target_size'] = image_shape[1:]
-            if fuse_normalize and key == 'NormalizeImage':
-                continue
-            p.update(value)
-            preprocess_list.append(p)
-    batch_transforms = reader_cfg.get('batch_transforms', None)
-    if batch_transforms:
-        for bt in batch_transforms:
-            for key, value in bt.items():
-                # for deploy/infer, use PadStride(stride) instead PadBatch(pad_to_stride)
-                if key == 'PadBatch':
-                    preprocess_list.append({
-                        'type': 'PadStride',
-                        'stride': value['pad_to_stride']
-                    })
-                    break
-
-    return preprocess_list, label_list
-
-
-def _parse_tracker(tracker_cfg):
-    tracker_params = {}
-    for k, v in tracker_cfg.items():
-        tracker_params.update({k: v})
-    return tracker_params
-
-
-def _dump_infer_config(config, path, image_shape, model):
-    arch_state = False
-    from paddlex.ppdet.core.config.yaml_helpers import setup_orderdict
-    setup_orderdict()
-    use_dynamic_shape = True if image_shape[2] == -1 else False
-    infer_cfg = OrderedDict({
-        'mode': 'fluid',
-        'draw_threshold': 0.5,
-        'metric': config['metric'],
-        'use_dynamic_shape': use_dynamic_shape
-    })
-    infer_arch = config['architecture']
-
-    if infer_arch in MOT_ARCH:
-        if infer_arch == 'DeepSORT':
-            tracker_cfg = config['DeepSORTTracker']
-        else:
-            tracker_cfg = config['JDETracker']
-        infer_cfg['tracker'] = _parse_tracker(tracker_cfg)
-
-    for arch, min_subgraph_size in TRT_MIN_SUBGRAPH.items():
-        if arch in infer_arch:
-            infer_cfg['arch'] = arch
-            infer_cfg['min_subgraph_size'] = min_subgraph_size
-            arch_state = True
-            break
-    if not arch_state:
-        logger.error(
-            'Architecture: {} is not supported for exporting model now.\n'.
-            format(infer_arch) +
-            'Please set TRT_MIN_SUBGRAPH in ppdet/engine/export_utils.py')
-        os._exit(0)
-    if 'mask_head' in config[config['architecture']] and config[config[
-            'architecture']]['mask_head']:
-        infer_cfg['mask'] = True
-    label_arch = 'detection_arch'
-    if infer_arch in KEYPOINT_ARCH:
-        label_arch = 'keypoint_arch'
-
-    if infer_arch in MOT_ARCH:
-        label_arch = 'mot_arch'
-        reader_cfg = config['TestMOTReader']
-        dataset_cfg = config['TestMOTDataset']
-    else:
-        reader_cfg = config['TestReader']
-        dataset_cfg = config['TestDataset']
-
-    infer_cfg['Preprocess'], infer_cfg['label_list'] = _parse_reader(
-        reader_cfg, dataset_cfg, config['metric'], label_arch, image_shape[1:])
-
-    if infer_arch == 'PicoDet':
-        infer_cfg['NMS'] = config['PicoHead']['nms']
-        # In order to speed up the prediction, the threshold of nms
-        # is adjusted here, which can be changed in infer_cfg.yml
-        config['PicoHead']['nms']["score_threshold"] = 0.3
-        config['PicoHead']['nms']["nms_threshold"] = 0.5
-        infer_cfg['fpn_stride'] = config['PicoHead']['fpn_stride']
-
-    yaml.dump(infer_cfg, open(path, 'w'))
-    logger.info("Export inference config file to {}".format(
-        os.path.join(path)))