paddlex 2.1.0__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex_restful/restful/project/train/params_v2.py

@@ -1,326 +0,0 @@
-# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-def get_base_params(model_type, per_gpu_memory, num_train_samples, num_gpu,
-                    gpu_list, num_classes):
-    params = dict()
-    params["model"] = model_type
-    params["cpu_num"] = 1
-    if num_gpu == 0:
-        batch_size = 4
-        params['cuda_visible_devices'] = ''
-    else:
-        params['cuda_visible_devices'] = str(gpu_list).strip("[]")
-        if model_type.startswith('MobileNet'):
-            batch_size = (per_gpu_memory - 513) // 57 * num_gpu
-            batch_size = min(batch_size, num_gpu * 125)
-        elif model_type.startswith('DenseNet') or model_type.startswith('ResNet') \
-                or model_type.startswith('Xception') or model_type.startswith('DarkNet') \
-                or model_type.startswith('ShuffleNet'):
-            batch_size = (per_gpu_memory - 739) // 211 * num_gpu
-            batch_size = min(batch_size, num_gpu * 16)
-        elif model_type.startswith('YOLOv3'):
-            batch_size = (per_gpu_memory - 1555) // 943 * num_gpu
-            batch_size = min(batch_size, num_gpu * 8)
-        elif model_type.startswith('PPYOLOTiny'):
-            batch_size = (per_gpu_memory - 579) // 1025 * num_gpu
-            batch_size = min(batch_size, num_gpu * 16)
-        elif model_type.startswith('PPYOLO'):
-            batch_size = (per_gpu_memory - 1691) // 1025 * num_gpu
-            batch_size = min(batch_size, num_gpu * 4)
-        elif model_type.startswith('FasterRCNN'):
-            batch_size = (per_gpu_memory - 1755) // 915 * num_gpu
-            batch_size = min(batch_size, num_gpu * 2)
-        elif model_type.startswith('MaskRCNN'):
-            batch_size = (per_gpu_memory - 2702) // 1188 * num_gpu
-            batch_size = min(batch_size, num_gpu * 2)
-        elif model_type.startswith('DeepLab'):
-            batch_size = (per_gpu_memory - 1469) // 1605 * num_gpu
-            batch_size = min(batch_size, num_gpu * 4)
-        elif model_type.startswith('BiSeNetV2'):
-            batch_size = (per_gpu_memory - 591) // 1605 * num_gpu
-            batch_size = min(batch_size, num_gpu * 4)
-        elif model_type.startswith('UNet'):
-            batch_size = (per_gpu_memory - 1275) // 1256 * num_gpu
-            batch_size = min(batch_size, num_gpu * 4)
-        elif model_type.startswith('HRNet_W18'):
-            batch_size = (per_gpu_memory - 800) // 682 * num_gpu
-            batch_size = min(batch_size, num_gpu * 4)
-        elif model_type.startswith('FastSCNN'):
-            batch_size = (per_gpu_memory - 636) // 144 * num_gpu
-            batch_size = min(batch_size, num_gpu * 4)
-    if batch_size > num_train_samples // 2:
-        batch_size = num_train_samples // 2
-    if batch_size < 1:
-        batch_size = 1
-
-    brightness_range = 0.5
-    contrast_range = 0.5
-    saturation_range = 0.5
-    saturation = False
-    hue = False
-    if model_type.startswith('DenseNet') or model_type.startswith('ResNet') \
-        or model_type.startswith('Xception') or model_type.startswith('DarkNet') \
-        or model_type.startswith('ShuffleNet') or model_type.startswith('MobileNet'):
-        if model_type.startswith('MobileNet'):
-            lr = (batch_size / 500.0) * 0.1
-        else:
-            lr = (batch_size / 256.0) * 0.1
-        shape = [224, 224]
-        save_interval_epochs = 5
-        num_epochs = 10
-        lr_decay_epochs = [4, 6, 8]
-        image_mean = [0.485, 0.456, 0.406]
-        image_std = [0.229, 0.224, 0.225]
-        brightness_range = 0.9
-        contrast_range = 0.9
-        saturation_range = 0.9
-        brightness = True
-        contrast = True
-    elif model_type.startswith('YOLOv3'):
-        shape = [608, 608]
-        save_interval_epochs = 30
-        num_epochs = 270
-        lr_decay_epochs = [216, 243]
-        lr = .005 / 12 * batch_size / 8
-        image_mean = [0.485, 0.456, 0.406]
-        image_std = [0.229, 0.224, 0.225]
-        brightness = True
-        contrast = True
-        saturation = True
-        hue = True
-        num_steps_each_epoch = num_train_samples // batch_size
-        min_warmup_step = max(3 * num_steps_each_epoch, 50 * num_classes)
-        if num_gpu == 0:
-            num_gpu = 1
-        warmup_step = min(min_warmup_step, int(400 * num_classes / num_gpu))
-    elif model_type.startswith('PPYOLOTiny'):
-        shape = [320, 320]
-        save_interval_epochs = 30
-        num_epochs = 550
-        lr_decay_epochs = [130, 540]
-        lr = .005 * batch_size / 16
-        image_mean = [0.485, 0.456, 0.406]
-        image_std = [0.229, 0.224, 0.225]
-        brightness = True
-        contrast = True
-        saturation = True
-        hue = True
-        num_steps_each_epoch = num_train_samples // batch_size
-        min_warmup_step = max(3 * num_steps_each_epoch, 50 * num_classes)
-        if num_gpu == 0:
-            num_gpu = 1
-        warmup_step = min(min_warmup_step, int(400 * num_classes / num_gpu))
-    elif model_type.startswith('PPYOLOv2'):
-        shape = [608, 608]
-        save_interval_epochs = 30
-        num_epochs = 170
-        lr_decay_epochs = [105, 135, 150]
-        lr = .005 / 12 * batch_size / 8
-        image_mean = [0.485, 0.456, 0.406]
-        image_std = [0.229, 0.224, 0.225]
-        brightness = True
-        contrast = True
-        saturation = True
-        hue = True
-        num_steps_each_epoch = num_train_samples // batch_size
-        min_warmup_step = max(3 * num_steps_each_epoch, 50 * num_classes)
-        if num_gpu == 0:
-            num_gpu = 1
-        warmup_step = min(min_warmup_step, int(400 * num_classes / num_gpu))
-    elif model_type.startswith('PPYOLO'):
-        shape = [608, 608]
-        save_interval_epochs = 30
-        num_epochs = 200
-        lr_decay_epochs = [85, 135]
-        lr = .005 / 12 * batch_size / 8
-        image_mean = [0.485, 0.456, 0.406]
-        image_std = [0.229, 0.224, 0.225]
-        brightness = True
-        contrast = True
-        saturation = True
-        hue = True
-        num_steps_each_epoch = num_train_samples // batch_size
-        min_warmup_step = max(3 * num_steps_each_epoch, 50 * num_classes)
-        if num_gpu == 0:
-            num_gpu = 1
-        warmup_step = min(min_warmup_step, int(400 * num_classes / num_gpu))
-    elif model_type.startswith('FasterRCNN') or model_type.startswith(
-            'MaskRCNN'):
-        shape = [800, 1333]
-        save_interval_epochs = 1
-        num_epochs = 12
-        lr_decay_epochs = [8, 11]
-        image_mean = [0.485, 0.456, 0.406]
-        image_std = [0.229, 0.224, 0.225]
-        brightness = False
-        contrast = False
-        lr = 0.02 * batch_size / 16
-        num_steps_each_epoch = num_train_samples // batch_size
-        min_warmup_step = max(num_steps_each_epoch, 50)
-        if num_gpu == 0:
-            num_gpu = 1
-        warmup_step = min(min_warmup_step, int(4000 / num_gpu))
-    elif model_type.startswith('DeepLab') or model_type.startswith('UNet') \
-        or model_type.startswith('HRNet_W18') or model_type.startswith('FastSCNN') or model_type.startswith('BiSeNetV2'):
-        shape = [512, 512]
-        save_interval_epochs = 10
-        num_epochs = 100
-        lr_decay_epochs = [10, 20]
-        image_mean = [0.5, 0.5, 0.5]
-        image_std = [0.5, 0.5, 0.5]
-        brightness = False
-        contrast = False
-        lr = 0.01 * batch_size / 2
-        loss_type = [False, False]
-        if 'loss_type' in params:
-            params['loss_type'] = loss_type
-
-    params['batch_size'] = batch_size
-    params['learning_rate'] = lr
-    params["image_shape"] = shape
-    params['save_interval_epochs'] = save_interval_epochs
-    params['num_epochs'] = num_epochs
-    params['lr_decay_epochs'] = lr_decay_epochs
-    params['resume_checkpoint'] = None
-    params["sensitivities_path"] = None
-    params["image_mean"] = image_mean
-    params["image_std"] = image_std
-    params["horizontal_flip_prob"] = 0.5
-    params['brightness'] = brightness
-    params["brightness_range"] = brightness_range
-    params["brightness_prob"] = 0.5
-    params['contrast'] = contrast
-    params['contrast_range'] = contrast_range
-    params['contrast_prob'] = 0.5
-    params['saturation'] = saturation
-    params['saturation_range'] = saturation_range
-    params['saturation_prob'] = 0.5
-    params['hue'] = hue
-    params['hue_range'] = 18
-    params['hue_prob'] = 0.5
-    params['horizontal_flip'] = True
-
-    if model_type in [
-            'YOLOv3', 'PPYOLO', 'PPYOLOTiny', 'PPYOLOv2', 'FasterRCNN',
-            'MaskRCNN'
-    ]:
-        num_epochs = params['num_epochs']
-        lr_decay_epochs = params['lr_decay_epochs']
-        if warmup_step >= lr_decay_epochs[0] * num_steps_each_epoch:
-            for i in range(len(lr_decay_epochs)):
-                lr_decay_epochs[i] += warmup_step // num_steps_each_epoch
-            num_epochs += warmup_step // num_steps_each_epoch
-        params['num_epochs'] = num_epochs
-        params['lr_decay_epochs'] = lr_decay_epochs
-        params['warmup_steps'] = warmup_step
-
-    return params
-
-
-def get_classification_params(params):
-    params["pretrain_weights"] = 'IMAGENET'
-    params["lr_policy"] = "Piecewise"
-    params['vertical_flip_prob'] = 0.5
-    params['vertical_flip'] = True
-    params['rotate'] = True
-    params['rotate_prob'] = 0.5
-    params['rotate_range'] = 30
-    return params
-
-
-def get_detection_params(params):
-    params['with_fpn'] = True
-    params["pretrain_weights"] = 'IMAGENET'
-    if params['model'].startswith('YOLOv3') or params['model'].startswith(
-            'PPYOLO'):
-        if params['model'].startswith('YOLOv3'):
-            params['backbone'] = 'DarkNet53'
-        elif params['model'].startswith('PPYOLO'):
-            params['backbone'] = 'ResNet50_vd_ssld'
-        params['use_mixup'] = True
-        params['mixup_alpha'] = 1.5
-        params['mixup_beta'] = 1.5
-        params['expand_prob'] = 0.5
-        params['expand_image'] = True
-        params['crop_image'] = True
-        params['random_shape'] = True
-        params['random_shape_sizes'] = [
-            320, 352, 384, 416, 448, 480, 512, 544, 576, 608
-        ]
-    elif params['model'].startswith('FasterRCNN') or params[
-            'model'].startswith('MaskRCNN'):
-        params['backbone'] = 'ResNet50'
-
-    return params
-
-
-def get_segmentation_params(params):
-    if params['model'].startswith('DeepLab'):
-        params['backbone'] = 'Xception65'
-        params["pretrain_weights"] = 'IMAGENET'
-    elif params['model'].startswith('UNet'):
-        params["pretrain_weights"] = 'COCO'
-    elif params['model'].startswith('HRNet_W18') or params['model'].startswith(
-            'FastSCNN'):
-        params["pretrain_weights"] = 'CITYSCAPES'
-    params['loss_type'] = [False, False]
-    params['lr_policy'] = 'Polynomial'
-    params['optimizer'] = 'SGD'
-    params['blur'] = False
-    params['blur_prob'] = 0.1
-    params['rotate'] = False
-    params['scale_aspect'] = False
-    params['min_ratio'] = 0.5
-    params['aspect_ratio'] = 0.33
-    params['vertical_flip'] = False
-    params['vertical_flip_prob'] = 0.5
-    return params
-
-
-def get_params(data, project_type, num_train_samples, num_classes, num_gpu,
-               per_gpu_memory, gpu_list):
-    if project_type == "classification":
-        if 'model_type' in data:
-            model_type = data['model_type']
-        else:
-            model_type = "MobileNetV2"
-        params = get_base_params(model_type, per_gpu_memory, num_train_samples,
-                                 num_gpu, gpu_list, num_classes)
-        return get_classification_params(params)
-    if project_type == "detection":
-        if 'model_type' in data:
-            model_type = data['model_type']
-        else:
-            model_type = "YOLOv3"
-        params = get_base_params(model_type, per_gpu_memory, num_train_samples,
-                                 num_gpu, gpu_list, num_classes)
-        return get_detection_params(params)
-    if project_type == "instance_segmentation":
-        if 'model_type' in data:
-            model_type = data['model_type']
-        else:
-            model_type = "MaskRCNN"
-        params = get_base_params(model_type, per_gpu_memory, num_train_samples,
-                                 num_gpu, gpu_list, num_classes)
-        return get_detection_params(params)
-    if project_type == 'segmentation' or project_type == "remote_segmentation":
-        if 'model_type' in data:
-            model_type = data['model_type']
-        else:
-            model_type = "DeepLabv3+"
-        params = get_base_params(model_type, per_gpu_memory, num_train_samples,
-                                 num_gpu, gpu_list, num_classes)
-        return get_segmentation_params(params)

paddlex_restful/restful/project/train/segmentation.py

@@ -1,191 +0,0 @@
-# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os.path as osp
-from paddleslim import L1NormFilterPruner
-
-
-def build_transforms(params):
-    from paddlex import transforms as T
-    seg_list = []
-    seg_list.extend([
-        T.Resize(target_size=params.image_shape),
-        T.RandomBlur(prob=params.blur_prob)
-    ])
-    if params.scale_aspect:
-        seg_list.append(
-            T.RandomScaleAspect(
-                min_scale=params.min_ratio, aspect_ratio=params.aspect_ratio))
-    seg_list.extend([
-        T.RandomDistort(
-            brightness_range=params.brightness_range,
-            brightness_prob=params.brightness_prob,
-            contrast_range=params.contrast_range,
-            contrast_prob=params.contrast_prob,
-            saturation_range=params.saturation_range,
-            saturation_prob=params.saturation_prob,
-            hue_range=params.hue_range,
-            hue_prob=params.hue_prob),
-        T.RandomVerticalFlip(prob=params.vertical_flip_prob),
-        T.RandomHorizontalFlip(prob=params.horizontal_flip_prob), T.Normalize(
-            mean=params.image_mean, std=params.image_std)
-    ])
-
-    train_transforms = T.Compose(seg_list)
-    eval_transforms = T.Compose([
-        T.Resize(target_size=params.image_shape), T.Normalize(
-            mean=params.image_mean, std=params.image_std)
-    ])
-    return train_transforms, eval_transforms
-
-
-def build_datasets(dataset_path, train_transforms, eval_transforms):
-    import paddlex as pdx
-    train_file_list = osp.join(dataset_path, 'train_list.txt')
-    eval_file_list = osp.join(dataset_path, 'val_list.txt')
-    label_list = osp.join(dataset_path, 'labels.txt')
-    train_dataset = pdx.datasets.SegDataset(
-        data_dir=dataset_path,
-        file_list=train_file_list,
-        label_list=label_list,
-        transforms=train_transforms,
-        shuffle=True)
-    eval_dataset = pdx.datasets.SegDataset(
-        data_dir=dataset_path,
-        file_list=eval_file_list,
-        label_list=label_list,
-        transforms=eval_transforms)
-    return train_dataset, eval_dataset
-
-
-def build_optimizer(parameters, step_each_epoch, params):
-    import paddle
-    from paddle.regularizer import L2Decay
-    learning_rate = params.learning_rate
-    num_epochs = params.num_epochs
-    if params.lr_policy == 'Piecewise':
-        lr_decay_epochs = params.lr_decay_epochs
-        gamma = 0.1
-        boundaries = [step_each_epoch * e for e in lr_decay_epochs]
-        values = [
-            learning_rate * (gamma**i)
-            for i in range(len(lr_decay_epochs) + 1)
-        ]
-        decayed_lr = paddle.optimizer.lr.PiecewiseDecay(
-            boundaries=boundaries, values=values)
-    elif params.lr_policy == 'Polynomial':
-        decay_step = num_epochs * step_each_epoch
-        decayed_lr = paddle.optimizer.lr.PolynomialDecay(
-            learning_rate=learning_rate,
-            decay_steps=decay_step,
-            end_lr=0.0,
-            power=.9)
-    elif params.lr_policy == 'Cosine':
-        decayed_lr = paddle.optimizer.lr.CosineAnnealingDecay(
-            learning_rate=.001, T_max=step_each_epoch * num_epochs)
-    else:
-        raise Exception(
-            'lr_policy only support Polynomial or Piecewise, but you set {}'.
-            format(params.lr_policy))
-
-    if params.optimizer.lower() == 'sgd':
-        momentum = 0.9
-        regularize_coef = 1e-4
-        optimizer = paddle.optimizer.Momentum(
-            learning_rate=decayed_lr,
-            momentum=momentum,
-            weight_decay=L2Decay(regularize_coef),
-            parameters=parameters)
-    elif params.optimizer.lower() == 'adam':
-        momentum = 0.9
-        momentum2 = 0.999
-        regularize_coef = 1e-4
-        optimizer = paddle.optimizer.Adam(
-            learning_rate=decayed_lr,
-            beta1=momentum,
-            beta2=momentum2,
-            weight_decay=L2Decay(regularize_coef),
-            parameters=parameters)
-
-    return optimizer
-
-
-def train(task_path, dataset_path, params):
-    import paddlex as pdx
-    pdx.log_level = 3
-    train_transforms, eval_transforms = build_transforms(params)
-    train_dataset, eval_dataset = build_datasets(
-        dataset_path=dataset_path,
-        train_transforms=train_transforms,
-        eval_transforms=eval_transforms)
-
-    step_each_epoch = train_dataset.num_samples // params.batch_size
-    save_interval_epochs = params.save_interval_epochs
-    save_dir = osp.join(task_path, 'output')
-    pretrain_weights = params.pretrain_weights
-    if pretrain_weights is not None and osp.exists(pretrain_weights):
-        pretrain_weights = osp.join(pretrain_weights, 'model.pdparams')
-
-    segmenter = getattr(pdx.seg, 'HRNet'
-                        if params.model.startswith('HRNet') else params.model)
-    use_dice_loss, use_bce_loss = params.loss_type
-    if use_bce_loss and use_dice_loss:
-        use_mixed_loss = [('CrossEntropyLoss', 1), ('DiceLoss', 1)]
-    elif use_bce_loss:
-        use_mixed_loss = [('CrossEntropyLoss', 1)]
-    elif use_dice_loss:
-        use_mixed_loss = [('DiceLoss', 1)]
-    else:
-        use_mixed_loss = False
-
-    backbone = params.backbone
-    sensitivities_path = params.sensitivities_path
-    pruned_flops = params.pruned_flops
-    if params.model in ['UNet', 'HRNet_W18', 'FastSCNN', 'BiSeNetV2']:
-        model = segmenter(
-            num_classes=len(train_dataset.labels),
-            use_mixed_loss=use_mixed_loss)
-    elif params.model == 'DeepLabV3P':
-        model = segmenter(
-            num_classes=len(train_dataset.labels),
-            backbone=backbone,
-            use_mixed_loss=use_mixed_loss)
-
-    if sensitivities_path is not None:
-        # load weights
-        model.net_initialize(pretrain_weights=osp.join(pretrain_weights,
-                                                       'model.pdparams'))
-        pretrain_weights = None
-        # prune
-        dataset = eval_dataset or train_dataset
-        inputs = [1, 3] + list(dataset[0]['image'].shape[:2])
-        model.pruner = L1NormFilterPruner(
-            model.net, inputs=inputs, sen_file=sensitivities_path)
-        #model.pruner.sensitive_prune(pruned_flops=pruned_flops)
-        model.prune(pruned_flops=pruned_flops)
-
-    optimizer = build_optimizer(model.net.parameters(), step_each_epoch,
-                                params)
-    model.train(
-        num_epochs=params.num_epochs,
-        train_dataset=train_dataset,
-        train_batch_size=params.batch_size,
-        eval_dataset=eval_dataset,
-        save_interval_epochs=save_interval_epochs,
-        log_interval_steps=2,
-        save_dir=save_dir,
-        pretrain_weights=pretrain_weights,
-        optimizer=optimizer,
-        use_vdl=True,
-        resume_checkpoint=params.resume_checkpoint)