paddlex 2.1.0__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

PaddleClas/ppcls/data/dataloader/dali.py

@@ -1,319 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import division
-
-import copy
-import os
-
-import numpy as np
-import nvidia.dali.ops as ops
-import nvidia.dali.types as types
-import paddle
-from nvidia.dali import fn
-from nvidia.dali.pipeline import Pipeline
-from nvidia.dali.plugin.base_iterator import LastBatchPolicy
-from nvidia.dali.plugin.paddle import DALIGenericIterator
-
-
-class HybridTrainPipe(Pipeline):
-    def __init__(self,
-                 file_root,
-                 file_list,
-                 batch_size,
-                 resize_shorter,
-                 crop,
-                 min_area,
-                 lower,
-                 upper,
-                 interp,
-                 mean,
-                 std,
-                 device_id,
-                 shard_id=0,
-                 num_shards=1,
-                 random_shuffle=True,
-                 num_threads=4,
-                 seed=42,
-                 pad_output=False,
-                 output_dtype=types.FLOAT,
-                 dataset='Train'):
-        super(HybridTrainPipe, self).__init__(
-            batch_size, num_threads, device_id, seed=seed)
-        self.input = ops.readers.File(
-            file_root=file_root,
-            file_list=file_list,
-            shard_id=shard_id,
-            num_shards=num_shards,
-            random_shuffle=random_shuffle)
-        # set internal nvJPEG buffers size to handle full-sized ImageNet images
-        # without additional reallocations
-        device_memory_padding = 211025920
-        host_memory_padding = 140544512
-        self.decode = ops.decoders.ImageRandomCrop(
-            device='mixed',
-            output_type=types.DALIImageType.RGB,
-            device_memory_padding=device_memory_padding,
-            host_memory_padding=host_memory_padding,
-            random_aspect_ratio=[lower, upper],
-            random_area=[min_area, 1.0],
-            num_attempts=100)
-        self.res = ops.Resize(
-            device='gpu', resize_x=crop, resize_y=crop, interp_type=interp)
-        self.cmnp = ops.CropMirrorNormalize(
-            device="gpu",
-            dtype=output_dtype,
-            output_layout='CHW',
-            crop=(crop, crop),
-            mean=mean,
-            std=std,
-            pad_output=pad_output)
-        self.coin = ops.random.CoinFlip(probability=0.5)
-        self.to_int64 = ops.Cast(dtype=types.DALIDataType.INT64, device="gpu")
-
-    def define_graph(self):
-        rng = self.coin()
-        jpegs, labels = self.input(name="Reader")
-        images = self.decode(jpegs)
-        images = self.res(images)
-        output = self.cmnp(images.gpu(), mirror=rng)
-        return [output, self.to_int64(labels.gpu())]
-
-    def __len__(self):
-        return self.epoch_size("Reader")
-
-
-class HybridValPipe(Pipeline):
-    def __init__(self,
-                 file_root,
-                 file_list,
-                 batch_size,
-                 resize_shorter,
-                 crop,
-                 interp,
-                 mean,
-                 std,
-                 device_id,
-                 shard_id=0,
-                 num_shards=1,
-                 random_shuffle=False,
-                 num_threads=4,
-                 seed=42,
-                 pad_output=False,
-                 output_dtype=types.FLOAT):
-        super(HybridValPipe, self).__init__(
-            batch_size, num_threads, device_id, seed=seed)
-        self.input = ops.readers.File(
-            file_root=file_root,
-            file_list=file_list,
-            shard_id=shard_id,
-            num_shards=num_shards,
-            random_shuffle=random_shuffle)
-        self.decode = ops.decoders.Image(device="mixed")
-        self.res = ops.Resize(
-            device="gpu", resize_shorter=resize_shorter, interp_type=interp)
-        self.cmnp = ops.CropMirrorNormalize(
-            device="gpu",
-            dtype=output_dtype,
-            output_layout='CHW',
-            crop=(crop, crop),
-            mean=mean,
-            std=std,
-            pad_output=pad_output)
-        self.to_int64 = ops.Cast(dtype=types.DALIDataType.INT64, device="gpu")
-
-    def define_graph(self):
-        jpegs, labels = self.input(name="Reader")
-        images = self.decode(jpegs)
-        images = self.res(images)
-        output = self.cmnp(images)
-        return [output, self.to_int64(labels.gpu())]
-
-    def __len__(self):
-        return self.epoch_size("Reader")
-
-
-def dali_dataloader(config, mode, device, seed=None):
-    assert "gpu" in device, "gpu training is required for DALI"
-    device_id = int(device.split(':')[1])
-    config_dataloader = config[mode]
-    seed = 42 if seed is None else seed
-    ops = [
-        list(x.keys())[0]
-        for x in config_dataloader["dataset"]["transform_ops"]
-    ]
-    support_ops_train = [
-        "DecodeImage", "NormalizeImage", "RandFlipImage", "RandCropImage"
-    ]
-    support_ops_eval = [
-        "DecodeImage", "ResizeImage", "CropImage", "NormalizeImage"
-    ]
-
-    if mode.lower() == 'train':
-        assert set(ops) == set(
-            support_ops_train
-        ), "The supported trasform_ops for train_dataset in dali is : {}".format(
-            ",".join(support_ops_train))
-    else:
-        assert set(ops) == set(
-            support_ops_eval
-        ), "The supported trasform_ops for eval_dataset in dali is : {}".format(
-            ",".join(support_ops_eval))
-
-    normalize_ops = [
-        op for op in config_dataloader["dataset"]["transform_ops"]
-        if "NormalizeImage" in op
-    ][0]["NormalizeImage"]
-    channel_num = normalize_ops.get("channel_num", 3)
-    output_dtype = types.FLOAT16 if normalize_ops.get("output_fp16",
-                                                      False) else types.FLOAT
-
-    env = os.environ
-    #  assert float(env.get('FLAGS_fraction_of_gpu_memory_to_use', 0.92)) < 0.9, \
-    #      "Please leave enough GPU memory for DALI workspace, e.g., by setting" \
-    #      " `export FLAGS_fraction_of_gpu_memory_to_use=0.8`"
-
-    gpu_num = paddle.distributed.get_world_size()
-
-    batch_size = config_dataloader["sampler"]["batch_size"]
-
-    file_root = config_dataloader["dataset"]["image_root"]
-    file_list = config_dataloader["dataset"]["cls_label_path"]
-
-    interp = 1  # settings.interpolation or 1  # default to linear
-    interp_map = {
-        0: types.DALIInterpType.INTERP_NN,  # cv2.INTER_NEAREST
-        1: types.DALIInterpType.INTERP_LINEAR,  # cv2.INTER_LINEAR
-        2: types.DALIInterpType.INTERP_CUBIC,  # cv2.INTER_CUBIC
-        3: types.DALIInterpType.
-        INTERP_LANCZOS3,  # XXX use LANCZOS3 for cv2.INTER_LANCZOS4
-    }
-
-    assert interp in interp_map, "interpolation method not supported by DALI"
-    interp = interp_map[interp]
-    pad_output = channel_num == 4
-
-    transforms = {
-        k: v
-        for d in config_dataloader["dataset"]["transform_ops"]
-        for k, v in d.items()
-    }
-
-    scale = transforms["NormalizeImage"].get("scale", 1.0 / 255)
-    scale = eval(scale) if isinstance(scale, str) else scale
-    mean = transforms["NormalizeImage"].get("mean", [0.485, 0.456, 0.406])
-    std = transforms["NormalizeImage"].get("std", [0.229, 0.224, 0.225])
-    mean = [v / scale for v in mean]
-    std = [v / scale for v in std]
-
-    sampler_name = config_dataloader["sampler"].get("name",
-                                                    "DistributedBatchSampler")
-    assert sampler_name in ["DistributedBatchSampler", "BatchSampler"]
-
-    if mode.lower() == "train":
-        resize_shorter = 256
-        crop = transforms["RandCropImage"]["size"]
-        scale = transforms["RandCropImage"].get("scale", [0.08, 1.])
-        ratio = transforms["RandCropImage"].get("ratio", [3.0 / 4, 4.0 / 3])
-        min_area = scale[0]
-        lower = ratio[0]
-        upper = ratio[1]
-
-        if 'PADDLE_TRAINER_ID' in env and 'PADDLE_TRAINERS_NUM' in env:
-            shard_id = int(env['PADDLE_TRAINER_ID'])
-            num_shards = int(env['PADDLE_TRAINERS_NUM'])
-            device_id = int(env['FLAGS_selected_gpus'])
-            pipe = HybridTrainPipe(
-                file_root,
-                file_list,
-                batch_size,
-                resize_shorter,
-                crop,
-                min_area,
-                lower,
-                upper,
-                interp,
-                mean,
-                std,
-                device_id,
-                shard_id,
-                num_shards,
-                seed=seed + shard_id,
-                pad_output=pad_output,
-                output_dtype=output_dtype)
-            pipe.build()
-            pipelines = [pipe]
-            #  sample_per_shard = len(pipe) // num_shards
-        else:
-            pipe = HybridTrainPipe(
-                file_root,
-                file_list,
-                batch_size,
-                resize_shorter,
-                crop,
-                min_area,
-                lower,
-                upper,
-                interp,
-                mean,
-                std,
-                device_id=device_id,
-                shard_id=0,
-                num_shards=1,
-                seed=seed,
-                pad_output=pad_output,
-                output_dtype=output_dtype)
-            pipe.build()
-            pipelines = [pipe]
-            #  sample_per_shard = len(pipelines[0])
-        return DALIGenericIterator(
-            pipelines, ['data', 'label'], reader_name='Reader')
-    else:
-        resize_shorter = transforms["ResizeImage"].get("resize_short", 256)
-        crop = transforms["CropImage"]["size"]
-        if 'PADDLE_TRAINER_ID' in env and 'PADDLE_TRAINERS_NUM' in env and sampler_name == "DistributedBatchSampler":
-            shard_id = int(env['PADDLE_TRAINER_ID'])
-            num_shards = int(env['PADDLE_TRAINERS_NUM'])
-            device_id = int(env['FLAGS_selected_gpus'])
-
-            pipe = HybridValPipe(
-                file_root,
-                file_list,
-                batch_size,
-                resize_shorter,
-                crop,
-                interp,
-                mean,
-                std,
-                device_id=device_id,
-                shard_id=shard_id,
-                num_shards=num_shards,
-                pad_output=pad_output,
-                output_dtype=output_dtype)
-        else:
-            pipe = HybridValPipe(
-                file_root,
-                file_list,
-                batch_size,
-                resize_shorter,
-                crop,
-                interp,
-                mean,
-                std,
-                device_id=device_id,
-                pad_output=pad_output,
-                output_dtype=output_dtype)
-        pipe.build()
-        return DALIGenericIterator(
-            [pipe], ['data', 'label'], reader_name="Reader")

PaddleClas/ppcls/data/dataloader/icartoon_dataset.py

@@ -1,36 +0,0 @@
-#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import numpy as np
-import os
-
-from .common_dataset import CommonDataset
-
-
-class ICartoonDataset(CommonDataset):
-    def _load_anno(self, seed=None):
-        assert os.path.exists(self._cls_path)
-        assert os.path.exists(self._img_root)
-        self.images = []
-        self.labels = []
-
-        with open(self._cls_path) as fd:
-            lines = fd.readlines()
-            for l in lines:
-                l = l.strip().split("\t")
-                self.images.append(os.path.join(self._img_root, l[0]))
-                self.labels.append(np.int64(l[1]))
-                assert os.path.exists(self.images[-1])

PaddleClas/ppcls/data/dataloader/imagenet_dataset.py

@@ -1,38 +0,0 @@
-#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import numpy as np
-import os
-
-from .common_dataset import CommonDataset
-
-
-class ImageNetDataset(CommonDataset):
-    def _load_anno(self, seed=None):
-        assert os.path.exists(self._cls_path)
-        assert os.path.exists(self._img_root)
-        self.images = []
-        self.labels = []
-
-        with open(self._cls_path) as fd:
-            lines = fd.readlines()
-            if seed is not None:
-                np.random.RandomState(seed).shuffle(lines)
-            for l in lines:
-                l = l.strip().split(" ")
-                self.images.append(os.path.join(self._img_root, l[0]))
-                self.labels.append(np.int64(l[1]))
-                assert os.path.exists(self.images[-1])

PaddleClas/ppcls/data/dataloader/logo_dataset.py

@@ -1,46 +0,0 @@
-#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import io
-import tarfile
-import numpy as np
-from PIL import Image  #all use default backend
-
-import paddle
-from paddle.io import Dataset
-import pickle
-import os
-import cv2
-import random
-
-from .common_dataset import CommonDataset
-
-
-class LogoDataset(CommonDataset):
-    def _load_anno(self):
-        assert os.path.exists(self._cls_path)
-        assert os.path.exists(self._img_root)
-        self.images = []
-        self.labels = []
-        with open(self._cls_path) as fd:
-            lines = fd.readlines()
-            for l in lines:
-                l = l.strip().split("\t")
-                if l[0] == 'image_id':
-                    continue
-                self.images.append(os.path.join(self._img_root, l[3]))
-                self.labels.append(np.int64(l[1]) - 1)
-                assert os.path.exists(self.images[-1])

PaddleClas/ppcls/data/dataloader/mix_dataset.py

@@ -1,49 +0,0 @@
-#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import numpy as np
-import os
-
-from paddle.io import Dataset
-from .. import dataloader
-
-
-class MixDataset(Dataset):
-    def __init__(self, datasets_config):
-        super().__init__()
-        self.dataset_list = []
-        start_idx = 0
-        end_idx = 0
-        for config_i in datasets_config:
-            dataset_name = config_i.pop('name')
-            dataset = getattr(dataloader, dataset_name)(**config_i)
-            end_idx += len(dataset)
-            self.dataset_list.append([end_idx, start_idx, dataset])
-            start_idx = end_idx
-
-        self.length = end_idx
-
-    def __getitem__(self, idx):
-        for dataset_i in self.dataset_list:
-            if dataset_i[0] > idx:
-                dataset_i_idx = idx - dataset_i[1]
-                return dataset_i[2][dataset_i_idx]
-
-    def __len__(self):
-        return self.length
-
-    def get_dataset_list(self):
-        return self.dataset_list

PaddleClas/ppcls/data/dataloader/mix_sampler.py

@@ -1,79 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-
-from paddle.io import DistributedBatchSampler, Sampler
-
-from ppcls.utils import logger
-from ppcls.data.dataloader.mix_dataset import MixDataset
-from ppcls.data import dataloader
-
-
-class MixSampler(DistributedBatchSampler):
-    def __init__(self, dataset, batch_size, sample_configs, iter_per_epoch):
-        super().__init__(dataset, batch_size)
-        assert isinstance(dataset,
-                          MixDataset), "MixSampler only support MixDataset"
-        self.sampler_list = []
-        self.batch_size = batch_size
-        self.start_list = []
-        self.length = iter_per_epoch
-        dataset_list = dataset.get_dataset_list()
-        batch_size_left = self.batch_size
-        self.iter_list = []
-        for i, config_i in enumerate(sample_configs):
-            self.start_list.append(dataset_list[i][1])
-            sample_method = config_i.pop("name")
-            ratio_i = config_i.pop("ratio")
-            if i < len(sample_configs) - 1:
-                batch_size_i = int(self.batch_size * ratio_i)
-                batch_size_left -= batch_size_i
-            else:
-                batch_size_i = batch_size_left
-            assert batch_size_i <= len(dataset_list[i][2])
-            config_i["batch_size"] = batch_size_i
-            if sample_method == "DistributedBatchSampler":
-                sampler_i = DistributedBatchSampler(dataset_list[i][2],
-                                                    **config_i)
-            else:
-                sampler_i = getattr(dataloader, sample_method)(
-                    dataset_list[i][2], **config_i)
-            self.sampler_list.append(sampler_i)
-            self.iter_list.append(iter(sampler_i))
-            self.length += len(dataset_list[i][2]) * ratio_i
-            self.iter_counter = 0
-
-    def __iter__(self):
-        while self.iter_counter < self.length:
-            batch = []
-            for i, iter_i in enumerate(self.iter_list):
-                batch_i = next(iter_i, None)
-                if batch_i is None:
-                    iter_i = iter(self.sampler_list[i])
-                    self.iter_list[i] = iter_i
-                    batch_i = next(iter_i, None)
-                    assert batch_i is not None, "dataset {} return None".format(
-                        i)
-                batch += [idx + self.start_list[i] for idx in batch_i]
-            if len(batch) == self.batch_size:
-                self.iter_counter += 1
-                yield batch
-            else:
-                logger.info("Some dataset reaches end")
-        self.iter_counter = 0
-
-    def __len__(self):
-        return self.length

PaddleClas/ppcls/data/dataloader/multilabel_dataset.py

@@ -1,59 +0,0 @@
-#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import numpy as np
-import os
-import cv2
-
-from ppcls.data.preprocess import transform
-from ppcls.utils import logger
-
-from .common_dataset import CommonDataset
-
-
-class MultiLabelDataset(CommonDataset):
-    def _load_anno(self):
-        assert os.path.exists(self._cls_path)
-        assert os.path.exists(self._img_root)
-        self.images = []
-        self.labels = []
-        with open(self._cls_path) as fd:
-            lines = fd.readlines()
-            for l in lines:
-                l = l.strip().split("\t")
-                self.images.append(os.path.join(self._img_root, l[0]))
-
-                labels = l[1].split(',')
-                labels = [np.int64(i) for i in labels]
-
-                self.labels.append(labels)
-                assert os.path.exists(self.images[-1])
-
-    def __getitem__(self, idx):
-        try:
-            with open(self.images[idx], 'rb') as f:
-                img = f.read()
-            if self._transform_ops:
-                img = transform(img, self._transform_ops)
-            img = img.transpose((2, 0, 1))
-            label = np.array(self.labels[idx]).astype("float32")
-            return (img, label)
-
-        except Exception as ex:
-            logger.error("Exception occured when parse line: {} with msg: {}".
-                         format(self.images[idx], ex))
-            rnd_idx = np.random.randint(self.__len__())
-            return self.__getitem__(rnd_idx)