paddlex 2.1.0__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/ppcls/data/dataloader/pk_sampler.py

@@ -1,105 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from collections import defaultdict
-import numpy as np
-import random
-from paddle.io import DistributedBatchSampler
-
-from paddlex.ppcls.utils import logger
-
-
-class PKSampler(DistributedBatchSampler):
-    """
-    First, randomly sample P identities.
-    Then for each identity randomly sample K instances.
-    Therefore batch size is P*K, and the sampler called PKSampler.
-    Args:
-        dataset (paddle.io.Dataset): list of (img_path, pid, cam_id).
-        sample_per_id(int): number of instances per identity in a batch.
-        batch_size (int): number of examples in a batch.
-        shuffle(bool): whether to shuffle indices order before generating
-            batch indices. Default False.
-    """
-
-    def __init__(self,
-                 dataset,
-                 batch_size,
-                 sample_per_id,
-                 shuffle=True,
-                 drop_last=True,
-                 sample_method="sample_avg_prob"):
-        super().__init__(
-            dataset, batch_size, shuffle=shuffle, drop_last=drop_last)
-        assert batch_size % sample_per_id == 0, \
-            "PKSampler configs error, Sample_per_id must be a divisor of batch_size."
-        assert hasattr(self.dataset,
-                       "labels"), "Dataset must have labels attribute."
-        self.sample_per_label = sample_per_id
-        self.label_dict = defaultdict(list)
-        self.sample_method = sample_method
-        for idx, label in enumerate(self.dataset.labels):
-            self.label_dict[label].append(idx)
-        self.label_list = list(self.label_dict)
-        assert len(self.label_list) * self.sample_per_label > self.batch_size, \
-            "batch size should be smaller than "
-        if self.sample_method == "id_avg_prob":
-            self.prob_list = np.array([1 / len(self.label_list)] *
-                                      len(self.label_list))
-        elif self.sample_method == "sample_avg_prob":
-            counter = []
-            for label_i in self.label_list:
-                counter.append(len(self.label_dict[label_i]))
-            self.prob_list = np.array(counter) / sum(counter)
-        else:
-            logger.error(
-                "PKSampler only support id_avg_prob and sample_avg_prob sample method, "
-                "but receive {}.".format(self.sample_method))
-        diff = np.abs(sum(self.prob_list) - 1)
-        if diff > 0.00000001:
-            self.prob_list[-1] = 1 - sum(self.prob_list[:-1])
-            if self.prob_list[-1] > 1 or self.prob_list[-1] < 0:
-                logger.error("PKSampler prob list error")
-            else:
-                logger.info(
-                    "PKSampler: sum of prob list not equal to 1, diff is {}, change the last prob".
-                    format(diff))
-
-    def __iter__(self):
-        label_per_batch = self.batch_size // self.sample_per_label
-        for _ in range(len(self)):
-            batch_index = []
-            batch_label_list = np.random.choice(
-                self.label_list,
-                size=label_per_batch,
-                replace=False,
-                p=self.prob_list)
-            for label_i in batch_label_list:
-                label_i_indexes = self.label_dict[label_i]
-                if self.sample_per_label <= len(label_i_indexes):
-                    batch_index.extend(
-                        np.random.choice(
-                            label_i_indexes,
-                            size=self.sample_per_label,
-                            replace=False))
-                else:
-                    batch_index.extend(
-                        np.random.choice(
-                            label_i_indexes,
-                            size=self.sample_per_label,
-                            replace=True))
-            if not self.drop_last or len(batch_index) == self.batch_size:
-                yield batch_index

paddlex/ppcls/data/dataloader/vehicle_dataset.py

@@ -1,138 +0,0 @@
-#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import numpy as np
-import paddle
-from paddle.io import Dataset
-import os
-import cv2
-
-from paddlex.ppcls.data import preprocess
-from paddlex.ppcls.data.preprocess import transform
-from paddlex.ppcls.utils import logger
-from .common_dataset import create_operators
-
-
-class CompCars(Dataset):
-    def __init__(self,
-                 image_root,
-                 cls_label_path,
-                 label_root=None,
-                 transform_ops=None,
-                 bbox_crop=False):
-        self._img_root = image_root
-        self._cls_path = cls_label_path
-        self._label_root = label_root
-        if transform_ops:
-            self._transform_ops = create_operators(transform_ops)
-        self._bbox_crop = bbox_crop
-        self._dtype = paddle.get_default_dtype()
-        self._load_anno()
-
-    def _load_anno(self):
-        assert os.path.exists(self._cls_path)
-        assert os.path.exists(self._img_root)
-        if self._bbox_crop:
-            assert os.path.exists(self._label_root)
-        self.images = []
-        self.labels = []
-        self.bboxes = []
-        with open(self._cls_path) as fd:
-            lines = fd.readlines()
-            for l in lines:
-                l = l.strip().split()
-                if not self._bbox_crop:
-                    self.images.append(os.path.join(self._img_root, l[0]))
-                    self.labels.append(int(l[1]))
-                else:
-                    label_path = os.path.join(self._label_root,
-                                              l[0].split('.')[0] + '.txt')
-                    assert os.path.exists(label_path)
-                    with open(label_path) as f:
-                        bbox = f.readlines()[-1].strip().split()
-                    bbox = [int(x) for x in bbox]
-                    self.images.append(os.path.join(self._img_root, l[0]))
-                    self.labels.append(int(l[1]))
-                    self.bboxes.append(bbox)
-                    assert os.path.exists(self.images[-1])
-
-    def __getitem__(self, idx):
-        img = cv2.imread(self.images[idx])
-        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
-        if self._bbox_crop:
-            bbox = self.bboxes[idx]
-            img = img[bbox[1]:bbox[3], bbox[0]:bbox[2], :]
-        if self._transform_ops:
-            img = transform(img, self._transform_ops)
-        img = img.transpose((2, 0, 1))
-        return (img, self.labels[idx])
-
-    def __len__(self):
-        return len(self.images)
-
-    @property
-    def class_num(self):
-        return len(set(self.labels))
-
-
-class VeriWild(Dataset):
-    def __init__(
-            self,
-            image_root,
-            cls_label_path,
-            transform_ops=None, ):
-        self._img_root = image_root
-        self._cls_path = cls_label_path
-        if transform_ops:
-            self._transform_ops = create_operators(transform_ops)
-        self._dtype = paddle.get_default_dtype()
-        self._load_anno()
-
-    def _load_anno(self):
-        assert os.path.exists(self._cls_path)
-        assert os.path.exists(self._img_root)
-        self.images = []
-        self.labels = []
-        self.cameras = []
-        with open(self._cls_path) as fd:
-            lines = fd.readlines()
-            for l in lines:
-                l = l.strip().split()
-                self.images.append(os.path.join(self._img_root, l[0]))
-                self.labels.append(np.int64(l[1]))
-                self.cameras.append(np.int64(l[2]))
-                assert os.path.exists(self.images[-1])
-
-    def __getitem__(self, idx):
-        try:
-            with open(self.images[idx], 'rb') as f:
-                img = f.read()
-            if self._transform_ops:
-                img = transform(img, self._transform_ops)
-            img = img.transpose((2, 0, 1))
-            return (img, self.labels[idx], self.cameras[idx])
-        except Exception as ex:
-            logger.error("Exception occured when parse line: {} with msg: {}".
-                         format(self.images[idx], ex))
-            rnd_idx = np.random.randint(self.__len__())
-            return self.__getitem__(rnd_idx)
-
-    def __len__(self):
-        return len(self.images)
-
-    @property
-    def class_num(self):
-        return len(set(self.labels))

paddlex/ppcls/data/postprocess/__init__.py

@@ -1,41 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import copy
-import importlib
-
-from . import topk
-
-from .topk import Topk, MultiLabelTopk
-
-
-def build_postprocess(config):
-    config = copy.deepcopy(config)
-    model_name = config.pop("name")
-    mod = importlib.import_module(__name__)
-    postprocess_func = getattr(mod, model_name)(**config)
-    return postprocess_func
-
-
-class DistillationPostProcess(object):
-    def __init__(self, model_name="Student", key=None, func="Topk", **kargs):
-        super().__init__()
-        self.func = eval(func)(**kargs)
-        self.model_name = model_name
-        self.key = key
-
-    def __call__(self, x, file_names=None):
-        x = x[self.model_name]
-        if self.key is not None:
-            x = x[self.key]
-        return self.func(x, file_names=file_names)

paddlex/ppcls/data/postprocess/topk.py

@@ -1,85 +0,0 @@
-# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import numpy as np
-import paddle
-import paddle.nn.functional as F
-
-
-class Topk(object):
-    def __init__(self, topk=1, class_id_map_file=None):
-        assert isinstance(topk, (int, ))
-        self.class_id_map = self.parse_class_id_map(class_id_map_file)
-        self.topk = topk
-
-    def parse_class_id_map(self, class_id_map_file):
-        if class_id_map_file is None:
-            return None
-        if not os.path.exists(class_id_map_file):
-            print(
-                "Warning: If want to use your own label_dict, please input legal path!\nOtherwise label_names will be empty!"
-            )
-            return None
-
-        try:
-            class_id_map = {}
-            with open(class_id_map_file, "r") as fin:
-                lines = fin.readlines()
-                for line in lines:
-                    partition = line.split("\n")[0].partition(" ")
-                    class_id_map[int(partition[0])] = str(partition[-1])
-        except Exception as ex:
-            print(ex)
-            class_id_map = None
-        return class_id_map
-
-    def __call__(self, x, file_names=None, multilabel=False):
-        assert isinstance(x, paddle.Tensor)
-        if file_names is not None:
-            assert x.shape[0] == len(file_names)
-        x = F.softmax(x, axis=-1) if not multilabel else F.sigmoid(x)
-        x = x.numpy()
-        y = []
-        for idx, probs in enumerate(x):
-            index = probs.argsort(axis=0)[-self.topk:][::-1].astype(
-                "int32") if not multilabel else np.where(
-                    probs >= 0.5)[0].astype("int32")
-            clas_id_list = []
-            score_list = []
-            label_name_list = []
-            for i in index:
-                clas_id_list.append(i.item())
-                score_list.append(probs[i].item())
-                if self.class_id_map is not None:
-                    label_name_list.append(self.class_id_map[i.item()])
-            result = {
-                "class_ids": clas_id_list,
-                "scores": np.around(
-                    score_list, decimals=5).tolist(),
-            }
-            if file_names is not None:
-                result["file_name"] = file_names[idx]
-            if label_name_list is not None:
-                result["label_names"] = label_name_list
-            y.append(result)
-        return y
-
-
-class MultiLabelTopk(Topk):
-    def __init__(self, topk=1, class_id_map_file=None):
-        super().__init__()
-
-    def __call__(self, x, file_names=None):
-        return super().__call__(x, file_names, multilabel=True)

paddlex/ppcls/data/preprocess/__init__.py

@@ -1,100 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from paddlex.ppcls.data.preprocess.ops.autoaugment import ImageNetPolicy as RawImageNetPolicy
-from paddlex.ppcls.data.preprocess.ops.randaugment import RandAugment as RawRandAugment
-from paddlex.ppcls.data.preprocess.ops.timm_autoaugment import RawTimmAutoAugment
-from paddlex.ppcls.data.preprocess.ops.cutout import Cutout
-
-from paddlex.ppcls.data.preprocess.ops.hide_and_seek import HideAndSeek
-from paddlex.ppcls.data.preprocess.ops.random_erasing import RandomErasing
-from paddlex.ppcls.data.preprocess.ops.grid import GridMask
-
-from paddlex.ppcls.data.preprocess.ops.operators import DecodeImage
-from paddlex.ppcls.data.preprocess.ops.operators import ResizeImage
-from paddlex.ppcls.data.preprocess.ops.operators import CropImage
-from paddlex.ppcls.data.preprocess.ops.operators import RandCropImage
-from paddlex.ppcls.data.preprocess.ops.operators import RandFlipImage
-from paddlex.ppcls.data.preprocess.ops.operators import NormalizeImage
-from paddlex.ppcls.data.preprocess.ops.operators import ToCHWImage
-from paddlex.ppcls.data.preprocess.ops.operators import AugMix
-
-from paddlex.ppcls.data.preprocess.batch_ops.batch_operators import MixupOperator, CutmixOperator, OpSampler, FmixOperator
-
-import numpy as np
-from PIL import Image
-
-
-def transform(data, ops=[]):
-    """ transform """
-    for op in ops:
-        data = op(data)
-    return data
-
-
-class AutoAugment(RawImageNetPolicy):
-    """ ImageNetPolicy wrapper to auto fit different img types """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def __call__(self, img):
-        if not isinstance(img, Image.Image):
-            img = np.ascontiguousarray(img)
-            img = Image.fromarray(img)
-
-        img = super().__call__(img)
-
-        if isinstance(img, Image.Image):
-            img = np.asarray(img)
-
-        return img
-
-
-class RandAugment(RawRandAugment):
-    """ RandAugment wrapper to auto fit different img types """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def __call__(self, img):
-        if not isinstance(img, Image.Image):
-            img = np.ascontiguousarray(img)
-            img = Image.fromarray(img)
-
-        img = super().__call__(img)
-
-        if isinstance(img, Image.Image):
-            img = np.asarray(img)
-
-        return img
-
-
-class TimmAutoAugment(RawTimmAutoAugment):
-    """ TimmAutoAugment wrapper to auto fit different img tyeps. """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def __call__(self, img):
-        if not isinstance(img, Image.Image):
-            img = np.ascontiguousarray(img)
-            img = Image.fromarray(img)
-
-        img = super().__call__(img)
-
-        if isinstance(img, Image.Image):
-            img = np.asarray(img)
-
-        return img