paddlex 2.1.0__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/ppdet/data/crop_utils/chip_box_utils.py

@@ -1,170 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import numpy as np
-
-
-def bbox_area(boxes):
-    return (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1])
-
-
-def intersection_over_box(chips, boxes):
-    """
-    intersection area over box area
-    :param chips:  C
-    :param boxes:  B
-    :return: iob, CxB
-    """
-    M = chips.shape[0]
-    N = boxes.shape[0]
-    if M * N == 0:
-        return np.zeros([M, N], dtype='float32')
-
-    box_area = bbox_area(boxes)  # B
-
-    inter_x2y2 = np.minimum(np.expand_dims(chips, 1)[:, :, 2:],
-                            boxes[:, 2:])  # CxBX2
-    inter_x1y1 = np.maximum(np.expand_dims(chips, 1)[:, :, :2],
-                            boxes[:, :2])  # CxBx2
-    inter_wh = inter_x2y2 - inter_x1y1
-    inter_wh = np.clip(inter_wh, a_min=0, a_max=None)
-    inter_area = inter_wh[:, :, 0] * inter_wh[:, :, 1]  # CxB
-
-    iob = inter_area / np.expand_dims(box_area, 0)
-    return iob
-
-
-def clip_boxes(boxes, im_shape):
-    """
-    Clip boxes to image boundaries.
-    :param boxes: [N, 4]
-    :param im_shape: tuple of 2, [h, w]
-    :return: [N, 4]
-    """
-    # x1 >= 0
-    boxes[:, 0] = np.clip(boxes[:, 0], 0, im_shape[1] - 1)
-    # y1 >= 0
-    boxes[:, 1] = np.clip(boxes[:, 1], 0, im_shape[0] - 1)
-    # x2 < im_shape[1]
-    boxes[:, 2] = np.clip(boxes[:, 2], 1, im_shape[1])
-    # y2 < im_shape[0]
-    boxes[:, 3] = np.clip(boxes[:, 3], 1, im_shape[0])
-    return boxes
-
-
-def transform_chip_box(gt_bbox: 'Gx4', boxes_idx: 'B', chip: '4'):
-    boxes_idx = np.array(boxes_idx)
-    cur_gt_bbox = gt_bbox[boxes_idx].copy()  # Bx4
-    x1, y1, x2, y2 = chip
-    cur_gt_bbox[:, 0] -= x1
-    cur_gt_bbox[:, 1] -= y1
-    cur_gt_bbox[:, 2] -= x1
-    cur_gt_bbox[:, 3] -= y1
-    h = y2 - y1
-    w = x2 - x1
-    cur_gt_bbox = clip_boxes(cur_gt_bbox, (h, w))
-    ws = (cur_gt_bbox[:, 2] - cur_gt_bbox[:, 0]).astype(np.int32)
-    hs = (cur_gt_bbox[:, 3] - cur_gt_bbox[:, 1]).astype(np.int32)
-    valid_idx = (ws >= 2) & (hs >= 2)
-    return cur_gt_bbox[valid_idx], boxes_idx[valid_idx]
-
-
-def find_chips_to_cover_overlaped_boxes(iob, overlap_threshold):
-    chip_ids, box_ids = np.nonzero(iob >= overlap_threshold)
-    chip_id2overlap_box_num = np.bincount(chip_ids)  # 1d array
-    chip_id2overlap_box_num = np.pad(
-        chip_id2overlap_box_num, (0, len(iob) - len(chip_id2overlap_box_num)),
-        constant_values=0)
-
-    chosen_chip_ids = []
-    while len(box_ids) > 0:
-        value_counts = np.bincount(chip_ids)  # 1d array
-        max_count_chip_id = np.argmax(value_counts)
-        assert max_count_chip_id not in chosen_chip_ids
-        chosen_chip_ids.append(max_count_chip_id)
-
-        box_ids_in_cur_chip = box_ids[chip_ids == max_count_chip_id]
-        ids_not_in_cur_boxes_mask = np.logical_not(
-            np.isin(box_ids, box_ids_in_cur_chip))
-        chip_ids = chip_ids[ids_not_in_cur_boxes_mask]
-        box_ids = box_ids[ids_not_in_cur_boxes_mask]
-    return chosen_chip_ids, chip_id2overlap_box_num
-
-
-def transform_chip_boxes2image_boxes(chip_boxes, chip, img_h, img_w):
-    chip_boxes = np.array(sorted(chip_boxes, key=lambda item: -item[1]))
-    xmin, ymin, _, _ = chip
-    # Transform to origin image loc
-    chip_boxes[:, 2] += xmin
-    chip_boxes[:, 4] += xmin
-    chip_boxes[:, 3] += ymin
-    chip_boxes[:, 5] += ymin
-    chip_boxes = clip_boxes(chip_boxes, (img_h, img_w))
-    return chip_boxes
-
-
-def nms(dets, thresh):
-    """Apply classic DPM-style greedy NMS."""
-    if dets.shape[0] == 0:
-        return dets[[], :]
-    scores = dets[:, 1]
-    x1 = dets[:, 2]
-    y1 = dets[:, 3]
-    x2 = dets[:, 4]
-    y2 = dets[:, 5]
-
-    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
-    order = scores.argsort()[::-1]
-
-    ndets = dets.shape[0]
-    suppressed = np.zeros((ndets), dtype=np.int)
-
-    # nominal indices
-    # _i, _j
-    # sorted indices
-    # i, j
-    # temp variables for box i's (the box currently under consideration)
-    # ix1, iy1, ix2, iy2, iarea
-
-    # variables for computing overlap with box j (lower scoring box)
-    # xx1, yy1, xx2, yy2
-    # w, h
-    # inter, ovr
-
-    for _i in range(ndets):
-        i = order[_i]
-        if suppressed[i] == 1:
-            continue
-        ix1 = x1[i]
-        iy1 = y1[i]
-        ix2 = x2[i]
-        iy2 = y2[i]
-        iarea = areas[i]
-        for _j in range(_i + 1, ndets):
-            j = order[_j]
-            if suppressed[j] == 1:
-                continue
-            xx1 = max(ix1, x1[j])
-            yy1 = max(iy1, y1[j])
-            xx2 = min(ix2, x2[j])
-            yy2 = min(iy2, y2[j])
-            w = max(0.0, xx2 - xx1 + 1)
-            h = max(0.0, yy2 - yy1 + 1)
-            inter = w * h
-            ovr = inter / (iarea + areas[j] - inter)
-            if ovr >= thresh:
-                suppressed[j] = 1
-    keep = np.where(suppressed == 0)[0]
-    dets = dets[keep, :]
-    return dets

paddlex/ppdet/data/reader.py

@@ -1,302 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import traceback
-import six
-import sys
-if sys.version_info >= (3, 0):
-    pass
-else:
-    pass
-import numpy as np
-
-from paddle.io import DataLoader, DistributedBatchSampler
-from paddle.fluid.dataloader.collate import default_collate_fn
-
-from paddlex.ppdet.core.workspace import register
-from . import transform
-from .shm_utils import _get_shared_memory_size_in_M
-
-from paddlex.ppdet.utils.logger import setup_logger
-logger = setup_logger('reader')
-
-MAIN_PID = os.getpid()
-
-
-class Compose(object):
-    def __init__(self, transforms, num_classes=80):
-        self.transforms = transforms
-        self.transforms_cls = []
-        for t in self.transforms:
-            for k, v in t.items():
-                op_cls = getattr(transform, k)
-                f = op_cls(**v)
-                if hasattr(f, 'num_classes'):
-                    f.num_classes = num_classes
-
-                self.transforms_cls.append(f)
-
-    def __call__(self, data):
-        for f in self.transforms_cls:
-            try:
-                data = f(data)
-            except Exception as e:
-                stack_info = traceback.format_exc()
-                logger.warning("fail to map sample transform [{}] "
-                               "with error: {} and stack:\n{}".format(
-                                   f, e, str(stack_info)))
-                raise e
-
-        return data
-
-
-class BatchCompose(Compose):
-    def __init__(self, transforms, num_classes=80, collate_batch=True):
-        super(BatchCompose, self).__init__(transforms, num_classes)
-        self.collate_batch = collate_batch
-
-    def __call__(self, data):
-        for f in self.transforms_cls:
-            try:
-                data = f(data)
-            except Exception as e:
-                stack_info = traceback.format_exc()
-                logger.warning("fail to map batch transform [{}] "
-                               "with error: {} and stack:\n{}".format(
-                                   f, e, str(stack_info)))
-                raise e
-
-        # remove keys which is not needed by model
-        extra_key = ['h', 'w', 'flipped']
-        for k in extra_key:
-            for sample in data:
-                if k in sample:
-                    sample.pop(k)
-
-        # batch data, if user-define batch function needed
-        # use user-defined here
-        if self.collate_batch:
-            batch_data = default_collate_fn(data)
-        else:
-            batch_data = {}
-            for k in data[0].keys():
-                tmp_data = []
-                for i in range(len(data)):
-                    tmp_data.append(data[i][k])
-                if not 'gt_' in k and not 'is_crowd' in k and not 'difficult' in k:
-                    tmp_data = np.stack(tmp_data, axis=0)
-                batch_data[k] = tmp_data
-        return batch_data
-
-
-class BaseDataLoader(object):
-    """
-    Base DataLoader implementation for detection models
-
-    Args:
-        sample_transforms (list): a list of transforms to perform
-                                  on each sample
-        batch_transforms (list): a list of transforms to perform
-                                 on batch
-        batch_size (int): batch size for batch collating, default 1.
-        shuffle (bool): whether to shuffle samples
-        drop_last (bool): whether to drop the last incomplete,
-                          default False
-        num_classes (int): class number of dataset, default 80
-        collate_batch (bool): whether to collate batch in dataloader.
-            If set to True, the samples will collate into batch according
-            to the batch size. Otherwise, the ground-truth will not collate,
-            which is used when the number of ground-truch is different in
-            samples.
-        use_shared_memory (bool): whether to use shared memory to
-                accelerate data loading, enable this only if you
-                are sure that the shared memory size of your OS
-                is larger than memory cost of input datas of model.
-                Note that shared memory will be automatically
-                disabled if the shared memory of OS is less than
-                1G, which is not enough for detection models.
-                Default False.
-    """
-
-    def __init__(self,
-                 sample_transforms=[],
-                 batch_transforms=[],
-                 batch_size=1,
-                 shuffle=False,
-                 drop_last=False,
-                 num_classes=80,
-                 collate_batch=True,
-                 use_shared_memory=False,
-                 **kwargs):
-        # sample transform
-        self._sample_transforms = Compose(
-            sample_transforms, num_classes=num_classes)
-
-        # batch transfrom
-        self._batch_transforms = BatchCompose(batch_transforms, num_classes,
-                                              collate_batch)
-        self.batch_size = batch_size
-        self.shuffle = shuffle
-        self.drop_last = drop_last
-        self.use_shared_memory = use_shared_memory
-        self.kwargs = kwargs
-
-    def __call__(self,
-                 dataset,
-                 worker_num,
-                 batch_sampler=None,
-                 return_list=False):
-        self.dataset = dataset
-        self.dataset.check_or_download_dataset()
-        self.dataset.parse_dataset()
-        # get data
-        self.dataset.set_transform(self._sample_transforms)
-        # set kwargs
-        self.dataset.set_kwargs(**self.kwargs)
-        # batch sampler
-        if batch_sampler is None:
-            self._batch_sampler = DistributedBatchSampler(
-                self.dataset,
-                batch_size=self.batch_size,
-                shuffle=self.shuffle,
-                drop_last=self.drop_last)
-        else:
-            self._batch_sampler = batch_sampler
-
-        # DataLoader do not start sub-process in Windows and Mac
-        # system, do not need to use shared memory
-        use_shared_memory = self.use_shared_memory and \
-                            sys.platform not in ['win32', 'darwin']
-        # check whether shared memory size is bigger than 1G(1024M)
-        if use_shared_memory:
-            shm_size = _get_shared_memory_size_in_M()
-            if shm_size is not None and shm_size < 1024.:
-                logger.warning("Shared memory size is less than 1G, "
-                               "disable shared_memory in DataLoader")
-                use_shared_memory = False
-
-        self.dataloader = DataLoader(
-            dataset=self.dataset,
-            batch_sampler=self._batch_sampler,
-            collate_fn=self._batch_transforms,
-            num_workers=worker_num,
-            return_list=return_list,
-            use_shared_memory=use_shared_memory)
-        self.loader = iter(self.dataloader)
-
-        return self
-
-    def __len__(self):
-        return len(self._batch_sampler)
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        try:
-            return next(self.loader)
-        except StopIteration:
-            self.loader = iter(self.dataloader)
-            six.reraise(*sys.exc_info())
-
-    def next(self):
-        # python2 compatibility
-        return self.__next__()
-
-
-@register
-class TrainReader(BaseDataLoader):
-    __shared__ = ['num_classes']
-
-    def __init__(self,
-                 sample_transforms=[],
-                 batch_transforms=[],
-                 batch_size=1,
-                 shuffle=True,
-                 drop_last=True,
-                 num_classes=80,
-                 collate_batch=True,
-                 **kwargs):
-        super(TrainReader, self).__init__(sample_transforms, batch_transforms,
-                                          batch_size, shuffle, drop_last,
-                                          num_classes, collate_batch, **kwargs)
-
-
-@register
-class EvalReader(BaseDataLoader):
-    __shared__ = ['num_classes']
-
-    def __init__(self,
-                 sample_transforms=[],
-                 batch_transforms=[],
-                 batch_size=1,
-                 shuffle=False,
-                 drop_last=True,
-                 num_classes=80,
-                 **kwargs):
-        super(EvalReader, self).__init__(sample_transforms, batch_transforms,
-                                         batch_size, shuffle, drop_last,
-                                         num_classes, **kwargs)
-
-
-@register
-class TestReader(BaseDataLoader):
-    __shared__ = ['num_classes']
-
-    def __init__(self,
-                 sample_transforms=[],
-                 batch_transforms=[],
-                 batch_size=1,
-                 shuffle=False,
-                 drop_last=False,
-                 num_classes=80,
-                 **kwargs):
-        super(TestReader, self).__init__(sample_transforms, batch_transforms,
-                                         batch_size, shuffle, drop_last,
-                                         num_classes, **kwargs)
-
-
-@register
-class EvalMOTReader(BaseDataLoader):
-    __shared__ = ['num_classes']
-
-    def __init__(self,
-                 sample_transforms=[],
-                 batch_transforms=[],
-                 batch_size=1,
-                 shuffle=False,
-                 drop_last=False,
-                 num_classes=1,
-                 **kwargs):
-        super(EvalMOTReader, self).__init__(
-            sample_transforms, batch_transforms, batch_size, shuffle,
-            drop_last, num_classes, **kwargs)
-
-
-@register
-class TestMOTReader(BaseDataLoader):
-    __shared__ = ['num_classes']
-
-    def __init__(self,
-                 sample_transforms=[],
-                 batch_transforms=[],
-                 batch_size=1,
-                 shuffle=False,
-                 drop_last=False,
-                 num_classes=1,
-                 **kwargs):
-        super(TestMOTReader, self).__init__(
-            sample_transforms, batch_transforms, batch_size, shuffle,
-            drop_last, num_classes, **kwargs)

paddlex/ppdet/data/shm_utils.py

@@ -1,67 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-
-SIZE_UNIT = ['K', 'M', 'G', 'T']
-SHM_QUERY_CMD = 'df -h'
-SHM_KEY = 'shm'
-SHM_DEFAULT_MOUNT = '/dev/shm'
-
-# [ shared memory size check ]
-# In detection models, image/target data occupies a lot of memory, and
-# will occupy lots of shared memory in multi-process DataLoader, we use
-# following code to get shared memory size and perform a size check to
-# disable shared memory use if shared memory size is not enough.
-# Shared memory getting process as follows:
-# 1. use `df -h` get all mount info
-# 2. pick up spaces whose mount info contains 'shm'
-# 3. if 'shm' space number is only 1, return its size
-# 4. if there are multiple 'shm' space, try to find the default mount
-#    directory '/dev/shm' is Linux-like system, otherwise return the
-#    biggest space size.
-
-
-def _parse_size_in_M(size_str):
-    num, unit = size_str[:-1], size_str[-1]
-    assert unit in SIZE_UNIT, \
-            "unknown shm size unit {}".format(unit)
-    return float(num) * \
-            (1024 ** (SIZE_UNIT.index(unit) - 1))
-
-
-def _get_shared_memory_size_in_M():
-    try:
-        df_infos = os.popen(SHM_QUERY_CMD).readlines()
-    except:
-        return None
-    else:
-        shm_infos = []
-        for df_info in df_infos:
-            info = df_info.strip()
-            if info.find(SHM_KEY) >= 0:
-                shm_infos.append(info.split())
-
-        if len(shm_infos) == 0:
-            return None
-        elif len(shm_infos) == 1:
-            return _parse_size_in_M(shm_infos[0][3])
-        else:
-            default_mount_infos = [
-                si for si in shm_infos if si[-1] == SHM_DEFAULT_MOUNT
-            ]
-            if default_mount_infos:
-                return _parse_size_in_M(default_mount_infos[0][3])
-            else:
-                return max([_parse_size_in_M(si[3]) for si in shm_infos])

paddlex/ppdet/data/source/__init__.py

@@ -1,29 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from . import coco
-from . import voc
-from . import widerface
-from . import category
-from . import keypoint_coco
-from . import mot
-from . import sniper_coco
-
-from .coco import *
-from .voc import *
-from .widerface import *
-from .category import *
-from .keypoint_coco import *
-from .mot import *
-from .sniper_coco import SniperCOCODataSet