paddlex 2.1.0__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/ppcls/arch/gears/cosmargin.py

@@ -1,55 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import paddle
-import math
-import paddle.nn as nn
-
-
-class CosMargin(paddle.nn.Layer):
-    def __init__(self, embedding_size, class_num, margin=0.35, scale=64.0):
-        super(CosMargin, self).__init__()
-        self.scale = scale
-        self.margin = margin
-        self.embedding_size = embedding_size
-        self.class_num = class_num
-
-        self.weight = self.create_parameter(
-            shape=[self.embedding_size, self.class_num],
-            is_bias=False,
-            default_initializer=paddle.nn.initializer.XavierNormal())
-
-    def forward(self, input, label):
-        label.stop_gradient = True
-
-        input_norm = paddle.sqrt(
-            paddle.sum(paddle.square(input), axis=1, keepdim=True))
-        input = paddle.divide(input, input_norm)
-
-        weight_norm = paddle.sqrt(
-            paddle.sum(paddle.square(self.weight), axis=0, keepdim=True))
-        weight = paddle.divide(self.weight, weight_norm)
-
-        cos = paddle.matmul(input, weight)
-        if not self.training or label is None:
-            return cos
-
-        cos_m = cos - self.margin
-
-        one_hot = paddle.nn.functional.one_hot(label, self.class_num)
-        one_hot = paddle.squeeze(one_hot, axis=[1])
-        output = paddle.multiply(one_hot, cos_m) + paddle.multiply(
-            (1.0 - one_hot), cos)
-        output = output * self.scale
-        return output

paddlex/ppcls/arch/gears/fc.py

@@ -1,35 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import paddle
-import paddle.nn as nn
-
-
-class FC(nn.Layer):
-    def __init__(self, embedding_size, class_num):
-        super(FC, self).__init__()
-        self.embedding_size = embedding_size
-        self.class_num = class_num
-        weight_attr = paddle.ParamAttr(
-            initializer=paddle.nn.initializer.XavierNormal())
-        self.fc = paddle.nn.Linear(
-            self.embedding_size, self.class_num, weight_attr=weight_attr)
-
-    def forward(self, input, label=None):
-        out = self.fc(input)
-        return out

paddlex/ppcls/arch/gears/identity_head.py

@@ -1,9 +0,0 @@
-from paddle import nn
-
-
-class IdentityHead(nn.Layer):
-    def __init__(self):
-        super(IdentityHead, self).__init__()
-
-    def forward(self, x, label=None):
-        return {"features": x, "logits": None}

paddlex/ppcls/arch/gears/vehicle_neck.py

@@ -1,52 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import, division, print_function
-
-import paddle
-import paddle.nn as nn
-
-
-class VehicleNeck(nn.Layer):
-    def __init__(self,
-                 in_channels,
-                 out_channels,
-                 kernel_size=1,
-                 stride=1,
-                 padding=0,
-                 dilation=1,
-                 groups=1,
-                 padding_mode='zeros',
-                 weight_attr=None,
-                 bias_attr=None,
-                 data_format='NCHW'):
-        super().__init__()
-        self.conv = nn.Conv2D(
-            in_channels=in_channels,
-            out_channels=out_channels,
-            kernel_size=kernel_size,
-            stride=stride,
-            padding=padding,
-            dilation=dilation,
-            groups=groups,
-            padding_mode=padding_mode,
-            weight_attr=weight_attr,
-            bias_attr=weight_attr,
-            data_format=data_format)
-        self.flatten = nn.Flatten()
-
-    def forward(self, x):
-        x = self.conv(x)
-        x = self.flatten(x)
-        return x

paddlex/ppcls/arch/utils.py

@@ -1,53 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import six
-import types
-from difflib import SequenceMatcher
-
-from . import backbone
-
-
-def get_architectures():
-    """
-    get all of model architectures
-    """
-    names = []
-    for k, v in backbone.__dict__.items():
-        if isinstance(v, (types.FunctionType, six.class_types)):
-            names.append(k)
-    return names
-
-
-def get_blacklist_model_in_static_mode():
-    from paddlex.ppcls.arch.backbone import distilled_vision_transformer
-    from paddlex.ppcls.arch.backbone import vision_transformer
-    blacklist = distilled_vision_transformer.__all__ + vision_transformer.__all__
-    return blacklist
-
-
-def similar_architectures(name='', names=[], thresh=0.1, topk=10):
-    """
-    inferred similar architectures
-    """
-    scores = []
-    for idx, n in enumerate(names):
-        if n.startswith('__'):
-            continue
-        score = SequenceMatcher(None, n.lower(), name.lower()).quick_ratio()
-        if score > thresh:
-            scores.append((idx, score))
-    scores.sort(key=lambda x: x[1], reverse=True)
-    similar_names = [names[s[0]] for s in scores[:min(topk, len(scores))]]
-    return similar_names

paddlex/ppcls/data/__init__.py

@@ -1,144 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import inspect
-import copy
-import paddle
-import numpy as np
-from paddle.io import DistributedBatchSampler, BatchSampler, DataLoader
-from paddlex.ppcls.utils import logger
-
-from paddlex.ppcls.data import dataloader
-# dataset
-from paddlex.ppcls.data.dataloader.imagenet_dataset import ImageNetDataset
-from paddlex.ppcls.data.dataloader.multilabel_dataset import MultiLabelDataset
-from paddlex.ppcls.data.dataloader.common_dataset import create_operators
-from paddlex.ppcls.data.dataloader.vehicle_dataset import CompCars, VeriWild
-from paddlex.ppcls.data.dataloader.logo_dataset import LogoDataset
-from paddlex.ppcls.data.dataloader.icartoon_dataset import ICartoonDataset
-from paddlex.ppcls.data.dataloader.mix_dataset import MixDataset
-
-# sampler
-from paddlex.ppcls.data.dataloader.DistributedRandomIdentitySampler import DistributedRandomIdentitySampler
-from paddlex.ppcls.data.dataloader.pk_sampler import PKSampler
-from paddlex.ppcls.data.dataloader.mix_sampler import MixSampler
-from paddlex.ppcls.data import preprocess
-from paddlex.ppcls.data.preprocess import transform
-
-
-def create_operators(params, class_num=None):
-    """
-    create operators based on the config
-
-    Args:
-        params(list): a dict list, used to create some operators
-    """
-    assert isinstance(params, list), ('operator config should be a list')
-    ops = []
-    for operator in params:
-        assert isinstance(operator,
-                          dict) and len(operator) == 1, "yaml format error"
-        op_name = list(operator)[0]
-        param = {} if operator[op_name] is None else operator[op_name]
-        op_func = getattr(preprocess, op_name)
-        if "class_num" in inspect.getfullargspec(op_func).args:
-            param.update({"class_num": class_num})
-        op = op_func(**param)
-        ops.append(op)
-
-    return ops
-
-
-def build_dataloader(config, mode, device, use_dali=False, seed=None):
-    assert mode in [
-        'Train', 'Eval', 'Test', 'Gallery', 'Query'
-    ], "Dataset mode should be Train, Eval, Test, Gallery, Query"
-    # build dataset
-    if use_dali:
-        from paddlex.ppcls.data.dataloader.dali import dali_dataloader
-        return dali_dataloader(config, mode, paddle.device.get_device(), seed)
-
-    class_num = config.get("class_num", None)
-    config_dataset = config[mode]['dataset']
-    config_dataset = copy.deepcopy(config_dataset)
-    dataset_name = config_dataset.pop('name')
-    if 'batch_transform_ops' in config_dataset:
-        batch_transform = config_dataset.pop('batch_transform_ops')
-    else:
-        batch_transform = None
-
-    dataset = eval(dataset_name)(**config_dataset)
-
-    logger.debug("build dataset({}) success...".format(dataset))
-
-    # build sampler
-    config_sampler = config[mode]['sampler']
-    if "name" not in config_sampler:
-        batch_sampler = None
-        batch_size = config_sampler["batch_size"]
-        drop_last = config_sampler["drop_last"]
-        shuffle = config_sampler["shuffle"]
-    else:
-        sampler_name = config_sampler.pop("name")
-        batch_sampler = eval(sampler_name)(dataset, **config_sampler)
-
-    logger.debug("build batch_sampler({}) success...".format(batch_sampler))
-
-    # build batch operator
-    def mix_collate_fn(batch):
-        batch = transform(batch, batch_ops)
-        # batch each field
-        slots = []
-        for items in batch:
-            for i, item in enumerate(items):
-                if len(slots) < len(items):
-                    slots.append([item])
-                else:
-                    slots[i].append(item)
-        return [np.stack(slot, axis=0) for slot in slots]
-
-    if isinstance(batch_transform, list):
-        batch_ops = create_operators(batch_transform, class_num)
-        batch_collate_fn = mix_collate_fn
-    else:
-        batch_collate_fn = None
-
-    # build dataloader
-    config_loader = config[mode]['loader']
-    num_workers = config_loader["num_workers"]
-    use_shared_memory = config_loader["use_shared_memory"]
-
-    if batch_sampler is None:
-        data_loader = DataLoader(
-            dataset=dataset,
-            places=device,
-            num_workers=num_workers,
-            return_list=True,
-            use_shared_memory=use_shared_memory,
-            batch_size=batch_size,
-            shuffle=shuffle,
-            drop_last=drop_last,
-            collate_fn=batch_collate_fn)
-    else:
-        data_loader = DataLoader(
-            dataset=dataset,
-            places=device,
-            num_workers=num_workers,
-            return_list=True,
-            use_shared_memory=use_shared_memory,
-            batch_sampler=batch_sampler,
-            collate_fn=batch_collate_fn)
-
-    logger.debug("build data_loader({}) success...".format(data_loader))
-    return data_loader

paddlex/ppcls/data/dataloader/DistributedRandomIdentitySampler.py

@@ -1,90 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from collections import defaultdict
-import numpy as np
-import copy
-import random
-from paddle.io import DistributedBatchSampler, Sampler
-
-
-class DistributedRandomIdentitySampler(DistributedBatchSampler):
-    """
-    Randomly sample N identities, then for each identity,
-    randomly sample K instances, therefore batch size is N*K.
-    Args:
-    - data_source (list): list of (img_path, pid, camid).
-    - num_instances (int): number of instances per identity in a batch.
-    - batch_size (int): number of examples in a batch.
-    """
-
-    def __init__(self, dataset, batch_size, num_instances, drop_last, **args):
-        self.dataset = dataset
-        self.batch_size = batch_size
-        self.num_instances = num_instances
-        self.drop_last = drop_last
-        self.num_pids_per_batch = self.batch_size // self.num_instances
-        self.index_dic = defaultdict(list)
-        for index, pid in enumerate(self.dataset.labels):
-            self.index_dic[pid].append(index)
-        self.pids = list(self.index_dic.keys())
-        # estimate number of examples in an epoch
-        self.length = 0
-        for pid in self.pids:
-            idxs = self.index_dic[pid]
-            num = len(idxs)
-            if num < self.num_instances:
-                num = self.num_instances
-            self.length += num - num % self.num_instances
-
-    def __iter__(self):
-        batch_idxs_dict = defaultdict(list)
-        for pid in self.pids:
-            idxs = copy.deepcopy(self.index_dic[pid])
-            if len(idxs) < self.num_instances:
-                idxs = np.random.choice(
-                    idxs, size=self.num_instances, replace=True)
-            random.shuffle(idxs)
-            batch_idxs = []
-            for idx in idxs:
-                batch_idxs.append(idx)
-                if len(batch_idxs) == self.num_instances:
-                    batch_idxs_dict[pid].append(batch_idxs)
-                    batch_idxs = []
-        avai_pids = copy.deepcopy(self.pids)
-        final_idxs = []
-        while len(avai_pids) >= self.num_pids_per_batch:
-            selected_pids = random.sample(avai_pids, self.num_pids_per_batch)
-            for pid in selected_pids:
-                batch_idxs = batch_idxs_dict[pid].pop(0)
-                final_idxs.extend(batch_idxs)
-                if len(batch_idxs_dict[pid]) == 0:
-                    avai_pids.remove(pid)
-        _sample_iter = iter(final_idxs)
-        batch_indices = []
-        for idx in _sample_iter:
-            batch_indices.append(idx)
-            if len(batch_indices) == self.batch_size:
-                yield batch_indices
-                batch_indices = []
-        if not self.drop_last and len(batch_indices) > 0:
-            yield batch_indices
-
-    def __len__(self):
-        if self.drop_last:
-            return self.length // self.batch_size
-        else:
-            return (self.length + self.batch_size - 1) // self.batch_size

paddlex/ppcls/data/dataloader/__init__.py

@@ -1,9 +0,0 @@
-from paddlex.ppcls.data.dataloader.imagenet_dataset import ImageNetDataset
-from paddlex.ppcls.data.dataloader.multilabel_dataset import MultiLabelDataset
-from paddlex.ppcls.data.dataloader.common_dataset import create_operators
-from paddlex.ppcls.data.dataloader.vehicle_dataset import CompCars, VeriWild
-from paddlex.ppcls.data.dataloader.logo_dataset import LogoDataset
-from paddlex.ppcls.data.dataloader.icartoon_dataset import ICartoonDataset
-from paddlex.ppcls.data.dataloader.mix_dataset import MixDataset
-from paddlex.ppcls.data.dataloader.mix_sampler import MixSampler
-from paddlex.ppcls.data.dataloader.pk_sampler import PKSampler

paddlex/ppcls/data/dataloader/common_dataset.py

@@ -1,84 +0,0 @@
-#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import numpy as np
-
-from paddle.io import Dataset
-import cv2
-
-from paddlex.ppcls.data import preprocess
-from paddlex.ppcls.data.preprocess import transform
-from paddlex.ppcls.utils import logger
-
-
-def create_operators(params):
-    """
-    create operators based on the config
-    Args:
-        params(list): a dict list, used to create some operators
-    """
-    assert isinstance(params, list), ('operator config should be a list')
-    ops = []
-    for operator in params:
-        assert isinstance(operator,
-                          dict) and len(operator) == 1, "yaml format error"
-        op_name = list(operator)[0]
-        param = {} if operator[op_name] is None else operator[op_name]
-        op = getattr(preprocess, op_name)(**param)
-        ops.append(op)
-
-    return ops
-
-
-class CommonDataset(Dataset):
-    def __init__(
-            self,
-            image_root,
-            cls_label_path,
-            transform_ops=None, ):
-        self._img_root = image_root
-        self._cls_path = cls_label_path
-        if transform_ops:
-            self._transform_ops = create_operators(transform_ops)
-
-        self.images = []
-        self.labels = []
-        self._load_anno()
-
-    def _load_anno(self):
-        pass
-
-    def __getitem__(self, idx):
-        try:
-            with open(self.images[idx], 'rb') as f:
-                img = f.read()
-            if self._transform_ops:
-                img = transform(img, self._transform_ops)
-            img = img.transpose((2, 0, 1))
-            return (img, self.labels[idx])
-
-        except Exception as ex:
-            logger.error("Exception occured when parse line: {} with msg: {}".
-                         format(self.images[idx], ex))
-            rnd_idx = np.random.randint(self.__len__())
-            return self.__getitem__(rnd_idx)
-
-    def __len__(self):
-        return len(self.images)
-
-    @property
-    def class_num(self):
-        return len(set(self.labels))