paddlex 2.1.0__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex/ppcls/utils/save_load.py

@@ -1,136 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import errno
-import os
-import re
-import shutil
-import tempfile
-
-import paddle
-from paddlex.ppcls.utils import logger
-from .download import get_weights_path_from_url
-
-__all__ = ['init_model', 'save_model', 'load_dygraph_pretrain']
-
-
-def _mkdir_if_not_exist(path):
-    """
-    mkdir if not exists, ignore the exception when multiprocess mkdir together
-    """
-    if not os.path.exists(path):
-        try:
-            os.makedirs(path)
-        except OSError as e:
-            if e.errno == errno.EEXIST and os.path.isdir(path):
-                logger.warning(
-                    'be happy if some process has already created {}'.format(
-                        path))
-            else:
-                raise OSError('Failed to mkdir {}'.format(path))
-
-
-def load_dygraph_pretrain(model, path=None):
-    if not (os.path.isdir(path) or os.path.exists(path + '.pdparams')):
-        raise ValueError("Model pretrain path {} does not "
-                         "exists.".format(path))
-    param_state_dict = paddle.load(path + ".pdparams")
-    model.set_dict(param_state_dict)
-    return
-
-
-def load_dygraph_pretrain_from_url(model, pretrained_url, use_ssld=False):
-    if use_ssld:
-        pretrained_url = pretrained_url.replace("_pretrained",
-                                                "_ssld_pretrained")
-    local_weight_path = get_weights_path_from_url(pretrained_url).replace(
-        ".pdparams", "")
-    load_dygraph_pretrain(model, path=local_weight_path)
-    return
-
-
-def load_distillation_model(model, pretrained_model):
-    logger.info("In distillation mode, teacher model will be "
-                "loaded firstly before student model.")
-
-    if not isinstance(pretrained_model, list):
-        pretrained_model = [pretrained_model]
-
-    teacher = model.teacher if hasattr(model,
-                                       "teacher") else model._layers.teacher
-    student = model.student if hasattr(model,
-                                       "student") else model._layers.student
-    load_dygraph_pretrain(teacher, path=pretrained_model[0])
-    logger.info("Finish initing teacher model from {}".format(
-        pretrained_model))
-    # load student model
-    if len(pretrained_model) >= 2:
-        load_dygraph_pretrain(student, path=pretrained_model[1])
-        logger.info("Finish initing student model from {}".format(
-            pretrained_model))
-
-
-def init_model(config, net, optimizer=None):
-    """
-    load model from checkpoint or pretrained_model
-    """
-    checkpoints = config.get('checkpoints')
-    if checkpoints and optimizer is not None:
-        assert os.path.exists(checkpoints + ".pdparams"), \
-            "Given dir {}.pdparams not exist.".format(checkpoints)
-        assert os.path.exists(checkpoints + ".pdopt"), \
-            "Given dir {}.pdopt not exist.".format(checkpoints)
-        para_dict = paddle.load(checkpoints + ".pdparams")
-        opti_dict = paddle.load(checkpoints + ".pdopt")
-        metric_dict = paddle.load(checkpoints + ".pdstates")
-        net.set_dict(para_dict)
-        optimizer.set_state_dict(opti_dict)
-        logger.info("Finish load checkpoints from {}".format(checkpoints))
-        return metric_dict
-
-    pretrained_model = config.get('pretrained_model')
-    use_distillation = config.get('use_distillation', False)
-    if pretrained_model:
-        if use_distillation:
-            load_distillation_model(net, pretrained_model)
-        else:  # common load
-            load_dygraph_pretrain(net, path=pretrained_model)
-            logger.info(
-                logger.coloring("Finish load pretrained model from {}".format(
-                    pretrained_model), "HEADER"))
-
-
-def save_model(net,
-               optimizer,
-               metric_info,
-               model_path,
-               model_name="",
-               prefix='ppcls'):
-    """
-    save model to the target path
-    """
-    if paddle.distributed.get_rank() != 0:
-        return
-    model_path = os.path.join(model_path, model_name)
-    _mkdir_if_not_exist(model_path)
-    model_path = os.path.join(model_path, prefix)
-
-    paddle.save(net.state_dict(), model_path + ".pdparams")
-    paddle.save(optimizer.state_dict(), model_path + ".pdopt")
-    paddle.save(metric_info, model_path + ".pdstates")
-    logger.info("Already save model in {}".format(model_path))

paddlex/ppdet/__init__.py

@@ -1,16 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from . import (core, data, engine, modeling, model_zoo, optimizer, metrics,
-               utils, slim)

paddlex/ppdet/core/__init__.py

@@ -1,15 +0,0 @@
-#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from . import config

paddlex/ppdet/core/config/__init__.py

@@ -1,13 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

paddlex/ppdet/core/config/schema.py

@@ -1,248 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import print_function
-from __future__ import division
-
-import inspect
-import importlib
-import re
-
-try:
-    from docstring_parser import parse as doc_parse
-except Exception:
-
-    def doc_parse(*args):
-        pass
-
-
-try:
-    from typeguard import check_type
-except Exception:
-
-    def check_type(*args):
-        pass
-
-
-__all__ = ['SchemaValue', 'SchemaDict', 'SharedConfig', 'extract_schema']
-
-
-class SchemaValue(object):
-    def __init__(self, name, doc='', type=None):
-        super(SchemaValue, self).__init__()
-        self.name = name
-        self.doc = doc
-        self.type = type
-
-    def set_default(self, value):
-        self.default = value
-
-    def has_default(self):
-        return hasattr(self, 'default')
-
-
-class SchemaDict(dict):
-    def __init__(self, **kwargs):
-        super(SchemaDict, self).__init__()
-        self.schema = {}
-        self.strict = False
-        self.doc = ""
-        self.update(kwargs)
-
-    def __setitem__(self, key, value):
-        # XXX also update regular dict to SchemaDict??
-        if isinstance(value, dict) and key in self and isinstance(self[key],
-                                                                  SchemaDict):
-            self[key].update(value)
-        else:
-            super(SchemaDict, self).__setitem__(key, value)
-
-    def __missing__(self, key):
-        if self.has_default(key):
-            return self.schema[key].default
-        elif key in self.schema:
-            return self.schema[key]
-        else:
-            raise KeyError(key)
-
-    def copy(self):
-        newone = SchemaDict()
-        newone.__dict__.update(self.__dict__)
-        newone.update(self)
-        return newone
-
-    def set_schema(self, key, value):
-        assert isinstance(value, SchemaValue)
-        self.schema[key] = value
-
-    def set_strict(self, strict):
-        self.strict = strict
-
-    def has_default(self, key):
-        return key in self.schema and self.schema[key].has_default()
-
-    def is_default(self, key):
-        if not self.has_default(key):
-            return False
-        if hasattr(self[key], '__dict__'):
-            return True
-        else:
-            return key not in self or self[key] == self.schema[key].default
-
-    def find_default_keys(self):
-        return [
-            k for k in list(self.keys()) + list(self.schema.keys())
-            if self.is_default(k)
-        ]
-
-    def mandatory(self):
-        return any([k for k in self.schema.keys() if not self.has_default(k)])
-
-    def find_missing_keys(self):
-        missing = [
-            k for k in self.schema.keys()
-            if k not in self and not self.has_default(k)
-        ]
-        placeholders = [k for k in self if self[k] in ('<missing>', '<value>')]
-        return missing + placeholders
-
-    def find_extra_keys(self):
-        return list(set(self.keys()) - set(self.schema.keys()))
-
-    def find_mismatch_keys(self):
-        mismatch_keys = []
-        for arg in self.schema.values():
-            if arg.type is not None:
-                try:
-                    check_type("{}.{}".format(self.name, arg.name),
-                               self[arg.name], arg.type)
-                except Exception:
-                    mismatch_keys.append(arg.name)
-        return mismatch_keys
-
-    def validate(self):
-        missing_keys = self.find_missing_keys()
-        if missing_keys:
-            raise ValueError("Missing param for class<{}>: {}".format(
-                self.name, ", ".join(missing_keys)))
-        extra_keys = self.find_extra_keys()
-        if extra_keys and self.strict:
-            raise ValueError("Extraneous param for class<{}>: {}".format(
-                self.name, ", ".join(extra_keys)))
-        mismatch_keys = self.find_mismatch_keys()
-        if mismatch_keys:
-            raise TypeError("Wrong param type for class<{}>: {}".format(
-                self.name, ", ".join(mismatch_keys)))
-
-
-class SharedConfig(object):
-    """
-    Representation class for `__shared__` annotations, which work as follows:
-
-    - if `key` is set for the module in config file, its value will take
-      precedence
-    - if `key` is not set for the module but present in the config file, its
-      value will be used
-    - otherwise, use the provided `default_value` as fallback
-
-    Args:
-        key: config[key] will be injected
-        default_value: fallback value
-    """
-
-    def __init__(self, key, default_value=None):
-        super(SharedConfig, self).__init__()
-        self.key = key
-        self.default_value = default_value
-
-
-def extract_schema(cls):
-    """
-    Extract schema from a given class
-
-    Args:
-        cls (type): Class from which to extract.
-
-    Returns:
-        schema (SchemaDict): Extracted schema.
-    """
-    ctor = cls.__init__
-    # python 2 compatibility
-    if hasattr(inspect, 'getfullargspec'):
-        argspec = inspect.getfullargspec(ctor)
-        annotations = argspec.annotations
-        has_kwargs = argspec.varkw is not None
-    else:
-        argspec = inspect.getfullargspec(ctor)
-        # python 2 type hinting workaround, see pep-3107
-        # however, since `typeguard` does not support python 2, type checking
-        # is still python 3 only for now
-        annotations = getattr(ctor, '__annotations__', {})
-        has_kwargs = argspec.varkw is not None
-
-    names = [arg for arg in argspec.args if arg != 'self']
-    defaults = argspec.defaults
-    num_defaults = argspec.defaults is not None and len(argspec.defaults) or 0
-    num_required = len(names) - num_defaults
-
-    docs = cls.__doc__
-    if docs is None and getattr(cls, '__category__', None) == 'op':
-        docs = cls.__call__.__doc__
-    try:
-        docstring = doc_parse(docs)
-    except Exception:
-        docstring = None
-
-    if docstring is None:
-        comments = {}
-    else:
-        comments = {}
-        for p in docstring.params:
-            match_obj = re.match('^([a-zA-Z_]+[a-zA-Z_0-9]*).*', p.arg_name)
-            if match_obj is not None:
-                comments[match_obj.group(1)] = p.description
-
-    schema = SchemaDict()
-    schema.name = cls.__name__
-    schema.doc = ""
-    if docs is not None:
-        start_pos = docs[0] == '\n' and 1 or 0
-        schema.doc = docs[start_pos:].split("\n")[0].strip()
-    # XXX handle paddle's weird doc convention
-    if '**' == schema.doc[:2] and '**' == schema.doc[-2:]:
-        schema.doc = schema.doc[2:-2].strip()
-    schema.category = hasattr(cls, '__category__') and getattr(
-        cls, '__category__') or 'module'
-    schema.strict = not has_kwargs
-    schema.pymodule = importlib.import_module(cls.__module__)
-    schema.inject = getattr(cls, '__inject__', [])
-    schema.shared = getattr(cls, '__shared__', [])
-    for idx, name in enumerate(names):
-        comment = name in comments and comments[name] or name
-        if name in schema.inject:
-            type_ = None
-        else:
-            type_ = name in annotations and annotations[name] or None
-        value_schema = SchemaValue(name, comment, type_)
-        if name in schema.shared:
-            assert idx >= num_required, "shared config must have default value"
-            default = defaults[idx - num_required]
-            value_schema.set_default(SharedConfig(name, default))
-        elif idx >= num_required:
-            default = defaults[idx - num_required]
-            value_schema.set_default(default)
-        schema.set_schema(name, value_schema)
-
-    return schema

paddlex/ppdet/core/config/yaml_helpers.py

@@ -1,118 +0,0 @@
-# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import importlib
-import inspect
-
-import yaml
-from .schema import SharedConfig
-
-__all__ = ['serializable', 'Callable']
-
-
-def represent_dictionary_order(self, dict_data):
-    return self.represent_mapping('tag:yaml.org,2002:map', dict_data.items())
-
-
-def setup_orderdict():
-    from collections import OrderedDict
-    yaml.add_representer(OrderedDict, represent_dictionary_order)
-
-
-def _make_python_constructor(cls):
-    def python_constructor(loader, node):
-        if isinstance(node, yaml.SequenceNode):
-            args = loader.construct_sequence(node, deep=True)
-            return cls(*args)
-        else:
-            kwargs = loader.construct_mapping(node, deep=True)
-            try:
-                return cls(**kwargs)
-            except Exception as ex:
-                print("Error when construct {} instance from yaml config".
-                      format(cls.__name__))
-                raise ex
-
-    return python_constructor
-
-
-def _make_python_representer(cls):
-    # python 2 compatibility
-    if hasattr(inspect, 'getfullargspec'):
-        argspec = inspect.getfullargspec(cls)
-    else:
-        argspec = inspect.getfullargspec(cls.__init__)
-    argnames = [arg for arg in argspec.args if arg != 'self']
-
-    def python_representer(dumper, obj):
-        if argnames:
-            data = {name: getattr(obj, name) for name in argnames}
-        else:
-            data = obj.__dict__
-        if '_id' in data:
-            del data['_id']
-        return dumper.represent_mapping(u'!{}'.format(cls.__name__), data)
-
-    return python_representer
-
-
-def serializable(cls):
-    """
-    Add loader and dumper for given class, which must be
-    "trivially serializable"
-
-    Args:
-        cls: class to be serialized
-
-    Returns: cls
-    """
-    yaml.add_constructor(u'!{}'.format(cls.__name__),
-                         _make_python_constructor(cls))
-    yaml.add_representer(cls, _make_python_representer(cls))
-    return cls
-
-
-yaml.add_representer(SharedConfig,
-                     lambda d, o: d.represent_data(o.default_value))
-
-
-@serializable
-class Callable(object):
-    """
-    Helper to be used in Yaml for creating arbitrary class objects
-
-    Args:
-        full_type (str): the full module path to target function
-    """
-
-    def __init__(self, full_type, args=[], kwargs={}):
-        super(Callable, self).__init__()
-        self.full_type = full_type
-        self.args = args
-        self.kwargs = kwargs
-
-    def __call__(self):
-        if '.' in self.full_type:
-            idx = self.full_type.rfind('.')
-            module = importlib.import_module(self.full_type[:idx])
-            func_name = self.full_type[idx + 1:]
-        else:
-            try:
-                module = importlib.import_module('builtins')
-            except Exception:
-                module = importlib.import_module('__builtin__')
-            func_name = self.full_type
-
-        func = getattr(module, func_name)
-        return func(*self.args, **self.kwargs)