pyxllib 0.3.96__py3-none-any.whl → 0.3.197__py3-none-any.whl

pyxlpr/ppocr/utils/profiler.py

@@ -1,110 +0,0 @@
-# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-import paddle
-
-# A global variable to record the number of calling times for profiler
-# functions. It is used to specify the tracing range of training steps.
-_profiler_step_id = 0
-
-# A global variable to avoid parsing from string every time.
-_profiler_options = None
-
-
-class ProfilerOptions(object):
-    '''
-    Use a string to initialize a ProfilerOptions.
-    The string should be in the format: "key1=value1;key2=value;key3=value3".
-    For example:
-      "profile_path=model.profile"
-      "batch_range=[50, 60]; profile_path=model.profile"
-      "batch_range=[50, 60]; tracer_option=OpDetail; profile_path=model.profile"
-    ProfilerOptions supports following key-value pair:
-      batch_range      - a integer list, e.g. [100, 110].
-      state            - a string, the optional values are 'CPU', 'GPU' or 'All'. 
-      sorted_key       - a string, the optional values are 'calls', 'total',
-                         'max', 'min' or 'ave.
-      tracer_option    - a string, the optional values are 'Default', 'OpDetail',
-                         'AllOpDetail'.
-      profile_path     - a string, the path to save the serialized profile data,
-                         which can be used to generate a timeline.
-      exit_on_finished - a boolean.
-    '''
-
-    def __init__(self, options_str):
-        assert isinstance(options_str, str)
-
-        self._options = {
-            'batch_range': [10, 20],
-            'state': 'All',
-            'sorted_key': 'total',
-            'tracer_option': 'Default',
-            'profile_path': '/tmp/profile',
-            'exit_on_finished': True
-        }
-        self._parse_from_string(options_str)
-
-    def _parse_from_string(self, options_str):
-        for kv in options_str.replace(' ', '').split(';'):
-            key, value = kv.split('=')
-            if key == 'batch_range':
-                value_list = value.replace('[', '').replace(']', '').split(',')
-                value_list = list(map(int, value_list))
-                if len(value_list) >= 2 and value_list[0] >= 0 and value_list[
-                        1] > value_list[0]:
-                    self._options[key] = value_list
-            elif key == 'exit_on_finished':
-                self._options[key] = value.lower() in ("yes", "true", "t", "1")
-            elif key in [
-                    'state', 'sorted_key', 'tracer_option', 'profile_path'
-            ]:
-                self._options[key] = value
-
-    def __getitem__(self, name):
-        if self._options.get(name, None) is None:
-            raise ValueError(
-                "ProfilerOptions does not have an option named %s." % name)
-        return self._options[name]
-
-
-def add_profiler_step(options_str=None):
-    '''
-    Enable the operator-level timing using PaddlePaddle's profiler.
-    The profiler uses a independent variable to count the profiler steps.
-    One call of this function is treated as a profiler step.
-    
-    Args:
-      profiler_options - a string to initialize the ProfilerOptions.
-                         Default is None, and the profiler is disabled.
-    '''
-    if options_str is None:
-        return
-
-    global _profiler_step_id
-    global _profiler_options
-
-    if _profiler_options is None:
-        _profiler_options = ProfilerOptions(options_str)
-
-    if _profiler_step_id == _profiler_options['batch_range'][0]:
-        paddle.utils.profiler.start_profiler(
-            _profiler_options['state'], _profiler_options['tracer_option'])
-    elif _profiler_step_id == _profiler_options['batch_range'][1]:
-        paddle.utils.profiler.stop_profiler(_profiler_options['sorted_key'],
-                                            _profiler_options['profile_path'])
-        if _profiler_options['exit_on_finished']:
-            sys.exit(0)
-
-    _profiler_step_id += 1

pyxlpr/ppocr/utils/save_load.py

@@ -1,150 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import errno
-import os
-import pickle
-import six
-
-import paddle
-
-from pyxlpr.ppocr.utils.logging import get_logger
-
-__all__ = ['load_model']
-
-
-def _mkdir_if_not_exist(path, logger):
-    """
-    mkdir if not exists, ignore the exception when multiprocess mkdir together
-    """
-    if not os.path.exists(path):
-        try:
-            os.makedirs(path)
-        except OSError as e:
-            if e.errno == errno.EEXIST and os.path.isdir(path):
-                logger.warning(
-                    'be happy if some process has already created {}'.format(
-                        path))
-            else:
-                raise OSError('Failed to mkdir {}'.format(path))
-
-
-def load_model(config, model, optimizer=None):
-    """
-    load model from checkpoint or pretrained_model
-    """
-    logger = get_logger()
-    global_config = config['Global']
-    checkpoints = global_config.get('checkpoints')
-    pretrained_model = global_config.get('pretrained_model')
-    best_model_dict = {}
-    if checkpoints:
-        if checkpoints.endswith('.pdparams'):
-            checkpoints = checkpoints.replace('.pdparams', '')
-        assert os.path.exists(checkpoints + ".pdparams"), \
-            "The {}.pdparams does not exists!".format(checkpoints)
-
-        # load params from trained model
-        params = paddle.load(checkpoints + '.pdparams')
-        state_dict = model.state_dict()
-        new_state_dict = {}
-        for key, value in state_dict.items():
-            if key not in params:
-                logger.warning("{} not in loaded params {} !".format(
-                    key, params.keys()))
-                continue
-            pre_value = params[key]
-            if list(value.shape) == list(pre_value.shape):
-                new_state_dict[key] = pre_value
-            else:
-                logger.warning(
-                    "The shape of model params {} {} not matched with loaded params shape {} !".
-                        format(key, value.shape, pre_value.shape))
-        model.set_state_dict(new_state_dict)
-
-        if optimizer is not None:
-            if os.path.exists(checkpoints + '.pdopt'):
-                optim_dict = paddle.load(checkpoints + '.pdopt')
-                optimizer.set_state_dict(optim_dict)
-            else:
-                logger.warning(
-                    "{}.pdopt is not exists, params of optimizer is not loaded".
-                        format(checkpoints))
-
-        if os.path.exists(checkpoints + '.states'):
-            with open(checkpoints + '.states', 'rb') as f:
-                states_dict = pickle.load(f) if six.PY2 else pickle.load(
-                    f, encoding='latin1')
-            best_model_dict = states_dict.get('best_model_dict', {})
-            if 'epoch' in states_dict:
-                best_model_dict['start_epoch'] = states_dict['epoch'] + 1
-        logger.info("resume from {}".format(checkpoints))
-    elif pretrained_model:
-        load_pretrained_params(model, pretrained_model)
-    else:
-        logger.info('train from scratch')
-    return best_model_dict
-
-
-def load_pretrained_params(model, path):
-    logger = get_logger()
-    if path.endswith('.pdparams'):
-        path = path.replace('.pdparams', '')
-    assert os.path.exists(path + ".pdparams"), \
-        "The {}.pdparams does not exists!".format(path)
-
-    params = paddle.load(path + '.pdparams')
-    state_dict = model.state_dict()
-    new_state_dict = {}
-    for k1 in params.keys():
-        if k1 not in state_dict.keys():
-            logger.warning("The pretrained params {} not in model".format(k1))
-        else:
-            if list(state_dict[k1].shape) == list(params[k1].shape):
-                new_state_dict[k1] = params[k1]
-            else:
-                logger.warning(
-                    "The shape of model params {} {} not matched with loaded params {} {} !".
-                        format(k1, state_dict[k1].shape, k1, params[k1].shape))
-    model.set_state_dict(new_state_dict)
-    logger.info("load pretrain successful from {}".format(path))
-    return model
-
-
-def save_model(model,
-               optimizer,
-               model_path,
-               logger,
-               is_best=False,
-               prefix='ppocr',
-               **kwargs):
-    """
-    save model to the target path
-    """
-    _mkdir_if_not_exist(model_path, logger)
-    model_prefix = os.path.join(model_path, prefix)
-    paddle.save(model.state_dict(), model_prefix + '.pdparams')
-    paddle.save(optimizer.state_dict(), model_prefix + '.pdopt')
-
-    # save metric and config
-    with open(model_prefix + '.states', 'wb') as f:
-        pickle.dump(kwargs, f, protocol=2)
-    if is_best:
-        logger.info('save best model is to {}'.format(model_prefix))
-    else:
-        logger.info("save model in {}".format(model_prefix))

pyxlpr/ppocr/utils/stats.py

@@ -1,72 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import collections
-import numpy as np
-import datetime
-
-__all__ = ['TrainingStats', 'Time']
-
-
-class SmoothedValue(object):
-    """Track a series of values and provide access to smoothed values over a
-    window or the global series average.
-    """
-
-    def __init__(self, window_size):
-        self.deque = collections.deque(maxlen=window_size)
-
-    def add_value(self, value):
-        self.deque.append(value)
-
-    def get_median_value(self):
-        return np.median(self.deque)
-
-
-def Time():
-    return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
-
-
-class TrainingStats(object):
-    def __init__(self, window_size, stats_keys):
-        self.window_size = window_size
-        self.smoothed_losses_and_metrics = {
-            key: SmoothedValue(window_size)
-            for key in stats_keys
-        }
-
-    def update(self, stats):
-        for k, v in stats.items():
-            if k not in self.smoothed_losses_and_metrics:
-                self.smoothed_losses_and_metrics[k] = SmoothedValue(
-                    self.window_size)
-            self.smoothed_losses_and_metrics[k].add_value(v)
-
-    def get(self, extras=None):
-        stats = collections.OrderedDict()
-        if extras:
-            for k, v in extras.items():
-                stats[k] = v
-        for k, v in self.smoothed_losses_and_metrics.items():
-            stats[k] = round(v.get_median_value(), 6)
-
-        return stats
-
-    def log(self, extras=None):
-        d = self.get(extras)
-        strs = []
-        for k, v in d.items():
-            strs.append('{}: {:x<6f}'.format(k, v))
-        strs = ', '.join(strs)
-        return strs

pyxlpr/ppocr/utils/utility.py

@@ -1,80 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-import os
-import imghdr
-import cv2
-
-
-def print_dict(d, logger, delimiter=0):
-    """
-    Recursively visualize a dict and
-    indenting acrrording by the relationship of keys.
-    """
-    for k, v in sorted(d.items()):
-        if isinstance(v, dict):
-            logger.info("{}{} : ".format(delimiter * " ", str(k)))
-            print_dict(v, logger, delimiter + 4)
-        elif isinstance(v, list) and len(v) >= 1 and isinstance(v[0], dict):
-            logger.info("{}{} : ".format(delimiter * " ", str(k)))
-            for value in v:
-                print_dict(value, logger, delimiter + 4)
-        else:
-            logger.info("{}{} : {}".format(delimiter * " ", k, v))
-
-
-def get_check_global_params(mode):
-    check_params = ['use_gpu', 'max_text_length', 'image_shape', \
-                    'image_shape', 'character_type', 'loss_type']
-    if mode == "train_eval":
-        check_params = check_params + [ \
-            'train_batch_size_per_card', 'test_batch_size_per_card']
-    elif mode == "test":
-        check_params = check_params + ['test_batch_size_per_card']
-    return check_params
-
-
-def get_image_file_list(img_file):
-    imgs_lists = []
-    if img_file is None or not os.path.exists(img_file):
-        raise Exception("not found any img file in {}".format(img_file))
-
-    img_end = {'jpg', 'bmp', 'png', 'jpeg', 'rgb', 'tif', 'tiff', 'gif', 'GIF'}
-    if os.path.isfile(img_file) and imghdr.what(img_file) in img_end:
-        imgs_lists.append(img_file)
-    elif os.path.isdir(img_file):
-        for single_file in os.listdir(img_file):
-            file_path = os.path.join(img_file, single_file)
-            if os.path.isfile(file_path) and imghdr.what(file_path) in img_end:
-                imgs_lists.append(file_path)
-    if len(imgs_lists) == 0:
-        raise Exception("not found any img file in {}".format(img_file))
-    imgs_lists = sorted(imgs_lists)
-    return imgs_lists
-
-
-def check_and_read_gif(img_path):
-    if os.path.basename(img_path)[-3:] in ['gif', 'GIF']:
-        gif = cv2.VideoCapture(img_path)
-        ret, frame = gif.read()
-        if not ret:
-            logger = logging.getLogger('ppocr')
-            logger.info("Cannot read {}. This gif image maybe corrupted.")
-            return None, False
-        if len(frame.shape) == 2 or frame.shape[-1] == 1:
-            frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
-        imgvalue = frame[:, :, ::-1]
-        return imgvalue, True
-    return None, False

pyxlpr/ppstructure/__init__.py

@@ -1,13 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

pyxlpr/ppstructure/predict_system.py

@@ -1,187 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import sys
-import subprocess
-
-__dir__ = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(__dir__)
-sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
-
-os.environ["FLAGS_allocator_strategy"] = 'auto_growth'
-import cv2
-import numpy as np
-import time
-import logging
-
-from pyxlpr.ppocr.utils.utility import get_image_file_list, check_and_read_gif
-from pyxlpr.ppocr.utils.logging import get_logger
-from pyxlpr.ppocr.tools.infer.predict_system import TextSystem
-from pyxlpr.ppstructure.table.predict_table import TableSystem, to_excel
-from pyxlpr.ppstructure.utility import parse_args, draw_structure_result
-
-logger = get_logger()
-
-
-class OCRSystem(object):
-    def __init__(self, args):
-        self.mode = args.mode
-        if self.mode == 'structure':
-            import layoutparser as lp
-            # args.det_limit_type = 'resize_long'
-            args.drop_score = 0
-            if not args.show_log:
-                logger.setLevel(logging.INFO)
-            self.text_system = TextSystem(args)
-            self.table_system = TableSystem(args,
-                                            self.text_system.text_detector,
-                                            self.text_system.text_recognizer)
-
-            config_path = None
-            model_path = None
-            if os.path.isdir(args.layout_path_model):
-                model_path = args.layout_path_model
-            else:
-                config_path = args.layout_path_model
-            self.table_layout = lp.PaddleDetectionLayoutModel(
-                config_path=config_path,
-                model_path=model_path,
-                threshold=0.5,
-                enable_mkldnn=args.enable_mkldnn,
-                enforce_cpu=not args.use_gpu,
-                thread_num=args.cpu_threads)
-            self.use_angle_cls = args.use_angle_cls
-            self.drop_score = args.drop_score
-        elif self.mode == 'vqa':
-            from pyxlpr.ppstructure.vqa.infer_ser_e2e import SerPredictor, draw_ser_results
-            self.vqa_engine = SerPredictor(args)
-
-    def __call__(self, img):
-        if self.mode == 'structure':
-            ori_im = img.copy()
-            layout_res = self.table_layout.detect(img[..., ::-1])
-            res_list = []
-            for region in layout_res:
-                x1, y1, x2, y2 = region.coordinates
-                x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
-                roi_img = ori_im[y1:y2, x1:x2, :]
-                if region.type == 'Table':
-                    res = self.table_system(roi_img)
-                else:
-                    filter_boxes, filter_rec_res = self.text_system(roi_img)
-                    filter_boxes = [x + [x1, y1] for x in filter_boxes]
-                    filter_boxes = [
-                        x.reshape(-1).tolist() for x in filter_boxes
-                    ]
-                    # remove style char
-                    style_token = [
-                        '<strike>', '<strike>', '<sup>', '</sub>', '<b>',
-                        '</b>', '<sub>', '</sup>', '<overline>', '</overline>',
-                        '<underline>', '</underline>', '<i>', '</i>'
-                    ]
-                    filter_rec_res_tmp = []
-                    for rec_res in filter_rec_res:
-                        rec_str, rec_conf = rec_res
-                        for token in style_token:
-                            if token in rec_str:
-                                rec_str = rec_str.replace(token, '')
-                        filter_rec_res_tmp.append((rec_str, rec_conf))
-                    res = (filter_boxes, filter_rec_res_tmp)
-                res_list.append({
-                    'type': region.type,
-                    'bbox': [x1, y1, x2, y2],
-                    'img': roi_img,
-                    'res': res
-                })
-        elif self.mode == 'vqa':
-            res_list, _ = self.vqa_engine(img)
-        return res_list
-
-
-def save_structure_res(res, save_folder, img_name):
-    excel_save_folder = os.path.join(save_folder, img_name)
-    os.makedirs(excel_save_folder, exist_ok=True)
-    # save res
-    with open(
-            os.path.join(excel_save_folder, 'res.txt'), 'w',
-            encoding='utf8') as f:
-        for region in res:
-            if region['type'] == 'Table':
-                excel_path = os.path.join(excel_save_folder,
-                                          '{}.xlsx'.format(region['bbox']))
-                to_excel(region['res'], excel_path)
-            if region['type'] == 'Figure':
-                roi_img = region['img']
-                img_path = os.path.join(excel_save_folder,
-                                        '{}.jpg'.format(region['bbox']))
-                cv2.imwrite(img_path, roi_img)
-            else:
-                for box, rec_res in zip(region['res'][0], region['res'][1]):
-                    f.write('{}\t{}\n'.format(
-                        np.array(box).reshape(-1).tolist(), rec_res))
-
-
-def main(args):
-    image_file_list = get_image_file_list(args.image_dir)
-    image_file_list = image_file_list
-    image_file_list = image_file_list[args.process_id::args.total_process_num]
-
-    structure_sys = OCRSystem(args)
-    img_num = len(image_file_list)
-    save_folder = os.path.join(args.output, structure_sys.mode)
-    os.makedirs(save_folder, exist_ok=True)
-
-    for i, image_file in enumerate(image_file_list):
-        logger.info("[{}/{}] {}".format(i, img_num, image_file))
-        img, flag = check_and_read_gif(image_file)
-        img_name = os.path.basename(image_file).split('.')[0]
-
-        if not flag:
-            img = cv2.imread(image_file)
-        if img is None:
-            logger.error("error in loading image:{}".format(image_file))
-            continue
-        starttime = time.time()
-        res = structure_sys(img)
-
-        if structure_sys.mode == 'structure':
-            save_structure_res(res, save_folder, img_name)
-            draw_img = draw_structure_result(img, res, args.vis_font_path)
-            img_save_path = os.path.join(save_folder, img_name, 'show.jpg')
-        elif structure_sys.mode == 'vqa':
-            draw_img = draw_ser_results(img, res, args.vis_font_path)
-            img_save_path = os.path.join(save_folder, img_name + '.jpg')
-        cv2.imwrite(img_save_path, draw_img)
-        logger.info('result save to {}'.format(img_save_path))
-        elapse = time.time() - starttime
-        logger.info("Predict time : {:.3f}s".format(elapse))
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    if args.use_mp:
-        p_list = []
-        total_process_num = args.total_process_num
-        for process_id in range(total_process_num):
-            cmd = [sys.executable, "-u"] + sys.argv + [
-                "--process_id={}".format(process_id),
-                "--use_mp={}".format(False)
-            ]
-            p = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stdout)
-            p_list.append(p)
-        for p in p_list:
-            p.wait()
-    else:
-        main(args)

pyxlpr/ppstructure/table/__init__.py

@@ -1,13 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.