pyxllib 0.3.96__py3-none-any.whl → 0.3.197__py3-none-any.whl

pyxlpr/ppocr/data/imaug/text_image_aug/__init__.py

@@ -1,17 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from .augment import tia_perspective, tia_distort, tia_stretch
-
-__all__ = ['tia_distort', 'tia_stretch', 'tia_perspective']

pyxlpr/ppocr/data/imaug/text_image_aug/augment.py

@@ -1,120 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-This code is refer from:
-https://github.com/RubanSeven/Text-Image-Augmentation-python/blob/master/augment.py
-"""
-
-import numpy as np
-from .warp_mls import WarpMLS
-
-
-def tia_distort(src, segment=4):
-    img_h, img_w = src.shape[:2]
-
-    cut = img_w // segment
-    thresh = cut // 3
-
-    src_pts = list()
-    dst_pts = list()
-
-    src_pts.append([0, 0])
-    src_pts.append([img_w, 0])
-    src_pts.append([img_w, img_h])
-    src_pts.append([0, img_h])
-
-    dst_pts.append([np.random.randint(thresh), np.random.randint(thresh)])
-    dst_pts.append(
-        [img_w - np.random.randint(thresh), np.random.randint(thresh)])
-    dst_pts.append(
-        [img_w - np.random.randint(thresh), img_h - np.random.randint(thresh)])
-    dst_pts.append(
-        [np.random.randint(thresh), img_h - np.random.randint(thresh)])
-
-    half_thresh = thresh * 0.5
-
-    for cut_idx in np.arange(1, segment, 1):
-        src_pts.append([cut * cut_idx, 0])
-        src_pts.append([cut * cut_idx, img_h])
-        dst_pts.append([
-            cut * cut_idx + np.random.randint(thresh) - half_thresh,
-            np.random.randint(thresh) - half_thresh
-        ])
-        dst_pts.append([
-            cut * cut_idx + np.random.randint(thresh) - half_thresh,
-            img_h + np.random.randint(thresh) - half_thresh
-        ])
-
-    trans = WarpMLS(src, src_pts, dst_pts, img_w, img_h)
-    dst = trans.generate()
-
-    return dst
-
-
-def tia_stretch(src, segment=4):
-    img_h, img_w = src.shape[:2]
-
-    cut = img_w // segment
-    thresh = cut * 4 // 5
-
-    src_pts = list()
-    dst_pts = list()
-
-    src_pts.append([0, 0])
-    src_pts.append([img_w, 0])
-    src_pts.append([img_w, img_h])
-    src_pts.append([0, img_h])
-
-    dst_pts.append([0, 0])
-    dst_pts.append([img_w, 0])
-    dst_pts.append([img_w, img_h])
-    dst_pts.append([0, img_h])
-
-    half_thresh = thresh * 0.5
-
-    for cut_idx in np.arange(1, segment, 1):
-        move = np.random.randint(thresh) - half_thresh
-        src_pts.append([cut * cut_idx, 0])
-        src_pts.append([cut * cut_idx, img_h])
-        dst_pts.append([cut * cut_idx + move, 0])
-        dst_pts.append([cut * cut_idx + move, img_h])
-
-    trans = WarpMLS(src, src_pts, dst_pts, img_w, img_h)
-    dst = trans.generate()
-
-    return dst
-
-
-def tia_perspective(src):
-    img_h, img_w = src.shape[:2]
-
-    thresh = img_h // 2
-
-    src_pts = list()
-    dst_pts = list()
-
-    src_pts.append([0, 0])
-    src_pts.append([img_w, 0])
-    src_pts.append([img_w, img_h])
-    src_pts.append([0, img_h])
-
-    dst_pts.append([0, np.random.randint(thresh)])
-    dst_pts.append([img_w, np.random.randint(thresh)])
-    dst_pts.append([img_w, img_h - np.random.randint(thresh)])
-    dst_pts.append([0, img_h - np.random.randint(thresh)])
-
-    trans = WarpMLS(src, src_pts, dst_pts, img_w, img_h)
-    dst = trans.generate()
-
-    return dst

pyxlpr/ppocr/data/imaug/text_image_aug/warp_mls.py

@@ -1,168 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-This code is refer from:
-https://github.com/RubanSeven/Text-Image-Augmentation-python/blob/master/warp_mls.py
-"""
-
-import numpy as np
-
-
-class WarpMLS:
-    def __init__(self, src, src_pts, dst_pts, dst_w, dst_h, trans_ratio=1.):
-        self.src = src
-        self.src_pts = src_pts
-        self.dst_pts = dst_pts
-        self.pt_count = len(self.dst_pts)
-        self.dst_w = dst_w
-        self.dst_h = dst_h
-        self.trans_ratio = trans_ratio
-        self.grid_size = 100
-        self.rdx = np.zeros((self.dst_h, self.dst_w))
-        self.rdy = np.zeros((self.dst_h, self.dst_w))
-
-    @staticmethod
-    def __bilinear_interp(x, y, v11, v12, v21, v22):
-        return (v11 * (1 - y) + v12 * y) * (1 - x) + (v21 *
-                                                      (1 - y) + v22 * y) * x
-
-    def generate(self):
-        self.calc_delta()
-        return self.gen_img()
-
-    def calc_delta(self):
-        w = np.zeros(self.pt_count, dtype=np.float32)
-
-        if self.pt_count < 2:
-            return
-
-        i = 0
-        while 1:
-            if self.dst_w <= i < self.dst_w + self.grid_size - 1:
-                i = self.dst_w - 1
-            elif i >= self.dst_w:
-                break
-
-            j = 0
-            while 1:
-                if self.dst_h <= j < self.dst_h + self.grid_size - 1:
-                    j = self.dst_h - 1
-                elif j >= self.dst_h:
-                    break
-
-                sw = 0
-                swp = np.zeros(2, dtype=np.float32)
-                swq = np.zeros(2, dtype=np.float32)
-                new_pt = np.zeros(2, dtype=np.float32)
-                cur_pt = np.array([i, j], dtype=np.float32)
-
-                k = 0
-                for k in range(self.pt_count):
-                    if i == self.dst_pts[k][0] and j == self.dst_pts[k][1]:
-                        break
-
-                    w[k] = 1. / (
-                        (i - self.dst_pts[k][0]) * (i - self.dst_pts[k][0]) +
-                        (j - self.dst_pts[k][1]) * (j - self.dst_pts[k][1]))
-
-                    sw += w[k]
-                    swp = swp + w[k] * np.array(self.dst_pts[k])
-                    swq = swq + w[k] * np.array(self.src_pts[k])
-
-                if k == self.pt_count - 1:
-                    pstar = 1 / sw * swp
-                    qstar = 1 / sw * swq
-
-                    miu_s = 0
-                    for k in range(self.pt_count):
-                        if i == self.dst_pts[k][0] and j == self.dst_pts[k][1]:
-                            continue
-                        pt_i = self.dst_pts[k] - pstar
-                        miu_s += w[k] * np.sum(pt_i * pt_i)
-
-                    cur_pt -= pstar
-                    cur_pt_j = np.array([-cur_pt[1], cur_pt[0]])
-
-                    for k in range(self.pt_count):
-                        if i == self.dst_pts[k][0] and j == self.dst_pts[k][1]:
-                            continue
-
-                        pt_i = self.dst_pts[k] - pstar
-                        pt_j = np.array([-pt_i[1], pt_i[0]])
-
-                        tmp_pt = np.zeros(2, dtype=np.float32)
-                        tmp_pt[0] = np.sum(pt_i * cur_pt) * self.src_pts[k][0] - \
-                                    np.sum(pt_j * cur_pt) * self.src_pts[k][1]
-                        tmp_pt[1] = -np.sum(pt_i * cur_pt_j) * self.src_pts[k][0] + \
-                                    np.sum(pt_j * cur_pt_j) * self.src_pts[k][1]
-                        tmp_pt *= (w[k] / miu_s)
-                        new_pt += tmp_pt
-
-                    new_pt += qstar
-                else:
-                    new_pt = self.src_pts[k]
-
-                self.rdx[j, i] = new_pt[0] - i
-                self.rdy[j, i] = new_pt[1] - j
-
-                j += self.grid_size
-            i += self.grid_size
-
-    def gen_img(self):
-        src_h, src_w = self.src.shape[:2]
-        dst = np.zeros_like(self.src, dtype=np.float32)
-
-        for i in np.arange(0, self.dst_h, self.grid_size):
-            for j in np.arange(0, self.dst_w, self.grid_size):
-                ni = i + self.grid_size
-                nj = j + self.grid_size
-                w = h = self.grid_size
-                if ni >= self.dst_h:
-                    ni = self.dst_h - 1
-                    h = ni - i + 1
-                if nj >= self.dst_w:
-                    nj = self.dst_w - 1
-                    w = nj - j + 1
-
-                di = np.reshape(np.arange(h), (-1, 1))
-                dj = np.reshape(np.arange(w), (1, -1))
-                delta_x = self.__bilinear_interp(
-                    di / h, dj / w, self.rdx[i, j], self.rdx[i, nj],
-                    self.rdx[ni, j], self.rdx[ni, nj])
-                delta_y = self.__bilinear_interp(
-                    di / h, dj / w, self.rdy[i, j], self.rdy[i, nj],
-                    self.rdy[ni, j], self.rdy[ni, nj])
-                nx = j + dj + delta_x * self.trans_ratio
-                ny = i + di + delta_y * self.trans_ratio
-                nx = np.clip(nx, 0, src_w - 1)
-                ny = np.clip(ny, 0, src_h - 1)
-                nxi = np.array(np.floor(nx), dtype=np.int32)
-                nyi = np.array(np.floor(ny), dtype=np.int32)
-                nxi1 = np.array(np.ceil(nx), dtype=np.int32)
-                nyi1 = np.array(np.ceil(ny), dtype=np.int32)
-
-                if len(self.src.shape) == 3:
-                    x = np.tile(np.expand_dims(ny - nyi, axis=-1), (1, 1, 3))
-                    y = np.tile(np.expand_dims(nx - nxi, axis=-1), (1, 1, 3))
-                else:
-                    x = ny - nyi
-                    y = nx - nxi
-                dst[i:i + h, j:j + w] = self.__bilinear_interp(
-                    x, y, self.src[nyi, nxi], self.src[nyi, nxi1],
-                    self.src[nyi1, nxi], self.src[nyi1, nxi1])
-
-        dst = np.clip(dst, 0, 255)
-        dst = np.array(dst, dtype=np.uint8)
-
-        return dst

pyxlpr/ppocr/data/lmdb_dataset.py

@@ -1,115 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import numpy as np
-import os
-from paddle.io import Dataset
-import lmdb
-import cv2
-
-from .imaug import transform, create_operators
-
-
-class LMDBDataSet(Dataset):
-    def __init__(self, config, mode, logger, seed=None):
-        super(LMDBDataSet, self).__init__()
-
-        global_config = config['Global']
-        dataset_config = config[mode]['dataset']
-        loader_config = config[mode]['loader']
-        batch_size = loader_config['batch_size_per_card']
-        data_dir = dataset_config['data_dir']
-        self.do_shuffle = loader_config['shuffle']
-
-        self.lmdb_sets = self.load_hierarchical_lmdb_dataset(data_dir)
-        logger.info("Initialize indexs of datasets:%s" % data_dir)
-        self.data_idx_order_list = self.dataset_traversal()
-        if self.do_shuffle:
-            np.random.shuffle(self.data_idx_order_list)
-        self.ops = create_operators(dataset_config['transforms'], global_config)
-
-    def load_hierarchical_lmdb_dataset(self, data_dir):
-        lmdb_sets = {}
-        dataset_idx = 0
-        for dirpath, dirnames, filenames in os.walk(data_dir + '/'):
-            if not dirnames:
-                env = lmdb.open(
-                    dirpath,
-                    max_readers=32,
-                    readonly=True,
-                    lock=False,
-                    readahead=False,
-                    meminit=False)
-                txn = env.begin(write=False)
-                num_samples = int(txn.get('num-samples'.encode()))
-                lmdb_sets[dataset_idx] = {"dirpath":dirpath, "env":env, \
-                    "txn":txn, "num_samples":num_samples}
-                dataset_idx += 1
-        return lmdb_sets
-
-    def dataset_traversal(self):
-        lmdb_num = len(self.lmdb_sets)
-        total_sample_num = 0
-        for lno in range(lmdb_num):
-            total_sample_num += self.lmdb_sets[lno]['num_samples']
-        data_idx_order_list = np.zeros((total_sample_num, 2))
-        beg_idx = 0
-        for lno in range(lmdb_num):
-            tmp_sample_num = self.lmdb_sets[lno]['num_samples']
-            end_idx = beg_idx + tmp_sample_num
-            data_idx_order_list[beg_idx:end_idx, 0] = lno
-            data_idx_order_list[beg_idx:end_idx, 1] \
-                = list(range(tmp_sample_num))
-            data_idx_order_list[beg_idx:end_idx, 1] += 1
-            beg_idx = beg_idx + tmp_sample_num
-        return data_idx_order_list
-
-    def get_img_data(self, value):
-        """get_img_data"""
-        if not value:
-            return None
-        imgdata = np.frombuffer(value, dtype='uint8')
-        if imgdata is None:
-            return None
-        imgori = cv2.imdecode(imgdata, 1)
-        if imgori is None:
-            return None
-        return imgori
-
-    def get_lmdb_sample_info(self, txn, index):
-        label_key = 'label-%09d'.encode() % index
-        label = txn.get(label_key)
-        if label is None:
-            return None
-        label = label.decode('utf-8')
-        img_key = 'image-%09d'.encode() % index
-        imgbuf = txn.get(img_key)
-        return imgbuf, label
-
-    def __getitem__(self, idx):
-        lmdb_idx, file_idx = self.data_idx_order_list[idx]
-        lmdb_idx = int(lmdb_idx)
-        file_idx = int(file_idx)
-        sample_info = self.get_lmdb_sample_info(self.lmdb_sets[lmdb_idx]['txn'],
-                                                file_idx)
-        if sample_info is None:
-            return self.__getitem__(np.random.randint(self.__len__()))
-        img, label = sample_info
-        data = {'image': img, 'label': label}
-        outs = transform(data, self.ops)
-        if outs is None:
-            return self.__getitem__(np.random.randint(self.__len__()))
-        return outs
-
-    def __len__(self):
-        return self.data_idx_order_list.shape[0]

pyxlpr/ppocr/data/pgnet_dataset.py

@@ -1,104 +0,0 @@
-# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import numpy as np
-import os
-from paddle.io import Dataset
-from .imaug import transform, create_operators
-import random
-
-
-class PGDataSet(Dataset):
-    def __init__(self, config, mode, logger, seed=None):
-        super(PGDataSet, self).__init__()
-
-        self.logger = logger
-        self.seed = seed
-        self.mode = mode
-        global_config = config['Global']
-        dataset_config = config[mode]['dataset']
-        loader_config = config[mode]['loader']
-
-        self.delimiter = dataset_config.get('delimiter', '\t')
-        label_file_list = dataset_config.pop('label_file_list')
-        data_source_num = len(label_file_list)
-        ratio_list = dataset_config.get("ratio_list", [1.0])
-        if isinstance(ratio_list, (float, int)):
-            ratio_list = [float(ratio_list)] * int(data_source_num)
-        assert len(
-            ratio_list
-        ) == data_source_num, "The length of ratio_list should be the same as the file_list."
-        self.data_dir = dataset_config['data_dir']
-        self.do_shuffle = loader_config['shuffle']
-
-        logger.info("Initialize indexs of datasets:%s" % label_file_list)
-        self.data_lines = self.get_image_info_list(label_file_list, ratio_list)
-        self.data_idx_order_list = list(range(len(self.data_lines)))
-        if mode.lower() == "train":
-            self.shuffle_data_random()
-
-        self.ops = create_operators(dataset_config['transforms'], global_config)
-
-    def shuffle_data_random(self):
-        if self.do_shuffle:
-            random.seed(self.seed)
-            random.shuffle(self.data_lines)
-        return
-
-    def get_image_info_list(self, file_list, ratio_list):
-        if isinstance(file_list, str):
-            file_list = [file_list]
-        data_lines = []
-        for idx, file in enumerate(file_list):
-            with open(file, "rb") as f:
-                lines = f.readlines()
-                if self.mode == "train" or ratio_list[idx] < 1.0:
-                    random.seed(self.seed)
-                    lines = random.sample(lines,
-                                          round(len(lines) * ratio_list[idx]))
-                data_lines.extend(lines)
-        return data_lines
-
-    def __getitem__(self, idx):
-        file_idx = self.data_idx_order_list[idx]
-        data_line = self.data_lines[file_idx]
-        img_id = 0
-        try:
-            data_line = data_line.decode('utf-8')
-            substr = data_line.strip("\n").split(self.delimiter)
-            file_name = substr[0]
-            label = substr[1]
-            img_path = os.path.join(self.data_dir, file_name)
-            if self.mode.lower() == 'eval':
-                try:
-                    img_id = int(data_line.split(".")[0][7:])
-                except:
-                    img_id = 0
-            data = {'img_path': img_path, 'label': label, 'img_id': img_id}
-            if not os.path.exists(img_path):
-                raise Exception("{} does not exist!".format(img_path))
-            with open(data['img_path'], 'rb') as f:
-                img = f.read()
-                data['image'] = img
-            outs = transform(data, self.ops)
-        except Exception as e:
-            self.logger.error(
-                "When parsing line {}, error happened with msg: {}".format(
-                    self.data_idx_order_list[idx], e))
-            outs = None
-        if outs is None:
-            return self.__getitem__(np.random.randint(self.__len__()))
-        return outs
-
-    def __len__(self):
-        return len(self.data_idx_order_list)

pyxlpr/ppocr/data/pubtab_dataset.py

@@ -1,107 +0,0 @@
-# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import numpy as np
-import os
-import random
-from paddle.io import Dataset
-import json
-
-from .imaug import transform, create_operators
-
-
-class PubTabDataSet(Dataset):
-    def __init__(self, config, mode, logger, seed=None):
-        super(PubTabDataSet, self).__init__()
-        self.logger = logger
-
-        global_config = config['Global']
-        dataset_config = config[mode]['dataset']
-        loader_config = config[mode]['loader']
-
-        label_file_path = dataset_config.pop('label_file_path')
-
-        self.data_dir = dataset_config['data_dir']
-        self.do_shuffle = loader_config['shuffle']
-        self.do_hard_select = False
-        if 'hard_select' in loader_config:
-            self.do_hard_select = loader_config['hard_select']
-            self.hard_prob = loader_config['hard_prob']
-        if self.do_hard_select:
-            self.img_select_prob = self.load_hard_select_prob()
-        self.table_select_type = None
-        if 'table_select_type' in loader_config:
-            self.table_select_type = loader_config['table_select_type']
-            self.table_select_prob = loader_config['table_select_prob']
-
-        self.seed = seed
-        logger.info("Initialize indexs of datasets:%s" % label_file_path)
-        with open(label_file_path, "rb") as f:
-            self.data_lines = f.readlines()
-        self.data_idx_order_list = list(range(len(self.data_lines)))
-        if mode.lower() == "train":
-            self.shuffle_data_random()
-        self.ops = create_operators(dataset_config['transforms'], global_config)
-
-    def shuffle_data_random(self):
-        if self.do_shuffle:
-            random.seed(self.seed)
-            random.shuffle(self.data_lines)
-        return
-
-    def __getitem__(self, idx):
-        try:
-            data_line = self.data_lines[idx]
-            data_line = data_line.decode('utf-8').strip("\n")
-            info = json.loads(data_line)
-            file_name = info['filename']
-            select_flag = True
-            if self.do_hard_select:
-                prob = self.img_select_prob[file_name]
-                if prob < random.uniform(0, 1):
-                    select_flag = False
-            
-            if self.table_select_type:
-                structure = info['html']['structure']['tokens'].copy()
-                structure_str = ''.join(structure)
-                table_type = "simple"
-                if 'colspan' in structure_str or 'rowspan' in structure_str:
-                    table_type = "complex"
-                if table_type == "complex":
-                    if self.table_select_prob < random.uniform(0, 1):
-                        select_flag = False                    
-            
-            if select_flag:
-                cells = info['html']['cells'].copy()
-                structure = info['html']['structure'].copy()
-                img_path = os.path.join(self.data_dir, file_name)
-                data = {'img_path': img_path, 'cells': cells, 'structure':structure}
-                if not os.path.exists(img_path):
-                    raise Exception("{} does not exist!".format(img_path))
-                with open(data['img_path'], 'rb') as f:
-                    img = f.read()
-                    data['image'] = img
-                outs = transform(data, self.ops)
-            else:
-                outs = None
-        except Exception as e:
-            self.logger.error(
-                "When parsing line {}, error happened with msg: {}".format(
-                    data_line, e))
-            outs = None
-        if outs is None:
-            return self.__getitem__(np.random.randint(self.__len__()))
-        return outs
-
-    def __len__(self):
-        return len(self.data_idx_order_list)