paddlex 2.1.0__py3-none-any.whl → 3.0.0rc0__py3-none-any.whl

paddlex_restful/restful/dataset/det_dataset.py

@@ -1,190 +0,0 @@
-# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os.path as osp
-from ..utils import list_files
-from .utils import is_pic, replace_ext, get_encoding, check_list_txt
-from .datasetbase import DatasetBase
-import xml.etree.ElementTree as ET
-
-
-class DetDataset(DatasetBase):
-    def __init__(self, dataset_id, path):
-        super().__init__(dataset_id, path)
-
-    def check_dataset(self, source_path):
-        if not osp.isdir(osp.join(source_path, 'Annotations')):
-            raise ValueError("标注文件应该放在{}目录下".format(
-                osp.join(source_path, 'Annotations')))
-        if not osp.isdir(osp.join(source_path, 'JPEGImages')):
-            raise ValueError("图片文件应该放在{}目录下".format(
-                osp.join(source_path, 'JPEGImages')))
-
-        self.all_files = list_files(source_path)
-        # 对检测数据集进行统计分析
-        self.file_info = dict()
-        self.label_info = dict()
-
-        if osp.exists(osp.join(source_path, 'train_list.txt')):
-            return self.check_splited_dataset(source_path)
-
-        for f in self.all_files:
-            if not is_pic(f):
-                continue
-            items = osp.split(f)
-            if len(items) == 2 and items[0] == "JPEGImages":
-                anno_name = replace_ext(items[1], "xml")
-                full_anno_path = osp.join(
-                    (osp.join(source_path, 'Annotations')), anno_name)
-                if osp.exists(full_anno_path):
-                    self.file_info[f] = osp.join('Annotations', anno_name)
-
-                # 解析XML文件，获取类别信息
-                try:
-                    tree = ET.parse(full_anno_path)
-                except:
-                    raise Exception("文件{}不是一个良构的xml文件".format(anno_name))
-                objs = tree.findall('object')
-                for i, obj in enumerate(objs):
-                    cname = obj.find('name').text
-                    if cname not in self.label_info:
-                        self.label_info[cname] = list()
-                    if f not in self.label_info[cname]:
-                        self.label_info[cname].append(f)
-
-        self.labels = sorted(self.label_info.keys())
-        for label in self.labels:
-            self.class_train_file_list[label] = list()
-            self.class_val_file_list[label] = list()
-            self.class_test_file_list[label] = list()
-        # 将数据集分析信息dump到本地
-        self.dump_statis_info()
-
-    def check_splited_dataset(self, source_path):
-        labels_txt = osp.join(source_path, "labels.txt")
-        train_list_txt = osp.join(source_path, "train_list.txt")
-        val_list_txt = osp.join(source_path, "val_list.txt")
-        test_list_txt = osp.join(source_path, "test_list.txt")
-        for txt_file in [labels_txt, train_list_txt, val_list_txt]:
-            if not osp.exists(txt_file):
-                raise Exception(
-                    "已切分的数据集下应该包含labels.txt, train_list.txt, val_list.txt文件")
-        check_list_txt([train_list_txt, val_list_txt, test_list_txt])
-
-        self.labels = open(
-            labels_txt, 'r',
-            encoding=get_encoding(labels_txt)).read().strip().split('\n')
-
-        for txt_file in [train_list_txt, val_list_txt, test_list_txt]:
-            if not osp.exists(txt_file):
-                continue
-            with open(txt_file, "r") as f:
-                for line in f:
-                    items = line.strip().split()
-                    img_file, xml_file = [items[0], items[1]]
-
-                    if not osp.isfile(osp.join(source_path, xml_file)):
-                        raise ValueError("数据目录{}中不存在标注文件{}".format(
-                            osp.split(txt_file)[-1], xml_file))
-                    if not osp.isfile(osp.join(source_path, img_file)):
-                        raise ValueError("数据目录{}中不存在图片文件{}".format(
-                            osp.split(txt_file)[-1], img_file))
-                    if not xml_file.split('.')[-1] == 'xml':
-                        raise ValueError("标注文件{}不是xml文件".format(xml_file))
-                    img_file_name = osp.split(img_file)[-1]
-                    if not is_pic(img_file_name) or img_file_name.startswith(
-                            '.'):
-                        raise ValueError("文件{}不是图片格式".format(img_file))
-
-                    self.file_info[img_file] = xml_file
-
-                    if txt_file == train_list_txt:
-                        self.train_files.append(img_file)
-                    elif txt_file == val_list_txt:
-                        self.val_files.append(img_file)
-                    elif txt_file == test_list_txt:
-                        self.test_files.append(img_file)
-
-                    try:
-                        tree = ET.parse(osp.join(source_path, xml_file))
-                    except:
-                        raise Exception("文件{}不是一个良构的xml文件".format(xml_file))
-                    objs = tree.findall('object')
-                    for i, obj in enumerate(objs):
-                        cname = obj.find('name').text
-                        if cname in self.labels:
-                            if cname not in self.label_info:
-                                self.label_info[cname] = list()
-                            if img_file not in self.label_info[cname]:
-                                self.label_info[cname].append(img_file)
-                                if txt_file == train_list_txt:
-                                    if cname in self.class_train_file_list:
-                                        self.class_train_file_list[
-                                            cname].append(img_file)
-                                    else:
-                                        self.class_train_file_list[
-                                            cname] = list()
-                                        self.class_train_file_list[
-                                            cname].append(img_file)
-                                elif txt_file == val_list_txt:
-                                    if cname in self.class_val_file_list:
-                                        self.class_val_file_list[cname].append(
-                                            img_file)
-                                    else:
-                                        self.class_val_file_list[cname] = list(
-                                        )
-                                        self.class_val_file_list[cname].append(
-                                            img_file)
-                                elif txt_file == test_list_txt:
-                                    if cname in self.class_test_file_list:
-                                        self.class_test_file_list[
-                                            cname].append(img_file)
-                                    else:
-                                        self.class_test_file_list[
-                                            cname] = list()
-                                        self.class_test_file_list[
-                                            cname].append(img_file)
-                        else:
-                            raise Exception("文件{}与labels.txt文件信息不对应".format(
-                                xml_file))
-
-        # 将数据集分析信息dump到本地
-        self.dump_statis_info()
-
-    def split(self, val_split, test_split):
-        super().split(val_split, test_split)
-        with open(
-                osp.join(self.path, 'train_list.txt'), mode='w',
-                encoding='utf-8') as f:
-            for x in self.train_files:
-                label = self.file_info[x]
-                f.write('{} {}\n'.format(x, label))
-        with open(
-                osp.join(self.path, 'val_list.txt'), mode='w',
-                encoding='utf-8') as f:
-            for x in self.val_files:
-                label = self.file_info[x]
-                f.write('{} {}\n'.format(x, label))
-        with open(
-                osp.join(self.path, 'test_list.txt'), mode='w',
-                encoding='utf-8') as f:
-            for x in self.test_files:
-                label = self.file_info[x]
-                f.write('{} {}\n'.format(x, label))
-        with open(
-                osp.join(self.path, 'labels.txt'), mode='w',
-                encoding='utf-8') as f:
-            for l in self.labels:
-                f.write('{}\n'.format(l))
-        self.dump_statis_info()

paddlex_restful/restful/dataset/ins_seg_dataset.py

@@ -1,312 +0,0 @@
-# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os.path as osp
-import random
-from ..utils import list_files
-from .utils import is_pic, replace_ext, MyEncoder, read_coco_ann, get_npy_from_coco_json
-from .datasetbase import DatasetBase
-import json
-from pycocotools.coco import COCO
-
-
-class InsSegDataset(DatasetBase):
-    def __init__(self, dataset_id, path):
-        super().__init__(dataset_id, path)
-        self.annotation_dict = None
-
-    def check_dataset(self, source_path):
-        if not osp.isdir(osp.join(source_path, 'JPEGImages')):
-            raise ValueError("图片文件应该放在{}目录下".format(
-                osp.join(source_path, 'JPEGImages')))
-
-        self.all_files = list_files(source_path)
-        # 对检测数据集进行统计分析
-        self.file_info = dict()
-        self.label_info = dict()
-        # 若数据集已切分
-        if osp.exists(osp.join(source_path, 'train.json')):
-            return self.check_splited_dataset(source_path)
-
-        if not osp.exists(osp.join(source_path, 'annotations.json')):
-            raise ValueError("标注文件annotations.json应该放在{}目录下".format(
-                source_path))
-
-        filename_set = set()
-        anno_set = set()
-        for f in self.all_files:
-            items = osp.split(f)
-            if len(items) == 2 and items[0] == "JPEGImages":
-                if not is_pic(f) or f.startswith('.'):
-                    continue
-                filename_set.add(items[1])
-        # 解析包含标注信息的json文件
-        try:
-            coco = COCO(osp.join(source_path, 'annotations.json'))
-            img_ids = coco.getImgIds()
-            cat_ids = coco.getCatIds()
-            anno_ids = coco.getAnnIds()
-
-            catid2clsid = dict({catid: i for i, catid in enumerate(cat_ids)})
-            cid2cname = dict({
-                clsid: coco.loadCats(catid)[0]['name']
-                for catid, clsid in catid2clsid.items()
-            })
-            for img_id in img_ids:
-                img_anno = coco.loadImgs(img_id)[0]
-                img_name = osp.split(img_anno['file_name'])[-1]
-                anno_set.add(img_name)
-                anno_dict = read_coco_ann(img_id, coco, cid2cname, catid2clsid)
-                img_path = osp.join("JPEGImages", img_name)
-                anno_path = osp.join("Annotations", img_name)
-                anno = replace_ext(anno_path, "npy")
-                self.file_info[img_path] = anno
-
-                img_class = list(set(anno_dict["gt_class"]))
-                for category_name in img_class:
-                    if not category_name in self.label_info:
-                        self.label_info[category_name] = [img_path]
-                    else:
-                        self.label_info[category_name].append(img_path)
-
-            for label in sorted(self.label_info.keys()):
-                self.labels.append(label)
-        except:
-            raise Exception("标注文件存在错误")
-
-        if len(anno_set) > len(filename_set):
-            sub_list = list(anno_set - filename_set)
-            raise Exception("标注文件中{}等{}个信息无对应图片".format(sub_list[0],
-                                                        len(sub_list)))
-
-        # 生成每个图片对应的标注信息npy文件
-        npy_path = osp.join(self.path, "Annotations")
-        get_npy_from_coco_json(coco, npy_path, self.file_info)
-
-        for label in self.labels:
-            self.class_train_file_list[label] = list()
-            self.class_val_file_list[label] = list()
-            self.class_test_file_list[label] = list()
-
-        # 将数据集分析信息dump到本地
-        self.dump_statis_info()
-
-    def check_splited_dataset(self, source_path):
-        train_files_json = osp.join(source_path, "train.json")
-        val_files_json = osp.join(source_path, "val.json")
-        test_files_json = osp.join(source_path, "test.json")
-
-        for json_file in [train_files_json, val_files_json]:
-            if not osp.exists(json_file):
-                raise Exception("已切分的数据集下应该包含train.json, val.json文件")
-
-        filename_set = set()
-        anno_set = set()
-        # 获取全部图片名称
-        for f in self.all_files:
-            items = osp.split(f)
-            if len(items) == 2 and items[0] == "JPEGImages":
-                if not is_pic(f) or f.startswith('.'):
-                    continue
-                filename_set.add(items[1])
-
-        img_id_index = 0
-        anno_id_index = 0
-        new_img_list = list()
-        new_cat_list = list()
-        new_anno_list = list()
-        for json_file in [train_files_json, val_files_json, test_files_json]:
-            if not osp.exists(json_file):
-                continue
-            coco = COCO(json_file)
-            img_ids = coco.getImgIds()
-            cat_ids = coco.getCatIds()
-            anno_ids = coco.getAnnIds()
-
-            catid2clsid = dict({catid: i for i, catid in enumerate(cat_ids)})
-            clsid2catid = dict({i: catid for i, catid in enumerate(cat_ids)})
-            cid2cname = dict({
-                clsid: coco.loadCats(catid)[0]['name']
-                for catid, clsid in catid2clsid.items()
-            })
-
-            # 由原train.json中的category生成新的category信息
-            if json_file == train_files_json:
-                cname2catid = dict({
-                    coco.loadCats(catid)[0]['name']: clsid2catid[clsid]
-                    for catid, clsid in catid2clsid.items()
-                })
-                new_cat_list = coco.loadCats(cat_ids)
-            # 获取json中全部标注图片的名字
-            for img_id in img_ids:
-                img_anno = coco.loadImgs(img_id)[0]
-                im_fname = img_anno['file_name']
-                anno_set.add(im_fname)
-
-                if json_file == train_files_json:
-                    self.train_files.append(osp.join("JPEGImages", im_fname))
-                elif json_file == val_files_json:
-                    self.val_files.append(osp.join("JPEGImages", im_fname))
-                elif json_file == test_files_json:
-                    self.test_files.append(osp.join("JPEGImages", im_fname))
-                # 获取每张图片的对应标注信息,并记录为npy格式
-                anno_dict = read_coco_ann(img_id, coco, cid2cname, catid2clsid)
-                img_path = osp.join("JPEGImages", im_fname)
-                anno_path = osp.join("Annotations", im_fname)
-                anno = replace_ext(anno_path, "npy")
-                self.file_info[img_path] = anno
-
-                # 生成label_info
-                img_class = list(set(anno_dict["gt_class"]))
-                for category_name in img_class:
-                    if not category_name in self.label_info:
-                        self.label_info[category_name] = [img_path]
-                    else:
-                        self.label_info[category_name].append(img_path)
-                    if json_file == train_files_json:
-                        if category_name in self.class_train_file_list:
-                            self.class_train_file_list[category_name].append(
-                                img_path)
-                        else:
-                            self.class_train_file_list[category_name] = list()
-                            self.class_train_file_list[category_name].append(
-                                img_path)
-                    elif json_file == val_files_json:
-                        if category_name in self.class_val_file_list:
-                            self.class_val_file_list[category_name].append(
-                                img_path)
-                        else:
-                            self.class_val_file_list[category_name] = list()
-                            self.class_val_file_list[category_name].append(
-                                img_path)
-                    elif json_file == test_files_json:
-                        if category_name in self.class_test_file_list:
-                            self.class_test_file_list[category_name].append(
-                                img_path)
-                        else:
-                            self.class_test_file_list[category_name] = list()
-                            self.class_test_file_list[category_name].append(
-                                img_path)
-
-                # 生成新的图片信息
-                new_img = img_anno
-                new_img["id"] = img_id_index
-                img_id_index += 1
-                new_img_list.append(new_img)
-                # 生成新的标注信息
-                ins_anno_ids = coco.getAnnIds(imgIds=img_id, iscrowd=0)
-                for ins_anno_id in ins_anno_ids:
-                    anno = coco.loadAnns(ins_anno_id)[0]
-                    new_anno = anno
-                    new_anno["image_id"] = new_img["id"]
-                    new_anno["id"] = anno_id_index
-                    anno_id_index += 1
-                    cat = coco.loadCats(anno["category_id"])[0]
-                    new_anno_list.append(new_anno)
-
-        if len(anno_set) > len(filename_set):
-            sub_list = list(anno_set - filename_set)
-            raise Exception("标注文件中{}等{}个信息无对应图片".format(sub_list[0],
-                                                        len(sub_list)))
-
-        for label in sorted(self.label_info.keys()):
-            self.labels.append(label)
-
-        self.annotation_dict = {
-            "images": new_img_list,
-            "categories": new_cat_list,
-            "annotations": new_anno_list
-        }
-
-        # 若原数据集已切分，无annotations.json文件
-        if not osp.exists(osp.join(self.path, "annotations.json")):
-            json_file = open(osp.join(self.path, "annotations.json"), 'w+')
-            json.dump(self.annotation_dict, json_file, cls=MyEncoder)
-            json_file.close()
-
-        # 生成每个图片对应的标注信息npy文件
-        coco = COCO(osp.join(self.path, "annotations.json"))
-        npy_path = osp.join(self.path, "Annotations")
-        get_npy_from_coco_json(coco, npy_path, self.file_info)
-
-        self.dump_statis_info()
-
-    def split(self, val_split, test_split):
-        all_files = list(self.file_info.keys())
-        val_num = int(len(all_files) * val_split)
-        test_num = int(len(all_files) * test_split)
-        train_num = len(all_files) - val_num - test_num
-        assert train_num > 0, "训练集样本数量需大于0"
-        assert val_num > 0, "验证集样本数量需大于0"
-        self.train_files = list()
-        self.val_files = list()
-        self.test_files = list()
-
-        coco = COCO(osp.join(self.path, 'annotations.json'))
-        img_ids = coco.getImgIds()
-        cat_ids = coco.getCatIds()
-        anno_ids = coco.getAnnIds()
-        random.shuffle(img_ids)
-
-        train_files_ids = img_ids[:train_num]
-        val_files_ids = img_ids[train_num:train_num + val_num]
-        test_files_ids = img_ids[train_num + val_num:]
-
-        for img_id_list in [train_files_ids, val_files_ids, test_files_ids]:
-            img_anno_ids = coco.getAnnIds(imgIds=img_id_list, iscrowd=0)
-            imgs = coco.loadImgs(img_id_list)
-            instances = coco.loadAnns(img_anno_ids)
-            categories = coco.loadCats(cat_ids)
-            img_dict = {
-                "annotations": instances,
-                "images": imgs,
-                "categories": categories
-            }
-
-            if img_id_list == train_files_ids:
-                for img in imgs:
-                    self.train_files.append(
-                        osp.join("JPEGImages", img["file_name"]))
-                json_file = open(osp.join(self.path, 'train.json'), 'w+')
-                json.dump(img_dict, json_file, cls=MyEncoder)
-            elif img_id_list == val_files_ids:
-                for img in imgs:
-                    self.val_files.append(
-                        osp.join("JPEGImages", img["file_name"]))
-                json_file = open(osp.join(self.path, 'val.json'), 'w+')
-                json.dump(img_dict, json_file, cls=MyEncoder)
-            elif img_id_list == test_files_ids:
-                for img in imgs:
-                    self.test_files.append(
-                        osp.join("JPEGImages", img["file_name"]))
-                json_file = open(osp.join(self.path, 'test.json'), 'w+')
-                json.dump(img_dict, json_file, cls=MyEncoder)
-
-            self.train_set = set(self.train_files)
-            self.val_set = set(self.val_files)
-            self.test_set = set(self.test_files)
-
-            for label, file_list in self.label_info.items():
-                self.class_train_file_list[label] = list()
-                self.class_val_file_list[label] = list()
-                self.class_test_file_list[label] = list()
-                for f in file_list:
-                    if f in self.test_set:
-                        self.class_test_file_list[label].append(f)
-                    if f in self.val_set:
-                        self.class_val_file_list[label].append(f)
-                    if f in self.train_set:
-                        self.class_train_file_list[label].append(f)
-
-        self.dump_statis_info()

paddlex_restful/restful/dataset/operate.py

@@ -1,155 +0,0 @@
-# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import pickle
-import traceback
-import os.path as osp
-import multiprocessing as mp
-from .cls_dataset import ClsDataset
-from .det_dataset import DetDataset
-from .seg_dataset import SegDataset
-from .ins_seg_dataset import InsSegDataset
-from ..utils import set_folder_status, get_folder_status, DatasetStatus, DownloadStatus, download, list_files
-
-dataset_url_list = [
-    'https://bj.bcebos.com/paddlex/demos/vegetables_cls.tar.gz',
-    'https://bj.bcebos.com/paddlex/demos/insect_det.tar.gz',
-    'https://bj.bcebos.com/paddlex/demos/optic_disc_seg.tar.gz',
-    'https://bj.bcebos.com/paddlex/demos/xiaoduxiong_ins_det.tar.gz',
-    'https://bj.bcebos.com/paddlex/demos/remote_sensing_seg.tar.gz'
-]
-dataset_url_dict = {
-    'classification':
-    'https://bj.bcebos.com/paddlex/demos/vegetables_cls.tar.gz',
-    'detection': 'https://bj.bcebos.com/paddlex/demos/insect_det.tar.gz',
-    'segmentation':
-    'https://bj.bcebos.com/paddlex/demos/optic_disc_seg.tar.gz',
-    'instance_segmentation':
-    'https://bj.bcebos.com/paddlex/demos/xiaoduxiong_ins_det.tar.gz'
-}
-
-
-def _check_and_copy(dataset, dataset_path, source_path):
-    try:
-        dataset.check_dataset(source_path)
-    except Exception as e:
-        error_info = traceback.format_exc()
-        set_folder_status(dataset_path, DatasetStatus.XCHECKFAIL, error_info)
-        return
-    set_folder_status(dataset_path, DatasetStatus.XCOPYING, os.getpid())
-    try:
-        dataset.copy_dataset(source_path, dataset.all_files)
-    except Exception as e:
-        error_info = traceback.format_exc()
-        set_folder_status(dataset_path, DatasetStatus.XCOPYFAIL, error_info)
-        return
-    # 若上传已切分好的数据集
-    if len(dataset.train_files) != 0:
-        set_folder_status(dataset_path, DatasetStatus.XSPLITED)
-
-
-def import_dataset(dataset_id, dataset_type, dataset_path, source_path):
-    if not osp.isdir(dataset_path):
-        if osp.exists(dataset_path):
-            os.remove(dataset_path)
-        os.makedirs(dataset_path)
-
-    set_folder_status(dataset_path, DatasetStatus.XCHECKING)
-    if dataset_type == 'classification':
-        ds = ClsDataset(dataset_id, dataset_path)
-    elif dataset_type == 'detection':
-        ds = DetDataset(dataset_id, dataset_path)
-    elif dataset_type == 'segmentation':
-        ds = SegDataset(dataset_id, dataset_path)
-    elif dataset_type == 'instance_segmentation':
-        ds = InsSegDataset(dataset_id, dataset_path)
-    p = mp.Process(
-        target=_check_and_copy, args=(ds, dataset_path, source_path))
-    p.start()
-    return p
-
-
-def _download_proc(url, target_path, dataset_type):
-    # 下载数据集压缩包
-    from paddlex.utils import decompress
-    target_path = osp.join(target_path, dataset_type)
-    fname = download(url, target_path)
-    # 解压
-    decompress(fname)
-    set_folder_status(target_path, DownloadStatus.XDDECOMPRESSED)
-
-
-def download_demo_dataset(prj_type, target_path):
-    url = dataset_url_list[prj_type.value]
-    dataset_type = prj_type.name
-    p = mp.Process(
-        target=_download_proc, args=(url, target_path, dataset_type))
-    p.start()
-    return p
-
-
-def get_dataset_status(dataset_id, dataset_type, dataset_path):
-    status, message = get_folder_status(dataset_path, True)
-    if status is None:
-        status = DatasetStatus.XEMPTY
-    if status == DatasetStatus.XCOPYING:
-        items = message.strip().split()
-        if len(items) < 2:
-            percent = 0.0
-        else:
-            pid = int(items[0])
-            if int(items[1]) == 0:
-                percent = 1.0
-            else:
-                copyed_files_num = len(list_files(dataset_path)) - 1
-                percent = copyed_files_num * 1.0 / int(items[1])
-        message = {'pid': pid, 'percent': percent}
-    if status == DatasetStatus.XCOPYDONE or status == DatasetStatus.XSPLITED:
-        if not osp.exists(osp.join(dataset_path, 'statis.pkl')):
-            p = import_dataset(dataset_id, dataset_type, dataset_path,
-                               dataset_path)
-            status = DatasetStatus.XCHECKING
-    return status, message
-
-
-def split_dataset(dataset_id, dataset_type, dataset_path, val_split,
-                  test_split):
-    status, message = get_folder_status(dataset_path, True)
-    if status != DatasetStatus.XCOPYDONE and status != DatasetStatus.XSPLITED:
-        raise Exception("数据集还未导入完成，请等数据集导入成功后再进行切分")
-    if not osp.exists(osp.join(dataset_path, 'statis.pkl')):
-        raise Exception("数据集需重新校验，请刷新数据集后再进行切分")
-
-    if dataset_type == 'classification':
-        ds = ClsDataset(dataset_id, dataset_path)
-    elif dataset_type == 'detection':
-        ds = DetDataset(dataset_id, dataset_path)
-    elif dataset_type == 'segmentation':
-        ds = SegDataset(dataset_id, dataset_path)
-    elif dataset_type == 'instance_segmentation':
-        ds = InsSegDataset(dataset_id, dataset_path)
-
-    ds.load_statis_info()
-    ds.split(val_split, test_split)
-    set_folder_status(dataset_path, DatasetStatus.XSPLITED)
-
-
-def get_dataset_details(dataset_path):
-    status, message = get_folder_status(dataset_path, True)
-    if status == DatasetStatus.XCOPYDONE or status == DatasetStatus.XSPLITED:
-        with open(osp.join(dataset_path, 'statis.pkl'), 'rb') as f:
-            details = pickle.load(f)
-        return details
-    return None