paddlex 2.1.0__py3-none-any.whl → 3.0.0b2__py3-none-any.whl

paddlex-3.0.0b2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

paddlex-3.0.0b2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+paddlex = paddlex.__main__:console_entry

paddlex-3.0.0b2.dist-info/top_level.txt

@@ -0,0 +1 @@
+paddlex

PaddleClas/__init__.py

@@ -1,16 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-__all__ = ['PaddleClas']
-from .paddleclas import PaddleClas

PaddleClas/deploy/__init__.py

@@ -1 +0,0 @@
-from . import utils

PaddleClas/deploy/paddleserving/classification_web_service.py

@@ -1,74 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import sys
-from paddle_serving_app.reader import Sequential, URL2Image, Resize, CenterCrop, RGB2BGR, Transpose, Div, Normalize, Base64ToImage
-try:
-    from paddle_serving_server_gpu.web_service import WebService, Op
-except ImportError:
-    from paddle_serving_server.web_service import WebService, Op
-import logging
-import numpy as np
-import base64, cv2
-
-
-class ImagenetOp(Op):
-    def init_op(self):
-        self.seq = Sequential([
-            Resize(256), CenterCrop(224), RGB2BGR(), Transpose((2, 0, 1)),
-            Div(255), Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225],
-                                True)
-        ])
-        self.label_dict = {}
-        label_idx = 0
-        with open("imagenet.label") as fin:
-            for line in fin:
-                self.label_dict[label_idx] = line.strip()
-                label_idx += 1
-
-    def preprocess(self, input_dicts, data_id, log_id):
-        (_, input_dict), = input_dicts.items()
-        batch_size = len(input_dict.keys())
-        imgs = []
-        for key in input_dict.keys():
-            data = base64.b64decode(input_dict[key].encode('utf8'))
-            data = np.fromstring(data, np.uint8)
-            im = cv2.imdecode(data, cv2.IMREAD_COLOR)
-            img = self.seq(im)
-            imgs.append(img[np.newaxis, :].copy())
-        input_imgs = np.concatenate(imgs, axis=0)
-        return {"inputs": input_imgs}, False, None, ""
-
-    def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
-        score_list = fetch_dict["prediction"]
-        result = {"label": [], "prob": []}
-        for score in score_list:
-            score = score.tolist()
-            max_score = max(score)
-            result["label"].append(self.label_dict[score.index(max_score)]
-                                   .strip().replace(",", ""))
-            result["prob"].append(max_score)
-        result["label"] = str(result["label"])
-        result["prob"] = str(result["prob"])
-        return result, None, ""
-
-
-class ImageService(WebService):
-    def get_pipeline_response(self, read_op):
-        image_op = ImagenetOp(name="imagenet", input_ops=[read_op])
-        return image_op
-
-
-uci_service = ImageService(name="imagenet")
-uci_service.prepare_pipeline_config("config.yml")
-uci_service.run_service()

PaddleClas/deploy/paddleserving/cpu_utilization.py

@@ -1,4 +0,0 @@
-import psutil
-cpu_utilization=psutil.cpu_percent(1,False)
-print('CPU_UTILIZATION:', cpu_utilization)
-

PaddleClas/deploy/paddleserving/pipeline_http_client.py

@@ -1,20 +0,0 @@
-import requests
-import json
-import base64
-import os
-
-
-def cv2_to_base64(image):
-    return base64.b64encode(image).decode('utf8')
-
-
-if __name__ == "__main__":
-    url = "http://127.0.0.1:18080/imagenet/prediction"
-    with open(os.path.join(".", "daisy.jpg"), 'rb') as file:
-        image_data1 = file.read()
-    image = cv2_to_base64(image_data1)
-
-    data = {"key": ["image"], "value": [image]}
-    for i in range(1):
-        r = requests.post(url=url, data=json.dumps(data))
-        print(r.json())

PaddleClas/deploy/paddleserving/pipeline_rpc_client.py

@@ -1,33 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-try:
-    from paddle_serving_server_gpu.pipeline import PipelineClient
-except ImportError:
-    from paddle_serving_server.pipeline import PipelineClient
-import base64
-
-client = PipelineClient()
-client.connect(['127.0.0.1:9993'])
-
-def cv2_to_base64(image):
-    return base64.b64encode(image).decode('utf8')
-
-if __name__ == "__main__":
-    with open("daisy.jpg", 'rb') as file:
-        image_data = file.read()
-    image = cv2_to_base64(image_data)
-
-    for i in range(1):
-        ret = client.predict(feed_dict={"image": image}, fetch=["label", "prob"])
-        print(ret)

PaddleClas/deploy/paddleserving/recognition/pipeline_http_client.py

@@ -1,21 +0,0 @@
-import requests
-import json
-import base64
-import os
-
-imgpath = "../../drink_dataset_v1.0/test_images/001.jpeg"
-
-def cv2_to_base64(image):
-    return base64.b64encode(image).decode('utf8')
-
-if __name__ == "__main__":
-    url = "http://127.0.0.1:18081/recognition/prediction"
-
-    with open(os.path.join(".",  imgpath), 'rb') as file:
-        image_data1 = file.read()
-    image = cv2_to_base64(image_data1)
-    data = {"key": ["image"], "value": [image]}
-
-    for i in range(1):
-        r = requests.post(url=url, data=json.dumps(data))
-        print(r.json())

PaddleClas/deploy/paddleserving/recognition/pipeline_rpc_client.py

@@ -1,34 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-try:
-    from paddle_serving_server_gpu.pipeline import PipelineClient
-except ImportError:
-    from paddle_serving_server.pipeline import PipelineClient
-import base64
-
-client = PipelineClient()
-client.connect(['127.0.0.1:9994'])
-imgpath = "../../drink_dataset_v1.0/test_images/001.jpeg"
-
-def cv2_to_base64(image):
-    return base64.b64encode(image).decode('utf8')
-
-if __name__ == "__main__":
-    with open(imgpath, 'rb') as file:
-        image_data = file.read()
-    image = cv2_to_base64(image_data)
-
-    for i in range(1):
-        ret = client.predict(feed_dict={"image": image}, fetch=["result"])
-        print(ret)

PaddleClas/deploy/paddleserving/recognition/recognition_web_service.py

@@ -1,209 +0,0 @@
-# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from paddle_serving_server.web_service import WebService, Op
-import logging
-import numpy as np
-import sys
-import cv2
-from paddle_serving_app.reader import *
-import base64
-import os
-import faiss
-import pickle
-import json
-
-
-class DetOp(Op):
-    def init_op(self):
-        self.img_preprocess = Sequential([
-            BGR2RGB(), Div(255.0),
-            Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], False),
-            Resize((640, 640)), Transpose((2, 0, 1))
-        ])
-
-        self.img_postprocess = RCNNPostprocess("label_list.txt", "output")
-        self.threshold = 0.2
-        self.max_det_results = 5
-
-    def generate_scale(self, im):
-        """
-        Args:
-            im (np.ndarray): image (np.ndarray)
-        Returns:
-            im_scale_x: the resize ratio of X
-            im_scale_y: the resize ratio of Y
-        """
-        target_size = [640, 640]
-        origin_shape = im.shape[:2]
-        resize_h, resize_w = target_size
-        im_scale_y = resize_h / float(origin_shape[0])
-        im_scale_x = resize_w / float(origin_shape[1])
-        return im_scale_y, im_scale_x
-
-    def preprocess(self, input_dicts, data_id, log_id):
-        (_, input_dict), = input_dicts.items()
-        imgs = []
-        raw_imgs = []
-        for key in input_dict.keys():
-            data = base64.b64decode(input_dict[key].encode('utf8'))
-            raw_imgs.append(data)
-            data = np.fromstring(data, np.uint8)
-            raw_im = cv2.imdecode(data, cv2.IMREAD_COLOR)
-
-            im_scale_y, im_scale_x = self.generate_scale(raw_im)
-            im = self.img_preprocess(raw_im)
-
-            im_shape = np.array(im.shape[1:]).reshape(-1)
-            scale_factor = np.array([im_scale_y, im_scale_x]).reshape(-1)
-            imgs.append({
-                "image": im[np.newaxis, :],
-                "im_shape": im_shape[np.newaxis, :],
-                "scale_factor": scale_factor[np.newaxis, :],
-            })
-        self.raw_img = raw_imgs
-
-        feed_dict = {
-            "image": np.concatenate(
-                [x["image"] for x in imgs], axis=0),
-            "im_shape": np.concatenate(
-                [x["im_shape"] for x in imgs], axis=0),
-            "scale_factor": np.concatenate(
-                [x["scale_factor"] for x in imgs], axis=0)
-        }
-        return feed_dict, False, None, ""
-
-    def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
-        boxes = self.img_postprocess(fetch_dict, visualize=False)
-        boxes.sort(key=lambda x: x["score"], reverse=True)
-        boxes = filter(lambda x: x["score"] >= self.threshold,
-                       boxes[:self.max_det_results])
-        boxes = list(boxes)
-        for i in range(len(boxes)):
-            boxes[i]["bbox"][2] += boxes[i]["bbox"][0] - 1
-            boxes[i]["bbox"][3] += boxes[i]["bbox"][1] - 1
-        result = json.dumps(boxes)
-        res_dict = {"bbox_result": result, "image": self.raw_img}
-        return res_dict, None, ""
-
-
-class RecOp(Op):
-    def init_op(self):
-        self.seq = Sequential([
-            BGR2RGB(), Resize((224, 224)), Div(255),
-            Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225],
-                      False), Transpose((2, 0, 1))
-        ])
-
-        index_dir = "../../drink_dataset_v1.0/index"
-        assert os.path.exists(os.path.join(
-            index_dir, "vector.index")), "vector.index not found ..."
-        assert os.path.exists(os.path.join(
-            index_dir, "id_map.pkl")), "id_map.pkl not found ... "
-
-        self.searcher = faiss.read_index(
-            os.path.join(index_dir, "vector.index"))
-
-        with open(os.path.join(index_dir, "id_map.pkl"), "rb") as fd:
-            self.id_map = pickle.load(fd)
-
-        self.rec_nms_thresold = 0.05
-        self.rec_score_thres = 0.5
-        self.feature_normalize = True
-        self.return_k = 1
-
-    def preprocess(self, input_dicts, data_id, log_id):
-        (_, input_dict), = input_dicts.items()
-        raw_img = input_dict["image"][0]
-        data = np.frombuffer(raw_img, np.uint8)
-        origin_img = cv2.imdecode(data, cv2.IMREAD_COLOR)
-        dt_boxes = input_dict["bbox_result"]
-        boxes = json.loads(dt_boxes)
-        boxes.append({
-            "category_id": 0,
-            "score": 1.0,
-            "bbox": [0, 0, origin_img.shape[1], origin_img.shape[0]]
-        })
-        self.det_boxes = boxes
-
-        #construct batch images for rec
-        imgs = []
-        for box in boxes:
-            box = [int(x) for x in box["bbox"]]
-            im = origin_img[box[1]:box[3], box[0]:box[2]].copy()
-            img = self.seq(im)
-            imgs.append(img[np.newaxis, :].copy())
-
-        input_imgs = np.concatenate(imgs, axis=0)
-        return {"x": input_imgs}, False, None, ""
-
-    def nms_to_rec_results(self, results, thresh=0.1):
-        filtered_results = []
-        x1 = np.array([r["bbox"][0] for r in results]).astype("float32")
-        y1 = np.array([r["bbox"][1] for r in results]).astype("float32")
-        x2 = np.array([r["bbox"][2] for r in results]).astype("float32")
-        y2 = np.array([r["bbox"][3] for r in results]).astype("float32")
-        scores = np.array([r["rec_scores"] for r in results])
-
-        areas = (x2 - x1 + 1) * (y2 - y1 + 1)
-        order = scores.argsort()[::-1]
-        while order.size > 0:
-            i = order[0]
-            xx1 = np.maximum(x1[i], x1[order[1:]])
-            yy1 = np.maximum(y1[i], y1[order[1:]])
-            xx2 = np.minimum(x2[i], x2[order[1:]])
-            yy2 = np.minimum(y2[i], y2[order[1:]])
-
-            w = np.maximum(0.0, xx2 - xx1 + 1)
-            h = np.maximum(0.0, yy2 - yy1 + 1)
-            inter = w * h
-            ovr = inter / (areas[i] + areas[order[1:]] - inter)
-            inds = np.where(ovr <= thresh)[0]
-            order = order[inds + 1]
-            filtered_results.append(results[i])
-        return filtered_results
-
-    def postprocess(self, input_dicts, fetch_dict, data_id, log_id):
-        batch_features = fetch_dict["features"]
-
-        if self.feature_normalize:
-            feas_norm = np.sqrt(
-                np.sum(np.square(batch_features), axis=1, keepdims=True))
-            batch_features = np.divide(batch_features, feas_norm)
-
-        scores, docs = self.searcher.search(batch_features, self.return_k)
-
-        results = []
-        for i in range(scores.shape[0]):
-            pred = {}
-            if scores[i][0] >= self.rec_score_thres:
-                pred["bbox"] = [int(x) for x in self.det_boxes[i]["bbox"]]
-                pred["rec_docs"] = self.id_map[docs[i][0]].split()[1]
-                pred["rec_scores"] = scores[i][0]
-                results.append(pred)
-
-        #do nms
-        results = self.nms_to_rec_results(results, self.rec_nms_thresold)
-        return {"result": str(results)}, None, ""
-
-
-class RecognitionService(WebService):
-    def get_pipeline_response(self, read_op):
-        det_op = DetOp(name="det", input_ops=[read_op])
-        rec_op = RecOp(name="rec", input_ops=[det_op])
-        return rec_op
-
-
-product_recog_service = RecognitionService(name="recognition")
-product_recog_service.prepare_pipeline_config("config.yml")
-product_recog_service.run_service()

PaddleClas/deploy/python/build_gallery.py

@@ -1,214 +0,0 @@
-# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-import sys
-
-__dir__ = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.abspath(os.path.join(__dir__, '../')))
-
-import cv2
-import faiss
-import numpy as np
-from tqdm import tqdm
-import pickle
-
-from python.predict_rec import RecPredictor
-
-from utils import logger
-from utils import config
-
-
-def split_datafile(data_file, image_root, delimiter="\t"):
-    '''
-        data_file: image path and info, which can be splitted by spacer
-        image_root: image path root
-        delimiter: delimiter
-    '''
-    gallery_images = []
-    gallery_docs = []
-    with open(data_file, 'r', encoding='utf-8') as f:
-        lines = f.readlines()
-        for _, ori_line in enumerate(lines):
-            line = ori_line.strip().split(delimiter)
-            text_num = len(line)
-            assert text_num >= 2, f"line({ori_line}) must be splitted into at least 2 parts, but got {text_num}"
-            image_file = os.path.join(image_root, line[0])
-
-            gallery_images.append(image_file)
-            gallery_docs.append(ori_line.strip())
-
-    return gallery_images, gallery_docs
-
-
-class GalleryBuilder(object):
-    def __init__(self, config):
-
-        self.config = config
-        self.rec_predictor = RecPredictor(config)
-        assert 'IndexProcess' in config.keys(), "Index config not found ... "
-        self.build(config['IndexProcess'])
-
-    def build(self, config):
-        '''
-            build index from scratch
-        '''
-        operation_method = config.get("index_operation", "new").lower()
-
-        gallery_images, gallery_docs = split_datafile(
-            config['data_file'], config['image_root'], config['delimiter'])
-
-        # when remove data in index, do not need extract fatures
-        if operation_method != "remove":
-            gallery_features = self._extract_features(gallery_images, config)
-        assert operation_method in [
-            "new", "remove", "append"
-        ], "Only append, remove and new operation are supported"
-
-        # vector.index: faiss index file
-        # id_map.pkl: use this file to map id to image_doc
-        if operation_method in ["remove", "append"]:
-            # if remove or append, vector.index and id_map.pkl must exist
-            assert os.path.join(
-                config["index_dir"], "vector.index"
-            ), "The vector.index dose not exist in {} when 'index_operation' is not None".format(
-                config["index_dir"])
-            assert os.path.join(
-                config["index_dir"], "id_map.pkl"
-            ), "The id_map.pkl dose not exist in {} when 'index_operation' is not None".format(
-                config["index_dir"])
-            index = faiss.read_index(
-                os.path.join(config["index_dir"], "vector.index"))
-            with open(os.path.join(config["index_dir"], "id_map.pkl"),
-                      'rb') as fd:
-                ids = pickle.load(fd)
-            assert index.ntotal == len(ids.keys(
-            )), "data number in index is not equal in in id_map"
-        else:
-            if not os.path.exists(config["index_dir"]):
-                os.makedirs(config["index_dir"], exist_ok=True)
-            index_method = config.get("index_method", "HNSW32")
-
-            # if IVF method, cal ivf number automaticlly
-            if index_method == "IVF":
-                index_method = index_method + str(
-                    min(int(len(gallery_images) // 8), 65536)) + ",Flat"
-
-            # for binary index, add B at head of index_method
-            if config["dist_type"] == "hamming":
-                index_method = "B" + index_method
-
-            #dist_type
-            dist_type = faiss.METRIC_INNER_PRODUCT if config[
-                "dist_type"] == "IP" else faiss.METRIC_L2
-
-            #build index
-            if config["dist_type"] == "hamming":
-                index = faiss.index_binary_factory(config["embedding_size"],
-                                                   index_method)
-            else:
-                index = faiss.index_factory(config["embedding_size"],
-                                            index_method, dist_type)
-                index = faiss.IndexIDMap2(index)
-            ids = {}
-
-        if config["index_method"] == "HNSW32":
-            logger.warning(
-                "The HNSW32 method dose not support 'remove' operation")
-
-        if operation_method != "remove":
-            # calculate id for new data
-            start_id = max(ids.keys()) + 1 if ids else 0
-            ids_now = (
-                np.arange(0, len(gallery_images)) + start_id).astype(np.int64)
-
-            # only train when new index file
-            if operation_method == "new":
-                if config["dist_type"] == "hamming":
-                    index.add(gallery_features)
-                else:
-                    index.train(gallery_features)
-
-            if not config["dist_type"] == "hamming":
-                index.add_with_ids(gallery_features, ids_now)
-
-            for i, d in zip(list(ids_now), gallery_docs):
-                ids[i] = d
-        else:
-            if config["index_method"] == "HNSW32":
-                raise RuntimeError(
-                    "The index_method: HNSW32 dose not support 'remove' operation"
-                )
-            # remove ids in id_map, remove index data in faiss index
-            remove_ids = list(
-                filter(lambda k: ids.get(k) in gallery_docs, ids.keys()))
-            remove_ids = np.asarray(remove_ids)
-            index.remove_ids(remove_ids)
-            for k in remove_ids:
-                del ids[k]
-
-        # store faiss index file and id_map file
-        if config["dist_type"] == "hamming":
-            faiss.write_index_binary(
-                index, os.path.join(config["index_dir"], "vector.index"))
-        else:
-            faiss.write_index(
-                index, os.path.join(config["index_dir"], "vector.index"))
-
-        with open(os.path.join(config["index_dir"], "id_map.pkl"), 'wb') as fd:
-            pickle.dump(ids, fd)
-
-    def _extract_features(self, gallery_images, config):
-        # extract gallery features
-        if config["dist_type"] == "hamming":
-            gallery_features = np.zeros(
-                [len(gallery_images), config['embedding_size'] // 8],
-                dtype=np.uint8)
-        else:
-            gallery_features = np.zeros(
-                [len(gallery_images), config['embedding_size']],
-                dtype=np.float32)
-
-        #construct batch imgs and do inference
-        batch_size = config.get("batch_size", 32)
-        batch_img = []
-        for i, image_file in enumerate(tqdm(gallery_images)):
-            img = cv2.imread(image_file)
-            if img is None:
-                logger.error("img empty, please check {}".format(image_file))
-                exit()
-            img = img[:, :, ::-1]
-            batch_img.append(img)
-
-            if (i + 1) % batch_size == 0:
-                rec_feat = self.rec_predictor.predict(batch_img)
-                gallery_features[i - batch_size + 1:i + 1, :] = rec_feat
-                batch_img = []
-
-        if len(batch_img) > 0:
-            rec_feat = self.rec_predictor.predict(batch_img)
-            gallery_features[-len(batch_img):, :] = rec_feat
-            batch_img = []
-
-        return gallery_features
-
-
-def main(config):
-    GalleryBuilder(config)
-    return
-
-
-if __name__ == "__main__":
-    args = config.parse_args()
-    config = config.get_config(args.config, overrides=args.override, show=True)
-    main(config)